diff --git a/src/modules/qrcode/comfy/ComfyClient.ts b/src/modules/qrcode/comfy/ComfyClient.ts index 691b7bcd..9ce8b854 100644 --- a/src/modules/qrcode/comfy/ComfyClient.ts +++ b/src/modules/qrcode/comfy/ComfyClient.ts @@ -11,6 +11,11 @@ interface HistoryResponseItem { 'subfolder': string 'type': 'output' }> + gifs: Array<{ + 'filename': string + 'subfolder': string + 'type': 'output' + }> }> } @@ -147,4 +152,9 @@ export class ComfyClient { const response = await this.httpClient.get(`/view?filename=${filename}&subfolder=&type=output`, { responseType: 'arraybuffer' }) return response.data } + + getFileUrl (filename: string): string { + // return `${this.host}/view?filename=${filename}&subfolder=&type=output` + return `${this.host}/view?filename=${filename}&subfolder=&type=output&format=image%2Fgif` + } } diff --git a/src/modules/sd-images/SDImagesBotBase.ts b/src/modules/sd-images/SDImagesBotBase.ts index b501c319..502ed79a 100644 --- a/src/modules/sd-images/SDImagesBotBase.ts +++ b/src/modules/sd-images/SDImagesBotBase.ts @@ -8,6 +8,7 @@ import { type ILora } from './api/loras-config' import { getParamsFromPrompt } from './api/helpers' import { type IBalancerOperation, OPERATION_STATUS, completeOperation, createOperation, getOperationById } from './balancer' import { now } from '../../utils/perf' +import { MEDIA_FORMAT } from './api/configs' export interface MessageExtras { caption?: string @@ -21,6 +22,7 @@ export interface ISession { prompt: string model: IModel lora?: ILora + format?: MEDIA_FORMAT all_seeds?: string[] seed?: number command: COMMAND @@ -71,13 +73,14 @@ export class SDImagesBotBase { prompt: string model: IModel lora?: ILora + format?: MEDIA_FORMAT command: COMMAND all_seeds?: string[] seed?: string } ): Promise => { // eslint-disable-next-line @typescript-eslint/naming-convention - const { prompt, model, command, all_seeds, lora } = params + const { prompt, model, command, all_seeds, lora, format } = params const authorObj = await ctx.getAuthor() const author = `@${authorObj.user.username}` @@ -93,7 +96,8 @@ export class SDImagesBotBase { lora, command, all_seeds, - message + message, + format } this.sessions.push(newSession) @@ -109,7 +113,11 @@ export class SDImagesBotBase { ): Promise<{ queueMessageId: number, balancerOperaton: IBalancerOperation }> => { const params = getParamsFromPrompt(session.prompt) - const lora = params.loraName ? `${params.loraName}.safetensors` : session.lora?.path + let lora = params.loraName ? `${params.loraName}.safetensors` : session.lora?.path + + if (!lora && session.format === MEDIA_FORMAT.GIF) { + lora = 'ym201.safetensors' + } let balancerOperaton = await createOperation({ model: session.model.path, @@ -140,7 +148,7 @@ export class SDImagesBotBase { session: ISession, specialMessage?: string ): Promise => { - const { model, prompt, seed, lora } = session + const { model, prompt, seed, lora, format } = session let balancerOperatonId @@ -154,7 +162,8 @@ export class SDImagesBotBase { prompt, model, seed, - lora + lora, + format }, balancerOperaton.server) if (balancerOperatonId) { @@ -166,10 +175,19 @@ export class SDImagesBotBase { ? session.message : `${session.message} ${prompt}` : `/${model.aliases[0]} ${prompt}` - await ctx.replyWithPhoto(new InputFile(imageBuffer), { - caption: specialMessage ?? reqMessage, - message_thread_id: ctx.message?.message_thread_id - }) + + if (format === MEDIA_FORMAT.GIF) { + await ctx.replyWithAnimation(new InputFile(imageBuffer, 'file.gif'), { + caption: specialMessage ?? reqMessage, + message_thread_id: ctx.message?.message_thread_id + }) + } else { + await ctx.replyWithPhoto(new InputFile(imageBuffer), { + caption: specialMessage ?? reqMessage, + message_thread_id: ctx.message?.message_thread_id + }) + } + if (ctx.chat?.id && queueMessageId) { await ctx.api.deleteMessage(ctx.chat?.id, queueMessageId) } diff --git a/src/modules/sd-images/api/configs/index.ts b/src/modules/sd-images/api/configs/index.ts index e118ebd1..0b8756dc 100644 --- a/src/modules/sd-images/api/configs/index.ts +++ b/src/modules/sd-images/api/configs/index.ts @@ -4,4 +4,6 @@ export * from './img2img_controlnet_config' export * from './img2img_controlnet_v2_config' export * from './text2img_config' export * from './text2img_lora_config' +export * from './text2gif_config' +export * from './text2gif_lora_config' export * from './types' diff --git a/src/modules/sd-images/api/configs/text2gif_config.ts b/src/modules/sd-images/api/configs/text2gif_config.ts new file mode 100644 index 00000000..202879ec --- /dev/null +++ b/src/modules/sd-images/api/configs/text2gif_config.ts @@ -0,0 +1,121 @@ +import { type Txt2ImgOptions } from './types' + +export function buildText2GifPrompt (options: Txt2ImgOptions & { clientId: string }): unknown { + return { + client_id: options.clientId, + prompt: { + 2: { + inputs: { vae_name: 'vae-ft-mse-840000-ema-pruned.ckpt' }, + class_type: 'VAELoader' + }, + 3: { + inputs: { + text: options.prompt, + clip: [ + '4', + 0 + ] + }, + class_type: 'CLIPTextEncode' + }, + 4: { + inputs: { + stop_at_clip_layer: -2, + clip: [ + '32', + 1 + ] + }, + class_type: 'CLIPSetLastLayer' + }, + 6: { + inputs: { + text: options.negativePrompt, + clip: [ + '4', + 0 + ] + }, + class_type: 'CLIPTextEncode' + }, + 7: { + inputs: { + seed: options.seed, + steps: options.steps ?? 20, + cfg: 8, + sampler_name: 'euler', + scheduler: 'normal', + denoise: 1, + model: [ + '27', + 0 + ], + positive: [ + '3', + 0 + ], + negative: [ + '6', + 0 + ], + latent_image: [ + '9', + 0 + ] + }, + class_type: 'KSampler' + }, + 9: { + inputs: { + width: options.width, + height: options.height, + batch_size: options.batchSize ?? 16 + }, + class_type: 'EmptyLatentImage' + }, + 10: { + inputs: { + samples: [ + '7', + 0 + ], + vae: [ + '2', + 0 + ] + }, + class_type: 'VAEDecode' + }, + 26: { + inputs: { + frame_rate: 8, + loop_count: 0, + filename_prefix: 'ComfyUI', + format: 'image/gif', + pingpong: false, + save_image: true, + images: [ + '10', + 0 + ] + }, + class_type: 'ADE_AnimateDiffCombine' + }, + 27: { + inputs: { + model_name: 'mm_sd_v14.ckpt', + beta_schedule: 'sqrt_linear (AnimateDiff)', + model: [ + '32', + 0 + ] + }, + class_type: 'ADE_AnimateDiffLoaderWithContext' + }, + 32: { + inputs: { ckpt_name: options.model }, + class_type: 'CheckpointLoaderSimple' + } + } + } +} diff --git a/src/modules/sd-images/api/configs/text2gif_lora_config.ts b/src/modules/sd-images/api/configs/text2gif_lora_config.ts new file mode 100644 index 00000000..63935f7d --- /dev/null +++ b/src/modules/sd-images/api/configs/text2gif_lora_config.ts @@ -0,0 +1,137 @@ +import { type Txt2ImgOptions } from './types' + +export function buildText2GifLoraPrompt (options: Txt2ImgOptions & { clientId: string }): unknown { + return { + client_id: options.clientId, + prompt: { + 2: { + inputs: { vae_name: 'vae-ft-mse-840000-ema-pruned.ckpt' }, + class_type: 'VAELoader' + }, + 3: { + inputs: { + text: options.prompt, + clip: [ + '4', + 0 + ] + }, + class_type: 'CLIPTextEncode' + }, + 4: { + inputs: { + stop_at_clip_layer: -2, + clip: [ + '35', + 1 + ] + }, + class_type: 'CLIPSetLastLayer' + }, + 6: { + inputs: { + text: options.negativePrompt, + clip: [ + '4', + 0 + ] + }, + class_type: 'CLIPTextEncode' + }, + 7: { + inputs: { + seed: options.seed, + steps: options.steps ?? 20, + cfg: 8, + sampler_name: 'euler', + scheduler: 'normal', + denoise: 1, + model: [ + '27', + 0 + ], + positive: [ + '3', + 0 + ], + negative: [ + '6', + 0 + ], + latent_image: [ + '9', + 0 + ] + }, + class_type: 'KSampler' + }, + 9: { + inputs: { + width: options.width, + height: options.height, + batch_size: options.batchSize ?? 16 + }, + class_type: 'EmptyLatentImage' + }, + 10: { + inputs: { + samples: [ + '7', + 0 + ], + vae: [ + '2', + 0 + ] + }, + class_type: 'VAEDecode' + }, + 26: { + inputs: { + frame_rate: 8, + loop_count: 0, + filename_prefix: 'ComfyUI', + format: 'image/gif', + pingpong: false, + save_image: true, + images: [ + '10', + 0 + ] + }, + class_type: 'ADE_AnimateDiffCombine' + }, + 27: { + inputs: { + model_name: 'mm_sd_v14.ckpt', + beta_schedule: 'sqrt_linear (AnimateDiff)', + model: [ + '35', + 0 + ] + }, + class_type: 'ADE_AnimateDiffLoaderWithContext' + }, + 32: { + inputs: { ckpt_name: options.model }, + class_type: 'CheckpointLoaderSimple' + }, + 35: { + inputs: { + lora_name: options.loraPath, + strength_model: 1, + strength_clip: 1, + model: [ + '32', + 0 + ], + clip: [ + '32', + 1 + ] + }, + class_type: 'LoraLoader' + } + } + } +} diff --git a/src/modules/sd-images/api/configs/types.ts b/src/modules/sd-images/api/configs/types.ts index d25858ff..f09c8d26 100644 --- a/src/modules/sd-images/api/configs/types.ts +++ b/src/modules/sd-images/api/configs/types.ts @@ -1,3 +1,8 @@ +export enum MEDIA_FORMAT { + JPEG = 'JPEG', + GIF = 'GIF' +} + export interface Txt2ImgOptions { hires?: { steps: number @@ -32,4 +37,5 @@ export interface Txt2ImgOptions { loraPath?: string sampler_name?: string scheduler?: string + format?: MEDIA_FORMAT } diff --git a/src/modules/sd-images/api/index.ts b/src/modules/sd-images/api/index.ts index 21ab1ad7..dc2bcbd2 100644 --- a/src/modules/sd-images/api/index.ts +++ b/src/modules/sd-images/api/index.ts @@ -1,8 +1,9 @@ import { Client, type ISDServerConfig } from './sd-node-client' -import { type IModel } from './models-config' +import { getModelByParam, type IModel } from './models-config' import { getLoraByParam, type ILora } from './loras-config' import { getParamsFromPrompt, NEGATIVE_PROMPT } from './helpers' import { type OnMessageContext, type OnCallBackQueryData } from '../../types' +import { MEDIA_FORMAT } from './configs' export * from './models-config' @@ -13,6 +14,7 @@ interface IGenImageOptions { seed?: number width?: number height?: number + format?: MEDIA_FORMAT } interface ITrainImageOptions { @@ -61,22 +63,43 @@ export class SDNodeApi { params.promptWithoutParams = `logo, ${params.promptWithoutParams}, LogoRedAF` } - const { images } = await this.client.txt2img({ - prompt: params.promptWithoutParams, - negativePrompt: params.negativePrompt, - width: params.width, - height: params.height, - steps: params.steps, - cfgScale: params.cfgScale, - loraPath: selectedLora?.path, - loraName: params.loraName, - loraStrength, - seed: options.seed ?? params.seed, - model: options.model.path, - batchSize: 1 - }, server) + if (options.format === MEDIA_FORMAT.GIF) { + const { images } = await this.client.txt2img({ + prompt: params.promptWithoutParams, + negativePrompt: params.negativePrompt, + width: 512, + height: 512, + steps: 20, + cfgScale: params.cfgScale, + loraPath: selectedLora?.path, + loraName: params.loraName, + loraStrength, + seed: options.seed ?? params.seed, + model: (params.modelAlias && getModelByParam(params.modelAlias)?.path) ?? options.model.path, + batchSize: 16, + format: options.format + }, server) - return images[0] + return images[0] + } else { + const { images } = await this.client.txt2img({ + prompt: params.promptWithoutParams, + negativePrompt: params.negativePrompt, + width: params.width, + height: params.height, + steps: params.steps, + cfgScale: params.cfgScale, + loraPath: selectedLora?.path, + loraName: params.loraName, + loraStrength, + seed: options.seed ?? params.seed, + model: options.model.path, + batchSize: 1, + format: options.format + }, server) + + return images[0] + } } generateImageByImage = async ( diff --git a/src/modules/sd-images/api/sd-node-client.ts b/src/modules/sd-images/api/sd-node-client.ts index 29711cf3..f538538f 100644 --- a/src/modules/sd-images/api/sd-node-client.ts +++ b/src/modules/sd-images/api/sd-node-client.ts @@ -8,8 +8,11 @@ import { buildImg2ImgPrompt, buildImg2ImgControlnetPrompt, buildImg2ImgControlnetV2Prompt, + buildText2GifPrompt, + buildText2GifLoraPrompt, type Txt2ImgOptions, - type Img2ImgOptions + type Img2ImgOptions, + MEDIA_FORMAT } from './configs' import { waitingExecute } from './helpers' import axios from 'axios' @@ -22,6 +25,7 @@ export interface ISDServerConfig { export interface Txt2ImgResponse { images: Buffer[] + imagesUrls: string[] parameters: object all_seeds: string[] info: string @@ -46,7 +50,13 @@ export class Client { const seed = options.seed ?? getRandomSeed() - const buildImgPromptMethod = options.loraPath ? buildImgPromptLora : buildImgPrompt + let buildImgPromptMethod + + if (options.format === MEDIA_FORMAT.GIF) { + buildImgPromptMethod = options.loraPath ? buildText2GifLoraPrompt : buildText2GifPrompt + } else { + buildImgPromptMethod = options.loraPath ? buildImgPromptLora : buildImgPrompt + } const prompt = buildImgPromptMethod({ ...options, @@ -60,14 +70,26 @@ export class Client { const history = await comfyClient.history(r.prompt_id) - const images = await Promise.all( - history.outputs['9'].images.map(async img => await comfyClient.downloadResult(img.filename)) - ) + let images: Buffer[] = [] + let imagesUrls: string[] = [] + + if (options.format === MEDIA_FORMAT.GIF) { + images = await Promise.all( + history.outputs['26'].gifs.map(async img => await comfyClient.downloadResult(img.filename)) + ) + + imagesUrls = history.outputs['26'].gifs.map(img => comfyClient.getFileUrl(img.filename)) + } else { + images = await Promise.all( + history.outputs['9'].images.map(async img => await comfyClient.downloadResult(img.filename)) + ) + } comfyClient.abortWebsocket() const result: Txt2ImgResponse = { images, + imagesUrls, parameters: {}, all_seeds: [String(seed)], info: '' @@ -162,6 +184,7 @@ export class Client { const result: Txt2ImgResponse = { images, + imagesUrls: [], parameters: {}, all_seeds: [String(seed)], info: '' diff --git a/src/modules/sd-images/helpers.ts b/src/modules/sd-images/helpers.ts index dc406d5f..e0a863e5 100644 --- a/src/modules/sd-images/helpers.ts +++ b/src/modules/sd-images/helpers.ts @@ -1,5 +1,6 @@ import { type OnCallBackQueryData, type OnMessageContext } from '../types' import { getModelByParam, type IModel, MODELS_CONFIGS } from './api' +import { MEDIA_FORMAT } from './api/configs' import { getLoraByParam, type ILora } from './api/loras-config' import { childrenWords, sexWords } from './words-blacklist' @@ -17,6 +18,7 @@ export interface IOperation { prompt: string model: IModel lora?: ILora + format?: MEDIA_FORMAT } export interface IMediaGroup { @@ -88,10 +90,10 @@ type Context = OnMessageContext | OnCallBackQueryData const hasCommand = (ctx: Context, cmd: string): boolean => { return ctx.hasCommand(cmd) || - ( - (ctx.message?.text?.startsWith(`${cmd} `) ?? false) && - ctx.chat?.type === 'private' - ) + ( + (ctx.message?.text?.startsWith(`${cmd} `) ?? false) && + ctx.chat?.type === 'private' + ) } export const parseCtx = (ctx: Context): IOperation | false => { @@ -114,6 +116,14 @@ export const parseCtx = (ctx: Context): IOperation | false => { let model = getModelByParam(modelId) let command let lora + let format: MEDIA_FORMAT = MEDIA_FORMAT.JPEG + + if (hasCommand(ctx, 'gif')) { + command = COMMAND.TEXT_TO_IMAGE + format = MEDIA_FORMAT.GIF + model = getModelByParam('22') + prompt = prompt || '1girl, solo, cherry blossom, hanami, pink flower, white flower, spring season, wisteria, petals, flower, plum blossoms, outdoors, falling petals, black eyes, upper body, from side' + } if ( (hasCommand(ctx, 'image') ?? hasCommand(ctx, 'imagine')) ?? hasCommand(ctx, 'img') @@ -206,7 +216,8 @@ export const parseCtx = (ctx: Context): IOperation | false => { command, model, lora, - prompt + prompt, + format } } } catch (e) {