Add initial Flux support including txt2img and LORA support

assets/js/src/App.tsx

@@ -24,14 +24,19 @@ import ScrollArea from "./components/ScrollArea";
 import useHistoryManager from "./hooks/useHistoryManager";
 import useScripts from "./hooks/useScripts";
 import { useAppDispatch, useAppSelector } from "./hooks";
-import { OptionsState, selectBackend } from "./state/optionsSlice";
+import {
+  OptionsState,
+  selectBackend,
+  selectSelectedModel,
+} from "./state/optionsSlice";
 import useBackend from "./hooks/useBackend";
 import useSchedulers from "./hooks/useSchedulers";
 import { ActionCreators as UndoActionCreators } from "redux-undo";
 import { useCustomEventListener } from "react-custom-events";
 import { HistoryItem } from "./state/historySlice";
 import Button from "./components/Button";
 import BatchImageResults from "./BatchImageResults";
+import ClipModelSelect from "./MainForm/ClipModelSelect";
 
 function App() {
   const { refetch: refetchOptions } = useOptions({ fetchPolicy: "eager" });
@@ -76,17 +81,21 @@ function App() {
     models,
     setModel,
     setVae,
+    setClipModel,
+    setClipModel2,
     selectedModel,
     selectedVae,
+    selectedClipModel,
+    selectedClipModel2,
     fetchData: refetchModels,
     fetchVaes: refetchVaes,
     vaes,
+    clipModels,
   } = useModels({
     fetchPolicy: "eager",
   });
 
   useIsConnected();
-
   // const panelRef = useRef<HTMLDivElement>(null);
   // const { width } = useResize({ container: panelRef });
 
@@ -133,6 +142,20 @@ function App() {
                 isVaeLoading={isVaeLoading}
                 isModelLoading={isModelLoading}
               />
+              {selectedModel?.isFlux && (
+                <>
+                  <ClipModelSelect
+                    clipModels={clipModels}
+                    setClipModel={setClipModel}
+                    selectedClipModel={selectedClipModel}
+                  />
+                  <ClipModelSelect
+                    clipModels={clipModels}
+                    setClipModel={setClipModel2}
+                    selectedClipModel={selectedClipModel2}
+                  />
+                </>
+              )}
             </div>
             <MainForm />
           </ScrollArea>
@@ -244,6 +267,7 @@ const VaeSelect = ({
   isModelLoading: boolean;
 }) => {
   const backend = useAppSelector(selectBackend);
+  const selectedModel = useAppSelector(selectSelectedModel);
 
   const handleVaeChange: SelectProps["onChange"] = useCallback(
     (value) => {
@@ -261,7 +285,10 @@ const VaeSelect = ({
   const title = "Select VAE";
   const name = "vae";
 
-  const vaeList = useMemo(() => ["Automatic", ...(vaes ?? [])], [vaes]);
+  const vaeList = useMemo(
+    () => [...(!selectedModel?.isFlux ? ["Automatic"] : []), ...(vaes ?? [])],
+    [selectedModel, vaes]
+  );
 
   return (
     <>

assets/js/src/MainForm/ClipModelSelect.tsx

@@ -0,0 +1,42 @@
+import { useCallback } from "react";
+import Select, { SelectProps } from "../components/Select";
+import useBackend from "../hooks/useBackend";
+
+type Props = {
+  clipModels: string[];
+  selectedClipModel?: string;
+  setClipModel: (clipModel: string) => void;
+};
+
+const ClipModelSelect = ({
+  clipModels,
+  selectedClipModel,
+  setClipModel,
+}: Props) => {
+  const { backend } = useBackend();
+
+  const handleClipModelChange: SelectProps["onChange"] = useCallback(
+    (value) => {
+      if (value && (selectedClipModel || backend === "comfy"))
+        setClipModel(value as string);
+    },
+    [backend, selectedClipModel, setClipModel]
+  );
+
+  if (backend !== "comfy") return null;
+
+  const title = "Select Clip Model";
+
+  return (
+    <Select
+      id="comfy_vae"
+      items={clipModels}
+      name={name}
+      value={selectedClipModel}
+      onChange={handleClipModelChange}
+      title={title}
+    />
+  );
+};
+
+export default ClipModelSelect;

assets/js/src/MainForm/MainForm.tsx

@@ -61,6 +61,8 @@ import {
 import useScripts from "../hooks/useScripts";
 import {
   selectBackend,
+  selectSelectedClipModel,
+  selectSelectedClipModel2,
   selectSelectedModel,
   selectSelectedVae,
 } from "../state/optionsSlice";
@@ -171,6 +173,9 @@ const MainForm = () => {
   const model = useAppSelector(selectSelectedModel);
   const vae = useAppSelector(selectSelectedVae);
 
+  const clipModel = useAppSelector(selectSelectedClipModel);
+  const clipModel2 = useAppSelector(selectSelectedClipModel2);
+
   const isConnected = useAppSelector(selectIsConnected);
 
   const isBatchDisabled =
@@ -231,14 +236,14 @@ const MainForm = () => {
   useEffect(() => {
     // Update scale if either width of height go below 512
     if (width && height) {
-      const referenceDim = model.isSdXl ? 1024 : 512;
+      const referenceDim = model.isSdXl || model.isFlux ? 1024 : 512;
 
       const minDimension = Math.min(+width, +height);
 
       if (minDimension < referenceDim) {
         const newScale = Math.min(
           10,
-          (model.isSdXl ? 1024 : 516.5) / minDimension
+          (model?.isSdXl || model?.isFlux ? 1024 : 516.5) / minDimension
         ).toFixed(2);
         setValue("scale", +newScale);
       } else if (
@@ -341,15 +346,16 @@ const MainForm = () => {
           ? processedPrompt
           : basePrompt,
       negative_prompt:
-        typeof negative_prompt === "string"
+        typeof (negative_prompt ?? "") === "string"
           ? prompt
           : editorJsonToText((negative_prompt as EditorState).doc.toJSON()),
       scheduler,
       ...rest,
       ...(!isSeedPinned && { seed: -1 }),
       mask: txt2img ? "" : maskDataUrl,
       init_images: txt2img
-        ? controlnetArgs.controlnet?.args.some(
+        ? hasControlnet &&
+          controlnetArgs.controlnet?.args.some(
             ({ overrideBaseLayer }) => !overrideBaseLayer
           )
           ? [initImageDataUrl]
@@ -387,9 +393,9 @@ const MainForm = () => {
             //_
             null,
             //tile_width,
-            model?.isSdXl ? 1024 : 512,
+            model?.isSdXl || model?.isFlux ? 1024 : 512,
             //tile_height
-            model?.isSdXl ? 1024 : 512,
+            model?.isSdXl || model?.isFlux ? 1024 : 512,
             //mask_blur
             8,
             // padding,
@@ -591,6 +597,8 @@ const MainForm = () => {
       invert_mask: invertMask,
       model: model?.name,
       vae,
+      clip_model: clipModel,
+      clip_model_2: clipModel2,
       ...((backend === "comfy" || schedulers?.length > 0) && {
         scheduler,
       }),
@@ -781,28 +789,30 @@ const MainForm = () => {
         className="flex flex-col p-4 px-6 pt-1 pb-10 gap-8  w-full"
         ref={formRef}
       >
-        <div className="flex place-items-center gap-3 justify-between">
-          <Label htmlFor="clip_skip" className="whitespace-nowrap">
-            Clip Skip
-          </Label>
-          <Controller
-            name="clip_skip"
-            control={control}
-            render={({ field }) => (
-              <Input
-                id="clip_skip"
-                className="text-center max-w-16"
-                type="number"
-                step={1}
-                min={1}
-                max={10}
-                {...field}
-                onChange={(event) => field.onChange(+event.target.value)}
-              />
-            )}
-            defaultValue={1}
-          />
-        </div>
+        {!model.isFlux && (
+          <div className="flex place-items-center gap-3 justify-between">
+            <Label htmlFor="clip_skip" className="whitespace-nowrap">
+              Clip Skip
+            </Label>
+            <Controller
+              name="clip_skip"
+              control={control}
+              render={({ field }) => (
+                <Input
+                  id="clip_skip"
+                  className="text-center max-w-16"
+                  type="number"
+                  step={1}
+                  min={1}
+                  max={10}
+                  {...field}
+                  onChange={(event) => field.onChange(+event.target.value)}
+                />
+              )}
+              defaultValue={1}
+            />
+          </div>
+        )}
 
         {!isGenerating || !isConnected ? (
           <Button
@@ -835,18 +845,21 @@ const MainForm = () => {
           />
         </div>
         {hasRegionalPrompting && <RegionalPromptsFields />}
-        <div className="flex flex-col gap-2">
-          <Label>Negative Prompt</Label>
-
-          <Controller
-            name="negative_prompt"
-            control={control}
-            render={({ field }) => (
-              <Editor placeholder="Negative Prompt" {...field} />
-            )}
-            defaultValue=""
-          />
-        </div>
+        {backend !== "comfy" ||
+          (!model.isFlux && (
+            <div className="flex flex-col gap-2">
+              <Label>Negative Prompt</Label>
+
+              <Controller
+                name="negative_prompt"
+                control={control}
+                render={({ field }) => (
+                  <Editor placeholder="Negative Prompt" {...field} />
+                )}
+                defaultValue=""
+              />
+            </div>
+          ))}
 
         <Controller
           name="txt2img"
@@ -951,15 +964,39 @@ const MainForm = () => {
             defaultValue={0.7}
           />
         )}
-        <Controller
-          name="cfg_scale"
-          control={control}
-          defaultValue={7}
-          // rules={{ required: true }}
-          render={({ field }) => (
-            <Slider step={0.1} min={1} max={30} label="CFG Scale" {...field} />
-          )}
-        />
+        {backend !== "comfy" || !model.isFlux ? (
+          <Controller
+            name="cfg_scale"
+            control={control}
+            defaultValue={7}
+            // rules={{ required: true }}
+            render={({ field }) => (
+              <Slider
+                step={0.1}
+                min={1}
+                max={30}
+                label="CFG Scale"
+                {...field}
+              />
+            )}
+          />
+        ) : (
+          <Controller
+            name="flux_guidance"
+            control={control}
+            defaultValue={3.5}
+            // rules={{ required: true }}
+            render={({ field }) => (
+              <Slider
+                step={0.1}
+                min={1}
+                max={30}
+                label="Flux Guidnace"
+                {...field}
+              />
+            )}
+          />
+        )}
         {/* TODO: Show image_cfg_scale only with ip2p */}
         {/* {true && (
         <Controller