v0.8.0-beta

Revamped the img2img workflow, added soft inapinting, added fooocus inpainting, added IPA integration with masking and added CN union integration

assets/js/src/LayersControl.tsx

@@ -449,6 +449,7 @@ const LayersControl = () => {
     controlnetDetect,
     ip_adapter_models,
     ip_adapter_weight_types,
+    unionControlnetTypes,
   } = useControlnet();
 
   // const { register, handleSubmit, setValue } = useForm();
@@ -905,6 +906,22 @@ const LayersControl = () => {
                 </Checkbox>
               )}
 
+              {backend === "comfy" && !activeControlnetLayer?.isIpAdapter && (
+                <Checkbox
+                  checked={activeControlnetLayer?.is_union}
+                  value={activeControlnetLayer?.is_union}
+                  onChange={(value) =>
+                    handleControlnetAttrsChange(
+                      "is_union",
+                      value,
+                      activeControlnetLayer.id
+                    )
+                  }
+                >
+                  Union Controlnet
+                </Checkbox>
+              )}
+
               {backend === "comfy" && activeControlnetLayer?.isIpAdapter ? (
                 <>
                   <div className="flex gap-2 h-full flex-col">
@@ -993,6 +1010,31 @@ const LayersControl = () => {
                 </>
               )}
 
+              {backend === "comfy" &&
+                !activeControlnetLayer?.isIpAdapter &&
+                activeControlnetLayer?.is_union && (
+                  <div className="flex gap-2 flex-col">
+                    <Label
+                      htmlFor={`module${activeControlnetLayer?.union_type}`}
+                    >
+                      Union Type
+                    </Label>
+                    <Select
+                      name="union_type"
+                      items={unionControlnetTypes}
+                      value={activeControlnetLayer?.union_type}
+                      onChange={(value) =>
+                        handleControlnetSelectChange({
+                          name: "union_type",
+                          type: "text",
+                          value,
+                          layerId: activeControlnetLayer.id,
+                        })
+                      }
+                    />
+                  </div>
+                )}
+
               {backend === "auto" &&
                 !/ip\S*adapter/gi.test(activeControlnetLayer.module) && (
                   <div className="flex gap-2 flex-col">

assets/js/src/MainForm/ComfySoftInpaintingFields/ComfySoftInpaintingFields.tsx

@@ -0,0 +1,46 @@
+import { Control, Controller } from "react-hook-form";
+import { comfySoftPaintingFields } from "./constants";
+import Slider from "../../components/Slider";
+import { MainFormValues } from "../MainForm";
+import ExpandCollapseCheckbox from "../../components/ExpandCollapseCheckbox";
+
+export type ComfySoftInpaintingArgs = {
+  isEnabled: boolean;
+  maskBlur: number;
+};
+
+const SoftInpaintingFields = ({
+  control,
+}: {
+  control: Control<MainFormValues>;
+}) => {
+  return (
+    <div className="flex flex-col gap-3">
+      <Controller
+        name={"comfySoftInpainting.isEnabled"}
+        control={control}
+        render={({ field }) => (
+          <ExpandCollapseCheckbox {...field} label="Soft Inpainting">
+            <div className="h-auto flex relative flex-col gap-8 bg-neutral-100/5 p-4 rounded-md overflow-hidden">
+              {comfySoftPaintingFields?.map(
+                ({ value: defaultValue, ...rest }) => (
+                  <div key={rest.name}>
+                    <Controller
+                      name={"comfySoftInpainting." + rest.name}
+                      control={control}
+                      render={({ field }) => <Slider {...rest} {...field} />}
+                      defaultValue={defaultValue}
+                    />
+                  </div>
+                )
+              )}
+            </div>
+          </ExpandCollapseCheckbox>
+        )}
+        defaultValue={false}
+      />
+    </div>
+  );
+};
+
+export default SoftInpaintingFields;

assets/js/src/MainForm/ComfySoftInpaintingFields/constants.ts

@@ -0,0 +1,24 @@
+import { ComfySoftInpaintingArgs } from "./ComfySoftInpaintingFields";
+
+export const defaultComfySoftPaintingArgs: ComfySoftInpaintingArgs = {
+  isEnabled: false,
+  maskBlur: 6,
+};
+
+export const comfySoftPaintingFields: {
+  label: string;
+  name: keyof ComfySoftInpaintingArgs;
+  value: number;
+  min: number;
+  max: number;
+  step: number;
+}[] = [
+  {
+    label: "Mask Blur",
+    name: "maskBlur",
+    value: 6,
+    min: 0,
+    max: 30,
+    step: 1,
+  },
+];

assets/js/src/MainForm/ComfySoftInpaintingFields/index.ts

@@ -0,0 +1 @@
+export { default } from "./ComfySoftInpaintingFields";

assets/js/src/MainForm/MainForm.tsx

@@ -74,13 +74,13 @@ import { REGIONAL_PROMPTS_SEPARATOR, weightTypesByName } from "./constants";
 import { selectPromptRegionLayers } from "../state/promptRegionsSlice";
 import { showNotification } from "../Notifications/utils";
 import Button from "../components/Button";
-import Modal from "../components/Modal";
-import ModelsModal from "../ModelsModal";
-import LorasModal from "../LorasModal";
 import { useCustomEventListener } from "react-custom-events";
+import ComfySoftInpaintingFields from "./ComfySoftInpaintingFields";
+import { ComfySoftInpaintingArgs } from "./ComfySoftInpaintingFields/ComfySoftInpaintingFields";
 
 export type MainFormValues = Record<string, any> & {
   softInpainting: SoftInpaintingArgs;
+  comfySoftInpainting: ComfySoftInpaintingArgs;
   regionalPrompts?: Record<string, { prompt: EditorState; weight }>;
   globalPromptWeight?: number;
 };
@@ -284,6 +284,7 @@ const MainForm = () => {
       negative_prompt,
       scheduler,
       softInpainting,
+      comfySoftInpainting,
       globalPromptWeight,
       fooocus_inpaint,
       ...rest
@@ -609,12 +610,17 @@ const MainForm = () => {
       ...(backend === "comfy" && iPAdapters?.length
         ? { ip_adapters: iPAdapters }
         : {}),
-      ...(backend === "comfy" && !txt2img ? { fooocus_inpaint } : {}),
+      ...(backend === "comfy" && !txt2img && model?.isSdXl
+        ? { fooocus_inpaint }
+        : {}),
+      ...(backend === "comfy" && comfySoftInpainting.isEnabled
+        ? { mask_blur: comfySoftInpainting.maskBlur }
+        : {}),
       ultimate_upscale: isUltimateUpscaleEnabled,
       clip_skip: clipSkip,
     };
 
-    console.log(image, { attrs });
+    // console.log(image, { attrs });
 
     batchImageResults?.length && dispatch(setBatchImageResults([]));
 
@@ -918,7 +924,9 @@ const MainForm = () => {
 
         {showSoftInpainting && <SoftInpaintingFields control={control} />}
 
-        {backend === "comfy" && !txt2img && (
+        {backend === "comfy" && <ComfySoftInpaintingFields control={control} />}
+
+        {backend === "comfy" && model?.isSdXl && !txt2img && (
           <Controller
             name="fooocus_inpaint"
             control={control}

assets/js/src/hooks/useControlnet.tsx

@@ -62,10 +62,19 @@ const useControlnet = ({ fetchPolicy }: Props = {}) => {
       fetchPolicy,
     });
 
+  const { fetchData: fetchUnionControlnetTypes, data: unionControlnetTypes } =
+    useData<string[]>({
+      name: "union_controlnet_types",
+      fetchPolicy,
+    });
+
   const {
     fetchData: fetchIpApapterModels,
     data: { ip_adapter_models, ip_adapter_weight_types } = {},
-  } = useData<string[]>({
+  } = useData<{
+    ip_adapter_models: string[];
+    ip_adapter_weight_types: string[];
+  }>({
     name: "ip_adapter_models",
     fetchPolicy,
     condition: backend === "comfy",
@@ -186,10 +195,12 @@ const useControlnet = ({ fetchPolicy }: Props = {}) => {
   return {
     controlnet_models,
     controlnet_preprocessors,
+    unionControlnetTypes,
     ip_adapter_models,
     ip_adapter_weight_types,
     fetchControlnetModels,
     fetchControlnetModules,
+    fetchUnionControlnetTypes,
     fetchIpApapterModels,
     controlnetDetect,
   };

assets/js/src/state/controlnetSlice.ts

@@ -50,8 +50,10 @@ type ControlnetUi = {
   weight_type?: keyof typeof weightTypesByName;
   composition_weight?: number;
   isIpAdapter: boolean;
+  is_union: boolean;
   iPAdapterModel?: string;
   iPAdapterWeightType?: string;
+  union_type?: string;
 };
 export type ControlnetLayer = {
   id?: string;
@@ -94,6 +96,8 @@ const controlnetLayerInitialState: ControlnetLayer = {
   control_mode: "Balanced",
   maskColor: "#FFFFFF",
   isIpAdapter: false,
+  is_union: false,
+  union_type: "auto",
 };
 const initialState: ControlnetState = {
   layers: [

lib/ex_sd/comfy_client.ex

@@ -214,6 +214,23 @@ defmodule ExSd.ComfyClient do
     end
   end
 
+  @spec get_union_controlnet_types() :: {:error, any()} | {:ok, list(binary())}
+  def get_union_controlnet_types() do
+    with response <- get("/object_info/SetUnionControlNetType"),
+         {:ok, body} <- handle_response(response) do
+      types =
+        body
+        |> get_in(["SetUnionControlNetType", "input", "required", "type"])
+        |> List.first()
+        |> Enum.sort()
+
+      {:ok, types}
+    else
+      {:error, _error} = res ->
+        res
+    end
+  end
+
   @spec get_ip_adapter_models() :: {:error, any()} | {:ok, list(binary())}
   def get_ip_adapter_models() do
     with response <- get("/object_info/IPAdapterUnifiedLoader"),

lib/ex_sd/sd.ex

@@ -65,6 +65,9 @@ defmodule ExSd.Sd do
   @spec get_controlnet_preprocessors :: {:ok, list}
   defdelegate get_controlnet_preprocessors(), to: SdServer
 
+  @spec get_union_controlnet_types :: {:ok, list}
+  defdelegate get_union_controlnet_types(), to: SdServer
+
   @spec get_ip_adapter_models :: {:ok, map()}
   defdelegate get_ip_adapter_models(), to: SdServer
 

lib/ex_sd/sd/comfy_prompt.ex

@@ -169,7 +169,7 @@ defmodule ExSd.Sd.ComfyPrompt do
         controlnet_args: controlnet_args
       )
 
-    File.write!("./prompt.json", Jason.encode!(prompt, pretty: true))
+    # File.write!("./prompt.json", Jason.encode!(prompt, pretty: true))
     prompt
   end
 
@@ -698,7 +698,7 @@ defmodule ExSd.Sd.ComfyPrompt do
         add_condition: has_ultimate_upscale
       )
 
-    File.write!("./prompt.json", Jason.encode!(prompt, pretty: true))
+    # File.write!("./prompt.json", Jason.encode!(prompt, pretty: true))
     prompt
   end
 
@@ -971,6 +971,31 @@ defmodule ExSd.Sd.ComfyPrompt do
     add_node(prompt, node)
   end
 
+  @spec maybe_add_set_union_controlnet_type(
+          prompt(),
+          binary(),
+          boolean(),
+          [
+            {:control_net, ref_node_value()},
+            {:type, binary()}
+          ]
+        ) :: prompt()
+  def maybe_add_set_union_controlnet_type(prompt, name, condition, options \\ [])
+
+  def maybe_add_set_union_controlnet_type(prompt, _name, false, _options) do
+    prompt
+  end
+
+  def maybe_add_set_union_controlnet_type(prompt, name, _condition, options) do
+    node =
+      node(name, "SetUnionControlNetType", %{
+        control_net: Keyword.get(options, :control_net),
+        type: Keyword.get(options, :type)
+      })
+
+    add_node(prompt, node)
+  end
+
   @spec add_image_loader(
           prompt(),
           [
@@ -1129,6 +1154,12 @@ defmodule ExSd.Sd.ComfyPrompt do
       |> add_controlnet_loader("cn#{entry.model}_controlnet_loader",
         control_net_name: entry.model
       )
+      |> maybe_add_set_union_controlnet_type(
+        "cn#{index}_union_controlnet_type",
+        entry.is_union,
+        control_net: node_ref("cn#{entry.model}_controlnet_loader", 0),
+        type: entry.union_type
+      )
       |> add_controlnet_apply_advanced(
         name: "cn#{index}_apply_controlnet",
         strength: entry.weight,
@@ -1155,7 +1186,10 @@ defmodule ExSd.Sd.ComfyPrompt do
         # TODO: reuse loaded models to avoid loading a model more than once for different layers
         control_net:
           node_ref(
-            "cn#{entry.model}_controlnet_loader",
+            if(entry.is_union,
+              do: "cn#{index}_union_controlnet_type",
+              else: "cn#{entry.model}_controlnet_loader"
+            ),
             0
           ),
         image:
@@ -1371,7 +1405,8 @@ defmodule ExSd.Sd.ComfyPrompt do
               node_ref(
                 "attention_couple_regions_#{max(0, ceil(length(attention_couple_regions) / 10) - 1)}",
                 0
-              )
+              ),
+            ip_adapter_active: not Enum.empty?(ip_adapters)
           }
         )
       )
@@ -1798,7 +1833,7 @@ defmodule ExSd.Sd.ComfyPrompt do
         image:
           if(Map.get(ip_adapter, "image"),
             do: node_ref("ip_adapter_#{index}_image_loader", 0),
-            else: nil
+            else: node_ref("image_input", 0)
           ),
         attn_mask:
           if(Map.get(ip_adapter, "mask"),

lib/ex_sd/sd/control_net_args.ex

@@ -28,6 +28,8 @@ defmodule ExSd.Sd.ControlNetArgs do
     field :pixel_perfect, :boolean, default: true
     field :control_mode, :string, default: "Balanced"
     field :advanced_weighting, {:array, :float}, default: nil
+    field :is_union, :boolean, default: false
+    field :union_type, :binary
   end
 
   def changeset(%__MODULE__{} = control_net, attrs) do
@@ -52,7 +54,9 @@ defmodule ExSd.Sd.ControlNetArgs do
         :guidance_end,
         :pixel_perfect,
         :control_mode,
-        :advanced_weighting
+        :advanced_weighting,
+        :is_union,
+        :union_type
       ]
     )
   end