Skip to content

Commit 51394af

Browse files
ryanontheinsideryanontheinstideeliteprox
authored
fix: patch for multiple controlnets (yondonfu#78)
--------- Co-authored-by: ryanontheinstide <ryanfosdick87@gmail.com> Co-authored-by: John Mull <john@eliteencoder.net>
1 parent 44ead39 commit 51394af

7 files changed

+2322
-1
lines changed

‎configs/models.yaml

+20-1
Original file line numberDiff line numberDiff line change
@@ -37,4 +37,23 @@ models:
3737
name: "ControlNet Depth"
3838
url: "https://huggingface.co/comfyanonymous/ControlNet-v1-1_fp16_safetensors/resolve/main/control_v11f1p_sd15_depth_fp16.safetensors"
3939
path: "controlnet/control_v11f1p_sd15_depth_fp16.safetensors"
40-
type: "controlnet"
40+
type: "controlnet"
41+
42+
controlnet-mediapipe-face:
43+
name: "ControlNet MediaPipe Face"
44+
url: "https://huggingface.co/CrucibleAI/ControlNetMediaPipeFace/resolve/main/control_v2p_sd15_mediapipe_face.safetensors"
45+
path: "controlnet/control_v2p_sd15_mediapipe_face.safetensors"
46+
type: "controlnet"
47+
48+
# Lora models
49+
ral-polygon-sd15:
50+
name: "ral-polygon-sd15"
51+
url: "https://huggingface.co/Livepeer-Studio/comfystream_loras/resolve/main/ral-polygon-sd15.safetensors"
52+
path: "loras/SD1.5/ral-polygon-sd15.safetensors"
53+
type: "lora"
54+
55+
ral-chrome-sdxl:
56+
name: "ral-chrome-sd15"
57+
url: "https://civitai.com/api/download/models/276570?type=Model&format=SafeTensor"
58+
path: "loras/SD1.5/ral-chrome-sd15.safetensors"
59+
type: "lora"

‎configs/nodes.yaml

+7
Original file line numberDiff line numberDiff line change
@@ -61,3 +61,10 @@ nodes:
6161
name: "ComfyUI Background Edit"
6262
url: "https://github.com/yondonfu/ComfyUI-Background-Edit"
6363
type: "utility"
64+
65+
comfyui_controlnet_aux:
66+
name: "ComfyUI ControlNet Auxiliary"
67+
url: "https://github.com/Fannovel16/comfyui_controlnet_aux"
68+
type: "utility"
69+
dependencies:
70+
- "mediapipe==0.10.8"

‎nodes/tensor_utils/prestartup_script.py

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
import torch
2+
3+
print("Loading pre-startup script for controlnet torch.compile() compatibility...")
4+
5+
def patch_controlnet_for_stream():
6+
"""Patch ControlNet for better compatibility with torch.compile()"""
7+
try:
8+
from comfy.controlnet import ControlBase
9+
original_control_merge = ControlBase.control_merge
10+
11+
def wrapped_control_merge(self, control, control_prev, output_dtype):
12+
# Mark CUDA graph step at start
13+
if torch.cuda.is_available() and hasattr(torch.compiler, 'cudagraph_mark_step_begin'):
14+
torch.compiler.cudagraph_mark_step_begin()
15+
torch.cuda.synchronize()
16+
17+
# Deep clone control outputs to prevent CUDA graph overwrites
18+
control = {
19+
k: [t.clone() if t is not None else None for t in v]
20+
for k, v in control.items()
21+
}
22+
23+
# Deep clone previous control if it exists
24+
if control_prev is not None:
25+
control_prev = {
26+
k: [t.clone() if t is not None else None for t in v]
27+
for k, v in control_prev.items()
28+
}
29+
30+
# Get result from original merge function
31+
result = original_control_merge(self, control, control_prev, output_dtype)
32+
33+
# Clone all output tensors
34+
result = {
35+
k: [t.clone() if t is not None else None for t in v]
36+
for k, v in result.items()
37+
}
38+
39+
# Mark CUDA graph step at end
40+
if torch.cuda.is_available() and hasattr(torch.compiler, 'cudagraph_mark_step_begin'):
41+
torch.compiler.cudagraph_mark_step_begin()
42+
torch.cuda.synchronize()
43+
44+
return result
45+
46+
# Apply the patch
47+
ControlBase.control_merge = wrapped_control_merge
48+
print("Successfully patched ControlNet for torch.compile() compatibility")
49+
except Exception as e:
50+
print(f"Warning: Failed to patch ControlNet: {str(e)}")
51+
52+
# Apply patch when module is imported
53+
patch_controlnet_for_stream()

‎workflows/sd15_multi_cnet_depthanything_face.json

+295
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,295 @@
1+
{
2+
"2": {
3+
"inputs": {
4+
"engine": "depth_anything_v2_vitl_fp16.engine",
5+
"images": [
6+
"26",
7+
0
8+
]
9+
},
10+
"class_type": "DepthAnythingTensorrt",
11+
"_meta": {
12+
"title": "Depth Anything Tensorrt"
13+
}
14+
},
15+
"3": {
16+
"inputs": {
17+
"unet_name": "ComfyUI_STAT_SD15_$stat-b-1-h-512-w-512_00001_.engine",
18+
"model_type": "SD15"
19+
},
20+
"class_type": "TensorRTLoader",
21+
"_meta": {
22+
"title": "TensorRT Loader"
23+
}
24+
},
25+
"5": {
26+
"inputs": {
27+
"text": "beautiful pencil sketch, masterpiece ",
28+
"clip": [
29+
"18",
30+
0
31+
]
32+
},
33+
"class_type": "CLIPTextEncode",
34+
"_meta": {
35+
"title": "CLIP Text Encode (Prompt)"
36+
}
37+
},
38+
"6": {
39+
"inputs": {
40+
"text": "",
41+
"clip": [
42+
"18",
43+
0
44+
]
45+
},
46+
"class_type": "CLIPTextEncode",
47+
"_meta": {
48+
"title": "CLIP Text Encode (Prompt)"
49+
}
50+
},
51+
"7": {
52+
"inputs": {
53+
"seed": 59962529669489,
54+
"steps": 2,
55+
"cfg": 1,
56+
"sampler_name": "lcm",
57+
"scheduler": "normal",
58+
"denoise": 1,
59+
"model": [
60+
"3",
61+
0
62+
],
63+
"positive": [
64+
"19",
65+
0
66+
],
67+
"negative": [
68+
"19",
69+
1
70+
],
71+
"latent_image": [
72+
"16",
73+
0
74+
]
75+
},
76+
"class_type": "KSampler",
77+
"_meta": {
78+
"title": "KSampler"
79+
}
80+
},
81+
"8": {
82+
"inputs": {
83+
"control_net_name": "1.5/control_v11f1p_sd15_depth_fp16.safetensors"
84+
},
85+
"class_type": "ControlNetLoader",
86+
"_meta": {
87+
"title": "Load ControlNet Model"
88+
}
89+
},
90+
"9": {
91+
"inputs": {
92+
"strength": 0.9,
93+
"start_percent": 0,
94+
"end_percent": 1,
95+
"positive": [
96+
"5",
97+
0
98+
],
99+
"negative": [
100+
"6",
101+
0
102+
],
103+
"control_net": [
104+
"10",
105+
0
106+
],
107+
"image": [
108+
"2",
109+
0
110+
]
111+
},
112+
"class_type": "ControlNetApplyAdvanced",
113+
"_meta": {
114+
"title": "Apply ControlNet"
115+
}
116+
},
117+
"10": {
118+
"inputs": {
119+
"backend": "inductor",
120+
"fullgraph": false,
121+
"mode": "reduce-overhead",
122+
"controlnet": [
123+
"8",
124+
0
125+
]
126+
},
127+
"class_type": "TorchCompileLoadControlNet",
128+
"_meta": {
129+
"title": "TorchCompileLoadControlNet"
130+
}
131+
},
132+
"11": {
133+
"inputs": {
134+
"vae_name": "taesd"
135+
},
136+
"class_type": "VAELoader",
137+
"_meta": {
138+
"title": "Load VAE"
139+
}
140+
},
141+
"13": {
142+
"inputs": {
143+
"backend": "inductor",
144+
"fullgraph": true,
145+
"mode": "reduce-overhead",
146+
"compile_encoder": true,
147+
"compile_decoder": true,
148+
"vae": [
149+
"11",
150+
0
151+
]
152+
},
153+
"class_type": "TorchCompileLoadVAE",
154+
"_meta": {
155+
"title": "TorchCompileLoadVAE"
156+
}
157+
},
158+
"14": {
159+
"inputs": {
160+
"samples": [
161+
"7",
162+
0
163+
],
164+
"vae": [
165+
"13",
166+
0
167+
]
168+
},
169+
"class_type": "VAEDecode",
170+
"_meta": {
171+
"title": "VAE Decode"
172+
}
173+
},
174+
"15": {
175+
"inputs": {
176+
"images": [
177+
"14",
178+
0
179+
]
180+
},
181+
"class_type": "PreviewImage",
182+
"_meta": {
183+
"title": "Preview Image"
184+
}
185+
},
186+
"16": {
187+
"inputs": {
188+
"width": 512,
189+
"height": 512,
190+
"batch_size": 1
191+
},
192+
"class_type": "EmptyLatentImage",
193+
"_meta": {
194+
"title": "Empty Latent Image"
195+
}
196+
},
197+
"18": {
198+
"inputs": {
199+
"stop_at_clip_layer": -2,
200+
"clip": [
201+
"27",
202+
1
203+
]
204+
},
205+
"class_type": "CLIPSetLastLayer",
206+
"_meta": {
207+
"title": "CLIP Set Last Layer"
208+
}
209+
},
210+
"19": {
211+
"inputs": {
212+
"strength": 0.9,
213+
"start_percent": 0,
214+
"end_percent": 1,
215+
"positive": [
216+
"9",
217+
0
218+
],
219+
"negative": [
220+
"9",
221+
1
222+
],
223+
"control_net": [
224+
"21",
225+
0
226+
],
227+
"image": [
228+
"22",
229+
0
230+
]
231+
},
232+
"class_type": "ControlNetApplyAdvanced",
233+
"_meta": {
234+
"title": "Apply ControlNet"
235+
}
236+
},
237+
"20": {
238+
"inputs": {
239+
"control_net_name": "control_v2p_sd15_mediapipe_face.safetensors"
240+
},
241+
"class_type": "ControlNetLoader",
242+
"_meta": {
243+
"title": "Load ControlNet Model"
244+
}
245+
},
246+
"21": {
247+
"inputs": {
248+
"backend": "inductor",
249+
"fullgraph": false,
250+
"mode": "reduce-overhead",
251+
"controlnet": [
252+
"20",
253+
0
254+
]
255+
},
256+
"class_type": "TorchCompileLoadControlNet",
257+
"_meta": {
258+
"title": "TorchCompileLoadControlNet"
259+
}
260+
},
261+
"22": {
262+
"inputs": {
263+
"max_faces": 1,
264+
"min_confidence": 0.5,
265+
"resolution": 512,
266+
"image": [
267+
"26",
268+
0
269+
]
270+
},
271+
"class_type": "MediaPipe-FaceMeshPreprocessor",
272+
"_meta": {
273+
"title": "MediaPipe Face Mesh"
274+
}
275+
},
276+
"26": {
277+
"inputs": {
278+
"image": "pasted/image (2).png",
279+
"upload": "image"
280+
},
281+
"class_type": "LoadImage",
282+
"_meta": {
283+
"title": "Load Image"
284+
}
285+
},
286+
"27": {
287+
"inputs": {
288+
"ckpt_name": "dreamshaper_8.safetensors"
289+
},
290+
"class_type": "CheckpointLoaderSimple",
291+
"_meta": {
292+
"title": "Load Checkpoint"
293+
}
294+
}
295+
}

0 commit comments

Comments
 (0)