ryanontheinside
diff --git a/‎.devcontainer/Dockerfile
+5-3 b/‎.devcontainer/Dockerfile
+5-3
diff --git a/‎.devcontainer/README.md
+49-37 b/‎.devcontainer/README.md
+49-37
diff --git a/‎docker/entrypoint.sh ‎.devcontainer/entrypoint.sh
+3-1 b/‎docker/entrypoint.sh ‎.devcontainer/entrypoint.sh
+3-1
diff --git a/‎.devcontainer/post-create.sh
+11-1 b/‎.devcontainer/post-create.sh
+11-1
diff --git a/‎README.md
+11 b/‎README.md
+11
diff --git a/‎configs/models.yaml
+20-1 b/‎configs/models.yaml
+20-1
diff --git a/‎configs/nodes.yaml
+9-2 b/‎configs/nodes.yaml
+9-2
diff --git a/‎docker/Dockerfile.base
+1 b/‎docker/Dockerfile.base
+1
diff --git a/‎nodes/audio_utils/__init__.py
+7 b/‎nodes/audio_utils/__init__.py
+7
diff --git a/‎nodes/audio_utils/load_audio_tensor.py
+52 b/‎nodes/audio_utils/load_audio_tensor.py
+52
diff --git a/‎nodes/audio_utils/pitch_shift.py
+32 b/‎nodes/audio_utils/pitch_shift.py
+32
diff --git a/‎nodes/audio_utils/save_audio_tensor.py
+25 b/‎nodes/audio_utils/save_audio_tensor.py
+25
diff --git a/‎nodes/tensor_utils/load_tensor.py
+3-2 b/‎nodes/tensor_utils/load_tensor.py
+3-2
@@ -33,6 +33,8 @@ RUN mkdir -p /etc/supervisor/conf.d
 COPY .devcontainer/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
 
 WORKDIR /workspace/comfystream
-COPY docker/entrypoint.sh /workspace/comfystream/docker/entrypoint.sh
-RUN chmod +x /workspace/comfystream/docker/entrypoint.sh
-ENTRYPOINT ["/workspace/comfystream/docker/entrypoint.sh"]
+COPY .devcontainer/entrypoint.sh /workspace/comfystream/.devcontainer/entrypoint.sh
+RUN chmod +x /workspace/comfystream/.devcontainer/entrypoint.sh
+
+ENTRYPOINT ["/workspace/comfystream/.devcontainer/entrypoint.sh"]
+CMD ["/bin/bash"]
@@ -65,22 +65,68 @@ Replace `/path/to/your/model-files` and `path/to/your/output-files` with the pat
 2. From VS Code, reload the folder as a devcontainer:
    - Open the Command Palette (`Ctrl+Shift+P` or `Cmd+Shift+P` on macOS).
    - Select `Remote-Containers: Reopen in Container`.
+3. Wait for the container to build and start.
 
-### Download models
+### Starting ComfyUI
+
+Start ComfyUI:
+
+```sh
+cd /workspace/comfystream/ComfyUI
+conda activate comfyui
+python main.py --listen
+```
+
+When using TensorRT engine enabled workflows, you should include the `---disable-cuda-malloc` flag as shown below:
+
+```sh
+cd /workspace/comfystream/ComfyUI
+conda activate comfyui
+python main.py --listen --disable-cuda-malloc
+```
+
+### Starting ComfyStream
+
+Start ComfyStream:
+
+```sh
+cd /workspace/comfystream
+conda activate comfystream
+python server/app.py --workspace /workspace/ComfyUI --media-ports=5678 --host=0.0.0.0 --port 8889
+```
+
+Optionally, you can also start the [ComfyStream UI](../README.md#run-ui) to view the stream:
+
+```sh
+cd /workspace/comfystream/ui
+npm run dev:https
+```
+
+### Running Example Workflows
+
+To run example workflows, you need to download models and build TensorRT engines. You can do this from within the dev container by running the following command in the terminal:
+
+```sh
+prepare_examples
+```
+
+Alternatively, you can follow the steps below.
+
+#### Download models
 
 From within the **dev container**, download models to run the example workflows:
 
 ```sh
 cd /workspace/comfystream
 conda activate comfystream
-python src/comfystream/scripts/setup_models.py --workspace /ComfyUI
+python src/comfystream/scripts/setup_models.py --workspace /workspace/ComfyUI
 ```
 
 For more info about configuring model downloads, see [src/comfystream/scripts/README.md](../src/comfystream/scripts/README.md)
 
 By following these steps, you should be able to set up and run your development container for ComfyStream efficiently.
 
-### Building the DepthAnything Engine
+#### Building the DepthAnything Engine
 
 After downloading models, it is necessary to compile TensorRT engines for the example workflow.
 
@@ -113,40 +159,6 @@ Alternatively, you may activate an environment manually with `conda activate com
 
 > [!NOTE] For more information, see [Python environments in VS Code](https://code.visualstudio.com/docs/python/environments)
 
-### Starting ComfyUI
-
-Start ComfyUI:
-
-```sh
-cd /workspace/comfystream/ComfyUI
-conda activate comfyui
-python main.py --listen
-```
-
-When using TensorRT engine enabled workflows, you should include the `---disable-cuda-malloc` flag as shown below:
-
-```sh
-cd /workspace/comfystream/ComfyUI
-conda activate comfyui
-python main.py --listen --disable-cuda-malloc
-```
-
-### Starting ComfyStream
-
-Start ComfyStream:
-
-```sh
-cd /workspace/comfystream
-conda activate comfystream
-python server/app.py --workspace /workspace/ComfyUI --media-ports=5678 --host=0.0.0.0 --port 8889
-```
-
-Optionally, you can also start the [ComfyStream UI](../README.md#run-ui) to view the stream:
-
-```sh
-cd /workspace/comfystream/ui
-npm run dev:https
-```
 
 ## Additional Resources
 
 
@@ -52,7 +52,9 @@ fi
 
 if [ "$1" = "--server" ]; then
     /usr/bin/supervisord -c /etc/supervisor/supervisord.conf
+    shift
 fi
 
 cd /workspace/comfystream
-/bin/bash
+
+exec "$@"
@@ -1,4 +1,14 @@
 #!/bin/bash
 
+chmod +x /workspace/comfystream/.devcontainer/entrypoint.sh
 cd /workspace/comfystream
-/bin/bash
+
+# Install Comfystream in editable mode.
+echo -e "\e[32mInstalling Comfystream in editable mode...\e[0m"
+/workspace/miniconda3/envs/comfystream/bin/python3 -m pip install -e . > /dev/null
+
+# Create a symlink to the entrypoint script.
+echo 'alias prepare_examples="/workspace/comfystream/.devcontainer/entrypoint.sh --download-models --build-engines"' >> ~/.bashrc
+echo -e "\e[32mContainer ready! Run 'prepare_examples' to download models and build engines for example workflows.\e[0m"
+
+/bin/bash
@@ -7,6 +7,7 @@ This repo also includes a WebRTC server and UI that uses comfystream to support
 - [comfystream](#comfystream)
   - [Quick Start](#quick-start)
     - [Docker DevContainer](#docker-devcontainer)
+    - [Docker Image](#docker-image)
   - [Download Models](#download-models)
   - [Install package](#install-package)
     - [Custom Nodes](#custom-nodes)
@@ -26,6 +27,16 @@ For other installation options, refer to [Install ComfyUI and ComfyStream](https
 
 For additional information, refer to the remaining sections below.
 
+### Docker Image
+
+You can quickly deploy ComfyStream using the docker image `livepeer/comfystream`
+
+Refer to the documentation at [https://pipelines.livepeer.org/docs/technical/getting-started/install-comfystream](https://pipelines.livepeer.org/docs/technical/getting-started/install-comfystream) for instructions to run locally or on a remote server.
+
+#### RunPod
+
+The RunPod template [livepeer-comfystream](https://runpod.io/console/deploy?template=w01m180vxx&ref=u8tlskew) can be used to deploy to RunPod.
+
 ## Download Models
 
 Refer to [src/comfystream/scripts/README.md](src/comfystream/scripts/README.md) for instructions to download commonly used models.
 
@@ -37,4 +37,23 @@ models:
     name: "ControlNet Depth"
     url: "https://huggingface.co/comfyanonymous/ControlNet-v1-1_fp16_safetensors/resolve/main/control_v11f1p_sd15_depth_fp16.safetensors"
     path: "controlnet/control_v11f1p_sd15_depth_fp16.safetensors"
-    type: "controlnet" 
+    type: "controlnet" 
+
+  controlnet-mediapipe-face:
+    name: "ControlNet MediaPipe Face"
+    url: "https://huggingface.co/CrucibleAI/ControlNetMediaPipeFace/resolve/main/control_v2p_sd15_mediapipe_face.safetensors"
+    path: "controlnet/control_v2p_sd15_mediapipe_face.safetensors"
+    type: "controlnet"
+
+  # Lora models
+  ral-polygon-sd15:
+    name: "ral-polygon-sd15"
+    url: "https://huggingface.co/Livepeer-Studio/comfystream_loras/resolve/main/ral-polygon-sd15.safetensors"
+    path: "loras/SD1.5/ral-polygon-sd15.safetensors"
+    type: "lora"
+
+  ral-chrome-sdxl:
+    name: "ral-chrome-sd15"
+    url: "https://civitai.com/api/download/models/276570?type=Model&format=SafeTensor"
+    path: "loras/SD1.5/ral-chrome-sd15.safetensors"
+    type: "lora"
@@ -2,8 +2,8 @@ nodes:
   # Core TensorRT nodes
   comfyui-tensorrt:
     name: "ComfyUI TensorRT"
-    url: "https://github.com/yondonfu/ComfyUI_TensorRT"
-    branch: "quantization_with_controlnet_fixes"
+    url: "https://github.com/ryanontheinside/ComfyUI_TensorRT"
+    branch: "hs-fix"
     type: "tensorrt"
 
   comfyui-depthanything-tensorrt:
@@ -61,3 +61,10 @@ nodes:
     name: "ComfyUI Background Edit"
     url: "https://github.com/yondonfu/ComfyUI-Background-Edit"
     type: "utility"
+
+  comfyui_controlnet_aux:
+    name: "ComfyUI ControlNet Auxiliary"
+    url: "https://github.com/Fannovel16/comfyui_controlnet_aux"
+    type: "utility"
+    dependencies: 
+      - "mediapipe==0.10.8"
@@ -9,6 +9,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -
     wget \
     nano \
     socat \
+    libsndfile1 \
     build-essential llvm tk-dev \
     && rm -rf /var/lib/apt/lists/*
 
 
@@ -0,0 +1,7 @@
+from .load_audio_tensor import LoadAudioTensor
+from .save_audio_tensor import SaveAudioTensor
+from .pitch_shift import PitchShifter
+
+NODE_CLASS_MAPPINGS = {"LoadAudioTensor": LoadAudioTensor, "SaveAudioTensor": SaveAudioTensor, "PitchShifter": PitchShifter}
+
+__all__ = ["NODE_CLASS_MAPPINGS"]
@@ -0,0 +1,52 @@
+import numpy as np
+
+from comfystream import tensor_cache
+
+class LoadAudioTensor:
+    CATEGORY = "audio_utils"
+    RETURN_TYPES = ("WAVEFORM", "INT")
+    FUNCTION = "execute"
+    
+    def __init__(self):
+        self.audio_buffer = np.empty(0, dtype=np.int16)
+        self.buffer_samples = None
+        self.sample_rate = None
+    
+    @classmethod
+    def INPUT_TYPES(s):
+        return {
+            "required": {
+                "buffer_size": ("FLOAT", {"default": 500.0}),
+            }
+        }
+    
+    @classmethod
+    def IS_CHANGED():
+        return float("nan")
+    
+    def execute(self, buffer_size):
+        if self.sample_rate is None or self.buffer_samples is None:
+            frame = tensor_cache.audio_inputs.get(block=True)
+            self.sample_rate = frame.sample_rate
+            self.buffer_samples = int(self.sample_rate * buffer_size / 1000)
+            self.leftover = frame.side_data.input
+        
+        if self.leftover.shape[0] < self.buffer_samples:
+            chunks = [self.leftover] if self.leftover.size > 0 else []
+            total_samples = self.leftover.shape[0]
+            
+            while total_samples < self.buffer_samples:
+                frame = tensor_cache.audio_inputs.get(block=True)
+                if frame.sample_rate != self.sample_rate:
+                    raise ValueError("Sample rate mismatch")
+                chunks.append(frame.side_data.input)
+                total_samples += frame.side_data.input.shape[0]
+            
+            merged_audio = np.concatenate(chunks, dtype=np.int16)
+            buffered_audio = merged_audio[:self.buffer_samples]
+            self.leftover = merged_audio[self.buffer_samples:]
+        else:
+            buffered_audio = self.leftover[:self.buffer_samples]
+            self.leftover = self.leftover[self.buffer_samples:]
+                
+        return buffered_audio, self.sample_rate
@@ -0,0 +1,32 @@
+import numpy as np
+import librosa
+
+class PitchShifter:
+    CATEGORY = "audio_utils"
+    RETURN_TYPES = ("WAVEFORM", "INT")
+    FUNCTION = "execute"
+    
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "audio": ("WAVEFORM",),
+                "sample_rate": ("INT",),
+                "pitch_shift": ("FLOAT", {
+                    "default": 4.0, 
+                    "min": 0.0, 
+                    "max": 12.0, 
+                    "step": 0.5
+                }),
+            }
+        }
+
+    @classmethod
+    def IS_CHANGED(cls):
+        return float("nan")
+
+    def execute(self, audio, sample_rate, pitch_shift):
+        audio_float = audio.astype(np.float32) / 32768.0
+        shifted_audio = librosa.effects.pitch_shift(y=audio_float, sr=sample_rate, n_steps=pitch_shift)
+        shifted_int16 = np.clip(shifted_audio * 32768.0, -32768, 32767).astype(np.int16)
+        return shifted_int16, sample_rate
@@ -0,0 +1,25 @@
+from comfystream import tensor_cache
+
+class SaveAudioTensor:
+    CATEGORY = "audio_utils"
+    RETURN_TYPES = ()
+    FUNCTION = "execute"
+    OUTPUT_NODE = True
+
+
+    @classmethod
+    def INPUT_TYPES(s):
+        return {
+            "required": {
+                "audio": ("WAVEFORM",)
+            }
+        }
+
+    @classmethod
+    def IS_CHANGED(s):
+        return float("nan")
+
+    def execute(self, audio):
+        tensor_cache.audio_outputs.put_nowait(audio)
+        return (audio,)
+
@@ -15,5 +15,6 @@ def IS_CHANGED():
         return float("nan")
 
     def execute(self):
-        input = tensor_cache.inputs.pop()
-        return (input,)
+        frame = tensor_cache.image_inputs.get(block=True)
+        frame.side_data.skipped = False
+        return (frame.side_data.input,)