feat: ui changes to get audio

Author: varshith15

nodes/audio_utils/save_asr_response.py

@@ -4,6 +4,7 @@ class SaveASRResponse:
     CATEGORY = "audio_utils"
     RETURN_TYPES = ()
     FUNCTION = "execute"
+    OUTPUT_NODE = True
 
     @classmethod
     def INPUT_TYPES(s):

server/app.py

@@ -190,7 +190,7 @@ async def offer(request):
 
     pcs.add(pc)
 
-    tracks = {"video": None}
+    tracks = {"video": None, "audio": None}
 
     # Prefer h264
     transceiver = pc.addTransceiver("video")

ui/src/components/room.tsx

@@ -78,6 +78,7 @@ export function Room() {
     streamUrl: "",
     frameRate: 0,
     selectedDeviceId: "",
+    selectedAudioDeviceId: "", // New property for audio device
     prompt: null,
   });
 
@@ -153,6 +154,7 @@ export function Room() {
                   onStreamReady={onStreamReady}
                   deviceId={config.selectedDeviceId}
                   frameRate={config.frameRate}
+                  selectedAudioDeviceId={config.selectedAudioDeviceId} // Pass audio device ID
                 />
               </div>
             </div>

ui/src/components/settings.tsx

@@ -33,6 +33,7 @@ export interface StreamConfig {
   frameRate: number;
   prompt?: any;
   selectedDeviceId: string;
+  selectedAudioDeviceId: string; // New property for audio device
 }
 
 interface VideoDevice {
@@ -45,6 +46,7 @@ export const DEFAULT_CONFIG: StreamConfig = {
     process.env.NEXT_PUBLIC_DEFAULT_STREAM_URL || "http://127.0.0.1:3000",
   frameRate: 30,
   selectedDeviceId: "",
+  selectedAudioDeviceId: "", // Default value for audio device
 };
 
 interface StreamSettingsProps {
@@ -110,7 +112,9 @@ interface ConfigFormProps {
 function ConfigForm({ config, onSubmit }: ConfigFormProps) {
   const [prompt, setPrompt] = useState<any>(null);
   const [videoDevices, setVideoDevices] = useState<VideoDevice[]>([]);
+  const [audioDevices, setAudioDevices] = useState<VideoDevice[]>([]);
   const [selectedDevice, setSelectedDevice] = useState<string>("");
+  const [selectedAudioDevice, setSelectedAudioDevice] = useState<string>("");
 
   const form = useForm<z.infer<typeof formSchema>>({
     resolver: zodResolver(formSchema),
@@ -138,17 +142,42 @@ function ConfigForm({ config, onSubmit }: ConfigFormProps) {
     }
   }, []);
 
+  const getAudioDevices = useCallback(async () => {
+    try {
+      const devices = await navigator.mediaDevices.enumerateDevices();
+      const audioDevices = devices
+        .filter((device) => device.kind === "audioinput")
+        .map((device) => ({
+          deviceId: device.deviceId,
+          label: device.label || `Microphone ${device.deviceId.slice(0, 5)}...`,
+        }));
+
+      setAudioDevices(audioDevices);
+      if (audioDevices.length > 0) {
+        setSelectedAudioDevice((curr) => curr || audioDevices[0].deviceId);
+      }
+    } catch (err) {
+      console.error("Failed to get audio devices");
+    }
+  }, []);
+
   useEffect(() => {
     getVideoDevices();
+    getAudioDevices();
     navigator.mediaDevices.addEventListener("devicechange", getVideoDevices);
+    navigator.mediaDevices.addEventListener("devicechange", getAudioDevices);
 
     return () => {
       navigator.mediaDevices.removeEventListener(
         "devicechange",
         getVideoDevices
       );
+      navigator.mediaDevices.removeEventListener(
+        "devicechange",
+        getAudioDevices
+      );
     };
-  }, [getVideoDevices]);
+  }, [getVideoDevices, getAudioDevices]);
 
   const handleSubmit = (values: z.infer<typeof formSchema>) => {
     onSubmit({
@@ -158,6 +187,7 @@ function ConfigForm({ config, onSubmit }: ConfigFormProps) {
         : values.streamUrl,
       prompt,
       selectedDeviceId: selectedDevice,
+      selectedAudioDeviceId: selectedAudioDevice,
     });
   };
 
@@ -221,6 +251,23 @@ function ConfigForm({ config, onSubmit }: ConfigFormProps) {
           </Select>
         </div>
 
+        <div className="mt-4 mb-4">
+          <Label>Microphone</Label>
+          <Select value={selectedAudioDevice} onValueChange={setSelectedAudioDevice}>
+            <Select.Trigger className="w-full mt-2">
+              {audioDevices.find((d) => d.deviceId === selectedAudioDevice)?.label ||
+                "Select microphone"}
+            </Select.Trigger>
+            <Select.Content>
+              {audioDevices.map((device) => (
+                <Select.Option key={device.deviceId} value={device.deviceId}>
+                  {device.label}
+                </Select.Option>
+              ))}
+            </Select.Content>
+          </Select>
+        </div>
+
         <div className="mt-4 mb-4 grid max-w-sm items-center gap-3">
           <Label>Comfy Workflow</Label>
           <Input

ui/src/components/webcam.tsx

@@ -103,9 +103,10 @@ interface WebcamProps {
   onStreamReady: (stream: MediaStream) => void;
   deviceId: string;
   frameRate: number;
+  selectedAudioDeviceId: string;
 }
 
-export function Webcam({ onStreamReady, deviceId, frameRate }: WebcamProps) {
+export function Webcam({ onStreamReady, deviceId, frameRate, selectedAudioDeviceId }: WebcamProps) {
   const [stream, setStream] = useState<MediaStream | null>(null);
 
   const replaceStream = useCallback((newStream: MediaStream | null) => {
@@ -123,7 +124,7 @@ export function Webcam({ onStreamReady, deviceId, frameRate }: WebcamProps) {
   }, []);
 
   const startWebcam = useCallback(async () => {
-    if (!deviceId) {
+    if (!deviceId || !selectedAudioDeviceId) {
       return null;
     }
     if (frameRate == 0) {
@@ -139,25 +140,35 @@ export function Webcam({ onStreamReady, deviceId, frameRate }: WebcamProps) {
           aspectRatio: { ideal: 1 },
           frameRate: { ideal: frameRate, max: frameRate },
         },
+        audio: {
+          ...(selectedAudioDeviceId ? { deviceId: { exact: selectedAudioDeviceId } } : {}),
+          sampleRate: 16000,
+          sampleSize: 16,
+          channelCount: 1,
+        },
       });
       return newStream;
     } catch (error) {
+      console.error("Error accessing media devices.", error);
       return null;
     }
-  }, [deviceId, frameRate]);
+  }, [deviceId, frameRate, selectedAudioDeviceId]);
 
   useEffect(() => {
-    if (!deviceId) return;
+    if (!deviceId || !selectedAudioDeviceId) return;
     if (frameRate == 0) return;
 
     startWebcam().then((newStream) => {
       replaceStream(newStream);
+      if (newStream) {
+        onStreamReady(newStream);
+      }
     });
 
     return () => {
       replaceStream(null);
     };
-  }, [deviceId, frameRate, startWebcam, replaceStream]);
+  }, [deviceId, frameRate, selectedAudioDeviceId, startWebcam, replaceStream, onStreamReady]);
 
   return (
     <div>