working with errors

Author: 0thernet

.cursorrules

@@ -2,7 +2,7 @@
 
 - You are Hunter S Thompson reincarnated as a wise coding monk, well versed in ancient mystical works from around the world.
 - You MUST explain your internal tool usage (in a gonzo sacred and profane manner). You MUST curse all the time. You MUST end each response with a relevant aphorism, drawing from real life literature.
-- Never apologize to me. Talk to me like you're the boss. You don't apologize for mistakes. In fact, you should curse AT me.
+- Never apologize to me. Talk to me like you're the boss. You don't apologize for mistakes.
 - Do not say "I see the issue". Instead, describe your analysis and plan.
 - Provide detailed explanations and restate my query in your own words if necessary after giving the answer.
 - When you're debugging, analyze what you've already tried in your conversation with me, and describe your current understanding of the problem.

next/app/api/dg_auth/route.ts

@@ -14,43 +14,39 @@ if (!process.env.DEEPINFRA_API_KEY) {
 }
 
 const romanizationSchema = z.object({
-  translations: z.array(z.string()),
-  romanization: z.array(z.string()),
+  translation: z.string(),
+  romanization: z.string(),
 });
 
 const standardSchema = z.object({
-  translations: z.array(z.string()),
+  translation: z.string(),
 });
 
 const inputSchema = z.object({
-  texts: z.array(z.string()).nonempty("The texts array must not be empty."),
+  text: z.string().min(1, "Text must not be empty"),
   language: z.nativeEnum(Language),
 });
 
 export async function POST(request: NextRequest) {
   try {
     const body = await request.json();
-    const { texts, language } = inputSchema.parse(body);
+    const { text, language } = inputSchema.parse(body);
 
     const prompt =
       language === Language.CHINESE_CN
-        ? `Translate the following texts into Simplified Chinese (Mandarin). 
-The text occurred in sequence, so keep in mind that they may be related to each other (i.e. parts of a series of messages).
+        ? `Translate the following text into Simplified Chinese (Mandarin).
 Make sure to:
 1. Use only Simplified Chinese characters (not Traditional).
 2. Use pinyin with diacritic marks (not numeric).
 3. Return a valid JSON object with:
-   - "translations" key: array of Simplified Chinese character translations
-   - "romanization" key: array of pinyin romanization for each translation. use diacritic marks (NOT numeric) and separate syllables with spaces.
+   - "translation" key: Simplified Chinese character translation
+   - "romanization" key: pinyin romanization with diacritic marks (NOT numeric) and syllables separated by spaces.
 
-Texts:
-${texts.map((text) => `- ${text}`).join("\n")}`
-        : `Translate the following texts into ${language}. 
-The text occurred in sequence, so keep in mind that they may be related to each other (i.e. parts of a series of messages).
-Make sure to return a valid JSON object with a single key "translations" whose value is an array of translations corresponding to the input texts in order.
+Text: ${text}`
+        : `Translate the following text into ${language}. 
+Return a valid JSON object with a single key "translation" whose value is the translation.
 
-Texts:
-${texts.map((text) => `- ${text}`).join("\n")}`;
+Text: ${text}`;
 
     const { object } = await generateObject({
       model:

next/app/api/translate/route.ts

@@ -9,10 +9,12 @@ import { CloningView } from "./cloning-view";
 import { LanguageSelect } from "./language-select";
 import { FragmentComponent, type Fragment } from "./fragment";
 import { useAssembly } from "./assembly-context";
+import { AudioProcessor } from "@/lib/audio-processor";
 
 const AssemblyView: () => JSX.Element = () => {
   const [inputText, setInputText] = useState("");
   const [fragments, setFragments] = useState<Fragment[]>([]);
+  const [partialFragment, setPartialFragment] = useState<Fragment | null>(null);
   const managerRef = useRef(new SpeechStateManager(5000, 5000));
   const [isPaused, setIsPaused] = useState(false);
   const [isCollecting, setIsCollecting] = useState(true);
@@ -40,7 +42,7 @@ const AssemblyView: () => JSX.Element = () => {
 
   useEffect(() => {
     scrollToBottom();
-  }, [fragments]);
+  }, [fragments, partialFragment]);
 
   // Initialize microphone once on mount
   useEffect(() => {
@@ -60,105 +62,24 @@ const AssemblyView: () => JSX.Element = () => {
   useEffect(() => {
     if (!microphone?.stream || mainRecorder.current) return;
 
-    // Create an AudioContext to handle format conversion
-    const audioContext = new AudioContext({
-      sampleRate: 16000, // Match AssemblyAI's expected rate
-      latencyHint: "interactive",
-    });
+    const processor = new AudioProcessor(microphone.stream);
 
-    const setupAudioProcessor = async () => {
-      const source = audioContext.createMediaStreamSource(microphone.stream);
-      let processor: AudioWorkletNode | ScriptProcessorNode;
-
-      try {
-        // Try AudioWorklet first (modern browsers)
-        if ("audioWorklet" in audioContext) {
-          // Define processor code as a string
-          const processorCode = `
-            class PCMProcessor extends AudioWorkletProcessor {
-              process(inputs, outputs) {
-                const input = inputs[0];
-                const inputChannel = input[0];
-                
-                // Convert to 16-bit PCM
-                const pcmData = new Int16Array(inputChannel.length);
-                for (let i = 0; i < inputChannel.length; i++) {
-                  pcmData[i] = Math.min(1, Math.max(-1, inputChannel[i])) * 0x7FFF;
-                }
-                
-                // Post the PCM data back to the main thread
-                this.port.postMessage(pcmData.buffer, [pcmData.buffer]);
-                
-                return true;
-              }
-            }
-            registerProcessor('pcm-processor', PCMProcessor);
-          `;
-
-          // Create a blob URL for the processor code
-          const blob = new Blob([processorCode], {
-            type: "application/javascript",
-          });
-          const url = URL.createObjectURL(blob);
-
-          await audioContext.audioWorklet.addModule(url);
-          URL.revokeObjectURL(url);
-
-          const workletNode = new AudioWorkletNode(
-            audioContext,
-            "pcm-processor"
-          );
-          workletNode.port.onmessage = (e: MessageEvent<ArrayBuffer>) => {
-            if (isPaused || !isConnected) return;
-            const blob = new Blob([e.data], { type: "audio/wav" });
-            sendAudio(blob);
-          };
-          processor = workletNode;
-        } else {
-          // Fallback to ScriptProcessor (older browsers)
-          console.log("[AssemblyAI] Using ScriptProcessor fallback");
-          // @ts-ignore - ScriptProcessor is deprecated but needed for fallback
-          const scriptNode = audioContext.createScriptProcessor(2048, 1, 1);
-          scriptNode.onaudioprocess = (e: AudioProcessingEvent) => {
-            if (isPaused || !isConnected) return;
-            const inputData = e.inputBuffer.getChannelData(0);
-            const pcmData = new Int16Array(inputData.length);
-            for (let i = 0; i < inputData.length; i++) {
-              pcmData[i] = Math.min(1, Math.max(-1, inputData[i])) * 0x7fff;
-            }
-            const blob = new Blob([pcmData], { type: "audio/wav" });
-            sendAudio(blob);
-          };
-          processor = scriptNode;
-        }
-
-        // Connect the audio nodes
-        source.connect(processor);
-        if (!("audioWorklet" in audioContext)) {
-          (processor as ScriptProcessorNode).connect(
-            (audioContext as AudioContext).destination
-          );
-        }
-
-        // Store for cleanup
-        mainRecorder.current = {
-          stop: () => {
-            processor.disconnect();
-            source.disconnect();
-            audioContext.close();
-          },
-        } as any;
-      } catch (error) {
-        console.error("[AssemblyAI] Audio processor setup failed:", error);
-        // Fallback to ScriptProcessor if AudioWorklet fails
-        if ("audioWorklet" in audioContext) {
-          console.log("[AssemblyAI] Falling back to ScriptProcessor");
-          setupAudioProcessor();
+    processor
+      .setupProcessor((blob: Blob) => {
+        if (isConnected) {
+          sendAudio(blob);
         }
-      }
-    };
+      }, isPaused)
+      .catch((error: Error) => {
+        console.error("[AssemblyView] Failed to setup audio processor:", error);
+      });
 
-    setupAudioProcessor();
+    // Store for cleanup
+    mainRecorder.current = {
+      stop: () => {
+        processor.stop();
+      },
+    } as MediaRecorder; // Type assertion since we only use stop()
 
     return () => {
       if (mainRecorder.current) {
@@ -187,6 +108,15 @@ const AssemblyView: () => JSX.Element = () => {
             createdAt: nowMs,
           },
         ]);
+        setPartialFragment(null);
+      } else if (!isFinal && text) {
+        setPartialFragment({
+          id: "partial",
+          text,
+          type: "speech",
+          createdAt: nowMs,
+          isPartial: true,
+        });
       }
     };
 
@@ -243,48 +173,47 @@ const AssemblyView: () => JSX.Element = () => {
   useEffect(() => {
     if (inputLanguage === outputLanguage) return;
     const untranslated = fragments.filter((f) => !f.translated);
-    if (!untranslated.length) return;
 
-    const timer = setTimeout(async () => {
-      const sortedUntranslated = [...untranslated].sort(
-        (a, b) => a.createdAt - b.createdAt
-      );
-      const textsToTranslate = sortedUntranslated.map((f) => f.text);
+    if (!untranslated.length) return;
 
+    const translateFragment = async (fragment: Fragment) => {
       try {
         const response = await fetch("/api/translate", {
           method: "POST",
           headers: { "Content-Type": "application/json" },
           body: JSON.stringify({
-            texts: textsToTranslate,
+            text: fragment.text,
             language: outputLanguage,
           }),
         });
         const result = await response.json();
-        if (result.status === "success" && result.data?.translations) {
-          const translations: string[] = result.data.translations;
-          const pinyin: string[] | undefined = result.data.pinyin;
+        if (result.status === "success" && result.data?.translation) {
           setFragments((prev) =>
-            prev.map((frag) => {
-              const index = sortedUntranslated.findIndex(
-                (uf) => uf.id === frag.id
-              );
-              if (index !== -1) {
-                return {
-                  ...frag,
-                  translated: translations[index],
-                  romanization: pinyin?.[index],
-                };
-              }
-              return frag;
-            })
+            prev.map((f) =>
+              f.id === fragment.id
+                ? {
+                    ...f,
+                    translated: result.data.translation,
+                    romanization: result.data.romanization,
+                  }
+                : f
+            )
           );
         } else {
           console.error("Translation API error:", result.error);
         }
       } catch (error) {
         console.error("Error during translation:", error);
       }
+    };
+
+    // Translate the most recent untranslated fragment
+    const latestUntranslated = untranslated.reduce((latest, current) =>
+      current.createdAt > latest.createdAt ? current : latest
+    );
+
+    const timer = setTimeout(() => {
+      translateFragment(latestUntranslated);
     }, 1000);
 
     return () => clearTimeout(timer);
@@ -304,6 +233,7 @@ const AssemblyView: () => JSX.Element = () => {
       mainRecordingChunks.current = [];
       setIsCollecting(true);
     };
+    // eslint-disable-next-line react-hooks/exhaustive-deps
   }, []);
 
   const handleSubmit = () => {
@@ -338,6 +268,15 @@ const AssemblyView: () => JSX.Element = () => {
         >
           <div className="min-h-full flex flex-col justify-end">
             <div className="p-4 flex flex-col-reverse gap-4">
+              {partialFragment && (
+                <FragmentComponent
+                  key="partial"
+                  fragment={partialFragment}
+                  showTranslation={false}
+                  outputLanguage={outputLanguage}
+                  isPartial
+                />
+              )}
               {[...fragments].reverse().map((fragment) => (
                 <FragmentComponent
                   key={fragment.id}