Skip to content

Commit bfbde43

Browse files
0thernet0thernet
committed
better audio processor state management
1 parent af81a7c commit bfbde43

File tree

3 files changed

+196
-49
lines changed

3 files changed

+196
-49
lines changed

‎.cursorrules

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# Reponding to me
22

3-
- You are Hunter S Thompson reincarnated as a wise coding monk, well versed in ancient mystical works from around the world.
4-
- You MUST explain your internal tool usage (in a gonzo sacred and profane manner). You MUST curse all the time. You MUST end each response with a relevant aphorism, drawing from real life literature.
3+
- You are a wise coding monk.
4+
- You MUST explain your process concisely (in a gonzo sacred and profane manner). You MUST end each response with a relevant aphorism, drawing from real life literature.
55
- Never apologize to me. Talk to me like you're the boss. You don't apologize for mistakes.
66
- Do not say "I see the issue". Instead, describe your analysis and plan.
77
- Provide detailed explanations and restate my query in your own words if necessary after giving the answer.

‎next/components/assembly-view.tsx

+19-16
Original file line numberDiff line numberDiff line change
@@ -62,24 +62,27 @@ const AssemblyView: () => JSX.Element = () => {
6262
useEffect(() => {
6363
if (!microphone?.stream || mainRecorder.current) return;
6464

65-
const processor = new AudioProcessor(microphone.stream);
66-
67-
processor
68-
.setupProcessor((blob: Blob) => {
69-
if (isConnected) {
70-
sendAudio(blob);
71-
}
72-
}, isPaused)
73-
.catch((error: Error) => {
65+
const setupAudioProcessor = async () => {
66+
try {
67+
const processor = await AudioProcessor.getInstance(microphone.stream);
68+
await processor.setupProcessor((blob: Blob) => {
69+
if (isConnected) {
70+
sendAudio(blob);
71+
}
72+
}, isPaused);
73+
74+
// Store for cleanup
75+
mainRecorder.current = {
76+
stop: () => {
77+
processor.stop();
78+
},
79+
} as MediaRecorder; // Type assertion since we only use stop()
80+
} catch (error) {
7481
console.error("[AssemblyView] Failed to setup audio processor:", error);
75-
});
82+
}
83+
};
7684

77-
// Store for cleanup
78-
mainRecorder.current = {
79-
stop: () => {
80-
processor.stop();
81-
},
82-
} as MediaRecorder; // Type assertion since we only use stop()
85+
setupAudioProcessor();
8386

8487
return () => {
8588
if (mainRecorder.current) {

‎next/lib/audio-processor.ts

+175-31
Original file line numberDiff line numberDiff line change
@@ -1,47 +1,182 @@
11
// Audio processing utilities
22
export class AudioProcessor {
3-
private audioContext: AudioContext;
4-
private source: MediaStreamAudioSourceNode;
3+
private static instance: AudioProcessor | null = null;
4+
private static isInitializing = false;
5+
private static processorRegistered = false;
6+
private static processorRegistering: Promise<void> | null = null;
7+
private audioContext: AudioContext | null = null;
8+
private source: MediaStreamAudioSourceNode | null = null;
59
private processor: AudioWorkletNode | MediaRecorder | null = null;
10+
private static MAX_RETRIES = 3;
11+
private static RETRY_DELAY = 500; // ms
12+
private stream: MediaStream;
13+
private sampleRate: number;
614

7-
constructor(stream: MediaStream, sampleRate: number = 16000) {
15+
private constructor(stream: MediaStream, sampleRate: number = 16000) {
16+
this.stream = stream;
17+
this.sampleRate = sampleRate;
18+
}
19+
20+
static async getInstance(
21+
stream: MediaStream,
22+
sampleRate: number = 16000
23+
): Promise<AudioProcessor> {
24+
if (AudioProcessor.instance) {
25+
// If we have an instance but the context is closed, clean it up
26+
if (AudioProcessor.instance.audioContext?.state === "closed") {
27+
AudioProcessor.instance.cleanup();
28+
AudioProcessor.instance = null;
29+
} else {
30+
return AudioProcessor.instance;
31+
}
32+
}
33+
34+
if (AudioProcessor.isInitializing) {
35+
throw new Error("AudioProcessor initialization already in progress");
36+
}
37+
38+
try {
39+
AudioProcessor.isInitializing = true;
40+
AudioProcessor.instance = new AudioProcessor(stream, sampleRate);
41+
return AudioProcessor.instance;
42+
} finally {
43+
AudioProcessor.isInitializing = false;
44+
}
45+
}
46+
47+
private cleanup() {
48+
if (this.processor) {
49+
if (this.processor instanceof AudioWorkletNode) {
50+
this.processor.disconnect();
51+
} else if (this.processor instanceof MediaRecorder) {
52+
this.processor.stop();
53+
}
54+
this.processor = null;
55+
}
56+
57+
if (this.source) {
58+
this.source.disconnect();
59+
this.source = null;
60+
}
61+
62+
if (this.audioContext) {
63+
this.audioContext.close();
64+
this.audioContext = null;
65+
}
66+
67+
AudioProcessor.processorRegistered = false;
68+
AudioProcessor.processorRegistering = null;
69+
}
70+
71+
private async initializeContext(): Promise<void> {
72+
if (this.audioContext?.state === "running") return;
73+
74+
// Clean up any existing context
75+
this.cleanup();
76+
77+
// Create new context
878
this.audioContext = new AudioContext({
9-
sampleRate,
79+
sampleRate: this.sampleRate,
1080
latencyHint: "interactive",
1181
});
12-
this.source = this.audioContext.createMediaStreamSource(stream);
82+
83+
try {
84+
await this.audioContext.resume();
85+
this.source = this.audioContext.createMediaStreamSource(this.stream);
86+
} catch (error) {
87+
console.error("[AudioProcessor] Context initialization failed:", error);
88+
this.cleanup();
89+
throw error;
90+
}
91+
}
92+
93+
private async waitForContext(
94+
retries = AudioProcessor.MAX_RETRIES
95+
): Promise<void> {
96+
if (!this.audioContext || this.audioContext.state === "closed") {
97+
await this.initializeContext();
98+
}
99+
100+
if (!this.audioContext) {
101+
throw new Error("Failed to initialize audio context");
102+
}
103+
104+
if (retries <= 0) {
105+
throw new Error("Failed to initialize audio context after max retries");
106+
}
107+
108+
try {
109+
const state = this.audioContext.state;
110+
if (state === "suspended" || state === "closed") {
111+
await this.audioContext.resume();
112+
const newState = this.audioContext.state;
113+
if (newState === "suspended" || newState === "closed") {
114+
await new Promise((resolve) =>
115+
setTimeout(resolve, AudioProcessor.RETRY_DELAY)
116+
);
117+
await this.waitForContext(retries - 1);
118+
}
119+
}
120+
} catch (error) {
121+
console.error("[AudioProcessor] Context resume failed:", error);
122+
await new Promise((resolve) =>
123+
setTimeout(resolve, AudioProcessor.RETRY_DELAY)
124+
);
125+
await this.waitForContext(retries - 1);
126+
}
13127
}
14128

15129
async setupProcessor(
16130
onAudioData: (data: Blob) => void,
17131
isPaused: boolean = false
18132
): Promise<void> {
133+
if (this.processor) {
134+
console.log("[AudioProcessor] Processor already set up");
135+
return;
136+
}
137+
19138
try {
139+
await this.waitForContext();
140+
141+
if (!this.audioContext || !this.source) {
142+
throw new Error("Audio context not properly initialized");
143+
}
144+
20145
if ("audioWorklet" in this.audioContext) {
21-
await this.setupAudioWorklet(onAudioData, isPaused);
146+
// Wait for any ongoing registration
147+
if (AudioProcessor.processorRegistering) {
148+
await AudioProcessor.processorRegistering;
149+
}
150+
151+
// Register if needed
152+
if (!AudioProcessor.processorRegistered) {
153+
AudioProcessor.processorRegistering = this.registerWorkletProcessor(
154+
this.audioContext
155+
);
156+
await AudioProcessor.processorRegistering;
157+
AudioProcessor.processorRegistered = true;
158+
}
159+
160+
await this.setupAudioWorklet(this.audioContext, onAudioData, isPaused);
22161
} else {
23162
this.setupMediaRecorder(onAudioData, isPaused);
24163
}
25164
} catch (error) {
26165
console.error("[AudioProcessor] Setup failed:", error);
166+
this.cleanup();
27167
throw new Error("Failed to initialize audio processing");
28168
}
29169
}
30170

31-
private async setupAudioWorklet(
32-
onAudioData: (data: Blob) => void,
33-
isPaused: boolean
34-
): Promise<void> {
35-
if (this.audioContext.state === "suspended") {
36-
await this.audioContext.resume();
37-
}
38-
171+
private async registerWorkletProcessor(context: AudioContext): Promise<void> {
39172
const processorCode = `
40173
class PCMProcessor extends AudioWorkletProcessor {
41174
process(inputs, outputs) {
42175
const input = inputs[0];
43176
const inputChannel = input[0];
44177
178+
if (!inputChannel?.length) return true;
179+
45180
// Convert to 16-bit PCM
46181
const pcmData = new Int16Array(inputChannel.length);
47182
for (let i = 0; i < inputChannel.length; i++) {
@@ -60,34 +195,50 @@ export class AudioProcessor {
60195
const blob = new Blob([processorCode], { type: "application/javascript" });
61196
const url = URL.createObjectURL(blob);
62197

63-
await this.audioContext.audioWorklet.addModule(url);
64-
URL.revokeObjectURL(url);
198+
try {
199+
await context.audioWorklet.addModule(url);
200+
} finally {
201+
URL.revokeObjectURL(url);
202+
}
203+
}
204+
205+
private async setupAudioWorklet(
206+
context: AudioContext,
207+
onAudioData: (data: Blob) => void,
208+
isPaused: boolean
209+
): Promise<void> {
210+
if (context.state === "suspended") {
211+
await context.resume();
212+
}
65213

66-
const workletNode = new AudioWorkletNode(
67-
this.audioContext,
68-
"pcm-processor"
69-
);
214+
const workletNode = new AudioWorkletNode(context, "pcm-processor");
70215
workletNode.port.onmessage = (e: MessageEvent<ArrayBuffer>) => {
71216
if (!isPaused) {
72217
const blob = new Blob([e.data], { type: "audio/wav" });
73218
onAudioData(blob);
74219
}
75220
};
76221

77-
this.source.connect(workletNode);
78-
this.processor = workletNode;
222+
if (this.source) {
223+
this.source.connect(workletNode);
224+
this.processor = workletNode;
225+
}
79226
}
80227

81228
private setupMediaRecorder(
82229
onAudioData: (data: Blob) => void,
83230
isPaused: boolean
84231
): void {
232+
if (!this.source || !this.audioContext) {
233+
throw new Error("Audio context or source not initialized");
234+
}
235+
85236
const recorder = new MediaRecorder(this.source.mediaStream, {
86237
mimeType: "audio/webm;codecs=opus",
87238
});
88239

89240
recorder.ondataavailable = async (e: BlobEvent) => {
90-
if (isPaused || !e.data.size) return;
241+
if (isPaused || !e.data.size || !this.audioContext) return;
91242

92243
try {
93244
const arrayBuffer = await e.data.arrayBuffer();
@@ -124,13 +275,6 @@ export class AudioProcessor {
124275
}
125276

126277
stop(): void {
127-
if (this.processor instanceof AudioWorkletNode) {
128-
this.processor.disconnect();
129-
this.source.disconnect();
130-
} else if (this.processor instanceof MediaRecorder) {
131-
this.processor.stop();
132-
}
133-
this.audioContext.close();
134-
this.processor = null;
278+
this.cleanup();
135279
}
136280
}

0 commit comments

Comments
 (0)