Tentative integration of audio worklet recorder

GoogleChromeLabs · Jun 19, 2024 · 8f4404d · 8f4404d
1 parent dc5a278
commit 8f4404d
Show file tree

Hide file tree

Showing 9 changed files with 285 additions and 28 deletions.
diff --git a/.gitignore b/.gitignore
@@ -5,3 +5,5 @@ _site/
 /playwright-report/
 /blob-report/
 /playwright/.cache/
+/.idea
+bun.lockb
diff --git a/src/tests/playwright/pages/realtime-sine.html b/src/tests/playwright/pages/realtime-sine.html
@@ -1,32 +1,8 @@
 <!DOCTYPE html>
 <html lang="en">
   <head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Hello Sine Test</title>
+    <script src="src/realtime-sine.js" defer type="module"></script>
+    <title></title>
   </head>
-  <body>
-    <h1>Realtime Sine Test</h1>
-    <p>Play 440Hz sine wave for 1 second, 880Hz for 1 second, then stop</p>
-
-    <script>
-      const audioContext = new AudioContext();
-      const osc = audioContext.createOscillator();
-      osc.type = 'sine';
-      osc.frequency.value = 440;
-      osc.connect(audioContext.destination);
-
-      let updateFrequencyPromise;
-      osc.start();
-
-      // Update frequency after 1 second
-      updateFrequencyPromise = new Promise((resolve, reject) => {
-        setTimeout(() => {
-          osc.frequency.value = 880;
-          osc.stop(audioContext.currentTime + 1);
-          resolve();
-        }, 1000);
-      });
-    </script>
-  </body>
-</html>
+  <body></body>
+</html>
diff --git a/src/tests/playwright/pages/src/audioBufferToWav.js b/src/tests/playwright/pages/src/audioBufferToWav.js
@@ -0,0 +1,162 @@
+// REF: https://github.com/hoch/canopy/blob/master/docs/js/canopy-exporter.js
+
+/**
+ * Writes a string to an array starting at a specified offset.
+ *
+ * @param {string} aString - The string to write to the array.
+ * @param {Uint8Array} targetArray - The array to write to.
+ * @param {number} offset - The offset in the array to start writing at.
+ */
+const _writeStringToArray = (aString, targetArray, offset) => {
+  for (let i = 0; i < aString.length; ++i) {
+    targetArray[offset + i] = aString.charCodeAt(i);
+  }
+};
+
+/**
+ * Writes a 16-bit integer to an array at the specified offset.
+ *
+ * @param {number} aNumber - The 16-bit integer to be written.
+ * @param {Uint8Array} targetArray - The array to write the integer to.
+ * @param {number} offset - The offset at which to write the integer in the
+ *                          array.
+ */
+const _writeInt16ToArray = (aNumber, targetArray, offset) => {
+  aNumber = Math.floor(aNumber);
+  targetArray[offset] = aNumber & 255; // byte 1
+  targetArray[offset + 1] = (aNumber >> 8) & 255; // byte 2
+};
+
+/**
+ * Writes a 32-bit integer to a target array at the specified offset.
+ *
+ * @param {number} aNumber - The number to be written.
+ * @param {Uint8Array} targetArray - The array to write the number to.
+ * @param {number} offset - The offset at which to start writing.
+ */
+const _writeInt32ToArray = (aNumber, targetArray, offset) => {
+  aNumber = Math.floor(aNumber);
+  targetArray[offset] = aNumber & 255; // byte 1
+  targetArray[offset + 1] = (aNumber >> 8) & 255; // byte 2
+  targetArray[offset + 2] = (aNumber >> 16) & 255; // byte 3
+  targetArray[offset + 3] = (aNumber >> 24) & 255; // byte 4
+};
+
+// Return the bits of the float as a 32-bit integer value.  This
+// produces the raw bits; no intepretation of the value is done.
+const _floatBits = (f) => {
+  const buf = new ArrayBuffer(4);
+  (new Float32Array(buf))[0] = f;
+  const bits = (new Uint32Array(buf))[0];
+  // Return as a signed integer.
+  return bits | 0;
+};
+
+/**
+ * Converts an audio buffer to an array with the specified bit depth.
+ *
+ * @param {AudioBuffer} audioBuffer - The audio buffer to convert.
+ * @param {Uint8Array} targetArray - The array to store the converted samples.
+ * @param {number} offset - The offset in the targetArray to start writing the
+ *                          converted samples.
+ * @param {number} bitDepth - The desired bit depth of the converted samples
+ *                            (16 or 32).
+ */
+const _writeAudioBufferToArray =
+  (audioBuffer, targetArray, offset, bitDepth) => {
+    let index; let channel = 0;
+    const length = audioBuffer.length;
+    const channels = audioBuffer.numberOfChannels;
+    let channelData; let sample;
+
+    // Clamping samples onto the 16-bit resolution.
+    for (index = 0; index < length; ++index) {
+      for (channel = 0; channel < channels; ++channel) {
+        channelData = audioBuffer.getChannelData(channel);
+
+        // Branches upon the requested bit depth
+        if (bitDepth === 16) {
+          sample = channelData[index] * 32768.0;
+          if (sample < -32768) {
+            sample = -32768;
+          } else if (sample > 32767) {
+            sample = 32767;
+          }
+          _writeInt16ToArray(sample, targetArray, offset);
+          offset += 2;
+        } else if (bitDepth === 32) {
+        // This assumes we're going to out 32-float, not 32-bit linear.
+          sample = _floatBits(channelData[index]);
+          _writeInt32ToArray(sample, targetArray, offset);
+          offset += 4;
+        } else {
+          console.error('Invalid bit depth for PCM encoding.');
+          return;
+        }
+      }
+    }
+  };
+
+/**
+ * Converts an AudioBuffer object into a WAV file in the form of a binary blob.
+ * The resulting WAV file can be used for audio playback or further processing.
+ * The function takes two parameters: audioBuffer which represents the audio
+ * data, and as32BitFloat which indicates whether the WAV file should be encoded
+ * as 32-bit float or 16-bit integer PCM. The unction performs various
+ * calculations and writes the necessary headers and data to create the WAV
+ * file. Finally, it returns the WAV file as a Blob object with the MIME type
+ * audio/wave.
+ *
+ * @param  {AudioBuffer} audioBuffer
+ * @param  {Boolean} as32BitFloat
+ * @return {Blob} Resulting binary blob.
+ */
+const audioBufferToWav = (audioBuffer, as32BitFloat) => {
+  // Encoding setup.
+  const frameLength = audioBuffer.length;
+  const numberOfChannels = audioBuffer.numberOfChannels;
+  const sampleRate = audioBuffer.sampleRate;
+  const bitsPerSample = as32BitFloat ? 32 : 16;
+  const bytesPerSample = bitsPerSample / 8;
+  const byteRate = sampleRate * numberOfChannels * bitsPerSample / 8;
+  const blockAlign = numberOfChannels * bitsPerSample / 8;
+  const wavDataByteLength = frameLength * numberOfChannels * bytesPerSample;
+  const headerByteLength = 44;
+  const totalLength = headerByteLength + wavDataByteLength;
+  const waveFileData = new Uint8Array(totalLength);
+  const subChunk1Size = 16;
+  const subChunk2Size = wavDataByteLength;
+  const chunkSize = 4 + (8 + subChunk1Size) + (8 + subChunk2Size);
+
+  _writeStringToArray('RIFF', waveFileData, 0);
+  _writeInt32ToArray(chunkSize, waveFileData, 4);
+  _writeStringToArray('WAVE', waveFileData, 8);
+  _writeStringToArray('fmt ', waveFileData, 12);
+
+  // SubChunk1Size (4)
+  _writeInt32ToArray(subChunk1Size, waveFileData, 16);
+  // AudioFormat (2): 3 means 32-bit float, 1 means integer PCM.
+  _writeInt16ToArray(as32BitFloat ? 3 : 1, waveFileData, 20);
+  // NumChannels (2)
+  _writeInt16ToArray(numberOfChannels, waveFileData, 22);
+  // SampleRate (4)
+  _writeInt32ToArray(sampleRate, waveFileData, 24);
+  // ByteRate (4)
+  _writeInt32ToArray(byteRate, waveFileData, 28);
+  // BlockAlign (2)
+  _writeInt16ToArray(blockAlign, waveFileData, 32);
+  // BitsPerSample (4)
+  _writeInt32ToArray(bitsPerSample, waveFileData, 34);
+  _writeStringToArray('data', waveFileData, 36);
+  // SubChunk2Size (4)
+  _writeInt32ToArray(subChunk2Size, waveFileData, 40);
+
+  // Write actual audio data starting at offset 44.
+  _writeAudioBufferToArray(audioBuffer, waveFileData, 44, bitsPerSample);
+
+  return new Blob([waveFileData], {
+    type: 'audio/wav',
+  });
+};
+
+export default audioBufferToWav;
diff --git a/src/tests/playwright/pages/src/concat.js b/src/tests/playwright/pages/src/concat.js
@@ -0,0 +1,16 @@
+export default (...arrays) => {
+  // Calculate the total length of the new Float32Array
+  const totalLength = arrays.reduce((acc, curr) => acc + curr.length, 0);
+
+  // Create a new Float32Array with the total length
+  const result = new Float32Array(totalLength);
+
+  // Copy elements from each input array into the new array
+  let offset = 0;
+  arrays.forEach((array) => {
+    result.set(array, offset);
+    offset += array.length;
+  });
+
+  return result;
+};
diff --git a/src/tests/playwright/pages/src/dom.js b/src/tests/playwright/pages/src/dom.js
@@ -0,0 +1,10 @@
+export default {
+  id: {
+    helloSine: {
+      btn: document.getElementById('hello-sine:btn'),
+      res: document.getElementById('hello-sine:res'),
+      time: document.getElementById('hello-sine:time'),
+      output: document.getElementById('hello-sine:output'),
+    },
+  },
+};
diff --git a/src/tests/playwright/pages/src/download.js b/src/tests/playwright/pages/src/download.js
@@ -0,0 +1,6 @@
+export default (blob) => {
+  const download = document.createElement('a');
+  download.href = URL.createObjectURL(blob);
+  download.download = 'out.wav';
+  download.click();
+};
diff --git a/src/tests/playwright/pages/src/realtime-sine.js b/src/tests/playwright/pages/src/realtime-sine.js
@@ -0,0 +1,27 @@
+import audioBufferToWav from './audioBufferToWav.js';
+import record from './recorder/main.js';
+
+// eslint-disable-next-line no-async-promise-executor
+window.updateFrequencyPromise = new Promise(async (resolve) => {
+  const ctx = new AudioContext();
+  const helloSine = new OscillatorNode(ctx);
+  const {recorder, buffer} = await record(ctx, helloSine);
+
+  helloSine.connect(recorder).connect(ctx.destination);
+
+  const start = performance.now();
+
+  helloSine.start();
+  helloSine.stop(ctx.currentTime + 1);
+
+  const latency = await new Promise((resolve) =>
+    helloSine.onended = () => resolve(ctx.baseLatency));
+
+  const end = performance.now();
+
+  const blob = audioBufferToWav(await buffer, false);
+
+  await ctx.close();
+
+  resolve({latency, time: end - start, blob});
+});
diff --git a/src/tests/playwright/pages/src/recorder/main.js b/src/tests/playwright/pages/src/recorder/main.js
@@ -0,0 +1,38 @@
+import concat from '../concat.js';
+
+export default async (ctx, scheduleNode) => {
+  console.assert(ctx instanceof AudioContext);
+  console.assert(scheduleNode instanceof AudioScheduledSourceNode);
+
+  const mutex = new Promise((resolve) =>
+    scheduleNode.addEventListener('ended', resolve));
+
+  await ctx.audioWorklet.addModule('./scripts/recorder/worker.js');
+
+  const recorder = new AudioWorkletNode(ctx, 'recorder');
+
+  const arrays = [];
+  recorder.port.onmessage = (e) => {
+    !(e.data.channel in arrays) && (arrays[e.data.channel] = []);
+    arrays[e.data.channel].push(e.data.data);
+  };
+
+  // eslint-disable-next-line no-async-promise-executor
+  const buffer = new Promise(async (resolve) => {
+    await mutex;
+    const res = [];
+    arrays.forEach((array, i) => res[i] = concat(...array));
+
+    const buf = new AudioBuffer({
+      length: res[0].byteLength,
+      sampleRate: ctx.sampleRate,
+      numberOfChannels: res.length,
+    });
+
+    res.forEach((array, i) => buf.copyToChannel(array, i));
+
+    resolve(buf);
+  });
+
+  return {recorder, buffer};
+};
diff --git a/src/tests/playwright/pages/src/recorder/worker.js b/src/tests/playwright/pages/src/recorder/worker.js
@@ -0,0 +1,20 @@
+// bypass-processor.js
+class RecorderProcessor extends AudioWorkletProcessor {
+  constructor() {
+    super();
+  }
+
+  process(inputs, outputs) {
+    const input = inputs[0];
+    const output = outputs[0];
+
+    for (let channel = 0; channel < input.length; channel++) {
+      output[channel].set(input[channel]);
+      this.port.postMessage({channel, data: input[channel]});
+    }
+
+    return true;
+  }
+}
+
+registerProcessor('recorder', RecorderProcessor);