Skip to content

Commit c0dc9cd

Browse files
mattyateau1-liquidtar-bin
authored andcommitted
feat(frontend): Audio player で波形を表示するように (MisskeyIO#827)
Co-authored-by: あわわわとーにゅ <17376330+u1-liquid@users.noreply.github.com> Co-authored-by: tar_bin <tar.bin.master@gmail.com>
1 parent 626b3d4 commit c0dc9cd

12 files changed

+579
-151
lines changed

packages/frontend/package.json

+5-2
Original file line numberDiff line numberDiff line change
@@ -72,8 +72,10 @@
7272
"uuid": "10.0.0",
7373
"v-code-diff": "1.13.1",
7474
"vite": "5.4.11",
75-
"vue": "3.5.12",
76-
"vuedraggable": "next"
75+
"vue": "3.5.13",
76+
"vue-gtag": "2.0.1",
77+
"vuedraggable": "next",
78+
"webgl-audiovisualizer": "github:tar-bin/webgl-audiovisualizer"
7779
},
7880
"devDependencies": {
7981
"@misskey-dev/summaly": "5.1.0",
@@ -104,6 +106,7 @@
104106
"@types/punycode": "2.1.4",
105107
"@types/sanitize-html": "2.13.0",
106108
"@types/seedrandom": "3.0.8",
109+
"@types/three": "0.170.0",
107110
"@types/throttle-debounce": "5.0.2",
108111
"@types/tinycolor2": "1.4.6",
109112
"@types/uuid": "10.0.0",
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,219 @@
1+
<template>
2+
<div ref="container" :class="$style.root">
3+
</div>
4+
</template>
5+
6+
<script lang="ts" setup>
7+
import { onMounted, onUnmounted, shallowRef, nextTick, ref } from 'vue';
8+
import * as Misskey from 'misskey-js';
9+
import * as THREE from 'three';
10+
import vertexShader from '../../node_modules/webgl-audiovisualizer/audiovisial-vertex.shader?raw';
11+
import fragmentShader from '../../node_modules/webgl-audiovisualizer/audiovisial-fragment.shader?raw';
12+
import circleMask from '../../node_modules/webgl-audiovisualizer/circlemask.png';
13+
14+
const props = defineProps<{
15+
user: Misskey.entities.UserLite;
16+
audioEl: HTMLAudioElement | undefined;
17+
analyser: AnalyserNode | undefined;
18+
}>();
19+
20+
const container = shallowRef<HTMLDivElement>();
21+
22+
const isPlaying = ref(false);
23+
const fftSize = 2048;
24+
25+
let prevTime = 0;
26+
let angle1 = 0;
27+
let angle2 = 0;
28+
29+
let scene, camera, renderer, width, height, uniforms, texture, maskTexture, dataArray1, dataArray2, dataArrayOrigin, bufferLength: number;
30+
31+
const init = () => {
32+
const parent = container.value ?? { offsetWidth: 0 };
33+
width = parent.offsetWidth;
34+
height = Math.floor(width * 9 / 16);
35+
36+
scene = new THREE.Scene();
37+
camera = new THREE.OrthographicCamera();
38+
camera.left = width / -2;
39+
camera.right = width / 2;
40+
camera.top = height / 2;
41+
camera.bottom = height / -2;
42+
camera.updateProjectionMatrix();
43+
44+
renderer = new THREE.WebGLRenderer({ antialias: true });
45+
renderer.setSize(width, height);
46+
47+
if (container.value) {
48+
container.value.appendChild(renderer.domElement);
49+
}
50+
51+
const loader = new THREE.TextureLoader();
52+
texture = loader.load(props.user.avatarUrl ?? '');
53+
maskTexture = loader.load(circleMask);
54+
uniforms = {
55+
enableAudio: {
56+
value: 0,
57+
},
58+
uTex: { value: texture },
59+
uMask: { value: maskTexture },
60+
time: {
61+
value: 0,
62+
},
63+
resolution: {
64+
value: new THREE.Vector2(width, height),
65+
},
66+
};
67+
68+
const material = new THREE.ShaderMaterial({
69+
uniforms: uniforms,
70+
vertexShader: vertexShader,
71+
fragmentShader: fragmentShader,
72+
});
73+
74+
const geometry = new THREE.PlaneGeometry(2, 2);
75+
76+
const mesh = new THREE.Mesh(geometry, material);
77+
scene.add(mesh);
78+
79+
renderer.setAnimationLoop(animate);
80+
};
81+
82+
const play = () => {
83+
if (props.analyser) {
84+
bufferLength = props.analyser.frequencyBinCount;
85+
dataArrayOrigin = new Uint8Array(bufferLength);
86+
dataArray1 = new Uint8Array(bufferLength);
87+
dataArray2 = new Uint8Array(bufferLength);
88+
uniforms = {
89+
enableAudio: {
90+
value: 1,
91+
},
92+
tAudioData1: { value: new THREE.DataTexture(dataArray1, fftSize / 2, 1, THREE.RedFormat) },
93+
tAudioData2: { value: new THREE.DataTexture(dataArray2, fftSize / 2, 1, THREE.RedFormat) },
94+
uTex: { value: texture },
95+
uMask: { value: maskTexture },
96+
time: {
97+
value: 0,
98+
},
99+
resolution: {
100+
value: new THREE.Vector2(width, height),
101+
},
102+
};
103+
const material = new THREE.ShaderMaterial({
104+
uniforms: uniforms,
105+
vertexShader: vertexShader,
106+
fragmentShader: fragmentShader,
107+
});
108+
109+
const geometry = new THREE.PlaneGeometry(2, 2);
110+
111+
const mesh = new THREE.Mesh(geometry, material);
112+
scene.add(mesh);
113+
114+
renderer.setAnimationLoop(animate);
115+
}
116+
};
117+
118+
const pauseAnimation = () => {
119+
isPlaying.value = false;
120+
};
121+
122+
const resumeAnimation = () => {
123+
isPlaying.value = true;
124+
renderer.setAnimationLoop(play);
125+
};
126+
127+
const animate = (time) => {
128+
if (angle1 >= bufferLength) {
129+
angle1 = 0;
130+
}
131+
if (angle2 >= bufferLength) {
132+
angle2 = 0;
133+
}
134+
135+
if (props.analyser) {
136+
if ((time - prevTime) > 10) {
137+
for (let i = 0; i < bufferLength; i++) {
138+
let n1 = (i + angle1) % bufferLength;
139+
let n2 = (i + angle2) % bufferLength;
140+
if (dataArrayOrigin[n1] > dataArray1[i]) {
141+
dataArray1[i] = dataArray1[i] < 255 ? (dataArrayOrigin[i] + dataArray1[i]) / 2 : 255;
142+
}
143+
if (dataArrayOrigin[n2] > dataArray2[i]) {
144+
dataArray2[i] = dataArray2[i] < 255 ? (dataArrayOrigin[i] + dataArray2[i]) / 2 : 255;
145+
}
146+
}
147+
}
148+
if ((time - prevTime) > 20) {
149+
for (let i = 0; i < bufferLength; i++) {
150+
let n1 = (i + angle1) % bufferLength;
151+
let n2 = (i + angle2) % bufferLength;
152+
if (dataArrayOrigin[n1] < dataArray1[i]) {
153+
dataArray1[i] = dataArray1[i] > 0 ? dataArray1[i] - 1 : 0;
154+
}
155+
if (dataArrayOrigin[n2] < dataArray2[i]) {
156+
dataArray2[i] = dataArray2[i] > 0 ? dataArray2[i] - 1 : 0;
157+
}
158+
}
159+
uniforms.tAudioData1.value.needsUpdate = true;
160+
uniforms.tAudioData2.value.needsUpdate = true;
161+
prevTime = time;
162+
}
163+
164+
if (isPlaying.value) {
165+
props.analyser.getByteTimeDomainData(dataArrayOrigin);
166+
} else {
167+
for (let i = 0; i < bufferLength; i++) {
168+
dataArrayOrigin[i] = 0;
169+
}
170+
}
171+
172+
angle1 += parseInt(String(bufferLength / 360 * 2)); //こうしないと型エラー出てうざい
173+
angle2 += parseInt(String(bufferLength / 360 * 3));
174+
}
175+
uniforms.time.value = time;
176+
renderer.render(scene, camera);
177+
};
178+
179+
const onResize = () => {
180+
const parent = container.value ?? { offsetWidth: 0 };
181+
width = parent.offsetWidth;
182+
height = Math.floor(width * 9 / 16);
183+
camera.left = width / -2;
184+
camera.right = width / 2;
185+
camera.top = height / 2;
186+
camera.bottom = height / -2;
187+
camera.updateProjectionMatrix();
188+
renderer.setSize(width, height);
189+
uniforms.resolution.value.set(width, height);
190+
};
191+
192+
onMounted(async () => {
193+
nextTick().then(() => {
194+
init();
195+
window.addEventListener('resize', onResize);
196+
});
197+
});
198+
199+
onUnmounted(() => {
200+
if (renderer) {
201+
renderer.dispose();
202+
}
203+
});
204+
205+
defineExpose({
206+
pauseAnimation,
207+
resumeAnimation,
208+
});
209+
210+
</script>
211+
212+
<style lang="scss" module>
213+
.root {
214+
position: relative;
215+
width: 100%;
216+
height: 100%;
217+
overflow: hidden;
218+
}
219+
</style>

packages/frontend/src/components/MkMediaAudio.vue

+36-10
Original file line numberDiff line numberDiff line change
@@ -74,15 +74,11 @@ SPDX-License-Identifier: AGPL-3.0-only
7474
<i v-else class="ti ti-volume"></i>
7575
</button>
7676
<MkMediaRange
77-
v-model="volume"
78-
:class="$style.volumeSeekbar"
77+
v-model="rangePercent"
78+
:class="$style.seekbarRoot"
79+
:buffer="bufferedDataRatio"
7980
/>
8081
</div>
81-
<MkMediaRange
82-
v-model="rangePercent"
83-
:class="$style.seekbarRoot"
84-
:buffer="bufferedDataRatio"
85-
/>
8682
</div>
8783
</div>
8884
</template>
@@ -97,11 +93,13 @@ import * as os from '@/os.js';
9793
import { type Keymap } from '@/scripts/hotkey.js';
9894
import bytes from '@/filters/bytes.js';
9995
import { hms } from '@/filters/hms.js';
96+
import MkAudioVisualizer from '@/components/MkAudioVisualizer.vue';
10097
import MkMediaRange from '@/components/MkMediaRange.vue';
10198
import { $i, iAmModerator } from '@/account.js';
10299

103100
const props = defineProps<{
104101
audio: Misskey.entities.DriveFile;
102+
user?: Misskey.entities.UserLite;
105103
}>();
106104

107105
const keymap = {
@@ -152,6 +150,7 @@ function hasFocus() {
152150

153151
const playerEl = shallowRef<HTMLDivElement>();
154152
const audioEl = shallowRef<HTMLAudioElement>();
153+
const audioVisualizer = ref<InstanceType<typeof MkAudioVisualizer>>();
155154

156155
// eslint-disable-next-line vue/no-setup-props-reactivity-loss
157156
const hide = ref((defaultStore.state.nsfw === 'force' || defaultStore.state.dataSaver.media) ? true : (props.audio.isSensitive && defaultStore.state.nsfw !== 'ignore'));
@@ -250,6 +249,11 @@ const isPlaying = ref(false);
250249
const isActuallyPlaying = ref(false);
251250
const elapsedTimeMs = ref(0);
252251
const durationMs = ref(0);
252+
const audioContext = ref<AudioContext | null>(null);
253+
const sourceNode = ref<MediaElementAudioSourceNode | null>(null);
254+
const gainNode = ref<GainNode | null>(null);
255+
const analyserGainNode = ref<GainNode | null>(null);
256+
const analyserNode = ref<AnalyserNode | null>(null);
253257
const rangePercent = computed({
254258
get: () => {
255259
return (elapsedTimeMs.value / durationMs.value) || 0;
@@ -272,11 +276,33 @@ const bufferedDataRatio = computed(() => {
272276
function togglePlayPause() {
273277
if (!isReady.value || !audioEl.value) return;
274278

279+
if (!sourceNode.value) {
280+
audioContext.value = new (window.AudioContext || window.webkitAudioContext)();
281+
sourceNode.value = audioContext.value.createMediaElementSource(audioEl.value);
282+
283+
analyserGainNode.value = audioContext.value.createGain();
284+
gainNode.value = audioContext.value.createGain();
285+
analyserNode.value = audioContext.value.createAnalyser();
286+
287+
sourceNode.value.connect(analyserGainNode.value);
288+
analyserGainNode.value.connect(analyserNode.value);
289+
analyserNode.value.connect(gainNode.value);
290+
gainNode.value.connect(audioContext.value.destination);
291+
292+
analyserNode.value.fftSize = 2048;
293+
294+
analyserGainNode.value.gain.setValueAtTime(0.8, audioContext.value.currentTime);
295+
296+
gainNode.value.gain.setValueAtTime(volume.value, audioContext.value.currentTime);
297+
}
298+
275299
if (isPlaying.value) {
276300
audioEl.value.pause();
301+
audioVisualizer.value?.pauseAnimation();
277302
isPlaying.value = false;
278303
} else {
279304
audioEl.value.play();
305+
audioVisualizer.value?.resumeAnimation();
280306
isPlaying.value = true;
281307
oncePlayed.value = true;
282308
}
@@ -334,6 +360,7 @@ function init() {
334360
oncePlayed.value = false;
335361
isActuallyPlaying.value = false;
336362
isPlaying.value = false;
363+
audioVisualizer.value?.pauseAnimation();
337364
});
338365

339366
durationMs.value = audioEl.value.duration * 1000;
@@ -342,16 +369,15 @@ function init() {
342369
durationMs.value = audioEl.value.duration * 1000;
343370
}
344371
});
345-
346-
audioEl.value.volume = volume.value;
372+
gainNode.value?.gain.setValueAtTime(volume.value, audioContext.value?.currentTime);
347373
}
348374
}, {
349375
immediate: true,
350376
});
351377
}
352378

353379
watch(volume, (to) => {
354-
if (audioEl.value) audioEl.value.volume = to;
380+
if (audioEl.value) gainNode.value?.gain.setValueAtTime(to, audioContext.value?.currentTime);
355381
});
356382

357383
watch(speed, (to) => {

packages/frontend/src/components/MkMediaBanner.vue

+6-4
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,12 @@ SPDX-License-Identifier: AGPL-3.0-only
55

66
<template>
77
<div :class="$style.root">
8-
<MkMediaAudio v-if="media.type.startsWith('audio') && media.type !== 'audio/midi'" :audio="media"/>
9-
<div v-else-if="media.isSensitive && hide" :class="$style.sensitive" @click="show">
8+
<div v-if="media.isSensitive && hide" :class="$style.sensitive" @click="showHiddenContent">
109
<span style="font-size: 1.6em;"><i class="ti ti-alert-triangle"></i></span>
1110
<b>{{ i18n.ts.sensitive }}</b>
1211
<span>{{ i18n.ts.clickToShow }}</span>
1312
</div>
13+
<MkMediaAudio v-else-if="media.type.startsWith('audio') && media.type !== 'audio/midi'" :audio="media" :user="user"/>
1414
<a
1515
v-else :class="$style.download"
1616
:href="media.url"
@@ -31,9 +31,11 @@ import { defaultStore } from '@/store.js';
3131
import * as os from '@/os.js';
3232
import MkMediaAudio from '@/components/MkMediaAudio.vue';
3333

34-
const props = defineProps<{
34+
const props = withDefaults(defineProps<{
3535
media: Misskey.entities.DriveFile;
36-
}>();
36+
user?: Misskey.entities.UserLite;
37+
}>(), {
38+
});
3739

3840
const hide = ref(true);
3941

0 commit comments

Comments
 (0)