173 lines
17 KiB
TypeScript
173 lines
17 KiB
TypeScript
import * as THREE from 'three';
|
|
import './style.css';
|
|
import vertexShader from './shaders/common.vert?raw';
|
|
import mosaicShader from './shaders/mosaic.frag?raw';
|
|
import reededShader from './shaders/reeded.frag?raw';
|
|
import frostedShader from './shaders/frosted.frag?raw';
|
|
import { MediaLoader } from './input/MediaLoader';
|
|
import { PointerController } from './input/PointerController';
|
|
import { CameraController } from './input/CameraController';
|
|
import { GyroController } from './input/GyroController';
|
|
import { StaticDepthProvider } from './depth/StaticDepthProvider';
|
|
import { DepthProcessor } from './depth/DepthProcessor';
|
|
import { PetDepthMesh } from './objects/PetDepthMesh';
|
|
import { DepthDebug } from './depth/DepthDebug';
|
|
import glassScreenVertex from './shaders/glassScreen.vert?raw';
|
|
|
|
const wrap = document.querySelector('#canvas-wrap') as HTMLElement;
|
|
const status = document.querySelector('#status') as HTMLElement;
|
|
const empty = document.querySelector('#empty') as HTMLElement;
|
|
const hintElement = document.querySelector('#hint') as HTMLElement;
|
|
hintElement.style.opacity = '0';
|
|
const renderer = new THREE.WebGLRenderer({ antialias: true });
|
|
renderer.setPixelRatio(Math.min(devicePixelRatio, 1.5));
|
|
wrap.append(renderer.domElement);
|
|
const scene = new THREE.Scene();
|
|
const glassScene = new THREE.Scene();
|
|
scene.background = new THREE.Color(0x111923);
|
|
const camera = new THREE.PerspectiveCamera(40, 1, 0.1, 100);
|
|
camera.position.z = 3;
|
|
const geometry = new THREE.PlaneGeometry(1, 1);
|
|
const emptyTex = new THREE.DataTexture(new Uint8Array([38, 48, 62, 255]), 1, 1);
|
|
emptyTex.needsUpdate = true;
|
|
const petMat = new THREE.MeshBasicMaterial({ map: emptyTex });
|
|
const pet = new THREE.Mesh(geometry, petMat);
|
|
scene.add(pet);
|
|
const depthMap: THREE.Texture = DepthProcessor.flatTexture();
|
|
const petDepth = new PetDepthMesh(emptyTex, depthMap, 0.25, 0);
|
|
const petDepthMaterial = petDepth.material;
|
|
const depthUniforms = petDepthMaterial.uniforms;
|
|
scene.add(petDepth);
|
|
const uniforms = { uSceneTexture: { value: emptyTex as THREE.Texture }, uContentViewProjection: { value: new THREE.Matrix4() }, uContentInverseViewProjection: { value: new THREE.Matrix4() }, uContentCameraPosition: { value: new THREE.Vector3() }, uLightPosition: { value: new THREE.Vector3(0.6, 0.8, 2.6) }, uGlassZ: { value: 0.6 }, uGridSize: { value: 28 }, uMosaicStyle: { value: 0 }, uStrength: { value: 0.9 }, uGlossStrength: { value: 0.55 }, uViewOffset: { value: new THREE.Vector2() }, uPaneRect: { value: new THREE.Vector4(0, 0, 1, 1) }, uResolution: { value: new THREE.Vector2() }, uTime: { value: 0 }, uPetSize: { value: new THREE.Vector2(1, 1) } };
|
|
const glassMat = new THREE.ShaderMaterial({ uniforms, vertexShader: glassScreenVertex, fragmentShader: mosaicShader });
|
|
const glass = new THREE.Mesh(new THREE.PlaneGeometry(2, 2), glassMat);
|
|
glassScene.add(glass);
|
|
const postCamera = new THREE.OrthographicCamera(-1, 1, 1, -1, 0, 1);
|
|
const renderTarget = new THREE.WebGLRenderTarget(1, 1, { colorSpace: THREE.NoColorSpace, type: THREE.HalfFloatType, depthBuffer: true });
|
|
let active: THREE.Texture = emptyTex;
|
|
let aspect = 1;
|
|
let mode = 'noglass';
|
|
let hintTimeout = 0;
|
|
let pendingPairDepth: THREE.Texture | null = null;
|
|
let mediaRequest = 0;
|
|
let depthRequest = 0;
|
|
let disposed = false;
|
|
const listenerAbort = new AbortController();
|
|
function clearPendingPair(): void { if (pendingPairDepth) pendingPairDepth.dispose(); pendingPairDepth = null; }
|
|
function resize(): void {
|
|
const bounds = wrap.getBoundingClientRect();
|
|
renderer.setSize(bounds.width, bounds.height, false);
|
|
camera.aspect = bounds.width / bounds.height;
|
|
camera.updateProjectionMatrix();
|
|
const height = 1.7;
|
|
const scale = Math.min(height * bounds.width / bounds.height / aspect, height);
|
|
pet.scale.set(scale * aspect, scale, 1);
|
|
petDepth.scale.copy(pet.scale);
|
|
uniforms.uPetSize.value.set(pet.scale.x, pet.scale.y);
|
|
uniforms.uResolution.value.set(bounds.width, bounds.height);
|
|
const glassHeight = 2 * (camera.position.z - 0.6) * Math.tan(THREE.MathUtils.degToRad(camera.fov / 2));
|
|
uniforms.uPetSize.value.set(glassHeight * camera.aspect, glassHeight);
|
|
const neutralGlassHeight = 2 * 3 * Math.tan(THREE.MathUtils.degToRad(40 / 2));
|
|
const neutralGlassWidth = neutralGlassHeight * camera.aspect;
|
|
uniforms.uPaneRect.value.set(Math.max(0, 0.5 - pet.scale.x / (2 * neutralGlassWidth)), Math.max(0, 0.5 - pet.scale.y / (2 * neutralGlassHeight)), Math.min(1, 0.5 + pet.scale.x / (2 * neutralGlassWidth)), Math.min(1, 0.5 + pet.scale.y / (2 * neutralGlassHeight)));
|
|
renderTarget.setSize(Math.max(1, Math.floor(bounds.width * Math.min(devicePixelRatio, 1.5))), Math.max(1, Math.floor(bounds.height * Math.min(devicePixelRatio, 1.5))));
|
|
}
|
|
function hint(): void {
|
|
window.clearTimeout(hintTimeout);
|
|
hintElement.style.opacity = '1';
|
|
hintTimeout = window.setTimeout(() => { hintElement.style.opacity = '0'; }, 2000);
|
|
}
|
|
function applyMedia(texture: THREE.Texture, mediaAspect: number, video: HTMLVideoElement | null): void {
|
|
if (active !== emptyTex) active.dispose();
|
|
active = texture; aspect = mediaAspect;
|
|
texture.colorSpace = THREE.SRGBColorSpace;
|
|
depthUniforms.uTexture.value = texture;
|
|
petMat.map = texture; petMat.needsUpdate = true;
|
|
empty.style.display = 'none'; resize();
|
|
}
|
|
function setDepth(texture: THREE.Texture): void {
|
|
if (depthUniforms.uDepthMap.value !== depthMap) depthUniforms.uDepthMap.value.dispose();
|
|
depthUniforms.uDepthMap.value = texture; texture.colorSpace = THREE.NoColorSpace;
|
|
if (mode === 'depth') { petMat.map = texture; petMat.needsUpdate = true; }
|
|
}
|
|
function setMode(next: string): void {
|
|
mode = next;
|
|
if (modeInput) modeInput.value = next;
|
|
pet.visible = next === 'rgb' || next === 'depth';
|
|
petDepth.visible = next === 'noglass' || next === 'final';
|
|
glass.visible = next === 'final' || next === 'glass';
|
|
petMat.map = next === 'depth' ? depthUniforms.uDepthMap.value : active;
|
|
petMat.needsUpdate = true;
|
|
petDepthMaterial.uniforms.uTexture.value = active;
|
|
}
|
|
const retry = document.createElement('button');
|
|
retry.textContent = '재생'; retry.hidden = true;
|
|
(document.querySelector('aside') as HTMLElement).append(retry);
|
|
const debugOutput = document.createElement('pre');
|
|
debugOutput.id = 'depth-debug';
|
|
debugOutput.hidden = true;
|
|
(document.querySelector('aside') as HTMLElement).append(debugOutput);
|
|
const depthDebug = new DepthDebug();
|
|
const loader = new MediaLoader((texture, mediaAspect, video) => { applyMedia(texture, mediaAspect, video); petDepth.setVideo(Boolean(video)); setDepth(pendingPairDepth ?? DepthProcessor.flatTexture()); pendingPairDepth = null; setMode('final'); hint(); }, (message) => { status.textContent = message; retry.hidden = !message.includes('자동'); });
|
|
retry.onclick = () => loader.retry();
|
|
const sampleDepthProvider = new StaticDepthProvider('/assets/sample-pet-depth.png');
|
|
const cameraController = new CameraController(camera);
|
|
const pointer = new PointerController(renderer.domElement, (x, y) => cameraController.add(x, y));
|
|
const gyro = new GyroController((x, y) => cameraController.target.set(THREE.MathUtils.clamp(x, -1, 1), THREE.MathUtils.clamp(y, -1, 1)), (message) => { status.textContent = message; });
|
|
const imageInput = document.querySelector('#image') as HTMLInputElement;
|
|
const videoInput = document.querySelector('#video') as HTMLInputElement;
|
|
const depthFileInput = document.querySelector('#depth-image') as HTMLInputElement;
|
|
const sampleButton = document.querySelector('#sample') as HTMLButtonElement | null;
|
|
const effectInput = document.querySelector('#effect') as HTMLSelectElement;
|
|
const modeInput = document.querySelector('#view-mode') as HTMLSelectElement | null;
|
|
const strengthInput = document.querySelector('#strength') as HTMLInputElement;
|
|
const sensitivityInput = document.querySelector('#sensitivity') as HTMLInputElement;
|
|
const depthInput = document.querySelector('#depth-scale') as HTMLInputElement | null;
|
|
const cameraOffsetInput = document.querySelector('#camera-offset') as HTMLInputElement;
|
|
const gridInput = document.querySelector('#grid-size') as HTMLInputElement;
|
|
const gridOutput = document.querySelector('#gridOut') as HTMLOutputElement;
|
|
const mosaicStyleInput = document.querySelector('#mosaic-style') as HTMLSelectElement;
|
|
const glossControls = document.querySelector('#gloss-controls') as HTMLElement | null;
|
|
const glossInput = document.querySelector('#gloss-strength') as HTMLInputElement | null;
|
|
const lightXInput = document.querySelector('#light-x') as HTMLInputElement | null;
|
|
const lightYInput = document.querySelector('#light-y') as HTMLInputElement | null;
|
|
const glossOutput = document.querySelector('#glossOut') as HTMLOutputElement | null;
|
|
const lightXOutput = document.querySelector('#lightXOut') as HTMLOutputElement | null;
|
|
const lightYOutput = document.querySelector('#lightYOut') as HTMLOutputElement | null;
|
|
const cameraOutput = document.querySelector('#cameraOut') as HTMLOutputElement;
|
|
const strengthOutput = document.querySelector('#strengthOut') as HTMLOutputElement;
|
|
const sensitivityOutput = document.querySelector('#sensitivityOut') as HTMLOutputElement;
|
|
const depthOutput = document.querySelector('#depthOut') as HTMLOutputElement | null;
|
|
const resetButton = document.querySelector('#reset') as HTMLButtonElement;
|
|
const gyroButton = document.querySelector('#gyro') as HTMLButtonElement;
|
|
const effects = [mosaicShader, reededShader, frostedShader];
|
|
let sensitivity = 1;
|
|
function setGlossControlsVisibility(): void {
|
|
const enabled = effectInput.value === 'mosaic';
|
|
if (glossControls) { glossControls.hidden = !enabled; glossControls.style.display = enabled ? 'grid' : 'none'; }
|
|
[glossInput, lightXInput, lightYInput].forEach((input) => { if (input) input.disabled = !enabled; });
|
|
}
|
|
imageInput.onchange = (event) => { const input = event.target as HTMLInputElement; mediaRequest += 1; depthRequest += 1; clearPendingPair(); depthDebug.clear(); if (input.files?.[0]) loader.load(input.files[0]); input.value = ''; };
|
|
videoInput.onchange = (event) => { const input = event.target as HTMLInputElement; mediaRequest += 1; depthRequest += 1; clearPendingPair(); depthDebug.clear(); if (input.files?.[0]) loader.load(input.files[0]); input.value = ''; };
|
|
sampleButton?.addEventListener('click', async () => { const request = ++mediaRequest; depthRequest += 1; clearPendingPair(); depthDebug.clear(); let url = ''; try { const [rgb, depth] = await Promise.all([fetch('/assets/sample-pet.png').then((r) => r.blob()), fetch('/assets/sample-pet-depth.png').then((r) => r.blob())]); if (disposed || request !== mediaRequest) return; url = URL.createObjectURL(depth); const image = new Image(); image.src = url; await image.decode(); if (disposed || request !== mediaRequest) return; const loadedDepth = await sampleDepthProvider.getDepthMap(image); if (disposed || request !== mediaRequest) { loadedDepth.dispose(); return; } clearPendingPair(); pendingPairDepth = loadedDepth; depthDebug.setSample(image); loader.load(new File([rgb], 'sample-pet.png', { type: 'image/png' })); status.textContent = '수작업 테스트 Depth · AI추정 아님'; } catch { if (!disposed && request === mediaRequest) status.textContent = '샘플 depth map을 읽지 못했습니다.'; } finally { if (url) URL.revokeObjectURL(url); } }, { signal: listenerAbort.signal });
|
|
effectInput.addEventListener('change', (event) => { const index = ['mosaic', 'reeded', 'frosted'].indexOf((event.target as HTMLSelectElement).value); glassMat.fragmentShader = effects[index]; glassMat.needsUpdate = true; setGlossControlsVisibility(); }, { signal: listenerAbort.signal });
|
|
modeInput?.addEventListener('change', (event) => setMode((event.target as HTMLSelectElement).value), { signal: listenerAbort.signal });
|
|
strengthInput.oninput = (event) => { const value = Number((event.target as HTMLInputElement).value); uniforms.uStrength.value = value; strengthOutput.value = value.toFixed(2); };
|
|
sensitivityInput.oninput = (event) => { const value = Number((event.target as HTMLInputElement).value); sensitivity = value; sensitivityOutput.value = value.toFixed(2); };
|
|
depthInput?.addEventListener('input', (event) => { const value = Math.min(Number((event.target as HTMLInputElement).value), 0.4); depthUniforms.uDepthScale.value = value; if (depthOutput) depthOutput.value = value.toFixed(2); }, { signal: listenerAbort.signal });
|
|
cameraOffsetInput.addEventListener('input', (event) => { const value = Number((event.target as HTMLInputElement).value); cameraController.setMaxOffset(value); cameraOutput.value = value.toFixed(2); }, { signal: listenerAbort.signal });
|
|
gridInput.addEventListener('input', (event) => { const value = Number((event.target as HTMLInputElement).value); uniforms.uGridSize.value = value; gridOutput.value = value.toFixed(0); }, { signal: listenerAbort.signal });
|
|
mosaicStyleInput.addEventListener('change', (event) => { const style = Number((event.target as HTMLSelectElement).value); uniforms.uMosaicStyle.value = style; uniforms.uGridSize.value = style === 0 ? 28 : 40; uniforms.uStrength.value = style === 0 ? 0.9 : 1; gridInput.value = String(uniforms.uGridSize.value); gridOutput.value = String(uniforms.uGridSize.value); strengthInput.value = String(uniforms.uStrength.value); strengthOutput.value = uniforms.uStrength.value.toFixed(2); }, { signal: listenerAbort.signal });
|
|
glossInput?.addEventListener('input', (event) => { const value = Number((event.target as HTMLInputElement).value); uniforms.uGlossStrength.value = value; if (glossOutput) glossOutput.value = value.toFixed(2); }, { signal: listenerAbort.signal });
|
|
lightXInput?.addEventListener('input', (event) => { const value = Number((event.target as HTMLInputElement).value); uniforms.uLightPosition.value.x = value; if (lightXOutput) lightXOutput.value = value.toFixed(2); }, { signal: listenerAbort.signal });
|
|
lightYInput?.addEventListener('input', (event) => { const value = Number((event.target as HTMLInputElement).value); uniforms.uLightPosition.value.y = value; if (lightYOutput) lightYOutput.value = value.toFixed(2); }, { signal: listenerAbort.signal });
|
|
depthFileInput?.addEventListener('change', async () => { const request = ++depthRequest; const file = depthFileInput.files?.[0]; if (!file) return; depthDebug.clear(); const url = URL.createObjectURL(file); let texture: THREE.Texture | null = null; try { const image = new Image(); image.src = url; await image.decode(); if (disposed || request !== depthRequest) return; if (aspect > 0 && Math.abs(image.width / image.height - aspect) > 0.02) throw new Error('aspect'); texture = DepthProcessor.toTexture(image); if (disposed || request !== depthRequest) { texture.dispose(); texture = null; return; } setDepth(texture); texture = null; status.textContent = '수동 depth map 적용됨 (0=뒤, 1=앞)'; } catch { if (!disposed && request === depthRequest) status.textContent = 'RGB와 종횡비가 다른 depth map입니다.'; } finally { texture?.dispose(); URL.revokeObjectURL(url); depthFileInput.value = ''; } }, { signal: listenerAbort.signal });
|
|
resetButton.onclick = () => { cameraController.reset(); gyro.reset(); };
|
|
gyroButton.onclick = async () => { gyroButton.textContent = `Gyroscope: ${(await gyro.toggle()) ? 'ON' : 'OFF'}`; };
|
|
const observer = new ResizeObserver(resize); observer.observe(wrap); resize(); setMode('final'); setGlossControlsVisibility();
|
|
let last = performance.now(); let raf = 0;
|
|
function frame(now: number): void { const dt = Math.min((now - last) / 1000, 0.1); last = now; uniforms.uViewOffset.value.copy(cameraController.update(dt, sensitivity)); camera.updateMatrixWorld(true); petDepth.updateMatrixWorld(true); uniforms.uContentViewProjection.value.multiplyMatrices(camera.projectionMatrix, camera.matrixWorldInverse); uniforms.uContentInverseViewProjection.value.copy(uniforms.uContentViewProjection.value).invert(); uniforms.uContentCameraPosition.value.copy(camera.position); uniforms.uTime.value = now / 1000; const diagnostic = depthDebug.update(camera, petDepth, depthUniforms.uDepthScale.value, depthUniforms.uDepthBias.value, wrap.clientWidth); debugOutput.textContent = diagnostic; debugOutput.hidden = !diagnostic; if (mode === 'final' || mode === 'glass') { renderer.setRenderTarget(renderTarget); renderer.render(scene, camera); renderer.setRenderTarget(null); uniforms.uSceneTexture.value = renderTarget.texture; renderer.render(glassScene, postCamera); } else { renderer.setRenderTarget(null); renderer.render(scene, camera); } raf = requestAnimationFrame(frame); }
|
|
raf = requestAnimationFrame(frame);
|
|
function teardown(): void { disposed = true; mediaRequest += 1; depthRequest += 1; clearPendingPair(); listenerAbort.abort(); cancelAnimationFrame(raf); window.clearTimeout(hintTimeout); observer.disconnect(); imageInput.onchange = null; videoInput.onchange = null; effectInput.onchange = null; strengthInput.oninput = null; sensitivityInput.oninput = null; resetButton.onclick = null; gyroButton.onclick = null; retry.onclick = null; pointer.dispose(); gyro.dispose(); loader.dispose(); renderTarget.dispose(); if (active !== emptyTex) active.dispose(); if (depthUniforms.uDepthMap.value !== depthMap) depthUniforms.uDepthMap.value.dispose(); depthMap.dispose(); petDepth.geometry.dispose(); petDepthMaterial.dispose(); geometry.dispose(); petMat.dispose(); glass.geometry.dispose(); glassMat.dispose(); emptyTex.dispose(); renderer.domElement.remove(); renderer.dispose(); hintElement.style.opacity = '0'; retry.remove(); debugOutput.remove(); }
|
|
const hot = import.meta as ImportMeta & { hot?: { dispose(fn: () => void): void } }; hot.hot?.dispose(teardown);
|