/* 3dads.js * * contains the implementation of the 3d-img and 3d-video components */ const isBrowser = typeof window !== "undefined" && typeof HTMLElement !== "undefined" && typeof document !== "undefined"; //-------------------------// let RGBD, RGBDimg, RGBDvideo; if(!isBrowser) { RGBD = class {}; RGBDimg = class {}; RGBDvideo = class {}; } else { const VERTEX_SHADER_SOURCE = `#version 300 es precision mediump float; precision mediump int; uniform sampler2D u_depthImage; uniform mat4 u_projMat; uniform mat4 u_viewMat; uniform vec2 u_imageSize; uniform float u_focal; uniform float u_depthMin; uniform float u_depthMax; uniform ivec2 u_grid; out vec2 v_uv; void main() { int vid = gl_VertexID; int quad = vid / 6; int v = vid - quad * 6; int widthGrid = u_grid.x; int heightGrid = u_grid.y; int x = quad % widthGrid; int y = quad / widthGrid; float fx = float(x) / float(widthGrid); float fy = float(y) / float(heightGrid); float fx1 = float(x + 1) / float(widthGrid); float fy1 = float(y + 1) / float(heightGrid); vec2 uv00 = vec2(fx , 1.0 - fy); vec2 uv10 = vec2(fx1, 1.0 - fy); vec2 uv01 = vec2(fx , 1.0 - fy1); vec2 uv11 = vec2(fx1, 1.0 - fy1); vec2 uv; if (v == 0) uv = uv00; else if (v == 1) uv = uv10; else if (v == 2) uv = uv01; else if (v == 3) uv = uv10; else if (v == 4) uv = uv11; else uv = uv01; float depth = texture(u_depthImage, uv).r; float z = mix(u_depthMin, u_depthMax, depth); vec2 pixelPos = uv * u_imageSize; vec2 pos = (pixelPos - u_imageSize * 0.5) * z / u_focal; v_uv = uv; gl_Position = u_projMat * u_viewMat * vec4(pos.x, pos.y, z, 1.0); } `; const FRAGMENT_SHADER_SOURCE = `#version 300 es precision mediump float; in vec2 v_uv; out vec4 fragColor; uniform sampler2D u_colorImage; uniform sampler2D u_depthImage; uniform vec2 u_targetRes; uniform float u_discardCutoff; void main() { float deriv = min( length(dFdx(v_uv) * u_targetRes), length(dFdy(v_uv) * u_targetRes) ); if(deriv < u_discardCutoff) discard; fragColor = texture(u_colorImage, v_uv); } `; //-------------------------// class _RGBD extends HTMLElement { constructor() { super(); //init params: //----------------- this._params = { // TODO: better defaults GRID_SIZE: 250, FOCAL_LENGTH: 1200.0, DEPTH_MIN: 1.5, DEPTH_MAX: 10.0, DISCARD_CUTOFF: 0.0, CAM_RADIUS: 1.5, CAM_MAX_THETA: Math.PI / 40, CAM_MAX_PHI: Math.PI / 64, CAM_FOV: Math.PI / 4.0, CAM_SMOOTHNESS: 0.9, CAM_ZOOM_FACTOR: 1.0, BOOMERANG_SPEED: 0.0, BACKGROUND_COLOR: [0.0, 0.0, 0.0] }; this._shadowRoot = null; this._canvas = null; this._colorSource = null; this._depthSource = null; this._gl = null; this._shaderProgram = null; this._textures = {}; this._uniforms = null; this._mouse = { x: 0, y: 0, hovered: false}; this._anim = {theta: Math.PI, phi: 0.0, radius: this._params.CAM_RADIUS}; this._lastTime = 0.0; this._animFrame = null; this._running = false; this._adMode = false; this._recording = false; this._isMobile = this._detectMobileDevice(); this._userViewMat = null; this._userProjMat = null; this._onResizeHandler = null; this._onMouseMoveHandler = null; this._onMouseLeaveHandler = null; this._onMouseEnterHandler = null; this._onRenderHandler = null; //set up DOM: //----------------- this._shadowRoot = this.attachShadow({mode: 'open'}); const wrapper = document.createElement('div'); wrapper.style.position = 'relative'; wrapper.style.width = '100%'; wrapper.style.height = '100%'; wrapper.style.display = 'flex'; wrapper.style.alignItems = 'center'; wrapper.style.justifyContent = 'center'; wrapper.style.background = 'transparent'; this._shadowRoot.appendChild(wrapper); const style = document.createElement('style'); style.textContent = ` :host { display: inline-block; width: 300px; height: 200px; } :host([fullbleed]) { display:block; width:100%; height:100%; } canvas { touch-action: none; } .cover-img, .cover-video { position: absolute; top: 0; left: 0; width: 100%; height: 100%; object-fit: contain; pointer-events: none; } `; this._shadowRoot.appendChild(style); //create canvas: //----------------- this._canvas = document.createElement('canvas'); this._canvas.style.width = '100%'; this._canvas.style.height = '100%'; this._canvas.style.objectFit = 'contain'; this._canvas.style.background = 'transparent'; this._canvas.setAttribute('role','img'); this._canvas.setAttribute('aria-label','3D image'); wrapper.appendChild(this._canvas); //bind handlers: //----------------- this._onResizeHandler = this._onResize.bind(this); this._onMouseMoveHandler = this._onMouseMove.bind(this); this._onMouseLeaveHandler = this._onMouseLeave.bind(this); this._onMouseEnterHandler = this._onMouseEnter.bind(this); this._onRenderHandler = this._onRender.bind(this); } _detectMobileDevice() { // Check for touch capability + mobile user agent const hasTouchScreen = ( 'ontouchstart' in window || navigator.maxTouchPoints > 0 || navigator.msMaxTouchPoints > 0 ); const mobileUserAgent = /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(navigator.userAgent); // Consider it mobile if it has touch AND matches mobile user agent // This excludes touch-enabled laptops/desktops return hasTouchScreen && mobileUserAgent; } connectedCallback() { //init: //----------------- this._readParams(); let fallback; try { this._init(); fallback = false; } catch(e) { fallback = true; } //add listeners: //----------------- window.addEventListener('resize', this._onResizeHandler); // On mobile devices, use scroll-based interaction (no boomerang) if (this._isMobile) { console.log('RGBD: Mobile device detected - scroll interaction enabled (boomerang disabled)'); // Disable boomerang on mobile this._params.BOOMERANG_SPEED = 0; // Initialize scroll-based interaction this._mouse.x = 0; this._mouse.y = 0; this._mouse.hovered = true; } else { // Desktop: add mouse interaction // In ad mode, track mouse on the wrapper (which includes the gray area) // so the boomerang effect only activates when mouse leaves the entire ad area if (this._adMode) { const wrapper = this._canvas.parentElement; wrapper.addEventListener('mouseenter', this._onMouseEnterHandler); wrapper.addEventListener('mousemove', this._onMouseMoveHandler); wrapper.addEventListener('mouseleave', this._onMouseLeaveHandler); } else { window.addEventListener('mousemove', this._onMouseMoveHandler); document.addEventListener('mouseleave', this._onMouseLeaveHandler); } } //load sources: //----------------- this._loadSources(); //fallback to static display, or start webgl rendering: //----------------- if(fallback) this._createFallback(); else { this._onResize(); this._running = true; this._lastTime = performance.now(); this._animFrame = requestAnimationFrame(this._onRenderHandler); // Setup scroll observer AFTER this._running is true (RAF loop checks this flag) if (this._isMobile) { this._setupScrollObserver(); } } } disconnectedCallback() { this._running = false; cancelAnimationFrame(this._animFrame); window.removeEventListener('resize', this._onResizeHandler); // Remove appropriate listeners based on device type if (this._isMobile) { // Remove mobile scroll listeners if (this._onMobileScroll) { window.removeEventListener('scroll', this._onMobileScroll); } if (this._onMobileTouchStart) { document.removeEventListener('touchstart', this._onMobileTouchStart); } if (this._onMobileTouchMove) { document.removeEventListener('touchmove', this._onMobileTouchMove); } if (this._onMobileTouchEnd) { document.removeEventListener('touchend', this._onMobileTouchEnd); } } else { // Remove mouse listeners for desktop if (this._adMode) { const wrapper = this._canvas.parentElement; if (wrapper) { wrapper.removeEventListener('mouseenter', this._onMouseEnterHandler); wrapper.removeEventListener('mousemove', this._onMouseMoveHandler); wrapper.removeEventListener('mouseleave', this._onMouseLeaveHandler); } } else { window.removeEventListener('mousemove', this._onMouseMoveHandler); document.removeEventListener('mouseleave', this._onMouseLeaveHandler); } } try { if(this._gl) { const gl = this._gl; if(this._shaderProgram) gl.deleteProgram(this._shaderProgram); for(let k in this._textures) gl.deleteTexture(this._textures[k]); } } catch (e) {} } setViewMatrix(mat) { this._userViewMat = (mat && mat.length === 16) ? new Float32Array(mat) : null; } setProjectionMatrix(mat) { this._userProjMat = (mat && mat.length === 16) ? new Float32Array(mat) : null; } get width() { return this._getSourceWidth(this._colorSource); } get height() { return this._getSourceHeight(this._colorSource); } _init() { //get context: //----------------- const gl = this._canvas.getContext('webgl2', { antialias: true, alpha: true, premultipliedAlpha: false }); if(!gl) throw new Error('WebGL not supported') this._gl = gl; //create shader program: //----------------- const program = this._createProgram(gl, VERTEX_SHADER_SOURCE, FRAGMENT_SHADER_SOURCE); if (!program) return; this._shaderProgram = program; gl.useProgram(program); //create textures: //----------------- const colorTex = this._createTexture(gl); const depthTex = this._createTexture(gl); this._textures.color = colorTex; this._textures.depth = depthTex; this._updateTexture('color', this._colorSource); this._updateTexture('depth', this._depthSource); //setup uniforms: //----------------- const uColor = gl.getUniformLocation(program, 'u_colorImage'); const uDepth = gl.getUniformLocation(program, 'u_depthImage'); gl.uniform1i(uColor, 0); gl.uniform1i(uDepth, 1); this._uniforms = { uProjection: gl.getUniformLocation(program, 'u_projMat'), uView: gl.getUniformLocation(program, 'u_viewMat'), uImageSize: gl.getUniformLocation(program, 'u_imageSize'), uFocal: gl.getUniformLocation(program, 'u_focal'), uDepthMin: gl.getUniformLocation(program, 'u_depthMin'), uDepthMax: gl.getUniformLocation(program, 'u_depthMax'), uGridSegs: gl.getUniformLocation(program, 'u_grid'), uTargetRes: gl.getUniformLocation(program, 'u_targetRes'), uDiscardCutoff: gl.getUniformLocation(program, 'u_discardCutoff') }; } _onRender(time) { if(!this._running) return; if(!this._gl) return const gl = this._gl; const dt = time - this._lastTime; this._lastTime = time; //update textures: //----------------- this._updateTexturesIfNeeded(); //clear: //----------------- gl.clearColor(...this._params.BACKGROUND_COLOR, 0.0); gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT); gl.enable(gl.DEPTH_TEST); gl.enable(gl.BLEND); gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA); //update camera: //----------------- // On mobile, keep static position (no boomerang animation) // On desktop, apply boomerang animation when not hovered if(!this._mouse.hovered && !this._isMobile) { this._mouse.x = Math.sin(time * this._params.BOOMERANG_SPEED); this._mouse.y = Math.sin(time * this._params.BOOMERANG_SPEED * 0.5 + Math.PI); } const targetTheta = this._mouse.x * this._params.CAM_MAX_THETA + Math.PI; const targetPhi = -this._mouse.y * this._params.CAM_MAX_PHI; const targetRadius = this._mouse.hovered ? this._params.CAM_RADIUS * this._params.CAM_ZOOM_FACTOR : this._params.CAM_RADIUS; this._anim.theta = this._animate(targetTheta, this._anim.theta, dt); this._anim.phi = this._animate(targetPhi, this._anim.phi, dt); this._anim.radius = this._animate(targetRadius, this._anim.radius, dt); //set uniforms: //----------------- const aspect = this._canvas.width / this._canvas.height; const gridW = Math.max(1, Math.floor(this._params.GRID_SIZE)); const gridH = Math.max(1, Math.round(gridW / aspect)); const sourceWidth = this._getSourceWidth(this._colorSource); const sourceHeight = this._getSourceHeight(this._colorSource); gl.uniform2f(this._uniforms.uImageSize, sourceWidth, sourceHeight); gl.uniform1f(this._uniforms.uFocal, this._params.FOCAL_LENGTH); gl.uniform1f(this._uniforms.uDepthMin, this._params.DEPTH_MIN); gl.uniform1f(this._uniforms.uDepthMax, this._params.DEPTH_MAX); gl.uniform2i(this._uniforms.uGridSegs, gridW, gridH); gl.uniform2f(this._uniforms.uTargetRes, this._canvas.width, this._canvas.height); gl.uniform1f(this._uniforms.uDiscardCutoff, this._params.DISCARD_CUTOFF); //set matrices: //----------------- const eye = [ this._anim.radius * Math.cos(this._anim.phi) * Math.sin(this._anim.theta), this._anim.radius * Math.sin(this._anim.phi), this._params.CAM_RADIUS + this._anim.radius * Math.cos(this._anim.phi) * Math.cos(this._anim.theta) ]; const target = [ 0.0, 0.0, this._params.CAM_RADIUS ]; const up = [0.0, -1.0, 0.0]; const view = this._userViewMat || new Float32Array(this._lookAt(eye, target, up)); const proj = this._userProjMat || new Float32Array(this._perspective(this._params.CAM_FOV, aspect, 0.1, 100.0)); gl.uniformMatrix4fv(this._uniforms.uView, false, view); gl.uniformMatrix4fv(this._uniforms.uProjection, false, proj); //draw: //----------------- gl.drawArrays(gl.TRIANGLES, 0, gridW * gridH * 6); //loop: //----------------- this._animFrame = requestAnimationFrame(this._onRenderHandler); } _onMouseMove(e) { // In ad mode, calculate mouse position relative to the canvas (for camera movement) // but hovered state is controlled by enter/leave on the wrapper const rect = this._canvas.getBoundingClientRect(); if (!this._adMode) { this._mouse.hovered = true; } this._mouse.x = ((e.clientX - rect.left) / rect.width) * 2 - 1; this._mouse.y = ((e.clientY - rect.top) / rect.height) * 2 - 1; } _onMouseEnter() { this._mouse.hovered = true; } _onMouseLeave() { this._mouse.hovered = false; } _setupScrollObserver() { // MOBILE-ONLY: Scroll direction-based camera panning // Scroll up = camera pans down, Scroll down = camera pans up let lastScrollY = window.scrollY; let lastTouchY = null; let targetY = 0; const scrollSensitivity = 0.008; const touchSensitivity = 0.004; const self = this; // Scroll handler: scroll down = pan up, scroll up = pan down this._onMobileScroll = function() { const delta = window.scrollY - lastScrollY; targetY = Math.max(-1, Math.min(1, targetY - delta * scrollSensitivity)); lastScrollY = window.scrollY; self._mouse.y = targetY; self._mouse.hovered = true; }; // Touch handlers for direct interaction this._onMobileTouchStart = function(e) { if (e.touches.length === 1) lastTouchY = e.touches[0].clientY; }; this._onMobileTouchMove = function(e) { if (lastTouchY !== null && e.touches.length === 1) { const delta = lastTouchY - e.touches[0].clientY; targetY = Math.max(-1, Math.min(1, targetY - delta * touchSensitivity)); lastTouchY = e.touches[0].clientY; self._mouse.y = targetY; self._mouse.hovered = true; } }; this._onMobileTouchEnd = function() { lastTouchY = null; }; // Attach listeners window.addEventListener('scroll', this._onMobileScroll, { passive: true }); document.addEventListener('touchstart', this._onMobileTouchStart, { passive: true }); document.addEventListener('touchmove', this._onMobileTouchMove, { passive: true }); document.addEventListener('touchend', this._onMobileTouchEnd, { passive: true }); } _onResize() { const dpr = window.devicePixelRatio || 1; const wrapper = this._canvas.parentElement; if(!wrapper) return; const displayW = wrapper.clientWidth; const displayH = wrapper.clientHeight; let srcW = this._getSourceWidth(this._colorSource) || 1; let srcH = this._getSourceHeight(this._colorSource) || 1; const srcAspect = srcW / srcH; let fittedW, fittedH; // In ad mode, fill the entire container without maintaining aspect ratio if(this._adMode) { fittedW = displayW; fittedH = displayH; } else { // In normal mode, maintain aspect ratio with contain behavior fittedW = displayW; fittedH = fittedW / srcAspect; if(fittedH > displayH) { fittedH = displayH; fittedW = fittedH * srcAspect; } } const internalW = Math.max(1, Math.floor(fittedW * dpr)); const internalH = Math.max(1, Math.floor(fittedH * dpr)); this._canvas.width = internalW; this._canvas.height = internalH; if(this._gl) this._gl.viewport(0, 0, internalW, internalH); } _readParams() { const p = this._params; if(this.hasAttribute('grid-size' )) p.GRID_SIZE = parseInt (this.getAttribute('grid-size' )) ?? p.GRID_SIZE; if(this.hasAttribute('focal-length' )) p.FOCAL_LENGTH = parseFloat(this.getAttribute('focal-length' )) ?? p.FOCAL_LENGTH; if(this.hasAttribute('depth-min' )) p.DEPTH_MIN = parseFloat(this.getAttribute('depth-min' )) ?? p.DEPTH_MIN; if(this.hasAttribute('depth-max' )) p.DEPTH_MAX = parseFloat(this.getAttribute('depth-max' )) ?? p.DEPTH_MAX; if(this.hasAttribute('discard-cutoff' )) p.DISCARD_CUTOFF = parseFloat(this.getAttribute('discard-cutoff' )) ?? p.DISCARD_CUTOFF; if(this.hasAttribute('cam-radius' )) p.CAM_RADIUS = parseFloat(this.getAttribute('cam-radius' )) ?? p.CAM_RADIUS; if(this.hasAttribute('cam-max-theta' )) p.CAM_MAX_THETA = parseFloat(this.getAttribute('cam-max-theta' )) ?? p.CAM_MAX_THETA; if(this.hasAttribute('cam-max-phi' )) p.CAM_MAX_PHI = parseFloat(this.getAttribute('cam-max-phi' )) ?? p.CAM_MAX_PHI; if(this.hasAttribute('cam-fov' )) p.CAM_FOV = parseFloat(this.getAttribute('cam-fov' )) ?? p.CAM_FOV; if(this.hasAttribute('cam-smoothness' )) p.CAM_SMOOTHNESS = parseFloat(this.getAttribute('cam-smoothness' )) ?? p.CAM_SMOOTHNESS; if(this.hasAttribute('cam-zoom-factor')) p.CAM_ZOOM_FACTOR = parseFloat(this.getAttribute('cam-zoom-factor')) ?? p.CAM_ZOOM_FACTOR; if(this.hasAttribute('boomerang-speed')) p.BOOMERANG_SPEED = parseFloat(this.getAttribute('boomerang-speed')) ?? p.BOOMERANG_SPEED; if(this.hasAttribute('ad-mode')) { this._adMode = this.getAttribute('ad-mode') === 'true' || this.getAttribute('ad-mode') === ''; } if(this.hasAttribute('background-color')) { try { const arr = this.getAttribute('background-color').split(',').map(Number); if (arr.length === 3) p.BACKGROUND_COLOR = arr; } catch(e) {} } } _createShader(gl, type, source) { const sh = gl.createShader(type); gl.shaderSource(sh, source); gl.compileShader(sh); if(!gl.getShaderParameter(sh, gl.COMPILE_STATUS)) { console.error(gl.getShaderInfoLog(sh)); gl.deleteShader(sh); return null; } return sh; } _createProgram(gl, vsSrc, fsSrc) { const vs = this._createShader(gl, gl.VERTEX_SHADER, vsSrc); const fs = this._createShader(gl, gl.FRAGMENT_SHADER, fsSrc); if(!vs || !fs) return null; const prog = gl.createProgram(); gl.attachShader(prog, vs); gl.attachShader(prog, fs); gl.linkProgram(prog); if (!gl.getProgramParameter(prog, gl.LINK_STATUS)) { console.error(gl.getProgramInfoLog(prog)); gl.deleteProgram(prog); return null; } gl.deleteShader(vs); gl.deleteShader(fs); return prog; } _createTexture(gl) { const t = gl.createTexture(); gl.bindTexture(gl.TEXTURE_2D, t); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); return t; } _updateTexture(name, source) { if(!this._gl) return; const gl = this._gl; const tex = this._textures[name]; gl.activeTexture(name === 'color' ? gl.TEXTURE0 : gl.TEXTURE1); gl.bindTexture(gl.TEXTURE_2D, tex); if(!this._isSourceReady(source)) { gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, 1,1,0, gl.RGBA, gl.UNSIGNED_BYTE, new Uint8Array([0,0,0,0])); return; } try { gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false); gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, source); } catch(e) { } } _perspective(fovy, aspect, near, far) { const f = 1.0 / Math.tan(fovy / 2); return [ f / aspect, 0.0, 0.0, 0.0, 0.0 , f , 0.0, 0.0, 0.0 , 0.0, (far + near) / (near - far), -1.0, 0.0 , 0.0, (2.0 * far * near) / (near - far), 0.0 ]; } _lookAt(eye, target, up) { function normalize(v) { const l = Math.hypot(...v) || 1; return v.map(x => x / l); } function cross(a,b) { return [ a[1] * b[2] - a[2] * b[1], a[2] * b[0] - a[0] * b[2], a[0] * b[1] - a[1] * b[0] ]; } function sub(a,b) { return a.map((x, i) => x - b[i]); } const z = normalize(sub(eye, target)); const x = normalize(cross(up, z)); const y = cross(z, x); return [ x[0], y[0], z[0], 0.0, x[1], y[1], z[1], 0.0, x[2], y[2], z[2], 0.0, -(x[0] * eye[0] + x[1] * eye[1] + x[2] * eye[2]), -(y[0] * eye[0] + y[1] * eye[1] + y[2] * eye[2]), -(z[0] * eye[0] + z[1] * eye[1] + z[2] * eye[2]), 1.0 ]; } _animate(target, cur, dt) { const amt = (1.0 - this._params.CAM_SMOOTHNESS) * dt / 16.67; const orgSign = Math.sign(target - cur); cur += (target - cur) * amt; if(Math.sign(target - cur) !== orgSign) cur = target; return cur; } _loadSources() { throw new Error('Must implement _loadSources'); } _createFallback() { throw new Error('Must implement _createFallback'); } _updateTexturesIfNeeded() { } _isSourceReady(source) { throw new Error('Must implement _isSourceReady'); } _getSourceWidth(source) { throw new Error('Must implement _getSourceWidth'); } _getSourceHeight(source) { throw new Error('Must implement _getSourceHeight'); } recordBoomerangVideo(options = {}) { const opts = Object.assign({ fps: 30, cycles: 1, mimeType: null, autoDownload: true, filename: 'boomerang.webm', overlayElement: null }, options); const periodMs = 2 * (2 * Math.PI) / this._params.BOOMERANG_SPEED; const durationMs = Math.max(1, Math.round(periodMs * opts.cycles)); let mimeType = opts.mimeType; if(!mimeType) { if(MediaRecorder.isTypeSupported('video/webm;codecs=vp9')) mimeType = 'video/webm;codecs=vp9'; else if(MediaRecorder.isTypeSupported('video/webm;codecs=vp8')) mimeType = 'video/webm;codecs=vp8'; else if(MediaRecorder.isTypeSupported('video/webm')) mimeType = 'video/webm'; else mimeType = ''; } return new Promise((resolve, reject) => { try { let stream; let compositeCanvas = null; let compositeCtx = null; let animationFrameId = null; // If overlay is provided, create a composite canvas if (opts.overlayElement) { compositeCanvas = document.createElement('canvas'); compositeCanvas.width = this._canvas.width; compositeCanvas.height = this._canvas.height; compositeCtx = compositeCanvas.getContext('2d'); // Function to draw composite frame const drawComposite = () => { // Clear canvas compositeCtx.clearRect(0, 0, compositeCanvas.width, compositeCanvas.height); // Draw WebGL canvas compositeCtx.drawImage(this._canvas, 0, 0); // Draw overlay on top if (opts.overlayElement && opts.overlayElement.complete) { // Save context state compositeCtx.save(); // Get the overlay's transform string const transformStyle = window.getComputedStyle(opts.overlayElement).transform; // Calculate base position and size for object-fit: contain const canvasAspect = compositeCanvas.width / compositeCanvas.height; const imgAspect = opts.overlayElement.naturalWidth / opts.overlayElement.naturalHeight; let drawWidth, drawHeight, drawX, drawY; if (imgAspect > canvasAspect) { // Image is wider than canvas drawWidth = compositeCanvas.width; drawHeight = drawWidth / imgAspect; drawX = 0; drawY = (compositeCanvas.height - drawHeight) / 2; } else { // Image is taller than canvas drawHeight = compositeCanvas.height; drawWidth = drawHeight * imgAspect; drawX = (compositeCanvas.width - drawWidth) / 2; drawY = 0; } // Parse and apply transform if (transformStyle && transformStyle !== 'none') { // Extract translate and scale from the transform // Format: translate(Xpx, Ypx) scale(S) const translateMatch = transformStyle.match(/translate\(([^,]+),\s*([^)]+)\)/); const scaleMatch = transformStyle.match(/scale\(([^)]+)\)/); let translateX = 0, translateY = 0, scale = 1; if (translateMatch) { translateX = parseFloat(translateMatch[1]); translateY = parseFloat(translateMatch[2]); } if (scaleMatch) { scale = parseFloat(scaleMatch[1]); } // Move to center of canvas compositeCtx.translate(compositeCanvas.width / 2, compositeCanvas.height / 2); // Apply user's translate compositeCtx.translate(translateX, translateY); // Apply user's scale compositeCtx.scale(scale, scale); // Move back so image draws from center compositeCtx.translate(-drawWidth / 2, -drawHeight / 2); // Adjust position to account for object-fit offset compositeCtx.translate(-drawX, -drawY); } compositeCtx.drawImage(opts.overlayElement, drawX, drawY, drawWidth, drawHeight); // Restore context state compositeCtx.restore(); } }; // Start rendering composite frames const renderLoop = () => { drawComposite(); animationFrameId = requestAnimationFrame(renderLoop); }; renderLoop(); stream = compositeCanvas.captureStream(opts.fps); } else { // No overlay, use original canvas stream = this._canvas.captureStream(opts.fps); } const recorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined); const chunks = []; recorder.ondataavailable = (ev) => { if (ev.data && ev.data.size) chunks.push(ev.data); }; recorder.onerror = (e) => { console.error('MediaRecorder error', e); }; recorder.onstop = () => { // Clean up composite canvas animation if (animationFrameId !== null) { cancelAnimationFrame(animationFrameId); } const blob = new Blob(chunks, { type: mimeType }); if(opts.autoDownload) { const url = URL.createObjectURL(blob); const a = document.createElement('a'); a.href = url; a.download = opts.filename; document.body.appendChild(a); a.click(); a.remove(); setTimeout(() => URL.revokeObjectURL(url), 10000); } resolve(blob); }; this._recording = true; recorder.start(); setTimeout(() => { if(recorder.state !== 'inactive') recorder.stop(); this._recording = false; }, durationMs); } catch(err) { reject(err); } }); } } RGBD = _RGBD; //-------------------------// class _RGBDimg extends RGBD { static get observedAttributes() { return [ 'color-src', 'depth-src', 'color-id', 'depth-id', 'grid-size', 'focal-length', 'depth-min', 'depth-max', 'discard-cutoff', 'cam-radius', 'cam-max-theta', 'cam-max-phi', 'cam-fov', 'cam-smoothness', 'cam-zoom-factor', 'boomerang-speed', 'background-color', 'ad-mode' ]; } constructor() { super(); this._colorSource = new Image(); this._depthSource = new Image(); this._colorSource.crossOrigin = 'anonymous'; this._depthSource.crossOrigin = 'anonymous'; } attributeChangedCallback(name, oldVal, newVal) { if(oldVal === newVal) return; if (name === 'color-src') this._colorSource.src = newVal || ''; else if(name === 'depth-src') this._depthSource.src = newVal || ''; else if(name === 'color-id') { const img = document.getElementById(newVal); if(img instanceof HTMLImageElement) { this._colorSource = img; if(img.complete) this._updateTexture('color', img); else img.addEventListener('load', () => this._updateTexture('color', img)); } } else if(name === 'depth-id') { const img = document.getElementById(newVal); if(img instanceof HTMLImageElement) { this._depthSource = img; if(img.complete) this._updateTexture('depth', img); else img.addEventListener('load', () => this._updateTexture('depth', img)); } } else this._readParams(); } _loadSources() { const colorId = this.getAttribute('color-id'); const depthId = this.getAttribute('depth-id'); const colorSrc = this.getAttribute('color-src'); const depthSrc = this.getAttribute('depth-src'); if(colorId) { const img = document.getElementById(colorId); if(img instanceof HTMLImageElement && img.complete) { this._colorSource = img; this._updateTexture('color', this._colorSource); } else if(img) { img.addEventListener('load', () => this._updateTexture('color', img)); this._colorSource = img; } } else if(colorSrc) { this._colorSource.src = colorSrc; this._colorSource.onload = () => this._updateTexture('color', this._colorSource); } if(depthId) { const img = document.getElementById(depthId); if(img instanceof HTMLImageElement && img.complete) { this._depthSource = img; this._updateTexture('depth', this._depthSource); } else if (img) { img.addEventListener('load', () => this._updateTexture('depth', img)); this._depthSource = img; } } else if(depthSrc) { this._depthSource.src = depthSrc; this._depthSource.onload = () => this._updateTexture('depth', this._depthSource); } } _createFallback() { const colorId = this.getAttribute('color-id'); if(colorId) { this._colorSource = this._colorSource.cloneNode(); this._colorSource.id += "_threedad-img"; } this._colorSource.style.display = "block"; this._colorSource.classList.add("cover-img"); this._canvas.parentElement.appendChild(this._colorSource); } _isSourceReady(source) { return source && source.complete && source.naturalWidth > 0; } _getSourceWidth(source) { return source ? source.width : 1; } _getSourceHeight(source) { return source ? source.height : 1; } } RGBDimg = _RGBDimg; //-------------------------// class _RGBDvideo extends RGBD { static get observedAttributes() { return [ 'color-src', 'depth-src', 'color-id', 'depth-id', 'grid-size', 'focal-length', 'depth-min', 'depth-max', 'discard-cutoff', 'cam-radius', 'cam-max-theta', 'cam-max-phi', 'cam-fov', 'cam-smoothness', 'cam-zoom-factor', 'boomerang-speed', 'background-color', 'ad-mode', 'autoplay', 'loop', 'muted' ]; } constructor() { super(); this._colorSource = document.createElement('video'); this._depthSource = document.createElement('video'); this._colorSource.crossOrigin = 'anonymous'; this._depthSource.crossOrigin = 'anonymous'; this._colorSource.playsInline = true; this._depthSource.playsInline = true; this._colorSource.muted = true; this._depthSource.muted = true; } attributeChangedCallback(name, oldVal, newVal) { if(oldVal === newVal) return; if (name === 'color-src') this._colorSource.src = newVal || ''; else if(name === 'depth-src') this._depthSource.src = newVal || ''; else if(name === 'color-id') { const vid = document.getElementById(newVal); if(vid instanceof HTMLVideoElement) { this._colorSource = vid; if(vid.readyState >= vid.HAVE_CURRENT_DATA) this._updateTexture('color', vid); else vid.addEventListener('loadeddata', () => this._updateTexture('color', vid)); } } else if(name === 'depth-id') { const vid = document.getElementById(newVal); if(vid instanceof HTMLVideoElement) { this._depthSource = vid; if(vid.readyState >= vid.HAVE_CURRENT_DATA) this._updateTexture('depth', vid); else vid.addEventListener('loadeddata', () => this._updateTexture('depth', vid)); } } else if(name === 'autoplay') { this._colorSource.autoplay = this.hasAttribute('autoplay'); this._depthSource.autoplay = this.hasAttribute('autoplay'); } else if(name === 'loop') { this._colorSource.loop = this.hasAttribute('loop'); this._depthSource.loop = this.hasAttribute('loop'); } else if(name === 'muted') { this._colorSource.muted = this.hasAttribute('muted'); this._depthSource.muted = this.hasAttribute('muted'); } else this._readParams(); } play() { return Promise.all([ this._colorSource?.play(), this._depthSource?.play() ]); } pause() { this._colorSource?.pause(); this._depthSource?.pause(); } get paused() { return this._colorSource ? this._colorSource.paused : true; } get currentTime() { return this._colorSource ? this._colorSource.currentTime : 0; } set currentTime(value) { console.log(value); if(this._colorSource) this._colorSource.currentTime = value; if(this._depthSource) this._depthSource.currentTime = value; } _loadSources() { const colorId = this.getAttribute('color-id'); const depthId = this.getAttribute('depth-id'); const colorSrc = this.getAttribute('color-src'); const depthSrc = this.getAttribute('depth-src'); //update attributes: //----------------- if(this.hasAttribute('autoplay')) { this._colorSource.autoplay = true; this._depthSource.autoplay = true; } if(this.hasAttribute('loop')) { this._colorSource.loop = true; this._depthSource.loop = true; } if(this.hasAttribute('muted')) { this._colorSource.muted = true; this._depthSource.muted = true; } //set sources: //----------------- if(colorId) { const vid = document.getElementById(colorId); if(vid instanceof HTMLVideoElement) this._colorSource = vid; } else if(colorSrc) this._colorSource.src = colorSrc; if(depthId) { const vid = document.getElementById(depthId); if(vid instanceof HTMLVideoElement) this._depthSource = vid; } else if(depthSrc) this._depthSource.src = depthSrc; //play if autoplay set: //----------------- if(this.hasAttribute('autoplay')) { this.play().then(() => { this._onResize(); }); } else { //TODO wtf? this.play().then(() => { this.pause(); this._onResize(); }); } } _createFallback() { const colorId = this.getAttribute('color-id'); if(colorId) { this._colorSource = this._colorSource.cloneNode(true); this._colorSource.id += "_threedad-video"; } this._colorSource.style.display = "block"; this._colorSource.classList.add("cover-video"); this._colorSource.controls = true; this._canvas.parentElement.appendChild(this._colorSource); } _updateTexturesIfNeeded() { if(this._isSourceReady(this._colorSource)) this._updateTexture('color', this._colorSource); if(this._isSourceReady(this._depthSource)) this._updateTexture('depth', this._depthSource); const drift = Math.abs(this._colorSource.currentTime - this._depthSource.currentTime); const MAX_DRIFT = 0.010; if(drift > MAX_DRIFT) { this._depthSource.currentTime = this._colorSource.currentTime; if(this._colorSource.paused !== this._depthSource.paused) { if(this._colorSource.paused) this._depthSource.pause(); else this._depthSource.play(); } } } _isSourceReady(source) { return source && source.readyState >= source.HAVE_CURRENT_DATA; } _getSourceWidth(source) { return source ? source.videoWidth : 1; } _getSourceHeight(source) { return source ? source.videoHeight : 1; } } RGBDvideo = _RGBDvideo; //-------------------------// customElements.define('rgbd-img-2', RGBDimg); customElements.define('rgbd-video', RGBDvideo); }