使用深度图绘制2D图像以实现伪3D效果的WebGL绘图

Question

我正在学习WebGL，并在WebGLFundamentals页面上得到了很大的帮助，这有助于我更好地理解缓冲区、着色器和所有相关内容。但现在我想实现一个特定的效果，就像我在这里看到的https://tympanus.net/Tutorials/HeatDistortionEffect/index3.html。 我知道如何制作热扭曲效果，我想要实现的效果是图像的深度。这个演示有一个教程，但它并没有真正解释如何做到这一点，它说我必须有一个灰度地图，其中白色部分离镜头最近，黑色部分离镜头最远。但我真的无法理解它是如何工作的。这是我的着色器代码:

var vertexShaderText = [
     "attribute vec2 a_position;",
     "attribute vec2 a_texCoord;",
     "uniform vec2 u_resolution;",
     "varying vec2 v_texCoord;",
     "void main() {",
     "  vec2 zeroToOne = a_position / u_resolution;",
     "  vec2 zeroToTwo = zeroToOne * 2.0;",
     "  vec2 clipSpace = zeroToTwo - 1.0;",
     "  gl_Position = vec4(clipSpace * vec2(1, -1), 0, 1);",
     "  v_texCoord = a_texCoord;",
     "}"
  ].join("\n")

  var fragShaderText = [
     "precision mediump float;",
     "uniform sampler2D u_image;",
     "uniform sampler2D u_depthMap;",
     "uniform vec2 mouse;",
     "varying vec2 v_texCoord;",
     "void main() {",
     "  float frequency=100.0;",
     "  float amplitude=0.010;",
     "  float distortion=sin(v_texCoord.y*frequency)*amplitude;",
     "  float map=texture2D(u_depthMap,v_texCoord).r;",
     "  vec4 color=texture2D(u_image,vec2(v_texCoord.x+distortion*map, v_texCoord.y));",
     "  gl_FragColor = color;",
     "}"
  ].join("\n")

我希望当我移动鼠标时，图像可以响应着色器来扭曲变形，就像我上面展示的链接中一样。但是我真的不知道如何在javascript部分实现它。

谢谢。

- NickyP

---

那个效果看起来不像图像的深度，更像是模糊。基本上，采取类似于这个模糊，但更多或更少地应用于某些部分。或者你想实现不同的效果吗？ - gman

谢谢您的回复，不，我正在尝试实现类似我发布链接中的深度效果，我发布的片段着色器可以实现热扭曲效果，但我需要将鼠标坐标传递给着色器并加载深度图以获得效果。但我真的不知道该怎么做... - NickyP

你指向的链接与深度无关。您可以使用WebGL Inspector或许多其他实用程序查看着色器。它所做的只是在未模糊的图像和预先模糊的图像之间进行混合。它使用一些波纹计算和这个扭曲映射来决定如何混合模糊图像和非模糊图像以及如何偏移像素。 - gman

Answer 1

从同一网站的图像处理教程中我们可以学习如何加载多张图片。你提供的示例和该教程很明确地展示了它的工作原理。

首先需要一个原始图像：

然后再应用正弦波畸变。

"use strict";

function main() {
  // Get A WebGL context
  /** @type {HTMLCanvasElement} */
  const canvas = document.getElementById("canvas");
  const gl = canvas.getContext("webgl");
  if (!gl) {
    return;
  }

  let originalImage = { width: 1, height: 1 }; // replaced after loading
  const originalTexture = twgl.createTexture(gl, {
    src: "https://i.imgur.com/xKYRSwu.jpg", crossOrigin: '',
  }, (err, texture, source) => {
    originalImage = source;
  });

  // compile shaders, link program, lookup location
  const programInfo = twgl.createProgramInfo(gl, ["vs", "fs"]);

  // calls gl.createBuffer, gl.bindBuffer, gl.bufferData for a quad
  const bufferInfo = twgl.primitives.createXYQuadBufferInfo(gl);

  requestAnimationFrame(render);
  
  function render(time) {
    time *= 0.001;  // seconds
    
    twgl.resizeCanvasToDisplaySize(gl.canvas);

    gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);

    gl.clearColor(0, 0, 0, 0);
    gl.clear(gl.COLOR_BUFFER_BIT);

    gl.useProgram(programInfo.program);

    // calls gl.bindBuffer, gl.enableVertexAttribArray, gl.vertexAttribPointer
    twgl.setBuffersAndAttributes(gl, programInfo, bufferInfo);

    const canvasAspect = gl.canvas.clientWidth / gl.canvas.clientHeight;
    const imageAspect = originalImage.width / originalImage.height;
    const mat = m3.scaling(imageAspect / canvasAspect, -1);
    
    // calls gl.activeTexture, gl.bindTexture, gl.uniformXXX
    twgl.setUniforms(programInfo, {
      u_matrix: mat,
      u_originalImage: originalTexture,
      u_distortionAmount: 0.003,  // .3%
      u_distortionRange: 100,
      u_time: time * 10,
    });

    // calls gl.drawArrays or gl.drawElements
    twgl.drawBufferInfo(gl, bufferInfo);
    
    requestAnimationFrame(render);
  }
}

main();

body { margin: 0; }
canvas { width: 100vw; height: 100vh; display: block; }

<script src="https://twgljs.org/dist/3.x/twgl-full.min.js"></script>
<canvas id="canvas"></canvas>

<!-- vertex shader -->
<script id="vs" type="f">
attribute vec2 position;
attribute vec2 texcoord;

uniform mat3 u_matrix;

varying vec2 v_texcoord;

void main() {
   gl_Position = vec4((u_matrix * vec3(position, 1)).xy, 0, 1);
   v_texcoord = texcoord;
}
</script>
<!-- fragment shader -->
<script id="fs" type="x-shader/x-fragment">
precision mediump float;

uniform float u_time;
uniform float u_distortionAmount;
uniform float u_distortionRange;

// our textures
uniform sampler2D u_originalImage;

// the texcoords passed in from the vertex shader.
varying vec2 v_texcoord;

void main() {
   vec2 distortion = vec2(
      sin(u_time + v_texcoord.y * u_distortionRange), 0) * u_distortionAmount;
   vec4 original = texture2D(u_originalImage, v_texcoord + distortion);
   gl_FragColor = original;
}
</script>
<script src="https://webglfundamentals.org/webgl/resources/m3.js"></script>

然后他们还会加载多个地图的纹理。这个纹理是在 Photoshop（或其他图片编辑程序）中手工创建的。绿色通道是扭曲程度的乘数。绿色越浓，扭曲就越大。

"use strict";

function main() {
  // Get A WebGL context
  /** @type {HTMLCanvasElement} */
  const canvas = document.getElementById("canvas");
  const gl = canvas.getContext("webgl");
  if (!gl) {
    return;
  }

  let originalImage = { width: 1, height: 1 }; // replaced after loading
  const originalTexture = twgl.createTexture(gl, {
    src: "https://i.imgur.com/xKYRSwu.jpg", 
    crossOrigin: '',
  }, (err, texture, source) => {
    originalImage = source;
  });
  
  const mapTexture = twgl.createTexture(gl, {
    src: "https://i.imgur.com/W9QazjL.jpg", crossOrigin: '',
  });

  // compile shaders, link program, lookup location
  const programInfo = twgl.createProgramInfo(gl, ["vs", "fs"]);

  // calls gl.createBuffer, gl.bindBuffer, gl.bufferData for a quad
  const bufferInfo = twgl.primitives.createXYQuadBufferInfo(gl);

  requestAnimationFrame(render);
  
  function render(time) {
    time *= 0.001;  // seconds
    
    twgl.resizeCanvasToDisplaySize(gl.canvas);

    gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);

    gl.clearColor(0, 0, 0, 0);
    gl.clear(gl.COLOR_BUFFER_BIT);

    gl.useProgram(programInfo.program);

    // calls gl.bindBuffer, gl.enableVertexAttribArray, gl.vertexAttribPointer
    twgl.setBuffersAndAttributes(gl, programInfo, bufferInfo);

    const canvasAspect = gl.canvas.clientWidth / gl.canvas.clientHeight;
    const imageAspect = originalImage.width / originalImage.height;
    const mat = m3.scaling(imageAspect / canvasAspect, -1);
    
    // calls gl.activeTexture, gl.bindTexture, gl.uniformXXX
    twgl.setUniforms(programInfo, {
      u_matrix: mat,
      u_originalImage: originalTexture,
      u_mapImage: mapTexture,
      u_distortionAmount: 0.003,  // .3%
      u_distortionRange: 100,
      u_time: time * 10,
    });

    // calls gl.drawArrays or gl.drawElements
    twgl.drawBufferInfo(gl, bufferInfo);
    
    requestAnimationFrame(render);
  }
}

main();

body { margin: 0; }
canvas { width: 100vw; height: 100vh; display: block; }

<canvas id="canvas"></canvas>

<script id="vs" type="f">
attribute vec2 position;
attribute vec2 texcoord;

uniform mat3 u_matrix;

varying vec2 v_texcoord;

void main() {
   gl_Position = vec4(u_matrix * vec3(position, 1), 1);
   v_texcoord = texcoord;
}
</script>
<script id="fs" type="f">
precision mediump float;

uniform float u_time;
uniform float u_distortionAmount;
uniform float u_distortionRange;

// our textures
uniform sampler2D u_originalImage;
uniform sampler2D u_mapImage;

// the texcoords passed in from the vertex shader.
varying vec2 v_texcoord;

void main() {
   vec4 depthDistortion = texture2D(u_mapImage, v_texcoord);
   float distortionMult = depthDistortion.g;  // just green channel
   
   vec2 distortion = vec2(
      sin(u_time + v_texcoord.y * u_distortionRange), 0) * u_distortionAmount;
   vec4 color0 = texture2D(u_originalImage, v_texcoord + distortion * distortionMult);
   gl_FragColor = color0;
}
</script>
<script src="https://twgljs.org/dist/3.x/twgl-full.min.js"></script>
<script src="https://webglfundamentals.org/webgl/resources/m3.js"></script>

接下来，还有一个偏移量与另一个手工制作的地图相乘来调整鼠标。这个地图是上面图片的红色通道，越红表示应用的鼠标偏移越大。这张地图可以代表深度。由于我们需要前景物体与背景物体相反移动，所以我们需要在着色器中将该通道从0到1转换为-0.5到+0.5。

"use strict";

function main() {
  // Get A WebGL context
  /** @type {HTMLCanvasElement} */
  const canvas = document.getElementById("canvas");
  const gl = canvas.getContext("webgl");
  if (!gl) {
    return;
  }

  let originalImage = { width: 1, height: 1 }; // replaced after loading
  const originalTexture = twgl.createTexture(gl, {
    src: "https://i.imgur.com/xKYRSwu.jpg", 
    crossOrigin: '',
  }, (err, texture, source) => {
    originalImage = source;
  });
  
  const mapTexture = twgl.createTexture(gl, {
    src: "https://i.imgur.com/W9QazjL.jpg", crossOrigin: '',
  });

  // compile shaders, link program, lookup location
  const programInfo = twgl.createProgramInfo(gl, ["vs", "fs"]);

  // calls gl.createBuffer, gl.bindBuffer, gl.bufferData for a quad
  const bufferInfo = twgl.primitives.createXYQuadBufferInfo(gl);

  const mouse = [0, 0];
  document.addEventListener('mousemove', (event) => {
    mouse[0] = (event.clientX / gl.canvas.clientWidth  * 2 - 1) * 0.05;
    mouse[1] = (event.clientY / gl.canvas.clientHeight * 2 - 1) * 0.05;
  });
  
  requestAnimationFrame(render);
  
  function render(time) {
    time *= 0.001;  // seconds
    
    twgl.resizeCanvasToDisplaySize(gl.canvas);

    gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);

    gl.clearColor(0, 0, 0, 0);
    gl.clear(gl.COLOR_BUFFER_BIT);

    gl.useProgram(programInfo.program);

    // calls gl.bindBuffer, gl.enableVertexAttribArray, gl.vertexAttribPointer
    twgl.setBuffersAndAttributes(gl, programInfo, bufferInfo);

    const canvasAspect = gl.canvas.clientWidth / gl.canvas.clientHeight;
    const imageAspect = originalImage.width / originalImage.height;
    const mat = m3.scaling(imageAspect / canvasAspect, -1);
    
    // calls gl.activeTexture, gl.bindTexture, gl.uniformXXX
    twgl.setUniforms(programInfo, {
      u_matrix: mat,
      u_originalImage: originalTexture,
      u_mapImage: mapTexture,
      u_distortionAmount: 0.003,  // .3%
      u_distortionRange: 100,
      u_time: time * 10,
      u_mouse: mouse,
    });

    // calls gl.drawArrays or gl.drawElements
    twgl.drawBufferInfo(gl, bufferInfo);
    
    requestAnimationFrame(render);
  }
}

main();

body { margin: 0; }
canvas { width: 100vw; height: 100vh; display: block; }

<canvas id="canvas"></canvas>

<!-- vertex shader -->
<script id="vs" type="f">
attribute vec2 position;
attribute vec2 texcoord;

uniform mat3 u_matrix;

varying vec2 v_texcoord;

void main() {
   gl_Position = vec4(u_matrix * vec3(position, 1), 1);
   v_texcoord = texcoord;
}
</script>
<!-- fragment shader -->
<script id="fs" type="f">
precision mediump float;

uniform float u_time;
uniform float u_distortionAmount;
uniform float u_distortionRange;
uniform vec2 u_mouse;

// our textures
uniform sampler2D u_originalImage;
uniform sampler2D u_mapImage;

// the texcoords passed in from the vertex shader.
varying vec2 v_texcoord;

void main() {
   vec4 depthDistortion = texture2D(u_mapImage, v_texcoord);
   float distortionMult = depthDistortion.g;     // just green channel
   float parallaxMult = 0.5 - depthDistortion.r; // just red channel
   
   vec2 distortion = vec2(
      sin(u_time + v_texcoord.y * u_distortionRange), 0) * u_distortionAmount  * distortionMult;
   vec2 parallax = u_mouse * parallaxMult;
      
   vec4 color0 = texture2D(u_originalImage, v_texcoord + distortion + parallax);
   gl_FragColor = color0;
}
</script>
<script src="https://twgljs.org/dist/3.x/twgl-full.min.js"></script>
<script src="https://webglfundamentals.org/webgl/resources/m3.js"></script>

最后，（不在教程中，但在示例中）它加载了原始图像的模糊版本（在某些图像编辑程序（如Photoshop）中进行了模糊处理）

由于模糊处理很微妙，可能很难看出来。

然后示例使用模糊图像使事物更加扭曲。

"use strict";

function main() {
  // Get A WebGL context
  /** @type {HTMLCanvasElement} */
  const canvas = document.getElementById("canvas");
  const gl = canvas.getContext("webgl");
  if (!gl) {
    return;
  }

  let originalImage = { width: 1, height: 1 }; // replaced after loading
  const originalTexture = twgl.createTexture(gl, {
    src: "https://i.imgur.com/xKYRSwu.jpg", 
    crossOrigin: '',
  }, (err, texture, source) => {
    originalImage = source;
  });
  
  const mapTexture = twgl.createTexture(gl, {
    src: "https://i.imgur.com/W9QazjL.jpg", crossOrigin: '',
  });
  
  const blurredTexture = twgl.createTexture(gl, {
    src: "https://i.imgur.com/Zw7mMLX.jpg", crossOrigin: '',
  });

  // compile shaders, link program, lookup location
  const programInfo = twgl.createProgramInfo(gl, ["vs", "fs"]);

  // calls gl.createBuffer, gl.bindBuffer, gl.bufferData for a quad
  const bufferInfo = twgl.primitives.createXYQuadBufferInfo(gl);

  const mouse = [0, 0];
  document.addEventListener('mousemove', (event) => {
    mouse[0] = (event.clientX / gl.canvas.clientWidth  * 2 - 1) * 0.05;
    mouse[1] = (event.clientY / gl.canvas.clientHeight * 2 - 1) * 0.05;
  });
  
  requestAnimationFrame(render);
  
  function render(time) {
    time *= 0.001;  // seconds
    
    twgl.resizeCanvasToDisplaySize(gl.canvas);

    gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);

    gl.clearColor(0, 0, 0, 0);
    gl.clear(gl.COLOR_BUFFER_BIT);

    gl.useProgram(programInfo.program);

    // calls gl.bindBuffer, gl.enableVertexAttribArray, gl.vertexAttribPointer
    twgl.setBuffersAndAttributes(gl, programInfo, bufferInfo);

    const canvasAspect = gl.canvas.clientWidth / gl.canvas.clientHeight;
    const imageAspect = originalImage.width / originalImage.height;
    const mat = m3.scaling(imageAspect / canvasAspect, -1);
    
    // calls gl.activeTexture, gl.bindTexture, gl.uniformXXX
    twgl.setUniforms(programInfo, {
      u_matrix: mat,
      u_originalImage: originalTexture,
      u_mapImage: mapTexture,
      u_blurredImage: blurredTexture,
      u_distortionAmount: 0.003,  // .3%
      u_distortionRange: 100,
      u_time: time * 10,
      u_mouse: mouse,
    });

    // calls gl.drawArrays or gl.drawElements
    twgl.drawBufferInfo(gl, bufferInfo);
    
    requestAnimationFrame(render);
  }
}

main();

body { margin: 0; }
canvas { width: 100vw; height: 100vh; display: block; }

<canvas id="canvas"></canvas>

<!-- vertex shader -->
<script id="vs" type="f">
attribute vec2 position;
attribute vec2 texcoord;

uniform mat3 u_matrix;

varying vec2 v_texcoord;

void main() {
   gl_Position = vec4(u_matrix * vec3(position, 1), 1);
   v_texcoord = texcoord;
}
</script>
<!-- fragment shader -->
<script id="fs" type="f">
precision mediump float;

uniform float u_time;
uniform float u_distortionAmount;
uniform float u_distortionRange;
uniform vec2 u_mouse;

// our textures
uniform sampler2D u_originalImage;
uniform sampler2D u_blurredImage;
uniform sampler2D u_mapImage;

// the texcoords passed in from the vertex shader.
varying vec2 v_texcoord;

void main() {
   vec4 depthDistortion = texture2D(u_mapImage, v_texcoord);
   float distortionMult = depthDistortion.g;     // just green channel
   float parallaxMult = 0.5 - depthDistortion.r; // just red channel
   
   vec2 distortion = vec2(
      sin(u_time + v_texcoord.y * u_distortionRange), 0) * u_distortionAmount  * distortionMult;
   vec2 parallax = u_mouse * parallaxMult;
      
   vec2 uv = v_texcoord + distortion + parallax;
   vec4 original = texture2D(u_originalImage, uv);
   vec4 blurred = texture2D(u_blurredImage, uv);
   gl_FragColor = mix(original, blurred, length(distortion) / u_distortionAmount);
}
</script>
<script src="https://twgljs.org/dist/3.x/twgl-full.min.js"></script>
<script src="https://webglfundamentals.org/webgl/resources/m3.js"></script>

最大的区别在于，与其使用简单的正弦波来产生扭曲效果，该示例中的着色器计算了更加复杂的内容。

封面

上述代码使用一个两个单位大小的四边形，它在X和Y轴上从-1到+1。如果你传入一个恒等矩阵（或一个1,1比例矩阵，这是相同的），它将覆盖画布。但我们希望图像不被扭曲。为了实现这一点，我们添加了以下代码：

const canvasAspect = gl.canvas.clientWidth / gl.canvas.clientHeight;
const imageAspect = originalImage.width / originalImage.height;
const mat = m3.scaling(imageAspect / canvasAspect, -1);

这段代码的作用是使图片在垂直方向上填满画布，并按照原始图片的比例进行缩放。-1 的作用是翻转四边形，否则图片会颠倒。

要实现 cover，我们只需要检查比例是否小于 1。如果比例小于 1，则不会填充整个画布，因此我们将水平比例设置为 1 并调整垂直比例。

// this assumes we want to fill vertically
let horizontalDrawAspect = imageAspect / canvasAspect;
let verticalDrawAspect = -1;
// does it fill horizontally?
if (horizontalDrawAspect < 1) {
  // no it does not so scale so we fill horizontally and
  // adjust vertical to match
  verticalDrawAspect /= horizontalDrawAspect;
  horizontalDrawAspect = 1;
}
const mat = m3.scaling(horizontalDrawAspect, verticalDrawAspect);

- gman

---

这太棒了！正是我一直在寻找的！我想 TWGL 库现在让它变得更容易了，我不知道有这样的东西存在。 最后一个问题，如何使图像按比例缩放以适应画布大小，以与 CSS 中的背景大小：覆盖相同的方式填充屏幕？ - NickyP

TWGL在这些教程中被提到，位于少代码多乐趣下。此外，还介绍了使用矩阵将剪裁空间转换为像素的内容。 - gman