最终答案

Question

我想要发生什么：

为了测试我想到的游戏艺术风格，我想以像素艺术形式呈现3D世界。例如，拍摄这样的场景（但是使用某种颜色/样式进行渲染，以便在像素化后看起来很好）：

让它看起来像这样：

通过使用不同的3D样式设计方式，我认为像素化输出看起来不错。当然要获得这种效果，只需将图像缩小到~80p，然后使用最近邻重新采样将其放大到1080p。但是直接渲染80p画布并开始进行升级更有效率。

通常不会使用着色器来调整最近邻格式的位图大小，但是它的性能优于我实时进行此类转换的任何其他方式。< / p>

我的代码：

我的位图缓冲区存储在行major中，如r1, g1, b1, a1, r2, g2, b2, a2...和我使用gpu.js，它实际上将此JS函数转换为着色器。我的目标是获取一个位图并以最大邻域缩放返回一个位图，因此每个像素变为2x2平方或3x3，依此类推。假设inputBuffer是由setOutput方法确定的输出大小的缩放分数。

var pixelateMatrix = gpu.createKernel(function(inputBuffer, width, height, scale) {
  var y = Math.floor((this.thread.x / (width[0] * 4)) / scale[0]);
  var x = Math.floor((this.thread.x % (width[0] * 4)) / scale[0]);
  var remainder = this.thread.x % 4;
  return inputBuffer[(x * y) + remainder]; 
}).setOutput([width * height * 4]);

JSFiddle

请记住它在全尺寸输出的新缓冲区上进行迭代，所以我必须找到将存在的正确坐标基于当前索引的较小sourceBuffer outputBuffer（索引由lib公开为this.thread.x）。

反而发生了什么：

这不是使最近的邻居高档，而是制作一个漂亮的小彩虹（上面是小的正常渲染，下面是着色器的结果，右边你可以看到一些调试记录与输入的统计数据和输出缓冲区）：

我做错了什么？

注意：我在这里问了一个相关问题，Is there a simpler (and still performant) way to upscale a canvas render with nearest neighbor resampling?

Answer 1

更新1 - 2018年5月25日

我能够解决大部分问题。有很多

转换的逻辑是错误的，数据也因某种原因而被翻转，所以我将cols和row翻转为从右下角开始

var pixelateMatrix = gpu.createKernel(function(inputBuffer, width, height, scale) {
var size = width[0] * height[0] * 4;
var current_index = Math.floor((size - this.thread.x)/4); 
var row = Math.floor(current_index / (width[0] * scale[0]) );
var col = Math.floor((current_index % width[0])/scale[0]);
var index_old = Math.floor(row * (width[0] / scale[0])) + width[0] - col;
var remainder = this.thread.x % 4;
return inputBuffer[index_old * 4 + remainder];

}).setOutput([width * height * 4]);

您在浮动中使用了宽度和高度，我已将其更改为先计算然后缩放

var smallWidth = Math.floor(window.innerWidth / scale);
var smallHeight = Math.floor(window.innerHeight / scale);

var width = smallWidth * scale;
var height = smallHeight * scale;


var rt = new THREE.WebGLRenderTarget(smallWidth, smallHeight);
var frameBuffer = new Uint8Array(smallHeight * smallHeight * 4);
var outputBuffer = new Uint8ClampedArray(width * height * 4);

画布大小设置为内部宽度和高度，您需要将其设置为图像宽度和高度
```
context = canvas.getContext('2d');
canvas.width = width;
canvas.height = height;
```

下面是相同的最终JSFiddle

https://jsfiddle.net/are5Lbw8/6/

结果：

最终守则供参考

var container;
var camera, scene, renderer;
var mouseX = 0;
var mouseY = 0;
var scale = 4;
var windowHalfX = window.innerWidth / 2;
var windowHalfY = window.innerHeight / 2;
var smallWidth = Math.floor(window.innerWidth / scale);
var smallHeight = Math.floor(window.innerHeight / scale);

var width = smallWidth * scale;
var height = smallHeight * scale;


var rt = new THREE.WebGLRenderTarget(smallWidth, smallHeight);
var frameBuffer = new Uint8Array(smallHeight * smallHeight * 4);
var outputBuffer = new Uint8ClampedArray(width * height * 4);
var output;
var divisor = 2;
var divisorHalf = divisor / 2;
var negativeDivisorHalf = -1 * divisorHalf;
var canvas;
var context;
var gpu = new GPU();

var pixelateMatrix = gpu.createKernel(function(inputBuffer, width, height, scale) {
/*   var y = Math.floor((this.thread.x / (width[0] * 4)) / scale[0]);
  var x = Math.floor((this.thread.x % (width[0] * 4)) / scale[0]);
  var remainder = this.thread.x % 4;
  return inputBuffer[(x * y) + remainder];
   */
   var size = width[0] * height[0] * 4;
   var current_index = Math.floor((size - this.thread.x)/4); 
   var row = Math.floor(current_index / (width[0] * scale[0]) );
   var col = Math.floor((current_index % width[0])/scale[0]);
   var index_old = Math.floor(row * (width[0] / scale[0])) + width[0] - col;
   var remainder = this.thread.x % 4;
   return inputBuffer[index_old * 4 + remainder];

}).setOutput([width * height * 4]);
console.log(window.innerWidth);
console.log(window.innerHeight);
init();
animate();

function init() {
  container = document.createElement('div');
  document.body.appendChild(container);
  canvas = document.createElement('canvas');
  document.body.appendChild(canvas);
  context = canvas.getContext('2d');
  canvas.width = width;
  canvas.height = height;
  camera = new THREE.PerspectiveCamera(45, window.innerWidth / window.innerHeight, 1, 2000);
  camera.position.z = 100;

  // scene
  scene = new THREE.Scene();
  var ambient = new THREE.AmbientLight(0xbbbbbb);
  scene.add(ambient);
  var directionalLight = new THREE.DirectionalLight(0xdddddd);
  directionalLight.position.set(0, 0, 1);
  scene.add(directionalLight);

  // texture
  var manager = new THREE.LoadingManager();
  manager.onProgress = function(item, loaded, total) {
    console.log(item, loaded, total);
  };
  var texture = new THREE.Texture();
  var onProgress = function(xhr) {
    if (xhr.lengthComputable) {
      var percentComplete = xhr.loaded / xhr.total * 100;
      console.log(Math.round(percentComplete, 2) + '% downloaded');
    }
  };
  var onError = function(xhr) {};
  var imgLoader = new THREE.ImageLoader(manager);
  imgLoader.load('https://i.imgur.com/P6158Su.jpg', function(image) {
    texture.image = image;
    texture.needsUpdate = true;
  });

  // model
  var objLoader = new THREE.OBJLoader(manager);
  objLoader.load('https://s3-us-west-2.amazonaws.com/s.cdpn.io/286022/Bulbasaur.obj', function(object) {
    object.traverse(function(child) {
      if (child instanceof THREE.Mesh) {
        child.material.map = texture;
      }
    });
    object.scale.x = 45;
    object.scale.y = 45;
    object.scale.z = 45;
    object.rotation.y = 3;
    object.position.y = -10.5;
    scene.add(object);
  }, onProgress, onError);
  renderer = new THREE.WebGLRenderer({
    alpha: true,
    antialias: false
  });
  renderer.setPixelRatio(window.devicePixelRatio);
  renderer.setSize(smallWidth, smallHeight);
  container.appendChild(renderer.domElement);
  renderer.context.webkitImageSmoothingEnabled = false;
  renderer.context.mozImageSmoothingEnabled = false;
  renderer.context.imageSmoothingEnabled = false;
  document.addEventListener('mousemove', onDocumentMouseMove, false);
  window.addEventListener('resize', onWindowResize, false);
}

function onWindowResize() {
  windowHalfX = (window.innerWidth / 2) / scale;
  windowHalfY = (window.innerHeight / 2) / scale;
  camera.aspect = (window.innerWidth / window.innerHeight) / scale;
  camera.updateProjectionMatrix();
  renderer.setSize(smallWidth, smallHeight);
}

function onDocumentMouseMove(event) {

  mouseX = (event.clientX - windowHalfX) / scale;
  mouseY = (event.clientY - windowHalfY) / scale;

}

function animate() {
  requestAnimationFrame(animate);
  render();
}

var flag = 0;

function render() {
  camera.position.x += (mouseX - camera.position.x) * .05;
  camera.position.y += (-mouseY - camera.position.y) * .05;
  camera.lookAt(scene.position);
  renderer.render(scene, camera);
  renderer.render(scene, camera, rt);
  renderer.readRenderTargetPixels(rt, 0, 0, smallWidth, smallHeight, frameBuffer);
  //console.time('gpu');
  console.log(frameBuffer, [width], [height], [scale]);
  var outputBufferRaw = pixelateMatrix(frameBuffer, [width], [height], [scale]);
  //console.timeEnd('gpu');
  if (flag < 15) {
    console.log('source', frameBuffer);
    console.log('output', outputBufferRaw);

    var count = 0;
    for (let i = 0; i < frameBuffer.length; i++) {
      if (frameBuffer[i] != 0) {
        count++;
      }
    }
    console.log('source buffer length', frameBuffer.length)
    console.log('source non zero', count);

    var count = 0;
    for (let i = 0; i < outputBufferRaw.length; i++) {
      if (outputBufferRaw[i] != 0) {
        count++;
      }
    }
    console.log('output buffer length', outputBufferRaw.length)
    console.log('output non zero', count);
  }
  outputBuffer = new Uint8ClampedArray(outputBufferRaw);
  output = new ImageData(outputBuffer, width, height);
  context.putImageData(output, 0, 0);
  flag++;
}

原始答案

我已经接近但仍有两个问题

图像反转
有时您的inputBuffer大小不是4的倍数，这会导致行为不端。

以下是我使用的代码

var pixelateMatrix = gpu.createKernel(function(inputBuffer, width, height, scale) {
   var current_index = Math.floor(this.thread.x/4); 
   var row = Math.floor(current_index / (width[0] * scale[0]) );
   var col = Math.floor((current_index % width[0])/scale[0]);
   var index_old = Math.floor(row * (width[0] / scale[0])) + col;
   var remainder = this.thread.x % 4;
   return inputBuffer[index_old * 4 + remainder];

}).setOutput([width * height * 4]);

下面是JSFiddle

https://jsfiddle.net/are5Lbw8/

以下是当前输出

Answer 2

我认为该功能应该如下：

var pixelateMatrix = gpu.createKernel(function(inputBuffer, width, height, scale) {
    var x = Math.floor((this.thread.x / (width[0] * 4)) / scale[0]);
    var y = Math.floor((this.thread.x % (width[0] * 4)) / scale[0]);
    var finalval = y * (Math.floor(width[0]/scale[0]) * 4) + (x * 4);
    var remainder = this.thread.x % 4;
    return inputBuffer[finalval + remainder];
}).setOutput([width * height * 4]);

基本上，以与你类似的方式得到x和y，缩放x和y，然后从（x，y）转换回来，将y值乘以新的缩放宽度并添加x值。不确定你是如何获得x * y的那部分。

Answer 3

最终答案

Tarun的回答帮助我找到了我的最终解决方案，所以他的赏金当之无愧，但我实际上了解了gpu.js的一个功能（图形输出与上下文共享，用于直接缓冲输出到渲染目标），允许渲染速度大约快30倍，使着色的总时间和将输出从30ms +渲染到~1ms，这是没有额外的优化我现在知道甚至可以将数组缓冲区发送到GPU更快，但我没有任何动机让阴影/渲染时间低于1毫秒。

var canvas = document.createElement('canvas');
canvas.width = width;
canvas.height = height;
document.body.appendChild(canvas);
var gl = canvas.getContext('webgl');
var gpu = new GPU({
  canvas,
  gl
});
var pixelateMatrix = gpu.createKernel(function(inputBuffer, width, scale, size, purity, div) {
    var subX = Math.floor(this.thread.x / scale[0]);
    var subY = Math.floor(this.thread.y / scale[0]);
    var subIndex = ((subX * 4) + (subY * width[0] * 4));
    var rIndex = subIndex;
    var gIndex = subIndex + 1;
    var bIndex = subIndex + 2;
    var r = ((inputBuffer[rIndex] * purity[0]) + inputBuffer[rIndex - 4] + inputBuffer[rIndex + 4]) / (div[0]);
    var g = ((inputBuffer[gIndex] * purity[0]) + inputBuffer[gIndex - 4] + inputBuffer[gIndex + 4]) / (div[0]);
    var b = ((inputBuffer[bIndex] * purity[0]) + inputBuffer[bIndex - 4] + inputBuffer[bIndex + 4]) / (div[0]);
    this.color(r / 255, g / 255, b / 255);
  }).setOutput([width, height]).setGraphical(true);

inputBuffer只是通过three.js的readRenderTargetPixels方法检索的缓冲区。

renderer.render(scene, camera, rt);
renderer.readRenderTargetPixels(rt, 0, 0, smallWidth, smallHeight, frameBuffer);
pixelateMatrix(frameBuffer, [smallWidth], [scale], [size], [purity], [div]);

旁注

我们能不能对WebGL为浏览器带来多少功能感到惊讶？仅仅约1ms就完成了829.44万个<强>多操作任务。根据我的计算，我的着色器每秒大约640亿次最大数学运算。那是疯了。那甚至可以吗？我的数学错了吗？ I see nvidia's self driving AI is performing 24 trillion ops/s，所以我猜我1060号的这些数字都在可能范围内。这真是令人难以置信。

GPU.js在优化矩阵操作以在GPU上运行而无需学习着色器代码方面做得非常棒，而且创建者在项目上非常活跃，通常在几个小时内响应问题。强烈建议你们试试lib。特别适合机器学习吞吐量。

如何在JS中正确编写这个着色器函数？

我想要发生什么：

我的代码：

反而发生了什么：

3 个答案:

最终答案

旁注