我正在使用 body-pix 从网络摄像头图像创建一个人的蒙版,并在 HTM 画布上使用直接 ImageData 字节操作实时替换背景,用背景图像中的像素替换非蒙版像素。
虽然这有效,但人物面具的边缘非常锋利:
我能够在透明背景上单独操作蒙版,所以我的想法是通过使蒙版边缘附近的像素半透明来柔化蒙版的边缘,越靠近边缘越透明。
是否有一种简单的算法可以通过迭代字节来做到这一点?我可以找到许多模糊/锐化等的例子,但我真的想在这种情况下考虑蒙版的边缘。
谢谢。
更新:根据 Blindman67 的建议,我尝试使用滤镜和合成,但结果只是背景图像模糊。
maskWithBackgroundImage: async function(outputCanvas, videoStream, backgroundImageName, frameWidth, frameHeight) {
service.FRAME_WIDTH = frameWidth;
service.FRAME_HEIGHT = frameHeight;
service.net = await bodyPix.load({
architecture: service.filterProps.architecture,
outputStride: service.filterProps.outputStride,
multiplier: service.filterProps.multiplier,
quantBytes: service.filterProps.quantBytes
});
service.videoCanvas = service.createCanvas();
service.maskCanvas = document.getElementById('maskCanvas');
service.videoCanvasContext = service.videoCanvas.getContext('2d');
service.setupBackgroundImage(backgroundImageName).then(() => {
service.segmentInRealTime(outputCanvas, videoStream);
});
},
segmentInRealTime: async function(outputCanvas, videoStream) {
let outputCanvasCtx = outputCanvas.getContext('2d');
//Draw he latest video frame
service.videoCanvasContext.drawImage(videoStream, 0, 0, service.FRAME_WIDTH, service.FRAME_HEIGHT);
//STEP 1 - Draw BG image out
outputCanvasCtx.drawImage(service.backgroundImage, 0, 0);
//STEP 2 - Get black and transparent mask
const personSegmentationImageData = await service.net.segmentPerson(service.videoCanvas, {
internalResolution: 'medium',
segmentationThreshold: service.filterProps.segmentationThreshold,
maxDetections: service.filterProps.maxDetections
});
let maskData = service.replaceNonHumanWithTransparency(personSegmentationImageData); //service.replaceNonHumanWithTransparency(personSegmentationImageData);
let maskCanvasCtx = service.maskCanvas.getContext('2d');
maskCanvasCtx.putImageData(maskData, 0, 0);
let maskImg = new Image();
maskImg.onload = function() {
//STEP 3 - Set blur and composite
outputCanvasCtx.filter = 'blur(1px)';
outputCanvasCtx.globalCompositeOperation = 'destination-out';
outputCanvasCtx.drawImage(maskImg, 0, 0);
outputCanvasCtx.globalCompositeOperation = 'destination-over';
outputCanvasCtx.drawImage(videoStream, 0, 0);
outputCanvasCtx.globalCompositeOperation = 'source-over';
window.requestAnimationFrame(() => {
service.segmentInRealTime(outputCanvas, videoStream);
});
};
maskImg.src = service.maskCanvas.toDataURL();
},
replaceNonHumanWithTransparency: function(
personOrPartSegmentation, foreground = {
r: 0,
g: 0,
b: 0,
a: 0
}, background = {
r: 0,
g: 0,
b: 0,
a: 255
}, foregroundIds = [1]) {
if (Array.isArray(personOrPartSegmentation) &&
personOrPartSegmentation.length === 0) {
return null;
}
let multiPersonOrPartSegmentation;
if (!Array.isArray(personOrPartSegmentation)) {
multiPersonOrPartSegmentation = [personOrPartSegmentation];
} else {
multiPersonOrPartSegmentation = personOrPartSegmentation;
}
const {width, height} = multiPersonOrPartSegmentation[0];
// eslint-disable-next-line no-undef
const bytes = new Uint8ClampedArray(width * height * 4);
for (let i = 0; i < height; i += 1) {
for (let j = 0; j < width; j += 1) {
const n = i * width + j;
bytes[4 * n + 0] = background.r; // background.r;
bytes[4 * n + 1] = background.g; //background.g;
bytes[4 * n + 2] = background.b; //background.b;
bytes[4 * n + 3] = background.a; //background.a;
for (let k = 0; k < multiPersonOrPartSegmentation.length; k++) {
if (foregroundIds.some(
id => id === multiPersonOrPartSegmentation[k].data[n])) {
bytes[4 * n] = foreground.r;
bytes[4 * n + 1] = foreground.g;
bytes[4 * n + 2] = foreground.b;
bytes[4 * n + 3] = foreground.a;
}
}
}
}
// eslint-disable-next-line no-undef
return new ImageData(bytes, width, height);
}
遮罩渲染良好,背景图像渲染也很好。将两者混合在一起给我带来了问题。任何想法都非常感谢。