Question

使用 Android CameraX Analyzer ，ImageProxy在引擎盖下使用ImageReader，默认图像格式为YUV_420_888。

我想在OpenCV Mat中进行转换，以便在分析仪中使用OpenCV：

override fun analyze(imageProxy: ImageProxy, rotationDegrees: Int) {
    try {
      imageProxy.image?.let {
        // ImageProxy uses an ImageReader under the hood:
        // https://developer.android.com/reference/androidx/camera/core/ImageProxy.html
        // That has a default format of YUV_420_888 if not changed that's the default
        // Android camera format.
        // https://developer.android.com/reference/android/graphics/ImageFormat.html#YUV_420_888
        // https://developer.android.com/reference/android/media/ImageReader.html

        // Sanity check
        if (it.format == ImageFormat.YUV_420_888
            && it.planes.size == 3
        ) {
           // TODO - convert ImageProxy.image to Mat
        } else {
          // Manage other image formats
          // TODO - https://developer.android.com/reference/android/media/Image.html
        }
      }
    } catch (ise: IllegalStateException) {
      ise.printStackTrace()
    }
  }

我该怎么做？

Answer 1

 private Mat convertYUVtoMat(@NonNull Image img) {
    byte[] nv21;

    ByteBuffer yBuffer = img.getPlanes()[0].getBuffer();
    ByteBuffer uBuffer = img.getPlanes()[1].getBuffer();
    ByteBuffer vBuffer = img.getPlanes()[2].getBuffer();

    int ySize = yBuffer.remaining();
    int uSize = uBuffer.remaining();
    int vSize = vBuffer.remaining();

    nv21 = new byte[ySize + uSize + vSize];

    yBuffer.get(nv21, 0, ySize);
    vBuffer.get(nv21, ySize, vSize);
    uBuffer.get(nv21, ySize + vSize, uSize);

    Mat yuv = new Mat(img.getHeight() + img.getHeight()/2, img.getWidth(), CvType.CV_8UC1);
    yuv.put(0, 0, nv21);
    Mat rgb = new Mat();
    Imgproc.cvtColor(yuv, rgb, Imgproc.COLOR_YUV2RGB_NV21, 3);
    Core.rotate(rgb, rgb, Core.ROTATE_90_CLOCKWISE);
    return  rgb;
}

此方法将 Camerax API YUV_420_888 图像转换为 OpenCV 的 Mat (RGB) 对象。（2021 年工作）

Answer 2

@shadowsheep解决方案很好，如果您需要获得OpenCV Mat。

但是如果您想获取位图并且不想在您的项目中添加opencv库，则可以在android/camera-samples repo中查看RenderScript解决方案

我还在github制作了一个Java文件库。如果您希望获得正确的ByteBuffer而没有任何行或像素步幅进行进一步处理（例如，使用神经网络引擎），将非常有用。

我也compared所有这些方法。 OpenCV是最快的。

Answer 3

在其GitHub存储库中查看OpenCV JavaCamera2Frame类，您可以编写一个Image扩展函数，如下所示：

（移植到Kotlin）

// Ported from opencv private class JavaCamera2Frame
fun Image.yuvToRgba(): Mat {
  val rgbaMat = Mat()

  if (format == ImageFormat.YUV_420_888
      && planes.size == 3) {

    val chromaPixelStride = planes[1].pixelStride

    if (chromaPixelStride == 2) { // Chroma channels are interleaved
      assert(planes[0].pixelStride == 1)
      assert(planes[2].pixelStride == 2)
      val yPlane = planes[0].buffer
      val uvPlane1 = planes[1].buffer
      val uvPlane2 = planes[2].buffer
      val yMat = Mat(height, width, CvType.CV_8UC1, yPlane)
      val uvMat1 = Mat(height / 2, width / 2, CvType.CV_8UC2, uvPlane1)
      val uvMat2 = Mat(height / 2, width / 2, CvType.CV_8UC2, uvPlane2)
      val addrDiff = uvMat2.dataAddr() - uvMat1.dataAddr()
      if (addrDiff > 0) {
        assert(addrDiff == 1L)
        Imgproc.cvtColorTwoPlane(yMat, uvMat1, rgbaMat, Imgproc.COLOR_YUV2RGBA_NV12)
      } else {
        assert(addrDiff == -1L)
        Imgproc.cvtColorTwoPlane(yMat, uvMat2, rgbaMat, Imgproc.COLOR_YUV2RGBA_NV21)
      }
    } else { // Chroma channels are not interleaved
      val yuvBytes = ByteArray(width * (height + height / 2))
      val yPlane = planes[0].buffer
      val uPlane = planes[1].buffer
      val vPlane = planes[2].buffer

      yPlane.get(yuvBytes, 0, width * height)

      val chromaRowStride = planes[1].rowStride
      val chromaRowPadding = chromaRowStride - width / 2

      var offset = width * height
      if (chromaRowPadding == 0) {
        // When the row stride of the chroma channels equals their width, we can copy
        // the entire channels in one go
        uPlane.get(yuvBytes, offset, width * height / 4)
        offset += width * height / 4
        vPlane.get(yuvBytes, offset, width * height / 4)
      } else {
        // When not equal, we need to copy the channels row by row
        for (i in 0 until height / 2) {
          uPlane.get(yuvBytes, offset, width / 2)
          offset += width / 2
          if (i < height / 2 - 1) {
            uPlane.position(uPlane.position() + chromaRowPadding)
          }
        }
        for (i in 0 until height / 2) {
          vPlane.get(yuvBytes, offset, width / 2)
          offset += width / 2
          if (i < height / 2 - 1) {
            vPlane.position(vPlane.position() + chromaRowPadding)
          }
        }
      }

      val yuvMat = Mat(height + height / 2, width, CvType.CV_8UC1)
      yuvMat.put(0, 0, yuvBytes)
      Imgproc.cvtColor(yuvMat, rgbaMat, Imgproc.COLOR_YUV2RGBA_I420, 4)
    }
  }

  return rgbaMat
}

所以您可以这样写：

override fun analyze(imageProxy: ImageProxy, rotationDegrees: Int) {
    try {
      imageProxy.image?.let {
        // ImageProxy uses an ImageReader under the hood:
        // https://developer.android.com/reference/androidx/camera/core/ImageProxy.html
        // That has a default format of YUV_420_888 if not changed that's the default
        // Android camera format.
        // https://developer.android.com/reference/android/graphics/ImageFormat.html#YUV_420_888
        // https://developer.android.com/reference/android/media/ImageReader.html

        // Sanity check
        if (it.format == ImageFormat.YUV_420_888
            && it.planes.size == 3
        ) {        
          val rgbaMat = it.yuvToRgba()
        } else {
          // Manage other image formats
          // TODO - https://developer.android.com/reference/android/media/Image.html
        }
      }
    } catch (ise: IllegalStateException) {
      ise.printStackTrace()
    }
  }

格式为YUV_420_888的Android CameraX Analyzer映像到OpenCV Mat

3 个答案: