Question

我正在尝试使用MediaCodec在onImageAvailable回调中记录ImageReader的原始帧，但是无法编写有效的代码。大多数示例都使用Camera 1 API或MediaRecorder。我的目的是捕获处理单个帧并从中创建mp4

原始YUV帧

        @Override
        public void onImageAvailable(ImageReader reader) {
            Image i = reader.acquireLatestImage();
            processImage(i);
            i.close();
            Log.d("hehe", "onImageAvailable");
        }
    };

MediaCodec

MediaCodec codec = MediaCodec.createByCodecName(name);
 MediaFormat mOutputFormat; // member variable
 codec.setCallback(new MediaCodec.Callback() {
   @Override
   void onInputBufferAvailable(MediaCodec mc, int inputBufferId) {
     ByteBuffer inputBuffer = codec.getInputBuffer(inputBufferId);
     // fill inputBuffer with valid data
     …
     codec.queueInputBuffer(inputBufferId, …);
   }

   @Override
   void onOutputBufferAvailable(MediaCodec mc, int outputBufferId, …) {
     ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId);
     MediaFormat bufferFormat = codec.getOutputFormat(outputBufferId); // option A
     // bufferFormat is equivalent to mOutputFormat
     // outputBuffer is ready to be processed or rendered.
     …
     codec.releaseOutputBuffer(outputBufferId, …);
   }

   @Override
   void onOutputFormatChanged(MediaCodec mc, MediaFormat format) {
     // Subsequent data will conform to new format.
     // Can ignore if using getOutputFormat(outputBufferId)
     mOutputFormat = format; // option B
   }

   @Override
   void onError(…) {
     …
   }
 });
 codec.configure(format, …);
 mOutputFormat = codec.getOutputFormat(); // option B
 codec.start();
 // wait for processing to complete
 codec.stop();
 codec.release();

我无法关联https://developer.android.com/reference/android/media/MediaCodec上给出的代码。请帮助

Answer 1

您为什么不尝试以下示例：https://github.com/googlesamples/android-Camera2Video

我认为它肯定会满足您的所有要求，如果您无法与上述示例中的代码相关联，您可以随时与我联系。

此示例使用Camera2 API以及您想要的从原始YUV帧的转换，可以使用它来完成。因此，如果您要遍历给定的样本并使用其代码在所需的应用中录制MP4视频，希望您不会有任何问题。

例如- a），您将必须实现CameraDevice.StateCallback才能接收有关摄像头设备状态变化的事件。覆盖其方法来设置CameraDevice实例，开始预览以及停止并释放相机。

b）开始预览时，将 MediaRecorder 设置为接受视频格式。

c）然后，在CameraDevice实例上使用createCaptureRequest（CameraDevice.TEMPLATE_RECORD）设置CaptureRequest.Builder。

d）然后，在CameraDevice实例上使用createCaptureSession（surfaces，new CameraCaptureSession.StateCallback（）{}）方法实现CameraCaptureSession.StateCallback，其中surface是一个列表，其中包括您的TextureView的表面视图和 MediaRecorder 实例的表面。

e）。在您的 MediaRecorder 实例上使用start（）和stop（）方法实际开始和停止记录。

f）最后，在onResume（）和onPause（）中设置并清理相机设备。

快乐的编码。

Answer 2

您必须创建一个Queue，将从 Image 平面创建的图像缓冲区推入队列，然后在void onInputBufferAvailable(MediaCodec mc, int inputBufferId)

中进行处理

1）创建一个类来包装缓冲区数据：

class MyData{
    byte[] buffer;
    long presentationTimeUs;
    // to tell your encoder that is a EOS, otherwise you can not know when to stop
    boolean isEOS; 
    public MyData(byte[] buffer,long presentationTimeUs, boolean isEOS){
        this.buffer = new byte[buffer.length];
        System.arraycopy(buffer, 0, this.buffer, 0, buffer.length);
        this.presentationTimeUs = presentationTimeUs;
        this.isEOS = isEOS;
    }

    public byte[] getBuffer() {
        return buffer;
    }

    public void setBuffer(byte[] buffer) {
        this.buffer = buffer;
    }

    public long getPresentationTimeUs() {
        return presentationTimeUs;
    }

    public void setPresentationTimeUs(long presentationTimeUs) {
        this.presentationTimeUs = presentationTimeUs;
    }

    public boolean isEOS() {
        return isEOS;
    }

    public void setEOS(boolean EOS) {
        isEOS = EOS;
    }

}

2）创建队列：

Queue<MyData> mQueue = new LinkedList<MyData>();

3）使用本机代码将图像平面转换为字节数组（byte []）：

向Gradle文件添加本机支持：

android {
compileSdkVersion 27
defaultConfig {
    ...

    externalNativeBuild {
        cmake {
            arguments "-DANDROID_STL=stlport_static"
            cppFlags "-std=c++11"
        }
    }
}

externalNativeBuild {
    cmake {
        path "CMakeLists.txt"
    }
}
...

}

创建一个将图像平面转换为字节数组的函数：（native-yuv-to-buffer.cpp）

外部“ C” JNIEXPORT jbyteArray JNICALL

Java_labs_farzi_camera2previewstream_MainActivity_yuvToBuffer （

JNIEnv *env,
jobject instance,
jobject yPlane,
jobject uPlane,
jobject vPlane,
jint yPixelStride,
jint yRowStride,
jint uPixelStride,
jint uRowStride,
jint vPixelStride,
jint vRowStride,
jint imgWidth,
jint imgHeight) {

    bbuf_yIn = static_cast<uint8_t *>(env->GetDirectBufferAddress(yPlane));
    bbuf_uIn = static_cast<uint8_t *>(env->GetDirectBufferAddress(uPlane));
    bbuf_vIn = static_cast<uint8_t *>(env->GetDirectBufferAddress(vPlane));

    buf = (uint8_t *) malloc(sizeof(uint8_t) * imgWidth * imgHeight +
                             2 * (imgWidth + 1) / 2 * (imgHeight + 1) / 2);

    bool isNV21;
    if (yPixelStride == 1) {
        // All pixels in a row are contiguous; copy one line at a time.
        for (int y = 0; y < imgHeight; y++)
            memcpy(buf + y * imgWidth, bbuf_yIn + y * yRowStride,
                   static_cast<size_t>(imgWidth));
    } else {
        // Highly improbable, but not disallowed by the API. In this case
        // individual pixels aren't stored consecutively but sparsely with
        // other data inbetween each pixel.
        for (int y = 0; y < imgHeight; y++)
            for (int x = 0; x < imgWidth; x++)
                buf[y * imgWidth + x] = bbuf_yIn[y * yRowStride + x * yPixelStride];
    }

    uint8_t *chromaBuf = &buf[imgWidth * imgHeight];
    int chromaBufStride = 2 * ((imgWidth + 1) / 2);
    if (uPixelStride == 2 && vPixelStride == 2 &&
        uRowStride == vRowStride && bbuf_vIn == bbuf_uIn + 1) {
        isNV21 = true;
        // The actual cb/cr planes happened to be laid out in
        // exact NV21 form in memory; copy them as is
        for (int y = 0; y < (imgHeight + 1) / 2; y++)
            memcpy(chromaBuf + y * chromaBufStride, bbuf_vIn + y * vRowStride,
                   static_cast<size_t>(chromaBufStride));
    } else if (vPixelStride == 2 && uPixelStride == 2 &&
               uRowStride == vRowStride && bbuf_vIn == bbuf_uIn + 1) {
        isNV21 = false;
        // The cb/cr planes happened to be laid out in exact NV12 form
        // in memory; if the destination API can use NV12 in addition to
        // NV21 do something similar as above, but using cbPtr instead of crPtr.
        // If not, remove this clause and use the generic code below.
    } else {
        isNV21 = true;
        if (vPixelStride == 1 && uPixelStride == 1) {
            // Continuous cb/cr planes; the input data was I420/YV12 or similar;
            // copy it into NV21 form
            for (int y = 0; y < (imgHeight + 1) / 2; y++) {
                for (int x = 0; x < (imgWidth + 1) / 2; x++) {
                    chromaBuf[y * chromaBufStride + 2 * x + 0] = bbuf_vIn[y * vRowStride + x];
                    chromaBuf[y * chromaBufStride + 2 * x + 1] = bbuf_uIn[y * uRowStride + x];
                }
            }
        } else {
            // Generic data copying into NV21
            for (int y = 0; y < (imgHeight + 1) / 2; y++) {
                for (int x = 0; x < (imgWidth + 1) / 2; x++) {
                    chromaBuf[y * chromaBufStride + 2 * x + 0] = bbuf_vIn[y * vRowStride +
                                                                          x * uPixelStride];
                    chromaBuf[y * chromaBufStride + 2 * x + 1] = bbuf_uIn[y * uRowStride +
                                                                          x * vPixelStride];
                }
            }
        }
    }

    uint8_t *I420Buff = (uint8_t *) malloc(sizeof(uint8_t) * imgWidth * imgHeight +
                                           2 * (imgWidth + 1) / 2 * (imgHeight + 1) / 2);
    SPtoI420(buf,I420Buff,imgWidth,imgHeight,isNV21);

    jbyteArray ret = env->NewByteArray(imgWidth * imgHeight *
                                       3/2);
    env->SetByteArrayRegion (ret, 0, imgWidth * imgHeight *
                                     3/2, (jbyte*)I420Buff);
    free(buf);
    free (I420Buff);
    return ret;
}

添加了将半平面转换为平面的功能：

布尔SPtoI420（const uint8_t * src，uint8_t * dst，int width，int height，bool isNV21） { 如果（！src ||！dst）{ 返回false； }

unsigned int YSize = width * height;
unsigned int UVSize = (YSize>>1);

// NV21: Y..Y + VUV...U
const uint8_t *pSrcY = src;
const uint8_t *pSrcUV = src + YSize;

// I420: Y..Y + U.U + V.V
uint8_t *pDstY = dst;
uint8_t *pDstU = dst + YSize;
uint8_t *pDstV = dst + YSize + (UVSize>>1);

// copy Y
memcpy(pDstY, pSrcY, YSize);

// copy U and V
for (int k=0; k < (UVSize>>1); k++) {
    if(isNV21) {
        pDstV[k] = pSrcUV[k * 2];     // copy V
        pDstU[k] = pSrcUV[k * 2 + 1];   // copy U
    }else{
        pDstU[k] = pSrcUV[k * 2];     // copy V
        pDstV[k] = pSrcUV[k * 2 + 1];   // copy U
    }
}

return true;}

4）将缓冲区推送到队列：

private final ImageReader.OnImageAvailableListener mOnGetPreviewListener
    = new ImageReader.OnImageAvailableListener() {

@Override
public void onImageAvailable(ImageReader reader) {
    Image image = reader.acquireLatestImage();
    if (image == null)
        return;
    final Image.Plane[] planes = image.getPlanes();
    Image.Plane yPlane = planes[0];
    Image.Plane uPlane = planes[1];
    Image.Plane vPlane = planes[2];
    byte[] mBuffer = yuvToBuffer(yPlane.getBuffer(),
            uPlane.getBuffer(),
            vPlane.getBuffer(),
            yPlane.getPixelStride(),
            yPlane.getRowStride(),
            uPlane.getPixelStride(),
            uPlane.getRowStride(),
            vPlane.getPixelStride(),
            vPlane.getRowStride(),
            image.getWidth(),
            image.getHeight());
    mQueue.add(new MyData(mBuffer, image.getTimestamp(), false));
    image.close();
    Log.d("hehe", "onImageAvailable");
}

};

5）对数据进行编码并保存一个h264视频文件（可通过VLC播放）：

        public void onInputBufferAvailable(MediaCodec mc, int inputBufferId) {
        ByteBuffer inputBuffer = mc.getInputBuffer(inputBufferId);
        Log.d(TAG, "onInputBufferAvailable: ");
        // fill inputBuffer with valid data
        MyData data = mQueue.poll();
        if (data != null) {
            // check if is EOS and process with EOS flag if is the case
            // else if NOT EOS
            if (inputBuffer != null) {
                Log.e(TAG, "onInputBufferAvailable: "+data.getBuffer().length);
                inputBuffer.clear();
                inputBuffer.put(data.getBuffer());

                mc.queueInputBuffer(inputBufferId,
                        0,
                        data.getBuffer().length,
                        data.getPresentationTimeUs(),
                        0);
            }

        } else {

            mc.queueInputBuffer(inputBufferId,
                    0,
                    0,
                    0,
                    0);
        }
    }

    @Override
    public void onOutputBufferAvailable(@NonNull MediaCodec codec, int index, @NonNull MediaCodec.BufferInfo info) {
        Log.d(TAG, "onOutputBufferAvailable: ");
        ByteBuffer outputBuffer = codec.getOutputBuffer(index);
        byte[] outData = new byte[info.size];
        if (outputBuffer != null) {
            outputBuffer.get(outData);
            try {
                fos.write(outData);
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        codec.releaseOutputBuffer(index,false);
    }

6）在void onOutputBufferAvailable(MediaCodec mc, int outputBufferId, …)中修改您的曲目，其处理类似于您可以在Internet上找到的具有同步模式的示例。

希望我的回答对您有帮助

Full example code here

使用MediaCodec和Camera2 API录制视频

2 个答案: