我正在尝试使用MediaCodec API通过ffmpeg解码来自PC的实时流屏幕捕获。
对于发件人（PC ffmpeg）
我使用此命令
ffmpeg -re -f gdigrab -s 1920x1080 -threads 4 -i desktop -vcodec libx264 -pix_fmt yuv420p -tune zerolatency -profile:v baseline -flags global_header -s 1280x720 -an -f rtp rtp://192.168.1.6:1234
和输出看起来像这样
Output #0, rtp, to 'rtp://192.168.1.6:1234':
Metadata:
encoder : Lavf56.15.104
Stream #0:0: Video: h264 (libx264), yuv420p, 1280x720, q=-1--1, 29.97 fps, 90k tbn, 29.97 tbc
Metadata:
encoder : Lavc56.14.100 libx264
Stream mapping:
Stream #0:0 -> #0:0 (bmp (native) -> h264 (libx264))
SDP:
v=0
o=- 0 0 IN IP4 127.0.0.1
s=No Name
c=IN IP4 192.168.1.6
t=0 0
a=tool:libavformat 56.15.104
m=video 1234 RTP/AVP 96
a=rtpmap:96 H264/90000
a=fmtp:96 packetization-mode=1; sprop-parameter-sets=Z0LAH9kAUAW6EAAAPpAADqYI8YMkgA==,aMuDyyA=; profile-level-id=42C01F
Press [q] to stop, [?] for help
frame= 19 fps=0.0 q=17.0 size= 141kB time=00:00:00.63 bitrate=1826.0kbits/
frame= 34 fps= 32 q=17.0 size= 164kB time=00:00:01.13 bitrate=1181.5kbits/
frame= 50 fps= 32 q=18.0 size= 173kB time=00:00:01.66 bitrate= 850.9kbits/
对于Receiver（Android MediaCodec）
我用surface创建了活动并实现了SurfaceHolder.Callback
在surfaceChanged中
@Override
public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
Log.i("sss", "surfaceChanged");
if( playerThread == null ) {
playerThread = new PlayerThread(holder.getSurface());
playerThread.start();
}
}
对于PlayerThread
class PlayerThread extends Thread {
MediaCodec decoder;
Surface surface;
public PlayerThread(Surface surface) {
this.surface = surface;
}
@Override
public void run() {
running = true;
try {
MediaFormat format = MediaFormat.createVideoFormat("video/avc", 1280, 720);
byte[] header = new byte[] {0,0,0,1};
byte[] sps = Base64.decode("Z0LAH9kAUAW6EAAAPpAADqYI8YMkgA==", Base64.DEFAULT);
byte[] pps = Base64.decode("aMuDyyA=", Base64.DEFAULT);
byte[] header_sps = new byte[sps.length + header.length];
System.arraycopy(header,0,header_sps,0,header.length);
System.arraycopy(sps,0,header_sps,header.length, sps.length);
byte[] header_pps = new byte[pps.length + header.length];
System.arraycopy(header,0, header_pps, 0, header.length);
System.arraycopy(pps, 0, header_pps, header.length, pps.length);
format.setByteBuffer("csd-0", ByteBuffer.wrap(header_sps));
format.setByteBuffer("csd-1", ByteBuffer.wrap(header_pps));
format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 1280 * 720);
// format.setInteger("durationUs", 63446722);
// format.setByteBuffer("csd-2", ByteBuffer.wrap((hexStringToByteArray("42C01E"))));
// format.setInteger(MediaFormat.KEY_COLOR_FORMAT ,MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar);
Log.i("sss", "Format = " + format);
try {
decoder = MediaCodec.createDecoderByType("video/avc");
decoder.configure(format, surface, null, 0);
decoder.start();
} catch (IOException ioEx) {
ioEx.printStackTrace();
}
DatagramSocket socket = new DatagramSocket(1234);
byte[] bytes = new byte[4096];
DatagramPacket packet = new DatagramPacket(bytes, bytes.length);
byte[] data;
ByteBuffer[] inputBuffers;
ByteBuffer[] outputBuffers;
ByteBuffer inputBuffer;
ByteBuffer outputBuffer;
MediaCodec.BufferInfo bufferInfo;
bufferInfo = new MediaCodec.BufferInfo();
int inputBufferIndex;
int outputBufferIndex;
byte[] outData;
inputBuffers = decoder.getInputBuffers();
outputBuffers = decoder.getOutputBuffers();
int minusCount = 0;
byte[] prevData = new byte[65535];
List<byte[]> playLoads = new ArrayList<>();
int playloadSize = 0;
while (true) {
try {
socket.receive(packet);
data = new byte[packet.getLength()];
System.arraycopy(packet.getData(), packet.getOffset(), data, 0, packet.getLength());
inputBufferIndex = decoder.dequeueInputBuffer(-1);
Log.i("sss", "inputBufferIndex = " + inputBufferIndex);
if (inputBufferIndex >= 0)
{
inputBuffer = inputBuffers[inputBufferIndex];
inputBuffer.clear();
inputBuffer.put(data);
decoder.queueInputBuffer(inputBufferIndex, 0, data.length, 0, 0);
// decoder.flush();
}
outputBufferIndex = decoder.dequeueOutputBuffer(bufferInfo, 10000);
Log.i("sss", "outputBufferIndex = " + outputBufferIndex);
while (outputBufferIndex >= 0)
{
outputBuffer = outputBuffers[outputBufferIndex];
outputBuffer.position(bufferInfo.offset);
outputBuffer.limit(bufferInfo.offset + bufferInfo.size);
outData = new byte[bufferInfo.size];
outputBuffer.get(outData);
decoder.releaseOutputBuffer(outputBufferIndex, false);
outputBufferIndex = decoder.dequeueOutputBuffer(bufferInfo, 0);
}
} catch (SocketTimeoutException e) {
Log.d("thread", "timeout");
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
我认为来自ffmpeg的流不是问题，因为我可以通过sdp文件从mxPlayer打开它。 如果我将此流传递到本地RTSP服务器（通过VLC），那么我使用MediaPlayer获取RTSP流，它可以工作但速度很慢。
在我查看数据包后，我意识到了
因此，我将前12个字节切掉并将数据包与相同的TimeStamp组合在一起。然后把它放在像这样的缓冲区
在收到数据包后的while（true）
Log.i("sss", "Received = " + data.length + " bytes");
Log.i("sss","prev " + prevData.length + " bytes = " + getBytesStr(prevData));
Log.i("sss","data " + data.length + " bytes = " + getBytesStr(data));
if(data[4] == prevData[4] && data[5] == prevData[5] && data[6] == prevData[6] && data[7] == prevData[7]){
byte[] playload = new byte[prevData.length -12];
System.arraycopy(prevData,12,playload, 0, prevData.length-12);
playLoads.add(playload);
playloadSize += playload.length;
Log.i("sss", "Same timeStamp playload " + playload.length + " bytes = " + getBytesStr(playload));
} else {
if(playLoads.size() > 0){
byte[] playload = new byte[prevData.length -12];
System.arraycopy(prevData,12,playload, 0, prevData.length-12);
playLoads.add(playload);
playloadSize += playload.length;
Log.i("sss", "last playload " + playload.length + " bytes = " + getBytesStr(playload));
inputBufferIndex = decoder.dequeueInputBuffer(-1);
if (inputBufferIndex >= 0){
inputBuffer = inputBuffers[inputBufferIndex];
inputBuffer.clear();
byte[] allPlayload = new byte[playloadSize];
int curLength = 0;
for(byte[] playLoad:playLoads){
System.arraycopy(playLoad,0,allPlayload, curLength, playLoad.length);
curLength += playLoad.length;
}
Log.i("sss", "diff timeStamp AlllayLoad " + allPlayload.length + "bytes = " + getBytesStr(allPlayload));
inputBuffer.put(allPlayload);
decoder.queueInputBuffer(inputBufferIndex, 0, data.length, 0, 0);
decoder.flush();
}
bufferInfo = new MediaCodec.BufferInfo();
outputBufferIndex = decoder.dequeueOutputBuffer(bufferInfo, 10000);
if(outputBufferIndex!= -1)
Log.i("sss", "outputBufferIndex = " + outputBufferIndex);
playLoads = new ArrayList<>();
prevData = new byte[65535];
playloadSize = 0;
}
}
prevData = data.clone();
outputBufferIndex仍返回-1
如果我将timeoutUS从10000更改为-1，它永远不会转到下一行
我已经搜索了一周，但仍然没有运气T_T
为什么dequeueOutputBuffer总是返回-1？
我的代码有什么问题？
你能否正确优化我的代码才能正常工作？
感谢您的帮助。
修改＃1
谢谢@mstorsjo指导我打包，我找到了有用的信息
然后我编辑了下面的代码
if((data[12] & 0x1f) == 28){
if((data[13] & 0x80) == 0x80){ //found start bit
inputBufferIndex = decoder.dequeueInputBuffer(-1);
if (inputBufferIndex >= 0){
inputBuffer = inputBuffers[inputBufferIndex];
inputBuffer.clear();
byte result = (byte)((bytes[12] & 0xe0) + (bytes[13] & 0x1f));
inputBuffer.put(new byte[] {0,0,1});
inputBuffer.put(result);
inputBuffer.put(data,14, data.length-14);
}
} else if((data[13] &0x40) == 0x40){ //found stop bit
inputBuffer.put(data, 14, data.length -14);
decoder.queueInputBuffer(inputBufferIndex, 0, data.length, 0, 0);
bufferInfo = new MediaCodec.BufferInfo();
outputBufferIndex = decoder.dequeueOutputBuffer(bufferInfo, 10000);
switch(outputBufferIndex)
{
case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
outputBuffers = decoder.getOutputBuffers();
Log.w("sss", "Output Buffers Changed");
break;
case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
Log.w("sss", "Output Format Changed");
MediaFormat newFormat = decoder.getOutputFormat();
Log.i("sss","New format : " + newFormat);
break;
case MediaCodec.INFO_TRY_AGAIN_LATER:
Log.w("sss", "Try Again Later");
break;
default:
outputBuffer = outputBuffers[outputBufferIndex];
outputBuffer.position(bufferInfo.offset);
outputBuffer.limit(bufferInfo.offset + bufferInfo.size);
decoder.releaseOutputBuffer(outputBufferIndex, true);
}
} else {
inputBuffer.put(data, 14, data.length -14);
}
}
现在我可以看到一些图片，但大部分屏幕都是灰色的
我接下来应该做什么？
谢谢。
答案 0 :(得分：1)
您不能只丢弃RTP标头并假装其余数据包是正常的H264帧 - 事实并非如此。有关将H264打包成RTP时使用的格式的说明，请参阅RFC 6184。您需要撤消此打包以将数据恢复为普通解码器可以处理的格式。您可以查看libav / ffmpeg中的libavformat/rtpdec_h264.c以获取有关如何执行此操作的示例。
答案 1 :(得分：0)
这可能会迟到，但我可以看到两个可能的问题。
1）你只看NAL 28型（FU-A）的NAL单位，但是ffmpeg正在发送类型为1,24和28的NAL单位.24种NAL单位可以忽略而没有风险，但是类型1 NAL单位不能忽略（他们有NRI> 0）。
2）rtp流不一定按照发送顺序到达。因此，可能以错误的顺序重建帧。要确保正确的顺序，您必须查看rtp标头中的时间戳。
我找到一个好的库，这是Anroid Streaming Client。您需要稍微修改它以在MediaFormat中使用正确的csd-0 / csd-1，并将输出缓冲区设置为任意曲面而不是SurfaceView中的一个。