我已经尝试了一切我能想到的，以改善从以太网摄像头解码h.264流的延迟。相机的制造商声称它使用硬件显示流的最小延迟为50ms，所以我知道它可能。我也能够在我的计算机上播放流，没有任何延迟。
我在Android中开发，我通过DatagramSocket接收UDP数据包，解析RTP数据包，组装NAL单元，将它们传递给MediaCodec，通过硬件解码器解码h.264流，最后显示流在SurfaceView上。
除了在录制的内容和显示的内容之间有大约610毫秒的延迟之外，流没有任何问题地播放完美。这款相机将用于车辆，因此610毫秒的延迟是不可接受的。
非常感谢任何关于如何改善这种延迟的建议。
以下是我从各种公共资源中改编的代码：
// configuration constants
private static final int SURFACE_WIDTH = 640;
private static final int SURFACE_HEIGHT = 480;
public static final String CSD_0 = "csd-0";
public static final String CSD_1 = "csd-1";
public static final String DURATION_US = "durationUs";
public static boolean DEBUGGING = false;
private final SurfaceView surfaceView;
private PlayerThread playerThread;
private RTPClientThread rtpSessionThread;
private ByteBuffer inputBuffer;
private MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
private MediaCodec decoder;
private Log log = LogFactory.getLog(RtpMediaDecoder.class);
private final byte[] byteStreamStartCodePrefix = {(byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x01};
private boolean useByteStreamFormat = true;
private int lastSequenceNumber = 0;
private boolean lastSequenceNumberIsValid = false;
private boolean sequenceError = false;
private boolean currentFrameHasError = false;
private BufferedSample currentFrame;
private ExecutorService executorService;
private enum NalType {
FULL,
STAPA,
STAPB,
MTAP16,
MTAP24,
FUA,
FUB,
UNKNOWN
}
public RtpMediaDecoder(SurfaceView surfaceView) {
this.surfaceView = surfaceView;
surfaceView.getHolder().addCallback(this);
}
public void start() {
rtpStartClient();
}
public void restart() {
rtpStopClient();
try {
sleep(500);
} catch (InterruptedException e) {
}
rtpStartClient();
}
public void release() {
rtpStopClient();
if (decoder != null) {
try {
decoder.stop();
} catch (Exception e) {
log.error("Encountered error while trying to stop decoder", e);
}
decoder.release();
decoder = null;
}
}
private void rtpStartClient() {
rtpSessionThread = new RTPClientThread();
executorService = Executors.newFixedThreadPool(1);
rtpSessionThread.start();
}
private void rtpStopClient() {
rtpSessionThread.interrupt();
executorService.shutdown();
}
public BufferedSample getSampleBuffer() throws Exception {
int inIndex = decoder.dequeueInputBuffer(-1);
if (inIndex < 0) {
throw new Exception("Didn't get a buffer from the MediaCodec");
}
inputBuffer = decoder.getInputBuffer(inIndex);
return new BufferedSample(inputBuffer, inIndex);
}
public void decodeFrame(BufferedSample decodeBuffer) throws Exception {
if (DEBUGGING) {
log.info(decodeBuffer.toString());
}
decoder.queueInputBuffer(decodeBuffer.getIndex(), 0,
decodeBuffer.getSampleSize(), 0, 0);
int outIndex = decoder.dequeueOutputBuffer(info, 0);
if (outIndex >= 0) {
// outputBuffer = decoder.getOutputBuffer(outIndex);
decoder.releaseOutputBuffer(outIndex,true);
}
// log.error("Completed frame decode: " + decodeBuffer.getRtpTimestamp() + " System Time: " + System.currentTimeMillis());
}
@Override
public void surfaceCreated(SurfaceHolder holder) {
android.view.ViewGroup.LayoutParams layoutParams = surfaceView.getLayoutParams();
layoutParams.width = SURFACE_WIDTH; // required width
layoutParams.height = SURFACE_HEIGHT; // required height
surfaceView.setLayoutParams(layoutParams);
}
@Override
public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
log.debug("Starting player thread.");
if (playerThread == null) {
playerThread = new PlayerThread(holder.getSurface());
playerThread.start();
}
}
@Override
public void surfaceDestroyed(SurfaceHolder holder) {
}
public MediaFormat getMediaFormat() {
String mimeType = "video/avc";
int width = 640;
int height = 480;
MediaFormat format = MediaFormat.createVideoFormat(mimeType, width, height);
// from avconv, when streaming sample.h264.mp4 from disk
// byte[] header_sps = {0, 0, 0, 1, // header
// 0x67, 0x64, (byte) 0x00, 0x1e, (byte) 0xac, (byte) 0xd9, 0x40, (byte) 0xa0, 0x3d,
// (byte) 0xa1, 0x00, 0x00, (byte) 0x03, 0x00, 0x01, 0x00, 0x00, 0x03, 0x00, 0x3C, 0x0F, 0x16, 0x2D, (byte) 0x96}; // sps
// byte[] header_pps = {0, 0, 0, 1, // header
// 0x68, (byte) 0xeb, (byte) 0xec, (byte) 0xb2, 0x2C}; // pps
byte[] header_sps = {0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0x00, 0x0a, (byte) 0xf8, 0x41, (byte) 0xa2};
byte[] header_pps = {0x00, 0x00, 0x00, 0x01, 0x68, (byte) 0xce, 0x38, (byte) 0x80};
format.setByteBuffer(CSD_0, ByteBuffer.wrap(header_sps));
format.setByteBuffer(CSD_1, ByteBuffer.wrap(header_pps));
//format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, width * height);
format.setInteger(DURATION_US, 12600000);
return format;
}
private class PlayerThread extends Thread {
private Surface surface;
public PlayerThread(Surface surface) {
this.surface = surface;
}
@Override
public void run() {
// MediaFormat mediaFormat = MediaFormat.createVideoFormat("video/avc", SURFACE_WIDTH, SURFACE_HEIGHT);
MediaFormat mediaFormat = getMediaFormat();
try {
String mime = mediaFormat.getString(MediaFormat.KEY_MIME);
if (mime.startsWith("video/")) {
decoder = MediaCodec.createDecoderByType(mime);
}
// decoder = MediaCodec.createByCodecName("OMX.Intel.hw_vd.h264");
// decoder = MediaCodec.createDecoderByType("video/avc");
} catch (IOException e) {
e.printStackTrace();
}
if (decoder == null) {
log.info("Can't find video info!");
return;
}
decoder.configure(mediaFormat, surface, null, 0);
// log.error("Decoder Started, System Time: " + System.currentTimeMillis());
decoder.start();
}
}
private class RTPClientThread extends Thread {
private DatagramSocket mDataGramSocket;
@Override
public void run() {
try {
sleep(200);
} catch (InterruptedException e) {
}
try {
mDataGramSocket = new DatagramSocket(50004);
mDataGramSocket.setReuseAddress(true);
mDataGramSocket.setSoTimeout(1000);
} catch (Exception e) {
e.printStackTrace();
}
byte[] recvPacket = {0};
int seqNum = 0, prevSeqNum = 0, length = 0;
byte[] message = new byte[1450];
DatagramPacket p = new DatagramPacket(message, message.length);
try {
while (!Thread.interrupted()) {
try {
mDataGramSocket.receive(p);
length = p.getLength();
recvPacket = new byte[length];
System.arraycopy(message,0,recvPacket,0,length);
seqNum = ((message[2] & 0xff) << 8) | (message[3] & 0xff);
if(seqNum != prevSeqNum) {
prevSeqNum = seqNum;
if (!executorService.isTerminated() && !executorService.isShutdown()) {
executorService.execute(new PacketRunnable(recvPacket, seqNum));
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
log.error("We Stopped");
mDataGramSocket.close();
} catch (Exception e) {
e.printStackTrace();
}
}
public class PacketRunnable implements Runnable {
private byte[] data;
private int localSeqNum;
private PacketRunnable(byte[] _data, int _seqNum) {
this.data = _data;
this.localSeqNum = _seqNum;
}
public void run() {
DataPacket packet = DataPacket.decode(data);
String debugging = "RTP data. ";
debugging += packet.getDataSize() + "b ";
debugging += "#" + packet.getSequenceNumber();
debugging += " " + packet.getTimestamp();
if (lastSequenceNumberIsValid && ((lastSequenceNumber + 1) != localSeqNum) && (localSeqNum != 0)) {
sequenceError = true;
log.error("Seq#: "+ localSeqNum + " PrevSeq#: " + lastSequenceNumber + " SKIPPED (" + (localSeqNum - lastSequenceNumber - 1) + ")");
debugging += " SKIPPED (" + (localSeqNum - lastSequenceNumber - 1) + ")";
} else {
sequenceError = false;
}
if (RtpMediaDecoder.DEBUGGING) {
log.error(debugging);
}
H264Packet h264Packet = new H264Packet(packet);
if (h264Packet.getNRIBits() > 0) {
switch (h264Packet.h264NalType) {
case FULL:
if (RtpMediaDecoder.DEBUGGING) {
log.info("NAL: full packet");
}
startFrame(packet.getTimestamp());
if (currentFrame != null) {
if (useByteStreamFormat) {
currentFrame.getBuffer().put(byteStreamStartCodePrefix);
}
currentFrame.getBuffer().put(packet.getData().toByteBuffer());
sendFrame();
}
break;
case FUA:
if (h264Packet.isStart()) {
if (RtpMediaDecoder.DEBUGGING) {
log.info("FU-A start found. Starting new frame");
}
startFrame(packet.getTimestamp());
if (currentFrame != null) {
// Add stream header
if (useByteStreamFormat) {
currentFrame.getBuffer().put(byteStreamStartCodePrefix);
}
byte reconstructedNalTypeOctet = h264Packet.getNalTypeOctet();
currentFrame.getBuffer().put(reconstructedNalTypeOctet);
}
}
if (currentFrame != null) {
if (packet.getTimestamp() != currentFrame.getRtpTimestamp()) {
if (RtpMediaDecoder.DEBUGGING) {
log.warn("Non-consecutive timestamp found");
}
currentFrameHasError = true;
}
if (sequenceError) {
currentFrameHasError = true;
}
// If we survived possible errors, collect data to the current frame buffer
if (!currentFrameHasError) {
currentFrame.getBuffer().put(packet.getData().toByteBuffer(2, packet.getDataSize() - 2));
} else {
if (RtpMediaDecoder.DEBUGGING) {
log.info("Dropping frame");
}
}
if (h264Packet.isEnd()) {
if (RtpMediaDecoder.DEBUGGING) {
log.info("FU-A end found. Sending frame!");
}
try {
sendFrame();
} catch (Throwable t) {
log.error("Error sending frame.", t);
}
}
}
break;
case STAPA:
if (RtpMediaDecoder.DEBUGGING) {
log.info("NAL: STAP-A");
}
ChannelBuffer buffer = packet.getData();
buffer.readByte();
while (buffer.readable()) {
short nalUnitSize = buffer.readShort();
byte[] nalUnitData = new byte[nalUnitSize];
buffer.readBytes(nalUnitData);
startFrame(packet.getTimestamp());
if (currentFrame != null) {
if (useByteStreamFormat) {
currentFrame.getBuffer().put(byteStreamStartCodePrefix);
}
currentFrame.getBuffer().put(nalUnitData);
sendFrame();
}
}
break;
case STAPB:
case MTAP16:
case MTAP24:
case FUB:
case UNKNOWN:
log.warn("NAL: Unimplemented unit type: " + h264Packet.getNalType());
break;
}
} else {
log.warn("Useless packet received.");
}
lastSequenceNumber = localSeqNum;
lastSequenceNumberIsValid = true;
}
}
}
private void startFrame(long rtpTimestamp) {
// Reset error bit
currentFrameHasError = false;
// Deal with potentially non-returned buffer due to error
if (currentFrame != null) {
currentFrame.getBuffer().clear();
// Otherwise, get a fresh buffer from the codec
} else {
try {
// Get buffer from decoder
currentFrame = getSampleBuffer();
currentFrame.getBuffer().clear();
} catch (Exception e) {
currentFrameHasError = true;
e.printStackTrace();
}
}
if (!currentFrameHasError) {
// Set the sample timestamp
currentFrame.setRtpTimestamp(rtpTimestamp);
}
}
private void sendFrame() {
currentFrame.setSampleSize(currentFrame.getBuffer().position());
currentFrame.getBuffer().flip();
// log.error("Sending Frame: " + currentFrame.getRtpTimestamp() + " System Time: " + System.currentTimeMillis());
try {
decodeFrame(currentFrame);
} catch (Exception e) {
log.error("Exception sending frame to decoder", e);
}
// Always make currentFrame null to indicate we have returned the buffer to the codec
currentFrame = null;
}
private class H264Packet {
private final byte nalFBits;
private final byte nalNriBits;
private final byte nalType;
private boolean fuStart = false;
private boolean fuEnd = false;
private byte fuNalType;
private NalType h264NalType = NalType.UNKNOWN;
public H264Packet(DataPacket packet) {
// Parsing the RTP Packet - http://www.ietf.org/rfc/rfc3984.txt section 5.3
byte nalUnitOctet = packet.getData().getByte(0);
nalFBits = (byte) (nalUnitOctet & 0x80);
nalNriBits = (byte) (nalUnitOctet & 0x60);
nalType = (byte) (nalUnitOctet & 0x1F);
// If it's a single NAL packet then the entire payload is here
if (nalType > 0 && nalType < 24) {
h264NalType = NalType.FULL;
} else if (nalType == 24) {
h264NalType = NalType.STAPA;
} else if (nalType == 25) {
h264NalType = NalType.STAPB;
} else if (nalType == 26) {
h264NalType = NalType.MTAP16;
} else if (nalType == 27) {
h264NalType = NalType.MTAP24;
} else if (nalType == 28) {
h264NalType = NalType.FUA;
} else if (nalType == 29) {
h264NalType = NalType.FUB;
}
byte fuHeader = packet.getData().getByte(1);
fuStart = ((fuHeader & 0x80) != 0);
fuEnd = ((fuHeader & 0x40) != 0);
fuNalType = (byte) (fuHeader & 0x1F);
}
public byte getNalTypeOctet() {
// Excerpt from the spec:
/* "The NAL unit type octet of the fragmented
NAL unit is not included as such in the fragmentation unit payload,
but rather the information of the NAL unit type octet of the
fragmented NAL unit is conveyed in F and NRI fields of the FU
indicator octet of the fragmentation unit and in the type field of
the FU header" */
return (byte) (fuNalType | nalFBits | nalNriBits);
}
public boolean isStart() {
return fuStart;
}
public boolean isEnd() {
return fuEnd;
}
public byte getNalType() {
return nalType;
}
public byte getNRIBits() {
return nalNriBits;
}
}
}