长篇故事：
不久：
如何处理这些原始UDP数据以便通过H264 / MPEG-4解码器过滤器进行解码？ 任何人都可以清楚地确定我必须对H264 / MPEG流做的步骤吗？
额外信息：
我能用FFmpeg做到这一点......但我无法弄清楚FFmpeg如何处理原始数据，以便解码器可以解码。
答案 0 :(得分：106)
答案 1 :(得分：3)
我有一个@ https://net7mma.codeplex.com/
的实现以下是相关代码
/// <summary>
/// Implements Packetization and Depacketization of packets defined in <see href="https://tools.ietf.org/html/rfc6184">RFC6184</see>.
/// </summary>
public class RFC6184Frame : Rtp.RtpFrame
{
/// <summary>
/// Emulation Prevention
/// </summary>
static byte[] NalStart = { 0x00, 0x00, 0x01 };
public RFC6184Frame(byte payloadType) : base(payloadType) { }
public RFC6184Frame(Rtp.RtpFrame existing) : base(existing) { }
public RFC6184Frame(RFC6184Frame f) : this((Rtp.RtpFrame)f) { Buffer = f.Buffer; }
public System.IO.MemoryStream Buffer { get; set; }
/// <summary>
/// Creates any <see cref="Rtp.RtpPacket"/>'s required for the given nal
/// </summary>
/// <param name="nal">The nal</param>
/// <param name="mtu">The mtu</param>
public virtual void Packetize(byte[] nal, int mtu = 1500)
{
if (nal == null) return;
int nalLength = nal.Length;
int offset = 0;
if (nalLength >= mtu)
{
//Make a Fragment Indicator with start bit
byte[] FUI = new byte[] { (byte)(1 << 7), 0x00 };
bool marker = false;
while (offset < nalLength)
{
//Set the end bit if no more data remains
if (offset + mtu > nalLength)
{
FUI[0] |= (byte)(1 << 6);
marker = true;
}
else if (offset > 0) //For packets other than the start
{
//No Start, No End
FUI[0] = 0;
}
//Add the packet
Add(new Rtp.RtpPacket(2, false, false, marker, PayloadTypeByte, 0, SynchronizationSourceIdentifier, HighestSequenceNumber + 1, 0, FUI.Concat(nal.Skip(offset).Take(mtu)).ToArray()));
//Move the offset
offset += mtu;
}
} //Should check for first byte to be 1 - 23?
else Add(new Rtp.RtpPacket(2, false, false, true, PayloadTypeByte, 0, SynchronizationSourceIdentifier, HighestSequenceNumber + 1, 0, nal));
}
/// <summary>
/// Creates <see cref="Buffer"/> with a H.264 RBSP from the contained packets
/// </summary>
public virtual void Depacketize() { bool sps, pps, sei, slice, idr; Depacketize(out sps, out pps, out sei, out slice, out idr); }
/// <summary>
/// Parses all contained packets and writes any contained Nal Units in the RBSP to <see cref="Buffer"/>.
/// </summary>
/// <param name="containsSps">Indicates if a Sequence Parameter Set was found</param>
/// <param name="containsPps">Indicates if a Picture Parameter Set was found</param>
/// <param name="containsSei">Indicates if Supplementatal Encoder Information was found</param>
/// <param name="containsSlice">Indicates if a Slice was found</param>
/// <param name="isIdr">Indicates if a IDR Slice was found</param>
public virtual void Depacketize(out bool containsSps, out bool containsPps, out bool containsSei, out bool containsSlice, out bool isIdr)
{
containsSps = containsPps = containsSei = containsSlice = isIdr = false;
DisposeBuffer();
this.Buffer = new MemoryStream();
//Get all packets in the frame
foreach (Rtp.RtpPacket packet in m_Packets.Values.Distinct())
ProcessPacket(packet, out containsSps, out containsPps, out containsSei, out containsSlice, out isIdr);
//Order by DON?
this.Buffer.Position = 0;
}
/// <summary>
/// Depacketizes a single packet.
/// </summary>
/// <param name="packet"></param>
/// <param name="containsSps"></param>
/// <param name="containsPps"></param>
/// <param name="containsSei"></param>
/// <param name="containsSlice"></param>
/// <param name="isIdr"></param>
internal protected virtual void ProcessPacket(Rtp.RtpPacket packet, out bool containsSps, out bool containsPps, out bool containsSei, out bool containsSlice, out bool isIdr)
{
containsSps = containsPps = containsSei = containsSlice = isIdr = false;
//Starting at offset 0
int offset = 0;
//Obtain the data of the packet (without source list or padding)
byte[] packetData = packet.Coefficients.ToArray();
//Cache the length
int count = packetData.Length;
//Must have at least 2 bytes
if (count <= 2) return;
//Determine if the forbidden bit is set and the type of nal from the first byte
byte firstByte = packetData[offset];
//bool forbiddenZeroBit = ((firstByte & 0x80) >> 7) != 0;
byte nalUnitType = (byte)(firstByte & Common.Binary.FiveBitMaxValue);
//o The F bit MUST be cleared if all F bits of the aggregated NAL units are zero; otherwise, it MUST be set.
//if (forbiddenZeroBit && nalUnitType <= 23 && nalUnitType > 29) throw new InvalidOperationException("Forbidden Zero Bit is Set.");
//Determine what to do
switch (nalUnitType)
{
//Reserved - Ignore
case 0:
case 30:
case 31:
{
return;
}
case 24: //STAP - A
case 25: //STAP - B
case 26: //MTAP - 16
case 27: //MTAP - 24
{
//Move to Nal Data
++offset;
//Todo Determine if need to Order by DON first.
//EAT DON for ALL BUT STAP - A
if (nalUnitType != 24) offset += 2;
//Consume the rest of the data from the packet
while (offset < count)
{
//Determine the nal unit size which does not include the nal header
int tmp_nal_size = Common.Binary.Read16(packetData, offset, BitConverter.IsLittleEndian);
offset += 2;
//If the nal had data then write it
if (tmp_nal_size > 0)
{
//For DOND and TSOFFSET
switch (nalUnitType)
{
case 25:// MTAP - 16
{
//SKIP DOND and TSOFFSET
offset += 3;
goto default;
}
case 26:// MTAP - 24
{
//SKIP DOND and TSOFFSET
offset += 4;
goto default;
}
default:
{
//Read the nal header but don't move the offset
byte nalHeader = (byte)(packetData[offset] & Common.Binary.FiveBitMaxValue);
if (nalHeader > 5)
{
if (nalHeader == 6)
{
Buffer.WriteByte(0);
containsSei = true;
}
else if (nalHeader == 7)
{
Buffer.WriteByte(0);
containsPps = true;
}
else if (nalHeader == 8)
{
Buffer.WriteByte(0);
containsSps = true;
}
}
if (nalHeader == 1) containsSlice = true;
if (nalHeader == 5) isIdr = true;
//Done reading
break;
}
}
//Write the start code
Buffer.Write(NalStart, 0, 3);
//Write the nal header and data
Buffer.Write(packetData, offset, tmp_nal_size);
//Move the offset past the nal
offset += tmp_nal_size;
}
}
return;
}
case 28: //FU - A
case 29: //FU - B
{
/*
Informative note: When an FU-A occurs in interleaved mode, it
always follows an FU-B, which sets its DON.
* Informative note: If a transmitter wants to encapsulate a single
NAL unit per packet and transmit packets out of their decoding
order, STAP-B packet type can be used.
*/
//Need 2 bytes
if (count > 2)
{
//Read the Header
byte FUHeader = packetData[++offset];
bool Start = ((FUHeader & 0x80) >> 7) > 0;
//bool End = ((FUHeader & 0x40) >> 6) > 0;
//bool Receiver = (FUHeader & 0x20) != 0;
//if (Receiver) throw new InvalidOperationException("Receiver Bit Set");
//Move to data
++offset;
//Todo Determine if need to Order by DON first.
//DON Present in FU - B
if (nalUnitType == 29) offset += 2;
//Determine the fragment size
int fragment_size = count - offset;
//If the size was valid
if (fragment_size > 0)
{
//If the start bit was set
if (Start)
{
//Reconstruct the nal header
//Use the first 3 bits of the first byte and last 5 bites of the FU Header
byte nalHeader = (byte)((firstByte & 0xE0) | (FUHeader & Common.Binary.FiveBitMaxValue));
//Could have been SPS / PPS / SEI
if (nalHeader > 5)
{
if (nalHeader == 6)
{
Buffer.WriteByte(0);
containsSei = true;
}
else if (nalHeader == 7)
{
Buffer.WriteByte(0);
containsPps = true;
}
else if (nalHeader == 8)
{
Buffer.WriteByte(0);
containsSps = true;
}
}
if (nalHeader == 1) containsSlice = true;
if (nalHeader == 5) isIdr = true;
//Write the start code
Buffer.Write(NalStart, 0, 3);
//Write the re-construced header
Buffer.WriteByte(nalHeader);
}
//Write the data of the fragment.
Buffer.Write(packetData, offset, fragment_size);
}
}
return;
}
default:
{
// 6 SEI, 7 and 8 are SPS and PPS
if (nalUnitType > 5)
{
if (nalUnitType == 6)
{
Buffer.WriteByte(0);
containsSei = true;
}
else if (nalUnitType == 7)
{
Buffer.WriteByte(0);
containsPps = true;
}
else if (nalUnitType == 8)
{
Buffer.WriteByte(0);
containsSps = true;
}
}
if (nalUnitType == 1) containsSlice = true;
if (nalUnitType == 5) isIdr = true;
//Write the start code
Buffer.Write(NalStart, 0, 3);
//Write the nal heaer and data data
Buffer.Write(packetData, offset, count - offset);
return;
}
}
}
internal void DisposeBuffer()
{
if (Buffer != null)
{
Buffer.Dispose();
Buffer = null;
}
}
public override void Dispose()
{
if (Disposed) return;
base.Dispose();
DisposeBuffer();
}
//To go to an Image...
//Look for a SliceHeader in the Buffer
//Decode Macroblocks in Slice
//Convert Yuv to Rgb
}
还有各种其他RFC的实现，它们有助于将媒体播放到MediaElement或其他软件中，或者只是将其保存到磁盘。
正在写入容器格式。
答案 2 :(得分：1)
使用UDP数据包，您会收到H.264流的比特，您希望将其拆分为H.264 NAL Units，而您通常会从您的过滤器推送到DirectShow管道。
NAL单位将被格式化为DirectShow媒体样本，也可能作为媒体类型（SPS/PPS NAL单位）的一部分格式化。
解包步骤在RFC 6184 - RTP Payload Format for H.264 Video中描述。这是RTP流量的有效负载部分，由RFC 3550 - RTP: A Transport Protocol for Real-Time Applications定义。
虽然很清楚，但并不是很短暂。
答案 3 :(得分：0)
我最近流传输了h264，并遇到了类似的问题。这是我的拆包器类。我写了一篇很长的博客文章，以节省其他时间来了解此过程http://cagneymoreau.com/stream-video-android/
Package networking;
import org.apache.commons.logging.Log;
import utility.Debug;
import java.io.Console;
import java.io.IOException;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.util.*;
/**
* This class is used to re-assemble udp packets filled with rtp packets into network abstraction layer units
*
*/
public class VideoDecoder {
private static final String TAG = "VideoDecoder";
private PipedOutputStream pipedOutputStream; //this is where we pass the nalus we extract
private Map<Integer, NaluBuffer> assemblyLine = new HashMap<>(); // This holds nalus we are building. Ideally only 1 and if it exceeds 3 there might be a problem
private final int thresh = 30;
private int assemblyThresh = thresh;
private final int trashDelay = 3000;
//unpacking
private final static int HEADER_SIZE = 12;
private final static int rtpByteHeader1 = 128; //rtp header byte 1 should always equal
private final static int typeSPSPPS = 24;
private final static byte typeFUA = 0b01111100;
private final static byte[] startcode = new byte[] { 0x00, 0x00, 0x00, 0x01};
//experimental bools that can mix piped data
private boolean annexB = true; //remove lengths and dd aprefix
private boolean mixed = false; //keep lengths and add pefix dont use with annexb
private boolean prelStyle = false; //include avcc 6 byte data
private boolean directPipe = false; //send in the data with no editing
public VideoDecoder(PipedOutputStream pipedOutputStream)
{
this.pipedOutputStream = pipedOutputStream;
}
// raw udp rtp packets come in here from the the udp.packet.getdata filled at socket
public void addPacket(byte[] incoming)
{
if (directPipe){
transferTOFFmpeg(incoming);
return;
}
if (incoming[0] != (byte) rtpByteHeader1){
System.out.println(TAG + " rtpHeaderError " + Byte.toString(incoming[0]));
}
if (incoming[1] == typeSPSPPS){
System.out.println(TAG + "addPacket type: 24" );
unpackType24(incoming);
}
else if (incoming[1] == typeFUA){
//System.out.println(TAG + "addPacket type: 28" );
unpackType28(incoming);
}
else if (incoming[1] == 1){
System.out.println(TAG + "addPacket type: 1" );
unpackType1(incoming);
}else if (incoming[1] == 5){
System.out.println(TAG + "addPacket type: 5" );
unpackType5(incoming);
}else{
System.out.println(TAG + "addPacket unknown type - ERROR " + String.valueOf(incoming[1]) );
}
}
//SPS & PPS this will get hit before every type 5
//im not rtp compliant.
// length sps length pps prel = 6length
// LL SPSPSPSPSP LL PPSPPSPPSPPS 123456
private void unpackType24(byte[] twentyFour)
{
if (annexB){
int sp = (twentyFour[13] << 8 | twentyFour[14] & 0XFF);
int pp = (twentyFour[sp + 15] << 8 | twentyFour[sp + 16] & 0XFF);
byte[] sps = new byte[sp];
byte[] pps = new byte[pp];
System.arraycopy(twentyFour,15, sps,0,sp);
System.arraycopy(twentyFour,sp + 17, pps,0,pps.length);
transferTOFFmpeg(sps);
transferTOFFmpeg(pps);
}else if (prelStyle)
{
//Debug.debugHex("unpack24 " , twentyFour, twentyFour.length);
int spsl = (twentyFour[14] & 0xff) + 2;
int ppsl = (twentyFour[14+ spsl] & 0xff) +2;
int prel = 6;
byte[] buf = new byte[spsl + ppsl + prel]; //rtp header length - type + experimental data
System.arraycopy(twentyFour, 13, buf, 6,spsl + ppsl);
System.arraycopy(twentyFour, spsl + ppsl + 13, buf,0, 6);
transferTOFFmpeg(buf);
}else{
int spsl = (twentyFour[14] & 0xff) + 2;
int ppsl = (twentyFour[14+ spsl] & 0xff) +2;
byte[] buf = new byte[spsl + ppsl ]; //rtp header length - type + experimental data
System.arraycopy(twentyFour, 13, buf, 0,spsl + ppsl);
//System.arraycopy(twentyFour, spsl + ppsl + 13, buf,0, 6);
transferTOFFmpeg(buf);
}
}
//Single NON IDR Nal - This seems liekly to never occur
private void unpackType1(byte[] one)
{
byte[] buf = new byte[one.length-12];
System.arraycopy(one, 12, buf, 0,buf.length);
transferTOFFmpeg(buf);
}
//Single IDR Nal - This seems likely to never occur
private void unpackType5(byte[] five)
{
byte[] buf = new byte[five.length-12];
System.arraycopy(five, 12, buf, 0,buf.length);
transferTOFFmpeg(buf);
}
// Unpack either any split up nalu - This will get 99.999999 of nalus
synchronized private void unpackType28(byte[] twentyEight)
{
//Debug.deBugHexTrailing("unpack 28 ", twentyEight, 20 );
int ts = (twentyEight[4] << 24 | twentyEight[5] << 16 | twentyEight[6] << 8 | twentyEight[7] & 0XFF); //each nalu has a unique timestamp
//int seqN = (twentyEight[2] << 8 | twentyEight[3] & 0xFF); //each part of that nalu is numbered in order.
// numbers are from every packet ever. not this nalu. no zero or 1 start
//check if already building this nalu
if (assemblyLine.containsKey(ts)){
assemblyLine.get(ts).addPiece(twentyEight);
}
//add a new nalu
else
{
assemblyLine.put(ts, new NaluBuffer(ts, twentyEight));
}
}
//this will transfer the assembled nal units to the media codec/trans-coder/decoder/whatever?!?
private void transferTOFFmpeg(byte[] nalu)
{
Debug.debugHex("VideoDecoder transferTOFFmpg -> ", nalu, 30);
try{
if (annexB || mixed){
pipedOutputStream.write(startcode);
}
pipedOutputStream.write(nalu,0,nalu.length);
}catch (IOException ioe){
System.out.println(TAG + " transferTOFFmpeg - unable to lay pipe ;)");
}
if (assemblyLine.size() > assemblyThresh){
System.err.println(TAG + "transferToFFmpeg -> assemblyLine grows to a count of " + String.valueOf(assemblyLine.size()));
assemblyThresh += thresh;
}
}
private void clearList()
{
String n = "\n";
List<Integer> toremove = new ArrayList<>();
StringBuilder description = new StringBuilder();
for(Map.Entry<Integer, NaluBuffer> entry : assemblyLine.entrySet()) {
Integer key = entry.getKey();
NaluBuffer value = entry.getValue();
if (value.age < System.currentTimeMillis() - trashDelay){
toremove.add(key);
description
.append(String.valueOf(value.timeStamp)).append(" timestamp").append(n)
.append(String.valueOf(value.payloadType)).append(" type").append(n)
.append(String.valueOf(value.count)).append(" count").append(n)
.append(String.valueOf(value.start)).append(" ").append(String.valueOf(value.finish)).append(n)
.append(n);
}
}
for (Integer i :
toremove) {
assemblyLine.remove(i);
}
if (toremove.size() > 0){
System.out.println(TAG + " cleaList current size : " + String.valueOf(assemblyLine.size()) + n + "deleting: " + toremove.size() + n + description);
assemblyThresh = thresh;
}
}
private void deletMe(int key)
{
assemblyLine.remove(key);
if (assemblyLine.size() > 3){
clearList();
}
}
/*
Once a multipart FU-A rtp packet is found it is added to a hashset containing this class
Here we do everything needed to either complete assembly and send or destroy if not completed due to presumable packet loss
** Example Packet From First FU-A with SER = 100 **
description-> |-------RTP--HEADER------| |FU-A--HEADER| |-NAL--HEADER|
byte index-> 0|1|2|3|4|5|6|7|8|9|10|11| 12|13 14|15|16|17|18
| | | | | | | | |S S R C| | |__header | | | | |__type
| | | | |TIMESTM| |__indicator | | | |__length
| | | |__sequence number | | |__length
| | |____sequence number | |___length
| |__payload |__length
|___version padding extension
*/
private class NaluBuffer
{
private final static String TAG = "NaluBuffer";
//private static final int BUFF_SIZE = 200005; // this is the max nalu size + 5 byte header we searched for in our androids nalu search
long age;
//List<String> sizes = new ArrayList<>();
NaluePiece[] buffer = new NaluePiece[167];
int count = 0;
int start;
int finish;
int timeStamp; //from rtp packets.
int completedSize; //this is number of nalu
int payloadType; //nalu type 5 or 1
int byteLength;
int naluByteArrayLength = 0;
//if it doesnt exist
NaluBuffer(int timeStamp, byte[] piece)
{
//System.out.println(TAG + " constructor " + String.valueOf(timeStamp) );
this.timeStamp = timeStamp;
age = System.currentTimeMillis();
addPieceToBuffer(piece);
count++;
}
//adding another piece
synchronized public void addPiece(byte[] piece)
{
//System.out.println(TAG + " addPiece " + String.valueOf(timeStamp));
addPieceToBuffer(piece);
count++;
}
//add to buffer. incoming data is still raw rtp packet
private void addPieceToBuffer(byte[] piece)
{
//System.out.println(TAG + " addPiecetobuffer " + String.valueOf(piece[13]));
int seqN = (piece[2] << 8 | piece[3] & 0xFF);
//add to buffer
buffer[count] = new NaluePiece(seqN, Arrays.copyOfRange(piece, 14,piece.length)); // 14 because we skip rtp header of 12 and fu-a header of 2
int in = ( piece.length - 14); //we save each byte[] copied size so we can easily construct a completed array later
//sizes.add(String.valueOf(in));
naluByteArrayLength += in;
//check if first or last, completed size type etc
if ((start == 0) && (piece[13] & 0b11000000) == 0b10000000){
//start of nalu
start = (piece[2] << 8 | piece[3] & 0xFF);
//type
payloadType = (piece[13] & 0b00011111); //could have used [18] //get type
byteLength = (piece[17]&0xFF | (piece[16]&0xFF)<<8 | (piece[15]&0xFF)<<16 | (piece[14]&0xFF)<<24); //get the h264 encoded length
byteLength += 4; //Now add 4 bytes for the length encoding itself
if (payloadType == 1 || payloadType == 5 && byteLength < 200000){
}else{
System.err.println(TAG + " addpiecetobuffer type: " + String.valueOf(payloadType) + "length: " + String.valueOf(byteLength) );
}
//System.out.println(TAG + " addpiecetobuffer start " + String.valueOf(start) + " type " + String.valueOf(payloadType));
}else if ((finish == 0) && (piece[13] & 0b11000000) == 0b01000000){
//end of nalu
finish = (piece[2] << 8 | piece[3] & 0xFF);
//System.out.println(TAG + " addpiecetobuffer finish " + String.valueOf(finish));
}
if (finish != 0 && start != 0 && completedSize == 0){
//completed size in packet sequnce number NOT in byte length
completedSize = finish - start;
//System.out.println(TAG + " addpiecetobuffer completedsize " + String.valueOf(completedSize));
//originally put in bytes but thats not what I was counting ...duh!
// (piece[14] <<24 | piece[15] << 16 | piece[16] << 8 | piece[17] & 0xFF);
}
//check if complete
if (completedSize != 0 && count == completedSize){
assembleDeliver();
}
}
// we have every sequence number accounted for.
// reconstruct the nalu and send it to the decoder
private void assembleDeliver()
{
count++; //make up for the ount that didn't get called following addpiecetobuffer method
// System.out.println(TAG + " assembleDeliver " + String.valueOf(timeStamp));
//create a new array the exact length needed and sort each nalu by sequence number
NaluePiece[] newbuf = new NaluePiece[count];
System.arraycopy(buffer,0,newbuf,0, count);
Arrays.sort(newbuf);
// TODO: 9/28/2018 we have no gaps in data here checking newbuff !!!!!
//this will be an array we feed/pipe to our videoprocessor
byte[] out;
if (annexB){
out = new byte[naluByteArrayLength-4]; //remove the 4 bytes of length
int tally = 0;
int destPos = 0;
int src = 4;
for (int i = 0; i < count; i++) {
if (i == 1){
src = 0;
}
tally += newbuf[i].piece.length;
System.arraycopy(newbuf[i].piece, src, out, destPos, newbuf[i].piece.length - src);
//Debug.fillCompleteNalData(out, destPos, newbuf[i].piece.length);
destPos += newbuf[i].piece.length - src;
}
/*
StringBuilder sb = new StringBuilder();
sb.append("VideoDecoder assembleDeliver out.length ").append(String.valueOf(out.length))
.append(" destPos ").append(String.valueOf(destPos)).append(" tally ").append(String.valueOf(tally))
.append(" count ").append(String.valueOf(count)).append(" obuf ").append(String.valueOf(completedSize));
for (String s :
sizes) {
sb.append(s).append(" ");
}
System.out.println(sb.toString());
*/
}else{
out = new byte[naluByteArrayLength];
int destPos = 0;
for (int i = 0; i < count; i++) {
System.arraycopy(newbuf[i].piece, 0, out, destPos, newbuf[i].piece.length);
destPos += newbuf[i].piece.length;
}
}
if (naluByteArrayLength != byteLength){
System.err.println(TAG + " assembleDeliver -> ERROR - h264 encoded length: " + String.valueOf(byteLength) + " and byte length found: " + String.valueOf(naluByteArrayLength) + " do not match");
}
// TODO: 9/28/2018 we have gaps in data here
//Debug.checkNaluData(out);
transferTOFFmpeg(out);
deletMe(timeStamp);
}
}
//This class stores the payload and ordering info
private class NaluePiece implements Comparable<NaluePiece>
{
int sequenceNumber; //here is the number we can access to order them
byte[] piece; //here we store the raw payload data to be aggregated
public NaluePiece(int sequenceNumber, byte[] piece)
{
this.sequenceNumber = sequenceNumber;
this.piece = piece;
//Debug.checkNaluPieceData(piece);
}
@Override
public int compareTo(NaluePiece o) {
return Integer.compare(this.sequenceNumber, o.sequenceNumber);
}
}
}