我正在尝试从远程流创建视频文件，以将其存储在硬盘驱动器上。我正在使用Humble Video进行编码。录制和创建视频文件效果很好。但是现在我想在视频中添加音频。
我正在从远程源接收JPG图像和音频数据作为字节数组。音频数据的采样率为11025，是1通道16位值。
我想我仍然缺少有关尝试插入视频文件的音频数据的时基和时间戳的信息。在大多数情况下，所有录制的音频都非常混乱，只能在视频的前几秒钟才能听到。
这是我的代码：
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.function.Consumer;
import javax.imageio.ImageIO;
import io.humble.video.AudioChannel.Layout;
import io.humble.video.AudioFormat.Type;
import io.humble.video.Codec;
import io.humble.video.Encoder;
import io.humble.video.MediaAudio;
import io.humble.video.MediaPacket;
import io.humble.video.MediaPicture;
import io.humble.video.Muxer;
import io.humble.video.MuxerFormat;
import io.humble.video.PixelFormat;
import io.humble.video.Rational;
import io.humble.video.awt.MediaPictureConverter;
import io.humble.video.awt.MediaPictureConverterFactory;
public class VideoStream {
private static final int FPS = 30;
private static final int AUDIO_SAMPLE_RATE = 11025;
private static final int AUDIO_SAMPLE_SIZE = 11025;
private final String filename;
private final long startTimestampVideo;
private final long startTimestampAudio;
private final Muxer muxer;
private final Encoder videoEncoder;
private final Encoder audioEncoder;
private final Rational timebase;
private MediaPicture picture;
private MediaPictureConverter converter;
private MediaPacket packet;
private MediaAudio sound;
private boolean finished = false;
private static class EncoderTask implements Runnable {
private final byte[] data;
private final long timestamp;
private final Consumer<EncoderTask> encoder;
public EncoderTask(byte[] data, long timestamp, Consumer<EncoderTask> encoder) {
if (encoder == null) {
throw new IllegalArgumentException("Encoder must not be null.");
}
this.data = data;
this.timestamp = timestamp;
this.encoder = encoder;
}
@Override
public void run() {
encoder.accept(this);
}
public byte[] getData() {
return data;
}
public long getTimestamp() {
return timestamp;
}
public Consumer<EncoderTask> getEncoder() {
return encoder;
}
}
private ExecutorService threadPool = Executors.newSingleThreadExecutor();
public VideoStream(String filename, int width, int height, long startTimestampVideo, long startTimestampAudio)
throws IOException, InterruptedException {
this.filename = filename;
this.startTimestampVideo = startTimestampVideo;
this.startTimestampAudio = startTimestampAudio;
this.timebase = Rational.make(1, FPS);
this.muxer = Muxer.make(filename, null, null);
PixelFormat.Type pixelFormat = PixelFormat.Type.PIX_FMT_YUV420P;
Codec videoCodec = Codec.findEncodingCodec(muxer.getFormat().getDefaultVideoCodecId());
Codec audioCodec = Codec.findEncodingCodec(muxer.getFormat().getDefaultAudioCodecId());
this.videoEncoder = createVideoEncoder(videoCodec, width, height, pixelFormat);
this.audioEncoder = createAudioEncoder(audioCodec);
videoEncoder.open(null, null);
audioEncoder.open(null, null);
muxer.addNewStream(videoEncoder);
muxer.addNewStream(audioEncoder);
muxer.open(null, null);
picture = MediaPicture.make(videoEncoder.getWidth(), videoEncoder.getHeight(), pixelFormat);
picture.setTimeBase(timebase);
sound = MediaAudio.make(AUDIO_SAMPLE_SIZE, AUDIO_SAMPLE_RATE, 1, Layout.CH_LAYOUT_MONO, Type.SAMPLE_FMT_S16);
sound.setTimeBase(timebase);
packet = MediaPacket.make();
}
private Encoder createVideoEncoder(Codec codec, int width, int height, PixelFormat.Type pixelFormat) {
Encoder encoder = Encoder.make(codec);
encoder.setWidth(width);
encoder.setHeight(height);
encoder.setPixelFormat(pixelFormat);
encoder.setTimeBase(timebase);
if (muxer.getFormat().getFlag(MuxerFormat.Flag.GLOBAL_HEADER)) {
encoder.setFlag(Encoder.Flag.FLAG_GLOBAL_HEADER, true);
}
return encoder;
}
private Encoder createAudioEncoder(Codec codec) {
Encoder encoder = Encoder.make(codec);
encoder.setSampleRate(AUDIO_SAMPLE_RATE);
encoder.setChannels(1);
encoder.setChannelLayout(Layout.CH_LAYOUT_MONO);
encoder.setSampleFormat(Type.SAMPLE_FMT_S16);
if (muxer.getFormat().getFlag(MuxerFormat.Flag.GLOBAL_HEADER)) {
encoder.setFlag(Encoder.Flag.FLAG_GLOBAL_HEADER, true);
}
return encoder;
}
public String getFilename() {
return filename;
}
public synchronized void addImage(byte[] imageData, long timestamp) {
if (!finished) {
System.out.println("Adding image: " + (timestamp - startTimestampVideo));
threadPool.execute(new EncoderTask(imageData, timestamp, this::encodeImage));
}
}
public synchronized void addAudio(byte[] audioData, long timestamp) {
if (!finished) {
System.out.println("Adding audio: " + (timestamp - startTimestampAudio));
threadPool.execute(new EncoderTask(audioData, timestamp, this::encodeAudio));
}
}
public synchronized void finish() {
if (!finished) {
threadPool.execute(new EncoderTask(null, 0, this::finish));
}
}
private synchronized void encodeImage(EncoderTask task) {
BufferedImage jpegImage = convertImageDataToBufferedImage(task.getData());
if (jpegImage == null) {
return;
}
BufferedImage image = convertToType(jpegImage, BufferedImage.TYPE_3BYTE_BGR);
if (converter == null) {
converter = MediaPictureConverterFactory.createConverter(image, picture);
}
long t = Math.round((task.getTimestamp() - startTimestampVideo) * timebase.getDouble());
converter.toPicture(picture, image, t);
System.out.println("Encoding video: " + t);
do {
videoEncoder.encode(packet, picture);
if (packet.isComplete()) {
muxer.write(packet, false);
}
} while (packet.isComplete());
}
private synchronized void encodeAudio(EncoderTask task) {
System.out.println(
"Audio delta: " + (task.getTimestamp() - startTimestampAudio) + " timebase: " + timebase.getDouble());
long t = Math.round((task.getTimestamp() - startTimestampAudio) * timebase.getDouble() * AUDIO_SAMPLE_RATE);
sound.getData(0).put(task.data, 0, 0, task.data.length);
sound.setNumSamples(task.data.length);
sound.setTimeStamp(t);
sound.setComplete(true);
System.out.println("Encoding audio: " + t);
do {
audioEncoder.encode(packet, sound);
if (packet.isComplete()) {
muxer.write(packet, false);
}
} while (packet.isComplete());
}
private synchronized void finish(EncoderTask task) {
do {
videoEncoder.encode(packet, null);
if (packet.isComplete()) {
muxer.write(packet, false);
}
} while (packet.isComplete());
do {
audioEncoder.encode(packet, null);
if (packet.isComplete()) {
muxer.write(packet, false);
}
} while (packet.isComplete());
muxer.close();
finished = true;
threadPool.shutdown();
}
private static BufferedImage convertImageDataToBufferedImage(byte[] imageData) {
try (InputStream in = new ByteArrayInputStream(imageData)) {
return ImageIO.read(in);
} catch (IOException e) {
return null;
}
}
private static BufferedImage convertToType(BufferedImage sourceImage, int targetType) {
BufferedImage image;
if (sourceImage.getType() == targetType) {
// if the source image is already the target type, return the source image
image = sourceImage;
} else {
// otherwise create a new image of the target type and draw the new
// image
image = new BufferedImage(sourceImage.getWidth(), sourceImage.getHeight(), targetType);
image.getGraphics().drawImage(sourceImage, 0, 0, null);
}
return image;
}
}
如果有人在相同文件中编码音频和视频的示例，那就太好了。 Humble Video git中的演示仅用于编码视频数据。