我想从stdin读取视频文件流,并使用分段上传将其上传到s3。但是,由于某些未知原因,文件被上传到s3之后已损坏。我不确定是否必须分别传递某种元数据。我的代码段如下。
OutputStream s3OutputStream = new S3UploadStream(s3Config.getS3Client(), bucketName,
mediaDownloadRequest.getVideoExtension(), s3MediaLink, threadCount);
int line;
while ((line = reader.read()) >= 0) {
s3OutputStream.write(line);
}
s3OutputStream.close();
S3UploadStream类:
public class S3UploadStream extends OutputStream {
private final static Integer PART_SIZE = 5 * 1024 * 1024;
private final AmazonS3 s3client;
private final String bucketName;
private final String objectUrl;
private final String uploadId;
private final List<PartETag> partETags = new LinkedList<>();
private final ThreadPoolExecutor executor;
private byte[] partData = new byte[PART_SIZE];
private int partDataIndex = 0;
private int totalPartCountIndex = 0;
private volatile Boolean closed = false;
public S3UploadStream(AmazonS3 s3client, String bucketName, VideoExtension ext, String objectUrl,
int uploadThreadsCount) {
this.s3client = s3client;
this.bucketName = bucketName;
this.objectUrl = objectUrl;
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentType("video/" + ext);
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucketName,
objectUrl, metadata);
InitiateMultipartUploadResult initResponse = s3client.initiateMultipartUpload(initRequest);
this.uploadId = initResponse.getUploadId();
this.executor = new ThreadPoolExecutor(uploadThreadsCount, uploadThreadsCount, 60,
TimeUnit.SECONDS,
new LinkedBlockingQueue<>(100));
}
@Override
public synchronized void write(int b) throws IOException {
if (closed) {
throw new IOException("Trying to write to a closed S3UploadStream");
}
partData[partDataIndex++] = (byte) b;
uploadPart(false);
}
@Override
public synchronized void close() {
if (closed) {
return;
}
closed = true;
uploadPart(true);
executor.shutdown();
try {
executor.awaitTermination(2, TimeUnit.MINUTES);
} catch (InterruptedException e) {
}
CompleteMultipartUploadRequest compRequest =
new CompleteMultipartUploadRequest(bucketName, objectUrl, uploadId, partETags);
s3client.completeMultipartUpload(compRequest);
}
private synchronized void uploadPart(Boolean force) {
if (!force && partDataIndex < PART_SIZE) {
return;
}
createUploadPartTask(partData, partDataIndex);
partData = new byte[PART_SIZE];
partDataIndex = 0;
}
private synchronized void createUploadPartTask(byte[] partData, int partDataIndex) {
InputStream stream = new ByteArrayInputStream(partData, 0, partDataIndex);
UploadPartRequest uploadRequest = new UploadPartRequest()
.withBucketName(bucketName)
.withKey(objectUrl)
.withUploadId(uploadId)
.withPartNumber(++totalPartCountIndex)
.withInputStream(stream)
.withPartSize(partDataIndex);
executor.execute(() -> {
PartETag partETag = s3client.uploadPart(uploadRequest).getPartETag();
synchronized (partETags) {
partETags.add(partETag);
}
});
}