将流视频分段上传到S3后,视频文件已损坏

时间:2019-11-29 07:41:02

标签: java amazon-s3

我想从stdin读取视频文件流,并使用分段上传将其上传到s3。但是,由于某些未知原因,文件被上传到s3之后已损坏。我不确定是否必须分别传递某种元数据。我的代码段如下。

OutputStream s3OutputStream = new S3UploadStream(s3Config.getS3Client(), bucketName,
          mediaDownloadRequest.getVideoExtension(), s3MediaLink, threadCount);

int line;
while ((line = reader.read()) >= 0) {
   s3OutputStream.write(line);
}
s3OutputStream.close();

S3UploadStream类:

public class S3UploadStream extends OutputStream {

  private final static Integer PART_SIZE = 5 * 1024 * 1024;
  private final AmazonS3 s3client;
  private final String bucketName;
  private final String objectUrl;
  private final String uploadId;
  private final List<PartETag> partETags = new LinkedList<>();
  private final ThreadPoolExecutor executor;
  private byte[] partData = new byte[PART_SIZE];
  private int partDataIndex = 0;
  private int totalPartCountIndex = 0;
  private volatile Boolean closed = false;

  public S3UploadStream(AmazonS3 s3client, String bucketName, VideoExtension ext, String objectUrl,
      int uploadThreadsCount) {
    this.s3client = s3client;
    this.bucketName = bucketName;
    this.objectUrl = objectUrl;
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentType("video/" + ext);
    InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucketName,
        objectUrl, metadata);
    InitiateMultipartUploadResult initResponse = s3client.initiateMultipartUpload(initRequest);
    this.uploadId = initResponse.getUploadId();
    this.executor = new ThreadPoolExecutor(uploadThreadsCount, uploadThreadsCount, 60,
        TimeUnit.SECONDS,
        new LinkedBlockingQueue<>(100));
  }


  @Override
  public synchronized void write(int b) throws IOException {
    if (closed) {
      throw new IOException("Trying to write to a closed S3UploadStream");
    }
    partData[partDataIndex++] = (byte) b;
    uploadPart(false);
  }

  @Override
  public synchronized void close() {
    if (closed) {
      return;
    }
    closed = true;
    uploadPart(true);
    executor.shutdown();
    try {
      executor.awaitTermination(2, TimeUnit.MINUTES);
    } catch (InterruptedException e) {
    }
    CompleteMultipartUploadRequest compRequest =
        new CompleteMultipartUploadRequest(bucketName, objectUrl, uploadId, partETags);
    s3client.completeMultipartUpload(compRequest);
  }

  private synchronized void uploadPart(Boolean force) {
    if (!force && partDataIndex < PART_SIZE) {
      return;
    }
    createUploadPartTask(partData, partDataIndex);
    partData = new byte[PART_SIZE];
    partDataIndex = 0;
  }

  private synchronized void createUploadPartTask(byte[] partData, int partDataIndex) {
    InputStream stream = new ByteArrayInputStream(partData, 0, partDataIndex);
    UploadPartRequest uploadRequest = new UploadPartRequest()
        .withBucketName(bucketName)
        .withKey(objectUrl)
        .withUploadId(uploadId)
        .withPartNumber(++totalPartCountIndex)
        .withInputStream(stream)
        .withPartSize(partDataIndex);
    executor.execute(() -> {
      PartETag partETag = s3client.uploadPart(uploadRequest).getPartETag();
      synchronized (partETags) {
        partETags.add(partETag);
      }
    });
  }

0 个答案:

没有答案