Question

我的目标是将一个大文件（任何文件类型）上传到Azure blob。

我想下面两种方法。

1. Convert file into stream and upload into Azure blob
2. Convert file into block streams and commit all the blocks to Azure blob in parallel

我正在使用Node.js Azure Blob服务API

方法1：我正在使用ReadStream将文件转换为fs.createReadStream(fileName, 'utf-8')。然后使用blobService.createBlockBlobFromStream()将流上传到Azure blob。下面是此方法的示例代码

    var blobName = "azure-blob-name.doc";
    var fileSize = fs.statSync(fileName).size;
    var readableStream = fs.createReadStream(fileName, 'utf-8');
        blobService.createBlockBlobFromStream(containerName, blobName, readableStream, fileSize, function (error, result, response) { 
            if(error) console.log(error);
            console.log({message : "blob uploaded"});
        });

方法2：我想通过使用ReadStream将blobService.createBlockFromStream()转换为天蓝色的blob块，并并行提交所有块。如何实现这种方法？

下面是我要尝试的示例代码，但最终会引发错误

var fileName = "local-file-path/filename.doc";
    var blobName = "azure-blob-name.doc";
    var fileSize = fs.statSync(fileName).size;
    var readableStream = fs.createReadStream(fileName, 'utf-8');
    var bl=1;
    var blockIds = [];
    readableStream.on("data", function (chunk) {
        var blockId = 'block'+bl;
        bl++;
        blobService.createBlockFromStream(blockId, containerName, blobName, chunk, chunk.length, function(error, response){
            if(error) console.log(error);

            blockIds.push(blockId);
                    console.log({message : "block created"});
        });
    });

以下是我遇到的错误

    D:\main-workspace\file-management\node_modules\azure-storage\lib\common\services\storageserviceclient.js:522
                body.outputStream.on('open', function () {
                                  ^
    TypeError: body.outputStream.on is not a function

方法1可以正常工作，但要花费太多时间来上传。（统计资料：花费了80秒上传了40 MB的文件）。
我的第二种方法是提高性能的正确方法吗？如果是，我如何实现这种方法？

是否有更好的方法来提高性能？在这方面有什么建议吗？

除了Azure Blob服务API之外，我们还有其他方法可以这样做吗？我的目标是在更短的时间内上传任何类型的大文件。例如，必须在20秒内上传40 MB的文件。

请朝正确的方向开车。感谢您的提前答复。

编辑-更新了代码

@Gaurav

下面是我更新的代码。但是我的readableStream.on("end", function(){})从未被要求提交这些块。在我的Azure帐户中，文件正在创建，但字节数为0

      const stream = require('stream');
      var fileName = "local-file-path/filename.doc";
      var blobName = "azure-blob-name.doc";
      var fileSize = fs.statSync(blobName).size;
      var customBlockSize = 1200; //1.2 MB CAP           
      var NoOfBlocks = Math.ceil(fileSize/customBlockSize);
      var blockIdLength = NoOfBlocks.toString().length;
      var readableStream = fs.createReadStream(fileName, 'utf-8');
      var bl=1;
      var blockIds = [];
      readableStream.on("data", function (chunk) {         
          var blockId = 'block'+ blockGuId(bl, blockIdLength);//Block ids must be of same length
          bl++;
          const bufferStream = new stream.PassThrough({
            highWaterMark: chunk.length
          });
          bufferStream.end(chunk);
          blobService.createBlockFromStream(blockId, containerName, blobName, bufferStream, chunk.length, function(error, response){
              if(error) console.log(error);

              blockIds.push(blockId);
              console.log({message : "block ("+blockId+") created"});
          });
      });

      readableStream.on("end", function(){

        console.log('readableStream.on "end"');

        blobService.commitBlocks(containerName, blobName, blockIds, function(error, result){
          if(error) console.log(error);

          console.log({message : "all blocks uploaded"});
          var hrend = process.hrtime(hrstart)
          console.info('Execution time (sec ms): %ds %dms', hrend[0], hrend[1] / 1000000)
        });
      });

      blockGuId(number, blockIdLength)
      {
        return Array(Math.max(blockIdLength - String(number).length + 1, 0)).join(0) + number;
      }

下面是套接字错误

{ Error: socket hang up at createHangUpError (_http_client.js:323:15) at TLSSocket.socketCloseListener (_http_client.js:364:25) at TLSSocket.emit (events.js:187:15) at _handle.close (net.js:610:12) at TCP.done (_tls_wrap.js:386:7) code: 'ECONNRESET' }

我在这里想念的是什么？为什么在创建块时发生套接字错误？

Answer 1

您可以尝试以下代码：

const stream = require('stream');

var fileName = "local-file-path/filename.doc";
    var blobName = "azure-blob-name.doc";
    var fileSize = fs.statSync(fileName).size;
    var readableStream = fs.createReadStream(fileName, 'utf-8');
    var bl=1;
    var blockIds = [];
    readableStream.on("data", function (chunk) {
        //var blockId = 'block'+bl;
        bl++;
        var blockId = UseGuidHere();//Block ids must be of same length
        const bufferStream = new stream.PassThrough({
          highWaterMark: chunk.length
        });
        bufferStream.end(chunk);
        blobService.createBlockFromStream(blockId, containerName, blobName, bufferStream, chunk.length, function(error, response){
            if(error) console.log(error);

            blockIds.push(blockId);
                    console.log({message : "block created"});
        });
    });

如代码中所述，请为您的块ID分配一个GUID，因为块ID的长度必须相同。根据您当前的逻辑，在9个块之后，块ID的长度将发生变化，这将导致上传过程失败。

将大文件上传到node.js中的Azure blob处理

1 个答案: