Question

无法使用jszip压缩所有文件。 JS zip从大约143个请求中从控制台读取快照中显示的所有402个文件，但仅压缩143个文件。我正在使用parallelimit同时干净地处理多个异步请求。我是如何获取结果中的所有403个文件？

private downloadUntouchedFiles = () => {


let requestObjectInfo = [];
let index = 0;
this._eventEmitter.emit(Constants.LOADER_CHANGE, { show: true });


this._eligibilitySubmissionInstance.getUntouchedFiles(this.state.filterObject).then((requests) => {
  debugger;
  if (!(!requests)) {
    if (requests.length > 0) {

      var zip = new JSZip();
      var zipFileName = "ES_Unviewed_Files";
      var promises = [];
      this._eventEmitter.emit(Constants.LOADER_CHANGE, { show: true });



      const downloadSubPromises = [];
      let i =0;
          requests.forEach((req) => {
            req.Folder.Files.forEach(f => {
              f.Name = this.state.initials + '_' + this.state.userId + '_' + f.Name;
              console.log(f.Name);
              i++;
              console.log(i);
              downloadSubPromises.push((submit: any) => {
                JSZipUtils.getBinaryContent(f.ServerRelativeUrl, (err, data) => {
                  try {
                    if (err) {
                      throw err;
                    }
                    zip.file(f.Name, data, { binary: true });

                    submit(null, true);

                  } catch (err) {
                    submit(err, true);
                    this._eventEmitter.emit(Constants.LOADER_CHANGE, { show: false });
                    this._loggerInstance.logException(Constants.SISCC_ES_EXCEPTIONS, {
                      Component: this._canonicalName,
                      Message: ErrorMessages.COM007,
                      UserName: !(!DataSingleton.getCurrentUser()) ? DataSingleton.getCurrentUser() : '',
                      Group: '',
                      Notes: err,
                      Source: Constants.EXCEPTION_UI_SOURCE,
                      ExceptionID: Guid.create().toString()
                    } as ExceptionObject).then(() => {
                    });
                  }
                });

              });
            });
            requestObjectInfo.push(req);
          });
          parallelLimit(downloadSubPromises, Constants.DOWNLOAD_BATCH_MAX_FILE_LIMIT,
            (err, results) => {
              try {
                console.log(results);
                debugger;
                zip
                  .generateInternalStream({ type: "blob" })
                  .accumulate()
                  .then((content) => {
                    saveAs(content, zipFileName + ".zip");
                  });
              }
              catch (err) {
                this._eventEmitter.emit(Constants.LOADER_CHANGE, { show: false });
                this._loggerInstance.logException(Constants.SISCC_ES_EXCEPTIONS, {
                  Component: this._canonicalName,
                  Message: ErrorMessages.COM007,
                  UserName: !(!DataSingleton.getCurrentUser()) ? DataSingleton.getCurrentUser() : '',
                  Group: '',
                  Notes: err,
                  Source: Constants.EXCEPTION_UI_SOURCE,
                  ExceptionID: Guid.create().toString()
                } as ExceptionObject).then(() => {
                });
              }});

      while (index < requestObjectInfo.length) {
        this.setState({ requestObject: requestObjectInfo[index] });
        if (this.state.requestObject.Status !== Constants.ES_DOWNLOADREQUEST_STATUS) {
          this.updateESRequestStatus(Constants.ES_DOWNLOADREQUEST_STATUS);
        }
        index++;
      }
      this._eventEmitter.emit(Constants.LOADER_CHANGE, { show: false });
    }
  }
});

}

在这种情况下，JS Heap仅使用55-75MB。

Answer 1

这将起作用

private downloadUntouchedFiles = () => {
this._eventEmitter.emit(Constants.LOADER_CHANGE, { show: true });
let statusUpdatePromises = [];
this._eligibilitySubmissionInstance.getUntouchedFiles(this.state.filterObject).then((requests) => {
  if (!(!requests)) {
    if (requests.length > 0) {
      var zip = new JSZip();
      var zipFileName = "ES_Unviewed_Files";
      const downloadSubPromises = [];
      requests.forEach((req: any) => {
        req.Folder.Files.forEach((f: any) => {
          f.Name = this.state.userId + '_' + f.Name;
          downloadSubPromises.push((submit: any) => {
            JSZipUtils.getBinaryContent(`${new Constants().BASE_URL}${encodeURIComponent(f.ServerRelativeUrl).replace('%2F', '/')}`, (err, data) => {
              try {
                if (err) {
                  submit(null, true);
                } else {
                  zip.file(f.Name, data, { binary: true });
                  submit(null, true);
                }
              } catch (err) {
                this._eventEmitter.emit(Constants.LOADER_CHANGE, { show: false });
                this._loggerInstance.logException(Constants.SISCC_ES_EXCEPTIONS, {
                  Component: this._canonicalName,
                  Message: ErrorMessages.COM007,
                  UserName: !(!DataSingleton.getCurrentUser()) ? DataSingleton.getCurrentUser() : '',
                  Group: '',
                  Notes: err,
                  Source: Constants.EXCEPTION_UI_SOURCE,
                  ExceptionID: Guid.create().toString()
                } as ExceptionObject).then(() => {
                });
                submit(null, false);
              }
            });

          });
        });
        statusUpdatePromises.push((submit: any) => {
          this.setState({ requestObject: req }, () => {
            if (this.state.requestObject.Status !== Constants.ES_DOWNLOADREQUEST_STATUS) {
              this.updateESRequestStatus(Constants.ES_DOWNLOADREQUEST_STATUS).then(res => {
                submit(true);
              });
            } else {
              submit(true);
            }
          });
        });
      });
      parallelLimit(downloadSubPromises, Constants.UPLOAD_BATCH_MAX_FILE_LIMIT,
        (err: any, results: any) => {
          parallelLimit(statusUpdatePromises, Constants.UPLOAD_BATCH_MAX_FILE_LIMIT,
            (subErr: any, subResults: any) => {
              try {
                zip
                  .generateInternalStream({ type: "blob" })
                  .accumulate()
                  .then((content) => {
                    this._eventEmitter.emit(Constants.LOADER_CHANGE, { show: false });
                    saveAs(content, zipFileName + ".zip");
                  });
              }
              catch (err) {
                this._eventEmitter.emit(Constants.LOADER_CHANGE, { show: false });
                this._loggerInstance.logException(Constants.SISCC_ES_EXCEPTIONS, {
                  Component: this._canonicalName,
                  Message: ErrorMessages.COM007,
                  UserName: !(!DataSingleton.getCurrentUser()) ? DataSingleton.getCurrentUser() : '',
                  Group: '',
                  Notes: err,
                  Source: Constants.EXCEPTION_UI_SOURCE,
                  ExceptionID: Guid.create().toString()
                } as ExceptionObject).then(() => {
                });
              }
            });
        });
    }
  }
});

}

Answer 2

您可以使用async和await使代码更易于理解并避免深层嵌套：

const ZIP_FILE_NAME = "ES_Unviewed_Files.zip"

const downloadUntouchedFiles = async () => {    
    const zip = new JSZip()
    const untouched = this._eligibilitySubmissionInstance.getUntouchedFiles(this.state.filterObject)    
    if (!untouched?.length) return

    const files = untouched.reduce((acc, { Folder: { Files } }) => 
        Files.forEach(f => acc.push({ name: `${this.state.userId}+${f.Name}`, ...f })), [])

    const downloads = files.map(({name, ServerRelativeUrl}) => 
        async () => ({ name, data: await fetchBinary(ServerRelativeUrl)}))

    const responses = await batched(downloads, Constants.DOWNLOAD_BATCH_MAX_FILE_LIMIT)

    responses.forEach(({ status, value: { name, data } }) => 
        status === 'fulfilled' && zip.file(name, data, { binary: true }))

    const content = await zip.generateInternalStream({ type: "blob" }).accumulate()
    await saveAs(content, ZIP_FILE_NAME)    
}

const fetchBinary = (file) => new Promise((resolve) => 
    JSZipUtils.getBinaryContent(url, (err, data) => err ? reject(err) : resolve(data)))

async function batched(fns, batchSize = 2) {
    const results = []    
    for(let start = 0, end = batchSize; start < fns.length; start += batchSize, end = start+batchSize) {
        const slice = fns.slice(start, end)        
        const promises = slice.map((fn) => fn())
        results.push([...await Promise.allSettled(promises)])
    }
    return results.flat()
}

使用jszip在Frontend中下载大量文件

2 个答案: