Question

我对Firestore还是很陌生，说实话，我才开始编程几个月。我了解了（或正在学习）这一点，以便更好地管理自己的SIM卡业务，该业务处理的是包含大量数据的CSV码。

我正在使用批写入方法将数据设置到我的firestore数据库中，并且正在通过云函数（index.ts）进行操作。它不会让我一次插入500条以上的记录，这是Firestore的限制。关于如何以某种方式将数据分成500个块，然后对每个数据进行批处理提交的任何建议，而不必一次手动上传500个？

我的CSV文件的长度通常约为10万至20万行。将csv上传到Firebase存储时，我还使用云功能读取csv。我很乐意将其作为Cloud功能的一部分，而不是依赖于第三方csv分离器。

import * as functions from 'firebase-functions';
import * as admin from 'firebase-admin';
import { tmpdir } from 'os';
import { join, dirname } from 'path';
import * as fs from 'fs-extra';
import * as storage from '@google-cloud/storage';
const gcs = new storage.Storage();
import * as papa from 'papaparse';

admin.initializeApp();

export const generateThumbs = functions.storage
  .object()
  .onFinalize(async object => {
    console.log('Version 19');
    const bucket = gcs.bucket(object.bucket);
    const filePath = object.name;
    const fileName = filePath.split('/').pop();
    const bucketDir = dirname(filePath);
    console.log(bucketDir);

    const workingDir = join(tmpdir(), 'thumbs');
    const tmpFilePath = join(workingDir, fileName);

    if (fileName.includes('thumb@')) {
      console.log('exiting function');
      return false;
    }

    // 1. Ensure dir exists
    await fs.ensureDir(workingDir);

    // 2. Download Source File
    await bucket.file(filePath).download({
      destination: tmpFilePath
    });

    const theFile = fs.createReadStream(tmpFilePath);

    var arrayOfData = []

    papa.parse(theFile, {
      worker: true, // Don't bog down the main thread if its a big file
      header: true,
      delimiter: ",",
      step: function(result) {
        // console.log(result.data, 'Result');
        arrayOfData.push(result.data)
        // do stuff with result


      },
      complete: function(results, theFile) {
        var simLength = arrayOfData.length;

        const db = admin.firestore(); 
        const batch = db.batch();


        for (var i = 0; i < simLength;i++) {
          const simArray = arrayOfData[i];
          const sim = simArray[0];
          var iccid = sim.iccid;
          const vcRef = db.collection('vc_uploads').doc(iccid);

          batch.set(vcRef, {simArray})

        }
        return batch.commit().then(function() {
          console.log('Batch Committed!')
        });

      }
    });

    // 5. Cleanup remove the tmp/thumbs from the filesystem
    return fs.remove(workingDir);
  });

Firebase功能+ Firestore：对批次进行排队

0 个答案: