Question

我目前正在尝试通过python apache Beam数据流管道将数据提交到firestore。

但是我一直在得到错误：

Traceback (most recent call last):
  File "/usr/local/lib/python2.7/dist-packages/dataflow_worker/batchworker.py", line 609, in do_work
    work_executor.execute()
  File "/usr/local/lib/python2.7/dist-packages/dataflow_worker/executor.py", line 167, in execute
    op.start()
  File "dataflow_worker/shuffle_operations.py", line 49, in dataflow_worker.shuffle_operations.GroupedShuffleReadOperation.start
    def start(self):
  File "dataflow_worker/shuffle_operations.py", line 50, in dataflow_worker.shuffle_operations.GroupedShuffleReadOperation.start
    with self.scoped_start_state:
  File "dataflow_worker/shuffle_operations.py", line 65, in dataflow_worker.shuffle_operations.GroupedShuffleReadOperation.start
    with self.shuffle_source.reader() as reader:
  File "dataflow_worker/shuffle_operations.py", line 69, in dataflow_worker.shuffle_operations.GroupedShuffleReadOperation.start
    self.output(windowed_value)
  File "apache_beam/runners/worker/operations.py", line 159, in apache_beam.runners.worker.operations.Operation.output
    cython.cast(Receiver, self.receivers[output_index]).receive(windowed_value)
  File "apache_beam/runners/worker/operations.py", line 85, in apache_beam.runners.worker.operations.ConsumerSet.receive
    cython.cast(Operation, consumer).process(windowed_value)
  File "dataflow_worker/shuffle_operations.py", line 233, in dataflow_worker.shuffle_operations.BatchGroupAlsoByWindowsOperation.process
    self.output(wvalue.with_value((k, wvalue.value)))
  File "apache_beam/runners/worker/operations.py", line 159, in apache_beam.runners.worker.operations.Operation.output
    cython.cast(Receiver, self.receivers[output_index]).receive(windowed_value)
  File "apache_beam/runners/worker/operations.py", line 85, in apache_beam.runners.worker.operations.ConsumerSet.receive
    cython.cast(Operation, consumer).process(windowed_value)
  File "apache_beam/runners/worker/operations.py", line 392, in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
  File "apache_beam/runners/worker/operations.py", line 393, in apache_beam.runners.worker.operations.DoOperation.process
    self.dofn_receiver.receive(o)
  File "apache_beam/runners/common.py", line 488, in apache_beam.runners.common.DoFnRunner.receive
    self.process(windowed_value)
  File "apache_beam/runners/common.py", line 496, in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn)
  File "apache_beam/runners/common.py", line 521, in apache_beam.runners.common.DoFnRunner._reraise_augmented
    raise
  File "apache_beam/runners/common.py", line 494, in apache_beam.runners.common.DoFnRunner.process
    self.do_fn_invoker.invoke_process(windowed_value)
  File "apache_beam/runners/common.py", line 395, in apache_beam.runners.common.PerWindowInvoker.invoke_process
    self._invoke_per_window(
  File "apache_beam/runners/common.py", line 432, in apache_beam.runners.common.PerWindowInvoker._invoke_per_window
    output_processor.process_outputs(
  File "apache_beam/runners/common.py", line 561, in apache_beam.runners.common._OutputProcessor.process_outputs
    def process_outputs(self, windowed_input_element, results):
  File "apache_beam/runners/common.py", line 592, in apache_beam.runners.common._OutputProcessor.process_outputs
    self.main_receivers.receive(windowed_value)
  File "apache_beam/runners/worker/operations.py", line 85, in apache_beam.runners.worker.operations.ConsumerSet.receive
    cython.cast(Operation, consumer).process(windowed_value)
  File "apache_beam/runners/worker/operations.py", line 392, in apache_beam.runners.worker.operations.DoOperation.process
    with self.scoped_process_state:
  File "apache_beam/runners/worker/operations.py", line 393, in apache_beam.runners.worker.operations.DoOperation.process
    self.dofn_receiver.receive(o)
  File "apache_beam/runners/common.py", line 488, in apache_beam.runners.common.DoFnRunner.receive
    self.process(windowed_value)
  File "apache_beam/runners/common.py", line 496, in apache_beam.runners.common.DoFnRunner.process
    self._reraise_augmented(exn)
  File "apache_beam/runners/common.py", line 537, in apache_beam.runners.common.DoFnRunner._reraise_augmented
    six.raise_from(new_exn, original_traceback)
  File "/usr/local/lib/python2.7/dist-packages/six.py", line 718, in raise_from
    raise value
ImportError: No module named firebase_admin [while running 'Batches to Firestore']

我目前正在函数内部调用firebase_admin，尽管与在文件开头调用它没什么不同。

class FireBatch(beam.DoFn):
    def process(self, element):
        """
        Make a batch element and send to firestore
        """
        import firebase_admin
        from firebase_admin import credentials
        from firebase_admin import firestore

        cred = credentials.Certificate('./creds/pipeCreds.json')
        firebase_admin.initialize_app(cred)

        db = firestore.Client()

有没有一种方法可以使数据流识别管道中的firebase_admin？还是将数据从管道提交到Firestore？

我已经安装了所有必需的软件包，升级了pip，刷新了凭据，尝试使用google.cloud.firestore而不是fire_base管理员。

感谢您的帮助！

Answer 1

此问题是由于未使用Google SDK进行初始化，我尝试使用GitBash进行初始化，显然配置未正确设置。

下一步是使用Requirements.txt文件初始化应用。

<table border='1'>
    <thead>
        <th>Sample Head</th>
        <th>Sample Head</th>
        <th>Sample Head</th>
        <th>Sample Head</th>
        <th>Sample Head</th>
    </thead>
    <tbody>
        <tr>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td><button type='submit'>X</button></td>
        </tr>
        <tr>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td><button type='submit'>X</button></td>
        </tr>
        <tr>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td><button type='submit'>X</button></td>
        </tr>
        <tr>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td><button type='submit'>X</button></td>
        </tr>
        <tr>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td><button type='submit'>X</button></td>
        </tr>
        <tr>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td><button type='submit'>X</button></td>
        </tr>
        <tr>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td><button type='submit'>X</button></td>
        </tr>
        <tr>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>  
            <td><button type='submit'>X</button></td>
        </tr>
        <tr>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td><button type='submit'>X</button></td>
        </tr>
        <tr>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td><button type='submit'>X</button></td>
        </tr>
        <tr>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td>Sample Data</td>
            <td><button type='submit'>X</button></td>
        </tr>
    </tbody>
</table>

此后，仅将应用程序所需的文件包括在requirements.txt文件中，然后通过添加以下内容来初始化应用程序：

pip freeze > requirements.txt

此外，有必要将--requirements_file requirements.txt更改为firebase_admin==2.11.0，因为这会引起错误。

ImportError：尝试将数据提交到数据流中的Firestore时，没有名为firebase_admin的模块

1 个答案: