Question

由于luigi Task，我想在S3上存储一个pickle文件。下面是定义任务的类：

class CreateItemVocabulariesTask(luigi.Task):
    def __init__(self):
        self.client = S3Client(AwsConfig().aws_access_key_id,
                               AwsConfig().aws_secret_access_key)
        super().__init__()

    def requires(self):
        return [GetItem2VecDataTask()]

    def run(self):
        filename = 'item2vec_results.tsv'
        data = self.client.get('s3://{}/item2vec_results.tsv'.format(AwsConfig().item2vec_path),
                               filename)
        df = pd.read_csv(filename, sep='\t', encoding='latin1')
        unique_users = df['CustomerId'].unique()
        unique_items = df['ProductNumber'].unique()
        item_to_int, int_to_item = utils.create_lookup_tables(unique_items)
        user_to_int, int_to_user = utils.create_lookup_tables(unique_users)

        with self.output()[0].open('wb') as out_file:
            pickle.dump(item_to_int, out_file)
        with self.output()[1].open('wb') as out_file:
            pickle.dump(int_to_item, out_file)
        with self.output()[2].open('wb') as out_file:
            pickle.dump(user_to_int, out_file)
        with self.output()[3].open('wb') as out_file:
            pickle.dump(int_to_user, out_file)

    def output(self):
        files = [S3Target('s3://{}/item2int.pkl'.format(AwsConfig().item2vec_path), client=self.client),
                 S3Target('s3://{}/int2item.pkl'.format(AwsConfig().item2vec_path), client=self.client),
                 S3Target('s3://{}/user2int.pkl'.format(AwsConfig().item2vec_path), client=self.client),
                 S3Target('s3://{}/int2user.pkl'.format(AwsConfig().item2vec_path), client=self.client),]
        return files

运行此任务时，出现错误ValueError: Unsupported open mode 'wb'。我尝试转储到pickle文件中的项目只是python词典。

完整追溯：

Traceback (most recent call last):
  File "C:\Anaconda3\lib\site-packages\luigi\worker.py", line 203, in run
    new_deps = self._run_get_new_deps()
  File "C:\Anaconda3\lib\site-packages\luigi\worker.py", line 140, in _run_get_new_deps
    task_gen = self.task.run()
  File "C:\Users\user\Documents\python workspace\pipeline.py", line 60, in run
    with self.output()[0].open('wb') as out_file:
  File "C:\Anaconda3\lib\site-packages\luigi\contrib\s3.py", line 714, in open
    raise ValueError("Unsupported open mode '%s'" % mode)
ValueError: Unsupported open mode 'wb'

Answer 1

这是一个仅在python 3.x上发生的问题，如here所述。为了使用python 3并编写二进制文件或目标文件（即使用“ wb”模式），只需将S3Target的格式参数设置为Nop。像这样：

S3Target('s3://path/to/file', client=self.client, format=luigi.format.Nop)

请注意，这只是一个把戏，不是那么直观也没有记录。

如何通过luigi Task将腌制文件写入S3？

1 个答案: