我正在尝试使用multiprocess（为了更好地挑选对象）和cassandra.concurrent来插入行。这是我的代码，
from multiprocess.pool import Pool
from cassandra.cluster import Cluster
from cassandra.concurrent import execute_concurrent_with_args
import itertools
class QueryManager(object):
concurrency = 100 # chosen to match the default in execute_concurrent_with_args
def __init__(self, cluster, process_count=None):
self.pool = Pool(processes=process_count, initializer=self._setup, initargs=(cluster,))
@classmethod
def _setup(cls, cluster):
cls.session = cluster.connect(keyspace=keyspace_name)
cls.prepared = cls.session.prepare("""
INSERT INTO test_table (key1, key2, key3, key4, key5) VALUES (?, ?, ?, ?, ?)
""")
def close_pool(self):
self.pool.close()
self.pool.join()
def get_results(self, params):
results = self.pool.map(_multiprocess_write, (params[n:n+self.concurrency] for n in range(0, len(params), self.concurrency)))
return list(itertools.chain(*results))
@classmethod
def _results_from_concurrent(cls, params):
return [results[1] for results in execute_concurrent_with_args(cls.session, cls.prepared, params)]
def _multiprocess_write(params):
return QueryManager._results_from_concurrent(params)
if __name__ == '__main__':
# connect cluster
cluster = Cluster()
session = cluster.connect()
qm = QueryManager(cluster, processes)
我收到了以下错误：
OSError: [Errno 22] Invalid argument
代码停在
self.pool = Pool(processes=process_count, initializer=self._setup, initargs=(cluster,))
错误跟踪：
Traceback (most recent call last):
File "<string>", line 1, in <module>
Traceback (most recent call last):
File "C:\Continuum\Anaconda3\lib\site-packages\multiprocess\popen_spawn_win32.py", line 66, in __init__
reduction.dump(process_obj, to_child)
File "C:\Continuum\Anaconda3\lib\site-packages\multiprocess\reduction.py", line 62, in dump
File "C:\Continuum\Anaconda3\lib\site-packages\multiprocess\spawn.py", line 109, in spawn_main
ForkingPickler(file, protocol).dump(obj)
File "C:\Continuum\Anaconda3\lib\pickle.py", line 408, in dump
exitcode = _main(fd)
File "C:\Continuum\Anaconda3\lib\site-packages\multiprocess\spawn.py", line 119, in _main
self = pickle.load(from_parent)
File "C:\Continuum\Anaconda3\lib\site-packages\dill\dill.py", line 250, in load
obj = pik.load()
File "C:\Continuum\Anaconda3\lib\pickle.py", line 1039, in load
self.save(obj)
File "C:\Continuum\Anaconda3\lib\pickle.py", line 520, in save
dispatch[key[0]](self)
File "C:\Continuum\Anaconda3\lib\pickle.py", line 1319, in load_newobj
self.save_reduce(obj=obj, *rv)
File "C:\Continuum\Anaconda3\lib\pickle.py", line 623, in save_reduce
save(state)
File "C:\Continuum\Anaconda3\lib\pickle.py", line 475, in save
obj = cls.__new__(cls, *args)
TypeError: __new__() missing 3 required positional arguments: 'ob', 'callback', and 'key'
f(self, obj) # Call unbound method with explicit self
File "C:\Continuum\Anaconda3\lib\site-packages\dill\dill.py", line 835, in save_module_dict
StockPickler.save_dict(pickler, obj)
File "C:\Continuum\Anaconda3\lib\pickle.py", line 810, in save_dict
self._batch_setitems(obj.items())
File "C:\Continuum\Anaconda3\lib\pickle.py", line 836, in _batch_setitems
save(v)
File "C:\Continuum\Anaconda3\lib\pickle.py", line 475, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Continuum\Anaconda3\lib\pickle.py", line 740, in save_tuple
save(element)
File "C:\Continuum\Anaconda3\lib\pickle.py", line 475, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Continuum\Anaconda3\lib\pickle.py", line 725, in save_tuple
save(element)
File "C:\Continuum\Anaconda3\lib\pickle.py", line 520, in save
self.save_reduce(obj=obj, *rv)
File "C:\Continuum\Anaconda3\lib\pickle.py", line 623, in save_reduce
save(state)
File "C:\Continuum\Anaconda3\lib\pickle.py", line 475, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Continuum\Anaconda3\lib\site-packages\dill\dill.py", line 835, in save_module_dict
StockPickler.save_dict(pickler, obj)
File "C:\Continuum\Anaconda3\lib\pickle.py", line 810, in save_dict
self._batch_setitems(obj.items())
File "C:\Continuum\Anaconda3\lib\pickle.py", line 836, in _batch_setitems
save(v)
File "C:\Continuum\Anaconda3\lib\pickle.py", line 520, in save
self.save_reduce(obj=obj, *rv)
File "C:\Continuum\Anaconda3\lib\pickle.py", line 623, in save_reduce
save(state)
File "C:\Continuum\Anaconda3\lib\pickle.py", line 475, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Continuum\Anaconda3\lib\site-packages\dill\dill.py", line 835, in save_module_dict
StockPickler.save_dict(pickler, obj)
File "C:\Continuum\Anaconda3\lib\pickle.py", line 810, in save_dict
self._batch_setitems(obj.items())
File "C:\Continuum\Anaconda3\lib\pickle.py", line 836, in _batch_setitems
save(v)
File "C:\Continuum\Anaconda3\lib\pickle.py", line 475, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Continuum\Anaconda3\lib\site-packages\dill\dill.py", line 835, in save_module_dict
StockPickler.save_dict(pickler, obj)
File "C:\Continuum\Anaconda3\lib\pickle.py", line 810, in save_dict
self._batch_setitems(obj.items())
File "C:\Continuum\Anaconda3\lib\pickle.py", line 836, in _batch_setitems
save(v)
File "C:\Continuum\Anaconda3\lib\pickle.py", line 520, in save
self.save_reduce(obj=obj, *rv)
File "C:\Continuum\Anaconda3\lib\pickle.py", line 623, in save_reduce
save(state)
File "C:\Continuum\Anaconda3\lib\pickle.py", line 475, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Continuum\Anaconda3\lib\site-packages\dill\dill.py", line 835, in save_module_dict
StockPickler.save_dict(pickler, obj)
File "C:\Continuum\Anaconda3\lib\pickle.py", line 810, in save_dict
self._batch_setitems(obj.items())
File "C:\Continuum\Anaconda3\lib\pickle.py", line 836, in _batch_setitems
save(v)
File "C:\Continuum\Anaconda3\lib\pickle.py", line 475, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Continuum\Anaconda3\lib\site-packages\dill\dill.py", line 835, in save_module_dict
StockPickler.save_dict(pickler, obj)
File "C:\Continuum\Anaconda3\lib\pickle.py", line 810, in save_dict
self._batch_setitems(obj.items())
File "C:\Continuum\Anaconda3\lib\pickle.py", line 841, in _batch_setitems
save(v)
File "C:\Continuum\Anaconda3\lib\pickle.py", line 520, in save
self.save_reduce(obj=obj, *rv)
File "C:\Continuum\Anaconda3\lib\pickle.py", line 623, in save_reduce
save(state)
File "C:\Continuum\Anaconda3\lib\pickle.py", line 475, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Continuum\Anaconda3\lib\site-packages\dill\dill.py", line 835, in save_module_dict
StockPickler.save_dict(pickler, obj)
File "C:\Continuum\Anaconda3\lib\pickle.py", line 810, in save_dict
self._batch_setitems(obj.items())
File "C:\Continuum\Anaconda3\lib\pickle.py", line 836, in _batch_setitems
save(v)
File "C:\Continuum\Anaconda3\lib\pickle.py", line 475, in save
f(self, obj) # Call unbound method with explicit self
File "C:\Continuum\Anaconda3\lib\pickle.py", line 703, in save_str
self.write(BINUNICODE + pack("<I", n) + encoded)
File "C:\Continuum\Anaconda3\lib\pickle.py", line 216, in write
return self.file_write(data)
BrokenPipeError: [Errno 32] Broken pipe
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:/Users/dweng/PycharmProjects/cassandra_tests/cassandra_multiprocess_concurrent.py", line 157, in <module>
qm = QueryManager(cluster, processes)
File "C:/Users/dweng/PycharmProjects/cassandra_tests/cassandra_multiprocess_concurrent.py", line 49, in __init__
self.pool = Pool(processes=process_count, initializer=self._setup, initargs=(cluster,))
File "C:\Continuum\Anaconda3\lib\site-packages\multiprocess\pool.py", line 168, in __init__
self._repopulate_pool()
File "C:\Continuum\Anaconda3\lib\site-packages\multiprocess\pool.py", line 233, in _repopulate_pool
w.start()
File "C:\Continuum\Anaconda3\lib\site-packages\multiprocess\process.py", line 105, in start
self._popen = self._Popen(self)
File "C:\Continuum\Anaconda3\lib\site-packages\multiprocess\context.py", line 313, in _Popen
return Popen(process_obj)
File "C:\Continuum\Anaconda3\lib\site-packages\multiprocess\popen_spawn_win32.py", line 68, in __init__
context.set_spawning_popen(None)
OSError: [Errno 22] Invalid argument
如何解决问题？
更新
通过将行与cassandra.cluster.ResultSet个对象分离，解决了这个问题。