我似乎错过了一些非常简单的事情。我想做的就是使用;
作为
CSV导出器中的分隔符,而不是,
。
我知道CSV导出器将kwargs传递给csv writer,但我似乎无法 弄清楚如何通过这个分隔符。
我这样称呼我的蜘蛛:
scrapy crawl spidername --set FEED_URI=output.csv --set FEED_FORMAT=csv
答案 0 :(得分:12)
在contrib/feedexport.py
,
class FeedExporter(object):
...
def open_spider(self, spider):
file = TemporaryFile(prefix='feed-')
exp = self._get_exporter(file) # <-- this is where the exporter is instantiated
exp.start_exporting()
self.slots[spider] = SpiderSlot(file, exp)
def _get_exporter(self, *a, **kw):
return self.exporters[self.format](*a, **kw) # <-- not passed in :(
你需要自己动手,这是一个例子:
from scrapy.conf import settings
from scrapy.contrib.exporter import CsvItemExporter
class CsvOptionRespectingItemExporter(CsvItemExporter):
def __init__(self, *args, **kwargs):
delimiter = settings.get('CSV_DELIMITER', ',')
kwargs['delimiter'] = delimiter
super(CsvOptionRespectingItemExporter, self).__init__(*args, **kwargs)
在抓取工具目录的settings.py
文件中,添加以下内容:
FEED_EXPORTERS = {
'csv': 'importable.path.to.CsvOptionRespectingItemExporter',
}
现在,您可以按如下方式执行蜘蛛:
scrapy crawl spidername --set FEED_URI=output.csv --set FEED_FORMAT=csv --set CSV_DELIMITER=';'
HTH。
答案 1 :(得分:0)
我也试过这也可以工作:
第1步:将C:\Python27\Lib\site-packages\scrapy\exporters.py
第21行修改为
__all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter',
'CsvItemExporter', 'TxtItemExporter', 'XmlItemExporter',
'JsonLinesItemExporter', 'JsonItemExporter', 'MarshalItemExporter']
这会将'TxtItemExporter'
添加到原始__all__
列表中。
第2步:将名为TxtItemExporter的新类添加到C:\Python27\Lib\site-packages\scrapy\exporters.py
:
class TxtItemExporter(BaseItemExporter):
def __init__(self, file, include_headers_line=True, join_multivalued=',', **kwargs):
self._configure(kwargs, dont_fail=True)
if not self.encoding:
self.encoding = 'utf-8'
self.include_headers_line = include_headers_line
self.stream = io.TextIOWrapper(
file,
line_buffering=False,
write_through=True,
encoding=self.encoding
) if six.PY3 else file
self.csv_writer = csv.writer(self.stream, delimiter='\t', **kwargs)
self._headers_not_written = True
self._join_multivalued = join_multivalued
def serialize_field(self, field, name, value):
serializer = field.get('serializer', self._join_if_needed)
return serializer(value)
def _join_if_needed(self, value):
if isinstance(value, (list, tuple)):
try:
return self._join_multivalued.join(value)
except TypeError: # list in value may not contain strings
pass
return value
def export_item(self, item):
if self._headers_not_written:
self._headers_not_written = False
self._write_headers_and_set_fields_to_export(item)
fields = self._get_serialized_fields(item, default_value='',
include_empty=True)
values = list(self._build_row(x for _, x in fields))
self.csv_writer.writerow(values)
def _build_row(self, values):
for s in values:
try:
yield to_native_str(s, self.encoding)
except TypeError:
yield s
def _write_headers_and_set_fields_to_export(self, item):
if self.include_headers_line:
if not self.fields_to_export:
if isinstance(item, dict):
# for dicts try using fields of the first item
self.fields_to_export = list(item.keys())
else:
# use fields declared in Item
self.fields_to_export = list(item.fields.keys())
row = list(self._build_row(self.fields_to_export))
self.csv_writer.writerow(row)
新类从CsvItemExporter复制,只将delimiter='\t'
添加到csv.writer()
第3步:将以下设置添加到settings.py
FEED_EXPORTERS = {
'txt': 'scrapy.contrib.exporter.TxtItemExporter',
}
FEED_FORMAT = 'txt'
FEED_URI = "your_output_file.txt"
第4步:运行scrapy crawl your_spider
然后您可以在蜘蛛项目目录中找到输出txt。
答案 2 :(得分:0)
scraper / exporters.py
from scrapy.exporters import CsvItemExporter
from scraper.settings import CSV_SEP
class CsvCustomSeperator(CsvItemExporter):
def __init__(self, *args, **kwargs):
kwargs['encoding'] = 'utf-8'
kwargs['delimiter'] = CSV_SEP
super(CsvCustomSeperator, self).__init__(*args, **kwargs)
scraper / settings.py
CSV_SEP = '|'
FEED_EXPORTERS = {
'csv': 'scraper.exporters.CsvCustomSeperator'
}
在终端机中
$ scrapy crawl spider -o file.csv -s CSV_SEP=<delimiter>