Question

import scrapy
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
from scrapy.selector import HtmlXPathSelector
import os
from Erowid.items import ErowidItem
import codecs

class ExperiencesSpider(CrawlSpider):
    name = "experiences"
    allowed_domains = ["www.erowid.org"]
    start_urls = ['https://www.erowid.org/experiences/exp_list.shtml']
    rules = [ 
        Rule(LinkExtractor(allow =('subs/exp_[a-zA-Z]+.shtml')), follow = True), 
        Rule(LinkExtractor(allow=r'/experiences/exp\.php\?ID=\d+$'),callback='parse_item', follow = True)
    ]
    def parse_item(self, response):
        selectors = response.css('div')
        for selector in selectors:
            experience = ErowidItem()
            experience['Author'] = selector.xpath('//div[@class="author"]/a/text()').extract()
            experience['Title'] = selector.xpath('//div[@class="title"]/text()').extract()
            experience['Substance'] = selector.xpath('//div[@class="substance"]/text()').extract()
            experience['Text'] = selector.xpath("//div[@class = 'report-text-surround']/text()").extract()
            experience['Title'] =  str(experience['Title']).replace('\\' , "")
            experience['Title'] =  str(experience['Title']).replace('?' , "")

            directory = os.path.join('Erowid/archive/',experience['Substance'][0].strip().lower())
            filename = os.path.join(directory,experience['Title'][0]+'.txt')

            if not os.path.exists(directory):
                os.makedirs(directory)

            with codecs.open(filename, encoding = 'utf-8', mode= 'wb') as fid:
                for symbols in experience['Text']:
                    fid.write(symbols) 
        yield experience

我正在尝试刮掉Erowid并且到目前为止这个代码是为了创建一个目录并主要根据经验和文本的名称编写文件。（根据经验，我指的是我正在废除Erowid的信息。）

问题在于，某些体验的名称字符不能写为文件，因为它们包含无法在windows filenames中使用的保留字符。

我正在尝试删除所有这些

的保留字符

以下保留字符：


<（小于）

>（大于）

:（冒号）

"（双引号）

/（正斜杠）

\（反斜杠）

|（垂直条或竖线）

?（问号）

*（星号）

我尝试用线

做点什么

 experience['Title'] =  str(experience['Title']).replace('\\' , "")
 experience['Title'] =  str(experience['Title']).replace('?' , "")

（大多数错误来自“\”和“？”）但我仍然收到错误

IOError: [Errno 22] invalid mode ('wb') or filename: u'Erowid/archive/syrian rue\\Meditative Help?.txt'

或我知道的其他一些文件名错误，因为不应该是问号或反斜杠。

我做错了什么导致了这些错误？

Answer 1

您未能替换特殊字符。试试这个：

filename = os.path.join(directory,experience['Title'][0]+'.txt')
filename = filename.replace('\\' , "").replace('?' , "")

<强>更新

您只想指定合法的文件名。所以我想出了这样一个想法。

directory = os.path.join('Erowid/archive/',experience['Substance'][0].strip().lower())
filename = experience['Substance']+experience['Title'][0]+'.txt'
filename = "".join([i for i in filename if i in string.ascii_letters]) 
#only use ascii letters as file name 
filename = os.path.join(directory, filename)

<强> string.ascii_letters

下面描述的ascii_lowercase和ascii_uppercase常量的串联。该值不依赖于语言环境。

Scrapy IOError：[Errno 22]无效模式（'wb'）或文件名

1 个答案: