Question

我目前正在使用selenium python来获取linkedin数据。我可以通过各种网页解析并抓取数据，但由于Unicode错误，该过程在前几页后被中断。这是我的代码：

from selenium import webdriver
from time import sleep

driver = webdriver.Firefox()
driver.get('https://www.linkedin.com/jobs/search?locationId=sg%3A0&f_TP=1%2C2&orig=FCTD&trk=jobs_jserp_posted_one_week')

result = []
while True:
    while True:
        try:
            sleep(1)
            result +=[i.text for i in driver.find_elements_by_class_name('job-title-text')]
        except:
            sleep(5)
        else:
            break
    try:
        for i in range(50):
            nextbutton = driver.find_element_by_class_name('next-btn')
            nextbutton.click()
    except:
        break

with open('jobtitles.csv', 'w') as f:
f.write('\n'.join(i for i in result).encode('utf-8').decode('utf-8'))

Answer 1

您可以使用UnicodeWriter（来自Python文档）：

import codecs
import cStringIO
import csv
from time import sleep

from selenium import webdriver


class UnicodeWriter:
    """
    A CSV writer which will write rows to CSV file "f",
    which is encoded in the given encoding.
    """

    def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
        # Redirect output to a queue
        self.queue = cStringIO.StringIO()
        self.writer = csv.writer(self.queue, dialect=dialect, **kwds)
        self.stream = f
        self.encoder = codecs.getincrementalencoder(encoding)()

    def writerow(self, row):
        self.writer.writerow([s.encode("utf-8") for s in row])
        # Fetch UTF-8 output from the queue ...
        data = self.queue.getvalue()
        data = data.decode("utf-8")
        # ... and reencode it into the target encoding
        data = self.encoder.encode(data)
        # write to the target stream
        self.stream.write(data)
        # empty queue
        self.queue.truncate(0)

    def writerows(self, rows):
        for row in rows:
            self.writerow(row)


driver = webdriver.Firefox()
driver.get('https://www.linkedin.com/jobs/search?locationId=sg%3A0&f_TP=1%2C2&orig=FCTD&trk=jobs_jserp_posted_one_week')

result = []
while True:
    while True:
        try:
            sleep(1)
            result +=[i.text for i in driver.find_elements_by_class_name('job-title-text')]
        except:
            sleep(5)
        else:
            break
    try:
        for i in range(50):
            nextbutton = driver.find_element_by_class_name('next-btn')
            nextbutton.click()
    except:
        break


with open('jobtitles.csv', 'w') as f:
    doc = UnicodeWriter(f)
    doc.writerows(result)

Answer 2

这是一种不正确的编码...你声称字节流是由UTF-8编码的，但事实并非如此在UTF-8实现的引用位置只允许ascii字符（0-127），因此UTF-8解码失败...我没有看到你的代码UTF-8解码失败的方式和时间，所以你应该自己追踪确切的位置检查带有type（）的变量，还请注意python 2和3在这个区域有差异

Answer 3

import sys 
reload(sys)
sys.setdefaultencoding("utf-8")
print sys.getdefaultencoding()

将其添加到代码顶部。

另外，您可能需要预处理代码以替换一些非英语单词

        words=word_tokenize(content)
        # print words
        word=[]
        for w in words:
            w= re.sub(r'[^\w\s]', '',w)
            w =re.sub("[^A-Za-z]+"," ",w,flags=re.MULTILINE)
            w =w .strip("\t\n\r")
            word.append(w)
        words=word
        # print words
        stop_words = set(stopwords.words('english'))
        filteredword = [w for w in words if not w in stop_words and 3 < len(w)]
        # print filteredword
        words=" ".join(filteredword)

UnicodeEncodeError：'ascii'编解码器无法对位置448中的字符u'\ u2013'进行编码：序数不在范围内（128）

3 个答案: