Question

我正在尝试使用来自我正在抓取的多个网址的数据创建一个数据框。代码工作但我无法递归地将数据存储在一个DataFrame中。 DataFrame（称为帧）每次都被新的url数据替换，而不是将新数据连接到同一帧。谢谢，我非常感谢你的帮助！

import urllib
import re
import json
import pandas
import pylab
import numpy
import matplotlib.pyplot
from pandas import *
from pylab import *
from threading import Thread
import sqlite3

urls = ['http://www.ratemyprofessors.com/ShowRatings.jsp?tid=1176131' , 'http://www.ratemyprofessors.com/ShowRatings.jsp?tid=795226', 'http://www.ratemyprofessors.com/ShowRatings.jsp?tid=1176131' , 'http://www.ratemyprofessors.com/ShowRatings.jsp?tid=1807944', 'http://www.ratemyprofessors.com/ShowRatings.jsp?tid=277459' , 'http://www.ratemyprofessors.com/ShowRatings.jsp?tid=1076779' , 'http://www.ratemyprofessors.com/ShowRatings.jsp?tid=971546']

i=0
regex = '<p class="commentText">(.+?)</p>'
regex2 = '<strong>Easiness</strong><span>(.+?)</span></p>'
regex3 = 'Helpfulness</strong><span>(.+?)</span></p>'
regex4 = 'Clarity</strong><span>(.+?)</span></p>'
regex5 = 'Rater Interest</strong><span>(.+?)</span></p>'
regex6 = '<div class="date">(.+?)</div>'
regex7 = '<div class="class"><p style="word-wrap:break-word;">(.+?)</p>'
regex8 = '<meta name="prof_name" content="(.+?)"/>'

pattern = re.compile(regex)
easiness = re.compile(regex2)
helpfulness = re.compile(regex3)
clarity = re.compile(regex4)
interest = re.compile(regex5)
date = re.compile(regex6)
mathclass = re.compile(regex7)
prof_name = re.compile(regex8)

while i < len(urls):
    htmlfile = urllib.urlopen(urls[i])
    htmltext = htmlfile.read()
    content = re.findall(pattern,htmltext)
    Easiness = re.findall(easiness,htmltext)
    Helpfulness = re.findall(helpfulness, htmltext)
    Clarity = re.findall(clarity, htmltext)
    Interest = re.findall(interest, htmltext)
    Date = re.findall(date, htmltext)
    Class = re.findall(mathclass, htmltext)
    PROFNAME=re.findall(prof_name, htmltext)
    i+=1

    frame = DataFrame({'Comments': content, 'Easiness': Easiness, 'Helpfulness': Helpfulness, 
    'Clarity': Clarity, 'Rater Interest': Interest, 'Class': Class,
    'Date': Date[1:len(Date)], 'Professor': PROFNAME[0]})

    print frame

Answer 1

使用pd.concat：

frames = []

while i < len(urls):
    htmlfile = urllib.urlopen(urls[i])
    htmltext = htmlfile.read()
    content = re.findall(pattern,htmltext)
    Easiness = re.findall(easiness,htmltext)
    Helpfulness = re.findall(helpfulness, htmltext)
    Clarity = re.findall(clarity, htmltext)
    Interest = re.findall(interest, htmltext)
    Date = re.findall(date, htmltext)
    Class = re.findall(mathclass, htmltext)
    PROFNAME=re.findall(prof_name, htmltext)
    i+=1

    frames.append(DataFrame({'Comments': content, 'Easiness': Easiness, 'Helpfulness': Helpfulness, 
    'Clarity': Clarity, 'Rater Interest': Interest, 'Class': Class,
    'Date': Date[1:len(Date)], 'Professor': PROFNAME[0]}))

pd.concat(frames)

Answer 2

您将在循环的每次迭代中覆盖您的帧。正如Phillip Cloud建议的那样，您可以创建每个循环附加的帧列表。我以不同的方式简化了您的代码，但我认为这可以为您提供所需的内容。

import urllib
import re
import pandas as pd

urls = ['http://www.ratemyprofessors.com/ShowRatings.jsp?tid=1176131',
        'http://www.ratemyprofessors.com/ShowRatings.jsp?tid=795226',
        'http://www.ratemyprofessors.com/ShowRatings.jsp?tid=1176131',
        'http://www.ratemyprofessors.com/ShowRatings.jsp?tid=1807944',
        'http://www.ratemyprofessors.com/ShowRatings.jsp?tid=277459', 
        'http://www.ratemyprofessors.com/ShowRatings.jsp?tid=1076779', 
        'http://www.ratemyprofessors.com/ShowRatings.jsp?tid=971546']

regex = {'pattern' : re.compile('<p class="commentText">(.+?)</p>'),
        'easiness' : re.compile('<strong>Easiness</strong><span>(.+?)</span></p>'),
        'helpfulness' : re.compile('Helpfulness</strong><span>(.+?)</span></p>'),
        'clarity' : re.compile('Clarity</strong><span>(.+?)</span></p>'),
        'interest' : re.compile('Rater Interest</strong><span>(.+?)</span></p>'),
        'date' : re.compile('<div class="date">(.+?)</div>'),
        'mathclass' : re.compile('<div class="class"><p style="word-wrap:break-word;">(.+?)</p>'),
        'prof_name' : re.compile('<meta name="prof_name" content="(.+?)"/>')}

# Make a dictionary with empty lists using the same keys
d = {}
for k in regex.keys():
    d[k] = []

# Now fill those lists
for url in urls:
    htmlfile = urllib.urlopen(url)
    htmltext = htmlfile.read()
    for k, v in regex.iteritems():
        d[k].append(re.findall(v, htmltext))
frame = pd.DataFrame(d) # Dump the dict into a DataFrame
print frame

在一个DataFrame内递归连接URLS中的数据

2 个答案: