Python优化脚本?

时间:2018-08-27 06:25:22

标签: python python-3.x selenium optimization phantomjs

我为Pinterest制作了一个刮板,该刮板可以正常运行,但在规格良好的硬件上运行速度非常慢。

逻辑

  • 用户通过参数输入引脚的url,csv文件的文件名
  • 启动硒(使用最新的phantomjs [ghost驱动程序]
  • 抓取数据并存储在数组中
  • Selenium脚本执行无限滚动并单击“查看更多”按钮以显示有关引脚的更多数据
  • 将数组保存到pandas数据框中并生成csv

问题 该脚本可以正常运行,但速度很慢,有时会冻结。

代码-精度

import time, argparse, pandas, datetime
from selenium import webdriver

# global lists for data storage
image_links = []
image_titles = []
image_descriptions = []
pin_links = []
pin_Board_Names = []
pin_Author_Names = []
scrape_timestamps = []

代码-makeCSV

def makeCsv(filename):
    # csv head title along with data list for each
    data={"Time Stamps":scrape_timestamps, "Author Names":pin_Author_Names, "Pins Board":pin_Board_Names, "Pin Links":pin_links, "Image URLs":image_links, "Image Titles":image_titles, "Image Descriptions":image_descriptions}

    # create pandas dataframe from above dictionary
    dataframe = pandas.DataFrame.from_dict(data, orient='index')

    # transpose the dataframe - lists data as rows instead of columns in csv
    dataframe = dataframe.transpose()

    # save csv file
    dataframe.to_csv(filename, sep=',')

代码-抓取/抓取

def grab(url, filename):
    driver = webdriver.PhantomJS(executable_path='phantomjs.exe')
    print("\n\nGhost Driver Invoked")
    driver.implicitly_wait(10) # if element not found, wait for (seconds) before next operation
    driver.get(url) # grab the url

    time.sleep(3) # seconds

    # Scrolling till the end of page
    print("Started Scroling ...")
    last_height = driver.execute_script("return document.body.scrollHeight")
    i = 0
    while True:
        if i == 2:
            break
        else:
            driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
            time.sleep(5) # seconds
            new_height = driver.execute_script("return document.body.scrollHeight")
            print("Scrolled: " + str(new_height))
            if new_height == last_height:
                break
            last_height = new_height
        i += 1

    time.sleep(3) # seconds

    # Get all pins , number of pins collected
    total_pins = []
    try:
        total_pins = driver.find_elements_by_class_name("Grid__Item")
    except:
        print("Unable to load pins")
    print("Total Pins: " + str(len(total_pins)))

    # get number of 'see more' buttons collected - for error checking
    moreButtons = driver.find_elements_by_xpath('//button[@data-test-id="seemoretoggle"]')
    print("Dynamic Elements: " + str(len(moreButtons)))
    print("Display: Dynamic Elements ...")

    # clicking all 'See More' buttons
    i = 0
    while i <= (len(moreButtons) - 1):
        moreButtons[i].click()
        print("Clicked: " + str(i+1))
        i += 1

    # Pin Image URLs
    i = 1
    while i < len(total_pins):
        try:

            temp_xpath = "/html/body/div/div[1]/div[1]/div/div/div/div/div[1]/div/div/div/div[" + str(i) + "]/div/div/div[1]/a/img"
            temp = driver.find_element_by_xpath(temp_xpath).get_attribute("src")
            image_links.append(temp)
            scrape_timestamps.append(str(datetime.datetime.now())) # save timestamps of scrape
            i += 1
        except:
            i += 1
            pass
    print("Total Images: " + str(i))

    # Image Titles
    imageTitle_Elements = driver.find_elements_by_class_name("PinAttributionTitle__title")
    i = 1
    for imageTitle_Element in imageTitle_Elements:
        image_titles.append(imageTitle_Element.text)
        i += 1
    print("Total Titles: " + str(i))

    # Image Descriptions
    imageDesc_Elements = driver.find_elements_by_class_name("PinDescription__desc")
    i = 1
    for imageDesc_Element in imageDesc_Elements:
        image_descriptions.append(imageDesc_Element.text)
        i += 1
    print("Total Descriptions: " + str(i))

    # Pin Page Link
    i = 1
    while i < len(total_pins):
        temp_xpath = "/html/body/div/div[1]/div[1]/div/div/div/div/div[1]/div/div/div/div[" + str(i) + "]/div/div/div[1]/a"
        temp = driver.find_element_by_xpath(temp_xpath).get_attribute("href")
        print("Pin Link:" + str(i))
        pin_links.append(temp)
        i += 1
    print("Total PinLinks: " + str(i))

    # Pin Board Names
    print("Extracting Board Names ... ")
    i = 1
    successful = False # for checking success of try | else not working
    while i <= len(total_pins):
        try:
            temp_xpath = "/html/body/div[1]/div[1]/div[1]/div/div/div/div/div[1]/div/div/div/div[" + str(i) + "]/div/div/div[2]/div[2]/h4/a[1]"
            temp = driver.find_element_by_xpath(temp_xpath)
            pin_Board_Names.append(temp.text)
            print("Board_No: " + str(i) + " > " + temp.text)
            successful = True
        except:
            temp_xpath = "/html/body/div[1]/div[1]/div[1]/div/div/div/div/div[1]/div/div/div/div[" + str(i) + "]/div/div/div[2]/div/h4/a[1]"
            temp = driver.find_element_by_xpath(temp_xpath)
            pin_Board_Names.append(temp.text)
            print("Board_No: " + str(i) + " > " + temp.text)
            successful = True
        if successful == False:
            print("Board_No: " + str(i) + " not found!")
        i += 1

    # Pin Author Names
    print("Extracting Author Names ... ")
    i = 1
    successful = False
    while i <= len(total_pins):
        try:
            temp_xpath = "/html/body/div[1]/div[1]/div[1]/div/div/div/div/div[1]/div/div/div/div[" + str(i) + "]/div/div/div[2]/div[2]/h4/a[2]"
            temp = driver.find_element_by_xpath(temp_xpath)
            pin_Author_Names.append(temp.text)
            print("Author_No: " + str(i) + " > " + temp.text)
            successful = True
        except:
            temp_xpath = "/html/body/div[1]/div[1]/div[1]/div/div/div/div/div[1]/div/div/div/div[" + str(i) + "]/div/div/div[2]/div/h4/a[2]"
            temp = driver.find_element_by_xpath(temp_xpath)
            pin_Author_Names.append(temp.text)
            print("Author_No: " + str(i) + " > " + temp.text)
            successful = True
        if successful == False:
            print("Author_No: " + str(i) + " not found!")
        i += 1

    # generate csv from collected items
    makeCsv(filename)

    # quit driver
    driver.quit()

代码-主

def main():
    # start argument parser
    parser = argparse.ArgumentParser()

    # custom arguments
    parser.add_argument("-url", "--url", help="Enter the URL")
    parser.add_argument("-fname", "--fname", help="Enter Filename")

    # parse arguments
    args = parser.parse_args()

    # all web driver functions happen here:
    grab(args.url, args.fname)

if __name__ == "__main__":
    main()

输出缓慢

>>> C:\Users\da74\Desktop\sites\pinscraper\ps>python -W ignore scrape.py --url https://www.pinterest.com/vijayakumargana/classic-kuala-lumpur/ --fname classic_kl.csv


Ghost Driver Invoked
Started Scroling ...
Scrolled: 11124
Scrolled: 18388
Total Pins: 126
Dynamic Elements: 125
Display: Dynamic Elements ...
Clicked: 1
Clicked: 2
Clicked: 3
Clicked: 4
Clicked: 5
Clicked: 6
Clicked: 7
Clicked: 8
Clicked: 9
Clicked: 10
Clicked: 11
Clicked: 12
Clicked: 13
Clicked: 14
Clicked: 15
Clicked: 16
Clicked: 17
Clicked: 18
Clicked: 19
Clicked: 20
Clicked: 21
Clicked: 22
Clicked: 23
Clicked: 24
Clicked: 25
Clicked: 26
Clicked: 27
Clicked: 28
Clicked: 29
Clicked: 30
Clicked: 31
Clicked: 32
Clicked: 33
Clicked: 34
Clicked: 35
Clicked: 36
Clicked: 37
Clicked: 38
Clicked: 39
Clicked: 40
Clicked: 41
Clicked: 42
Clicked: 43
Clicked: 44
Clicked: 45
Clicked: 46
Clicked: 47
Clicked: 48
Clicked: 49
Clicked: 50
Clicked: 51
Clicked: 52
Clicked: 53
Clicked: 54
Clicked: 55
Clicked: 56
Clicked: 57
Clicked: 58
Clicked: 59
Clicked: 60
Clicked: 61
Clicked: 62
Clicked: 63
Clicked: 64
Clicked: 65
Clicked: 66
Clicked: 67
Clicked: 68
Clicked: 69
Clicked: 70
Clicked: 71
Clicked: 72
Clicked: 73
Clicked: 74
Clicked: 75
Clicked: 76
Clicked: 77
Clicked: 78
Clicked: 79
Clicked: 80
Clicked: 81
Clicked: 82
Clicked: 83
Clicked: 84
Clicked: 85
Clicked: 86
Clicked: 87
Clicked: 88
Clicked: 89
Clicked: 90
Clicked: 91
Clicked: 92
Clicked: 93
Clicked: 94
Clicked: 95
Clicked: 96
Clicked: 97
Clicked: 98
Clicked: 99
Clicked: 100
Clicked: 101
Clicked: 102
Clicked: 103
Clicked: 104
Clicked: 105
Clicked: 106
Clicked: 107
Clicked: 108
Clicked: 109
Clicked: 110
Clicked: 111
Clicked: 112
Clicked: 113
Clicked: 114
Clicked: 115
Clicked: 116
Clicked: 117
Clicked: 118
Clicked: 119
Clicked: 120
Clicked: 121
Clicked: 122
Clicked: 123
Clicked: 124
Clicked: 125

单击所有js按钮后,脚本冻结。即使单击按钮,它也会减慢很多速度。使用Python3.6,Windows 10 x64专业版,8GB RAM,2TB HDD。

如果有太多事情或更大页面,谁能帮助并提出优化它的方法并阻止它冻结? 谢谢。

0 个答案:

没有答案