Question

我试图抓取一个网址列表：

https://www.jobsbank.gov.sg/ICMSPortal/portlets/JobBankHandler/SearchDetail.do?id=JOB-2016-0010810 https://www.jobsbank.gov.sg/ICMSPortal/portlets/JobBankHandler/SearchDetail.do?id=JOB-2015-0374997

使用此代码，使用selenium webdriver

def jobdesc(link):
    driver = webdriver.Firefox()
    driver.get(link)
    jobname = driver.find_element_by_xpath('//*[@id="wrapper"]/div[1]/div/div[3]/div/div[2]/div[1]/div[2]/div[1]/h3').text
    print "job title" + " " + jobname
    postingdate = driver.find_element_by_xpath('//*[@id="wrapper"]/div[1]/div/div[3]/div/div[2]/div[1]/div[2]/div[1]/table/tbody/tr/td[1]/div/p[1]').text
    print "posting date" + " " + postingdate
    expirydate = driver.find_element_by_xpath('//*[@id="wrapper"]/div[1]/div/div[3]/div/div[2]/div[1]/div[2]/div[1]/table/tbody/tr/td[1]/div/p[2]').text
    print "expiry date" + " " + expirydate
    jobid = driver.find_element_by_xpath('//*[@id="wrapper"]/div[1]/div/div[3]/div/div[2]/div[1]/div[2]/div[1]/table/tbody/tr/td[2]/div/p').text
    print "job id" + " " + jobid
    location = driver.find_element_by_xpath('//*[@id="wrapper"]/div[1]/div/div[3]/div/div[2]/div[2]/div/dl[1]/table/tbody/tr/td/p[1]').text
    print "location" + " " + location
    category = driver.find_element_by_xpath('//*[@id="wrapper"]/div[1]/div/div[3]/div/div[2]/div[2]/div/dl[2]/table/tbody/tr/td/ul/li/span[1]').text
    print "category" + " " + category
    industry = driver.find_element_by_xpath('//*[@id="wrapper"]/div[1]/div/div[3]/div/div[2]/div[2]/div/dl[3]/ul/li/table/tbody/tr/td/ul/li/span[1]').text
    print "industry" + " " + industry
    emptype = driver.find_element_by_xpath('//*[@id="wrapper"]/div[1]/div/div[3]/div/div[2]/div[2]/div/ul[1]/li/table/tbody/tr/td/ul/li/span[1]').text
    print "emptype" + " " + emptype
    workhrs = driver.find_element_by_xpath('//*[@id="wrapper"]/div[1]/div/div[3]/div/div[2]/div[2]/div/ul[2]/li/span[1]').text
    print "workhrs" + " " + workhrs
    shiftpattern = driver.find_element_by_xpath('//*[@id="wrapper"]/div[1]/div/div[3]/div/div[2]/div[2]/div/ul[3]/li/span[1]').text
    print "shiftpattern" + " " + shiftpattern
    salary = driver.find_element_by_xpath('//*[@id="wrapper"]/div[1]/div/div[3]/div/div[2]/div[2]/div/ul[4]/li/table/tbody/tr/td/span[1]').text
    print "salary" + " " + salary
    joblevel = driver.find_element_by_xpath('//*[@id="wrapper"]/div[1]/div/div[3]/div/div[2]/div[2]/div/ul[5]/li/table/tbody/tr/td/ul/li/span[1]').text
    print "joblevel" + " " + joblevel
    yearsofexp = driver.find_element_by_xpath('//*[@id="wrapper"]/div[1]/div/div[3]/div/div[2]/div[2]/div/ul[6]/li').text
    print "yearofexp" + " " + yearsofexp
    vacancyno = driver.find_element_by_xpath('//*[@id="wrapper"]/div[1]/div/div[3]/div/div[2]/div[2]/div/span[2]').text
    print "vacancyno" + " " + vacancyno
    jobadviews = driver.find_element_by_xpath('//*[@id="wrapper"]/div[1]/div/div[3]/div/div[2]/div[2]/div/span[4]').text
    print "jobadviews" + " " + jobadviews
    jobapplicants = driver.find_element_by_xpath('//*[@id="wrapper"]/div[1]/div/div[3]/div/div[2]/div[2]/div/span[6]').text
    print "nojobapplicants" + " " + jobapplicants


    driver.switch_to_frame('frameJobDescription')
    jobdesc = driver.page_source
    jobdesctree = html.fromstring(jobdesc)
    jobdesctrees = jobdesctree.xpath('//*[@id="divMainJobDescription"]//text()')
    print jobdesctrees

    driver.switch_to_default_content()
    driver.switch_to_frame('frameSkillsRequired')
    jobskills = driver.page_source
    jobskillstree = html.fromstring(jobskills)
    jobskillstrees = jobskillstree.xpath('////*[@id="divMainSkillsRequired"]//text()')
    print jobskillstrees

有时当我试图刮一个网址时，Selenium能够检索所有必需的字段。但是，当我将抓取放入一个尝试所有url的循环时，它会在第二个url中快速停止工作，并返回此错误：

> Traceback (most recent call last):   File "E:\jobsbank15012016.py",
> line 123, in <module>
>     main()   File "E:\jobsbank15012016.py", line 120, in main
>     jobdesc(url)   File "E:\jobsbank15012016.py", line 59, in jobdesc
>     postingdate = driver.find_element_by_xpath('//*[@id="wrapper"]/div[1]/div/div[3]/div/div[2]/div[1]/div[2]/div[1]/table/tbody/tr/td[1]/div/p[1]').text
> File
> "C:\Python27\lib\site-packages\selenium\webdriver\remote\webdriver.py",
> line 252, in find_element_by_xpath
>     return self.find_element(by=By.XPATH, value=xpath)   File "C:\Python27\lib\site-packages\selenium\webdriver\remote\webdriver.py",
> line 684, in find_element
>     {'using': by, 'value': value})['value']   File "C:\Python27\lib\site-packages\selenium\webdriver\remote\webdriver.py",
> line 195, in execute
>     self.error_handler.check_response(response)   File "C:\Python27\lib\site-packages\selenium\webdriver\remote\errorhandler.py",
> line 170, in check_response
>     raise exception_class(message, screen, stacktrace) NoSuchElementException: Message: Unable to locate element:
> {"method":"xpath","selector":"//*[@id=\"wrapper\"]/div[1]/div/div[3]/div/div[2]/div[1]/div[2]/div[1]/table/tbody/tr/td[1]/div/p[1]"}
> Stacktrace:
>     at FirefoxDriver.prototype.findElementInternal_ (file:///c:/users/anon/appdata/local/temp/tmprizeob/extensions/fxdriver@googlecode.com/components/driver-component.js:10299)
>     at FirefoxDriver.prototype.findElement (file:///c:/users/anon/appdata/local/temp/tmprizeob/extensions/fxdriver@googlecode.com/components/driver-component.js:10308)
>     at DelayedCommand.prototype.executeInternal_/h (file:///c:/users/anon/appdata/local/temp/tmprizeob/extensions/fxdriver@googlecode.com/components/command-processor.js:12282)
>     at DelayedCommand.prototype.executeInternal_ (file:///c:/users/anon/appdata/local/temp/tmprizeob/extensions/fxdriver@googlecode.com/components/command-processor.js:12287)
>     at DelayedCommand.prototype.execute/< (file:///c:/users/anon/appdata/local/temp/tmprizeob/extensions/fxdriver@googlecode.com/components/command-processor.js:12229)

Answer 1

第一个问题是，您应该通过添加wait为页面加载一些时间。您还应该简化定位器并减少它们对HTML结构的依赖：

gulp.task('scripts', function() {
    return gulp.src(['./js/script.js'])
        .pipe(include())
        .pipe(plumber(
            //{errorHandler: errorScripts};
            function() {
                console.log('There was an issue compiling scripts');
                this.emit('end');
            }
        ))
        .pipe(concat('script-dist.js'))
        //.pipe(stripDebug())
        //.pipe(jshint())
        .pipe(uglify())
        .pipe(gulp.dest('./js/'))
        .pipe(livereload());
});

在这里，对于职位名称，我们使用一个简单的CSS选择器 - 基本上找到一个from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC driver = webdriver.Firefox() driver.get(link) # wait until title is present wait = WebDriverWait(driver, 10) wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "h3.title"))) job_name = driver.find_element_by_css_selector("h3.title").text类的h3元素。这里的XPath替代方案是title。

Selenium无法找到元素SOMETIMES

1 个答案: