用Python执行网页脚本

时间:2014-09-16 03:56:29

标签: javascript python windows web-scraping pyqt4

我正在尝试抓一个充满javascript的页面。网址是:

http://www.nasdaqomxnordic.com/index/index_info?Instrument=DK0016268840

我使用以下代码来获取数据。显然这段代码应该处理javascript并返回一个完整的html文件,但事实并非如此。可能存在时间问题,如果是这样,我不太明白你在哪里推迟了proram以允许完整的html。

import sys
from PyQt4.QtGui import *
from PyQt4.QtCore import *
from PyQt4.QtWebKit import *

class Render(QWebPage):
    def __init__(self, url):
        self.app = QApplication(sys.argv)
        QWebPage.__init__(self)
        self.loadFinished.connect(self._loadFinished)
        self.mainFrame().load(QUrl(url))
        self.app.exec_()

    def _loadFinished(self, result):
        self.frame = self.mainFrame()
        self.app.quit()

def getHtml(str_url):
    r_html = Render(str_url)
    html = r_html.frame.toHtml()
    return html

str_url = 'http://www.nasdaqomxnordic.com/index/index_info?Instrument=DK0016268840'
str_html = getHtml(str_url)
print(str_html)

如果您从Web浏览器请求页面源,这将为您提供html。当然,页面上有更多内容,因为所有表都填充了javascript函数。使用Firebug,我正在寻找的表的ID是" sharesInIndexTable。我真正想要抓住的项目是每个公司名下的链接 - 但是访问整个表格来解析beautifulsoup会更好。从这张表中,人们应该能够找到“#Carlsberg" (作为一个潜在的测试,看看AJAX是否满载)。然后我试图找出解析DOM的东西,我尝试了这个:

import sys
from PyQt4 import QtGui, QtCore, QtWebKit

class Sp():
    def printit(self):        
        data = self.webView.page().mainFrame().findFirstElement('id="sharesInIndexTable"')
    print(data)       

def main(self):
    self.webView = QtWebKit.QWebView()
    self.webView.load(QtCore.QUrl("http://www.nasdaqomxnordic.com/index/index_info?Instrument=DK0016268840"))
    QtCore.QObject.connect(self.webView,QtCore.SIGNAL("loadFinished(bool)"),self.printit)

    app = QtGui.QApplication(sys.argv)
    s = Sp()
    s.main()
    sys.exit(app.exec_())

我从中获得的是PyQt4.QtWebkit.QWebElement对象,位于0x03294830(您的结果可能会有所不同)。无论我试图将这个地址变成可读格式都失败了。这段代码似乎也运行了两次。 然后我尝试了这个(有点适合我的需要):

#!/usr/bin/python

# These lines will get us the modules we need.
from PyQt4.QtCore import QUrl, SIGNAL
from PyQt4.QtGui import QApplication
from PyQt4.QtWebKit import QWebPage, QWebView

class Scrape(QApplication):
  def __init__(self):
  # only work with ["test"] as it normally takes an array of args
  super(Scrape, self).__init__(["test"])
  # Create a QWebView instance and store it.
  self.webView = QWebView()
  # Connect our searchform method to the searchform signal of this new
  # QWebView.
  self.webView.loadFinished.connect(self.searchForm)

  def load(self, url):
  # In the __init__ we stored a QWebView instance into self.webView so
  # we can load a url into it. It needs a QUrl instance though.
  self.webView.load(QUrl(url))

  def searchForm(self):
  # We landed here because the load is finished. Now, load the root document
  # element. It'll be a QWebElement instance. QWebElement is a QT4.6
  # addition and it allows easier DOM interaction.
  documentElement = self.webView.page().currentFrame().documentElement()
  # Let's find the search input element.
  print("Begin search")
  inputSearch = documentElement.findFirst('id="sharesInIndexTable"')
  # Disconnect ourselves from the signal.
  self.webView.loadFinished.disconnect(self.searchForm)
  print("End search")
  # And connect the next function.
  self.webView.loadFinished.connect(self.searchResults)

  def searchResults(self):
  # As seen above, first grab the root document element and then load all g
  # classed list items.
  print("Begin results")
  results = self.webView.page().currentFrame().documentElement().findAll('td')

  # Change the resulting QWebElementCollection into a list so we can easily
  # iterate over it.
  for e in results.toList():
    # Just print the results.
    print(e.tohtml())
  # We are inside a QT application and need to terminate that properly.
  print("End results")
  self.exit()

# Instantiate our class.
my_scrape = Scrape()
# Load the Google homepage.
my_scrape.load('http://www.nasdaqomxnordic.com/index/index_info?Instrument=DK0016268840')
# Start the QT event loop.
my_scrape.exec_()

我添加了print()语句来确定程序是否完全执行了命令。这根本不产生任何东西(打印陈述除外)

检查源页面,我可以找到填充表格的脚本:

var sharesInIndex = { 
load: function () {
var index = webCore.getInstrument();
var nLabel = 'nm';
var hiddenAttributes = ",lists,tp,hlp,isin,note,";
var xslt = "inst_table.xsl";
var options =  ",noflag,sectoridicon,";
var xpath = "//index//instruments";
// Check if swedish r�nteindex or Icelandic r�nteindex.
if ( index.indexOf('OMFSE') >= 0 || webCore.getInstrument().indexOf('IS00000') >= 0 ) {
    hiddenAttributes += ",to,sectid,";
    nLabel = 'fnm';
}

// Check if weights index present (typeof)
var shbindex = ",SE0002834820,SE0002834838,SE0002834846,SE0002977397,";
if ( shbindex.indexOf(index) >= 0 ) {
    xslt = "inst_table_windex.xsl";
    options += "windex,";
    xpath = "//index";
}

var query = webCore.createQuery(
    Utils.Constants.marketAction.getIndexInstrument, {
    inst__a: "0,1,2,5,37,4,20,21,23,24,33,34,97,129,98,10", /* 87,*/
    Instrument: index,
    XPath: xpath,
    ext_xslt: xslt,
    ext_xslt_lang: currentLanguage,
    ext_xslt_tableId: "sharesInIndexTable",
    ext_xslt_hiddenattrs: hiddenAttributes,
    ext_xslt_notlabel: nLabel,
    ext_xslt_options: options
  });

  $("#sharesInIndexOutput").empty().loading("/static/nordic/css/img/loading.gif");
  $("#sharesInIndexOutput").load( webCore.getProxyURL('prod'), {xmlquery: query},
    function( responseText, textStatus, XMLHttpRequest) {
      $("#sharesInIndexTable").tablesorter({
        widgets: ['zebra'], 
        textExtraction: 'complex', 
        numberFormat: Utils.Constants.numberFormat[currentLanguage]
        });
      $("#sharesInIndexTable a").each( function() {
        $(this).attr("href",webCore.getURL( Utils.Constants.pages.micrositeShare, $(this).attr('name') ));
      });
    });
  }
};

$(document).ready( sharesInIndex.load );

我知道有一个" execute_script"命令,但我不知道你是如何实现它的,也没有找到任何适合这个的例子 - 如果结果是Json或HTML或纯文本,我不介意。我相信这就是答案所在:(1)加载页面,(2)运行页面脚本,(3)获取结果,(4)解析/打印/保存结果......

我最好有一个无头解决方案,如果有一个甚至Phantomjs在Windows上不是完全无头,因为它弹出一个cmd窗口(我知道你可以通过Linux上的虚拟显示摆脱这一点 - 但是& #39; s不是环境)。另外,只是告诉我:哦,你必须轮询它以查看数据是否已加载然后你检索它不是很有帮助:你能告诉我(即使在伪代码中)如何进行民意调查,更重要的是大致在哪里进行民意调查程序会进行轮询(这就是我发布完全可执行代码的原因 - 如果其他人遇到同样的问题,他们应该有一个完整且易于理解的答案)。

我的最新尝试(1 - 插入延迟以允许加载AJAX)

import sys  
from PyQt4.QtGui import *  
from PyQt4.QtCore import *  
from PyQt4.QtWebKit import *
import time

class Render(QWebPage):  
  def __init__(self, url):  
    self.app = QApplication(sys.argv)  
    QWebPage.__init__(self)
    self.mainFrame().load(QUrl(url))  
    self.loadFinished.connect(self._loadFinished)   
    self.app.exec_()  

  def _loadFinished(self, result):
    time.sleep(5)
    self.frame = self.currentFrame()  
    self.app.quit()  

url = 'http://www.nasdaqomxnordic.com/index/index_info?Instrument=DK0016268840'  
r = Render(url)  
html = r.frame.toHtml()
print(html)

(2 - 对源页面中的已知项目进行轮询) - 使用firebug检查器找到的项目 - 可能是findFirst参数的语法错误。

import sys  
from PyQt4.QtGui import *  
from PyQt4.QtCore import *  
from PyQt4.QtWebKit import *
import time

class Render(QWebPage):  
  def __init__(self, url):  
    self.app = QApplication(sys.argv)  
    QWebPage.__init__(self)
    self.mainFrame().load(QUrl(url))  
    self.loadFinished.connect(self._loadFinished)   
    self.app.exec_()  

  def _loadFinished(self, result):
    counter = 0
    while(self.mainFrame().documentElement().findFirst("id=sharesInIndexTable")):
      counter+=1
      print(counter)
      time.sleep(1)    
    self.frame = self.currentFrame()  
    self.app.quit()  

url = 'http://www.nasdaqomxnordic.com/index/index_info?Instrument=DK0016268840'  
r = Render(url)  
html = r.frame.toHtml()
print(html)

这最后一个有一个计数器,以显示是否发生了什么。它永远计数,必须用ctrl-c停止。

(3 - 使用WebElement的另一个变体)

import sys  
from PyQt4.QtGui import *  
from PyQt4.QtCore import *  
from PyQt4.QtWebKit import *
import time

class Render(QWebPage):  
  def __init__(self, url):  
    self.app = QApplication(sys.argv)  
    QWebPage.__init__(self)
    self.mainFrame().load(QUrl(url))  
    self.loadFinished.connect(self._loadFinished)   
    self.app.exec_()  

  def _loadFinished(self, result):
    table = self.mainFrame().documentElement().findFirst("id=sharesInIndexTable")
    print(table)    #prints: <PyQt4.QtWebKit.QWebElement object at 0x0319FB0>
    print("Attributes:")
    print(table.attributeNames())    #prints: [] i.e. None 
    print("Classes: ")
    print(table.classes())      #prints: [] i.e. None
    print("InnerXML: " + table.toInnerXml())   #prints nothing
    print("OuterXML: " + table.toOuterXml())   #prints nothing
    print("Done")
    self.frame = self.currentFrame()  
    self.app.quit()  

url = 'http://www.nasdaqomxnordic.com/index/index_info?Instrument=DK0016268840'  
r = Render(url)  
html = r.frame.toHtml()

这个没有成功。我把代码打印出来。那里显然有一个对象,但我看不出里面是什么。

1 个答案:

答案 0 :(得分:1)

我知道这已经很久了,但这个答案适用于类似情况的后来访客

我遇到了类似的问题,我尝试了各种各样的事情,例如等待来自QWebPage和QWebFrame的loadFinished的信号,等待来自QWebFrame.intialLayoutCompleted()等的信号。

最终对我有用的是:

我只是在普通浏览器中呈现页面。检查了因为javascript而没有在PyQt中呈现的元素,得到了该元素的id(如果它是一个div,它又包含多个元素,表等,然后得到div id)。现在在yourPage.loadFinished函数的python代码中调用yourFrame.evaluateJavaScript(“document.getElementById(element_id_retrieved_earlier')”)。

这将等待检索id,然后等待嵌入式脚本执行。