Question

我运行此代码：

import argparse

from tqdm import tqdm
from sys import argv
from pprint import pformat

from twisted.internet.task import react
from twisted.web.client import Agent, readBody
from twisted.web.http_headers import Headers
from twisted.internet.task import cooperate
from twisted.internet.defer import gatherResults

import sys
from twisted.python import log

log.startLogging(sys.stdout)

import lxml.html

from geoip import geolite2
import pycountry

from tld import get_tld
import json
import socket

poweredby = ""
server = ""
ip = ""

f = open("errors.txt", "w")


def error(response, url):
    f.write("Error: "+url+"\n") 


def cbRequest(response, url):
    global poweredby, server, ip
    # print 'Response version:', response.version
    # print 'Response code:', response.code
    # print 'Response phrase:', response.phrase
    # print 'Response headers:'
    # print pformat(list(response.headers.getAllRawHeaders()))
    poweredby = response.headers.getRawHeaders("X-Powered-By")[0]
    server = response.headers.getRawHeaders("Server")[0]

    #print poweredby
    #print server

    d = readBody(response)
    d.addCallback(cbBody, url)
    return d


def cbBody(body, ourl):
    global poweredby, server,ip

    #print body
    html_element = lxml.html.fromstring(body)
    generator = html_element.xpath("//meta[@name='generator']/@content")

    ip = socket.gethostbyname(ourl)

    try:
        match = geolite2.lookup(ip)
        if match is not None:
            country = match.country
            try:

                c = pycountry.countries.lookup(country)
                country = c.name
            except:
                country = ""

    except:
        country = ""
    try:
        res = get_tld("http://www" + ourl, as_object=True)
        tld = res.suffix
    except:
        tld = ""

    try:
        match = re.search(r'[\w\.-]+@[\w\.-]+', body)
        email = match.group(0)
    except:
        email = ""

    permalink=ourl.rstrip().replace(".","-")

    try:
        item = generator[0]
        val = "{ \"Domain\":" + json.dumps(
            "http://" + ourl.rstrip()) + ",\"IP\":\"" + ip + "\",\"Server\":" + json.dumps(
            str(server)) + ",\"PoweredBy\":" + json.dumps(
                str(poweredby)) + ",\"MetaGenerator\":" + json.dumps(item) + ",\"Email\":" + json.dumps(
                    email) + ",\"Suffix\":\"" + tld + "\",\"CountryHosted\":\"" + country+"\",\"permalink\":\""+permalink+"\" }"
    except:
        val = "{ \"Domain\":" + json.dumps(
            "http://" + ourl.rstrip()) + ",\"IP\":\"" + ip + "\"," + "\"Server\":" + json.dumps(
            str(server)) + ",\"PoweredBy\":" + json.dumps(
                str(poweredby)) + ",\"MetaGenerator\":\"\",\"Email\":" + json.dumps(
                    email) + ",\"Suffix\":\"" + tld + "\",\"CountryHosted\":\"" + country+"\",\"permalink\":\""+permalink+"\" }"


    print val


def main(reactor, url_path):
    urls = open(url_path)
    return mainjob(reactor, (url.strip() for url in urls))

def mainjob(reactor, urls=argv[2:]):
    #for url in urls:
    #  print url
    agent = Agent(reactor)
    work = (process(agent, url) for url in tqdm(urls))
    tasks = list(cooperate(work) for i in range(100))
    return gatherResults(list(task.whenDone() for task in tasks))



def process(agent, url):
    d = agent.request(
        'GET', "http://" + url,
        Headers({'User-Agent': ['crawler']}),
        None)
    d.debug=1
    d.addCallback(cbRequest, url)
    d.addErrback(error, url)  
    return d

react(main, ["./test.txt"])

f.close()

我启用了调试，并以某种方式为这两个条目调用addErrback（4次尝试中有4次），因此错误条目：

 user@laptop:~/crawler$ python scanner.py 
2017-02-25 20:35:36+0100 [-] Log opened.
0it [00:00, ?it/s]2017-02-25 20:35:36+0100 [-] Starting factory <twisted.web.client._HTTP11ClientFactory instance at 0x7fffefac1248>
2017-02-25 20:35:36+0100 [-] Starting factory <twisted.web.client._HTTP11ClientFactory instance at 0x7fffefac16c8>
2it [00:00, 660.31it/s]
2017-02-25 20:35:37+0100 [-] Stopping factory <twisted.web.client._HTTP11ClientFactory instance at 0x7fffefac1248>
2017-02-25 20:35:37+0100 [-] Stopping factory <twisted.web.client._HTTP11ClientFactory instance at 0x7fffefac16c8>
2017-02-25 20:35:37+0100 [-] Main loop terminated.
user@laptop:~/crawler$ cat errors.txt 
Error: google.al
Error: fau.edu.al

我很困惑，因为在捕获时，我看到这些请求被发出并收到了响应。

如何在addErrback（）中打印确切的错误原因以更好地理解行为？

这与这个问题有关：

Twisted/Python - processing a large file line by line

我感谢任何帮助。 Python和Twisted的新手。

谢谢，

更新1：

我将错误功能修改为：

def error(failure, url):
    f.write("Error: "+url+"\n") 
    print type(failure.value), failure # catch error here
    print failure.value.reasons[0].printTraceback()

这是输出，在每次运行时似乎都是相同的：

user@laptop:~/crawler$ python scanner.py 
2017-02-25 21:24:48+0100 [-] Log opened.
0it [00:00, ?it/s]2017-02-25 21:24:48+0100 [-] Starting factory <twisted.web.client._HTTP11ClientFactory instance at 0x7fffefac1200>
2017-02-25 21:24:48+0100 [-] Starting factory <twisted.web.client._HTTP11ClientFactory instance at 0x7fffefac1680>
2it [00:00, 788.33it/s]
2017-02-25 21:24:48+0100 [-] <type 'exceptions.TypeError'> [Failure instance: Traceback: <type 'exceptions.TypeError'>: 'NoneType' object has no attribute '__getitem__'
2017-02-25 21:24:48+0100 [-] /usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py:565:_startRunCallbacks
2017-02-25 21:24:48+0100 [-] /usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py:651:_runCallbacks
2017-02-25 21:24:48+0100 [-] /usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py:457:callback
2017-02-25 21:24:48+0100 [-] /usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py:565:_startRunCallbacks
2017-02-25 21:24:48+0100 [-] --- <exception caught here> ---
2017-02-25 21:24:48+0100 [-] /usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py:651:_runCallbacks
2017-02-25 21:24:48+0100 [-] scanner.py:47:cbRequest
2017-02-25 21:24:48+0100 [-] ]
2017-02-25 21:24:48+0100 [HTTP11ClientProtocol,client] main function encountered error
    Traceback (most recent call last):
    Failure: twisted.internet.defer.FirstError: FirstError[#0, [Failure instance: Traceback: <type 'exceptions.AttributeError'>: 'exceptions.TypeError' object has no attribute 'reasons'
    /usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py:565:_startRunCallbacks
    /usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py:651:_runCallbacks
    /usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py:457:callback
    /usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py:565:_startRunCallbacks
    --- <exception caught here> ---
    /usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py:651:_runCallbacks
    scanner.py:37:error
    ]]

2017-02-25 21:24:48+0100 [-] <class 'twisted.internet.error.ConnectError'> [Failure instance: Traceback (failure with no frames): <class 'twisted.internet.error.ConnectError'>: An error occurred while connecting: [Failure instance: Traceback (failure with no frames): <class 'twisted.internet.error.ConnectionLost'>: Connection to the other side was lost in a non-clean fashion: Connection lost.
2017-02-25 21:24:48+0100 [-] ].
2017-02-25 21:24:48+0100 [-] ]
2017-02-25 21:24:48+0100 [-] Stopping factory <twisted.web.client._HTTP11ClientFactory instance at 0x7fffefac1680>
2017-02-25 21:24:48+0100 [-] Stopping factory <twisted.web.client._HTTP11ClientFactory instance at 0x7fffefac1200>
2017-02-25 21:24:48+0100 [-] Main loop terminated.
2017-02-25 21:24:48+0100 [-] Unhandled error in Deferred:
2017-02-25 21:24:48+0100 [-] Unhandled Error
    Traceback (most recent call last):
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/base.py", line 1084, in connectionFailed
        self.factory.clientConnectionFailed(self, reason)
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/endpoints.py", line 246, in clientConnectionFailed
        self._onConnection.errback(reason)
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py", line 498, in errback
        self._startRunCallbacks(fail)
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py", line 565, in _startRunCallbacks
        self._runCallbacks()
    --- <exception caught here> ---
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py", line 651, in _runCallbacks
        current.result = callback(current.result, *args, **kw)
      File "scanner.py", line 37, in error
        print failure.value.reasons[0].printTraceback()
    exceptions.AttributeError: 'ConnectError' object has no attribute 'reasons'

2017-02-25 21:24:48+0100 [-] Unhandled error in Deferred:
2017-02-25 21:24:48+0100 [-] Unhandled Error
    Traceback (most recent call last):
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py", line 565, in _startRunCallbacks
        self._runCallbacks()
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py", line 651, in _runCallbacks
        current.result = callback(current.result, *args, **kw)
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py", line 457, in callback
        self._startRunCallbacks(result)
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py", line 565, in _startRunCallbacks
        self._runCallbacks()
    --- <exception caught here> ---
      File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py", line 651, in _runCallbacks
        current.result = callback(current.result, *args, **kw)
      File "scanner.py", line 37, in error
        print failure.value.reasons[0].printTraceback()
    exceptions.AttributeError: 'exceptions.TypeError' object has no attribute 'reasons'

更新2：

将错误功能修改为@ Jean-Paul Calderone给出的示例

我得到了这个输出：

2017-02-27 17:46:57+0100 [-] Log opened.
0it [00:00, ?it/s]2017-02-27 17:46:57+0100 [-] Starting factory <twisted.web.client._HTTP11ClientFactory instance at 0x7fffefac0320>
2017-02-27 17:46:57+0100 [-] Starting factory <twisted.web.client._HTTP11ClientFactory instance at 0x7fffefac07a0>
2it [00:00, 763.36it/s]
2017-02-27 17:46:57+0100 [-] Traceback (most recent call last):
2017-02-27 17:46:57+0100 [-]   File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py", line 565, in _startRunCallbacks
2017-02-27 17:46:57+0100 [-]     self._runCallbacks()
2017-02-27 17:46:57+0100 [-]   File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py", line 651, in _runCallbacks
2017-02-27 17:46:57+0100 [-]     current.result = callback(current.result, *args, **kw)
2017-02-27 17:46:57+0100 [-]   File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py", line 457, in callback
2017-02-27 17:46:57+0100 [-]     self._startRunCallbacks(result)
2017-02-27 17:46:57+0100 [-]   File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py", line 565, in _startRunCallbacks
2017-02-27 17:46:57+0100 [-]     self._runCallbacks()
2017-02-27 17:46:57+0100 [-] --- <exception caught here> ---
2017-02-27 17:46:57+0100 [-]   File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py", line 651, in _runCallbacks
2017-02-27 17:46:57+0100 [-]     current.result = callback(current.result, *args, **kw)
2017-02-27 17:46:57+0100 [-]   File "scan.py", line 59, in cbRequest
2017-02-27 17:46:57+0100 [-]     poweredby = response.headers.getRawHeaders("X-Powered-By")[0]
2017-02-27 17:46:57+0100 [-] exceptions.TypeError: 'NoneType' object has no attribute '__getitem__'
2017-02-27 17:46:57+0100 [-] Stopping factory <twisted.web.client._HTTP11ClientFactory instance at 0x7fffefac0320>
2017-02-27 17:46:59+0100 [-] Traceback (most recent call last):
2017-02-27 17:46:59+0100 [-]   File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py", line 565, in _startRunCallbacks
2017-02-27 17:46:59+0100 [-]     self._runCallbacks()
2017-02-27 17:46:59+0100 [-]   File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py", line 651, in _runCallbacks
2017-02-27 17:46:59+0100 [-]     current.result = callback(current.result, *args, **kw)
2017-02-27 17:46:59+0100 [-]   File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py", line 457, in callback
2017-02-27 17:46:59+0100 [-]     self._startRunCallbacks(result)
2017-02-27 17:46:59+0100 [-]   File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py", line 565, in _startRunCallbacks
2017-02-27 17:46:59+0100 [-]     self._runCallbacks()
2017-02-27 17:46:59+0100 [-] --- <exception caught here> ---
2017-02-27 17:46:59+0100 [-]   File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py", line 651, in _runCallbacks
2017-02-27 17:46:59+0100 [-]     current.result = callback(current.result, *args, **kw)
2017-02-27 17:46:59+0100 [-]   File "scan.py", line 59, in cbRequest
2017-02-27 17:46:59+0100 [-]     poweredby = response.headers.getRawHeaders("X-Powered-By")[0]
2017-02-27 17:46:59+0100 [-] exceptions.TypeError: 'NoneType' object has no attribute '__getitem__'
2017-02-27 17:46:59+0100 [-] Stopping factory <twisted.web.client._HTTP11ClientFactory instance at 0x7fffefac07a0>
2017-02-27 17:46:59+0100 [-] Main loop terminated.

正如你所看到的，它抱怨响应是NoneType对象，因此没有提到真正的原因。

但是，当我运行旧版本的错误函数时，它会打印出您在Update 1中看到的内容：

<class 'twisted.internet.error.ConnectError'> [Failure instance: Traceback (failure with no frames): <class 'twisted.internet.error.ConnectError'>: An error occurred while connecting: [Failure instance: Traceback (failure with no frames): <class 'twisted.internet.error.ConnectionLost'>: Connection to the other side was lost in a non-clean fashion: Connection lost.

为什么它不会在第一个版本中打印出来（@ Jean-Paul Calderone建议的答案）版本？为什么它一般会出现这个错误？

我验证它在我连接到这些域时有效，也可以通过线程版本的爬虫（也在Python中）工作，因为你可以看到它也在线上（Wireshark截图）。看起来Twisted 没有看到它（看作是丢失的连接？）

Answer 1

您的error功能存在错误。回溯告诉你：

  File "scanner.py", line 37, in error
    print failure.value.reasons[0].printTraceback()
exceptions.AttributeError: 'exceptions.TypeError' object has no attribute 'reasons'

scanner.py，第37行，在名为error的函数中，行print failure.value.reasons[0].printTraceback()会引发AttributeError，因为TypeError实例没有reasons属性

我认为您查看的reasons属性属于RequestGenerationFailed，RequestTransmissionFailed或ResponseFailed。

所以，请像这样定义error，而不是：

from twisted.web._newclient import (
    RequestGenerationFailed, 
    RequestTransmissionFailed,
    ResponseFailed,
)

def error(failure, url):
    f.write("Error: "+url+"\n") 
    if failure.check(
        RequestGenerationFailed,
        RequestTransmissionFailed,
        ResponseFailed,
    ):
        failure.value.reasons[0].printTraceback()
    else:
        failure.printTraceback()

如果这证明有用，请提交针对Twisted的票证以使这些例外类型公开（因为不能保证_newclient导入将来继续有效）。

Python / Twisted - 如何打印更详细的错误消息

1 个答案: