Question

我不知道这段代码有什么问题。我正在尝试从99acres.com抓取数据。我已经通过了帖子参数。这是代码

from scrapy import Spider
from scrapy.http import FormRequest
from scrapy.selector import HtmlXPathSelector


class aagSpider(Spider):
    name = "acre"
    start_urls = ["http://www.99acres.com"]

    def parse(self, response):
        frmdata3 = {"Refine_Localities": "Refine Localities", "action": "/do/quicksearch/search", "bedroom_num": "",
                    "budget_max": "", "budget_min": "", "city": 4,
                    "class": "", "fullSelectedSuggestions": "laxmi nagar, delhi east", "isvoicesearch": "N",
                    "keyword": "",
                    "keyword_suggest": "laxmi nagar, delhi east;",
                    "locality_array[]": "233",
                    "locality_array[]": "233",
                    "locality_array[]": "233",
                    "lstAcn": "HP_R",
                    "lstAcnId": "0",
                    "np_search_type": "NL,NP,R2M",
                    "preference": "S",
                    "property_type": "23",
                    "refine_results": "Y",
                    "res_com": "R",
                    "search_location": "HP",
                    "search_type": "QS",
                    "searchform": "1",
                    "selected_tab": "3",
                    "src": "CLUSTER",
                    "strEntityMap": "[{'type':'locality'},{'1':['laxmi nagar, delhi east','CITY_4, LOCALITY_233, PREFERENCE_S, RESCOM_R']}]",
                    "suggestion": "CITY_4, LOCALITY_233, PREFERENCE_S, RESCOM_R",
                    "texttypedtillsuggestion": "laxmi"}

        yield FormRequest(response.url, callback=self.fourth, formdata=frmdata3)

    def fourth(self, response):
        print "11111111111111111111111111111111111111111111111111"

我试图在传递上述参数后进入页面，但继续得到这个错误

 Traceback (most recent call last):
    File "/home/user/.local/lib/python2.7/site-packages/scrapy/utils /defer.py", line 102, in iter_errback
    yield next(it)
  File "/home/user/.local/lib/python2.7/site-packages/scrapy/spidermiddlewares/offsite.py", line 29, in process_spider_output
    for x in result:
  File "/home/user/.local/lib/python2.7/site-packages/scrapy/spidermiddlewares/referer.py", line 22, in <genexpr>
    return (_set_referer(r) for r in result or ())
  File "/home/user/.local/lib/python2.7/site-packages/scrapy/spidermiddlewares/urllength.py", line 37, in <genexpr>
    return (r for r in result or () if _filter(r))
  File "/home/user/.local/lib/python2.7/site-packages/scrapy/spidermiddlewares/depth.py", line 58, in <genexpr>
    return (r for r in result or () if _filter(r))
  File "/home/user/tutorial/tutorial/spiders/acre.py", line 37, in parse
    yield FormRequest(response.url,callback=self.fourth,formdata=frmdata3)
  File "/home/user/.local/lib/python2.7/site-packages/scrapy/http/request/form.py", line 28, in __init__
    querystr = _urlencode(items, self.encoding)
  File "/home/user/.local/lib/python2.7/site-packages/scrapy/http/request/form.py", line 61, in _urlencode
    for v in (vs if is_listlike(vs) else [vs])]
  File "/home/user/.local/lib/python2.7/site-packages/scrapy/utils/python.py", line 117, in to_bytes
    'object, got %s' % type(text).__name__)
TypeError: to_bytes must receive a unicode, str or bytes object, got int

Answer 1

这个问题背后的原因是，在Form Data中，永远不会像处理它那样处理Null，Boolean（True / False）和数字。始终将其视为字符串。在python中，Null是None，但在表单数据中，它总是'null'。 True或False写为'true'或'false'。对于数字，请将其设为字符串。

Python Scrapy：TypeError：to_bytes必须接收unicode，str或bytes对象，得到int

1 个答案: