Question

我正在使用以下代码从Twitter用户ID获取朋友列表。我在python中使用Oauth来做到这一点：

import oauth2 as oauth
import json
import twitter
import time

CONSUMER_KEY = 'xxxxx'
CONSUMER_SECRET = 'xxxxxxx'
ACCESS_KEY = 'xxxxxxx'
ACCESS_SECRET = 'xxxxxxxxx'


consumer = oauth.Consumer(key=CONSUMER_KEY, secret=CONSUMER_SECRET)
access_token = oauth.Token(key=ACCESS_KEY, secret=ACCESS_SECRET)
client = oauth.Client(consumer, access_token)
api_path = "https://api.twitter.com/1.1/friends/ids.json?user_id="

def friendsids(user_id):
    cursor='-1'
    all_ids=[]
    errors=[]
    while (cursor!='0'):
        url_with_cursor = api_path+str(user_id) +"&cursor="+cursor
        response,data = client.request( url_with_cursor )
        data=data.decode('utf8')
        data = json.loads(data)
        if 'ids' in data.keys():
            for each in data['ids']:
                all_ids.append(each)
            cursor=data['next_cursor_str']
        elif 'errors' in data.keys():
            time.sleep(60*15)
            continue
        else:

            errors.append(str('error')+str(user_id))
            cursor='0'
    if len(all_ids)>0:
        return all_ids
    else:
        return errors


with open('a.txt') as f:
    for line in f:
        line=line.rstrip()
        twitter_id=line[2]
        frnds=friendsids(twitter_id)
        print(twitter_id+'\t'+str(frnds))

文件a.txt包含数据。该程序在somtime结束时出现以下错误：

  File "import2.py", line 54, in <module>
    frnds=friendsids(twitter_id)
  File "import2.py", line 23, in friendsids
    response,data = client.request( url_with_cursor )
  File "/usr/local/lib/python3.5/dist-packages/oauth2/__init__.py", line 687, in request
    connection_type=connection_type)
  File "/usr/lib/python3/dist-packages/httplib2/__init__.py", line 1312, in request
    (response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
  File "/usr/lib/python3/dist-packages/httplib2/__init__.py", line 1062, in _request
    (response, content) = self._conn_request(conn, request_uri, method, body, headers)
  File "/usr/lib/python3/dist-packages/httplib2/__init__.py", line 1015, in _conn_request
    response = conn.getresponse()
  File "/usr/lib/python3.5/http/client.py", line 1197, in getresponse
    response.begin()
  File "/usr/lib/python3.5/http/client.py", line 297, in begin
    version, status, reason = self._read_status()
  File "/usr/lib/python3.5/http/client.py", line 258, in _read_status
    line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
  File "/usr/lib/python3.5/socket.py", line 575, in readinto
    return self._sock.recv_into(b)
  File "/usr/lib/python3.5/ssl.py", line 929, in recv_into
    return self.read(nbytes, buffer)
  File "/usr/lib/python3.5/ssl.py", line 791, in read
    return self._sslobj.read(len, buffer)
  File "/usr/lib/python3.5/ssl.py", line 575, in read
    v = self._sslobj.read(len, buffer)
ConnectionResetError: [Errno 104] Connection reset by peer

有什么办法可以阻止这种情况的发生？我使用的方法非常慢。它几乎无法在2小时内收集100个用户数据。有没有更快的方法来收集多达1000位用户的python中的twitter朋友列表（所有朋友）？

从Twitter API提取数据时，对等方重置连接

0 个答案: