我是Python的新手，正在编写一个简单的查询来从USPS API获取信息，并将结果存储在我以后可以参考的.csv文件中。我可以成功查询API，但我想将查询扩展到大约220万个查询。在for循环中执行此操作需要数周时间，因此我将多线程作为一种并行运行请求的方法。我有两个问题：
当我执行超过15个左右的线程时，我收到连接错误。该错误类似于this question，但由于我的较小查询有效，我认为必须从服务器进行限制。
如何保留字典的键值，而不是将其更改为＆＃34; 0,1,2，...＆＃34;？
如果我必须一次将查询限制为只有小批量，我可以保留一个我连续追加的主文件作为for循环运行吗？我知道Python有this structure用于添加到字典
以下是我的代码的最小示例（数据集必须很大，因为当我执行大量卷时出现错误）：
from xml.etree import ElementTree as ET
from threading import Thread
import numpy as np
import pandas as pd
import requests
import csv
# API Information
usps_uname = '536UNIVE4362'
usps_pw = '462YK79VT194'
url = 'http://production.shippingapis.com/ShippingAPITest.dll?'
req = "StandardB"
# Single API query
def delivery(origin, destination):
query = url + 'API=' + str(req) + '&XML=%3C' + str(req) + \
'Request%20USERID=%22' + str(usps_uname) + '%22%3E' + \
'%3COriginZip%3E' + "%05d" % origin + '%3C/OriginZip%3E' + \
'%3CDestinationZip%3E' + "%05d" % destination + '%3C/DestinationZip%3E' + \
'%3C/' + str(req) + 'Request%3E'
data = requests.get(query, auth = (usps_uname, usps_pw))
root = ET.fromstring(data.content)
if root[2].text == "No Data":
DeliveryTime = 99
else:
DeliveryTime = int(root[2].text)
return DeliveryTime
# Returns delivery times from all origins to specified destination
def delivery_range(origin_range, destination, thread_index, store=None):
store[thread_index] = [0] * len(origin_range)
for i, x in enumerate(origin_range):
store[thread_index][i] = delivery(x, destination)
return store
# Threading attempt
def threaded_process(nthreads, origin_range):
store = {}
threads = []
for i in range(nthreads):
ids = origin_range.values()[i]
destination = origin_range.keys()[i]
t = Thread(target=delivery_range, args=(ids, destination, i, store))
threads.append(t)
[ t.start() for t in threads ]
[ t.join() for t in threads ]
return store
origin_range = {
2072: [ 2072, 3063, 6095, 7001, 8085, 8691, 15205, 17013, 17015,
17015, 17339, 18031, 18202, 19709, 19720, 21224, 23803, 23836,
28027, 29172, 29303, 30344, 33182, 33570, 33811, 33897, 37090,
37127, 37310, 37416, 40165, 40218, 40511, 41048, 42718, 46075,
46075, 46168, 46168, 46231, 47130, 53144, 66219, 67337, 75019,
75261, 76155, 76177, 77338, 78154, 85043, 85043, 85338, 85906,
89030, 89408, 92374, 92408, 92408, 92551, 94560, 95206, 95304,
95363, 98004, 98032, 98327, 98390],
3063: [ 2072, 3063, 6095, 7001, 8085, 8691, 15205, 17013, 17015,
17015, 17339, 18031, 18202, 19709, 19720, 21224, 23803, 23836,
28027, 29172, 29303, 30344, 33182, 33570, 33811, 33897, 37090,
37127, 37310, 37416, 40165, 40218, 40511, 41048, 42718, 46075,
46075, 46168, 46168, 46231, 47130, 53144, 66219, 67337, 75019,
75261, 76155, 76177, 77338, 78154, 85043, 85043, 85338, 85906,
89030, 89408, 92374, 92408, 92408, 92551, 94560, 95206, 95304,
95363, 98004, 98032, 98327, 98390],
6095: [ 2072, 3063, 6095, 7001, 8085, 8691, 15205, 17013, 17015,
17015, 17339, 18031, 18202, 19709, 19720, 21224, 23803, 23836,
28027, 29172, 29303, 30344, 33182, 33570, 33811, 33897, 37090,
37127, 37310, 37416, 40165, 40218, 40511, 41048, 42718, 46075,
46075, 46168, 46168, 46231, 47130, 53144, 66219, 67337, 75019,
75261, 76155, 76177, 77338, 78154, 85043, 85043, 85338, 85906,
89030, 89408, 92374, 92408, 92408, 92551, 94560, 95206, 95304,
95363, 98004, 98032, 98327, 98390],
7001: [ 2072, 3063, 6095, 7001, 8085, 8691, 15205, 17013, 17015,
17015, 17339, 18031, 18202, 19709, 19720, 21224, 23803, 23836,
28027, 29172, 29303, 30344, 33182, 33570, 33811, 33897, 37090,
37127, 37310, 37416, 40165, 40218, 40511, 41048, 42718, 46075,
46075, 46168, 46168, 46231, 47130, 53144, 66219, 67337, 75019,
75261, 76155, 76177, 77338, 78154, 85043, 85043, 85338, 85906,
89030, 89408, 92374, 92408, 92408, 92551, 94560, 95206, 95304,
95363, 98004, 98032, 98327, 98390],
8085: [ 2072, 3063, 6095, 7001, 8085, 8691, 15205, 17013, 17015,
17015, 17339, 18031, 18202, 19709, 19720, 21224, 23803, 23836,
28027, 29172, 29303, 30344, 33182, 33570, 33811, 33897, 37090,
37127, 37310, 37416, 40165, 40218, 40511, 41048, 42718, 46075,
46075, 46168, 46168, 46231, 47130, 53144, 66219, 67337, 75019,
75261, 76155, 76177, 77338, 78154, 85043, 85043, 85338, 85906,
89030, 89408, 92374, 92408, 92408, 92551, 94560, 95206, 95304,
95363, 98004, 98032, 98327, 98390],
8691: [ 2072, 3063, 6095, 7001, 8085, 8691, 15205, 17013, 17015,
17015, 17339, 18031, 18202, 19709, 19720, 21224, 23803, 23836,
28027, 29172, 29303, 30344, 33182, 33570, 33811, 33897, 37090,
37127, 37310, 37416, 40165, 40218, 40511, 41048, 42718, 46075,
46075, 46168, 46168, 46231, 47130, 53144, 66219, 67337, 75019,
75261, 76155, 76177, 77338, 78154, 85043, 85043, 85338, 85906,
89030, 89408, 92374, 92408, 92408, 92551, 94560, 95206, 95304,
95363, 98004, 98032, 98327, 98390],
15205: [ 2072, 3063, 6095, 7001, 8085, 8691, 15205, 17013, 17015,
17015, 17339, 18031, 18202, 19709, 19720, 21224, 23803, 23836,
28027, 29172, 29303, 30344, 33182, 33570, 33811, 33897, 37090,
37127, 37310, 37416, 40165, 40218, 40511, 41048, 42718, 46075,
46075, 46168, 46168, 46231, 47130, 53144, 66219, 67337, 75019,
75261, 76155, 76177, 77338, 78154, 85043, 85043, 85338, 85906,
89030, 89408, 92374, 92408, 92408, 92551, 94560, 95206, 95304,
95363, 98004, 98032, 98327, 98390],
17013: [ 2072, 3063, 6095, 7001, 8085, 8691, 15205, 17013, 17015,
17015, 17339, 18031, 18202, 19709, 19720, 21224, 23803, 23836,
28027, 29172, 29303, 30344, 33182, 33570, 33811, 33897, 37090,
37127, 37310, 37416, 40165, 40218, 40511, 41048, 42718, 46075,
46075, 46168, 46168, 46231, 47130, 53144, 66219, 67337, 75019,
75261, 76155, 76177, 77338, 78154, 85043, 85043, 85338, 85906,
89030, 89408, 92374, 92408, 92408, 92551, 94560, 95206, 95304,
95363, 98004, 98032, 98327, 98390],
17015: [ 2072, 3063, 6095, 7001, 8085, 8691, 15205, 17013, 17015,
17015, 17339, 18031, 18202, 19709, 19720, 21224, 23803, 23836,
28027, 29172, 29303, 30344, 33182, 33570, 33811, 33897, 37090,
37127, 37310, 37416, 40165, 40218, 40511, 41048, 42718, 46075,
46075, 46168, 46168, 46231, 47130, 53144, 66219, 67337, 75019,
75261, 76155, 76177, 77338, 78154, 85043, 85043, 85338, 85906,
89030, 89408, 92374, 92408, 92408, 92551, 94560, 95206, 95304,
95363, 98004, 98032, 98327, 98390],
17339: [ 2072, 3063, 6095, 7001, 8085, 8691, 15205, 17013, 17015,
17015, 17339, 18031, 18202, 19709, 19720, 21224, 23803, 23836,
28027, 29172, 29303, 30344, 33182, 33570, 33811, 33897, 37090,
37127, 37310, 37416, 40165, 40218, 40511, 41048, 42718, 46075,
46075, 46168, 46168, 46231, 47130, 53144, 66219, 67337, 75019,
75261, 76155, 76177, 77338, 78154, 85043, 85043, 85338, 85906,
89030, 89408, 92374, 92408, 92408, 92551, 94560, 95206, 95304,
95363, 98004, 98032, 98327, 98390],
18031: [ 2072, 3063, 6095, 7001, 8085, 8691, 15205, 17013, 17015,
17015, 17339, 18031, 18202, 19709, 19720, 21224, 23803, 23836,
28027, 29172, 29303, 30344, 33182, 33570, 33811, 33897, 37090,
37127, 37310, 37416, 40165, 40218, 40511, 41048, 42718, 46075,
46075, 46168, 46168, 46231, 47130, 53144, 66219, 67337, 75019,
75261, 76155, 76177, 77338, 78154, 85043, 85043, 85338, 85906,
89030, 89408, 92374, 92408, 92408, 92551, 94560, 95206, 95304,
95363, 98004, 98032, 98327, 98390],
18202: [ 2072, 3063, 6095, 7001, 8085, 8691, 15205, 17013, 17015,
17015, 17339, 18031, 18202, 19709, 19720, 21224, 23803, 23836,
28027, 29172, 29303, 30344, 33182, 33570, 33811, 33897, 37090,
37127, 37310, 37416, 40165, 40218, 40511, 41048, 42718, 46075,
46075, 46168, 46168, 46231, 47130, 53144, 66219, 67337, 75019,
75261, 76155, 76177, 77338, 78154, 85043, 85043, 85338, 85906,
89030, 89408, 92374, 92408, 92408, 92551, 94560, 95206, 95304,
95363, 98004, 98032, 98327, 98390],
19709: [ 2072, 3063, 6095, 7001, 8085, 8691, 15205, 17013, 17015,
17015, 17339, 18031, 18202, 19709, 19720, 21224, 23803, 23836,
28027, 29172, 29303, 30344, 33182, 33570, 33811, 33897, 37090,
37127, 37310, 37416, 40165, 40218, 40511, 41048, 42718, 46075,
46075, 46168, 46168, 46231, 47130, 53144, 66219, 67337, 75019,
75261, 76155, 76177, 77338, 78154, 85043, 85043, 85338, 85906,
89030, 89408, 92374, 92408, 92408, 92551, 94560, 95206, 95304,
95363, 98004, 98032, 98327, 98390],
19720: [ 2072, 3063, 6095, 7001, 8085, 8691, 15205, 17013, 17015,
17015, 17339, 18031, 18202, 19709, 19720, 21224, 23803, 23836,
28027, 29172, 29303, 30344, 33182, 33570, 33811, 33897, 37090,
37127, 37310, 37416, 40165, 40218, 40511, 41048, 42718, 46075,
46075, 46168, 46168, 46231, 47130, 53144, 66219, 67337, 75019,
75261, 76155, 76177, 77338, 78154, 85043, 85043, 85338, 85906,
89030, 89408, 92374, 92408, 92408, 92551, 94560, 95206, 95304,
95363, 98004, 98032, 98327, 98390],
21224: [ 2072, 3063, 6095, 7001, 8085, 8691, 15205, 17013, 17015,
17015, 17339, 18031, 18202, 19709, 19720, 21224, 23803, 23836,
28027, 29172, 29303, 30344, 33182, 33570, 33811, 33897, 37090,
37127, 37310, 37416, 40165, 40218, 40511, 41048, 42718, 46075,
46075, 46168, 46168, 46231, 47130, 53144, 66219, 67337, 75019,
75261, 76155, 76177, 77338, 78154, 85043, 85043, 85338, 85906,
89030, 89408, 92374, 92408, 92408, 92551, 94560, 95206, 95304,
95363, 98004, 98032, 98327, 98390],
23803: [ 2072, 3063, 6095, 7001, 8085, 8691, 15205, 17013, 17015,
17015, 17339, 18031, 18202, 19709, 19720, 21224, 23803, 23836,
28027, 29172, 29303, 30344, 33182, 33570, 33811, 33897, 37090,
37127, 37310, 37416, 40165, 40218, 40511, 41048, 42718, 46075,
46075, 46168, 46168, 46231, 47130, 53144, 66219, 67337, 75019,
75261, 76155, 76177, 77338, 78154, 85043, 85043, 85338, 85906,
89030, 89408, 92374, 92408, 92408, 92551, 94560, 95206, 95304,
95363, 98004, 98032, 98327, 98390],
23836: [ 2072, 3063, 6095, 7001, 8085, 8691, 15205, 17013, 17015,
17015, 17339, 18031, 18202, 19709, 19720, 21224, 23803, 23836,
28027, 29172, 29303, 30344, 33182, 33570, 33811, 33897, 37090,
37127, 37310, 37416, 40165, 40218, 40511, 41048, 42718, 46075,
46075, 46168, 46168, 46231, 47130, 53144, 66219, 67337, 75019,
75261, 76155, 76177, 77338, 78154, 85043, 85043, 85338, 85906,
89030, 89408, 92374, 92408, 92408, 92551, 94560, 95206, 95304,
95363, 98004, 98032, 98327, 98390],
28027: [ 2072, 3063, 6095, 7001, 8085, 8691, 15205, 17013, 17015,
17015, 17339, 18031, 18202, 19709, 19720, 21224, 23803, 23836,
28027, 29172, 29303, 30344, 33182, 33570, 33811, 33897, 37090,
37127, 37310, 37416, 40165, 40218, 40511, 41048, 42718, 46075,
46075, 46168, 46168, 46231, 47130, 53144, 66219, 67337, 75019,
75261, 76155, 76177, 77338, 78154, 85043, 85043, 85338, 85906,
89030, 89408, 92374, 92408, 92408, 92551, 94560, 95206, 95304,
95363, 98004, 98032, 98327, 98390],
29172: [ 2072, 3063, 6095, 7001, 8085, 8691, 15205, 17013, 17015,
17015, 17339, 18031, 18202, 19709, 19720, 21224, 23803, 23836,
28027, 29172, 29303, 30344, 33182, 33570, 33811, 33897, 37090,
37127, 37310, 37416, 40165, 40218, 40511, 41048, 42718, 46075,
46075, 46168, 46168, 46231, 47130, 53144, 66219, 67337, 75019,
75261, 76155, 76177, 77338, 78154, 85043, 85043, 85338, 85906,
89030, 89408, 92374, 92408, 92408, 92551, 94560, 95206, 95304,
95363, 98004, 98032, 98327, 98390],
29303: [ 2072, 3063, 6095, 7001, 8085, 8691, 15205, 17013, 17015,
17015, 17339, 18031, 18202, 19709, 19720, 21224, 23803, 23836,
28027, 29172, 29303, 30344, 33182, 33570, 33811, 33897, 37090,
37127, 37310, 37416, 40165, 40218, 40511, 41048, 42718, 46075,
46075, 46168, 46168, 46231, 47130, 53144, 66219, 67337, 75019,
75261, 76155, 76177, 77338, 78154, 85043, 85043, 85338, 85906,
89030, 89408, 92374, 92408, 92408, 92551, 94560, 95206, 95304,
95363, 98004, 98032, 98327, 98390],
30344: [ 2072, 3063, 6095, 7001, 8085, 8691, 15205, 17013, 17015,
17015, 17339, 18031, 18202, 19709, 19720, 21224, 23803, 23836,
28027, 29172, 29303, 30344, 33182, 33570, 33811, 33897, 37090,
37127, 37310, 37416, 40165, 40218, 40511, 41048, 42718, 46075,
46075, 46168, 46168, 46231, 47130, 53144, 66219, 67337, 75019,
75261, 76155, 76177, 77338, 78154, 85043, 85043, 85338, 85906,
89030, 89408, 92374, 92408, 92408, 92551, 94560, 95206, 95304,
95363, 98004, 98032, 98327, 98390],
}
ans = threaded_process(len(origin_range), origin_range)
writer = csv.writer(open('DeliveryTimes.csv', 'wb'))
for key, value in ans.items():
writer.writerow([key, value])
如果您第一次没有收到错误，请再次运行该代码，它应该会出错。
答案 0 :(得分：0)
您可以将python脚本设置为通过参数列表将原点和目标作为输入（例如，使用argparse模块），然后使用GNU Parallel（http://www.gnu.org/software/parallel/）来调用此脚本可能的组合作为参数传递。
GNU Parallel将对您的代码进行操作系统级并行化，因此您不必担心在Python中执行此操作。