我编写了一个python程序,使用请求库和多线程(使用队列)下载文件。即使我已经给出了超时选项,它仍然会在某个时刻陷入困境。我已经发布了完整的源代码,请帮我解决这个问题。
def downloadqueue():
qwork = Queue.Queue()
kill_process = False
#File Queue Downloader
def downloadpipe():
while not kill_process:
try:
#Get the Records for download
dwdetails = qwork.get(True,5)
except Queue.Empty:
continue
#Download the file and save the filename as MD5
downstatus = request_download(os.path.join(storepath,dwdetails[0]),dwdetails[1],filetype)
if downstatus == True:
#Here i am using sql procedure to update the table status
print "Success"
else:
#Here i am using sql procedure to update the table status
print "Fails"
qwork.task_done()
#Create Download Threads
threads = []
for unindex in range(int(threadcnt)):
thread = threading.Thread(target=downloadpipe)
thread.daemon = True
thread.start()
threads.append(thread)
#Loop over to download urls
while not kill_process:
try:
# rectuple - a List of urls fetching from sql table
if qwork.qsize() > 500:
time.sleep(20)
continue
if len(rectuple) == 0:
time.sleep(10)
continue
bufmem = len(os.listdir(storepath))
if bufmem >= int(folsize):
kill_process = True
mvdestpath = os.path.join(destpath,filetype+"_"+str(tstamp()))
for thread in threads:
if thread.is_alive():
thread.join()
shutil.move(storepath,mvdestpath)
#Create the Storage folder if not exists
if not os.path.exists(storepath):
os.mkdir(storepath)
kill_process = False
threads = []
for unindex in range(int(threadcnt)):
thread = threading.Thread(target=downloadpipe)
thread.daemon = True
thread.start()
threads.append(thread)
#Put the records to the download queue
for drecords in rectuple:
qwork.put(drecords)
#Wait until all the downloads has been completed
qwork.join()
except KeyboardInterrupt:
#Wait until all the downloads has been completed
kill_process = True
for thread in threads:
if thread.is_alive():
thread.join()
except Exception,e:
#Report the Error
time.sleep(5)
def request_download(fullfilename,url,filetype):
attempts = 0
resp = 0
while attempts < 3:
try:
#Request the url to download file
response = requests.get(url,stream=True,timeout=90,verify=False)
resp = response.status_code
if response.status_code == 200:
with open(fullfilename, 'wb') as fobj:
for chunk in response.iter_content(chunk_size=1024):
if chunk:
fobj.write(chunk)
fobj.flush()
return True
else:
attempts += 1
except Exception,e:
attempts += 1
return resp