Python程序被挂起了

时间:2014-01-28 06:23:51

标签: python multithreading

我编写了一个python程序,使用请求库多线程(使用队列)下载文件。即使我已经给出了超时选项,它仍然会在某个时刻陷入困境。我已经发布了完整的源代码,请帮我解决这个问题。

def downloadqueue():
    qwork = Queue.Queue()
    kill_process = False
    #File Queue Downloader
    def downloadpipe():
        while not kill_process:
            try:
                #Get the Records for download
                dwdetails = qwork.get(True,5) 
            except Queue.Empty:
                continue
            #Download the file and save the filename as MD5
            downstatus = request_download(os.path.join(storepath,dwdetails[0]),dwdetails[1],filetype)
            if downstatus == True:
                #Here i am using sql procedure to update the table status   
                print "Success"
            else:
                #Here i am using sql procedure to update the table status
                print "Fails"
            qwork.task_done()
    #Create Download Threads
    threads = []
    for unindex in range(int(threadcnt)):
        thread = threading.Thread(target=downloadpipe)
        thread.daemon = True
        thread.start()
        threads.append(thread)      
    #Loop over to download urls 
    while not kill_process:
        try:
            # rectuple - a List of urls fetching from sql table

            if qwork.qsize() > 500:
                time.sleep(20)
                continue
            if len(rectuple) == 0:
                time.sleep(10)
                continue
            bufmem = len(os.listdir(storepath))
            if bufmem >= int(folsize):
                kill_process = True
                mvdestpath = os.path.join(destpath,filetype+"_"+str(tstamp()))
                for thread in threads:
                    if thread.is_alive():
                        thread.join()
                shutil.move(storepath,mvdestpath)
                #Create the Storage folder if not exists
                if not os.path.exists(storepath):
                    os.mkdir(storepath)
                kill_process = False
                threads = []
                for unindex in range(int(threadcnt)):
                    thread = threading.Thread(target=downloadpipe)
                    thread.daemon = True
                    thread.start()
                    threads.append(thread)              
            #Put the records to the download queue
            for drecords in rectuple:
                qwork.put(drecords)
            #Wait until all the downloads has been completed 
            qwork.join()
        except KeyboardInterrupt:
            #Wait until all the downloads has been completed 
            kill_process = True
            for thread in threads:
                if thread.is_alive():
                    thread.join()
        except Exception,e:
            #Report the Error
            time.sleep(5)

def request_download(fullfilename,url,filetype):
    attempts = 0
    resp = 0
    while attempts < 3:
        try:
            #Request the url to download file
            response = requests.get(url,stream=True,timeout=90,verify=False)
            resp = response.status_code
            if response.status_code == 200:
                with open(fullfilename, 'wb') as fobj:
                    for chunk in response.iter_content(chunk_size=1024): 
                        if chunk:
                            fobj.write(chunk)
                            fobj.flush()
                return True
            else:
                attempts += 1
        except Exception,e:
            attempts += 1
    return resp

0 个答案:

没有答案