Question

我有python脚本将文件上传到云帐户。它工作了一段时间，但现在我开始收到'Exception in thread Thread-1 (most likely raised during interpreter shutdown)'错误。在研究之后，我发现了这个python问题http://bugs.python.org/issue14623，该问题表明问题不会得到解决。

但是，我不确定这会适用于我，我希望有人可以指出修复。我想继续使用python的线程并尽量避免使用多处理，因为这是I / O绑定的。这是精简版本（也有此问题），但在完整版本中，upload.py有一个我想分享的列表，所以我希望它在同一个内存中运行。

它总是在完成并且上传所有文件后中断。我尝试删除't.daemon = True'，它会在同一点挂起（而不是中断）（在上传所有文件之后）。我还尝试删除q.join()和't.daemon = True'，它将在完成后挂起。如果没有t.daemon = True和q.join（），我认为在脚本执行结束时只是在item = q.get()处阻塞（只是猜测）。

主：

 import logging 
import os 
import sys
import json
from os.path import expanduser
from Queue import Queue
from threading import Thread
from auth import Authenticate
from getinfo import get_containers, get_files, get_link
from upload import upload_file 
from container_util import create_containers
from filter import MyFilter

home = expanduser("~") + '/'

directory = home + "krunchuploader_logs"

if not os.path.exists(directory):
    os.makedirs(directory)

debug = directory + "/krunchuploader__debug_" + str(os.getpid())
error = directory + "/krunchuploader__error_" + str(os.getpid())
info = directory + "/krunchuploader__info_" + str(os.getpid())

os.open(debug, os.O_CREAT | os.O_EXCL)
os.open(error, os.O_CREAT | os.O_EXCL)
os.open(info, os.O_CREAT | os.O_EXCL)

formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')


logging.basicConfig(level=logging.DEBUG,
    format='%(asctime)s - %(levelname)s - %(message)s',
    filename=debug,
    filemode='w')

logger = logging.getLogger("krunch")

fh_error = logging.FileHandler(error)
fh_error.setLevel(logging.ERROR)
fh_error.setFormatter(formatter)
fh_error.addFilter(MyFilter(logging.ERROR))

fh_info = logging.FileHandler(info)
fh_info.setLevel(logging.INFO)
fh_info.setFormatter(formatter)
fh_info.addFilter(MyFilter(logging.INFO))

std_out_error = logging.StreamHandler()
std_out_error.setLevel(logging.ERROR)
std_out_info = logging.StreamHandler()
std_out_info.setLevel(logging.INFO)

logger.addHandler(fh_error)
logger.addHandler(fh_info)
logger.addHandler(std_out_error)
logger.addHandler(std_out_info)



def main():
    sys.stdout.write("\x1b[2J\x1b[H")
    print title
    authenticate = Authenticate()
    cloud_url = get_link(authenticate.jsonresp)
    #per 1 million files the list will take
    #approx 300MB of memory.
    file_container_list, file_list = get_files(authenticate, cloud_url)
    cloud_container_list = get_containers(authenticate, cloud_url)
    create_containers(cloud_container_list,
        file_container_list, authenticate, cloud_url)

    return file_list

def do_the_uploads(file_list):
    def worker():
    while True:
        item = q.get()
        upload_file(item)
        q.task_done()

    q = Queue()
    for i in range(5):
    t = Thread(target=worker)
    t.daemon = True
    t.start()

    for item in file_list:
    q.put(item)

    q.join()


if __name__ == '__main__':
    file_list = main()
    value = raw_input("\nProceed to upload files? Enter [Y/y] for yes: ").upper()
    if value == "Y":
    do_the_uploads(file_list)

upload.py：

def upload_file(file_obj):
    absolute_path_filename, filename, dir_name, token, url = file_obj
    url = url + dir_name + '/' + filename

    header_collection = {
        "X-Auth-Token": token}

    print "Uploading " + absolute_path_filename
    with open(absolute_path_filename) as f:
    r = requests.put(url, data=f, headers=header_collection)
    print "done"

错误输出：

Fetching Cloud Container List... Got it!
All containers exist, none need to be added

Proceed to upload files? Enter [Y/y] for yes: y
Uploading /home/one/huh/one/green
Uploading /home/one/huh/one/red
Uploading /home/one/huh/two/white
Uploading /home/one/huh/one/blue
 Uploading /home/one/huh/two/yellow
done
Uploading /home/one/huh/two/purple
done
done
done
done
done
Exception in thread Thread-1 (most likely raised during interpreter shutdown):
Traceback (most recent call last):
  File "/usr/lib64/python2.7/threading.py", line 808, in __bootstrap_inner
  File "/usr/lib64/python2.7/threading.py", line 761, in run
  File "krunchuploader.py", line 97, in worker
  File "/usr/lib64/python2.7/Queue.py", line 168, in get
  File "/usr/lib64/python2.7/threading.py", line 332, in wait
<type 'exceptions.TypeError'>: 'NoneType' object is not callable

更新：我在脚本的末尾放置了一个time.sleep（2），似乎解决了这个问题。我想睡眠允许守护进程在脚本结束并关闭之前完成？我认为主要过程必须等待守护进程完成。

Answer 1

你可以使用“毒丸”来优雅地杀死工人。将所有工作放入队列后，添加一个特殊对象，每个工作一个，工人识别并退出。你可以使线程非守护进程，因此Python会在关闭进程之前等待它们完成。

让worker识别poison并退出的简明方法是在for循环中使用iter()内置的双参数形式：

def do_the_uploads(file_list):
    def worker():
        for item in iter(q.get, poison):
            upload_file(item)

    poison = object()
    num_workers = 5
    q = Queue()
    for i in range(num_workers):
        t = Thread(target=worker)
        t.start()

    for item in file_list:
        q.put(item)

    for i in range(num_workers):
        q.put(poison)

Python线程和解释器关闭 - 这是可修复的还是问题Python问题＃Issue14623

1 个答案: