如何从异步芹菜任务（python / flask）中的循环中连续运行函数

Question

我在Flask应用程序中有一个与Celery异步运行的函数

task = import_files.apply_async()

@celery.task()
def import_files():
    while True:
        for filename in filenames:
            urllib.request.urlretrieve(filename)
            #[...]
            aFunction(filenameFromFle)
            #[...]
            os.remove(filename)

def aFunction(filenameFromFile):
    urllib.request.urlretrieve(filenameFromFile)
    parse file(xml) # xml.dom minidom
    os.remove(filenameFromFile)

该任务由

调用

  

celery -A app.celery worker -l info

在加载下一个文件之前，应该加载解析和删除文件，这只是作为一个函数运行时工作正常，但现在文件正在立即下载并同时解析。

我创建了一个最小的工作示例，但它按预期连续执行

from flask import Flask
from celery import Celery

import time

app = Flask(__name__)
app.config.update(
    CELERY_BROKER_URL='redis://localhost:6379',
    CELERY_RESULT_BACKEND='redis://localhost:6379'
)
celery = Celery(app.import_name, backend=app.config['CELERY_RESULT_BACKEND'],
                broker=app.config['CELERY_BROKER_URL'])
celery.conf.update(app.config)
TaskBase = celery.Task
class ContextTask(TaskBase):
    abstract = True
def __call__(self, *args, **kwargs):
    with app.app_context():
        return TaskBase.__call__(self, *args, **kwargs)
celery.Task = ContextTask


@celery.task()
def foo():
    for x in range(1,23):
        print ('start '+str(x))
        y = bar(x)
        print ('end '+str(x))
    return "foo"

def bar(x):
    print('x= ')
    time.sleep(5);
    print(x)
    return 'baz'


task = foo.apply_async()

所以我简化了调试功能

def bar(x):
    print('x= ')
    time.sleep(5);
    print(x)

@celery.task()
def import_files():
    while True:
        print("import data")
        utc = pytz.UTC
        filenames = get_filenames_()
        baseURL = get_baseurl_()

        for filename in filenames:
            outFilePath = filename[:-3]
            metaFile = outFilePath[:-3]+"meta"
            try:
                urllib.request.urlretrieve(baseURL + metaFile,metaFile)
                with open(metaFile, 'r') as f:
                    first_line = f.readline()
                file_last_modified = dateutil.parser.parse(first_line[17:]).replace(tzinfo=utc)
                file_last_import = ImportDates.query.filter_by(filename=outFilePath).first()
                if (file_last_import is None):
                    bar("test")
                else:
                    file_last_import = file_last_import.last_import.replace(tzinfo=utc)
                    if file_last_import<file_last_modified:
                        bar ("test")
                os.remove(metaFile)
            except:
                pass
        print("done")
    return "1"

现在输出

[2017-05-08 14:35:27,002: WARNING/PoolWorker-4] x=
[2017-05-08 14:35:27,247: WARNING/PoolWorker-1] x=
[2017-05-08 14:35:27,357: WARNING/PoolWorker-2] x=
[2017-05-08 14:35:32,007: WARNING/PoolWorker-4] test
[2017-05-08 14:35:32,252: WARNING/PoolWorker-1] test
[2017-05-08 14:35:32,362: WARNING/PoolWorker-2] test   

而不是预期的

[2017-05-08 14:35:27,002: WARNING/PoolWorker-4] x=
[2017-05-08 14:35:32,007: WARNING/PoolWorker-4] test
[2017-05-08 14:35:27,247: WARNING/PoolWorker-1] x=
[2017-05-08 14:35:32,252: WARNING/PoolWorker-1] test
[2017-05-08 14:35:27,357: WARNING/PoolWorker-2] x=
[2017-05-08 14:35:32,362: WARNING/PoolWorker-2] test