Question

我正在尝试设置动态序列etl作业，这些作业将使用XCOM从第一个运行的任务中获取数据。这是当前的代码：

from airflow import DAG
from airflow.operators.bash_operator import BashOperator
from datetime import datetime as dt, timedelta as td, date
from airflow.models import BaseOperator
from airflow.operators.sensors import ExternalTaskSensor
from airflow.operators.dummy_operator import DummyOperator
from airflow.operators.python_operator import PythonOperator
from airflow.models import Variable

START_DT = dt.combine(dt.today(), dt.min.time())
END_DT = dt.combine(dt.today(), dt.max.time())
NOW = dt.now()
CURRENT_EXEC = '{{ execution_date }}'
TODAY_MD = dt.today().strftime("%m%d")

def datetime_range(start, end, delta):
    """Generates the date range with time separation"""
    current = start
    if not isinstance(delta, td):
            delta = td(**delta)
    while current < end:
        yield current
        current += delta

default_args = {
        'owner': 'test',
        'depends_on_past': False,
        'start_date': START_DT, 
        'email': ['test@test.com'],
        'email_on_failure': False,
        'email_on_retry': False,
        'queue': 'etl',
        'retries': 1,
        'retry_delay': td(minutes=1),
}

dag_name = 'SEQ_TEST_01'

dag = DAG(dag_id=dag_name, default_args=default_args, schedule_interval=td(minutes=30))

def seq_job(sq_dt, **kwargs):
    for count, dt_in in enumerate(datetime_range(START_DT, END_DT, {'minutes':30}), 1):
        if sq_dt < str(dt_in):
            curr_seq = count, dt_in, dt_in + td(minutes=29, seconds=59)
            sequence = int(curr_seq[0])
            return sequence

pycall = PythonOperator(
    task_id='seq_sensor',
    provide_context=True,
    python_callable=seq_job,
    op_kwargs={'sq_dt': CURRENT_EXEC},
    dag=dag)

def group(grp, **context):
    sequence = context['task_instance'].xcom_pull(task_ids='seq_sensor')
    grp = '%0.2d' % grp
    database = 'TEST'
    today_date = '{{ ds_nodash }}'
    return BashOperator(
           task_id='ETL_GRP{}_{}_{}'.format(database, sequence, gap),
           bash_command='script.sh {} {} {} {}'.format(today_date, sequence, database, grp), 
           dag=dag)

complete = DummyOperator(
        task_id='All_Sequences_complete',
        dag=dag)

pycall >> group(1) >> complete
pycall >> group(2) >> complete
pycall >> group(3) >> complete

问题在于，无论我尝试什么，我都会收到此错误：

Traceback (most recent call last):
  File "/usr/local/lib/python2.7/site-packages/airflow/models.py", line 263, in process_file
    m = imp.load_source(mod_name, filepath)
  File "/opt/airflow/incubator-airflow/airflow/dags/new_dag_seq.py", line 66, in <module>
    pycall >> group(1) >> complete
  File "/opt/airflow/incubator-airflow/airflow/dags/new_dag_seq.py", line 56, in group
    sequence = context['task_instance'].xcom_pull(task_ids='seq_sensor')
KeyError: 'task_instance'

不确定它是否是我失踪的小东西，或者我是否有错。对气流仍然不熟悉，并试图设置我们的ETL测试环境，每30分钟运行一次由datetime_range生成的唯一序列号，并基于execution_date变量。

Answer 1

我通过将bash运算符移动到另一个函数并通过以下方式从python运算符中提取数据来解决它：

def bash_out(group, **kwargs):
        sequence = "{{ task_instance.xcom_pull(task_ids='seq_sensor') }}"
        return BashOperator(task_id='ETL_{}_GRP{}'.format(database, group), bash_command='script.sh {} {} {} {}'.format(today_date, database, sequence, group), dag=dag)

并设置依赖项：

pycall >> [bash_out('01'), bash_out('02'), bash_out('03')] >> complete

Answer 2

尝试使用{{1}}代替。

Answer 3

确保 'provide_context': True 存在于 default_args 中。

气流XCOM KeyError：＆＃39; task_instance＆＃39;

3 个答案: