Question

嗨，我尝试使用气流并行运行python脚本。但是脚本不是并行运行，而是以顺序方式运行。可以帮忙一下吗： python脚本与通过命令行传递的参数并行运行。在这种情况下，在气流中，我已经通过列表传递了动态参数。这是代码：


#from airflow import models
from airflow.operators import bash_operator,python_operator
from  datetime import timedelta
#from airflow.operators import python_operator
from airflow import DAG
from airflow.utils.email import send_email
from airflow.operators.hive_operator import HiveOperator

yesterday = datetime.datetime.combine(
    datetime.datetime.today() - datetime.timedelta(1),
    datetime.datetime.min.time())


default_dag_args = {
    # Setting start date as yesterday starts the DAG immediately when it is
    # detected in the Cloud Storage bucket.
    'start_date': yesterday,
    'retries' : 1,
    'retry_delay': timedelta(minutes=2),
    #'on_failure_callback': notify_email
}


#dag_id = "sample_test"

dag = DAG('sample_test_parallel', default_args=default_dag_args)
          #schedule_interval=timedelta(days=1)
          #schedule_interval='*/20 * * * *' )

#with DAG(dag_id=dag_id, start_date=datetime(2018, 11, 14),
#             schedule_interval=datetime.timedelta(days=1)) as dag:



from_channels = ['D_A', 'D_B', 'D_C']


for channel in from_channels:
    #env={}
    #env['DATE'] = '{{ ds }}'
    ch=channel
    run_parallel = bash_operator.BashOperator(

        task_id='run_parallel' + ch,
        bash_command="python /root/airflow/dags/test.py '{{ params.channel }}' & ",
        #bash_command="python /root/airflow/dags/test.py {{ ch }} & ",
        params={'channel': ch
                },
        dag=dag)
    run_parallel

如何使用动态参数在气流中并行运行python脚本？

0 个答案: