我正在尝试使自定义Airflow操作员如下所示：
QRandomGenerator但是由于某种原因，尽管显式设置了{{1}，但class NewCheckOperator(BaseOperator):
__mapper_args__ = {
'polymorphic_identity': 'NewCheckOperator'
}
template_fields = ('sql1', 'sql2') # type: Iterable[str]
template_ext = ('.hql', '.sql',) # type: Iterable[str]
ui_color = '#91E6F2'
ratio_formulas = {
'max_over_min': lambda cur, ref: float(max(cur, ref)) / min(cur, ref),
'relative_diff': lambda cur, ref: float(abs(cur - ref)) / ref,
}
@apply_defaults
def __init__(
self,
table: str,
template_time: str,
metrics_thresholds: Dict[str, int],
random_param: int,
ratio_formula: Optional[str] = 'max_over_min',
ignore_zero: Optional[bool] = True,
conn_id: Optional[str] = None,
*args, **kwargs
):
super().__init__(*args, **kwargs)
#some logic here
self.sql1 = " '{{ ds }}' "
self.sql2 = "'{{ macros.ds_add(ds, " + str(self.random_param) + ") }}'"
def execute(self, context=None):
hook = self.get_db_hook()
self.log.info('Using ratio formula: %s', self.ratio_formula)
self.log.info('Executing SQL check: %s', self.sql2)
row2 = hook.get_first(self.sql2)
self.log.info('Executing SQL check: %s', self.sql1)
row1 = hook.get_first(self.sql1)
if not row2:
raise AirflowException("The query {} returned None".format(self.sql2))
if not row1:
raise AirflowException("The query {} returned None".format(self.sql1))
#other logic
def get_db_hook(self):
return BaseHook.get_hook(conn_id=self.conn_id)
class BigQueryNewCheckOperator(NewCheckOperator):
template_fields = ('table', 'gcp_conn_id', )
@apply_defaults
def __init__(self,
table: str,
metrics_thresholds: dict,
random_param: int,
date_filter_column: str = 'ds',
gcp_conn_id: str = 'google_cloud_default',
bigquery_conn_id: Optional[str] = None,
use_legacy_sql: bool = True,
*args,
**kwargs) -> None:
super().__init__(
table=table, metrics_thresholds=metrics_thresholds,
date_filter_column=date_filter_column, days_back=days_back,
*args, **kwargs)
if bigquery_conn_id:
warnings.warn(
"The bigquery_conn_id parameter has been deprecated. You should pass "
"the gcp_conn_id parameter.", DeprecationWarning, stacklevel=3)
gcp_conn_id = bigquery_conn_id
self.gcp_conn_id = gcp_conn_id
self.use_legacy_sql = use_legacy_sql
def get_db_hook(self):
return BigQueryHook(bigquery_conn_id=self.gcp_conn_id,
use_legacy_sql=self.use_legacy_sql)
中的模板却无法正常工作（这意味着它们像NewCheckOperator一样被视为str文字） }和" '{{ ds }}' "作为sql1。这是在sql2和template_fields之后进行的，因此我不确定为什么气流宏不起作用。在这里的任何帮助将不胜感激！
答案 0 :(得分：0)
我认为是由于以下原因：
ROLE
roles/bigquery.dataViewer
roles/owner
您应该只在JINJA模板中使用这些宏，或者可以将这些宏作为DAG的参数传递到运算符中。
运算符周围可能有一些装饰器/包装器，用于接收模板化的字段并在输入运算符之前对其进行渲染。本质上，您是在使用呈现的参数，然后将其更改为文字字符串。