From dfa9b24f7c9007f4044536d941b4508df0d81349 Mon Sep 17 00:00:00 2001 From: Werikson1 Date: Fri, 5 May 2023 01:07:43 -0300 Subject: [PATCH 1/3] teste --- .../job_trusted_tb_teste_developer_teste.yaml | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 dag_factory/config/job_trusted_tb_teste_developer_teste.yaml diff --git a/dag_factory/config/job_trusted_tb_teste_developer_teste.yaml b/dag_factory/config/job_trusted_tb_teste_developer_teste.yaml new file mode 100644 index 0000000..102a8e3 --- /dev/null +++ b/dag_factory/config/job_trusted_tb_teste_developer_teste.yaml @@ -0,0 +1,27 @@ +Head: + DocId: "tb_teste_developer1_teste" + Version: "v1" + StartTime: "2023-01-01 00:00:00" + Schedule: "0 9 * * *" + Retries: "Low" + RetryDelay: "Short" + Gerencia: "Front" + Coord: "RGM" + ValueStream: "Franqueado" + Tags: "factory" + +Table: + ProjectId: "trusted-zone" + DatasetId: "sellout" + TableId: "tb_real_dia_cupom" + +Task: + Operator: "BigQueryExecuteQueryOperator" + Sql: "CALL prc_load_tb_real_dia_cupom" + +DataQuality: + Operator: "DummyOperator" + +Dependencies: + - TableId: tb_loja_venda_so + PokeInterval: "Short" From 83790e903751d202caaa69398dd7c05ec7e0faa5 Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Fri, 5 May 2023 04:10:01 +0000 Subject: [PATCH 2/3] DAG gerada com sucesso!!! --- dags/generated/dag_tb_airflow_devops.py | 60 +++++++++++++++++++ .../dag_tb_teste_developer1_teste.py | 60 +++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 dags/generated/dag_tb_airflow_devops.py create mode 100644 dags/generated/dag_tb_teste_developer1_teste.py diff --git a/dags/generated/dag_tb_airflow_devops.py b/dags/generated/dag_tb_airflow_devops.py new file mode 100644 index 0000000..6d0d973 --- /dev/null +++ b/dags/generated/dag_tb_airflow_devops.py @@ -0,0 +1,60 @@ +from datetime import datetime, timedelta + +from airflow import DAG +from airflow.operators.dummy import DummyOperator +from libs.bigquery_operator import BigQueryExecuteQueryOperator +from libs.last_execution_external_task_sensor import LastExecutionExternalTaskSensor + +default_args = { + 'owner': 'Gerencia: Front, Coord: RGM', + 'depends_on_past': False, + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=1) +} + +dag = DAG( + 'dag_tb_airflow_devops', + start_date=datetime.strptime('2023-01-01 00:00:00', '%Y-%m-%d %H:%M:%S'), + schedule_interval='0 9 * * *', + default_args=default_args, + catchup=False, + tags=['factory'] +) + +with dag: + + + sensor_tb_loja_venda_so = LastExecutionExternalTaskSensor( + task_id='sensor_tb_loja_venda_so', + external_dag_id='dag_tb_loja_venda_so', + external_task_id='dq_tb_loja_venda_so', + poke_interval=2*60, + timeout=10*60, + failed_states=['failed', 'skipped', 'upstream_failed'] + ) + + dq_tb_loja_venda_so = BigQueryExecuteQueryOperator( + task_id='dq_tb_loja_venda_so', + sql= 'CALL prc_data_quality', + use_legacy_sql=False, + depends_on_past=False, + priority="BATCH" + ) + + job_tb_airflow_devops = BigQueryExecuteQueryOperator( + task_id='job_tb_airflow_devops', + sql= 'CALL prc_load_tb_real_dia_cupom', + use_legacy_sql=False, + depends_on_past=False, + priority="BATCH" + ) + + dq_tb_airflow_devops = DummyOperator( + task_id='dq_tb_airflow_devops' + ) + + sensor_tb_loja_venda_so >> dq_tb_loja_venda_so >> job_tb_airflow_devops + + job_tb_airflow_devops >> dq_tb_airflow_devops \ No newline at end of file diff --git a/dags/generated/dag_tb_teste_developer1_teste.py b/dags/generated/dag_tb_teste_developer1_teste.py new file mode 100644 index 0000000..6bacdda --- /dev/null +++ b/dags/generated/dag_tb_teste_developer1_teste.py @@ -0,0 +1,60 @@ +from datetime import datetime, timedelta + +from airflow import DAG +from airflow.operators.dummy import DummyOperator +from libs.bigquery_operator import BigQueryExecuteQueryOperator +from libs.last_execution_external_task_sensor import LastExecutionExternalTaskSensor + +default_args = { + 'owner': 'Gerencia: Front, Coord: RGM', + 'depends_on_past': False, + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=1) +} + +dag = DAG( + 'dag_tb_teste_developer1_teste', + start_date=datetime.strptime('2023-01-01 00:00:00', '%Y-%m-%d %H:%M:%S'), + schedule_interval='0 9 * * *', + default_args=default_args, + catchup=False, + tags=['factory'] +) + +with dag: + + + sensor_tb_loja_venda_so = LastExecutionExternalTaskSensor( + task_id='sensor_tb_loja_venda_so', + external_dag_id='dag_tb_loja_venda_so', + external_task_id='dq_tb_loja_venda_so', + poke_interval=2*60, + timeout=10*60, + failed_states=['failed', 'skipped', 'upstream_failed'] + ) + + dq_tb_loja_venda_so = BigQueryExecuteQueryOperator( + task_id='dq_tb_loja_venda_so', + sql= 'CALL prc_data_quality', + use_legacy_sql=False, + depends_on_past=False, + priority="BATCH" + ) + + job_tb_teste_developer1_teste = BigQueryExecuteQueryOperator( + task_id='job_tb_teste_developer1_teste', + sql= 'CALL prc_load_tb_real_dia_cupom', + use_legacy_sql=False, + depends_on_past=False, + priority="BATCH" + ) + + dq_tb_teste_developer1_teste = DummyOperator( + task_id='dq_tb_teste_developer1_teste' + ) + + sensor_tb_loja_venda_so >> dq_tb_loja_venda_so >> job_tb_teste_developer1_teste + + job_tb_teste_developer1_teste >> dq_tb_teste_developer1_teste \ No newline at end of file From 69920492e0cdcd6bdc1b07a430d23dd28cc0e1ee Mon Sep 17 00:00:00 2001 From: Werikson Rodrigues <49258737+Werikson1@users.noreply.github.com> Date: Fri, 5 May 2023 01:15:02 -0300 Subject: [PATCH 3/3] Update job_trusted_tb_teste_developer_teste.yaml --- dag_factory/config/job_trusted_tb_teste_developer_teste.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dag_factory/config/job_trusted_tb_teste_developer_teste.yaml b/dag_factory/config/job_trusted_tb_teste_developer_teste.yaml index 102a8e3..cc27ca3 100644 --- a/dag_factory/config/job_trusted_tb_teste_developer_teste.yaml +++ b/dag_factory/config/job_trusted_tb_teste_developer_teste.yaml @@ -1,5 +1,5 @@ Head: - DocId: "tb_teste_developer1_teste" + DocId: "tb_teste_developer1_teste1" Version: "v1" StartTime: "2023-01-01 00:00:00" Schedule: "0 9 * * *"