diff --git a/dag_factory/config/job_raw_tb_loja_venda_so.yaml b/dag_factory/config/job_raw_tb_loja_venda_so.yaml deleted file mode 100644 index 66501ca..0000000 --- a/dag_factory/config/job_raw_tb_loja_venda_so.yaml +++ /dev/null @@ -1,25 +0,0 @@ -Head: - DocId: "tb_loja_venda_so" - Version: "v1" - StartTime: "2023-01-01 00:00:00" - Schedule: "0 9 * * *" - Retries: "Low" - RetryDelay: "Short" - Gerencia: "Front" - Coord: "RGM" - ValueStream: "Franqueado" - Tags: "factory" - -Table: - ProjectId: "raw-zone-005" - DatasetId: "raw_hana" - TableId: "tb_loja_venda_so" - -Task: - Operator: "SapDataServicesOperator" - NameJob: "JOB_GCP_BLZ_RAW_TB_EUD_BLZWEB_ADDRESS" - Pool: "sap-ds-pool" - Timeout: 480 - -DataQuality: - Operator: "DummyOperator" diff --git a/dag_factory/config/job_trusted_tb_airflow_devops.yaml b/dag_factory/config/job_tb_job_dec.yaml similarity index 88% rename from dag_factory/config/job_trusted_tb_airflow_devops.yaml rename to dag_factory/config/job_tb_job_dec.yaml index 7e50c6d..465b33a 100644 --- a/dag_factory/config/job_trusted_tb_airflow_devops.yaml +++ b/dag_factory/config/job_tb_job_dec.yaml @@ -1,5 +1,5 @@ Head: - DocId: "tb_airflow_devops" + DocId: "tb_teste_dev" Version: "v1" StartTime: "2023-01-01 00:00:00" Schedule: "0 9 * * *" @@ -13,7 +13,7 @@ Head: Table: ProjectId: "trusted-zone" DatasetId: "sellout" - TableId: "tb_real_dia_cupom" + TableId: "tb_teste_dev" Task: Operator: "BigQueryExecuteQueryOperator" diff --git a/dag_factory/config/job_trusted_tb_real_dia.yaml b/dag_factory/config/job_trusted_tb_real_dia.yaml deleted file mode 100644 index a6152d3..0000000 --- a/dag_factory/config/job_trusted_tb_real_dia.yaml +++ /dev/null @@ -1,27 +0,0 @@ -Head: - DocId: "tb_real_dia_cupomdsd" - Version: "v1" - StartTime: "2023-01-01 00:00:00" - Schedule: "0 9 * * *" - Retries: "Low" - RetryDelay: "Short" - Gerencia: "Front" - Coord: "RGM" - ValueStream: "Franqueado" - Tags: "factory" - -Table: - ProjectId: "trusted-zone" - DatasetId: "sellout" - TableId: "tb_real_dia_cupom" - -Task: - Operator: "BigQueryExecuteQueryOperator" - Sql: "CALL prc_load_tb_real_dia_cupom" - -DataQuality: - Operator: "DummyOperator" - -Dependencies: - - TableId: tb_loja_venda_so - PokeInterval: "Short" diff --git a/dag_factory/config/job_trusted_tb_real_dia_cupm.yaml b/dag_factory/config/job_trusted_tb_real_dia_cupm.yaml deleted file mode 100644 index 2a1ee20..0000000 --- a/dag_factory/config/job_trusted_tb_real_dia_cupm.yaml +++ /dev/null @@ -1,27 +0,0 @@ -Head: - DocId: "tb_real_dia_cupom" - Version: "v1" - StartTime: "2023-01-01 00:00:00" - Schedule: "0 9 * * *" - Retries: "Low" - RetryDelay: "Short" - Gerencia: "Front" - Coord: "RGM" - ValueStream: "Franqueado" - Tags: "factory" - -Table: - ProjectId: "trusted-zone" - DatasetId: "sellout" - TableId: "tb_real_dia_cupom" - -Task: - Operator: "BigQueryExecuteQueryOperator" - Sql: "CALL prc_load_tb_real_dia_cupom" - -DataQuality: - Operator: "DummyOperator" - -Dependencies: - - TableId: tb_loja_venda_so - PokeInterval: "Short" diff --git a/dag_factory/config/job_trusted_tb_real_dia_cupom _cp1.yaml b/dag_factory/config/job_trusted_tb_real_dia_cupom _cp1.yaml deleted file mode 100644 index b636f6f..0000000 --- a/dag_factory/config/job_trusted_tb_real_dia_cupom _cp1.yaml +++ /dev/null @@ -1,27 +0,0 @@ -Head: - DocId: "tb_real_dia_TESTE" - Version: "v1" - StartTime: "2023-01-01 00:00:00" - Schedule: "0 9 * * *" - Retries: "Low" - RetryDelay: "Short" - Gerencia: "Front" - Coord: "RGM" - ValueStream: "Franqueado" - Tags: "factory" - -Table: - ProjectId: "trusted-zone" - DatasetId: "sellout" - TableId: "tb_real_dia_cupom" - -Task: - Operator: "BigQueryExecuteQueryOperator" - Sql: "CALL prc_load_tb_real_dia_cupom" - -DataQuality: - Operator: "DummyOperator" - -Dependencies: - - TableId: tb_loja_venda_so - PokeInterval: "Short" \ No newline at end of file diff --git a/dag_factory/config/job_trusted_tb_teste_developer.yaml b/dag_factory/config/job_trusted_tb_teste_developer.yaml deleted file mode 100644 index 7f57a1c..0000000 --- a/dag_factory/config/job_trusted_tb_teste_developer.yaml +++ /dev/null @@ -1,27 +0,0 @@ -Head: - DocId: "tb_teste_developer" - Version: "v1" - StartTime: "2023-01-01 00:00:00" - Schedule: "0 9 * * *" - Retries: "Low" - RetryDelay: "Short" - Gerencia: "Front" - Coord: "RGM" - ValueStream: "Franqueado" - Tags: "factory" - -Table: - ProjectId: "trusted-zone" - DatasetId: "sellout" - TableId: "tb_real_dia_cupom" - -Task: - Operator: "BigQueryExecuteQueryOperator" - Sql: "CALL prc_load_tb_real_dia_cupom" - -DataQuality: - Operator: "DummyOperator" - -Dependencies: - - TableId: tb_loja_venda_so - PokeInterval: "Short" diff --git a/dag_factory/config/job_trusted_tb_teste_developer1.yaml b/dag_factory/config/job_trusted_tb_teste_developer1.yaml deleted file mode 100644 index 143bc34..0000000 --- a/dag_factory/config/job_trusted_tb_teste_developer1.yaml +++ /dev/null @@ -1,27 +0,0 @@ -Head: - DocId: "tb_teste_developer1" - Version: "v1" - StartTime: "2023-01-01 00:00:00" - Schedule: "0 9 * * *" - Retries: "Low" - RetryDelay: "Short" - Gerencia: "Front" - Coord: "RGM" - ValueStream: "Franqueado" - Tags: "factory" - -Table: - ProjectId: "trusted-zone" - DatasetId: "sellout" - TableId: "tb_real_dia_cupom" - -Task: - Operator: "BigQueryExecuteQueryOperator" - Sql: "CALL prc_load_tb_real_dia_cupom" - -DataQuality: - Operator: "DummyOperator" - -Dependencies: - - TableId: tb_loja_venda_so - PokeInterval: "Short" diff --git a/dags/generated/dag_tb_teste_dev.py b/dags/generated/dag_tb_teste_dev.py new file mode 100644 index 0000000..4efbcfe --- /dev/null +++ b/dags/generated/dag_tb_teste_dev.py @@ -0,0 +1,60 @@ +from datetime import datetime, timedelta + +from airflow import DAG +from airflow.operators.dummy import DummyOperator +from libs.bigquery_operator import BigQueryExecuteQueryOperator +from libs.last_execution_external_task_sensor import LastExecutionExternalTaskSensor + +default_args = { + 'owner': 'Gerencia: Front, Coord: RGM', + 'depends_on_past': False, + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=1) +} + +dag = DAG( + 'dag_tb_teste_dev', + start_date=datetime.strptime('2023-01-01 00:00:00', '%Y-%m-%d %H:%M:%S'), + schedule_interval='0 9 * * *', + default_args=default_args, + catchup=False, + tags=['factory'] +) + +with dag: + + + sensor_tb_loja_venda_so = LastExecutionExternalTaskSensor( + task_id='sensor_tb_loja_venda_so', + external_dag_id='dag_tb_loja_venda_so', + external_task_id='dq_tb_loja_venda_so', + poke_interval=2*60, + timeout=10*60, + failed_states=['failed', 'skipped', 'upstream_failed'] + ) + + dq_tb_loja_venda_so = BigQueryExecuteQueryOperator( + task_id='dq_tb_loja_venda_so', + sql= 'CALL prc_data_quality', + use_legacy_sql=False, + depends_on_past=False, + priority="BATCH" + ) + + job_tb_teste_dev = BigQueryExecuteQueryOperator( + task_id='job_tb_teste_dev', + sql= 'CALL prc_load_tb_real_dia_cupom', + use_legacy_sql=False, + depends_on_past=False, + priority="BATCH" + ) + + dq_tb_teste_dev = DummyOperator( + task_id='dq_tb_teste_dev' + ) + + sensor_tb_loja_venda_so >> dq_tb_loja_venda_so >> job_tb_teste_dev + + job_tb_teste_dev >> dq_tb_teste_dev \ No newline at end of file