Skip to content

Commit 9efb989

Browse files
o-nikolaso-nikolas
authored
Clean-up of google cloud example dags (#19436)
- Use start start_date - Use catchup=False - Tidy up the chaining of tasks in some cases
1 parent 3a7e687 commit 9efb989

24 files changed

+118
-64
lines changed

β€Žairflow/providers/google/cloud/example_dags/example_automl_nl_text_classification.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
Example Airflow DAG that uses Google AutoML services.
2121
"""
2222
import os
23+
from datetime import datetime
2324

2425
from airflow import models
2526
from airflow.providers.google.cloud.hooks.automl import CloudAutoMLHook
@@ -30,7 +31,6 @@
3031
AutoMLImportDataOperator,
3132
AutoMLTrainModelOperator,
3233
)
33-
from airflow.utils.dates import days_ago
3434

3535
GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "your-project-id")
3636
GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1")
@@ -60,7 +60,8 @@
6060
with models.DAG(
6161
"example_automl_text_cls",
6262
schedule_interval=None, # Override to match your needs
63-
start_date=days_ago(1),
63+
start_date=datetime(2021, 1, 1),
64+
catchup=False,
6465
tags=['example'],
6566
) as example_dag:
6667
create_dataset_task = AutoMLCreateDatasetOperator(

β€Žairflow/providers/google/cloud/example_dags/example_automl_nl_text_extraction.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
Example Airflow DAG that uses Google AutoML services.
2121
"""
2222
import os
23+
from datetime import datetime
2324

2425
from airflow import models
2526
from airflow.providers.google.cloud.hooks.automl import CloudAutoMLHook
@@ -30,7 +31,6 @@
3031
AutoMLImportDataOperator,
3132
AutoMLTrainModelOperator,
3233
)
33-
from airflow.utils.dates import days_ago
3434

3535
GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "your-project-id")
3636
GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1")
@@ -59,7 +59,8 @@
5959
with models.DAG(
6060
"example_automl_text",
6161
schedule_interval=None, # Override to match your needs
62-
start_date=days_ago(1),
62+
start_date=datetime(2021, 1, 1),
63+
catchup=False,
6364
user_defined_macros={"extract_object_id": extract_object_id},
6465
tags=['example'],
6566
) as example_dag:

β€Žairflow/providers/google/cloud/example_dags/example_automl_nl_text_sentiment.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
Example Airflow DAG that uses Google AutoML services.
2121
"""
2222
import os
23+
from datetime import datetime
2324

2425
from airflow import models
2526
from airflow.providers.google.cloud.hooks.automl import CloudAutoMLHook
@@ -30,7 +31,6 @@
3031
AutoMLImportDataOperator,
3132
AutoMLTrainModelOperator,
3233
)
33-
from airflow.utils.dates import days_ago
3434

3535
GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "your-project-id")
3636
GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1")
@@ -60,7 +60,8 @@
6060
with models.DAG(
6161
"example_automl_text_sentiment",
6262
schedule_interval=None, # Override to match your needs
63-
start_date=days_ago(1),
63+
start_date=datetime(2021, 1, 1),
64+
catchup=False,
6465
user_defined_macros={"extract_object_id": extract_object_id},
6566
tags=['example'],
6667
) as example_dag:

β€Žairflow/providers/google/cloud/example_dags/example_automl_tables.py

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
"""
2222
import os
2323
from copy import deepcopy
24+
from datetime import datetime
2425
from typing import Dict, List
2526

2627
from airflow import models
@@ -40,7 +41,8 @@
4041
AutoMLTablesUpdateDatasetOperator,
4142
AutoMLTrainModelOperator,
4243
)
43-
from airflow.utils.dates import days_ago
44+
45+
START_DATE = datetime(2021, 1, 1)
4446

4547
GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "your-project-id")
4648
GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1")
@@ -85,7 +87,8 @@ def get_target_column_spec(columns_specs: List[Dict], column_name: str) -> str:
8587
with models.DAG(
8688
"example_create_and_deploy",
8789
schedule_interval='@once', # Override to match your needs
88-
start_date=days_ago(1),
90+
start_date=START_DATE,
91+
catchup=False,
8992
user_defined_macros={
9093
"get_target_column_spec": get_target_column_spec,
9194
"target": TARGET,
@@ -197,7 +200,8 @@ def get_target_column_spec(columns_specs: List[Dict], column_name: str) -> str:
197200
with models.DAG(
198201
"example_automl_dataset",
199202
schedule_interval='@once', # Override to match your needs
200-
start_date=days_ago(1),
203+
start_date=START_DATE,
204+
catchup=False,
201205
user_defined_macros={"extract_object_id": extract_object_id},
202206
) as example_dag:
203207
create_dataset_task = AutoMLCreateDatasetOperator(
@@ -265,7 +269,8 @@ def get_target_column_spec(columns_specs: List[Dict], column_name: str) -> str:
265269
with models.DAG(
266270
"example_gcp_get_deploy",
267271
schedule_interval='@once', # Override to match your needs
268-
start_date=days_ago(1),
272+
start_date=START_DATE,
273+
catchup=False,
269274
tags=["example"],
270275
) as get_deploy_dag:
271276
# [START howto_operator_get_model]
@@ -290,7 +295,8 @@ def get_target_column_spec(columns_specs: List[Dict], column_name: str) -> str:
290295
with models.DAG(
291296
"example_gcp_predict",
292297
schedule_interval='@once', # Override to match your needs
293-
start_date=days_ago(1),
298+
start_date=START_DATE,
299+
catchup=False,
294300
tags=["example"],
295301
) as predict_dag:
296302
# [START howto_operator_prediction]

β€Žairflow/providers/google/cloud/example_dags/example_automl_translation.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
Example Airflow DAG that uses Google AutoML services.
2121
"""
2222
import os
23+
from datetime import datetime
2324

2425
from airflow import models
2526
from airflow.providers.google.cloud.hooks.automl import CloudAutoMLHook
@@ -30,7 +31,6 @@
3031
AutoMLImportDataOperator,
3132
AutoMLTrainModelOperator,
3233
)
33-
from airflow.utils.dates import days_ago
3434

3535
GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "your-project-id")
3636
GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1")
@@ -66,7 +66,8 @@
6666
with models.DAG(
6767
"example_automl_translation",
6868
schedule_interval=None, # Override to match your needs
69-
start_date=days_ago(1),
69+
start_date=datetime(2021, 1, 1),
70+
catchup=False,
7071
user_defined_macros={"extract_object_id": extract_object_id},
7172
tags=['example'],
7273
) as example_dag:

β€Žairflow/providers/google/cloud/example_dags/example_automl_video_intelligence_classification.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
Example Airflow DAG that uses Google AutoML services.
2121
"""
2222
import os
23+
from datetime import datetime
2324

2425
from airflow import models
2526
from airflow.providers.google.cloud.hooks.automl import CloudAutoMLHook
@@ -30,7 +31,6 @@
3031
AutoMLImportDataOperator,
3132
AutoMLTrainModelOperator,
3233
)
33-
from airflow.utils.dates import days_ago
3434

3535
GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "your-project-id")
3636
GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1")
@@ -63,7 +63,8 @@
6363
with models.DAG(
6464
"example_automl_video",
6565
schedule_interval=None, # Override to match your needs
66-
start_date=days_ago(1),
66+
start_date=datetime(2021, 1, 1),
67+
catchup=False,
6768
user_defined_macros={"extract_object_id": extract_object_id},
6869
tags=['example'],
6970
) as example_dag:

β€Žairflow/providers/google/cloud/example_dags/example_automl_video_intelligence_tracking.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
Example Airflow DAG that uses Google AutoML services.
2121
"""
2222
import os
23+
from datetime import datetime
2324

2425
from airflow import models
2526
from airflow.providers.google.cloud.hooks.automl import CloudAutoMLHook
@@ -30,7 +31,6 @@
3031
AutoMLImportDataOperator,
3132
AutoMLTrainModelOperator,
3233
)
33-
from airflow.utils.dates import days_ago
3434

3535
GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "your-project-id")
3636
GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1")
@@ -64,7 +64,8 @@
6464
with models.DAG(
6565
"example_automl_video_tracking",
6666
schedule_interval=None, # Override to match your needs
67-
start_date=days_ago(1),
67+
start_date=datetime(2021, 1, 1),
68+
catchup=False,
6869
user_defined_macros={"extract_object_id": extract_object_id},
6970
tags=['example'],
7071
) as example_dag:

β€Žairflow/providers/google/cloud/example_dags/example_automl_vision_classification.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
Example Airflow DAG that uses Google AutoML services.
2121
"""
2222
import os
23+
from datetime import datetime
2324

2425
from airflow import models
2526
from airflow.providers.google.cloud.hooks.automl import CloudAutoMLHook
@@ -30,7 +31,6 @@
3031
AutoMLImportDataOperator,
3132
AutoMLTrainModelOperator,
3233
)
33-
from airflow.utils.dates import days_ago
3434

3535
GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "your-project-id")
3636
GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1")
@@ -61,7 +61,8 @@
6161
with models.DAG(
6262
"example_automl_vision",
6363
schedule_interval=None, # Override to match your needs
64-
start_date=days_ago(1),
64+
start_date=datetime(2021, 1, 1),
65+
catchup=False,
6566
user_defined_macros={"extract_object_id": extract_object_id},
6667
tags=['example'],
6768
) as example_dag:

β€Žairflow/providers/google/cloud/example_dags/example_automl_vision_object_detection.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
Example Airflow DAG that uses Google AutoML services.
2121
"""
2222
import os
23+
from datetime import datetime
2324

2425
from airflow import models
2526
from airflow.providers.google.cloud.hooks.automl import CloudAutoMLHook
@@ -30,7 +31,6 @@
3031
AutoMLImportDataOperator,
3132
AutoMLTrainModelOperator,
3233
)
33-
from airflow.utils.dates import days_ago
3434

3535
GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "your-project-id")
3636
GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1")
@@ -63,7 +63,8 @@
6363
with models.DAG(
6464
"example_automl_vision_detection",
6565
schedule_interval=None, # Override to match your needs
66-
start_date=days_ago(1),
66+
start_date=datetime(2021, 1, 1),
67+
catchup=False,
6768
user_defined_macros={"extract_object_id": extract_object_id},
6869
tags=['example'],
6970
) as example_dag:

β€Žairflow/providers/google/cloud/example_dags/example_bigquery_dts.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
"""
2222
import os
2323
import time
24+
from datetime import datetime
2425

2526
from airflow import models
2627
from airflow.providers.google.cloud.operators.bigquery_dts import (
@@ -29,7 +30,6 @@
2930
BigQueryDeleteDataTransferConfigOperator,
3031
)
3132
from airflow.providers.google.cloud.sensors.bigquery_dts import BigQueryDataTransferServiceTransferRunSensor
32-
from airflow.utils.dates import days_ago
3333

3434
GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "example-project")
3535
BUCKET_URI = os.environ.get("GCP_DTS_BUCKET_URI", "gs://INVALID BUCKET NAME/bank-marketing.csv")
@@ -65,7 +65,8 @@
6565
with models.DAG(
6666
"example_gcp_bigquery_dts",
6767
schedule_interval='@once', # Override to match your needs
68-
start_date=days_ago(1),
68+
start_date=datetime(2021, 1, 1),
69+
catchup=False,
6970
tags=['example'],
7071
) as dag:
7172
# [START howto_bigquery_create_data_transfer]

0 commit comments

Comments
 (0)