File tree Expand file tree Collapse file tree 2 files changed +14
-4
lines changed
airflow/providers/google/cloud/operators
tests/system/providers/google/cloud/dataproc Expand file tree Collapse file tree 2 files changed +14
-4
lines changed Original file line number Diff line number Diff line change @@ -1096,6 +1096,10 @@ def on_kill(self) -> None:
1096
1096
class DataprocSubmitPigJobOperator (DataprocJobBaseOperator ):
1097
1097
"""Start a Pig query Job on a Cloud DataProc cluster.
1098
1098
1099
+ .. seealso::
1100
+ This operator is deprecated, please use
1101
+ :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocSubmitJobOperator`:
1102
+
1099
1103
The parameters of the operation will be passed to the cluster.
1100
1104
1101
1105
It's a good practice to define dataproc_* parameters in the default_args of the dag
Original file line number Diff line number Diff line change 31
31
)
32
32
from airflow .utils .trigger_rule import TriggerRule
33
33
34
- ENV_ID = os .environ .get ("SYSTEM_TESTS_ENV_ID" )
35
34
DAG_ID = "dataproc_pig"
35
+ ENV_ID = os .environ .get ("SYSTEM_TESTS_ENV_ID" )
36
36
PROJECT_ID = os .environ .get ("SYSTEM_TESTS_GCP_PROJECT" )
37
37
38
38
CLUSTER_NAME = f"cluster-dataproc-pig-{ ENV_ID } "
41
41
42
42
43
43
# Cluster definition
44
-
45
44
CLUSTER_CONFIG = {
46
45
"master_config" : {
47
46
"num_instances" : 1 ,
72
71
schedule = "@once" ,
73
72
start_date = datetime (2021 , 1 , 1 ),
74
73
catchup = False ,
75
- tags = ["example" , "dataproc" ],
74
+ tags = ["example" , "dataproc" , "pig" ],
76
75
) as dag :
77
76
create_cluster = DataprocCreateClusterOperator (
78
77
task_id = "create_cluster" ,
94
93
trigger_rule = TriggerRule .ALL_DONE ,
95
94
)
96
95
97
- create_cluster >> pig_task >> delete_cluster
96
+ (
97
+ # TEST SETUP
98
+ create_cluster
99
+ # TEST BODY
100
+ >> pig_task
101
+ # TEST TEARDOWN
102
+ >> delete_cluster
103
+ )
98
104
99
105
from tests .system .utils .watcher import watcher
100
106
You canβt perform that action at this time.
0 commit comments