Skip to content

Commit fbeddc3

Browse files
authored
Fixup docstring for deprecated DataprocSubmitPigJobOperator (#32739)
1 parent 440c9eb commit fbeddc3

File tree

2 files changed

+14
-4
lines changed

2 files changed

+14
-4
lines changed

β€Žairflow/providers/google/cloud/operators/dataproc.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1096,6 +1096,10 @@ def on_kill(self) -> None:
10961096
class DataprocSubmitPigJobOperator(DataprocJobBaseOperator):
10971097
"""Start a Pig query Job on a Cloud DataProc cluster.
10981098
1099+
.. seealso::
1100+
This operator is deprecated, please use
1101+
:class:`~airflow.providers.google.cloud.operators.dataproc.DataprocSubmitJobOperator`:
1102+
10991103
The parameters of the operation will be passed to the cluster.
11001104
11011105
It's a good practice to define dataproc_* parameters in the default_args of the dag

β€Žtests/system/providers/google/cloud/dataproc/example_dataproc_pig.py

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@
3131
)
3232
from airflow.utils.trigger_rule import TriggerRule
3333

34-
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
3534
DAG_ID = "dataproc_pig"
35+
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
3636
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT")
3737

3838
CLUSTER_NAME = f"cluster-dataproc-pig-{ENV_ID}"
@@ -41,7 +41,6 @@
4141

4242

4343
# Cluster definition
44-
4544
CLUSTER_CONFIG = {
4645
"master_config": {
4746
"num_instances": 1,
@@ -72,7 +71,7 @@
7271
schedule="@once",
7372
start_date=datetime(2021, 1, 1),
7473
catchup=False,
75-
tags=["example", "dataproc"],
74+
tags=["example", "dataproc", "pig"],
7675
) as dag:
7776
create_cluster = DataprocCreateClusterOperator(
7877
task_id="create_cluster",
@@ -94,7 +93,14 @@
9493
trigger_rule=TriggerRule.ALL_DONE,
9594
)
9695

97-
create_cluster >> pig_task >> delete_cluster
96+
(
97+
# TEST SETUP
98+
create_cluster
99+
# TEST BODY
100+
>> pig_task
101+
# TEST TEARDOWN
102+
>> delete_cluster
103+
)
98104

99105
from tests.system.utils.watcher import watcher
100106

0 commit comments

Comments
 (0)