Skip to content

Commit 0fbb05a

Browse files
authored
Fixup system test for DataprocSubmitJobOperator (SparkSQL job) (#32745)
1 parent fbeddc3 commit 0fbb05a

File tree

2 files changed

+12
-4
lines changed

2 files changed

+12
-4
lines changed

β€Žairflow/providers/google/cloud/operators/dataproc.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1286,6 +1286,10 @@ def execute(self, context: Context):
12861286
class DataprocSubmitSparkSqlJobOperator(DataprocJobBaseOperator):
12871287
"""Start a Spark SQL query Job on a Cloud DataProc cluster.
12881288
1289+
.. seealso::
1290+
This operator is deprecated, please use
1291+
:class:`~airflow.providers.google.cloud.operators.dataproc.DataprocSubmitJobOperator`:
1292+
12891293
:param query: The query or reference to the query file (q extension). (templated)
12901294
:param query_uri: The HCFS URI of the script that contains the SQL queries.
12911295
:param variables: Map of named parameters for the query. (templated)

β€Žtests/system/providers/google/cloud/dataproc/example_dataproc_spark_sql.py

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@
3737

3838
CLUSTER_NAME = f"dataproc-spark-sql-{ENV_ID}"
3939
REGION = "europe-west1"
40-
ZONE = "europe-west1-b"
4140

4241
# Cluster definition
4342
CLUSTER_CONFIG = {
@@ -53,8 +52,6 @@
5352
},
5453
}
5554

56-
TIMEOUT = {"seconds": 1 * 24 * 60 * 60}
57-
5855
# Jobs definitions
5956
# [START how_to_cloud_dataproc_sparksql_config]
6057
SPARK_SQL_JOB = {
@@ -92,7 +89,14 @@
9289
trigger_rule=TriggerRule.ALL_DONE,
9390
)
9491

95-
create_cluster >> spark_sql_task >> delete_cluster
92+
(
93+
# TEST SETUP
94+
create_cluster
95+
# TEST BODY
96+
>> spark_sql_task
97+
# TEST TEARDOWN
98+
>> delete_cluster
99+
)
96100

97101
from tests.system.utils.watcher import watcher
98102

0 commit comments

Comments
 (0)