Skip to content

Commit aae3b8f

Browse files
potiukpotiuk
authored
Individual package READMEs (#8012)
1 parent 5b0f541 commit aae3b8f

File tree

7 files changed

+620
-20
lines changed

7 files changed

+620
-20
lines changed

β€Ž.pre-commit-config.yaml

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,10 +132,16 @@ repos:
132132
hooks:
133133
- id: doctoc
134134
name: Add TOC for md files
135-
files: ^README\.md$|^CONTRIBUTING\.md$|^UPDATING.md$|^dev/README.md$
135+
files: ^README\.md$|^CONTRIBUTING\.md$|^UPDATING.md$|^dev/README\.md$
136136
args:
137137
- "--maxlevel"
138138
- "2"
139+
- repo: https://github.com/thlorenz/doctoc.git
140+
rev: v1.4.0
141+
hooks:
142+
- id: doctoc
143+
name: Add TOC for backport readme files
144+
files: BACKPORT_README\.md$
139145
- repo: meta
140146
hooks:
141147
- id: check-hooks-apply

β€ŽCONTRIBUTING.rst

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1019,5 +1019,13 @@ prepare such packages on your own easily.
10191019
* You can also build sdist (source distribution packages) by running
10201020
``python setup.py <PROVIDER_NAME> sdist`` but this is only needed in case of distribution of the packages.
10211021

1022+
Each package has description generated from the the general ``backport_packages/README.md`` file with the
1023+
following replacements:
1024+
1025+
* ``{{ PACKAGE_NAME }}`` is replaced with the name of the package (``apache-airflow-providers-<NAME>``)
1026+
* ``{{ PACKAGE_DEPENDENCIES }}`` is replaced with list of optional dependencies for the package
1027+
* ``{{ PACKAGE_BACKPORT_README }}`` is replaced with the content of ``BACKPORT_README.md`` file in the
1028+
package folder if it exists.
1029+
10221030
Note that those are unofficial packages yet - they are not yet released in PyPi, but you might use them to
10231031
test the master versions of operators/hooks/sensors in Airflow 1.10.* environment with Python3.6+

β€Žairflow/providers/google/BACKPORT_README.md

Lines changed: 469 additions & 0 deletions
Large diffs are not rendered by default.

β€Žbackport_packages/README.md

Lines changed: 0 additions & 1 deletion
This file was deleted.

β€Žbackport_packages/README.md

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
<!--
2+
Licensed to the Apache Software Foundation (ASF) under one
3+
or more contributor license agreements. See the NOTICE file
4+
distributed with this work for additional information
5+
regarding copyright ownership. The ASF licenses this file
6+
to you under the Apache License, Version 2.0 (the
7+
"License"); you may not use this file except in compliance
8+
with the License. You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing,
13+
software distributed under the License is distributed on an
14+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
KIND, either express or implied. See the License for the
16+
specific language governing permissions and limitations
17+
under the License.
18+
-->
19+
# Backport package {{ PACKAGE_NAME }}
20+
21+
## Description
22+
23+
This is a backport package for Apache Airflow 1.10.* series. It provides Hooks, Operators, Sensors
24+
and Secrets (in Airflow 1.10.10+) that are developed for Apache Airflow 2.0 in a package that
25+
is compatible with Airflow 1.10.* series.
26+
27+
It provides the classes under airflow.providers.{{ PACKAGE_FOLDER }} package.
28+
29+
This is only one of a number of packages released. The current status and description of all
30+
packages are available in the
31+
[Backport Providers Packages document](https://cwiki.apache.org/confluence/display/AIRFLOW/Backported+providers+packages+for+Airflow+1.10.*+series)
32+
33+
## Installation
34+
35+
You can install this package with pip via 'pip install {{ PACKAGE }}' for the existing airflow 1.10 version.
36+
37+
{{ PACKAGE_DEPENDENCIES }}
38+
39+
{{ PACKAGE_BACKPORT_README }}

β€Žbackport_packages/setup_backport_packages.py

Lines changed: 52 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -248,41 +248,76 @@ def _remover(node: LN, capture: Capture, filename: Filename) -> None:
248248
qry.execute(write=True, silent=False, interactive=False)
249249

250250

251+
def get_source_providers_folder():
252+
return os.path.join(dirname(__file__), os.pardir, "airflow", "providers")
253+
254+
255+
def get_providers_folder():
256+
return os.path.join(dirname(__file__), "airflow", "providers")
257+
258+
259+
def get_providers_package_folder(provider_package: str):
260+
return os.path.join(get_providers_folder(), *provider_package.split("."))
261+
262+
263+
def get_provider_package_name(provider_package: str):
264+
return "apache-airflow-providers-" + provider_package.replace(".", "-")
265+
266+
251267
def copy_provider_sources():
252-
build_dir = os.path.join(dirname(__file__), "build")
253-
if os.path.isdir(build_dir):
254-
rmtree(build_dir)
255-
package_providers_dir = os.path.join(dirname(__file__), "airflow", "providers")
268+
rm_build_dir()
269+
package_providers_dir = get_providers_folder()
256270
if os.path.isdir(package_providers_dir):
257271
rmtree(package_providers_dir)
258-
copytree(os.path.join(dirname(__file__), os.pardir, "airflow", "providers"),
259-
os.path.join(dirname(__file__), "airflow", "providers"))
272+
copytree(get_source_providers_folder(), get_providers_folder())
260273

261274

262-
def get_provider_package_name(provider_module: str):
263-
return "apache-airflow-providers-" + provider_module.replace(".", "-")
275+
def rm_build_dir():
276+
build_dir = os.path.join(dirname(__file__), "build")
277+
if os.path.isdir(build_dir):
278+
rmtree(build_dir)
264279

265280

266281
def copy_and_refactor_sources():
267282
copy_provider_sources()
268283
change_import_paths_to_deprecated()
269284

270285

271-
def do_setup_package_providers(provider_module: str, deps: List[str], extras: Dict[str, List[str]]):
286+
def get_long_description(provider_package: str):
287+
providers_folder = get_providers_folder()
288+
package_name = get_provider_package_name(provider_package)
289+
package_folder = get_providers_package_folder(provider_package)
290+
with open(os.path.join(providers_folder, "dependencies.json"), "rt") as dependencies_file:
291+
dependent_packages = json.load(dependencies_file).get(provider_package) or ""
292+
package_dependencies = ""
293+
if dependent_packages:
294+
package_dependencies = "This package has those optional dependencies:\n"
295+
for dependent_package in dependent_packages:
296+
package_dependencies += f" * {get_provider_package_name(dependent_package)}\n"
297+
package_backport_readme = ""
298+
backport_readme_file_path = os.path.join(package_folder, "BACKPORT_README.md")
299+
if os.path.isfile(backport_readme_file_path):
300+
with open(backport_readme_file_path, "tr") as backport_readme:
301+
package_backport_readme = backport_readme.read()
302+
# No jinja here - we do not have any more dependencies in setup.py
303+
return long_description.replace("{{ PACKAGE_NAME }}", package_name) \
304+
.replace("{{ PACKAGE_BACKPORT_README }}", package_backport_readme) \
305+
.replace("{{ PACKAGE_DEPENDENCIES }}", package_dependencies)
306+
307+
308+
def do_setup_package_providers(provider_package: str, deps: List[str], extras: Dict[str, List[str]]):
272309
setup.write_version()
273-
provider_package_name = get_provider_package_name(provider_module)
274-
package_name = f'{provider_package_name}' if provider_module != "providers" \
310+
provider_package_name = get_provider_package_name(provider_package)
311+
package_name = f'{provider_package_name}' if provider_package != "providers" \
275312
else f'apache-airflow-providers'
276-
package_prefix = f'airflow.providers.{provider_module}' if provider_module != 'providers' \
313+
package_prefix = f'airflow.providers.{provider_package}' if provider_package != 'providers' \
277314
else 'airflow.providers'
278315
found_packages = find_packages()
279316
found_packages = [package for package in found_packages if package.startswith(package_prefix)]
280317
setuptools_setup(
281318
name=package_name,
282-
description=f'Back-porting ${package_name} package for Airflow 1.10.*',
283-
long_description=f"""
284-
Back-ported {package_name} to 1.10.* series of Airflow.
285-
""",
319+
description=f'Back-ported ${package_name} package for Airflow 1.10.*',
320+
long_description=get_long_description(provider_package),
286321
long_description_content_type='text/markdown',
287322
license='Apache License 2.0',
288323
version='0.0.1',
@@ -379,6 +414,6 @@ def usage():
379414
del sys.argv[1]
380415
print(f"Building backport package: {provider_package}")
381416
dependencies = find_package_dependencies(package=provider_package)
382-
do_setup_package_providers(provider_module=provider_package,
417+
do_setup_package_providers(provider_package=provider_package,
383418
deps=dependencies,
384419
extras=find_package_extras(provider_package))

β€Žscripts/ci/ci_prepare_packages.sh

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
1919

2020
# shellcheck source=scripts/ci/_script_init.sh
2121
. "$( dirname "${BASH_SOURCE[0]}" )/_script_init.sh"
22-
2322
LIST_OF_DIRS_FILE=$(mktemp)
2423

2524
cd "${MY_DIR}/../../airflow/providers" || exit 1

β€Žtests/test_core_to_contrib.py

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1389,6 +1389,51 @@
13891389
'airflow.providers.mysql.operators.presto_to_mysql.PrestoToMySqlTransfer',
13901390
'airflow.operators.presto_to_mysql.PrestoToMySqlTransfer',
13911391
),
1392+
(
1393+
'airflow.providers.google.cloud.operators.cloud_sql.CloudSQLBaseOperator',
1394+
'airflow.contrib.operators.gcp_sql_operator.CloudSqlBaseOperator'
1395+
),
1396+
(
1397+
'airflow.providers.google.cloud.operators.cloud_sql.CloudSQLCreateInstanceDatabaseOperator',
1398+
'airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabaseCreateOperator'
1399+
),
1400+
(
1401+
'airflow.providers.google.cloud.operators.cloud_sql.CloudSQLCreateInstanceOperator',
1402+
'airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceCreateOperator'
1403+
),
1404+
(
1405+
'airflow.providers.google.cloud.operators.cloud_sql.CloudSQLDeleteInstanceDatabaseOperator',
1406+
'airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabaseDeleteOperator'
1407+
),
1408+
(
1409+
'airflow.providers.google.cloud.operators.cloud_sql.CloudSQLDeleteInstanceOperator',
1410+
'airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDeleteOperator'
1411+
),
1412+
(
1413+
'airflow.providers.google.cloud.operators.cloud_sql.CloudSQLExecuteQueryOperator',
1414+
'airflow.contrib.operators.gcp_sql_operator.CloudSqlQueryOperator'
1415+
),
1416+
(
1417+
'airflow.providers.google.cloud.operators.cloud_sql.CloudSQLExportInstanceOperator',
1418+
'airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceExportOperator'
1419+
),
1420+
(
1421+
'airflow.providers.google.cloud.operators.cloud_sql.CloudSQLImportInstanceOperator',
1422+
'airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceImportOperator'
1423+
),
1424+
(
1425+
'airflow.providers.google.cloud.operators.cloud_sql.CloudSQLInstancePatchOperator',
1426+
'airflow.contrib.operators.gcp_sql_operator.CloudSqlInstancePatchOperator'
1427+
),
1428+
(
1429+
'airflow.providers.google.cloud.operators.cloud_sql.CloudSQLPatchInstanceDatabaseOperator',
1430+
'airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabasePatchOperator'
1431+
),
1432+
(
1433+
'airflow.providers.google.cloud.operators.cloud_storage_transfer_service'
1434+
'.CloudDataTransferServiceS3ToGCSOperator',
1435+
'airflow.contrib.operators.s3_to_gcs_transfer_operator.CloudDataTransferServiceS3ToGCSOperator'
1436+
),
13921437
]
13931438

13941439
SENSOR = [

0 commit comments

Comments
 (0)