Fixes versioning for pre-release provider packages (#11586)

When we prepare pre-release versions, they are not intended to be
converted to final release versions, so there is no need to replace
version number for them artificially,

For release candidates on the other hand, we should internally use the
"final" version because those packages might be simply renamed to the
final "production" versions.

Fixes #11585

Author: potiuk

.github/workflows/ci.yml

@@ -281,6 +281,7 @@ jobs:
       INSTALL_AIRFLOW_VERSION: "1.10.12"
       PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
       BACKPORT_PACKAGES: "true"
+      VERSION_SUFFIX_FOR_SVN: "rc1"
     if: github.repository == 'apache/airflow' || github.event_name != 'schedule'
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -316,6 +317,8 @@ jobs:
     env:
       INSTALL_AIRFLOW_VERSION: "2.0.0-dev"   # Note that this causes local installation
       PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
+      VERSION_SUFFIX_FOR_PYPI: "a0"
+      VERSION_SUFFIX_FOR_SVN: "a0"
     if: github.repository == 'apache/airflow' || github.event_name != 'schedule'
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"

BREEZE.rst

@@ -1945,14 +1945,17 @@ This is the current syntax for  `./breeze <./breeze>`_:
         Prepares backport packages. You can provide (after --) optional list of packages to prepare.
         If no packages are specified, readme for all packages are generated. You can specify optional
         --version-suffix-for-svn flag to generate rc candidate packages to upload to SVN or
-        --version-suffix-for-pypi flag to generate rc candidates for PyPI packages.
+        --version-suffix-for-pypi flag to generate rc candidates for PyPI packages. You can also
+        provide both suffixes in case you prepare alpha/beta versions.
 
         Examples:
 
         'breeze prepare-provider-packages' or
         'breeze prepare-provider-packages google' or
         'breeze prepare-provider-packages --version-suffix-for-svn rc1 http google amazon' or
         'breeze prepare-provider-packages --version-suffix-for-pypi rc1 http google amazon'
+        'breeze prepare-provider-packages --version-suffix-for-pypi a1
+                                              --version-suffix-for-svn a1 http google amazon'
 
         General form:
 

breeze

@@ -1099,24 +1099,12 @@ function breeze::parse_arguments() {
             shift 2
             ;;
         -S | --version-suffix-for-pypi)
-            if [[ -n ${VERSION_SUFFIX_FOR_SVN=} ]]; then
-                echo
-                echo "You can only set one version suffix - either for PyPI or for SVN"
-                echo
-                exit 1
-            fi
             export VERSION_SUFFIX_FOR_PYPI="${2}"
             echo "Version suffix for PyPI ${VERSION_SUFFIX_FOR_PYPI}"
             echo
             shift 2
             ;;
         -N | --version-suffix-for-svn)
-            if [[ -n ${VERSION_SUFFIX_FOR_PYPI=} ]]; then
-                echo
-                echo "You can only set one version suffix - either for PyPI or for SVN"
-                echo
-                exit 1
-            fi
             export VERSION_SUFFIX_FOR_SVN="${2}"
             echo "Version suffix for SVN ${VERSION_SUFFIX_FOR_SVN}"
             echo
@@ -1725,14 +1713,17 @@ ${CMDNAME} prepare-provider-packages [FLAGS] [PACKAGE_ID ...]
       Prepares backport packages. You can provide (after --) optional list of packages to prepare.
       If no packages are specified, readme for all packages are generated. You can specify optional
       --version-suffix-for-svn flag to generate rc candidate packages to upload to SVN or
-      --version-suffix-for-pypi flag to generate rc candidates for PyPI packages.
+      --version-suffix-for-pypi flag to generate rc candidates for PyPI packages. You can also
+      provide both suffixes in case you prepare alpha/beta versions.
 
       Examples:
 
       '${CMDNAME} prepare-provider-packages' or
       '${CMDNAME} prepare-provider-packages google' or
       '${CMDNAME} prepare-provider-packages --version-suffix-for-svn rc1 http google amazon' or
       '${CMDNAME} prepare-provider-packages --version-suffix-for-pypi rc1 http google amazon'
+      '${CMDNAME} prepare-provider-packages --version-suffix-for-pypi a1
+                                            --version-suffix-for-svn a1 http google amazon'
 
       General form:
 

dev/README.md

@@ -1458,13 +1458,15 @@ Details about maintaining the SEMVER version are going to be discussed and imple
 You can iterate and re-generate the same readme content as many times as you want.
 Generated readme files should be eventually committed to the repository.
 
-### Build an Alpha release for SVN apache upload
+### Build regular provider packages for SVN apache upload
 
-The Alpha artifacts we vote upon should be the exact ones in the future we vote against, without any
-modification than renaming i.e. the contents of the files must be the same between voted
-release candidate and final release. Because of this the version in the built artifacts
-that will become the official Apache releases must not include the rcN suffix. They also need
-to be signed and have checksum files. You can generate the checksum/signature files by running
+There is a slightly different procedure if you build pre-release (alpha/beta) packages and the
+release candidates. For the Alpha artifacts there is no voting and signature/checksum check, so
+we do not need to care about this part. For release candidates - those packages might get promoted
+to "final" packages by just renaming the files, so internally they should keep the final version
+number without the rc suffix, even if they are rc1/rc2/... candidates.
+
+They also need to be signed and have checksum files. You can generate the checksum/signature files by running
 the "dev/sign.sh" script (assuming you have the right PGP key set-up for signing). The script
 generates corresponding .asc and .sha512 files for each file to sign.
 
@@ -1474,20 +1476,37 @@ Currently, we are releasing alpha provider packages together with the main sourc
 we are going to add procedure to release the sources of released provider packages separately.
 Details are in [the related issue](https://github.com/apache/airflow/issues/11425)
 
-* Generate the packages - since we are preparing packages for SVN repo, we should use the right switch. Note
-  that this will clean up dist folder before generating the packages, so it will only contain the packages
-  you intended to build.
+For alpha/beta releases you need to specify both - svn and pyp i - suffixes, and they have to match. This is
+verified by the breeze script. Note that the script will clean up dist folder before generating the
+packages, so it will only contain the packages you intended to build.
+
+* Pre-release packages:
 
 ```shell script
 export VERSION=0.0.1alpha1
 
-./breeze prepare-provider-packages --version-suffix-for-svn alpha1
+./breeze prepare-provider-packages --version-suffix-for-svn a1 --version-suffix-for-pypi a1
 ```
 
 if you ony build few packages, run:
 
 ```shell script
-./breeze prepare-provider-packages --version-suffix-for-svn alpha1 PACKAGE PACKAGE ....
+./breeze prepare-provider-packages --version-suffix-for-svn a1 --version-suffix-for-pypi a1 \
+    PACKAGE PACKAGE ....
+```
+
+* Release candidate packages:
+
+```shell script
+export VERSION=0.0.1alpha1
+
+./breeze prepare-provider-packages --version-suffix-for-svn rc1
+```
+
+if you ony build few packages, run:
+
+```shell script
+./breeze prepare-provider-packages --version-suffix-for-svn rc1 PACKAGE PACKAGE ....
 ```
 
 * Sign all your packages
@@ -1527,24 +1546,30 @@ cd ${AIRFLOW_REPO_ROOT}
 Verify that the files are available at
 [backport-providers](https://dist.apache.org/repos/dist/dev/airflow/backport-providers/)
 
-### Publish the Alpha convenience package to PyPI
+### Publish the Regular convenience package to PyPI
 
-In order to publish to PyPI you just need to build and release packages. The packages should however
-contain the rcN suffix in the version name as well, so you need to use `--version-suffix-for-pypi` switch
-to prepare those packages. Note that these are different packages than the ones used for SVN upload
+
+In case of pre-release versions you build the same packages for both PyPI and SVN so you can simply use
+packages generated in the previous step and you can skip the "prepare" step below.
+
+In order to publish release candidate to PyPI you just need to build and release packages.
+The packages should however contain the rcN suffix in the version file name but not internally in the package,
+so you need to use `--version-suffix-for-pypi` switch to prepare those packages.
+Note that these are different packages than the ones used for SVN upload
 though they should be generated from the same sources.
 
 * Generate the packages with the right RC version (specify the version suffix with PyPI switch). Note that
 this will clean up dist folder before generating the packages, so you will only have the right packages there.
 
 ```shell script
-./breeze prepare-provider-packages --version-suffix-for-pypi alpha1
+./breeze prepare-provider-packages --version-suffix-for-pypi a1 --version-suffix-for-SVN a1
 ```
 
 if you ony build few packages, run:
 
 ```shell script
-./breeze prepare-provider-packages --version-suffix-for-pypi alpha1 PACKAGE PACKAGE ....
+./breeze prepare-provider-packages --version-suffix-for-pypi a1 \
+    PACKAGE PACKAGE ....
 ```
 
 * Verify the artifacts that would be uploaded:

scripts/in_container/run_prepare_provider_packages.sh

@@ -89,16 +89,81 @@ fi
 python3 refactor_provider_packages.py
 
 VERSION_SUFFIX_FOR_PYPI=${VERSION_SUFFIX_FOR_PYPI:=""}
+readonly VERSION_SUFFIX_FOR_PYPI
+
 VERSION_SUFFIX_FOR_SVN=${VERSION_SUFFIX_FOR_SVN:=""}
+readonly VERSION_SUFFIX_FOR_SVN
+
+if [[ ${VERSION_SUFFIX_FOR_PYPI} != "" ]]; then
+    echo
+    echo "Version suffix for PyPI = ${VERSION_SUFFIX_FOR_PYPI}"
+    echo
+fi
+if [[ ${VERSION_SUFFIX_FOR_SVN} != "" ]]; then
+    echo
+    echo "Version suffix for SVN  = ${VERSION_SUFFIX_FOR_SVN}"
+    echo
+fi
 
-echo "Version suffix for PyPI= ${VERSION_SUFFIX_FOR_PYPI}"
-echo "Version suffix for SVN = ${VERSION_SUFFIX_FOR_SVN}"
+if [[ ${VERSION_SUFFIX_FOR_PYPI} != '' && ${VERSION_SUFFIX_FOR_SVN} != '' ]]; then
+    if [[ ${VERSION_SUFFIX_FOR_PYPI} != "${VERSION_SUFFIX_FOR_SVN}" ]]; then
+        >&2 echo
+        >&2 echo "If you specify both version suffixes they must match"
+        >&2 echo "However they are different: '${VERSION_SUFFIX_FOR_PYPI}' vs. '${VERSION_SUFFIX_FOR_SVN}'"
+        >&2 echo
+        exit 1
+    else
+        if [[ ${VERSION_SUFFIX_FOR_PYPI} =~ ^rc ]]; then
+            >&2 echo
+            >&2 echo "If you prepare an RC candidate, you need to specify only one of the PyPI/SVN suffixes"
+            >&2 echo "However you specified both: '${VERSION_SUFFIX_FOR_PYPI}' vs. '${VERSION_SUFFIX_FOR_SVN}'"
+            >&2 echo
+            exit 2
+        fi
+        # Just use one of them - they are both the same:
+        TARGET_VERSION_SUFFIX=${VERSION_SUFFIX_FOR_PYPI}
+    fi
+else
+    if [[ ${VERSION_SUFFIX_FOR_PYPI} == '' && ${VERSION_SUFFIX_FOR_SVN} == '' ]]; then
+        # Preparing "official version"
+        TARGET_VERSION_SUFFIX=""
+    else
+        TARGET_VERSION_SUFFIX=${VERSION_SUFFIX_FOR_PYPI}${VERSION_SUFFIX_FOR_SVN}
+        if [[ ! ${TARGET_VERSION_SUFFIX} =~ rc.* ]]; then
+            >&2 echo
+            >&2 echo "If you prepare an alpha/beta release, you need to specify both PyPI/SVN suffixes"
+            >&2 echo "And they have to match. You specified only one."
+            >&2 echo
+            exit 2
+        fi
+    fi
+fi
+
+readonly TARGET_VERSION_SUFFIX
 
 for PROVIDER_PACKAGE in ${PROVIDER_PACKAGES}
 do
     LOG_FILE=$(mktemp)
     echo "==================================================================================="
-    echo " Preparing ${PACKAGE_TYPE} package ${PROVIDER_PACKAGE}"
+    echo " Preparing ${PACKAGE_TYPE} package ${PROVIDER_PACKAGE} "
+    if [[ "${VERSION_SUFFIX_FOR_PYPI}" == '' && "${VERSION_SUFFIX_FOR_SVN}" == '' ]]; then
+        echo
+        echo " Official version"
+        echo
+    elif [[ "${VERSION_SUFFIX_FOR_PYPI}" == '' ]]; then
+        echo
+        echo " Package Version for SVN release candidate: ${TARGET_VERSION_SUFFIX}"
+        echo
+    elif [[ "${VERSION_SUFFIX_FOR_SVN}" == '' ]]; then
+        echo
+        echo " Package Version for PyPI release candidate: ${TARGET_VERSION_SUFFIX}"
+        echo
+    else
+        # Both SV/PYPI are set to the same version here!
+        echo
+        echo " Pre-release version: ${TARGET_VERSION_SUFFIX}"
+        echo
+    fi
     echo "-----------------------------------------------------------------------------------"
     set +e
     python3 setup_provider_packages.py "${PROVIDER_PACKAGE}" clean --all >"${LOG_FILE}" 2>&1
@@ -160,7 +225,8 @@ cd "${AIRFLOW_SOURCES}" || exit 1
 
 pushd dist
 
-if [[ -n ${VERSION_SUFFIX_FOR_SVN=} ]]; then
+if [[ ${VERSION_SUFFIX_FOR_PYPI} != "${TARGET_VERSION_SUFFIX}" ]]; then
+    # In case we prepare different suffix for SVN and PYPI, rename the generated files
     for FILE in *.tar.gz
     do
         mv "${FILE}" "${FILE//\.tar\.gz/${VERSION_SUFFIX_FOR_SVN}-bin.tar.gz}"
@@ -173,7 +239,7 @@ fi
 
 popd
 
-AIRFLOW_PACKAGES_TGZ_FILE="/files/airflow-packages-$(date +"%Y%m%d-%H%M%S")-${VERSION_SUFFIX_FOR_SVN}${VERSION_SUFFIX_FOR_PYPI}.tar.gz"
+AIRFLOW_PACKAGES_TGZ_FILE="/files/airflow-packages-$(date +"%Y%m%d-%H%M%S")-${TARGET_VERSION_SUFFIX}.tar.gz"
 
 tar -cvzf "${AIRFLOW_PACKAGES_TGZ_FILE}" dist/*.whl dist/*.tar.gz
 echo