Adds secrets backend/logging/auth information to provider yaml (#17625)

This is preparatory work to automatically generate summary of
secret backends, logging handlers, API auth backends
for all providers.

It adds logging/secrets-backends/auth-backends sections in the
provider.yaml and refactors the code a little to extract common
initialization routines to a decorator as well as common sanity
check that imports classes/modules and checks if community
providers are using correct package prefix.

The provider manager is also simplified by removing the
unnecessary `_add` methods.

Author: potiuk

airflow/cli/cli_parser.py

@@ -1267,6 +1267,24 @@ class GroupCommand(NamedTuple):
         func=lazy_load_command('airflow.cli.commands.provider_command.connection_field_behaviours'),
         args=(ARG_OUTPUT, ARG_VERBOSE),
     ),
+    ActionCommand(
+        name='logging',
+        help='Get information about task logging handlers provided',
+        func=lazy_load_command('airflow.cli.commands.provider_command.logging_list'),
+        args=(ARG_OUTPUT, ARG_VERBOSE),
+    ),
+    ActionCommand(
+        name='secrets',
+        help='Get information about secrets backends provided',
+        func=lazy_load_command('airflow.cli.commands.provider_command.secrets_backends_list'),
+        args=(ARG_OUTPUT, ARG_VERBOSE),
+    ),
+    ActionCommand(
+        name='auth',
+        help='Get information about API auth backends provided',
+        func=lazy_load_command('airflow.cli.commands.provider_command.auth_backend_list'),
+        args=(ARG_OUTPUT, ARG_VERBOSE),
+    ),
 )
 
 USERS_COMMANDS = (

airflow/cli/commands/provider_command.py

@@ -113,3 +113,39 @@ def extra_links_list(args):
             "extra_link_class_name": x,
         },
     )
+
+
+@suppress_logs_and_warning
+def logging_list(args):
+    """Lists all log task handlers at the command line"""
+    AirflowConsole().print_as(
+        data=list(ProvidersManager().logging_class_names),
+        output=args.output,
+        mapper=lambda x: {
+            "logging_class_name": x,
+        },
+    )
+
+
+@suppress_logs_and_warning
+def secrets_backends_list(args):
+    """Lists all secrets backends at the command line"""
+    AirflowConsole().print_as(
+        data=list(ProvidersManager().secrets_backend_class_names),
+        output=args.output,
+        mapper=lambda x: {
+            "secrets_backend_class_name": x,
+        },
+    )
+
+
+@suppress_logs_and_warning
+def auth_backend_list(args):
+    """Lists all API auth backend modules at the command line"""
+    AirflowConsole().print_as(
+        data=list(ProvidersManager().auth_backend_module_names),
+        output=args.output,
+        mapper=lambda x: {
+            "api_auth_backand_module": x,
+        },
+    )

airflow/provider.yaml.schema.json

@@ -210,6 +210,27 @@
     "additional-extras": {
       "type": "object",
       "description": "Additional extras that the provider should have"
+    },
+    "secrets-backends": {
+      "type": "array",
+      "description": "Secrets Backend class names",
+      "items": {
+          "type": "string"
+      }
+    },
+    "logging": {
+      "type": "array",
+      "description": "Logging Task Handlers class names",
+      "items": {
+          "type": "string"
+      }
+    },
+    "auth-backends": {
+      "type": "array",
+      "description": "API Auth Backend module names",
+      "items": {
+          "type": "string"
+      }
     }
   },
   "additionalProperties": false,

airflow/provider_info.schema.json

@@ -27,6 +27,27 @@
       "items": {
         "type": "string"
       }
+    },
+    "secrets-backends": {
+      "type": "array",
+      "description": "Secrets Backend class names",
+      "items": {
+        "type": "string"
+      }
+    },
+    "logging": {
+      "type": "array",
+      "description": "Logging Task Handlers class names",
+      "items": {
+          "type": "string"
+      }
+    },
+    "auth-backends": {
+      "type": "array",
+      "description": "API Auth Backend module names",
+      "items": {
+          "type": "string"
+      }
     }
   },
   "required": [

airflow/providers/amazon/provider.yaml

@@ -399,3 +399,11 @@ hook-class-names:
   - airflow.providers.amazon.aws.hooks.s3.S3Hook
   - airflow.providers.amazon.aws.hooks.base_aws.AwsBaseHook
   - airflow.providers.amazon.aws.hooks.emr.EmrHook
+
+secrets-backends:
+  - airflow.providers.amazon.aws.secrets.secrets_manager.SecretsManagerBackend
+  - airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend
+
+logging:
+  - airflow.providers.amazon.aws.log.s3_task_handler.S3TaskHandler
+  - airflow.providers.amazon.aws.log.cloudwatch_task_handler.CloudwatchTaskHandler

airflow/providers/elasticsearch/provider.yaml

@@ -46,3 +46,6 @@ hooks:
 
 hook-class-names:
   - airflow.providers.elasticsearch.hooks.elasticsearch.ElasticsearchHook
+
+logging:
+  - airflow.providers.elasticsearch.log.es_task_handler.ElasticsearchTaskHandler

airflow/providers/google/provider.yaml

@@ -757,3 +757,13 @@ extra-links:
 additional-extras:
   apache.beam: apache-beam[gcp]
   leveldb: plyvel
+
+secrets-backends:
+  - airflow.providers.google.cloud.secrets.secret_manager.CloudSecretManagerBackend
+
+auth-backends:
+  - airflow.providers.google.common.auth_backend.google_openid
+
+logging:
+  - airflow.providers.google.cloud.log.gcs_task_handler.GCSTaskHandler
+  - airflow.providers.google.cloud.log.stackdriver_task_handler.StackdriverTaskHandler

airflow/providers/hashicorp/provider.yaml

@@ -43,3 +43,6 @@ hooks:
 
 hook-class-names:
   - airflow.providers.hashicorp.hooks.vault.VaultHook
+
+secrets-backends:
+  - airflow.providers.hashicorp.secrets.vault.VaultBackend

airflow/providers/microsoft/azure/provider.yaml

@@ -165,3 +165,9 @@ hook-class-names:
   - airflow.providers.microsoft.azure.hooks.wasb.WasbHook
   - airflow.providers.microsoft.azure.hooks.azure_data_factory.AzureDataFactoryHook
   - airflow.providers.microsoft.azure.hooks.azure_container_registry.AzureContainerRegistryHook
+
+secrets-backends:
+  - airflow.providers.microsoft.azure.secrets.azure_key_vault.AzureKeyVaultBackend
+
+logging:
+  - airflow.providers.microsoft.azure.log.wasb_task_handler.WasbTaskHandler