[C++] Azure SDK's identity.hpp fails on builds with -Werror=attributes

[anjakefala]

Describe the bug, including details regarding any error messages, version, and platform.

Since this commit: 91b2243

I have had this build failure on a Linux machine:

FAILED: src/arrow/CMakeFiles/arrow_objlib.dir/filesystem/azurefs.cc.o 
/home/anja/miniconda3/envs/pyarrow-dev/bin/ccache /usr/bin/g++ -DARROW_EXPORTING -DARROW_EXTRA_ERROR_CONTEXT -DARROW_HAVE_RUNTIME_AVX2 -DARROW_HAVE_RUNTIME_AVX512 -DARROW_HAVE_RUNTIME_BMI2 -DARROW_HAVE_RUNTIME_SSE4_2 -DARROW_HAVE_SSE4_2 -DARROW_HDFS -DARROW_MIMALLOC -DARROW_WITH_BACKTRACE -DARROW_WITH_BROTLI -DARROW_WITH_BZ2 -DARROW_WITH_LZ4 -DARROW_WITH_RE2 -DARROW_WITH_SNAPPY -DARROW_WITH_TIMING_TESTS -DARROW_WITH_UTF8PROC -DARROW_WITH_ZLIB -DARROW_WITH_ZSTD -DAWS_AUTH_USE_IMPORT_EXPORT -DAWS_CAL_USE_IMPORT_EXPORT -DAWS_CHECKSUMS_USE_IMPORT_EXPORT -DAWS_COMMON_USE_IMPORT_EXPORT -DAWS_COMPRESSION_USE_IMPORT_EXPORT -DAWS_CRT_CPP_USE_IMPORT_EXPORT -DAWS_EVENT_STREAM_USE_IMPORT_EXPORT -DAWS_HTTP_USE_IMPORT_EXPORT -DAWS_IO_USE_IMPORT_EXPORT -DAWS_MQTT_USE_IMPORT_EXPORT -DAWS_S3_USE_IMPORT_EXPORT -DAWS_SDKUTILS_USE_IMPORT_EXPORT -DAWS_SDK_VERSION_MAJOR=1 -DAWS_SDK_VERSION_MINOR=11 -DAWS_SDK_VERSION_PATCH=68 -DAWS_USE_EPOLL -DAZ_RTTI -DURI_STATIC_BUILD -I/home/anja/git/arrow/cpp/build/anja-python-debug/src -I/home/anja/git/arrow/cpp/src -I/home/anja/git/arrow/cpp/src/generated -isystem /home/anja/git/arrow/cpp/thirdparty/flatbuffers/include -isystem /home/anja/git/arrow/cpp/thirdparty/hadoop/include -isystem /home/anja/git/arrow/cpp/build/anja-python-debug/jemalloc_ep-prefix/src -isystem /home/anja/git/arrow/cpp/build/anja-python-debug/mimalloc_ep/src/mimalloc_ep/include/mimalloc-2.0 -Wno-noexcept-type -fvisibility-inlines-hidden -fmessage-length=0 -march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-strong -fno-plt -O2 -ffunction-sections -pipe -isystem /home/anja/miniconda3/envs/pyarrow-dev/include -fdiagnostics-color=always  -Wall -Wno-conversion -Wno-sign-conversion -Wunused-result -Wdate-time -fno-semantic-interposition -msse4.2  -g -Werror -O0 -ggdb  -std=c++17 -fPIC -fPIC -MD -MT src/arrow/CMakeFiles/arrow_objlib.dir/filesystem/azurefs.cc.o -MF src/arrow/CMakeFiles/arrow_objlib.dir/filesystem/azurefs.cc.o.d -o src/arrow/CMakeFiles/arrow_objlib.dir/filesystem/azurefs.cc.o -c /home/anja/git/arrow/cpp/src/arrow/filesystem/azurefs.cc
/home/anja/git/arrow/cpp/src/arrow/filesystem/azurefs.cc: In member function ‘arrow::Status arrow::fs::AzureOptions::ConfigureWorkloadIdentityCredential()’:
/home/anja/git/arrow/cpp/src/arrow/filesystem/azurefs.cc:141:57: error: ‘WorkloadIdentityCredential’ is not a member of ‘Azure::Identity’; did you mean ‘ManagedIdentityCredential’?
  141 |   token_credential_ = std::make_shared<Azure::Identity::WorkloadIdentityCredential>();
      |                                                         ^~~~~~~~~~~~~~~~~~~~~~~~~~
      |                                                         ManagedIdentityCredential
/home/anja/git/arrow/cpp/src/arrow/filesystem/azurefs.cc:141:84: error: no matching function for call to ‘make_shared<<expression error> >()’
  141 |   token_credential_ = std::make_shared<Azure::Identity::WorkloadIdentityCredential>();
      |                       ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^~
In file included from /usr/include/c++/11/memory:77,
                 from /home/anja/git/arrow/cpp/src/arrow/filesystem/azurefs.h:20,
                 from /home/anja/git/arrow/cpp/src/arrow/filesystem/azurefs.cc:18:
/usr/include/c++/11/bits/shared_ptr.h:875:5: note: candidate: ‘template<class _Tp, class ... _Args> std::shared_ptr<_Tp> std::make_shared(_Args&& ...)’
  875 |     make_shared(_Args&&... __args)
      |     ^~~~~~~~~~~
/usr/include/c++/11/bits/shared_ptr.h:875:5: note:   template argument deduction/substitution failed:
/home/anja/git/arrow/cpp/src/arrow/filesystem/azurefs.cc:141:84: error: template argument 1 is invalid
  141 |   token_credential_ = std::make_shared<Azure::Identity::WorkloadIdentityCredential>();
      |                       ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^~

This is my CMakeUserPresets.json:

{
    "version": 3,
    "cmakeMinimumRequired": {
      "major": 3,
      "minor": 21,
      "patch": 0
    },
    "configurePresets": [
        {
            "name": "user-base",
            "hidden": true,
            "binaryDir": "${sourceDir}/build/${presetName}",
            "cacheVariables": {
              "ARROW_INSTALL_NAME_RPATH": "OFF"
            }
        },
        {
            "name": "anja-python-debug",
            "inherits": ["ninja-debug-python", "features-filesystems", "user-base"],
            "cacheVariables": {
                "CMAKE_INSTALL_PREFIX": "/home/anja/miniconda3/envs/pyarrow-dev",
                "CMAKE_CXX_STANDARD": "17",
                "ARROW_BUILD_EXAMPLES": "ON",
                "PARQUET_REQUIRE_ENCRYPTION": "ON",
                "ARROW_FILESYSTEM": "ON"
            }
        }
    ]
}

@felipecrv

I have tried clearing my existing build directory before building!

Component(s)

C++

[felipecrv]

@anjakefala some other changes were made to this file after my PR. Is this still an issue on latest main?

[anjakefala]

Unfortunately, yes. =(

[149/534] Building CXX object src/arrow/CMakeFiles/arrow_objlib.dir/filesystem/azurefs.cc.o                                                                                                                                                 
FAILED: src/arrow/CMakeFiles/arrow_objlib.dir/filesystem/azurefs.cc.o                                                                                                                                                                       
/home/anja/miniconda3/envs/pyarrow-dev/bin/ccache /usr/bin/g++ -DARROW_EXPORTING -DARROW_EXTRA_ERROR_CONTEXT -DARROW_HAVE_RUNTIME_AVX2 -DARROW_HAVE_RUNTIME_AVX512 -DARROW_HAVE_RUNTIME_BMI2 -DARROW_HAVE_RUNTIME_SSE4_2 -DARROW_HAVE_SSE4_2
 -DARROW_HDFS -DARROW_MIMALLOC -DARROW_WITH_BACKTRACE -DARROW_WITH_BROTLI -DARROW_WITH_BZ2 -DARROW_WITH_LZ4 -DARROW_WITH_RE2 -DARROW_WITH_SNAPPY -DARROW_WITH_TIMING_TESTS -DARROW_WITH_UTF8PROC -DARROW_WITH_ZLIB -DARROW_WITH_ZSTD -DAWS_A
UTH_USE_IMPORT_EXPORT -DAWS_CAL_USE_IMPORT_EXPORT -DAWS_CHECKSUMS_USE_IMPORT_EXPORT -DAWS_COMMON_USE_IMPORT_EXPORT -DAWS_COMPRESSION_USE_IMPORT_EXPORT -DAWS_CRT_CPP_USE_IMPORT_EXPORT -DAWS_EVENT_STREAM_USE_IMPORT_EXPORT -DAWS_HTTP_USE_I
MPORT_EXPORT -DAWS_IO_USE_IMPORT_EXPORT -DAWS_MQTT_USE_IMPORT_EXPORT -DAWS_S3_USE_IMPORT_EXPORT -DAWS_SDKUTILS_USE_IMPORT_EXPORT -DAWS_SDK_VERSION_MAJOR=1 -DAWS_SDK_VERSION_MINOR=11 -DAWS_SDK_VERSION_PATCH=68 -DAWS_USE_EPOLL -DAZ_RTTI -
DURI_STATIC_BUILD -I/home/anja/git/arrow/cpp/build/anja-python-debug/src -I/home/anja/git/arrow/cpp/src -I/home/anja/git/arrow/cpp/src/generated -isystem /home/anja/git/arrow/cpp/thirdparty/flatbuffers/include -isystem /home/anja/git/ar
row/cpp/thirdparty/hadoop/include -isystem /home/anja/git/arrow/cpp/build/anja-python-debug/jemalloc_ep-prefix/src -isystem /home/anja/git/arrow/cpp/build/anja-python-debug/mimalloc_ep/src/mimalloc_ep/include/mimalloc-2.0 -Wno-noexcept-
type -fvisibility-inlines-hidden -fmessage-length=0 -march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-strong -fno-plt -O2 -ffunction-sections -pipe -isystem /home/anja/miniconda3/envs/pyarrow-dev/include -fdiagnostic
s-color=always  -Wall -Wno-conversion -Wno-sign-conversion -Wunused-result -Wdate-time -fno-semantic-interposition -msse4.2  -g -Werror -O0 -ggdb  -std=c++17 -fPIC -fPIC -MD -MT src/arrow/CMakeFiles/arrow_objlib.dir/filesystem/azurefs.cc.o -MF src/arrow/CMakeFiles/arrow_objlib.dir/filesystem/azurefs.cc.o.d -o src/arrow/CMakeFiles/arrow_objlib.dir/filesystem/azurefs.cc.o -c /home/anja/git/arrow/cpp/src/arrow/filesystem/azurefs.cc
/home/anja/git/arrow/cpp/src/arrow/filesystem/azurefs.cc: In member function ‘arrow::Status arrow::fs::AzureOptions::ConfigureWorkloadIdentityCredential()’:
/home/anja/git/arrow/cpp/src/arrow/filesystem/azurefs.cc:141:57: error: ‘WorkloadIdentityCredential’ is not a member of ‘Azure::Identity’; did you mean ‘ManagedIdentityCredential’?
  141 |   token_credential_ = std::make_shared<Azure::Identity::WorkloadIdentityCredential>();
      |                                                         ^~~~~~~~~~~~~~~~~~~~~~~~~~
      |                                                         ManagedIdentityCredential
/home/anja/git/arrow/cpp/src/arrow/filesystem/azurefs.cc:141:84: error: no matching function for call to ‘make_shared<<expression error> >()’
  141 |   token_credential_ = std::make_shared<Azure::Identity::WorkloadIdentityCredential>();
      |                       ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^~
In file included from /usr/include/c++/11/memory:77,
                 from /home/anja/git/arrow/cpp/src/arrow/filesystem/azurefs.h:20,
                 from /home/anja/git/arrow/cpp/src/arrow/filesystem/azurefs.cc:18:
/usr/include/c++/11/bits/shared_ptr.h:875:5: note: candidate: ‘template<class _Tp, class ... _Args> std::shared_ptr<_Tp> std::make_shared(_Args&& ...)’
  875 |     make_shared(_Args&&... __args)
      |     ^~~~~~~~~~~
/usr/include/c++/11/bits/shared_ptr.h:875:5: note:   template argument deduction/substitution failed:
/home/anja/git/arrow/cpp/src/arrow/filesystem/azurefs.cc:141:84: error: template argument 1 is invalid
  141 |   token_credential_ = std::make_shared<Azure::Identity::WorkloadIdentityCredential>();
      |                       ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^~
[164/534] Building CXX object src/arrow/CMakeFiles/arrow_objlib.dir/ipc/json_simple.cc.o
ninja: build stopped: subcommand failed.

[anjakefala]

I decided to completely rebuild from scratch, including the environment.

I got this error:

CMake Error at cmake_modules/FindAzure.cmake:32 (find_package):
  Could not find a package configuration file provided by "azure-core-cpp"
  with any of the following names:

    azure-core-cppConfig.cmake
    azure-core-cpp-config.cmake

  Add the installation prefix of "azure-core-cpp" to CMAKE_PREFIX_PATH or set
  "azure-core-cpp_DIR" to a directory containing one of the above files.  If
  "azure-core-cpp" provides a separate development package or SDK, be sure it
  has been installed.
Call Stack (most recent call first):
  cmake_modules/ThirdpartyToolchain.cmake:307 (find_package)
  cmake_modules/ThirdpartyToolchain.cmake:5138 (resolve_dependency)
  CMakeLists.txt:542 (include)

[anjakefala]

Some additional information:

For this log: message(STATUS "Using ${ARROW_DEPENDENCY_SOURCE} approach to find dependencies"). Mine is CONDA. This is equivalent to if it was SYSTEM. if SYSTEM is set explicitly the bundled build of azure-core-cpp does not happen.

[anjakefala]

The fix was to set: "Azure_SOURCE":"BUNDLED" in my CmakeUserPreset, and then to do conda install anaconda::libxml2.

And now I again have:

FAILED: src/arrow/CMakeFiles/arrow_objlib.dir/filesystem/azurefs.cc.o 
/home/anja/miniconda3/envs/pyarrow-dev/bin/ccache /home/anja/miniconda3/envs/pyarrow-dev/bin/x86_64-conda-linux-gnu-c++ -DARROW_EXPORTING -DARROW_EXTRA_ERROR_CONTEXT -DARROW_HAVE_RUNTIME_AVX2 -DARROW_HAVE_RUNTIME_AVX512 -DARROW_HAVE_RUNTIME_BMI2 -DARROW_HAVE_RUNTIME_SSE4_2 -DARROW_HAVE_SSE4_2 -DARROW_HDFS -DARROW_MIMALLOC -DARROW_WITH_BACKTRACE -DARROW_WITH_BROTLI -DARROW_WITH_BZ2 -DARROW_WITH_LZ4 -DARROW_WITH_RE2 -DARROW_WITH_SNAPPY -DARROW_WITH_TIMING_TESTS -DARROW_WITH_UTF8PROC -DARROW_WITH_ZLIB -DARROW_WITH_ZSTD -DAWS_AUTH_USE_IMPORT_EXPORT -DAWS_CAL_USE_IMPORT_EXPORT -DAWS_CHECKSUMS_USE_IMPORT_EXPORT -DAWS_COMMON_USE_IMPORT_EXPORT -DAWS_COMPRESSION_USE_IMPORT_EXPORT -DAWS_CRT_CPP_USE_IMPORT_EXPORT -DAWS_EVENT_STREAM_USE_IMPORT_EXPORT -DAWS_HTTP_USE_IMPORT_EXPORT -DAWS_IO_USE_IMPORT_EXPORT -DAWS_MQTT_USE_IMPORT_EXPORT -DAWS_S3_USE_IMPORT_EXPORT -DAWS_SDKUTILS_USE_IMPORT_EXPORT -DAWS_SDK_VERSION_MAJOR=1 -DAWS_SDK_VERSION_MINOR=11 -DAWS_SDK_VERSION_PATCH=68 -DAWS_USE_EPOLL -DAZ_RTTI -DURI_STATIC_BUILD -I/home/anja/git/arrow/cpp/build/anja-python-debug/src -I/home/anja/git/arrow/cpp/src -I/home/anja/git/arrow/cpp/src/generated -I/home/anja/git/arrow/cpp/build/anja-python-debug/_deps/azure_sdk-src/sdk/storage/azure-storage-files-datalake/inc -I/home/anja/git/arrow/cpp/build/anja-python-debug/_deps/azure_sdk-src/sdk/storage/azure-storage-blobs/inc -I/home/anja/git/arrow/cpp/build/anja-python-debug/_deps/azure_sdk-src/sdk/storage/azure-storage-common/inc -I/home/anja/git/arrow/cpp/build/anja-python-debug/_deps/azure_sdk-src/sdk/core/azure-core/inc -I/home/anja/git/arrow/cpp/build/anja-python-debug/_deps/azure_sdk-src/sdk/identity/azure-identity/inc -isystem /home/anja/git/arrow/cpp/thirdparty/flatbuffers/include -isystem /home/anja/git/arrow/cpp/thirdparty/hadoop/include -isystem /home/anja/git/arrow/cpp/build/anja-python-debug/jemalloc_ep-prefix/src -isystem /home/anja/git/arrow/cpp/build/anja-python-debug/mimalloc_ep/src/mimalloc_ep/include/mimalloc-2.0 -Wno-noexcept-type -Wno-self-move -fvisibility-inlines-hidden -fmessage-length=0 -march=nocona -mtune=haswell -ftree-vectorize -fPIC -fstack-protector-strong -fno-plt -O2 -ffunction-sections -pipe -isystem /home/anja/miniconda3/envs/pyarrow-dev/include -fdiagnostics-color=always  -Wall -Wno-conversion -Wno-sign-conversion -Wunused-result -Wdate-time -fno-semantic-interposition -msse4.2  -g -Werror -O0 -ggdb  -std=c++17 -fPIC -pthread -fPIC -MD -MT src/arrow/CMakeFiles/arrow_objlib.dir/filesystem/azurefs.cc.o -MF src/arrow/CMakeFiles/arrow_objlib.dir/filesystem/azurefs.cc.o.d -o src/arrow/CMakeFiles/arrow_objlib.dir/filesystem/azurefs.cc.o -c /home/anja/git/arrow/cpp/src/arrow/filesystem/azurefs.cc
In file included from /home/anja/git/arrow/cpp/build/anja-python-debug/_deps/azure_sdk-src/sdk/identity/azure-identity/inc/azure/identity.hpp:13,
                 from /home/anja/git/arrow/cpp/src/arrow/filesystem/azurefs.cc:21:
/home/anja/git/arrow/cpp/build/anja-python-debug/_deps/azure_sdk-src/sdk/identity/azure-identity/inc/azure/identity/client_certificate_credential.hpp:68:9: error: 'Azure::Identity::ClientCertificateCredential' declared with greater visibility than the type of its field 'Azure::Identity::ClientCertificateCredential::m_pkey' [-Werror=attributes]
   68 |   class ClientCertificateCredential final : public Core::Credentials::TokenCredential {
      |         ^~~~~~~~~~~~~~~~~~~~~~~~~~~

[anjakefala]

It seems that despite this:

5104   message(STATUS "Building Azure SDK for C++ from source")                                                                                          
5105   fetchcontent_declare(azure_sdk                                                                                                                    
5106                        ${FC_DECLARE_COMMON_OPTIONS}                                                                                                 
5107                        URL ${ARROW_AZURE_SDK_URL}                                                                                                   
5108                        URL_HASH "SHA256=${ARROW_AZURE_SDK_BUILD_SHA256_CHECKSUM}")                                                                  
5109   prepare_fetchcontent() 

and

1023 macro(prepare_fetchcontent)                                                                                                                         
1024   set(BUILD_SHARED_LIBS OFF)                                                                                                                        
1025   set(BUILD_STATIC_LIBS ON)                                                                                                                         
1026   set(CMAKE_COMPILE_WARNING_AS_ERROR FALSE)   

in cpp/cmake_modules/ThirdpartyToolchain.cmake a -Werror is getting added at some point in the process. That -Werror results in my build failure.

[anjakefala]

Ah, "ninja-debug-python" builds it in debug mode, and then the warning gets treated as an error.

We might want to take care of that warning, since developers build on debug-mode. But that feels like a seperate issue.