[Java][FlightSQL] useSystemTrustStore on MacOS returns user's keychain instead of System Roots keychain

[ravjotbrar]

Describe the bug, including details regarding any error messages, version, and platform.

When attempting to connect to a flight endpoint with the jdbc flight driver, I get the following error even though the Root CA of the endpoint is available in the System Roots keychain:

java.sql.SQLException: cfjd.org.apache.arrow.flight.FlightRuntimeException: UNAVAILABLE: io exception
Channel Pipeline: [SslHandler#0, ProtocolNegotiators$ClientTlsHandler#0, WriteBufferingAndExceptionHandler#0, DefaultChannelPipeline$TailContext#0]
	at org.apache.arrow.driver.jdbc.client.ArrowFlightSqlClientHandler$Builder.build(ArrowFlightSqlClientHandler.java:813)
	at org.apache.arrow.driver.jdbc.ArrowFlightConnection.createNewClientHandler(ArrowFlightConnection.java:114)
	at org.apache.arrow.driver.jdbc.ArrowFlightConnection.createNewConnection(ArrowFlightConnection.java:88)
	at org.apache.arrow.driver.jdbc.ArrowFlightJdbcDriver.connect(ArrowFlightJdbcDriver.java:85)
	at org.apache.arrow.driver.jdbc.ArrowFlightJdbcDriver.connect(ArrowFlightJdbcDriver.java:49)
	at java.sql.DriverManager.getConnection(DriverManager.java:664)
	at java.sql.DriverManager.getConnection(DriverManager.java:208)
	at JdbcClient.main(JdbcClient.java:30)
Caused by: cfjd.org.apache.arrow.flight.FlightRuntimeException: UNAVAILABLE: io exception
Channel Pipeline: [SslHandler#0, ProtocolNegotiators$ClientTlsHandler#0, WriteBufferingAndExceptionHandler#0, DefaultChannelPipeline$TailContext#0]
	at cfjd.org.apache.arrow.flight.CallStatus.toRuntimeException(CallStatus.java:131)
	at cfjd.org.apache.arrow.flight.grpc.StatusUtils.fromGrpcRuntimeException(StatusUtils.java:164)
	at cfjd.org.apache.arrow.flight.grpc.StatusUtils.fromThrowable(StatusUtils.java:185)
	at cfjd.org.apache.arrow.flight.auth2.ClientHandshakeWrapper.doClientHandshake(ClientHandshakeWrapper.java:59)
	at cfjd.org.apache.arrow.flight.FlightClient.handshake(FlightClient.java:212)
	at org.apache.arrow.driver.jdbc.client.utils.ClientAuthenticationUtils.getAuthenticate(ClientAuthenticationUtils.java:73)
	at org.apache.arrow.driver.jdbc.client.ArrowFlightSqlClientHandler$Builder.build(ArrowFlightSqlClientHandler.java:806)
	... 7 more
Caused by: javax.net.ssl.SSLHandshakeException: General OpenSslEngine problem
	at cfjd.io.netty.handler.ssl.ReferenceCountedOpenSslEngine.handshakeException(ReferenceCountedOpenSslEngine.java:1945)
	at cfjd.io.netty.handler.ssl.ReferenceCountedOpenSslEngine.wrap(ReferenceCountedOpenSslEngine.java:852)
	at javax.net.ssl.SSLEngine.wrap(SSLEngine.java:511)
	at cfjd.io.netty.handler.ssl.SslHandler.wrap(SslHandler.java:1130)
	at cfjd.io.netty.handler.ssl.SslHandler.wrapNonAppData(SslHandler.java:974)
	at cfjd.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1507)
	at cfjd.io.netty.handler.ssl.SslHandler.unwrapNonAppData(SslHandler.java:1425)
	at cfjd.io.netty.handler.ssl.SslHandler.access$1800(SslHandler.java:170)
	at cfjd.io.netty.handler.ssl.SslHandler$SslTasksRunner.resumeOnEventExecutor(SslHandler.java:1816)
	at cfjd.io.netty.handler.ssl.SslHandler$SslTasksRunner.access$2000(SslHandler.java:1707)
	at cfjd.io.netty.handler.ssl.SslHandler$SslTasksRunner$2.run(SslHandler.java:1868)
	at cfjd.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173)
	at cfjd.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166)
	at cfjd.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)
	at cfjd.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569)
	at cfjd.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at cfjd.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at cfjd.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	at java.lang.Thread.run(Thread.java:748)
Caused by: sun.security.validator.ValidatorException: PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target
	at sun.security.validator.PKIXValidator.doBuild(PKIXValidator.java:456)
	at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:323)
	at sun.security.validator.Validator.validate(Validator.java:271)
	at sun.security.ssl.X509TrustManagerImpl.validate(X509TrustManagerImpl.java:315)
	at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:278)
	at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:141)
	at cfjd.io.netty.handler.ssl.EnhancingX509ExtendedTrustManager.checkServerTrusted(EnhancingX509ExtendedTrustManager.java:69)
	at cfjd.io.netty.handler.ssl.ReferenceCountedOpenSslClientContext$ExtendedTrustManagerVerifyCallback.verify(ReferenceCountedOpenSslClientContext.java:235)
	at cfjd.io.netty.handler.ssl.ReferenceCountedOpenSslContext$AbstractCertificateVerifier.verify(ReferenceCountedOpenSslContext.java:790)
	at cfjd.io.netty.internal.tcnative.CertificateVerifierTask.runTask(CertificateVerifierTask.java:36)
	at cfjd.io.netty.internal.tcnative.SSLTask.run(SSLTask.java:48)
	at cfjd.io.netty.internal.tcnative.SSLTask.run(SSLTask.java:42)
	at cfjd.io.netty.handler.ssl.ReferenceCountedOpenSslEngine.runAndResetNeedTask(ReferenceCountedOpenSslEngine.java:1534)
	at cfjd.io.netty.handler.ssl.ReferenceCountedOpenSslEngine.access$700(ReferenceCountedOpenSslEngine.java:96)
	at cfjd.io.netty.handler.ssl.ReferenceCountedOpenSslEngine$TaskDecorator.run(ReferenceCountedOpenSslEngine.java:1509)
	at cfjd.io.netty.handler.ssl.SslHandler$SslTasksRunner.run(SslHandler.java:1885)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	... 1 more
Caused by: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target
	at sun.security.provider.certpath.SunCertPathBuilder.build(SunCertPathBuilder.java:141)
	at sun.security.provider.certpath.SunCertPathBuilder.engineBuild(SunCertPathBuilder.java:126)
	at java.security.cert.CertPathBuilder.build(CertPathBuilder.java:280)
	at sun.security.validator.PKI

We dug into this issue deeper and noticed the bug lies in the ClientAuthenticationUtils.getCertificateInputStreamFromSystem method where it tries to get the keystore this way:
keyStoreList.add(getKeyStoreInstance("KeychainStore"));

A simple app to test this method found that the KeychainStore was only returning user's certificates. The following code can help repro the problem:

import java.io.IOException;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.NoSuchProviderException;
import java.security.cert.Certificate;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.*;

public class Main {
    public static void main(String[] args) throws KeyStoreException, NoSuchProviderException, CertificateException, IOException, NoSuchAlgorithmException {
        KeyStore keyStore = KeyStore.getInstance("KeychainStore", "Apple");
        keyStore.load(null, null);
        
        Collections.list(keyStore.aliases()).forEach(alias -> {
            try {
                Certificate certificate = keyStore.getCertificate(alias);
                if (certificate instanceof X509Certificate) {
                    X509Certificate x509Certificate = (X509Certificate) certificate;

                    // Access certificate information
                    System.out.println("Alias: " + alias);
                    System.out.println("Subject: " + x509Certificate.getSubjectDN());
                    System.out.println("Issuer: " + x509Certificate.getIssuerDN());
                    System.out.println("Serial Number: " + x509Certificate.getSerialNumber());
                    System.out.println("Valid From: " + x509Certificate.getNotBefore());
                    System.out.println("Valid Until: " + x509Certificate.getNotAfter());
                    System.out.println("-----");
                }
            } catch (KeyStoreException e) {
                throw new RuntimeException(e);
            }
        });
    }
}

Component(s)

Java