From 1f1396f3f0c0cad6379cce65e609111f4f4296dd Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Wed, 8 Sep 2021 09:10:20 -0700 Subject: [PATCH 01/76] Start 1.42.0 development cycle --- build.gradle | 2 +- .../src/test/golden/TestDeprecatedService.java.txt | 2 +- compiler/src/test/golden/TestService.java.txt | 2 +- .../src/testLite/golden/TestDeprecatedService.java.txt | 2 +- compiler/src/testLite/golden/TestService.java.txt | 2 +- core/src/main/java/io/grpc/internal/GrpcUtil.java | 2 +- examples/android/clientcache/app/build.gradle | 10 +++++----- examples/android/helloworld/app/build.gradle | 8 ++++---- examples/android/routeguide/app/build.gradle | 8 ++++---- examples/android/strictmode/app/build.gradle | 8 ++++---- examples/build.gradle | 2 +- examples/example-alts/build.gradle | 2 +- examples/example-gauth/build.gradle | 2 +- examples/example-gauth/pom.xml | 4 ++-- examples/example-hostname/build.gradle | 2 +- examples/example-hostname/pom.xml | 4 ++-- examples/example-jwt-auth/build.gradle | 2 +- examples/example-jwt-auth/pom.xml | 4 ++-- examples/example-tls/build.gradle | 2 +- examples/example-tls/pom.xml | 4 ++-- examples/example-xds/build.gradle | 2 +- examples/pom.xml | 4 ++-- 22 files changed, 40 insertions(+), 40 deletions(-) diff --git a/build.gradle b/build.gradle index 6c099e0cf39..fbd55703aea 100644 --- a/build.gradle +++ b/build.gradle @@ -18,7 +18,7 @@ subprojects { apply plugin: "net.ltgt.errorprone" group = "io.grpc" - version = "1.41.0-SNAPSHOT" // CURRENT_GRPC_VERSION + version = "1.42.0-SNAPSHOT" // CURRENT_GRPC_VERSION repositories { maven { // The google mirror is less flaky than mavenCentral() diff --git a/compiler/src/test/golden/TestDeprecatedService.java.txt b/compiler/src/test/golden/TestDeprecatedService.java.txt index 018849586de..3d1476a5bee 100644 --- a/compiler/src/test/golden/TestDeprecatedService.java.txt +++ b/compiler/src/test/golden/TestDeprecatedService.java.txt @@ -8,7 +8,7 @@ import static io.grpc.MethodDescriptor.generateFullMethodName; * */ @javax.annotation.Generated( - value = "by gRPC proto compiler (version 1.41.0-SNAPSHOT)", + value = "by gRPC proto compiler (version 1.42.0-SNAPSHOT)", comments = "Source: grpc/testing/compiler/test.proto") @io.grpc.stub.annotations.GrpcGenerated @java.lang.Deprecated diff --git a/compiler/src/test/golden/TestService.java.txt b/compiler/src/test/golden/TestService.java.txt index 18af83a9119..929ab5af817 100644 --- a/compiler/src/test/golden/TestService.java.txt +++ b/compiler/src/test/golden/TestService.java.txt @@ -8,7 +8,7 @@ import static io.grpc.MethodDescriptor.generateFullMethodName; * */ @javax.annotation.Generated( - value = "by gRPC proto compiler (version 1.41.0-SNAPSHOT)", + value = "by gRPC proto compiler (version 1.42.0-SNAPSHOT)", comments = "Source: grpc/testing/compiler/test.proto") @io.grpc.stub.annotations.GrpcGenerated public final class TestServiceGrpc { diff --git a/compiler/src/testLite/golden/TestDeprecatedService.java.txt b/compiler/src/testLite/golden/TestDeprecatedService.java.txt index 30f22366765..e088d7c0ede 100644 --- a/compiler/src/testLite/golden/TestDeprecatedService.java.txt +++ b/compiler/src/testLite/golden/TestDeprecatedService.java.txt @@ -8,7 +8,7 @@ import static io.grpc.MethodDescriptor.generateFullMethodName; * */ @javax.annotation.Generated( - value = "by gRPC proto compiler (version 1.41.0-SNAPSHOT)", + value = "by gRPC proto compiler (version 1.42.0-SNAPSHOT)", comments = "Source: grpc/testing/compiler/test.proto") @io.grpc.stub.annotations.GrpcGenerated @java.lang.Deprecated diff --git a/compiler/src/testLite/golden/TestService.java.txt b/compiler/src/testLite/golden/TestService.java.txt index 38626900571..17005100271 100644 --- a/compiler/src/testLite/golden/TestService.java.txt +++ b/compiler/src/testLite/golden/TestService.java.txt @@ -8,7 +8,7 @@ import static io.grpc.MethodDescriptor.generateFullMethodName; * */ @javax.annotation.Generated( - value = "by gRPC proto compiler (version 1.41.0-SNAPSHOT)", + value = "by gRPC proto compiler (version 1.42.0-SNAPSHOT)", comments = "Source: grpc/testing/compiler/test.proto") @io.grpc.stub.annotations.GrpcGenerated public final class TestServiceGrpc { diff --git a/core/src/main/java/io/grpc/internal/GrpcUtil.java b/core/src/main/java/io/grpc/internal/GrpcUtil.java index 54f6d2f41d5..98e1d00585e 100644 --- a/core/src/main/java/io/grpc/internal/GrpcUtil.java +++ b/core/src/main/java/io/grpc/internal/GrpcUtil.java @@ -202,7 +202,7 @@ public byte[] parseAsciiString(byte[] serialized) { public static final Splitter ACCEPT_ENCODING_SPLITTER = Splitter.on(',').trimResults(); - private static final String IMPLEMENTATION_VERSION = "1.41.0-SNAPSHOT"; // CURRENT_GRPC_VERSION + private static final String IMPLEMENTATION_VERSION = "1.42.0-SNAPSHOT"; // CURRENT_GRPC_VERSION /** * The default timeout in nanos for a keepalive ping request. diff --git a/examples/android/clientcache/app/build.gradle b/examples/android/clientcache/app/build.gradle index fab0934405e..99ac5e5db1c 100644 --- a/examples/android/clientcache/app/build.gradle +++ b/examples/android/clientcache/app/build.gradle @@ -34,7 +34,7 @@ android { protobuf { protoc { artifact = 'com.google.protobuf:protoc:3.17.2' } plugins { - grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.41.0-SNAPSHOT' // CURRENT_GRPC_VERSION + grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION } } generateProtoTasks { @@ -54,12 +54,12 @@ dependencies { implementation 'com.android.support:appcompat-v7:27.0.2' // You need to build grpc-java to obtain these libraries below. - implementation 'io.grpc:grpc-okhttp:1.41.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-protobuf-lite:1.41.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-stub:1.41.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-okhttp:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-protobuf-lite:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-stub:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION implementation 'org.apache.tomcat:annotations-api:6.0.53' testImplementation 'junit:junit:4.12' testImplementation 'com.google.truth:truth:1.0.1' - testImplementation 'io.grpc:grpc-testing:1.41.0-SNAPSHOT' // CURRENT_GRPC_VERSION + testImplementation 'io.grpc:grpc-testing:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION } diff --git a/examples/android/helloworld/app/build.gradle b/examples/android/helloworld/app/build.gradle index 1fddbd6d481..f93118f8c66 100644 --- a/examples/android/helloworld/app/build.gradle +++ b/examples/android/helloworld/app/build.gradle @@ -32,7 +32,7 @@ android { protobuf { protoc { artifact = 'com.google.protobuf:protoc:3.17.2' } plugins { - grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.41.0-SNAPSHOT' // CURRENT_GRPC_VERSION + grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION } } generateProtoTasks { @@ -52,8 +52,8 @@ dependencies { implementation 'com.android.support:appcompat-v7:27.0.2' // You need to build grpc-java to obtain these libraries below. - implementation 'io.grpc:grpc-okhttp:1.41.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-protobuf-lite:1.41.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-stub:1.41.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-okhttp:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-protobuf-lite:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-stub:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION implementation 'org.apache.tomcat:annotations-api:6.0.53' } diff --git a/examples/android/routeguide/app/build.gradle b/examples/android/routeguide/app/build.gradle index 250b10c3653..ef9f5593895 100644 --- a/examples/android/routeguide/app/build.gradle +++ b/examples/android/routeguide/app/build.gradle @@ -32,7 +32,7 @@ android { protobuf { protoc { artifact = 'com.google.protobuf:protoc:3.17.2' } plugins { - grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.41.0-SNAPSHOT' // CURRENT_GRPC_VERSION + grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION } } generateProtoTasks { @@ -52,8 +52,8 @@ dependencies { implementation 'com.android.support:appcompat-v7:27.0.2' // You need to build grpc-java to obtain these libraries below. - implementation 'io.grpc:grpc-okhttp:1.41.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-protobuf-lite:1.41.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-stub:1.41.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-okhttp:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-protobuf-lite:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-stub:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION implementation 'org.apache.tomcat:annotations-api:6.0.53' } diff --git a/examples/android/strictmode/app/build.gradle b/examples/android/strictmode/app/build.gradle index f68e8584ef0..98374f10b4e 100644 --- a/examples/android/strictmode/app/build.gradle +++ b/examples/android/strictmode/app/build.gradle @@ -33,7 +33,7 @@ android { protobuf { protoc { artifact = 'com.google.protobuf:protoc:3.17.2' } plugins { - grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.41.0-SNAPSHOT' // CURRENT_GRPC_VERSION + grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION } } generateProtoTasks { @@ -53,8 +53,8 @@ dependencies { implementation 'com.android.support:appcompat-v7:28.0.0' // You need to build grpc-java to obtain these libraries below. - implementation 'io.grpc:grpc-okhttp:1.41.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-protobuf-lite:1.41.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-stub:1.41.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-okhttp:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-protobuf-lite:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-stub:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION implementation 'org.apache.tomcat:annotations-api:6.0.53' } diff --git a/examples/build.gradle b/examples/build.gradle index 03967a41c0d..db8d9b77c17 100644 --- a/examples/build.gradle +++ b/examples/build.gradle @@ -22,7 +22,7 @@ targetCompatibility = 1.7 // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. -def grpcVersion = '1.41.0-SNAPSHOT' // CURRENT_GRPC_VERSION +def grpcVersion = '1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION def protobufVersion = '3.17.2' def protocVersion = protobufVersion diff --git a/examples/example-alts/build.gradle b/examples/example-alts/build.gradle index b1265e89440..2925085f57f 100644 --- a/examples/example-alts/build.gradle +++ b/examples/example-alts/build.gradle @@ -23,7 +23,7 @@ targetCompatibility = 1.7 // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. -def grpcVersion = '1.41.0-SNAPSHOT' // CURRENT_GRPC_VERSION +def grpcVersion = '1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION def protocVersion = '3.17.2' dependencies { diff --git a/examples/example-gauth/build.gradle b/examples/example-gauth/build.gradle index f7f332e6962..bef50494821 100644 --- a/examples/example-gauth/build.gradle +++ b/examples/example-gauth/build.gradle @@ -23,7 +23,7 @@ targetCompatibility = 1.7 // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. -def grpcVersion = '1.41.0-SNAPSHOT' // CURRENT_GRPC_VERSION +def grpcVersion = '1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION def protobufVersion = '3.17.2' def protocVersion = protobufVersion diff --git a/examples/example-gauth/pom.xml b/examples/example-gauth/pom.xml index 1cc4c3ba6a7..ba08911a924 100644 --- a/examples/example-gauth/pom.xml +++ b/examples/example-gauth/pom.xml @@ -6,13 +6,13 @@ jar - 1.41.0-SNAPSHOT + 1.42.0-SNAPSHOT example-gauth https://github.com/grpc/grpc-java UTF-8 - 1.41.0-SNAPSHOT + 1.42.0-SNAPSHOT 3.17.2 1.7 diff --git a/examples/example-hostname/build.gradle b/examples/example-hostname/build.gradle index 44048a78e34..f6e050d0570 100644 --- a/examples/example-hostname/build.gradle +++ b/examples/example-hostname/build.gradle @@ -21,7 +21,7 @@ targetCompatibility = 1.7 // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. -def grpcVersion = '1.41.0-SNAPSHOT' // CURRENT_GRPC_VERSION +def grpcVersion = '1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION def protobufVersion = '3.17.2' dependencies { diff --git a/examples/example-hostname/pom.xml b/examples/example-hostname/pom.xml index 9af512c3952..0a22488d3e3 100644 --- a/examples/example-hostname/pom.xml +++ b/examples/example-hostname/pom.xml @@ -6,13 +6,13 @@ jar - 1.41.0-SNAPSHOT + 1.42.0-SNAPSHOT example-hostname https://github.com/grpc/grpc-java UTF-8 - 1.41.0-SNAPSHOT + 1.42.0-SNAPSHOT 3.17.2 1.7 diff --git a/examples/example-jwt-auth/build.gradle b/examples/example-jwt-auth/build.gradle index cf59da51d4a..851fa5ce095 100644 --- a/examples/example-jwt-auth/build.gradle +++ b/examples/example-jwt-auth/build.gradle @@ -22,7 +22,7 @@ targetCompatibility = 1.7 // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. -def grpcVersion = '1.41.0-SNAPSHOT' // CURRENT_GRPC_VERSION +def grpcVersion = '1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION def protobufVersion = '3.17.2' def protocVersion = protobufVersion diff --git a/examples/example-jwt-auth/pom.xml b/examples/example-jwt-auth/pom.xml index c4ae09cda90..019d0cf4131 100644 --- a/examples/example-jwt-auth/pom.xml +++ b/examples/example-jwt-auth/pom.xml @@ -7,13 +7,13 @@ jar - 1.41.0-SNAPSHOT + 1.42.0-SNAPSHOT example-jwt-auth https://github.com/grpc/grpc-java UTF-8 - 1.41.0-SNAPSHOT + 1.42.0-SNAPSHOT 3.17.2 3.17.2 diff --git a/examples/example-tls/build.gradle b/examples/example-tls/build.gradle index 61f13e050de..a1696cfabdd 100644 --- a/examples/example-tls/build.gradle +++ b/examples/example-tls/build.gradle @@ -23,7 +23,7 @@ targetCompatibility = 1.7 // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. -def grpcVersion = '1.41.0-SNAPSHOT' // CURRENT_GRPC_VERSION +def grpcVersion = '1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION def protocVersion = '3.17.2' dependencies { diff --git a/examples/example-tls/pom.xml b/examples/example-tls/pom.xml index d83a0937725..4a1dfe1be15 100644 --- a/examples/example-tls/pom.xml +++ b/examples/example-tls/pom.xml @@ -6,13 +6,13 @@ jar - 1.41.0-SNAPSHOT + 1.42.0-SNAPSHOT example-tls https://github.com/grpc/grpc-java UTF-8 - 1.41.0-SNAPSHOT + 1.42.0-SNAPSHOT 3.17.2 2.0.34.Final diff --git a/examples/example-xds/build.gradle b/examples/example-xds/build.gradle index 01ef4ba9266..9b5a5ee745f 100644 --- a/examples/example-xds/build.gradle +++ b/examples/example-xds/build.gradle @@ -22,7 +22,7 @@ targetCompatibility = 1.7 // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. -def grpcVersion = '1.41.0-SNAPSHOT' // CURRENT_GRPC_VERSION +def grpcVersion = '1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION def nettyTcNativeVersion = '2.0.31.Final' def protocVersion = '3.17.2' diff --git a/examples/pom.xml b/examples/pom.xml index 156b11fb7ac..93b9e502b90 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -6,13 +6,13 @@ jar - 1.41.0-SNAPSHOT + 1.42.0-SNAPSHOT examples https://github.com/grpc/grpc-java UTF-8 - 1.41.0-SNAPSHOT + 1.42.0-SNAPSHOT 3.17.2 3.17.2 From fb004630019ad6cc7de737e1230d993529a01e21 Mon Sep 17 00:00:00 2001 From: ZhenLian Date: Wed, 8 Sep 2021 11:43:23 -0700 Subject: [PATCH 02/76] fix a flaky test in advanced TLS (#8474) * fix a flaky test in advanced tls --- .../grpc/util/AdvancedTlsX509KeyManager.java | 21 ++++++++------- .../util/AdvancedTlsX509TrustManager.java | 19 ++++++++----- .../java/io/grpc/netty/AdvancedTlsTest.java | 27 ++++++++++++++++--- 3 files changed, 47 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/io/grpc/util/AdvancedTlsX509KeyManager.java b/core/src/main/java/io/grpc/util/AdvancedTlsX509KeyManager.java index adaa1e6e69a..8541c6b5280 100644 --- a/core/src/main/java/io/grpc/util/AdvancedTlsX509KeyManager.java +++ b/core/src/main/java/io/grpc/util/AdvancedTlsX509KeyManager.java @@ -23,12 +23,11 @@ import java.io.FileInputStream; import java.io.IOException; import java.net.Socket; -import java.security.NoSuchAlgorithmException; +import java.security.GeneralSecurityException; import java.security.Principal; import java.security.PrivateKey; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; -import java.security.spec.InvalidKeySpecException; import java.util.Arrays; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; @@ -107,8 +106,7 @@ public String chooseEngineServerAlias(String keyType, Principal[] issuers, * @param key the private key that is going to be used * @param certs the certificate chain that is going to be used */ - public void updateIdentityCredentials(PrivateKey key, X509Certificate[] certs) - throws CertificateException { + public void updateIdentityCredentials(PrivateKey key, X509Certificate[] certs) { // TODO(ZhenLian): explore possibilities to do a crypto check here. this.keyInfo = new KeyInfo(checkNotNull(key, "key"), checkNotNull(certs, "certs")); } @@ -126,10 +124,16 @@ public void updateIdentityCredentials(PrivateKey key, X509Certificate[] certs) * @return an object that caller should close when the file refreshes are not needed */ public Closeable updateIdentityCredentialsFromFile(File keyFile, File certFile, - long period, TimeUnit unit, ScheduledExecutorService executor) { + long period, TimeUnit unit, ScheduledExecutorService executor) throws IOException, + GeneralSecurityException { + UpdateResult newResult = readAndUpdate(keyFile, certFile, 0, 0); + if (!newResult.success) { + throw new GeneralSecurityException( + "Files were unmodified before their initial update. Probably a bug."); + } final ScheduledFuture future = executor.scheduleWithFixedDelay( - new LoadFilePathExecution(keyFile, certFile), 0, period, unit); + new LoadFilePathExecution(keyFile, certFile), period, period, unit); return new Closeable() { @Override public void close() { future.cancel(false); @@ -170,8 +174,7 @@ public void run() { this.currentKeyTime = newResult.keyTime; this.currentCertTime = newResult.certTime; } - } catch (CertificateException | IOException | NoSuchAlgorithmException - | InvalidKeySpecException e) { + } catch (IOException | GeneralSecurityException e) { log.log(Level.SEVERE, "Failed refreshing private key and certificate chain from files. " + "Using previous ones", e); } @@ -201,7 +204,7 @@ public UpdateResult(boolean success, long keyTime, long certTime) { * @return the result of this update execution */ private UpdateResult readAndUpdate(File keyFile, File certFile, long oldKeyTime, long oldCertTime) - throws IOException, CertificateException, NoSuchAlgorithmException, InvalidKeySpecException { + throws IOException, GeneralSecurityException { long newKeyTime = keyFile.lastModified(); long newCertTime = certFile.lastModified(); // We only update when both the key and the certs are updated. diff --git a/core/src/main/java/io/grpc/util/AdvancedTlsX509TrustManager.java b/core/src/main/java/io/grpc/util/AdvancedTlsX509TrustManager.java index f6e366d3219..ad69fe4abfa 100644 --- a/core/src/main/java/io/grpc/util/AdvancedTlsX509TrustManager.java +++ b/core/src/main/java/io/grpc/util/AdvancedTlsX509TrustManager.java @@ -21,6 +21,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.net.Socket; +import java.security.GeneralSecurityException; import java.security.KeyStore; import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; @@ -124,8 +125,8 @@ public void useSystemDefaultTrustCerts() throws CertificateException, KeyStoreEx * * @param trustCerts the trust certificates that are going to be used */ - public void updateTrustCredentials(X509Certificate[] trustCerts) throws CertificateException, - KeyStoreException, NoSuchAlgorithmException, IOException { + public void updateTrustCredentials(X509Certificate[] trustCerts) throws IOException, + GeneralSecurityException { KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); keyStore.load(null, null); int i = 1; @@ -219,10 +220,15 @@ private void checkTrusted(X509Certificate[] chain, String authType, SSLEngine ss * @return an object that caller should close when the file refreshes are not needed */ public Closeable updateTrustCredentialsFromFile(File trustCertFile, long period, TimeUnit unit, - ScheduledExecutorService executor) { + ScheduledExecutorService executor) throws IOException, GeneralSecurityException { + long updatedTime = readAndUpdate(trustCertFile, 0); + if (updatedTime == 0) { + throw new GeneralSecurityException( + "Files were unmodified before their initial update. Probably a bug."); + } final ScheduledFuture future = executor.scheduleWithFixedDelay( - new LoadFilePathExecution(trustCertFile), 0, period, unit); + new LoadFilePathExecution(trustCertFile), period, period, unit); return new Closeable() { @Override public void close() { future.cancel(false); @@ -243,8 +249,7 @@ public LoadFilePathExecution(File file) { public void run() { try { this.currentTime = readAndUpdate(this.file, this.currentTime); - } catch (CertificateException | IOException | KeyStoreException - | NoSuchAlgorithmException e) { + } catch (IOException | GeneralSecurityException e) { log.log(Level.SEVERE, "Failed refreshing trust CAs from file. Using previous CAs", e); } } @@ -259,7 +264,7 @@ public void run() { * @return oldTime if failed or the modified time is not changed, otherwise the new modified time */ private long readAndUpdate(File trustCertFile, long oldTime) - throws CertificateException, IOException, KeyStoreException, NoSuchAlgorithmException { + throws IOException, GeneralSecurityException { long newTime = trustCertFile.lastModified(); if (newTime == oldTime) { return oldTime; diff --git a/netty/src/test/java/io/grpc/netty/AdvancedTlsTest.java b/netty/src/test/java/io/grpc/netty/AdvancedTlsTest.java index 7dd5ec75e54..9e0d8170c40 100644 --- a/netty/src/test/java/io/grpc/netty/AdvancedTlsTest.java +++ b/netty/src/test/java/io/grpc/netty/AdvancedTlsTest.java @@ -45,6 +45,7 @@ import java.io.File; import java.io.IOException; import java.net.Socket; +import java.security.GeneralSecurityException; import java.security.NoSuchAlgorithmException; import java.security.PrivateKey; import java.security.cert.CertificateException; @@ -169,7 +170,6 @@ public void advancedTlsKeyManagerTrustManagerMutualTlsTest() throws Exception { .clientAuth(ClientAuth.REQUIRE).build(); server = Grpc.newServerBuilderForPort(0, serverCredentials).addService( new SimpleServiceImpl()).build().start(); - TimeUnit.SECONDS.sleep(5); // Create a client with the key manager and trust manager. AdvancedTlsX509KeyManager clientKeyManager = new AdvancedTlsX509KeyManager(); clientKeyManager.updateIdentityCredentials(clientKey0, clientCert0); @@ -232,7 +232,6 @@ public void verifyPeerCertificate(X509Certificate[] peerCertChain, String authTy .clientAuth(ClientAuth.REQUIRE).build(); server = Grpc.newServerBuilderForPort(0, serverCredentials).addService( new SimpleServiceImpl()).build().start(); - TimeUnit.SECONDS.sleep(5); AdvancedTlsX509KeyManager clientKeyManager = new AdvancedTlsX509KeyManager(); clientKeyManager.updateIdentityCredentials(clientKey0, clientCert0); @@ -307,7 +306,6 @@ public void verifyPeerCertificate(X509Certificate[] peerCertChain, String authTy .clientAuth(ClientAuth.REQUIRE).build(); server = Grpc.newServerBuilderForPort(0, serverCredentials).addService( new SimpleServiceImpl()).build().start(); - TimeUnit.SECONDS.sleep(5); AdvancedTlsX509KeyManager clientKeyManager = new AdvancedTlsX509KeyManager(); clientKeyManager.updateIdentityCredentials(clientKey0, clientCert0); @@ -359,7 +357,6 @@ public void onFileReloadingKeyManagerTrustManagerTest() throws Exception { .clientAuth(ClientAuth.REQUIRE).build(); server = Grpc.newServerBuilderForPort(0, serverCredentials).addService( new SimpleServiceImpl()).build().start(); - TimeUnit.SECONDS.sleep(5); // Create a client to connect. AdvancedTlsX509KeyManager clientKeyManager = new AdvancedTlsX509KeyManager(); Closeable clientKeyShutdown = clientKeyManager.updateIdentityCredentialsFromFile(clientKey0File, @@ -391,6 +388,28 @@ public void onFileReloadingKeyManagerTrustManagerTest() throws Exception { clientTrustShutdown.close(); } + @Test + public void onFileReloadingKeyManagerBadInitialContentTest() throws Exception { + exceptionRule.expect(GeneralSecurityException.class); + AdvancedTlsX509KeyManager keyManager = new AdvancedTlsX509KeyManager(); + // We swap the order of key and certificates to intentionally create an exception. + Closeable keyShutdown = keyManager.updateIdentityCredentialsFromFile(serverCert0File, + serverKey0File, 100, TimeUnit.MILLISECONDS, executor); + keyShutdown.close(); + } + + @Test + public void onFileReloadingTrustManagerBadInitialContentTest() throws Exception { + exceptionRule.expect(GeneralSecurityException.class); + AdvancedTlsX509TrustManager trustManager = AdvancedTlsX509TrustManager.newBuilder() + .setVerification(Verification.CERTIFICATE_ONLY_VERIFICATION) + .build(); + // We pass in a key as the trust certificates to intentionally create an exception. + Closeable trustShutdown = trustManager.updateTrustCredentialsFromFile(serverKey0File, + 100, TimeUnit.MILLISECONDS, executor); + trustShutdown.close(); + } + @Test public void keyManagerAliasesTest() throws Exception { AdvancedTlsX509KeyManager km = new AdvancedTlsX509KeyManager(); From f71eedff40ca809d93acc8ef17816fad657e53cf Mon Sep 17 00:00:00 2001 From: sanjaypujare Date: Wed, 8 Sep 2021 22:38:26 +0000 Subject: [PATCH 03/76] xds: remove hashCode() and equals() for SslContextProviderSupplier (#8496) --- .../sds/SslContextProviderSupplier.java | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/internal/sds/SslContextProviderSupplier.java b/xds/src/main/java/io/grpc/xds/internal/sds/SslContextProviderSupplier.java index 3300c22b2bf..17fc442e7da 100644 --- a/xds/src/main/java/io/grpc/xds/internal/sds/SslContextProviderSupplier.java +++ b/xds/src/main/java/io/grpc/xds/internal/sds/SslContextProviderSupplier.java @@ -25,7 +25,6 @@ import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; import io.grpc.xds.TlsContextManager; import io.netty.handler.ssl.SslContext; -import java.util.Objects; /** * Enables Client or server side to initialize this object with the received {@link BaseTlsContext} @@ -119,26 +118,6 @@ public synchronized void close() { shutdown = true; } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - SslContextProviderSupplier that = (SslContextProviderSupplier) o; - return shutdown == that.shutdown - && Objects.equals(tlsContext, that.tlsContext) - && Objects.equals(tlsContextManager, that.tlsContextManager) - && Objects.equals(sslContextProvider, that.sslContextProvider); - } - - @Override - public int hashCode() { - return Objects.hash(tlsContext, tlsContextManager, sslContextProvider, shutdown); - } - @Override public String toString() { return MoreObjects.toStringHelper(this) From 22603810b97ed2a3f7ff82aed1ec7fe136aea7ff Mon Sep 17 00:00:00 2001 From: sanjaypujare Date: Wed, 8 Sep 2021 23:06:21 +0000 Subject: [PATCH 04/76] xds: use the new cert-provider instances if present (#8494) --- .../java/io/grpc/xds/ClientXdsClient.java | 95 +++++++++++-------- .../CertProviderClientSslContextProvider.java | 28 ++---- .../CertProviderServerSslContextProvider.java | 28 ++---- .../CertProviderSslContextProvider.java | 48 ++++++++++ .../internal/sds/CommonTlsContextUtil.java | 14 ++- .../io/grpc/xds/ClientXdsClientDataTest.java | 32 +++++-- .../io/grpc/xds/ClientXdsClientTestBase.java | 41 +++++++- .../io/grpc/xds/ClientXdsClientV2Test.java | 6 ++ .../io/grpc/xds/ClientXdsClientV3Test.java | 16 ++++ ...tProviderClientSslContextProviderTest.java | 84 ++++++++++++++++ ...tProviderServerSslContextProviderTest.java | 94 ++++++++++++++++++ .../sds/CommonTlsContextTestsUtil.java | 83 ++++++++++++++++ 12 files changed, 474 insertions(+), 95 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/ClientXdsClient.java b/xds/src/main/java/io/grpc/xds/ClientXdsClient.java index 83845515978..21cf78b1269 100644 --- a/xds/src/main/java/io/grpc/xds/ClientXdsClient.java +++ b/xds/src/main/java/io/grpc/xds/ClientXdsClient.java @@ -456,10 +456,6 @@ static void validateCommonTlsContext( if (commonTlsContext.hasTlsParams()) { throw new ResourceInvalidException("common-tls-context with tls_params is not supported"); } - if (commonTlsContext.hasValidationContext()) { - throw new ResourceInvalidException( - "common-tls-context with validation_context is not supported"); - } if (commonTlsContext.hasValidationContextSdsSecretConfig()) { throw new ResourceInvalidException( "common-tls-context with validation_context_sds_secret_config is not supported"); @@ -473,54 +469,50 @@ static void validateCommonTlsContext( "common-tls-context with validation_context_certificate_provider_instance is not" + " supported"); } - String certInstanceName = null; - if (!commonTlsContext.hasTlsCertificateCertificateProviderInstance()) { + String certInstanceName = getIdentityCertInstanceName(commonTlsContext); + if (certInstanceName == null) { if (server) { throw new ResourceInvalidException( - "tls_certificate_certificate_provider_instance is required in downstream-tls-context"); + "tls_certificate_provider_instance is required in downstream-tls-context"); } if (commonTlsContext.getTlsCertificatesCount() > 0) { throw new ResourceInvalidException( - "common-tls-context with tls_certificates is not supported"); + "tls_certificate_provider_instance is unset"); } if (commonTlsContext.getTlsCertificateSdsSecretConfigsCount() > 0) { throw new ResourceInvalidException( - "common-tls-context with tls_certificate_sds_secret_configs is not supported"); + "tls_certificate_provider_instance is unset"); } if (commonTlsContext.hasTlsCertificateCertificateProvider()) { throw new ResourceInvalidException( - "common-tls-context with tls_certificate_certificate_provider is not supported"); - } - } else { - certInstanceName = commonTlsContext.getTlsCertificateCertificateProviderInstance() - .getInstanceName(); - } - if (certInstanceName != null) { - if (certProviderInstances == null || !certProviderInstances.contains(certInstanceName)) { - throw new ResourceInvalidException( - "CertificateProvider instance name '" + certInstanceName - + "' not defined in the bootstrap file."); + "tls_certificate_provider_instance is unset"); } + } else if (certProviderInstances == null || !certProviderInstances.contains(certInstanceName)) { + throw new ResourceInvalidException( + "CertificateProvider instance name '" + certInstanceName + + "' not defined in the bootstrap file."); } - String rootCaInstanceName = null; - if (!commonTlsContext.hasCombinedValidationContext()) { + String rootCaInstanceName = getRootCertInstanceName(commonTlsContext); + if (rootCaInstanceName == null) { if (!server) { throw new ResourceInvalidException( - "combined_validation_context is required in upstream-tls-context"); + "ca_certificate_provider_instance is required in upstream-tls-context"); } } else { - CommonTlsContext.CombinedCertificateValidationContext combinedCertificateValidationContext - = commonTlsContext.getCombinedValidationContext(); - if (!combinedCertificateValidationContext.hasValidationContextCertificateProviderInstance()) { + if (certProviderInstances == null || !certProviderInstances.contains(rootCaInstanceName)) { throw new ResourceInvalidException( - "validation_context_certificate_provider_instance is required in" - + " combined_validation_context"); - } - rootCaInstanceName = combinedCertificateValidationContext - .getValidationContextCertificateProviderInstance().getInstanceName(); - if (combinedCertificateValidationContext.hasDefaultValidationContext()) { - CertificateValidationContext certificateValidationContext - = combinedCertificateValidationContext.getDefaultValidationContext(); + "ca_certificate_provider_instance name '" + rootCaInstanceName + + "' not defined in the bootstrap file."); + } + CertificateValidationContext certificateValidationContext = null; + if (commonTlsContext.hasValidationContext()) { + certificateValidationContext = commonTlsContext.getValidationContext(); + } else if (commonTlsContext.hasCombinedValidationContext() && commonTlsContext + .getCombinedValidationContext().hasDefaultValidationContext()) { + certificateValidationContext = commonTlsContext.getCombinedValidationContext() + .getDefaultValidationContext(); + } + if (certificateValidationContext != null) { if (certificateValidationContext.getMatchSubjectAltNamesCount() > 0 && server) { throw new ResourceInvalidException( "match_subject_alt_names only allowed in upstream_tls_context"); @@ -547,13 +539,38 @@ static void validateCommonTlsContext( } } } - if (rootCaInstanceName != null) { - if (certProviderInstances == null || !certProviderInstances.contains(rootCaInstanceName)) { - throw new ResourceInvalidException( - "ValidationContextProvider instance name '" + rootCaInstanceName - + "' not defined in the bootstrap file."); + } + + private static String getIdentityCertInstanceName(CommonTlsContext commonTlsContext) { + if (commonTlsContext.hasTlsCertificateProviderInstance()) { + return commonTlsContext.getTlsCertificateProviderInstance().getInstanceName(); + } else if (commonTlsContext.hasTlsCertificateCertificateProviderInstance()) { + return commonTlsContext.getTlsCertificateCertificateProviderInstance().getInstanceName(); + } + return null; + } + + private static String getRootCertInstanceName(CommonTlsContext commonTlsContext) { + if (commonTlsContext.hasValidationContext()) { + if (commonTlsContext.getValidationContext().hasCaCertificateProviderInstance()) { + return commonTlsContext.getValidationContext().getCaCertificateProviderInstance() + .getInstanceName(); + } + } else if (commonTlsContext.hasCombinedValidationContext()) { + CommonTlsContext.CombinedCertificateValidationContext combinedCertificateValidationContext + = commonTlsContext.getCombinedValidationContext(); + if (combinedCertificateValidationContext.hasDefaultValidationContext() + && combinedCertificateValidationContext.getDefaultValidationContext() + .hasCaCertificateProviderInstance()) { + return combinedCertificateValidationContext.getDefaultValidationContext() + .getCaCertificateProviderInstance().getInstanceName(); + } else if (combinedCertificateValidationContext + .hasValidationContextCertificateProviderInstance()) { + return combinedCertificateValidationContext + .getValidationContextCertificateProviderInstance().getInstanceName(); } } + return null; } private static void checkForUniqueness(Set uniqueSet, diff --git a/xds/src/main/java/io/grpc/xds/internal/certprovider/CertProviderClientSslContextProvider.java b/xds/src/main/java/io/grpc/xds/internal/certprovider/CertProviderClientSslContextProvider.java index 2ee21e7db6a..ce9ef3de680 100644 --- a/xds/src/main/java/io/grpc/xds/internal/certprovider/CertProviderClientSslContextProvider.java +++ b/xds/src/main/java/io/grpc/xds/internal/certprovider/CertProviderClientSslContextProvider.java @@ -22,7 +22,6 @@ import io.envoyproxy.envoy.config.core.v3.Node; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CertificateValidationContext; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CommonTlsContext; -import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CommonTlsContext.CombinedCertificateValidationContext; import io.grpc.Internal; import io.grpc.netty.GrpcSslContexts; import io.grpc.xds.Bootstrapper.CertificateProviderInfo; @@ -94,27 +93,12 @@ public CertProviderClientSslContextProvider getProvider( @Nullable Map certProviders) { checkNotNull(upstreamTlsContext, "upstreamTlsContext"); CommonTlsContext commonTlsContext = upstreamTlsContext.getCommonTlsContext(); - CommonTlsContext.CertificateProviderInstance rootCertInstance = null; - CertificateValidationContext staticCertValidationContext = null; - if (commonTlsContext.hasCombinedValidationContext()) { - CombinedCertificateValidationContext combinedValidationContext = - commonTlsContext.getCombinedValidationContext(); - if (combinedValidationContext.hasValidationContextCertificateProviderInstance()) { - rootCertInstance = - combinedValidationContext.getValidationContextCertificateProviderInstance(); - } - if (combinedValidationContext.hasDefaultValidationContext()) { - staticCertValidationContext = combinedValidationContext.getDefaultValidationContext(); - } - } else if (commonTlsContext.hasValidationContextCertificateProviderInstance()) { - rootCertInstance = commonTlsContext.getValidationContextCertificateProviderInstance(); - } else if (commonTlsContext.hasValidationContext()) { - staticCertValidationContext = commonTlsContext.getValidationContext(); - } - CommonTlsContext.CertificateProviderInstance certInstance = null; - if (commonTlsContext.hasTlsCertificateCertificateProviderInstance()) { - certInstance = commonTlsContext.getTlsCertificateCertificateProviderInstance(); - } + CertificateValidationContext staticCertValidationContext = getStaticValidationContext( + commonTlsContext); + CommonTlsContext.CertificateProviderInstance rootCertInstance = getRootCertProviderInstance( + commonTlsContext); + CommonTlsContext.CertificateProviderInstance certInstance = getCertProviderInstance( + commonTlsContext); return new CertProviderClientSslContextProvider( node, certProviders, diff --git a/xds/src/main/java/io/grpc/xds/internal/certprovider/CertProviderServerSslContextProvider.java b/xds/src/main/java/io/grpc/xds/internal/certprovider/CertProviderServerSslContextProvider.java index 1f33e1de789..a7f0849d00b 100644 --- a/xds/src/main/java/io/grpc/xds/internal/certprovider/CertProviderServerSslContextProvider.java +++ b/xds/src/main/java/io/grpc/xds/internal/certprovider/CertProviderServerSslContextProvider.java @@ -22,7 +22,6 @@ import io.envoyproxy.envoy.config.core.v3.Node; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CertificateValidationContext; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CommonTlsContext; -import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CommonTlsContext.CombinedCertificateValidationContext; import io.grpc.Internal; import io.grpc.netty.GrpcSslContexts; import io.grpc.xds.Bootstrapper.CertificateProviderInfo; @@ -97,27 +96,12 @@ public CertProviderServerSslContextProvider getProvider( @Nullable Map certProviders) { checkNotNull(downstreamTlsContext, "downstreamTlsContext"); CommonTlsContext commonTlsContext = downstreamTlsContext.getCommonTlsContext(); - CommonTlsContext.CertificateProviderInstance rootCertInstance = null; - CertificateValidationContext staticCertValidationContext = null; - if (commonTlsContext.hasCombinedValidationContext()) { - CombinedCertificateValidationContext combinedValidationContext = - commonTlsContext.getCombinedValidationContext(); - if (combinedValidationContext.hasValidationContextCertificateProviderInstance()) { - rootCertInstance = - combinedValidationContext.getValidationContextCertificateProviderInstance(); - } - if (combinedValidationContext.hasDefaultValidationContext()) { - staticCertValidationContext = combinedValidationContext.getDefaultValidationContext(); - } - } else if (commonTlsContext.hasValidationContextCertificateProviderInstance()) { - rootCertInstance = commonTlsContext.getValidationContextCertificateProviderInstance(); - } else if (commonTlsContext.hasValidationContext()) { - staticCertValidationContext = commonTlsContext.getValidationContext(); - } - CommonTlsContext.CertificateProviderInstance certInstance = null; - if (commonTlsContext.hasTlsCertificateCertificateProviderInstance()) { - certInstance = commonTlsContext.getTlsCertificateCertificateProviderInstance(); - } + CertificateValidationContext staticCertValidationContext = getStaticValidationContext( + commonTlsContext); + CommonTlsContext.CertificateProviderInstance rootCertInstance = getRootCertProviderInstance( + commonTlsContext); + CommonTlsContext.CertificateProviderInstance certInstance = getCertProviderInstance( + commonTlsContext); return new CertProviderServerSslContextProvider( node, certProviders, diff --git a/xds/src/main/java/io/grpc/xds/internal/certprovider/CertProviderSslContextProvider.java b/xds/src/main/java/io/grpc/xds/internal/certprovider/CertProviderSslContextProvider.java index 1af9e1670d3..1ec58764196 100644 --- a/xds/src/main/java/io/grpc/xds/internal/certprovider/CertProviderSslContextProvider.java +++ b/xds/src/main/java/io/grpc/xds/internal/certprovider/CertProviderSslContextProvider.java @@ -18,9 +18,11 @@ import io.envoyproxy.envoy.config.core.v3.Node; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CertificateValidationContext; +import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CommonTlsContext; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CommonTlsContext.CertificateProviderInstance; import io.grpc.xds.Bootstrapper.CertificateProviderInfo; import io.grpc.xds.EnvoyServerProtoData.BaseTlsContext; +import io.grpc.xds.internal.sds.CommonTlsContextUtil; import io.grpc.xds.internal.sds.DynamicSslContextProvider; import java.security.PrivateKey; import java.security.cert.X509Certificate; @@ -88,6 +90,52 @@ private static CertificateProviderInfo getCertProviderConfig( return certProviders != null ? certProviders.get(pluginInstanceName) : null; } + @Nullable + protected static CertificateProviderInstance getCertProviderInstance( + CommonTlsContext commonTlsContext) { + if (commonTlsContext.hasTlsCertificateProviderInstance()) { + return CommonTlsContextUtil.convert(commonTlsContext.getTlsCertificateProviderInstance()); + } else if (commonTlsContext.hasTlsCertificateCertificateProviderInstance()) { + return commonTlsContext.getTlsCertificateCertificateProviderInstance(); + } + return null; + } + + @Nullable + protected static CertificateValidationContext getStaticValidationContext( + CommonTlsContext commonTlsContext) { + if (commonTlsContext.hasValidationContext()) { + return commonTlsContext.getValidationContext(); + } else if (commonTlsContext.hasCombinedValidationContext()) { + CommonTlsContext.CombinedCertificateValidationContext combinedValidationContext = + commonTlsContext.getCombinedValidationContext(); + if (combinedValidationContext.hasDefaultValidationContext()) { + return combinedValidationContext.getDefaultValidationContext(); + } + } + return null; + } + + @Nullable + protected static CommonTlsContext.CertificateProviderInstance getRootCertProviderInstance( + CommonTlsContext commonTlsContext) { + CertificateValidationContext certValidationContext = getStaticValidationContext( + commonTlsContext); + if (certValidationContext != null && certValidationContext.hasCaCertificateProviderInstance()) { + return CommonTlsContextUtil.convert(certValidationContext.getCaCertificateProviderInstance()); + } + if (commonTlsContext.hasCombinedValidationContext()) { + CommonTlsContext.CombinedCertificateValidationContext combinedValidationContext = + commonTlsContext.getCombinedValidationContext(); + if (combinedValidationContext.hasValidationContextCertificateProviderInstance()) { + return combinedValidationContext.getValidationContextCertificateProviderInstance(); + } + } else if (commonTlsContext.hasValidationContextCertificateProviderInstance()) { + return commonTlsContext.getValidationContextCertificateProviderInstance(); + } + return null; + } + @Override public final void updateCertificate(PrivateKey key, List certChain) { savedKey = key; diff --git a/xds/src/main/java/io/grpc/xds/internal/sds/CommonTlsContextUtil.java b/xds/src/main/java/io/grpc/xds/internal/sds/CommonTlsContextUtil.java index 234989ad115..0c28c79ee22 100644 --- a/xds/src/main/java/io/grpc/xds/internal/sds/CommonTlsContextUtil.java +++ b/xds/src/main/java/io/grpc/xds/internal/sds/CommonTlsContextUtil.java @@ -16,11 +16,12 @@ package io.grpc.xds.internal.sds; +import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CertificateProviderPluginInstance; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CommonTlsContext; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CommonTlsContext.CombinedCertificateValidationContext; /** Class for utility functions for {@link CommonTlsContext}. */ -final class CommonTlsContextUtil { +public final class CommonTlsContextUtil { private CommonTlsContextUtil() {} @@ -38,4 +39,15 @@ private static boolean hasCertProviderValidationContext(CommonTlsContext commonT } return commonTlsContext.hasValidationContextCertificateProviderInstance(); } + + /** + * Converts {@link CertificateProviderPluginInstance} to + * {@link CommonTlsContext.CertificateProviderInstance}. + */ + public static CommonTlsContext.CertificateProviderInstance convert( + CertificateProviderPluginInstance pluginInstance) { + return CommonTlsContext.CertificateProviderInstance.newBuilder() + .setInstanceName(pluginInstance.getInstanceName()) + .setCertificateName(pluginInstance.getCertificateName()).build(); + } } diff --git a/xds/src/test/java/io/grpc/xds/ClientXdsClientDataTest.java b/xds/src/test/java/io/grpc/xds/ClientXdsClientDataTest.java index fb5c349f123..80cd2a8046e 100644 --- a/xds/src/test/java/io/grpc/xds/ClientXdsClientDataTest.java +++ b/xds/src/test/java/io/grpc/xds/ClientXdsClientDataTest.java @@ -77,6 +77,7 @@ import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager; import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpFilter; import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.Rds; +import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CertificateProviderPluginInstance; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CertificateValidationContext; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CommonTlsContext; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CommonTlsContext.CertificateProviderInstance; @@ -1551,7 +1552,7 @@ public void validateCommonTlsContext_validationContext() throws ResourceInvalidE .setValidationContext(CertificateValidationContext.getDefaultInstance()) .build(); thrown.expect(ResourceInvalidException.class); - thrown.expectMessage("common-tls-context with validation_context is not supported"); + thrown.expectMessage("ca_certificate_provider_instance is required in upstream-tls-context"); ClientXdsClient.validateCommonTlsContext(commonTlsContext, null, false); } @@ -1603,14 +1604,26 @@ public void validateCommonTlsContext_tlsCertificateProviderInstance_isRequiredFo .build(); thrown.expect(ResourceInvalidException.class); thrown.expectMessage( - "tls_certificate_certificate_provider_instance is required in downstream-tls-context"); + "tls_certificate_provider_instance is required in downstream-tls-context"); ClientXdsClient.validateCommonTlsContext(commonTlsContext, null, true); } + @Test + @SuppressWarnings("deprecation") + public void validateCommonTlsContext_tlsNewCertificateProviderInstance() + throws ResourceInvalidException { + CommonTlsContext commonTlsContext = CommonTlsContext.newBuilder() + .setTlsCertificateProviderInstance( + CertificateProviderPluginInstance.newBuilder().setInstanceName("name1").build()) + .build(); + ClientXdsClient + .validateCommonTlsContext(commonTlsContext, ImmutableSet.of("name1", "name2"), true); + } + @Test @SuppressWarnings("deprecation") public void validateCommonTlsContext_tlsCertificateProviderInstance() - throws ResourceInvalidException { + throws ResourceInvalidException { CommonTlsContext commonTlsContext = CommonTlsContext.newBuilder() .setTlsCertificateCertificateProviderInstance( CertificateProviderInstance.newBuilder().setInstanceName("name1").build()) @@ -1662,7 +1675,7 @@ public void validateCommonTlsContext_validationContextProviderInstance_absentInB .build(); thrown.expect(ResourceInvalidException.class); thrown.expectMessage( - "ValidationContextProvider instance name 'bad-name' not defined in the bootstrap file."); + "ca_certificate_provider_instance name 'bad-name' not defined in the bootstrap file."); ClientXdsClient .validateCommonTlsContext(commonTlsContext, ImmutableSet.of("name1", "name2"), false); } @@ -1674,7 +1687,7 @@ public void validateCommonTlsContext_tlsCertificatesCount() throws ResourceInval .addTlsCertificates(TlsCertificate.getDefaultInstance()) .build(); thrown.expect(ResourceInvalidException.class); - thrown.expectMessage("common-tls-context with tls_certificates is not supported"); + thrown.expectMessage("tls_certificate_provider_instance is unset"); ClientXdsClient.validateCommonTlsContext(commonTlsContext, null, false); } @@ -1686,7 +1699,7 @@ public void validateCommonTlsContext_tlsCertificateSdsSecretConfigsCount() .build(); thrown.expect(ResourceInvalidException.class); thrown.expectMessage( - "common-tls-context with tls_certificate_sds_secret_configs is not supported"); + "tls_certificate_provider_instance is unset"); ClientXdsClient.validateCommonTlsContext(commonTlsContext, null, false); } @@ -1700,7 +1713,7 @@ public void validateCommonTlsContext_tlsCertificateCertificateProvider() .build(); thrown.expect(ResourceInvalidException.class); thrown.expectMessage( - "common-tls-context with tls_certificate_certificate_provider is not supported"); + "tls_certificate_provider_instance is unset"); ClientXdsClient.validateCommonTlsContext(commonTlsContext, null, false); } @@ -1710,7 +1723,7 @@ public void validateCommonTlsContext_combinedValidationContext_isRequiredForClie CommonTlsContext commonTlsContext = CommonTlsContext.newBuilder() .build(); thrown.expect(ResourceInvalidException.class); - thrown.expectMessage("combined_validation_context is required in upstream-tls-context"); + thrown.expectMessage("ca_certificate_provider_instance is required in upstream-tls-context"); ClientXdsClient.validateCommonTlsContext(commonTlsContext, null, false); } @@ -1723,8 +1736,7 @@ public void validateCommonTlsContext_combinedValidationContextWithoutCertProvide .build(); thrown.expect(ResourceInvalidException.class); thrown.expectMessage( - "validation_context_certificate_provider_instance is required in " - + "combined_validation_context"); + "ca_certificate_provider_instance is required in upstream-tls-context"); ClientXdsClient.validateCommonTlsContext(commonTlsContext, null, false); } diff --git a/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java b/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java index 9e4d92fb344..55bd6ba3e9d 100644 --- a/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java +++ b/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java @@ -39,6 +39,7 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Message; import io.envoyproxy.envoy.config.route.v3.FilterConfig; +import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CertificateProviderPluginInstance; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CommonTlsContext; import io.grpc.BindableService; import io.grpc.Context; @@ -1353,6 +1354,42 @@ public void cdsResponseWithUpstreamTlsContext() { verifySubscribedResourcesMetadataSizes(0, 1, 0, 0); } + /** + * CDS response containing new UpstreamTlsContext for a cluster. + */ + @Test + @SuppressWarnings("deprecation") + public void cdsResponseWithNewUpstreamTlsContext() { + Assume.assumeTrue(useProtocolV3()); + DiscoveryRpcCall call = startResourceWatcher(CDS, CDS_RESOURCE, cdsResourceWatcher); + + // Management server sends back CDS response with UpstreamTlsContext. + Any clusterEds = + Any.pack(mf.buildEdsCluster(CDS_RESOURCE, "eds-cluster-foo.googleapis.com", "round_robin", + null, true, + mf.buildNewUpstreamTlsContext("cert-instance-name", "cert1"), + "envoy.transport_sockets.tls", null)); + List clusters = ImmutableList.of( + Any.pack(mf.buildLogicalDnsCluster("cluster-bar.googleapis.com", + "dns-service-bar.googleapis.com", 443, "round_robin", null, false, null, null)), + clusterEds, + Any.pack(mf.buildEdsCluster("cluster-baz.googleapis.com", null, "round_robin", null, false, + null, "envoy.transport_sockets.tls", null))); + call.sendResponse(CDS, clusters, VERSION_1, "0000"); + + // Client sent an ACK CDS request. + call.verifyRequest(CDS, CDS_RESOURCE, VERSION_1, "0000", NODE); + verify(cdsResourceWatcher, times(1)).onChanged(cdsUpdateCaptor.capture()); + CdsUpdate cdsUpdate = cdsUpdateCaptor.getValue(); + CertificateProviderPluginInstance certificateProviderInstance = + cdsUpdate.upstreamTlsContext().getCommonTlsContext().getValidationContext() + .getCaCertificateProviderInstance(); + assertThat(certificateProviderInstance.getInstanceName()).isEqualTo("cert-instance-name"); + assertThat(certificateProviderInstance.getCertificateName()).isEqualTo("cert1"); + verifyResourceMetadataAcked(CDS, CDS_RESOURCE, clusterEds, VERSION_1, TIME_INCREMENT); + verifySubscribedResourcesMetadataSizes(0, 1, 0, 0); + } + /** * CDS response containing bad UpstreamTlsContext for a cluster. */ @@ -1373,7 +1410,7 @@ public void cdsResponseErrorHandling_badUpstreamTlsContext() { "CDS response Cluster 'cluster.googleapis.com' validation error: " + "Cluster cluster.googleapis.com: malformed UpstreamTlsContext: " + "io.grpc.xds.ClientXdsClient$ResourceInvalidException: " - + "combined_validation_context is required in upstream-tls-context")); + + "ca_certificate_provider_instance is required in upstream-tls-context")); verifyNoInteractions(cdsResourceWatcher); } @@ -2400,6 +2437,8 @@ protected abstract Message buildRingHashLbConfig(String hashFunction, long minRi protected abstract Message buildUpstreamTlsContext(String instanceName, String certName); + protected abstract Message buildNewUpstreamTlsContext(String instanceName, String certName); + protected abstract Message buildCircuitBreakers(int highPriorityMaxRequests, int defaultPriorityMaxRequests); diff --git a/xds/src/test/java/io/grpc/xds/ClientXdsClientV2Test.java b/xds/src/test/java/io/grpc/xds/ClientXdsClientV2Test.java index 409613aecf7..39f5d1a1a2e 100644 --- a/xds/src/test/java/io/grpc/xds/ClientXdsClientV2Test.java +++ b/xds/src/test/java/io/grpc/xds/ClientXdsClientV2Test.java @@ -515,6 +515,12 @@ protected Message buildUpstreamTlsContext(String instanceName, String certName) .build(); } + @Override + protected Message buildNewUpstreamTlsContext(String instanceName, String certName) { + return buildUpstreamTlsContext(instanceName, certName); + } + + @Override protected Message buildCircuitBreakers(int highPriorityMaxRequests, int defaultPriorityMaxRequests) { diff --git a/xds/src/test/java/io/grpc/xds/ClientXdsClientV3Test.java b/xds/src/test/java/io/grpc/xds/ClientXdsClientV3Test.java index eddba1040d4..dfd407ef016 100644 --- a/xds/src/test/java/io/grpc/xds/ClientXdsClientV3Test.java +++ b/xds/src/test/java/io/grpc/xds/ClientXdsClientV3Test.java @@ -77,6 +77,8 @@ import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager; import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpFilter; import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.Rds; +import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CertificateProviderPluginInstance; +import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CertificateValidationContext; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CommonTlsContext; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext; import io.envoyproxy.envoy.service.discovery.v3.AggregatedDiscoveryServiceGrpc.AggregatedDiscoveryServiceImplBase; @@ -555,6 +557,20 @@ protected Message buildUpstreamTlsContext(String instanceName, String certName) .build(); } + @Override + protected Message buildNewUpstreamTlsContext(String instanceName, String certName) { + CommonTlsContext.Builder commonTlsContextBuilder = CommonTlsContext.newBuilder(); + if (instanceName != null && certName != null) { + commonTlsContextBuilder.setValidationContext(CertificateValidationContext.newBuilder() + .setCaCertificateProviderInstance( + CertificateProviderPluginInstance.newBuilder().setInstanceName(instanceName) + .setCertificateName(certName).build())); + } + return UpstreamTlsContext.newBuilder() + .setCommonTlsContext(commonTlsContextBuilder) + .build(); + } + @Override protected Message buildCircuitBreakers(int highPriorityMaxRequests, int defaultPriorityMaxRequests) { diff --git a/xds/src/test/java/io/grpc/xds/internal/certprovider/CertProviderClientSslContextProviderTest.java b/xds/src/test/java/io/grpc/xds/internal/certprovider/CertProviderClientSslContextProviderTest.java index 00b29014648..1eed5488aa0 100644 --- a/xds/src/test/java/io/grpc/xds/internal/certprovider/CertProviderClientSslContextProviderTest.java +++ b/xds/src/test/java/io/grpc/xds/internal/certprovider/CertProviderClientSslContextProviderTest.java @@ -87,6 +87,27 @@ private CertProviderClientSslContextProvider getSslContextProvider( bootstrapInfo.getCertProviders()); } + /** Helper method to build CertProviderClientSslContextProvider. */ + private CertProviderClientSslContextProvider getNewSslContextProvider( + String certInstanceName, + String rootInstanceName, + Bootstrapper.BootstrapInfo bootstrapInfo, + Iterable alpnProtocols, + CertificateValidationContext staticCertValidationContext) { + EnvoyServerProtoData.UpstreamTlsContext upstreamTlsContext = + CommonTlsContextTestsUtil.buildNewUpstreamTlsContextForCertProviderInstance( + certInstanceName, + "cert-default", + rootInstanceName, + "root-default", + alpnProtocols, + staticCertValidationContext); + return certProviderClientSslContextProviderFactory.getProvider( + upstreamTlsContext, + bootstrapInfo.getNode().toEnvoyProtoNode(), + bootstrapInfo.getCertProviders()); + } + @Test public void testProviderForClient_mtls() throws Exception { final CertificateProvider.DistributorWatcher[] watcherCaptor = @@ -150,6 +171,69 @@ public void testProviderForClient_mtls() throws Exception { assertThat(testCallback1.updatedSslContext).isNotSameInstanceAs(testCallback.updatedSslContext); } + @Test + public void testProviderForClient_mtls_newXds() throws Exception { + final CertificateProvider.DistributorWatcher[] watcherCaptor = + new CertificateProvider.DistributorWatcher[1]; + TestCertificateProvider.createAndRegisterProviderProvider( + certificateProviderRegistry, watcherCaptor, "testca", 0); + CertProviderClientSslContextProvider provider = + getNewSslContextProvider( + "gcp_id", + "gcp_id", + CommonBootstrapperTestUtils.getTestBootstrapInfo(), + /* alpnProtocols= */ null, + /* staticCertValidationContext= */ null); + + assertThat(provider.savedKey).isNull(); + assertThat(provider.savedCertChain).isNull(); + assertThat(provider.savedTrustedRoots).isNull(); + assertThat(provider.getSslContext()).isNull(); + + // now generate cert update + watcherCaptor[0].updateCertificate( + CommonCertProviderTestUtils.getPrivateKey(CLIENT_KEY_FILE), + ImmutableList.of(getCertFromResourceName(CLIENT_PEM_FILE))); + assertThat(provider.savedKey).isNotNull(); + assertThat(provider.savedCertChain).isNotNull(); + assertThat(provider.getSslContext()).isNull(); + + // now generate root cert update + watcherCaptor[0].updateTrustedRoots(ImmutableList.of(getCertFromResourceName(CA_PEM_FILE))); + assertThat(provider.getSslContext()).isNotNull(); + assertThat(provider.savedKey).isNull(); + assertThat(provider.savedCertChain).isNull(); + assertThat(provider.savedTrustedRoots).isNull(); + + TestCallback testCallback = + CommonTlsContextTestsUtil.getValueThruCallback(provider); + + doChecksOnSslContext(false, testCallback.updatedSslContext, /* expectedApnProtos= */ null); + TestCallback testCallback1 = + CommonTlsContextTestsUtil.getValueThruCallback(provider); + assertThat(testCallback1.updatedSslContext).isSameInstanceAs(testCallback.updatedSslContext); + + // just do root cert update: sslContext should still be the same + watcherCaptor[0].updateTrustedRoots( + ImmutableList.of(getCertFromResourceName(SERVER_0_PEM_FILE))); + assertThat(provider.savedKey).isNull(); + assertThat(provider.savedCertChain).isNull(); + assertThat(provider.savedTrustedRoots).isNotNull(); + testCallback1 = CommonTlsContextTestsUtil.getValueThruCallback(provider); + assertThat(testCallback1.updatedSslContext).isSameInstanceAs(testCallback.updatedSslContext); + + // now update id cert: sslContext should be updated i.e.different from the previous one + watcherCaptor[0].updateCertificate( + CommonCertProviderTestUtils.getPrivateKey(SERVER_1_KEY_FILE), + ImmutableList.of(getCertFromResourceName(SERVER_1_PEM_FILE))); + assertThat(provider.savedKey).isNull(); + assertThat(provider.savedCertChain).isNull(); + assertThat(provider.savedTrustedRoots).isNull(); + assertThat(provider.getSslContext()).isNotNull(); + testCallback1 = CommonTlsContextTestsUtil.getValueThruCallback(provider); + assertThat(testCallback1.updatedSslContext).isNotSameInstanceAs(testCallback.updatedSslContext); + } + @Test public void testProviderForClient_queueExecutor() throws Exception { final CertificateProvider.DistributorWatcher[] watcherCaptor = diff --git a/xds/src/test/java/io/grpc/xds/internal/certprovider/CertProviderServerSslContextProviderTest.java b/xds/src/test/java/io/grpc/xds/internal/certprovider/CertProviderServerSslContextProviderTest.java index ef801ccc2c1..783ce2b11f7 100644 --- a/xds/src/test/java/io/grpc/xds/internal/certprovider/CertProviderServerSslContextProviderTest.java +++ b/xds/src/test/java/io/grpc/xds/internal/certprovider/CertProviderServerSslContextProviderTest.java @@ -31,12 +31,14 @@ import com.google.common.util.concurrent.MoreExecutors; import io.envoyproxy.envoy.config.core.v3.DataSource; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CertificateValidationContext; +import io.envoyproxy.envoy.type.matcher.v3.StringMatcher; import io.grpc.xds.Bootstrapper; import io.grpc.xds.CommonBootstrapperTestUtils; import io.grpc.xds.EnvoyServerProtoData; import io.grpc.xds.internal.certprovider.CertProviderClientSslContextProviderTest.QueuedExecutor; import io.grpc.xds.internal.sds.CommonTlsContextTestsUtil; import io.grpc.xds.internal.sds.CommonTlsContextTestsUtil.TestCallback; +import java.util.Arrays; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -81,6 +83,30 @@ private CertProviderServerSslContextProvider getSslContextProvider( bootstrapInfo.getCertProviders()); } + /** Helper method to build CertProviderServerSslContextProvider. */ + private CertProviderServerSslContextProvider getNewSslContextProvider( + String certInstanceName, + String rootInstanceName, + Bootstrapper.BootstrapInfo bootstrapInfo, + Iterable alpnProtocols, + CertificateValidationContext staticCertValidationContext, + boolean requireClientCert) { + EnvoyServerProtoData.DownstreamTlsContext downstreamTlsContext = + CommonTlsContextTestsUtil.buildNewDownstreamTlsContextForCertProviderInstance( + certInstanceName, + "cert-default", + rootInstanceName, + "root-default", + alpnProtocols, + staticCertValidationContext, + requireClientCert); + return certProviderServerSslContextProviderFactory.getProvider( + downstreamTlsContext, + bootstrapInfo.getNode().toEnvoyProtoNode(), + bootstrapInfo.getCertProviders()); + } + + @Test public void testProviderForServer_mtls() throws Exception { final CertificateProvider.DistributorWatcher[] watcherCaptor = @@ -145,6 +171,74 @@ public void testProviderForServer_mtls() throws Exception { assertThat(testCallback1.updatedSslContext).isNotSameInstanceAs(testCallback.updatedSslContext); } + @Test + public void testProviderForServer_mtls_newXds() throws Exception { + final CertificateProvider.DistributorWatcher[] watcherCaptor = + new CertificateProvider.DistributorWatcher[1]; + TestCertificateProvider.createAndRegisterProviderProvider( + certificateProviderRegistry, watcherCaptor, "testca", 0); + CertificateValidationContext staticCertValidationContext = + CertificateValidationContext.newBuilder().addAllMatchSubjectAltNames(Arrays + .asList(StringMatcher.newBuilder().setExact("foo.com").build(), + StringMatcher.newBuilder().setExact("bar.com").build())).build(); + CertProviderServerSslContextProvider provider = + getNewSslContextProvider( + "gcp_id", + "gcp_id", + CommonBootstrapperTestUtils.getTestBootstrapInfo(), + /* alpnProtocols= */ null, + staticCertValidationContext, + /* requireClientCert= */ true); + + assertThat(provider.savedKey).isNull(); + assertThat(provider.savedCertChain).isNull(); + assertThat(provider.savedTrustedRoots).isNull(); + assertThat(provider.getSslContext()).isNull(); + + // now generate cert update + watcherCaptor[0].updateCertificate( + CommonCertProviderTestUtils.getPrivateKey(SERVER_0_KEY_FILE), + ImmutableList.of(getCertFromResourceName(SERVER_0_PEM_FILE))); + assertThat(provider.savedKey).isNotNull(); + assertThat(provider.savedCertChain).isNotNull(); + assertThat(provider.getSslContext()).isNull(); + + // now generate root cert update + watcherCaptor[0].updateTrustedRoots(ImmutableList.of(getCertFromResourceName(CA_PEM_FILE))); + assertThat(provider.getSslContext()).isNotNull(); + assertThat(provider.savedKey).isNull(); + assertThat(provider.savedCertChain).isNull(); + assertThat(provider.savedTrustedRoots).isNull(); + + TestCallback testCallback = + CommonTlsContextTestsUtil.getValueThruCallback(provider); + + doChecksOnSslContext(true, testCallback.updatedSslContext, /* expectedApnProtos= */ null); + TestCallback testCallback1 = + CommonTlsContextTestsUtil.getValueThruCallback(provider); + assertThat(testCallback1.updatedSslContext).isSameInstanceAs(testCallback.updatedSslContext); + + // just do root cert update: sslContext should still be the same + watcherCaptor[0].updateTrustedRoots( + ImmutableList.of(getCertFromResourceName(CLIENT_PEM_FILE))); + assertThat(provider.savedKey).isNull(); + assertThat(provider.savedCertChain).isNull(); + assertThat(provider.savedTrustedRoots).isNotNull(); + testCallback1 = CommonTlsContextTestsUtil.getValueThruCallback(provider); + assertThat(testCallback1.updatedSslContext).isSameInstanceAs(testCallback.updatedSslContext); + + // now update id cert: sslContext should be updated i.e.different from the previous one + watcherCaptor[0].updateCertificate( + CommonCertProviderTestUtils.getPrivateKey(SERVER_1_KEY_FILE), + ImmutableList.of(getCertFromResourceName(SERVER_1_PEM_FILE))); + assertThat(provider.savedKey).isNull(); + assertThat(provider.savedCertChain).isNull(); + assertThat(provider.savedTrustedRoots).isNull(); + assertThat(provider.getSslContext()).isNotNull(); + testCallback1 = CommonTlsContextTestsUtil.getValueThruCallback(provider); + assertThat(testCallback1.updatedSslContext).isNotSameInstanceAs(testCallback.updatedSslContext); + } + @Test public void testProviderForServer_queueExecutor() throws Exception { final CertificateProvider.DistributorWatcher[] watcherCaptor = diff --git a/xds/src/test/java/io/grpc/xds/internal/sds/CommonTlsContextTestsUtil.java b/xds/src/test/java/io/grpc/xds/internal/sds/CommonTlsContextTestsUtil.java index 81fbda9bde4..840cced424f 100644 --- a/xds/src/test/java/io/grpc/xds/internal/sds/CommonTlsContextTestsUtil.java +++ b/xds/src/test/java/io/grpc/xds/internal/sds/CommonTlsContextTestsUtil.java @@ -22,6 +22,7 @@ import com.google.common.io.CharStreams; import com.google.common.util.concurrent.MoreExecutors; import com.google.protobuf.BoolValue; +import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CertificateProviderPluginInstance; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CertificateValidationContext; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CommonTlsContext; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CommonTlsContext.CertificateProviderInstance; @@ -234,6 +235,30 @@ private static CommonTlsContext buildCommonTlsContextForCertProviderInstance( return builder.build(); } + private static CommonTlsContext buildNewCommonTlsContextForCertProviderInstance( + String certInstanceName, + String certName, + String rootInstanceName, + String rootCertName, + Iterable alpnProtocols, + CertificateValidationContext staticCertValidationContext) { + CommonTlsContext.Builder builder = CommonTlsContext.newBuilder(); + if (certInstanceName != null) { + builder = + builder.setTlsCertificateProviderInstance( + CertificateProviderPluginInstance.newBuilder() + .setInstanceName(certInstanceName) + .setCertificateName(certName)); + } + builder = + addNewCertificateValidationContext( + builder, rootInstanceName, rootCertName, staticCertValidationContext); + if (alpnProtocols != null) { + builder.addAllAlpnProtocols(alpnProtocols); + } + return builder.build(); + } + @SuppressWarnings("deprecation") private static CommonTlsContext.Builder addCertificateValidationContext( CommonTlsContext.Builder builder, @@ -259,6 +284,26 @@ private static CommonTlsContext.Builder addCertificateValidationContext( return builder; } + private static CommonTlsContext.Builder addNewCertificateValidationContext( + CommonTlsContext.Builder builder, + String rootInstanceName, + String rootCertName, + CertificateValidationContext staticCertValidationContext) { + if (rootInstanceName != null) { + CertificateProviderPluginInstance providerInstance = + CertificateProviderPluginInstance.newBuilder() + .setInstanceName(rootInstanceName) + .setCertificateName(rootCertName) + .build(); + CertificateValidationContext.Builder validationContextBuilder = + staticCertValidationContext != null ? staticCertValidationContext.toBuilder() + : CertificateValidationContext.newBuilder(); + return builder.setValidationContext( + validationContextBuilder.setCaCertificateProviderInstance(providerInstance)); + } + return builder; + } + /** Helper method to build UpstreamTlsContext for CertProvider tests. */ public static EnvoyServerProtoData.UpstreamTlsContext buildUpstreamTlsContextForCertProviderInstance( @@ -278,6 +323,25 @@ private static CommonTlsContext.Builder addCertificateValidationContext( staticCertValidationContext)); } + /** Helper method to build UpstreamTlsContext for CertProvider tests. */ + public static EnvoyServerProtoData.UpstreamTlsContext + buildNewUpstreamTlsContextForCertProviderInstance( + @Nullable String certInstanceName, + @Nullable String certName, + @Nullable String rootInstanceName, + @Nullable String rootCertName, + Iterable alpnProtocols, + CertificateValidationContext staticCertValidationContext) { + return buildUpstreamTlsContext( + buildNewCommonTlsContextForCertProviderInstance( + certInstanceName, + certName, + rootInstanceName, + rootCertName, + alpnProtocols, + staticCertValidationContext)); + } + /** Helper method to build DownstreamTlsContext for CertProvider tests. */ public static EnvoyServerProtoData.DownstreamTlsContext buildDownstreamTlsContextForCertProviderInstance( @@ -298,6 +362,25 @@ private static CommonTlsContext.Builder addCertificateValidationContext( staticCertValidationContext), requireClientCert); } + /** Helper method to build DownstreamTlsContext for CertProvider tests. */ + public static EnvoyServerProtoData.DownstreamTlsContext + buildNewDownstreamTlsContextForCertProviderInstance( + @Nullable String certInstanceName, + @Nullable String certName, + @Nullable String rootInstanceName, + @Nullable String rootCertName, + Iterable alpnProtocols, + CertificateValidationContext staticCertValidationContext, + boolean requireClientCert) { + return buildInternalDownstreamTlsContext( + buildNewCommonTlsContextForCertProviderInstance( + certInstanceName, + certName, + rootInstanceName, + rootCertName, + alpnProtocols, + staticCertValidationContext), requireClientCert); + } /** Perform some simple checks on sslContext. */ public static void doChecksOnSslContext(boolean server, SslContext sslContext, From 9870db1f479380301cb68be70650f0ca34f0f83a Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Mon, 16 Aug 2021 11:45:21 -0700 Subject: [PATCH 05/76] stub: Document that noop onCancelHandler is useful setOnCancelHandler tells gRPC that the application is handling cancellation. But it's fine to have noop behavior within the handler itself if the application doesn't need it. It is just a way to opt-in to the more recent no-exception-from-onNext behavior. Let's mention this use-case in the docs to make it more obvious it is a possibility. Came up as part of #8409. --- stub/src/main/java/io/grpc/stub/ServerCallStreamObserver.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/stub/src/main/java/io/grpc/stub/ServerCallStreamObserver.java b/stub/src/main/java/io/grpc/stub/ServerCallStreamObserver.java index a4d4564a46d..2ac008b269a 100644 --- a/stub/src/main/java/io/grpc/stub/ServerCallStreamObserver.java +++ b/stub/src/main/java/io/grpc/stub/ServerCallStreamObserver.java @@ -54,7 +54,9 @@ public abstract class ServerCallStreamObserver extends CallStreamObserver * service returns its {@code StreamObserver}. * *

Setting the onCancelHandler will suppress the on-cancel exception thrown by - * {@link #onNext}. + * {@link #onNext}. If the caller is already handling cancellation via polling or cannot + * substantially benefit from observing cancellation, using a no-op {@code onCancelHandler} is + * useful just to suppress the {@code onNext()} exception. * * @param onCancelHandler to call when client has cancelled the call. */ From be7aa504417824cf035652b8753a5498361d9167 Mon Sep 17 00:00:00 2001 From: yifeizhuang Date: Wed, 8 Sep 2021 18:32:26 -0700 Subject: [PATCH 06/76] xds: referenciate server routing config (#8491) * routing config ref * atomic ref virtual host list * Revert "routing config ref" This reverts commit cbcad5744fb3b57bfb2f43f854cd6a21a998af78. * test: noop config non-static, better validation --- .../java/io/grpc/xds/XdsServerWrapper.java | 68 +++---- ...rChainMatchingProtocolNegotiatorsTest.java | 51 ++--- .../io/grpc/xds/XdsServerWrapperTest.java | 184 +++++++++++------- 3 files changed, 150 insertions(+), 153 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java b/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java index fa8ecf8a822..faa6e9d34b2 100644 --- a/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java +++ b/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java @@ -396,7 +396,7 @@ public void run() { } routeDiscoveryStates.keySet().retainAll(allRds); if (pendingRds.isEmpty()) { - updateSelector(true); + updateSelector(); } } }); @@ -450,14 +450,7 @@ private void shutdown() { releaseSuppliersInFlight(); } - /** - * Use firstTimeNoPendingRds to indicate that the previous SslContextProviderSuppliers in - * filterChainSelectorRef should be released. Call updateSelector(true) when all routing are - * just complete and the newest filter chain is ready to be applied to the - * filterChainSelectorRef. Call updateSelector(false) for subsequent routing update - * corresponding to the same filter chain list. - */ - private void updateSelector(boolean firstTimeNoPendingRds) { + private void updateSelector() { Map filterChainRouting = new HashMap<>(); for (FilterChain filterChain: filterChains) { filterChainRouting.put(filterChain, generateRoutingConfig(filterChain)); @@ -466,10 +459,7 @@ private void updateSelector(boolean firstTimeNoPendingRds) { Collections.unmodifiableMap(filterChainRouting), defaultFilterChain == null ? null : defaultFilterChain.getSslContextProviderSupplier(), defaultFilterChain == null ? null : generateRoutingConfig(defaultFilterChain)); - List toRelease = Collections.emptyList(); - if (firstTimeNoPendingRds) { - toRelease = getSuppliersInUse(); - } + List toRelease = getSuppliersInUse(); filterChainSelectorRef.set(selector); for (SslContextProviderSupplier e: toRelease) { e.close(); @@ -480,14 +470,12 @@ private void updateSelector(boolean firstTimeNoPendingRds) { private ServerRoutingConfig generateRoutingConfig(FilterChain filterChain) { HttpConnectionManager hcm = filterChain.getHttpConnectionManager(); if (hcm.virtualHosts() != null) { - return ServerRoutingConfig.create(hcm.httpFilterConfigs(), hcm.virtualHosts()); + return ServerRoutingConfig.create(hcm.httpFilterConfigs(), + new AtomicReference<>(hcm.virtualHosts())); } else { RouteDiscoveryState rds = routeDiscoveryStates.get(hcm.rdsName()); - if (rds != null && rds.savedVirtualHosts != null) { - return ServerRoutingConfig.create(hcm.httpFilterConfigs(), rds.savedVirtualHosts); - } else { - return ServerRoutingConfig.FAILING_ROUTING_CONFIG; - } + checkNotNull(rds, "rds"); + return ServerRoutingConfig.create(hcm.httpFilterConfigs(), rds.savedVirtualHosts); } } @@ -555,8 +543,8 @@ private void releaseSuppliersInFlight() { private final class RouteDiscoveryState implements RdsResourceWatcher { private final String resourceName; - @Nullable - private List savedVirtualHosts; + private AtomicReference> savedVirtualHosts = + new AtomicReference<>(); private boolean isPending = true; private RouteDiscoveryState(String resourceName) { @@ -571,7 +559,7 @@ public void run() { if (!routeDiscoveryStates.containsKey(resourceName)) { return; } - savedVirtualHosts = update.virtualHosts; + savedVirtualHosts.set(ImmutableList.copyOf(update.virtualHosts)); maybeUpdateSelector(); } }); @@ -586,7 +574,7 @@ public void run() { return; } logger.log(Level.WARNING, "Rds {0} unavailable", resourceName); - savedVirtualHosts = null; + savedVirtualHosts.set(null); maybeUpdateSelector(); } }); @@ -608,13 +596,13 @@ public void run() { } // Update the selector to use the most recently updated configs only after all rds have been - // discovered, i.e. pendingRds is empty. Do the updateSelector even after rds are already - // fully discovered and new change comes. + // discovered for the first time. Later changes on rds will be applied through virtual host + // list atomic ref. private void maybeUpdateSelector() { isPending = false; - boolean isLastPending = pendingRds.remove(resourceName); - if (pendingRds.isEmpty()) { - updateSelector(isLastPending); + boolean isLastPending = pendingRds.remove(resourceName) && pendingRds.isEmpty(); + if (isLastPending) { + updateSelector(); } } } @@ -644,15 +632,19 @@ public Listener interceptCall(ServerCall call, public Listener interceptCall(ServerCall call, Metadata headers, ServerCallHandler next) { ServerRoutingConfig routingConfig = call.getAttributes().get(ATTR_SERVER_ROUTING_CONFIG); - if (routingConfig == null - || routingConfig.equals(ServerRoutingConfig.FAILING_ROUTING_CONFIG)) { - String errorMsg = "Missing xDS routing config. " + (routingConfig == null ? "" : - "RDS config unavailable."); + if (routingConfig == null) { + String errorMsg = "Missing xDS routing config."; + call.close(Status.UNAVAILABLE.withDescription(errorMsg), new Metadata()); + return new Listener() {}; + } + List virtualHosts = routingConfig.virtualHosts().get(); + if (virtualHosts == null) { + String errorMsg = "Missing xDS routing config VirtualHosts due to RDS config unavailable."; call.close(Status.UNAVAILABLE.withDescription(errorMsg), new Metadata()); return new Listener() {}; } VirtualHost virtualHost = RoutingUtils.findVirtualHostForHostName( - routingConfig.virtualHosts(), call.getAuthority()); + virtualHosts, call.getAuthority()); if (virtualHost == null) { call.close( Status.UNAVAILABLE.withDescription("Could not find xDS virtual host matching RPC"), @@ -727,24 +719,20 @@ public Listener interceptCall(ServerCall call, */ @AutoValue abstract static class ServerRoutingConfig { - private static final ServerRoutingConfig FAILING_ROUTING_CONFIG = - new AutoValue_XdsServerWrapper_ServerRoutingConfig( - ImmutableList.of(), ImmutableList.of()); - // Top level http filter configs. abstract ImmutableList httpFilterConfigs(); - abstract ImmutableList virtualHosts(); + abstract AtomicReference> virtualHosts(); /** * Server routing configuration. * */ public static ServerRoutingConfig create(List httpFilterConfigs, - List virtualHosts) { + AtomicReference> virtualHosts) { checkNotNull(httpFilterConfigs, "httpFilterConfigs"); checkNotNull(virtualHosts, "virtualHosts"); return new AutoValue_XdsServerWrapper_ServerRoutingConfig( - ImmutableList.copyOf(httpFilterConfigs), ImmutableList.copyOf(virtualHosts)); + ImmutableList.copyOf(httpFilterConfigs), virtualHosts); } } } diff --git a/xds/src/test/java/io/grpc/xds/FilterChainMatchingProtocolNegotiatorsTest.java b/xds/src/test/java/io/grpc/xds/FilterChainMatchingProtocolNegotiatorsTest.java index d79785c9f32..167f3f03c6b 100644 --- a/xds/src/test/java/io/grpc/xds/FilterChainMatchingProtocolNegotiatorsTest.java +++ b/xds/src/test/java/io/grpc/xds/FilterChainMatchingProtocolNegotiatorsTest.java @@ -23,6 +23,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.SettableFuture; import io.grpc.internal.TestUtils.NoopChannelLogger; @@ -62,6 +63,8 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; @@ -88,6 +91,8 @@ public class FilterChainMatchingProtocolNegotiatorsTest { private static final String LOCAL_IP = "10.1.2.3"; // dest private static final String REMOTE_IP = "10.4.2.3"; // source private static final int PORT = 7000; + private final ServerRoutingConfig noopConfig = ServerRoutingConfig.create( + new ArrayList(), new AtomicReference>()); @Test public void nofilterChainMatch_defaultSslContext() throws Exception { @@ -98,8 +103,6 @@ public void nofilterChainMatch_defaultSslContext() throws Exception { SslContextProviderSupplier defaultSsl = new SslContextProviderSupplier(createTls(), tlsContextManager); - ServerRoutingConfig noopConfig = ServerRoutingConfig.create( - new ArrayList(), new ArrayList()); FilterChainSelector selector = new FilterChainSelector( new HashMap(), defaultSsl, noopConfig); FilterChainMatchingHandler filterChainMatchingHandler = @@ -154,8 +157,6 @@ public void singleFilterChainWithoutAlpn() throws Exception { "filter-chain-foo", filterChainMatch, HTTP_CONNECTION_MANAGER, tlsContext, tlsContextManager); - ServerRoutingConfig noopConfig = ServerRoutingConfig.create( - new ArrayList(), new ArrayList()); FilterChainSelector selector = new FilterChainSelector(ImmutableMap.of(filterChain, noopConfig), null, null); FilterChainMatchingHandler filterChainMatchingHandler = @@ -195,8 +196,6 @@ public void singleFilterChainWithAlpn() throws Exception { EnvoyServerProtoData.FilterChain defaultFilterChain = new EnvoyServerProtoData.FilterChain( "filter-chain-bar", null, HTTP_CONNECTION_MANAGER, defaultTlsContext, tlsContextManager); - ServerRoutingConfig noopConfig = ServerRoutingConfig.create( - new ArrayList(), new ArrayList()); FilterChainSelector selector = new FilterChainSelector( ImmutableMap.of(filterChain, randomConfig("no-match")), defaultFilterChain.getSslContextProviderSupplier(), noopConfig); @@ -241,12 +240,11 @@ public void destPortFails_returnDefaultFilterChain() throws Exception { tlsContextForDefaultFilterChain, tlsContextManager); ServerRoutingConfig routingConfig = ServerRoutingConfig.create( - new ArrayList(), Arrays.asList(createVirtualHost("virtual"))); - ServerRoutingConfig defaultRoutingConfig = ServerRoutingConfig.create( - new ArrayList(), new ArrayList()); + new ArrayList(), new AtomicReference<>( + ImmutableList.of(createVirtualHost("virtual")))); FilterChainSelector selector = new FilterChainSelector( ImmutableMap.of(filterChainWithDestPort, routingConfig), - defaultFilterChain.getSslContextProviderSupplier(), defaultRoutingConfig); + defaultFilterChain.getSslContextProviderSupplier(), noopConfig); FilterChainMatchingHandler filterChainMatchingHandler = new FilterChainMatchingHandler(grpcHandler, selector, mockDelegate); @@ -259,7 +257,7 @@ public void destPortFails_returnDefaultFilterChain() throws Exception { pipeline.fireUserEventTriggered(event); channel.runPendingTasks(); assertThat(sslSet.get()).isEqualTo(defaultFilterChain.getSslContextProviderSupplier()); - assertThat(routingSettable.get()).isEqualTo(defaultRoutingConfig); + assertThat(routingSettable.get()).isEqualTo(noopConfig); assertThat(sslSet.get().getTlsContext()) .isSameInstanceAs(tlsContextForDefaultFilterChain); } @@ -287,8 +285,6 @@ public void destPrefixRangeMatch() throws Exception { "filter-chain-bar", null, HTTP_CONNECTION_MANAGER, tlsContextForDefaultFilterChain, tlsContextManager); - ServerRoutingConfig noopConfig = ServerRoutingConfig.create( - new ArrayList(), new ArrayList()); FilterChainSelector selector = new FilterChainSelector( ImmutableMap.of(filterChainWithMatch, noopConfig), defaultFilterChain.getSslContextProviderSupplier(), randomConfig("no-match")); @@ -333,8 +329,6 @@ public void destPrefixRangeMismatch_returnDefaultFilterChain() EnvoyServerProtoData.FilterChain defaultFilterChain = new EnvoyServerProtoData.FilterChain( "filter-chain-bar", null, HTTP_CONNECTION_MANAGER, tlsContextForDefaultFilterChain, tlsContextManager); - ServerRoutingConfig noopConfig = ServerRoutingConfig.create( - new ArrayList(), new ArrayList()); FilterChainSelector selector = new FilterChainSelector( ImmutableMap.of(filterChainWithMismatch, randomConfig("no-match")), defaultFilterChain.getSslContextProviderSupplier(), noopConfig); @@ -380,8 +374,6 @@ public void dest0LengthPrefixRange() "filter-chain-bar", null, HTTP_CONNECTION_MANAGER, tlsContextForDefaultFilterChain, tlsContextManager); - ServerRoutingConfig noopConfig = ServerRoutingConfig.create( - new ArrayList(), new ArrayList()); FilterChainSelector selector = new FilterChainSelector( ImmutableMap.of(filterChain0Length, noopConfig), defaultFilterChain.getSslContextProviderSupplier(), null); @@ -439,8 +431,6 @@ public void destPrefixRange_moreSpecificWins() tlsContextManager); EnvoyServerProtoData.FilterChain defaultFilterChain = new EnvoyServerProtoData.FilterChain( "filter-chain-baz", null, HTTP_CONNECTION_MANAGER, null, tlsContextManager); - ServerRoutingConfig noopConfig = ServerRoutingConfig.create( - new ArrayList(), new ArrayList()); FilterChainSelector selector = new FilterChainSelector( ImmutableMap.of(filterChainLessSpecific, randomConfig("no-match"), filterChainMoreSpecific, noopConfig), @@ -500,8 +490,6 @@ public void destPrefixRange_emptyListLessSpecific() tlsContextManager); EnvoyServerProtoData.FilterChain defaultFilterChain = new EnvoyServerProtoData.FilterChain( "filter-chain-baz", null, HTTP_CONNECTION_MANAGER, null, tlsContextManager); - ServerRoutingConfig noopConfig = ServerRoutingConfig.create( - new ArrayList(), new ArrayList()); FilterChainSelector selector = new FilterChainSelector( ImmutableMap.of(filterChainLessSpecific, randomConfig("no-match"), filterChainMoreSpecific, noopConfig), @@ -559,8 +547,6 @@ public void destPrefixRangeIpv6_moreSpecificWins() tlsContextMoreSpecific, tlsContextManager); EnvoyServerProtoData.FilterChain defaultFilterChain = new EnvoyServerProtoData.FilterChain( "filter-chain-baz", null, HTTP_CONNECTION_MANAGER, null, tlsContextManager); - ServerRoutingConfig noopConfig = ServerRoutingConfig.create( - new ArrayList(), new ArrayList()); FilterChainSelector selector = new FilterChainSelector( ImmutableMap.of(filterChainLessSpecific, randomConfig("no-match"), filterChainMoreSpecific, noopConfig), @@ -624,8 +610,6 @@ public void destPrefixRange_moreSpecificWith2Wins() EnvoyServerProtoData.FilterChain defaultFilterChain = new EnvoyServerProtoData.FilterChain( "filter-chain-baz", null, HTTP_CONNECTION_MANAGER, null, tlsContextManager); - ServerRoutingConfig noopConfig = ServerRoutingConfig.create( - new ArrayList(), new ArrayList()); FilterChainSelector selector = new FilterChainSelector( ImmutableMap.of(filterChainMoreSpecificWith2, noopConfig, filterChainLessSpecific, randomConfig("no-match")), @@ -669,8 +653,6 @@ public void sourceTypeMismatch_returnDefaultFilterChain() throws Exception { EnvoyServerProtoData.FilterChain defaultFilterChain = new EnvoyServerProtoData.FilterChain( "filter-chain-bar", null, HTTP_CONNECTION_MANAGER,tlsContextForDefaultFilterChain, tlsContextManager); - ServerRoutingConfig noopConfig = ServerRoutingConfig.create( - new ArrayList(), new ArrayList()); FilterChainSelector selector = new FilterChainSelector( ImmutableMap.of(filterChainWithMismatch, randomConfig("no-match")), defaultFilterChain.getSslContextProviderSupplier(), noopConfig); @@ -716,8 +698,6 @@ public void sourceTypeLocal() throws Exception { "filter-chain-bar", null, HTTP_CONNECTION_MANAGER, tlsContextForDefaultFilterChain, tlsContextManager); - ServerRoutingConfig noopConfig = ServerRoutingConfig.create( - new ArrayList(), new ArrayList()); FilterChainSelector selector = new FilterChainSelector( ImmutableMap.of(filterChainWithMatch, noopConfig), defaultFilterChain.getSslContextProviderSupplier(), randomConfig("default")); @@ -777,8 +757,6 @@ public void sourcePrefixRange_moreSpecificWith2Wins() EnvoyServerProtoData.FilterChain defaultFilterChain = new EnvoyServerProtoData.FilterChain( "filter-chain-baz", null, HTTP_CONNECTION_MANAGER, null, tlsContextManager); - ServerRoutingConfig noopConfig = ServerRoutingConfig.create( - new ArrayList(), new ArrayList()); FilterChainSelector selector = new FilterChainSelector( ImmutableMap.of(filterChainMoreSpecificWith2, noopConfig, filterChainLessSpecific, randomConfig("no-match")), @@ -845,8 +823,6 @@ public void userEventTriggered(ChannelHandlerContext ctx, Object evt) { EnvoyServerProtoData.FilterChain defaultFilterChain = new EnvoyServerProtoData.FilterChain( "filter-chain-baz", null, HTTP_CONNECTION_MANAGER, null, null); - ServerRoutingConfig noopConfig = ServerRoutingConfig.create( - new ArrayList(), new ArrayList()); FilterChainSelector selector = new FilterChainSelector( ImmutableMap.of(filterChain1, noopConfig, filterChain2, noopConfig), defaultFilterChain.getSslContextProviderSupplier(), noopConfig); @@ -908,8 +884,6 @@ public void sourcePortMatch_exactMatchWinsOverEmptyList() throws Exception { EnvoyServerProtoData.FilterChain defaultFilterChain = new EnvoyServerProtoData.FilterChain( "filter-chain-baz", null, HTTP_CONNECTION_MANAGER, null, tlsContextManager); - ServerRoutingConfig noopConfig = ServerRoutingConfig.create( - new ArrayList(), new ArrayList()); FilterChainSelector selector = new FilterChainSelector( ImmutableMap.of(filterChainEmptySourcePorts, randomConfig("no-match"), filterChainSourcePortMatch, noopConfig), @@ -1059,8 +1033,6 @@ public void filterChain_5stepMatch() throws Exception { EnvoyServerProtoData.FilterChain defaultFilterChain = new EnvoyServerProtoData.FilterChain( "filter-chain-7", null, HTTP_CONNECTION_MANAGER, null, tlsContextManager); - ServerRoutingConfig noopConfig = ServerRoutingConfig.create( - new ArrayList(), new ArrayList()); Map map = new HashMap<>(); map.put(filterChain1, randomConfig("1")); map.put(filterChain2, randomConfig("2")); @@ -1142,8 +1114,6 @@ public void filterChainMatch_unsupportedMatchers() throws Exception { "filter-chain-baz", defaultFilterChainMatch, HTTP_CONNECTION_MANAGER, tlsContext3, mock(TlsContextManager.class)); - ServerRoutingConfig noopConfig = ServerRoutingConfig.create( - new ArrayList(), new ArrayList()); FilterChainSelector selector = new FilterChainSelector( ImmutableMap.of(filterChain1, randomConfig("1"), filterChain2, randomConfig("2")), defaultFilterChain.getSslContextProviderSupplier(), noopConfig); @@ -1177,7 +1147,8 @@ private static VirtualHost createVirtualHost(String name) { private static ServerRoutingConfig randomConfig(String domain) { return ServerRoutingConfig.create( - new ArrayList(), Arrays.asList(createVirtualHost(domain))); + new ArrayList(), new AtomicReference<>( + ImmutableList.of(createVirtualHost(domain)))); } private EnvoyServerProtoData.DownstreamTlsContext createTls() { diff --git a/xds/src/test/java/io/grpc/xds/XdsServerWrapperTest.java b/xds/src/test/java/io/grpc/xds/XdsServerWrapperTest.java index 4c91d5758f9..876b0913742 100644 --- a/xds/src/test/java/io/grpc/xds/XdsServerWrapperTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsServerWrapperTest.java @@ -376,12 +376,11 @@ public void run() { tlsContextManager); xdsClient.deliverLdsUpdate(Collections.singletonList(filterChain), null); start.get(5000, TimeUnit.MILLISECONDS); - FilterChainSelector selector = selectorRef.get(); assertThat(ldsWatched).isEqualTo("grpc/server?udpa.resource.listening_address=0.0.0.0:1"); - assertThat(selector.getRoutingConfigs()).isEqualTo(ImmutableMap.of( - filterChain, ServerRoutingConfig.create(httpConnectionManager.httpFilterConfigs(), - httpConnectionManager.virtualHosts()) - )); + assertThat(selectorRef.get().getRoutingConfigs().size()).isEqualTo(1); + ServerRoutingConfig realConfig = selectorRef.get().getRoutingConfigs().get(filterChain); + assertThat(realConfig.virtualHosts().get()).isEqualTo(httpConnectionManager.virtualHosts()); + assertThat(realConfig.httpFilterConfigs()).isEqualTo(httpConnectionManager.httpFilterConfigs()); verify(listener).onServing(); verify(mockServer).start(); } @@ -427,15 +426,22 @@ public void run() { Collections.singletonList(createVirtualHost("virtual-host-2"))); start.get(5000, TimeUnit.MILLISECONDS); verify(mockServer).start(); - assertThat(selectorRef.get().getRoutingConfigs()).isEqualTo(ImmutableMap.of( - f0, ServerRoutingConfig.create( - hcmVirtual.httpFilterConfigs(), hcmVirtual.virtualHosts()), - f2, ServerRoutingConfig.create(f2.getHttpConnectionManager().httpFilterConfigs(), - Collections.singletonList(createVirtualHost("virtual-host-1"))) - )); - assertThat(selectorRef.get().getDefaultRoutingConfig()).isEqualTo( - ServerRoutingConfig.create(f3.getHttpConnectionManager().httpFilterConfigs(), - Collections.singletonList(createVirtualHost("virtual-host-2")))); + ServerRoutingConfig realConfig = selectorRef.get().getRoutingConfigs().get(f0); + assertThat(realConfig.virtualHosts().get()).isEqualTo( + Collections.singletonList(createVirtualHost("virtual-host-0"))); + assertThat(realConfig.httpFilterConfigs()).isEqualTo( + f0.getHttpConnectionManager().httpFilterConfigs()); + assertThat(selectorRef.get().getRoutingConfigs().size()).isEqualTo(2); + realConfig = selectorRef.get().getRoutingConfigs().get(f2); + assertThat(realConfig.virtualHosts().get()).isEqualTo( + Collections.singletonList(createVirtualHost("virtual-host-1"))); + assertThat(realConfig.httpFilterConfigs()).isEqualTo( + f2.getHttpConnectionManager().httpFilterConfigs()); + realConfig = selectorRef.get().getDefaultRoutingConfig(); + assertThat(realConfig.virtualHosts().get()).isEqualTo( + Collections.singletonList(createVirtualHost("virtual-host-2"))); + assertThat(realConfig.httpFilterConfigs()).isEqualTo( + f3.getHttpConnectionManager().httpFilterConfigs()); assertThat(selectorRef.get().getDefaultSslContextProviderSupplier()).isEqualTo( f3.getSslContextProviderSupplier()); } @@ -469,38 +475,53 @@ public void run() { Collections.singletonList(createVirtualHost("virtual-host-0"))); start.get(5000, TimeUnit.MILLISECONDS); verify(mockServer, times(1)).start(); - assertThat(selectorRef.get().getRoutingConfigs()).isEqualTo(ImmutableMap.of( - f0, ServerRoutingConfig.create( - f0.getHttpConnectionManager().httpFilterConfigs(), - Collections.singletonList(createVirtualHost("virtual-host-0"))), - f1, ServerRoutingConfig.create(f1.getHttpConnectionManager().httpFilterConfigs(), - Collections.singletonList(createVirtualHost("virtual-host-0"))) - )); - assertThat(selectorRef.get().getDefaultRoutingConfig()).isEqualTo( - ServerRoutingConfig.create(f2.getHttpConnectionManager().httpFilterConfigs(), - Collections.singletonList(createVirtualHost("virtual-host-0")))); + ServerRoutingConfig realConfig = selectorRef.get().getRoutingConfigs().get(f0); + assertThat(realConfig.virtualHosts().get()).isEqualTo( + Collections.singletonList(createVirtualHost("virtual-host-0"))); + assertThat(realConfig.httpFilterConfigs()).isEqualTo( + f0.getHttpConnectionManager().httpFilterConfigs()); + realConfig = selectorRef.get().getRoutingConfigs().get(f1); + assertThat(realConfig.virtualHosts().get()).isEqualTo( + Collections.singletonList(createVirtualHost("virtual-host-0"))); + assertThat(realConfig.httpFilterConfigs()).isEqualTo( + f1.getHttpConnectionManager().httpFilterConfigs()); + + realConfig = selectorRef.get().getDefaultRoutingConfig(); + assertThat(realConfig.virtualHosts().get()).isEqualTo( + Collections.singletonList(createVirtualHost("virtual-host-0"))); + assertThat(realConfig.httpFilterConfigs()).isEqualTo( + f2.getHttpConnectionManager().httpFilterConfigs()); assertThat(selectorRef.get().getDefaultSslContextProviderSupplier()).isSameInstanceAs( f2.getSslContextProviderSupplier()); EnvoyServerProtoData.FilterChain f3 = createFilterChain("filter-chain-3", createRds("r0")); EnvoyServerProtoData.FilterChain f4 = createFilterChain("filter-chain-4", createRds("r1")); + EnvoyServerProtoData.FilterChain f5 = createFilterChain("filter-chain-4", createRds("r1")); xdsClient.rdsCount = new CountDownLatch(1); - xdsClient.deliverLdsUpdate(Arrays.asList(f1, f3), f4); + xdsClient.deliverLdsUpdate(Arrays.asList(f5, f3), f4); xdsClient.rdsCount.await(5, TimeUnit.SECONDS); xdsClient.deliverRdsUpdate("r1", Collections.singletonList(createVirtualHost("virtual-host-1"))); xdsClient.deliverRdsUpdate("r0", Collections.singletonList(createVirtualHost("virtual-host-0"))); - assertThat(selectorRef.get().getRoutingConfigs()).isEqualTo(ImmutableMap.of( - f1, ServerRoutingConfig.create( - f1.getHttpConnectionManager().httpFilterConfigs(), - Collections.singletonList(createVirtualHost("virtual-host-0"))), - f3, ServerRoutingConfig.create(f3.getHttpConnectionManager().httpFilterConfigs(), - Collections.singletonList(createVirtualHost("virtual-host-0"))) - )); - assertThat(selectorRef.get().getDefaultRoutingConfig()).isEqualTo( - ServerRoutingConfig.create(f4.getHttpConnectionManager().httpFilterConfigs(), - Collections.singletonList(createVirtualHost("virtual-host-1")))); + + assertThat(selectorRef.get().getRoutingConfigs().size()).isEqualTo(2); + realConfig = selectorRef.get().getRoutingConfigs().get(f5); + assertThat(realConfig.virtualHosts().get()).isEqualTo( + Collections.singletonList(createVirtualHost("virtual-host-1"))); + assertThat(realConfig.httpFilterConfigs()).isEqualTo( + f5.getHttpConnectionManager().httpFilterConfigs()); + realConfig = selectorRef.get().getRoutingConfigs().get(f3); + assertThat(realConfig.virtualHosts().get()).isEqualTo( + Collections.singletonList(createVirtualHost("virtual-host-0"))); + assertThat(realConfig.httpFilterConfigs()).isEqualTo( + f3.getHttpConnectionManager().httpFilterConfigs()); + + realConfig = selectorRef.get().getDefaultRoutingConfig(); + assertThat(realConfig.virtualHosts().get()).isEqualTo( + Collections.singletonList(createVirtualHost("virtual-host-1"))); + assertThat(realConfig.httpFilterConfigs()).isEqualTo( + f4.getHttpConnectionManager().httpFilterConfigs()); assertThat(selectorRef.get().getDefaultSslContextProviderSupplier()).isSameInstanceAs( f4.getSslContextProviderSupplier()); verify(mockServer, times(1)).start(); @@ -535,27 +556,38 @@ public void run() { xdsClient.rdsCount.await(); xdsClient.rdsWatchers.get("r0").onError(Status.CANCELLED); start.get(5000, TimeUnit.MILLISECONDS); - assertThat(selectorRef.get().getRoutingConfigs().get(f1)).isEqualTo(ServerRoutingConfig.create( - ImmutableList.of(), ImmutableList.of()) - ); + assertThat(selectorRef.get().getRoutingConfigs().size()).isEqualTo(2); + ServerRoutingConfig realConfig = selectorRef.get().getRoutingConfigs().get(f1); + assertThat(realConfig.virtualHosts().get()).isNull(); + assertThat(realConfig.httpFilterConfigs()).isEqualTo( + f1.getHttpConnectionManager().httpFilterConfigs()); + realConfig = selectorRef.get().getRoutingConfigs().get(f0); + assertThat(realConfig.virtualHosts().get()).isEqualTo(hcmVirtual.virtualHosts()); + assertThat(realConfig.httpFilterConfigs()).isEqualTo( + f0.getHttpConnectionManager().httpFilterConfigs()); + xdsClient.deliverRdsUpdate("r0", Collections.singletonList(createVirtualHost("virtual-host-1"))); - assertThat(selectorRef.get().getRoutingConfigs().get(f1)).isEqualTo( - ServerRoutingConfig.create(f1.getHttpConnectionManager().httpFilterConfigs(), - Collections.singletonList(createVirtualHost("virtual-host-1")))); + realConfig = selectorRef.get().getRoutingConfigs().get(f1); + assertThat(realConfig.virtualHosts().get()).isEqualTo( + Collections.singletonList(createVirtualHost("virtual-host-1"))); + assertThat(realConfig.httpFilterConfigs()).isEqualTo( + f1.getHttpConnectionManager().httpFilterConfigs()); xdsClient.rdsWatchers.get("r0").onError(Status.CANCELLED); - assertThat(selectorRef.get().getRoutingConfigs().get(f1)).isEqualTo( - ServerRoutingConfig.create(f1.getHttpConnectionManager().httpFilterConfigs(), - Collections.singletonList(createVirtualHost("virtual-host-1")))); + realConfig = selectorRef.get().getRoutingConfigs().get(f1); + assertThat(realConfig.virtualHosts().get()).isEqualTo( + Collections.singletonList(createVirtualHost("virtual-host-1"))); + assertThat(realConfig.httpFilterConfigs()).isEqualTo( + f1.getHttpConnectionManager().httpFilterConfigs()); xdsClient.rdsWatchers.get("r0").onResourceDoesNotExist("r0"); - assertThat(selectorRef.get().getRoutingConfigs().get(f1)).isEqualTo(ServerRoutingConfig.create( - ImmutableList.of(), ImmutableList.of()) - ); + realConfig = selectorRef.get().getRoutingConfigs().get(f1); + assertThat(realConfig.virtualHosts().get()).isNull(); + assertThat(realConfig.httpFilterConfigs()).isEqualTo( + f1.getHttpConnectionManager().httpFilterConfigs()); } - @Test public void error() throws Exception { final SettableFuture start = SettableFuture.create(); @@ -602,11 +634,12 @@ public void run() { verify(mockBuilder, times(1)).build(); verify(mockServer, times(2)).start(); verify(listener, times(1)).onServing(); - assertThat(selectorRef.get().getRoutingConfigs()).isEqualTo(ImmutableMap.of( - filterChain1, ServerRoutingConfig.create( - filterChain1.getHttpConnectionManager().httpFilterConfigs(), - Collections.singletonList(createVirtualHost("virtual-host-1"))) - )); + assertThat(selectorRef.get().getRoutingConfigs().size()).isEqualTo(1); + ServerRoutingConfig realConfig = selectorRef.get().getRoutingConfigs().get(filterChain1); + assertThat(realConfig.virtualHosts().get()).isEqualTo( + Collections.singletonList(createVirtualHost("virtual-host-1"))); + assertThat(realConfig.httpFilterConfigs()).isEqualTo( + filterChain1.getHttpConnectionManager().httpFilterConfigs()); // xds update after start xdsClient.deliverRdsUpdate("rds", Collections.singletonList(createVirtualHost("virtual-host-2"))); @@ -615,11 +648,12 @@ public void run() { verify(mockBuilder, times(1)).build(); verify(mockServer, times(2)).start(); verify(listener, times(2)).onNotServing(any(StatusException.class)); - assertThat(selectorRef.get().getRoutingConfigs()).isEqualTo(ImmutableMap.of( - filterChain1, ServerRoutingConfig.create( - filterChain1.getHttpConnectionManager().httpFilterConfigs(), - Collections.singletonList(createVirtualHost("virtual-host-2"))) - )); + assertThat(selectorRef.get().getRoutingConfigs().size()).isEqualTo(1); + realConfig = selectorRef.get().getRoutingConfigs().get(filterChain1); + assertThat(realConfig.virtualHosts().get()).isEqualTo( + Collections.singletonList(createVirtualHost("virtual-host-2"))); + assertThat(realConfig.httpFilterConfigs()).isEqualTo( + filterChain1.getHttpConnectionManager().httpFilterConfigs()); assertThat(sslSupplier1.isShutdown()).isFalse(); // not serving after serving @@ -652,11 +686,12 @@ public void run() { verify(mockServer, times(3)).start(); verify(listener, times(1)).onServing(); verify(listener, times(3)).onNotServing(any(StatusException.class)); - assertThat(selectorRef.get().getRoutingConfigs()).isEqualTo(ImmutableMap.of( - filterChain2, ServerRoutingConfig.create( - filterChain2.getHttpConnectionManager().httpFilterConfigs(), - Collections.singletonList(createVirtualHost("virtual-host-1"))) - )); + assertThat(selectorRef.get().getRoutingConfigs().size()).isEqualTo(1); + realConfig = selectorRef.get().getRoutingConfigs().get(filterChain2); + assertThat(realConfig.virtualHosts().get()).isEqualTo( + Collections.singletonList(createVirtualHost("virtual-host-1"))); + assertThat(realConfig.httpFilterConfigs()).isEqualTo( + filterChain2.getHttpConnectionManager().httpFilterConfigs()); assertThat(executor.numPendingTasks()).isEqualTo(1); xdsClient.ldsWatcher.onResourceDoesNotExist(ldsResource); verify(mockServer, times(4)).shutdown(); @@ -676,11 +711,13 @@ public void run() { verify(listener, times(1)).onServing(); when(mockServer.isShutdown()).thenReturn(false); verify(listener, times(4)).onNotServing(any(StatusException.class)); - assertThat(selectorRef.get().getRoutingConfigs()).isEqualTo(ImmutableMap.of( - filterChain3, ServerRoutingConfig.create( - filterChain3.getHttpConnectionManager().httpFilterConfigs(), - Collections.singletonList(createVirtualHost("virtual-host-1"))) - )); + + assertThat(selectorRef.get().getRoutingConfigs().size()).isEqualTo(1); + realConfig = selectorRef.get().getRoutingConfigs().get(filterChain3); + assertThat(realConfig.virtualHosts().get()).isEqualTo( + Collections.singletonList(createVirtualHost("virtual-host-1"))); + assertThat(realConfig.httpFilterConfigs()).isEqualTo( + filterChain3.getHttpConnectionManager().httpFilterConfigs()); xdsServerWrapper.shutdown(); verify(mockServer, times(5)).shutdown(); assertThat(sslSupplier3.isShutdown()).isTrue(); @@ -828,7 +865,8 @@ public void run() { verify(mockBuilder).intercept(interceptorCaptor.capture()); ConfigApplyingInterceptor interceptor = interceptorCaptor.getValue(); ServerRoutingConfig failingConfig = ServerRoutingConfig.create( - ImmutableList.of(), ImmutableList.of()); + ImmutableList.of(), new AtomicReference>() + ); ServerCall serverCall = mock(ServerCall.class); when(serverCall.getAttributes()).thenReturn( Attributes.newBuilder().set(ATTR_SERVER_ROUTING_CONFIG, failingConfig).build()); @@ -841,7 +879,7 @@ public void run() { Status status = statusCaptor.getValue(); assertThat(status.getCode()).isEqualTo(Status.UNAVAILABLE.getCode()); assertThat(status.getDescription()).isEqualTo( - "Missing xDS routing config. RDS config unavailable."); + "Missing xDS routing config VirtualHosts due to RDS config unavailable."); } @Test @@ -900,7 +938,7 @@ public ServerCall.Listener interceptCall(ServerCall(ImmutableList.of(virtualHost)) ); ServerCall serverCall = mock(ServerCall.class); ServerCallHandler mockNext = mock(ServerCallHandler.class); @@ -984,8 +1022,8 @@ private static ServerRoutingConfig createRoutingConfig(String path, String domai FilterConfig f0 = mock(FilterConfig.class); when(f0.typeUrl()).thenReturn(filterType); return ServerRoutingConfig.create( - Arrays.asList(new NamedFilterConfig("filter-config-name-0", f0)), - Collections.singletonList(virtualHost) + Arrays.asList(new NamedFilterConfig("filter-config-name-0", f0)), + new AtomicReference<>(ImmutableList.of(virtualHost)) ); } From 67d5f1b0d66e16c360cca156b036a29ca8bc23bf Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Thu, 9 Sep 2021 09:06:13 -0700 Subject: [PATCH 07/76] stub: update CallStreamObserver stabilization issue --- stub/src/main/java/io/grpc/stub/CallStreamObserver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stub/src/main/java/io/grpc/stub/CallStreamObserver.java b/stub/src/main/java/io/grpc/stub/CallStreamObserver.java index 7b3d4e55b3e..b014e9cfc25 100644 --- a/stub/src/main/java/io/grpc/stub/CallStreamObserver.java +++ b/stub/src/main/java/io/grpc/stub/CallStreamObserver.java @@ -50,7 +50,7 @@ *

DO NOT MOCK: The API is too complex to reliably mock. Use InProcessChannelBuilder to create * "real" RPCs suitable for testing. */ -@ExperimentalApi("https://github.com/grpc/grpc-java/issues/1788") +@ExperimentalApi("https://github.com/grpc/grpc-java/issues/8499") public abstract class CallStreamObserver implements StreamObserver { /** From a6df9de7bb9388e3f014c06755c7f4a2752ed782 Mon Sep 17 00:00:00 2001 From: yifeizhuang Date: Thu, 9 Sep 2021 09:55:27 -0700 Subject: [PATCH 08/76] xds: add terminal http filter verification, remove lame route filter, add hcm as terminal network filter verification (#8342) * xds: add terminal filter verification, remove lame route filter * move last filter check inline * add server validate terminal filter --- .../java/io/grpc/xds/ClientXdsClient.java | 79 +++++++----- xds/src/main/java/io/grpc/xds/LameFilter.java | 121 ------------------ .../java/io/grpc/xds/XdsNameResolver.java | 42 +----- .../io/grpc/xds/ClientXdsClientDataTest.java | 94 ++++++++++++-- .../io/grpc/xds/ClientXdsClientTestBase.java | 27 ++-- .../io/grpc/xds/ClientXdsClientV2Test.java | 5 + .../io/grpc/xds/ClientXdsClientV3Test.java | 21 +++ .../java/io/grpc/xds/XdsNameResolverTest.java | 24 ---- 8 files changed, 174 insertions(+), 239 deletions(-) delete mode 100644 xds/src/main/java/io/grpc/xds/LameFilter.java diff --git a/xds/src/main/java/io/grpc/xds/ClientXdsClient.java b/xds/src/main/java/io/grpc/xds/ClientXdsClient.java index 21cf78b1269..693848897c6 100644 --- a/xds/src/main/java/io/grpc/xds/ClientXdsClient.java +++ b/xds/src/main/java/io/grpc/xds/ClientXdsClient.java @@ -334,41 +334,32 @@ static FilterChain parseFilterChain( TlsContextManager tlsContextManager, FilterRegistry filterRegistry, Set uniqueSet, Set certProviderInstances, boolean parseHttpFilters) throws ResourceInvalidException { - io.grpc.xds.HttpConnectionManager httpConnectionManager = null; - HashSet uniqueNames = new HashSet<>(); - for (io.envoyproxy.envoy.config.listener.v3.Filter filter : proto.getFiltersList()) { - if (!uniqueNames.add(filter.getName())) { - throw new ResourceInvalidException( - "FilterChain " + proto.getName() + " with duplicated filter: " + filter.getName()); - } - if (!filter.hasTypedConfig()) { - throw new ResourceInvalidException( - "FilterChain " + proto.getName() + " contains filter " + filter.getName() - + " without typed_config"); - } - Any any = filter.getTypedConfig(); - // HttpConnectionManager is the only supported network filter at the moment. - if (!any.getTypeUrl().equals(TYPE_URL_HTTP_CONNECTION_MANAGER)) { - throw new ResourceInvalidException( - "FilterChain " + proto.getName() + " contains filter " + filter.getName() - + " with unsupported typed_config type " + any.getTypeUrl()); - } - if (httpConnectionManager == null) { - HttpConnectionManager hcmProto; - try { - hcmProto = any.unpack(HttpConnectionManager.class); - } catch (InvalidProtocolBufferException e) { - throw new ResourceInvalidException("FilterChain " + proto.getName() + " with filter " - + filter.getName() + " failed to unpack message", e); - } - httpConnectionManager = parseHttpConnectionManager( - hcmProto, rdsResources, filterRegistry, parseHttpFilters, false /* isForClient */); - } - } - if (httpConnectionManager == null) { + if (proto.getFiltersCount() != 1) { throw new ResourceInvalidException("FilterChain " + proto.getName() - + " missing required HttpConnectionManager filter"); + + " should contain exact one HttpConnectionManager filter"); } + io.envoyproxy.envoy.config.listener.v3.Filter filter = proto.getFiltersList().get(0); + if (!filter.hasTypedConfig()) { + throw new ResourceInvalidException( + "FilterChain " + proto.getName() + " contains filter " + filter.getName() + + " without typed_config"); + } + Any any = filter.getTypedConfig(); + // HttpConnectionManager is the only supported network filter at the moment. + if (!any.getTypeUrl().equals(TYPE_URL_HTTP_CONNECTION_MANAGER)) { + throw new ResourceInvalidException( + "FilterChain " + proto.getName() + " contains filter " + filter.getName() + + " with unsupported typed_config type " + any.getTypeUrl()); + } + HttpConnectionManager hcmProto; + try { + hcmProto = any.unpack(HttpConnectionManager.class); + } catch (InvalidProtocolBufferException e) { + throw new ResourceInvalidException("FilterChain " + proto.getName() + " with filter " + + filter.getName() + " failed to unpack message", e); + } + io.grpc.xds.HttpConnectionManager httpConnectionManager = parseHttpConnectionManager( + hcmProto, rdsResources, filterRegistry, parseHttpFilters, false /* isForClient */); EnvoyServerProtoData.DownstreamTlsContext downstreamTlsContext = null; if (proto.hasTransportSocket()) { @@ -762,10 +753,14 @@ static io.grpc.xds.HttpConnectionManager parseHttpConnectionManager( // Parse http filters. List filterConfigs = null; if (parseHttpFilter) { + if (proto.getHttpFiltersList().isEmpty()) { + throw new ResourceInvalidException("Missing HttpFilter in HttpConnectionManager."); + } filterConfigs = new ArrayList<>(); Set names = new HashSet<>(); - for (io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpFilter - httpFilter : proto.getHttpFiltersList()) { + for (int i = 0; i < proto.getHttpFiltersCount(); i++) { + io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpFilter + httpFilter = proto.getHttpFiltersList().get(i); String filterName = httpFilter.getName(); if (!names.add(filterName)) { throw new ResourceInvalidException( @@ -773,6 +768,11 @@ static io.grpc.xds.HttpConnectionManager parseHttpConnectionManager( } StructOrError filterConfig = parseHttpFilter(httpFilter, filterRegistry, isForClient); + if ((i == proto.getHttpFiltersCount() - 1) + && (filterConfig == null || !isTerminalFilter(filterConfig.struct))) { + throw new ResourceInvalidException("The last HttpFilter must be a terminal filter: " + + filterName); + } if (filterConfig == null) { continue; } @@ -781,6 +781,10 @@ static io.grpc.xds.HttpConnectionManager parseHttpConnectionManager( "HttpConnectionManager contains invalid HttpFilter: " + filterConfig.getErrorDetail()); } + if ((i < proto.getHttpFiltersCount() - 1) && isTerminalFilter(filterConfig.getStruct())) { + throw new ResourceInvalidException("A terminal HttpFilter must be the last filter: " + + filterName); + } filterConfigs.add(new NamedFilterConfig(filterName, filterConfig.struct)); } } @@ -821,6 +825,11 @@ static io.grpc.xds.HttpConnectionManager parseHttpConnectionManager( "HttpConnectionManager neither has inlined route_config nor RDS"); } + // hard-coded: currently router config is the only terminal filter. + private static boolean isTerminalFilter(FilterConfig filterConfig) { + return RouterFilter.ROUTER_CONFIG.equals(filterConfig); + } + @VisibleForTesting @Nullable // Returns null if the filter is optional but not supported. static StructOrError parseHttpFilter( diff --git a/xds/src/main/java/io/grpc/xds/LameFilter.java b/xds/src/main/java/io/grpc/xds/LameFilter.java deleted file mode 100644 index 4dd1d3c96ed..00000000000 --- a/xds/src/main/java/io/grpc/xds/LameFilter.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright 2021 The gRPC Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.grpc.xds; - -import com.google.common.util.concurrent.MoreExecutors; -import com.google.protobuf.Message; -import io.grpc.CallOptions; -import io.grpc.Channel; -import io.grpc.ClientCall; -import io.grpc.ClientInterceptor; -import io.grpc.Context; -import io.grpc.LoadBalancer.PickSubchannelArgs; -import io.grpc.Metadata; -import io.grpc.MethodDescriptor; -import io.grpc.Status; -import io.grpc.xds.Filter.ClientInterceptorBuilder; -import java.util.concurrent.Executor; -import java.util.concurrent.ScheduledExecutorService; -import javax.annotation.Nullable; - -/** - * A filter that fails all RPCs. To be added to the end of filter chain if RouterFilter is absent. - */ -enum LameFilter implements Filter, ClientInterceptorBuilder { - INSTANCE; - - static final FilterConfig LAME_CONFIG = new FilterConfig() { - @Override - public String typeUrl() { - throw new UnsupportedOperationException("shouldn't be called"); - } - - @Override - public String toString() { - return "LAME_CONFIG"; - } - }; - - @Override - public String[] typeUrls() { - return new String[0]; - } - - @Override - public ConfigOrError parseFilterConfig(Message rawProtoMessage) { - throw new UnsupportedOperationException(); - } - - @Override - public ConfigOrError parseFilterConfigOverride(Message rawProtoMessage) { - throw new UnsupportedOperationException(); - } - - @Nullable - @Override - public ClientInterceptor buildClientInterceptor( - FilterConfig config, @Nullable FilterConfig overrideConfig, PickSubchannelArgs args, - ScheduledExecutorService scheduler) { - class LameInterceptor implements ClientInterceptor { - - @Override - public ClientCall interceptCall( - MethodDescriptor method, final CallOptions callOptions, Channel next) { - final Context context = Context.current(); - return new ClientCall() { - @Override - public void start(final Listener listener, Metadata headers) { - Executor callExecutor = callOptions.getExecutor(); - if (callExecutor == null) { // This should never happen in practice because - // ManagedChannelImpl.ConfigSelectingClientCall always provides CallOptions with - // a callExecutor. - // TODO(https://github.com/grpc/grpc-java/issues/7868) - callExecutor = MoreExecutors.directExecutor(); - } - callExecutor.execute( - new Runnable() { - @Override - public void run() { - Context previous = context.attach(); - try { - listener.onClose( - Status.UNAVAILABLE.withDescription("No router filter"), new Metadata()); - } finally { - context.detach(previous); - } - } - }); - } - - @Override - public void request(int numMessages) {} - - @Override - public void cancel(@Nullable String message, @Nullable Throwable cause) {} - - @Override - public void halfClose() {} - - @Override - public void sendMessage(ReqT message) {} - }; - } - } - - return new LameInterceptor(); - } -} diff --git a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java index 787336e4b54..bb73f3c1823 100644 --- a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java +++ b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java @@ -23,7 +23,6 @@ import com.google.common.base.Joiner; import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; import com.google.common.collect.Sets; import com.google.gson.Gson; import com.google.protobuf.util.Durations; @@ -95,8 +94,6 @@ final class XdsNameResolver extends NameResolver { CallOptions.Key.create("io.grpc.xds.CLUSTER_SELECTION_KEY"); static final CallOptions.Key RPC_HASH_KEY = CallOptions.Key.create("io.grpc.xds.RPC_HASH_KEY"); - private static final NamedFilterConfig LAME_FILTER = - new NamedFilterConfig(null, LameFilter.LAME_CONFIG); @VisibleForTesting static boolean enableTimeout = Strings.isNullOrEmpty(System.getenv("GRPC_XDS_EXPERIMENTAL_ENABLE_TIMEOUT")) @@ -374,10 +371,6 @@ public Result selectConfig(PickSubchannelArgs args) { do { routingCfg = routingConfig; selectedOverrideConfigs = new HashMap<>(routingCfg.virtualHostOverrideConfig); - if (routingCfg.filterChain != null - && Iterables.getLast(routingCfg.filterChain).equals(LAME_FILTER)) { - break; - } for (Route route : routingCfg.routes) { if (matchRoute(route.routeMatch(), "/" + args.getMethodDescriptor().getFullMethodName(), headers, random)) { @@ -442,12 +435,7 @@ public Result selectConfig(PickSubchannelArgs args) { if (routingCfg.filterChain != null) { for (NamedFilterConfig namedFilter : routingCfg.filterChain) { FilterConfig filterConfig = namedFilter.filterConfig; - Filter filter; - if (namedFilter.equals(LAME_FILTER)) { - filter = LameFilter.INSTANCE; - } else { - filter = filterRegistry.get(filterConfig.typeUrl()); - } + Filter filter = filterRegistry.get(filterConfig.typeUrl()); if (filter instanceof ClientInterceptorBuilder) { ClientInterceptor interceptor = ((ClientInterceptorBuilder) filter) .buildClientInterceptor( @@ -458,12 +446,6 @@ public Result selectConfig(PickSubchannelArgs args) { } } } - if (Iterables.getLast(routingCfg.filterChain).equals(LAME_FILTER)) { - return Result.newBuilder() - .setConfig(config) - .setInterceptor(combineInterceptors(filterInterceptors)) - .build(); - } } final String finalCluster = cluster; final long hash = generateHash(selectedRoute.routeAction().hashPolicies(), headers); @@ -754,27 +736,7 @@ private void updateRoutes(List virtualHosts, long httpMaxStreamDura return; } - // A router filter is required for request routing. For backward compatibility, routing - // is always enabled for gRPC clients without HttpFilter support. List routes = virtualHost.routes(); - List filterChain = null; - if (filterConfigs != null) { - boolean hasRouter = false; - filterChain = new ArrayList<>(filterConfigs.size()); - for (NamedFilterConfig namedFilter : filterConfigs) { - filterChain.add(namedFilter); - if (namedFilter.filterConfig.equals(RouterFilter.ROUTER_CONFIG)) { - hasRouter = true; - break; - } - } - if (!hasRouter) { - // Fail all RPCs if a router filter is not present. Reference counts for all currently - // selectable clusters should be reclaimed. - filterChain.add(LAME_FILTER); - routes = Collections.emptyList(); - } - } // Populate all clusters to which requests can be routed to through the virtual host. Set clusters = new HashSet<>(); @@ -815,7 +777,7 @@ private void updateRoutes(List virtualHosts, long httpMaxStreamDura // selectable. routingConfig = new RoutingConfig( - httpMaxStreamDurationNano, routes, filterChain, + httpMaxStreamDurationNano, routes, filterConfigs, virtualHost.filterConfigOverrides()); shouldUpdateResult = false; for (String cluster : deletedClusters) { diff --git a/xds/src/test/java/io/grpc/xds/ClientXdsClientDataTest.java b/xds/src/test/java/io/grpc/xds/ClientXdsClientDataTest.java index 80cd2a8046e..807f512b0f4 100644 --- a/xds/src/test/java/io/grpc/xds/ClientXdsClientDataTest.java +++ b/xds/src/test/java/io/grpc/xds/ClientXdsClientDataTest.java @@ -129,7 +129,7 @@ public class ClientXdsClientDataTest { @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467 @Rule public final ExpectedException thrown = ExpectedException.none(); - private final FilterRegistry filterRegistry = FilterRegistry.newRegistry(); + private final FilterRegistry filterRegistry = FilterRegistry.getDefaultRegistry(); private boolean originalEnableRetry; @Before @@ -1132,6 +1132,9 @@ public void parseHttpConnectionManager_duplicateHttpFilters() throws ResourceInv HttpFilter.newBuilder().setName("envoy.filter.foo").setIsOptional(true)) .addHttpFilters( HttpFilter.newBuilder().setName("envoy.filter.foo").setIsOptional(true)) + .addHttpFilters( + HttpFilter.newBuilder().setName("terminal").setTypedConfig( + Any.pack(Router.newBuilder().build())).setIsOptional(true)) .build(); thrown.expect(ResourceInvalidException.class); thrown.expectMessage("HttpConnectionManager contains duplicate HttpFilter: envoy.filter.foo"); @@ -1140,6 +1143,70 @@ public void parseHttpConnectionManager_duplicateHttpFilters() throws ResourceInv true /* does not matter */); } + @Test + public void parseHttpConnectionManager_lastNotTerminal() throws ResourceInvalidException { + filterRegistry.register(FaultFilter.INSTANCE); + HttpConnectionManager hcm = + HttpConnectionManager.newBuilder() + .addHttpFilters( + HttpFilter.newBuilder().setName("envoy.filter.foo").setIsOptional(true)) + .addHttpFilters( + HttpFilter.newBuilder().setName("envoy.filter.bar").setIsOptional(true) + .setTypedConfig(Any.pack(HTTPFault.newBuilder().build()))) + .build(); + thrown.expect(ResourceInvalidException.class); + thrown.expectMessage("The last HttpFilter must be a terminal filter: envoy.filter.bar"); + ClientXdsClient.parseHttpConnectionManager( + hcm, new HashSet(), filterRegistry, true /* parseHttpFilter */, + true /* does not matter */); + } + + @Test + public void parseHttpConnectionManager_terminalNotLast() throws ResourceInvalidException { + filterRegistry.register(RouterFilter.INSTANCE); + HttpConnectionManager hcm = + HttpConnectionManager.newBuilder() + .addHttpFilters( + HttpFilter.newBuilder().setName("terminal").setTypedConfig( + Any.pack(Router.newBuilder().build())).setIsOptional(true)) + .addHttpFilters( + HttpFilter.newBuilder().setName("envoy.filter.foo").setIsOptional(true)) + .build(); + thrown.expect(ResourceInvalidException.class); + thrown.expectMessage("A terminal HttpFilter must be the last filter: terminal"); + ClientXdsClient.parseHttpConnectionManager( + hcm, new HashSet(), filterRegistry, true /* parseHttpFilter */, + true); + } + + @Test + public void parseHttpConnectionManager_unknownFilters() throws ResourceInvalidException { + HttpConnectionManager hcm = + HttpConnectionManager.newBuilder() + .addHttpFilters( + HttpFilter.newBuilder().setName("envoy.filter.foo").setIsOptional(true)) + .addHttpFilters( + HttpFilter.newBuilder().setName("envoy.filter.bar").setIsOptional(true)) + .build(); + thrown.expect(ResourceInvalidException.class); + thrown.expectMessage("The last HttpFilter must be a terminal filter: envoy.filter.bar"); + ClientXdsClient.parseHttpConnectionManager( + hcm, new HashSet(), filterRegistry, true /* parseHttpFilter */, + true /* does not matter */); + } + + @Test + public void parseHttpConnectionManager_emptyFilters() throws ResourceInvalidException { + HttpConnectionManager hcm = + HttpConnectionManager.newBuilder() + .build(); + thrown.expect(ResourceInvalidException.class); + thrown.expectMessage("Missing HttpFilter in HttpConnectionManager."); + ClientXdsClient.parseHttpConnectionManager( + hcm, new HashSet(), filterRegistry, true /* parseHttpFilter */, + true /* does not matter */); + } + @Test public void parseCluster_ringHashLbPolicy_defaultLbConfig() throws ResourceInvalidException { Cluster cluster = Cluster.newBuilder() @@ -1280,7 +1347,8 @@ public void parseServerSideListener_useOriginalDst() throws ResourceInvalidExcep @Test public void parseServerSideListener_nonUniqueFilterChainMatch() throws ResourceInvalidException { Filter filter1 = buildHttpConnectionManagerFilter( - HttpFilter.newBuilder().setName("http-filter-1").setIsOptional(true).build()); + HttpFilter.newBuilder().setName("http-filter-1").setTypedConfig( + Any.pack(Router.newBuilder().build())).setIsOptional(true).build()); FilterChainMatch filterChainMatch1 = FilterChainMatch.newBuilder() .addAllSourcePorts(Arrays.asList(80, 8080)) @@ -1296,7 +1364,8 @@ public void parseServerSideListener_nonUniqueFilterChainMatch() throws ResourceI .addFilters(filter1) .build(); Filter filter2 = buildHttpConnectionManagerFilter( - HttpFilter.newBuilder().setName("http-filter-2").setIsOptional(true).build()); + HttpFilter.newBuilder().setName("http-filter-2").setTypedConfig( + Any.pack(Router.newBuilder().build())).setIsOptional(true).build()); FilterChainMatch filterChainMatch2 = FilterChainMatch.newBuilder() .addAllSourcePorts(Arrays.asList(443, 8080)) @@ -1328,7 +1397,8 @@ public void parseServerSideListener_nonUniqueFilterChainMatch() throws ResourceI public void parseServerSideListener_nonUniqueFilterChainMatch_sameFilter() throws ResourceInvalidException { Filter filter1 = buildHttpConnectionManagerFilter( - HttpFilter.newBuilder().setName("http-filter-1").setIsOptional(true).build()); + HttpFilter.newBuilder().setName("http-filter-1").setTypedConfig( + Any.pack(Router.newBuilder().build())).setIsOptional(true).build()); FilterChainMatch filterChainMatch1 = FilterChainMatch.newBuilder() .addAllSourcePorts(Arrays.asList(80, 8080)) @@ -1343,7 +1413,8 @@ public void parseServerSideListener_nonUniqueFilterChainMatch_sameFilter() .addFilters(filter1) .build(); Filter filter2 = buildHttpConnectionManagerFilter( - HttpFilter.newBuilder().setName("http-filter-2").setIsOptional(true).build()); + HttpFilter.newBuilder().setName("http-filter-2").setTypedConfig( + Any.pack(Router.newBuilder().build())).setIsOptional(true).build()); FilterChainMatch filterChainMatch2 = FilterChainMatch.newBuilder() .addAllSourcePorts(Arrays.asList(443, 8080)) @@ -1374,7 +1445,8 @@ public void parseServerSideListener_nonUniqueFilterChainMatch_sameFilter() @Test public void parseServerSideListener_uniqueFilterChainMatch() throws ResourceInvalidException { Filter filter1 = buildHttpConnectionManagerFilter( - HttpFilter.newBuilder().setName("http-filter-1").setIsOptional(true).build()); + HttpFilter.newBuilder().setName("http-filter-1").setTypedConfig( + Any.pack(Router.newBuilder().build())).setIsOptional(true).build()); FilterChainMatch filterChainMatch1 = FilterChainMatch.newBuilder() .addAllSourcePorts(Arrays.asList(80, 8080)) @@ -1391,7 +1463,8 @@ public void parseServerSideListener_uniqueFilterChainMatch() throws ResourceInva .addFilters(filter1) .build(); Filter filter2 = buildHttpConnectionManagerFilter( - HttpFilter.newBuilder().setName("http-filter-2").setIsOptional(true).build()); + HttpFilter.newBuilder().setName("http-filter-2").setTypedConfig( + Any.pack(Router.newBuilder().build())).setIsOptional(true).build()); FilterChainMatch filterChainMatch2 = FilterChainMatch.newBuilder() .addAllSourcePorts(Arrays.asList(443, 8080)) @@ -1428,7 +1501,7 @@ public void parseFilterChain_noHcm() throws ResourceInvalidException { .build(); thrown.expect(ResourceInvalidException.class); thrown.expectMessage( - "FilterChain filter-chain-foo missing required HttpConnectionManager filter"); + "FilterChain filter-chain-foo should contain exact one HttpConnectionManager filter"); ClientXdsClient.parseFilterChain( filterChain, new HashSet(), null, filterRegistry, null, null, true /* does not matter */); @@ -1447,7 +1520,7 @@ public void parseFilterChain_duplicateFilter() throws ResourceInvalidException { .build(); thrown.expect(ResourceInvalidException.class); thrown.expectMessage( - "FilterChain filter-chain-foo with duplicated filter: envoy.http_connection_manager"); + "FilterChain filter-chain-foo should contain exact one HttpConnectionManager filter"); ClientXdsClient.parseFilterChain( filterChain, new HashSet(), null, filterRegistry, null, null, true /* does not matter */); @@ -1504,6 +1577,7 @@ public void parseFilterChain_noName_generatedUuid() throws ResourceInvalidExcept HttpFilter.newBuilder() .setName("http-filter-foo") .setIsOptional(true) + .setTypedConfig(Any.pack(Router.newBuilder().build())) .build())) .build(); FilterChain filterChain2 = @@ -1512,6 +1586,7 @@ public void parseFilterChain_noName_generatedUuid() throws ResourceInvalidExcept .addFilters(buildHttpConnectionManagerFilter( HttpFilter.newBuilder() .setName("http-filter-bar") + .setTypedConfig(Any.pack(Router.newBuilder().build())) .setIsOptional(true) .build())) .build(); @@ -1525,7 +1600,6 @@ public void parseFilterChain_noName_generatedUuid() throws ResourceInvalidExcept assertThat(parsedFilterChain1.getName()).isNotEqualTo(parsedFilterChain2.getName()); } - @Test public void validateCommonTlsContext_tlsParams() throws ResourceInvalidException { CommonTlsContext commonTlsContext = CommonTlsContext.newBuilder() diff --git a/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java b/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java index 55bd6ba3e9d..a2a29ffe989 100644 --- a/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java +++ b/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java @@ -39,6 +39,7 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Message; import io.envoyproxy.envoy.config.route.v3.FilterConfig; +import io.envoyproxy.envoy.extensions.filters.http.router.v3.Router; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CertificateProviderPluginInstance; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CommonTlsContext; import io.grpc.BindableService; @@ -658,7 +659,8 @@ public void ldsResourceUpdate_withFaultInjection() { mf.buildHttpFaultTypedConfig( 1L, 2, "cluster1", ImmutableList.of(), 3, null, null, null), - false)))); + false), + mf.buildHttpFilter("terminal", Any.pack(Router.newBuilder().build()), true)))); call.sendResponse(LDS, listener, VERSION_1, "0000"); // Client sends an ACK LDS request. @@ -993,7 +995,7 @@ public void rdsResourcesDeletedByLdsTcpListener() { verifySubscribedResourcesMetadataSizes(1, 0, 1, 0); Message hcmFilter = mf.buildHttpConnectionManagerFilter( - RDS_RESOURCE, null, Collections.emptyList()); + RDS_RESOURCE, null, Collections.singletonList(mf.buildTerminalFilter())); Message downstreamTlsContext = CommonTlsContextTestsUtil.buildTestDownstreamTlsContext( "google-sds-config-default", "ROOTCA", false); Message filterChain = mf.buildFilterChain( @@ -1028,7 +1030,7 @@ public void rdsResourcesDeletedByLdsTcpListener() { null, mf.buildRouteConfiguration( "route-bar.googleapis.com", mf.buildOpaqueVirtualHosts(VHOST_SIZE)), - Collections.emptyList()); + Collections.singletonList(mf.buildTerminalFilter())); filterChain = mf.buildFilterChain( Collections.emptyList(), downstreamTlsContext, "envoy.transport_sockets.tls", hcmFilter); @@ -2203,7 +2205,8 @@ public void serverSideListenerFound() { ClientXdsClientTestBase.DiscoveryRpcCall call = startResourceWatcher(LDS, LISTENER_RESOURCE, ldsResourceWatcher); Message hcmFilter = mf.buildHttpConnectionManagerFilter( - "route-foo.googleapis.com", null, Collections.emptyList()); + "route-foo.googleapis.com", null, + Collections.singletonList(mf.buildTerminalFilter())); Message downstreamTlsContext = CommonTlsContextTestsUtil.buildTestDownstreamTlsContext( "google-sds-config-default", "ROOTCA", false); Message filterChain = mf.buildFilterChain( @@ -2226,7 +2229,8 @@ public void serverSideListenerFound() { assertThat(parsedFilterChain.getFilterChainMatch().getApplicationProtocols()).isEmpty(); assertThat(parsedFilterChain.getHttpConnectionManager().rdsName()) .isEqualTo("route-foo.googleapis.com"); - assertThat(parsedFilterChain.getHttpConnectionManager().httpFilterConfigs()).isEmpty(); + assertThat(parsedFilterChain.getHttpConnectionManager().httpFilterConfigs().get(0).filterConfig) + .isEqualTo(RouterFilter.ROUTER_CONFIG); assertThat(fakeClock.getPendingTasks(LDS_RESOURCE_FETCH_TIMEOUT_TASK_FILTER)).isEmpty(); } @@ -2237,7 +2241,8 @@ public void serverSideListenerNotFound() { ClientXdsClientTestBase.DiscoveryRpcCall call = startResourceWatcher(LDS, LISTENER_RESOURCE, ldsResourceWatcher); Message hcmFilter = mf.buildHttpConnectionManagerFilter( - "route-foo.googleapis.com", null, Collections.emptyList()); + "route-foo.googleapis.com", null, + Collections.singletonList(mf.buildTerminalFilter())); Message downstreamTlsContext = CommonTlsContextTestsUtil.buildTestDownstreamTlsContext( "google-sds-config-default", "ROOTCA", false); Message filterChain = mf.buildFilterChain( @@ -2263,7 +2268,8 @@ public void serverSideListenerResponseErrorHandling_badDownstreamTlsContext() { ClientXdsClientTestBase.DiscoveryRpcCall call = startResourceWatcher(LDS, LISTENER_RESOURCE, ldsResourceWatcher); Message hcmFilter = mf.buildHttpConnectionManagerFilter( - "route-foo.googleapis.com", null, Collections.emptyList()); + "route-foo.googleapis.com", null, + Collections.singletonList(mf.buildTerminalFilter())); Message downstreamTlsContext = CommonTlsContextTestsUtil.buildTestDownstreamTlsContext( null, null,false); Message filterChain = mf.buildFilterChain( @@ -2286,7 +2292,8 @@ public void serverSideListenerResponseErrorHandling_badTransportSocketName() { ClientXdsClientTestBase.DiscoveryRpcCall call = startResourceWatcher(LDS, LISTENER_RESOURCE, ldsResourceWatcher); Message hcmFilter = mf.buildHttpConnectionManagerFilter( - "route-foo.googleapis.com", null, Collections.emptyList()); + "route-foo.googleapis.com", null, + Collections.singletonList(mf.buildTerminalFilter())); Message downstreamTlsContext = CommonTlsContextTestsUtil.buildTestDownstreamTlsContext( "cert1", "cert2",false); Message filterChain = mf.buildFilterChain( @@ -2385,7 +2392,7 @@ protected abstract static class MessageFactory { /** Throws {@link InvalidProtocolBufferException} on {@link Any#unpack(Class)}. */ protected static final Any FAILING_ANY = Any.newBuilder().setTypeUrl("fake").build(); - protected final Message buildListenerWithApiListener(String name, Message routeConfiguration) { + protected Message buildListenerWithApiListener(String name, Message routeConfiguration) { return buildListenerWithApiListener( name, routeConfiguration, Collections.emptyList()); } @@ -2470,5 +2477,7 @@ protected abstract Message buildListenerWithFilterChain( protected abstract Message buildHttpConnectionManagerFilter( @Nullable String rdsName, @Nullable Message routeConfig, List httpFilters); + + protected abstract Message buildTerminalFilter(); } } diff --git a/xds/src/test/java/io/grpc/xds/ClientXdsClientV2Test.java b/xds/src/test/java/io/grpc/xds/ClientXdsClientV2Test.java index 39f5d1a1a2e..1a69b6fc650 100644 --- a/xds/src/test/java/io/grpc/xds/ClientXdsClientV2Test.java +++ b/xds/src/test/java/io/grpc/xds/ClientXdsClientV2Test.java @@ -641,6 +641,11 @@ protected Message buildHttpConnectionManagerFilter( @Nullable String rdsName, @Nullable Message routeConfig, List httpFilters) { throw new UnsupportedOperationException(); } + + @Override + protected Message buildTerminalFilter() { + throw new UnsupportedOperationException(); + } } /** diff --git a/xds/src/test/java/io/grpc/xds/ClientXdsClientV3Test.java b/xds/src/test/java/io/grpc/xds/ClientXdsClientV3Test.java index dfd407ef016..6df36e1c31e 100644 --- a/xds/src/test/java/io/grpc/xds/ClientXdsClientV3Test.java +++ b/xds/src/test/java/io/grpc/xds/ClientXdsClientV3Test.java @@ -74,6 +74,7 @@ import io.envoyproxy.envoy.extensions.filters.http.fault.v3.FaultAbort; import io.envoyproxy.envoy.extensions.filters.http.fault.v3.FaultAbort.HeaderAbort; import io.envoyproxy.envoy.extensions.filters.http.fault.v3.HTTPFault; +import io.envoyproxy.envoy.extensions.filters.http.router.v3.Router; import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager; import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpFilter; import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.Rds; @@ -276,6 +277,15 @@ protected Message buildListenerWithApiListener( .build(); } + @Override + protected Message buildListenerWithApiListener(String name, Message routeConfiguration) { + return buildListenerWithApiListener(name, routeConfiguration, Arrays.asList( + HttpFilter.newBuilder() + .setName("terminal") + .setTypedConfig(Any.pack(Router.newBuilder().build())).build() + )); + } + @Override protected Message buildListenerWithApiListenerForRds(String name, String rdsResourceName) { return Listener.newBuilder() @@ -291,6 +301,10 @@ protected Message buildListenerWithApiListenerForRds(String name, String rdsReso .setConfigSource( ConfigSource.newBuilder() .setAds(AggregatedConfigSource.getDefaultInstance()))) + .addHttpFilters( + HttpFilter.newBuilder() + .setName("terminal") + .setTypedConfig(Any.pack(Router.newBuilder().build()))) .build()))) .build(); } @@ -742,6 +756,13 @@ protected Message buildHttpConnectionManagerFilter( Any.pack(hcmBuilder.build(), "type.googleapis.com")) .build(); } + + @Override + protected Message buildTerminalFilter() { + return HttpFilter.newBuilder() + .setName("terminal") + .setTypedConfig(Any.pack(Router.newBuilder().build())).build(); + } } /** diff --git a/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java b/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java index 22d7302f207..7a8fec5f74a 100644 --- a/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java @@ -1384,20 +1384,6 @@ public long nanoTime() { + " Deadline exceeded after 0.000004000s. ")); } - @Test - public void resolved_withNoRouterFilter() { - resolver.start(mockListener); - FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); - xdsClient.deliverLdsUpdateWithNoRouterFilter(); - verify(mockListener).onResult(resolutionResultCaptor.capture()); - ResolutionResult result = resolutionResultCaptor.getValue(); - InternalConfigSelector configSelector = result.getAttributes().get(InternalConfigSelector.KEY); - ClientCall.Listener observer = startNewCall( - TestMethodDescriptors.voidMethod(), configSelector, Collections.emptyMap(), - CallOptions.DEFAULT); - verifyRpcFailed(observer, Status.UNAVAILABLE.withDescription("No router filter")); - } - @Test public void resolved_faultAbortAndDelayInLdsUpdateInLdsUpdate() { resolver.start(mockListener); @@ -1826,16 +1812,6 @@ void deliverLdsUpdateWithFaultInjection( 0L, Collections.singletonList(virtualHost), filterChain))); } - void deliverLdsUpdateWithNoRouterFilter() { - VirtualHost virtualHost = VirtualHost.create( - "virtual-host", - Collections.singletonList(AUTHORITY), - Collections.emptyList(), - Collections.emptyMap()); - ldsWatcher.onChanged(LdsUpdate.forApiListener(HttpConnectionManager.forVirtualHosts( - 0L, Collections.singletonList(virtualHost), ImmutableList.of()))); - } - void deliverLdsUpdateForRdsNameWithFaultInjection( final String rdsName, @Nullable FaultConfig httpFilterFaultConfig) { if (httpFilterFaultConfig == null) { From 7ad7876e99e0217914783308920701ca0b8b4dd6 Mon Sep 17 00:00:00 2001 From: yifeizhuang Date: Thu, 9 Sep 2021 12:15:27 -0700 Subject: [PATCH 09/76] fix header matcher for null value (#8503) --- .../java/io/grpc/xds/XdsNameResolver.java | 32 +------------------ .../java/io/grpc/xds/internal/Matchers.java | 9 ++---- .../io/grpc/xds/internal/MatcherTest.java | 13 ++++++++ 3 files changed, 17 insertions(+), 37 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java index bb73f3c1823..e2905dee26e 100644 --- a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java +++ b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java @@ -576,7 +576,7 @@ static boolean matchRoute(RouteMatch routeMatch, String fullMethodName, return false; } for (HeaderMatcher headerMatcher : routeMatch.headerMatchers()) { - if (!matchHeader(headerMatcher, getHeaderValue(headers, headerMatcher.name()))) { + if (!headerMatcher.matches(getHeaderValue(headers, headerMatcher.name()))) { return false; } } @@ -597,36 +597,6 @@ private static boolean matchPath(PathMatcher pathMatcher, String fullMethodName) return pathMatcher.regEx().matches(fullMethodName); } - // TODO(zivy): consider reuse Matchers.HeaderMatcher.matches() - private static boolean matchHeader(HeaderMatcher headerMatcher, @Nullable String value) { - if (headerMatcher.present() != null) { - return (value == null) == headerMatcher.present().equals(headerMatcher.inverted()); - } - if (value == null) { - return false; - } - boolean baseMatch; - if (headerMatcher.exactValue() != null) { - baseMatch = headerMatcher.exactValue().equals(value); - } else if (headerMatcher.safeRegEx() != null) { - baseMatch = headerMatcher.safeRegEx().matches(value); - } else if (headerMatcher.range() != null) { - long numValue; - try { - numValue = Long.parseLong(value); - baseMatch = numValue >= headerMatcher.range().start() - && numValue <= headerMatcher.range().end(); - } catch (NumberFormatException ignored) { - baseMatch = false; - } - } else if (headerMatcher.prefix() != null) { - baseMatch = value.startsWith(headerMatcher.prefix()); - } else { - baseMatch = value.endsWith(headerMatcher.suffix()); - } - return baseMatch != headerMatcher.inverted(); - } - @Nullable private static String getHeaderValue(Metadata headers, String headerName) { if (headerName.endsWith(Metadata.BINARY_HEADER_SUFFIX)) { diff --git a/xds/src/main/java/io/grpc/xds/internal/Matchers.java b/xds/src/main/java/io/grpc/xds/internal/Matchers.java index 28ec8418297..3bf7b7723e2 100644 --- a/xds/src/main/java/io/grpc/xds/internal/Matchers.java +++ b/xds/src/main/java/io/grpc/xds/internal/Matchers.java @@ -117,13 +117,8 @@ private static HeaderMatcher create(String name, @Nullable String exactValue, /** Returns the matching result. */ public boolean matches(@Nullable String value) { - if (present() != null) { - return (value == null) == present().equals(inverted()); - } - // FIXME(zivy@): invert result for null value. - // https://github.com/envoyproxy/envoy/blob/0fae6970ddaf93f024908ba304bbd2b34e997a51/source/common/http/header_utility.cc#L130 if (value == null) { - return false; + return present() != null && present() == inverted(); } boolean baseMatch; if (exactValue() != null) { @@ -141,6 +136,8 @@ public boolean matches(@Nullable String value) { } } else if (prefix() != null) { baseMatch = value.startsWith(prefix()); + } else if (present() != null) { + baseMatch = present(); } else { baseMatch = value.endsWith(suffix()); } diff --git a/xds/src/test/java/io/grpc/xds/internal/MatcherTest.java b/xds/src/test/java/io/grpc/xds/internal/MatcherTest.java index 4fb4acc41f6..93a9b7087d6 100644 --- a/xds/src/test/java/io/grpc/xds/internal/MatcherTest.java +++ b/xds/src/test/java/io/grpc/xds/internal/MatcherTest.java @@ -127,45 +127,58 @@ public void headerMatcher() { HeaderMatcher matcher = HeaderMatcher.forExactValue("version", "v1", false); assertThat(matcher.matches("v1")).isTrue(); assertThat(matcher.matches("v2")).isFalse(); + assertThat(matcher.matches(null)).isFalse(); matcher = HeaderMatcher.forExactValue("version", "v1", true); assertThat(matcher.matches("v1")).isFalse(); assertThat(matcher.matches( "v2")).isTrue(); + assertThat(matcher.matches(null)).isFalse(); matcher = HeaderMatcher.forPresent("version", true, false); assertThat(matcher.matches("any")).isTrue(); assertThat(matcher.matches(null)).isFalse(); matcher = HeaderMatcher.forPresent("version", true, true); assertThat(matcher.matches("version")).isFalse(); + assertThat(matcher.matches(null)).isTrue(); matcher = HeaderMatcher.forPresent("version", false, true); assertThat(matcher.matches("tag")).isTrue(); + assertThat(matcher.matches(null)).isFalse(); matcher = HeaderMatcher.forPresent("version", false, false); assertThat(matcher.matches("tag")).isFalse(); + assertThat(matcher.matches(null)).isTrue(); matcher = HeaderMatcher.forPrefix("version", "v2", false); assertThat(matcher.matches("v22")).isTrue(); + assertThat(matcher.matches(null)).isFalse(); matcher = HeaderMatcher.forPrefix("version", "v2", true); assertThat(matcher.matches("v22")).isFalse(); + assertThat(matcher.matches(null)).isFalse(); matcher = HeaderMatcher.forSuffix("version", "v1", false); assertThat(matcher.matches("xv1")).isTrue(); assertThat(matcher.matches("v1x")).isFalse(); + assertThat(matcher.matches(null)).isFalse(); matcher = HeaderMatcher.forSuffix("version", "v2", true); assertThat(matcher.matches("xv1")).isTrue(); assertThat(matcher.matches("1v2")).isFalse(); + assertThat(matcher.matches(null)).isFalse(); matcher = HeaderMatcher.forSafeRegEx("version", Pattern.compile("v2.*"), false); assertThat(matcher.matches("v2..")).isTrue(); assertThat(matcher.matches("v1")).isFalse(); + assertThat(matcher.matches(null)).isFalse(); matcher = HeaderMatcher.forSafeRegEx("version", Pattern.compile("v1\\..*"), true); assertThat(matcher.matches("v1.43")).isFalse(); assertThat(matcher.matches("v2")).isTrue(); + assertThat(matcher.matches(null)).isFalse(); matcher = HeaderMatcher.forRange("version", Range.create(8080L, 8090L), false); assertThat(matcher.matches("8080")).isTrue(); assertThat(matcher.matches("1")).isFalse(); + assertThat(matcher.matches(null)).isFalse(); matcher = HeaderMatcher.forRange("version", Range.create(8080L, 8090L), true); assertThat(matcher.matches("1")).isTrue(); assertThat(matcher.matches("8080")).isFalse(); + assertThat(matcher.matches(null)).isFalse(); } } From 7a65c7428324670c9549cb4f413f7ca73e3f7666 Mon Sep 17 00:00:00 2001 From: ZHANG Dapeng Date: Sat, 11 Sep 2021 21:57:47 -0700 Subject: [PATCH 10/76] xds: apply valid resources while NACKing update (#8506) Implementing [gRFC A46](https://github.com/grpc/proposal/pull/260) --- .../java/io/grpc/xds/ClientXdsClient.java | 135 +++++------ .../io/grpc/xds/ClientXdsClientTestBase.java | 209 +++++++++++++++--- 2 files changed, 248 insertions(+), 96 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/ClientXdsClient.java b/xds/src/main/java/io/grpc/xds/ClientXdsClient.java index 693848897c6..dcd69e427c3 100644 --- a/xds/src/main/java/io/grpc/xds/ClientXdsClient.java +++ b/xds/src/main/java/io/grpc/xds/ClientXdsClient.java @@ -190,6 +190,7 @@ final class ClientXdsClient extends AbstractXdsClient { protected void handleLdsResponse(String versionInfo, List resources, String nonce) { Map parsedResources = new HashMap<>(resources.size()); Set unpackedResources = new HashSet<>(resources.size()); + Set invalidResources = new HashSet<>(); List errors = new ArrayList<>(); Set retainedRdsResources = new HashSet<>(); @@ -222,6 +223,7 @@ protected void handleLdsResponse(String versionInfo, List resources, String } catch (ResourceInvalidException e) { errors.add( "LDS response Listener '" + listenerName + "' validation error: " + e.getMessage()); + invalidResources.add(listenerName); continue; } @@ -231,19 +233,9 @@ protected void handleLdsResponse(String versionInfo, List resources, String getLogger().log(XdsLogLevel.INFO, "Received LDS Response version {0} nonce {1}. Parsed resources: {2}", versionInfo, nonce, unpackedResources); - - if (!errors.isEmpty()) { - handleResourcesRejected(ResourceType.LDS, unpackedResources, versionInfo, nonce, errors); - return; - } - - handleResourcesAccepted(ResourceType.LDS, parsedResources, versionInfo, nonce); - for (String resource : rdsResourceSubscribers.keySet()) { - if (!retainedRdsResources.contains(resource)) { - ResourceSubscriber subscriber = rdsResourceSubscribers.get(resource); - subscriber.onAbsent(); - } - } + handleResourceUpdate( + ResourceType.LDS, parsedResources, invalidResources, retainedRdsResources, versionInfo, + nonce, errors); } private LdsUpdate processClientSideListener( @@ -1313,6 +1305,7 @@ static StructOrError parseClusterWeight( protected void handleRdsResponse(String versionInfo, List resources, String nonce) { Map parsedResources = new HashMap<>(resources.size()); Set unpackedResources = new HashSet<>(resources.size()); + Set invalidResources = new HashSet<>(); List errors = new ArrayList<>(); for (int i = 0; i < resources.size(); i++) { @@ -1340,6 +1333,7 @@ protected void handleRdsResponse(String versionInfo, List resources, String errors.add( "RDS response RouteConfiguration '" + routeConfigName + "' validation error: " + e .getMessage()); + invalidResources.add(routeConfigName); continue; } @@ -1348,12 +1342,9 @@ protected void handleRdsResponse(String versionInfo, List resources, String getLogger().log(XdsLogLevel.INFO, "Received RDS Response version {0} nonce {1}. Parsed resources: {2}", versionInfo, nonce, unpackedResources); - - if (!errors.isEmpty()) { - handleResourcesRejected(ResourceType.RDS, unpackedResources, versionInfo, nonce, errors); - } else { - handleResourcesAccepted(ResourceType.RDS, parsedResources, versionInfo, nonce); - } + handleResourceUpdate( + ResourceType.RDS, parsedResources, invalidResources, Collections.emptySet(), + versionInfo, nonce, errors); } private static RdsUpdate processRouteConfiguration( @@ -1377,6 +1368,7 @@ private static RdsUpdate processRouteConfiguration( protected void handleCdsResponse(String versionInfo, List resources, String nonce) { Map parsedResources = new HashMap<>(resources.size()); Set unpackedResources = new HashSet<>(resources.size()); + Set invalidResources = new HashSet<>(); List errors = new ArrayList<>(); Set retainedEdsResources = new HashSet<>(); @@ -1413,6 +1405,7 @@ protected void handleCdsResponse(String versionInfo, List resources, String } catch (ResourceInvalidException e) { errors.add( "CDS response Cluster '" + clusterName + "' validation error: " + e.getMessage()); + invalidResources.add(clusterName); continue; } parsedResources.put(clusterName, new ParsedResource(cdsUpdate, resource)); @@ -1420,21 +1413,9 @@ protected void handleCdsResponse(String versionInfo, List resources, String getLogger().log(XdsLogLevel.INFO, "Received CDS Response version {0} nonce {1}. Parsed resources: {2}", versionInfo, nonce, unpackedResources); - - if (!errors.isEmpty()) { - handleResourcesRejected(ResourceType.CDS, unpackedResources, versionInfo, nonce, errors); - return; - } - - handleResourcesAccepted(ResourceType.CDS, parsedResources, versionInfo, nonce); - // CDS responses represents the state of the world, EDS resources not referenced in CDS - // resources should be deleted. - for (String resource : edsResourceSubscribers.keySet()) { - ResourceSubscriber subscriber = edsResourceSubscribers.get(resource); - if (!retainedEdsResources.contains(resource)) { - subscriber.onAbsent(); - } - } + handleResourceUpdate( + ResourceType.CDS, parsedResources, invalidResources, retainedEdsResources, versionInfo, + nonce, errors); } @VisibleForTesting @@ -1615,6 +1596,7 @@ private static StructOrError parseNonAggregateCluster( protected void handleEdsResponse(String versionInfo, List resources, String nonce) { Map parsedResources = new HashMap<>(resources.size()); Set unpackedResources = new HashSet<>(resources.size()); + Set invalidResources = new HashSet<>(); List errors = new ArrayList<>(); for (int i = 0; i < resources.size(); i++) { @@ -1649,16 +1631,17 @@ protected void handleEdsResponse(String versionInfo, List resources, String } catch (ResourceInvalidException e) { errors.add("EDS response ClusterLoadAssignment '" + clusterName + "' validation error: " + e.getMessage()); + invalidResources.add(clusterName); continue; } parsedResources.put(clusterName, new ParsedResource(edsUpdate, resource)); } - - if (!errors.isEmpty()) { - handleResourcesRejected(ResourceType.EDS, unpackedResources, versionInfo, nonce, errors); - } else { - handleResourcesAccepted(ResourceType.EDS, parsedResources, versionInfo, nonce); - } + getLogger().log( + XdsLogLevel.INFO, "Received EDS Response version {0} nonce {1}. Parsed resources: {2}", + versionInfo, nonce, unpackedResources); + handleResourceUpdate( + ResourceType.EDS, parsedResources, invalidResources, Collections.emptySet(), + versionInfo, nonce, errors); } private static EdsUpdate processClusterLoadAssignment(ClusterLoadAssignment assignment) @@ -2048,43 +2031,67 @@ private void cleanUpResourceTimers() { } } - private void handleResourcesAccepted( - ResourceType type, Map parsedResources, String version, - String nonce) { - ackResponse(type, version, nonce); - + private void handleResourceUpdate( + ResourceType type, Map parsedResources, Set invalidResources, + Set retainedResources, String version, String nonce, List errors) { + String errorDetail = null; + if (errors.isEmpty()) { + checkArgument(invalidResources.isEmpty(), "found invalid resources but missing errors"); + ackResponse(type, version, nonce); + } else { + errorDetail = Joiner.on('\n').join(errors); + getLogger().log(XdsLogLevel.WARNING, + "Failed processing {0} Response version {1} nonce {2}. Errors:\n{3}", + type, version, nonce, errorDetail); + nackResponse(type, nonce, errorDetail); + } long updateTime = timeProvider.currentTimeNanos(); for (Map.Entry entry : getSubscribedResourcesMap(type).entrySet()) { String resourceName = entry.getKey(); ResourceSubscriber subscriber = entry.getValue(); + // Attach error details to the subscribed resources that included in the ADS update. + if (invalidResources.contains(resourceName)) { + subscriber.onRejected(version, updateTime, errorDetail); + } // Notify the watchers. if (parsedResources.containsKey(resourceName)) { subscriber.onData(parsedResources.get(resourceName), version, updateTime); } else if (type == ResourceType.LDS || type == ResourceType.CDS) { + if (subscriber.data != null && invalidResources.contains(resourceName)) { + // Update is rejected but keep using the cached data. + if (type == ResourceType.LDS) { + LdsUpdate ldsUpdate = (LdsUpdate) subscriber.data; + io.grpc.xds.HttpConnectionManager hcm = ldsUpdate.httpConnectionManager(); + if (hcm != null) { + String rdsName = hcm.rdsName(); + if (rdsName != null) { + retainedResources.add(rdsName); + } + } + } else { + CdsUpdate cdsUpdate = (CdsUpdate) subscriber.data; + String edsName = cdsUpdate.edsServiceName(); + if (edsName == null) { + edsName = cdsUpdate.clusterName(); + } + retainedResources.add(edsName); + } + continue; + } // For State of the World services, notify watchers when their watched resource is missing // from the ADS update. subscriber.onAbsent(); } } - } - - private void handleResourcesRejected( - ResourceType type, Set unpackedResourceNames, String version, - String nonce, List errors) { - String errorDetail = Joiner.on('\n').join(errors); - getLogger().log(XdsLogLevel.WARNING, - "Failed processing {0} Response version {1} nonce {2}. Errors:\n{3}", - type, version, nonce, errorDetail); - nackResponse(type, nonce, errorDetail); - - long updateTime = timeProvider.currentTimeNanos(); - for (Map.Entry entry : getSubscribedResourcesMap(type).entrySet()) { - String resourceName = entry.getKey(); - ResourceSubscriber subscriber = entry.getValue(); - - // Attach error details to the subscribed resources that included in the ADS update. - if (unpackedResourceNames.contains(resourceName)) { - subscriber.onRejected(version, updateTime, errorDetail); + // LDS/CDS responses represents the state of the world, RDS/EDS resources not referenced in + // LDS/CDS resources should be deleted. + if (type == ResourceType.LDS || type == ResourceType.CDS) { + Map dependentSubscribers = + type == ResourceType.LDS ? rdsResourceSubscribers : edsResourceSubscribers; + for (String resource : dependentSubscribers.keySet()) { + if (!retainedResources.contains(resource)) { + dependentSubscribers.get(resource).onAbsent(); + } } } } diff --git a/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java b/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java index a2a29ffe989..e66c73163be 100644 --- a/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java +++ b/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java @@ -473,11 +473,11 @@ public void ldsResponseErrorHandling_someResourcesFailedUnpack() { List errors = ImmutableList.of( "LDS response Resource index 0 - can't decode Listener: ", "LDS response Resource index 2 - can't decode Listener: "); - verifyResourceMetadataNacked(LDS, LDS_RESOURCE, null, "", 0, VERSION_1, TIME_INCREMENT, errors); + verifyResourceMetadataAcked(LDS, LDS_RESOURCE, testListenerRds, VERSION_1, TIME_INCREMENT); verifySubscribedResourcesMetadataSizes(1, 0, 0, 0); // The response is NACKed with the same error message. call.verifyRequestNack(LDS, LDS_RESOURCE, "", "0000", NODE, errors); - verifyNoInteractions(ldsResourceWatcher); + verify(ldsResourceWatcher).onChanged(any(LdsUpdate.class)); } /** @@ -517,14 +517,14 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid() { "A", Any.pack(mf.buildListenerWithApiListenerForRds("A", "A.2")), "B", Any.pack(mf.buildListenerWithApiListenerInvalid("B"))); call.sendResponse(LDS, resourcesV2.values().asList(), VERSION_2, "0001"); - // {A, B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B - // {C} -> ACK, version 1 + // {A} -> ACK, version 2 + // {B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B + // {C} -> does not exist List errorsV2 = ImmutableList.of("LDS response Listener 'B' validation error: "); - verifyResourceMetadataNacked(LDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT, - VERSION_2, TIME_INCREMENT * 2, errorsV2); + verifyResourceMetadataAcked(LDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); verifyResourceMetadataNacked(LDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT, VERSION_2, TIME_INCREMENT * 2, errorsV2); - verifyResourceMetadataAcked(LDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); + verifyResourceMetadataDoesNotExist(LDS, "C"); call.verifyRequestNack(LDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); // LDS -> {B, C} version 3 @@ -532,7 +532,7 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid() { "B", Any.pack(mf.buildListenerWithApiListenerForRds("B", "B.3")), "C", Any.pack(mf.buildListenerWithApiListenerForRds("C", "C.3"))); call.sendResponse(LDS, resourcesV3.values().asList(), VERSION_3, "0002"); - // {A} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B + // {A} -> does not exist // {B, C} -> ACK, version 3 verifyResourceMetadataDoesNotExist(LDS, "A"); verifyResourceMetadataAcked(LDS, "B", resourcesV3.get("B"), VERSION_3, TIME_INCREMENT * 3); @@ -541,6 +541,73 @@ public void ldsResponseErrorHandling_subscribedResourceInvalid() { verifySubscribedResourcesMetadataSizes(3, 0, 0, 0); } + @Test + public void ldsResponseErrorHandling_subscribedResourceInvalid_withRdsSubscriptioin() { + List subscribedResourceNames = ImmutableList.of("A", "B", "C"); + xdsClient.watchLdsResource("A", ldsResourceWatcher); + xdsClient.watchRdsResource("A.1", rdsResourceWatcher); + xdsClient.watchLdsResource("B", ldsResourceWatcher); + xdsClient.watchRdsResource("B.1", rdsResourceWatcher); + xdsClient.watchLdsResource("C", ldsResourceWatcher); + xdsClient.watchRdsResource("C.1", rdsResourceWatcher); + DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); + assertThat(call).isNotNull(); + verifyResourceMetadataRequested(LDS, "A"); + verifyResourceMetadataRequested(LDS, "B"); + verifyResourceMetadataRequested(LDS, "C"); + verifyResourceMetadataRequested(RDS, "A.1"); + verifyResourceMetadataRequested(RDS, "B.1"); + verifyResourceMetadataRequested(RDS, "C.1"); + verifySubscribedResourcesMetadataSizes(3, 0, 3, 0); + + // LDS -> {A, B, C}, version 1 + ImmutableMap resourcesV1 = ImmutableMap.of( + "A", Any.pack(mf.buildListenerWithApiListenerForRds("A", "A.1")), + "B", Any.pack(mf.buildListenerWithApiListenerForRds("B", "B.1")), + "C", Any.pack(mf.buildListenerWithApiListenerForRds("C", "C.1"))); + call.sendResponse(LDS, resourcesV1.values().asList(), VERSION_1, "0000"); + // {A, B, C} -> ACK, version 1 + verifyResourceMetadataAcked(LDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT); + verifyResourceMetadataAcked(LDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT); + verifyResourceMetadataAcked(LDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); + call.verifyRequest(LDS, subscribedResourceNames, VERSION_1, "0000", NODE); + + // RDS -> {A.1, B.1, C.1}, version 1 + List vhostsV1 = mf.buildOpaqueVirtualHosts(1); + ImmutableMap resourcesV11 = ImmutableMap.of( + "A.1", Any.pack(mf.buildRouteConfiguration("A.1", vhostsV1)), + "B.1", Any.pack(mf.buildRouteConfiguration("B.1", vhostsV1)), + "C.1", Any.pack(mf.buildRouteConfiguration("C.1", vhostsV1))); + call.sendResponse(RDS, resourcesV11.values().asList(), VERSION_1, "0000"); + // {A.1, B.1, C.1} -> ACK, version 1 + verifyResourceMetadataAcked(RDS, "A.1", resourcesV11.get("A.1"), VERSION_1, TIME_INCREMENT * 2); + verifyResourceMetadataAcked(RDS, "B.1", resourcesV11.get("B.1"), VERSION_1, TIME_INCREMENT * 2); + verifyResourceMetadataAcked(RDS, "C.1", resourcesV11.get("C.1"), VERSION_1, TIME_INCREMENT * 2); + + // LDS -> {A, B}, version 2 + // Failed to parse endpoint B + ImmutableMap resourcesV2 = ImmutableMap.of( + "A", Any.pack(mf.buildListenerWithApiListenerForRds("A", "A.2")), + "B", Any.pack(mf.buildListenerWithApiListenerInvalid("B"))); + call.sendResponse(LDS, resourcesV2.values().asList(), VERSION_2, "0001"); + // {A} -> ACK, version 2 + // {B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B + // {C} -> does not exist + List errorsV2 = ImmutableList.of("LDS response Listener 'B' validation error: "); + verifyResourceMetadataAcked(LDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 3); + verifyResourceMetadataNacked( + LDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT, VERSION_2, TIME_INCREMENT * 3, + errorsV2); + verifyResourceMetadataDoesNotExist(LDS, "C"); + call.verifyRequestNack(LDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); + // {A.1} -> does not exist + // {B.1} -> version 1 + // {C.1} -> does not exist + verifyResourceMetadataDoesNotExist(RDS, "A.1"); + verifyResourceMetadataAcked(RDS, "B.1", resourcesV11.get("B.1"), VERSION_1, TIME_INCREMENT * 2); + verifyResourceMetadataDoesNotExist(RDS, "C.1"); + } + @Test public void ldsResourceFound_containsVirtualHosts() { DiscoveryRpcCall call = startResourceWatcher(LDS, LDS_RESOURCE, ldsResourceWatcher); @@ -807,11 +874,11 @@ public void rdsResponseErrorHandling_someResourcesFailedUnpack() { List errors = ImmutableList.of( "RDS response Resource index 0 - can't decode RouteConfiguration: ", "RDS response Resource index 2 - can't decode RouteConfiguration: "); - verifyResourceMetadataNacked(RDS, RDS_RESOURCE, null, "", 0, VERSION_1, TIME_INCREMENT, errors); + verifyResourceMetadataAcked(RDS, RDS_RESOURCE, testRouteConfig, VERSION_1, TIME_INCREMENT); verifySubscribedResourcesMetadataSizes(0, 0, 1, 0); // The response is NACKed with the same error message. call.verifyRequestNack(RDS, RDS_RESOURCE, "", "0000", NODE, errors); - verifyNoInteractions(rdsResourceWatcher); + verify(rdsResourceWatcher).onChanged(any(RdsUpdate.class)); } /** @@ -852,12 +919,12 @@ public void rdsResponseErrorHandling_subscribedResourceInvalid() { "A", Any.pack(mf.buildRouteConfiguration("A", mf.buildOpaqueVirtualHosts(2))), "B", Any.pack(mf.buildRouteConfigurationInvalid("B"))); call.sendResponse(RDS, resourcesV2.values().asList(), VERSION_2, "0001"); - // {A, B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B + // {A} -> ACK, version 2 + // {B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B // {C} -> ACK, version 1 List errorsV2 = ImmutableList.of("RDS response RouteConfiguration 'B' validation error: "); - verifyResourceMetadataNacked(RDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT, - VERSION_2, TIME_INCREMENT * 2, errorsV2); + verifyResourceMetadataAcked(RDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); verifyResourceMetadataNacked(RDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT, VERSION_2, TIME_INCREMENT * 2, errorsV2); verifyResourceMetadataAcked(RDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); @@ -869,10 +936,9 @@ public void rdsResponseErrorHandling_subscribedResourceInvalid() { "B", Any.pack(mf.buildRouteConfiguration("B", vhostsV3)), "C", Any.pack(mf.buildRouteConfiguration("C", vhostsV3))); call.sendResponse(RDS, resourcesV3.values().asList(), VERSION_3, "0002"); - // {A} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B + // {A} -> ACK, version 2 // {B, C} -> ACK, version 3 - verifyResourceMetadataNacked(RDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT, - VERSION_2, TIME_INCREMENT * 2, errorsV2); + verifyResourceMetadataAcked(RDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); verifyResourceMetadataAcked(RDS, "B", resourcesV3.get("B"), VERSION_3, TIME_INCREMENT * 3); verifyResourceMetadataAcked(RDS, "C", resourcesV3.get("C"), VERSION_3, TIME_INCREMENT * 3); call.verifyRequest(RDS, subscribedResourceNames, VERSION_3, "0002", NODE); @@ -1146,11 +1212,12 @@ public void cdsResponseErrorHandling_someResourcesFailedUnpack() { List errors = ImmutableList.of( "CDS response Resource index 0 - can't decode Cluster: ", "CDS response Resource index 2 - can't decode Cluster: "); - verifyResourceMetadataNacked(CDS, CDS_RESOURCE, null, "", 0, VERSION_1, TIME_INCREMENT, errors); + verifyResourceMetadataAcked( + CDS, CDS_RESOURCE, testClusterRoundRobin, VERSION_1, TIME_INCREMENT); verifySubscribedResourcesMetadataSizes(0, 1, 0, 0); // The response is NACKed with the same error message. call.verifyRequestNack(CDS, CDS_RESOURCE, "", "0000", NODE, errors); - verifyNoInteractions(cdsResourceWatcher); + verify(cdsResourceWatcher).onChanged(any(CdsUpdate.class)); } /** @@ -1198,14 +1265,14 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid() { )), "B", Any.pack(mf.buildClusterInvalid("B"))); call.sendResponse(CDS, resourcesV2.values().asList(), VERSION_2, "0001"); - // {A, B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B - // {C} -> ACK, version 1 + // {A} -> ACK, version 2 + // {B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B + // {C} -> does not exist List errorsV2 = ImmutableList.of("CDS response Cluster 'B' validation error: "); - verifyResourceMetadataNacked(CDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT, - VERSION_2, TIME_INCREMENT * 2, errorsV2); + verifyResourceMetadataAcked(CDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); verifyResourceMetadataNacked(CDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT, VERSION_2, TIME_INCREMENT * 2, errorsV2); - verifyResourceMetadataAcked(CDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); + verifyResourceMetadataDoesNotExist(CDS, "C"); call.verifyRequestNack(CDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); // CDS -> {B, C} version 3 @@ -1217,7 +1284,7 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid() { "envoy.transport_sockets.tls", null ))); call.sendResponse(CDS, resourcesV3.values().asList(), VERSION_3, "0002"); - // {A} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B + // {A} -> does not exit // {B, C} -> ACK, version 3 verifyResourceMetadataDoesNotExist(CDS, "A"); verifyResourceMetadataAcked(CDS, "B", resourcesV3.get("B"), VERSION_3, TIME_INCREMENT * 3); @@ -1225,6 +1292,82 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid() { call.verifyRequest(CDS, subscribedResourceNames, VERSION_3, "0002", NODE); } + @Test + public void cdsResponseErrorHandling_subscribedResourceInvalid_withEdsSubscription() { + List subscribedResourceNames = ImmutableList.of("A", "B", "C"); + xdsClient.watchCdsResource("A", cdsResourceWatcher); + xdsClient.watchEdsResource("A.1", edsResourceWatcher); + xdsClient.watchCdsResource("B", cdsResourceWatcher); + xdsClient.watchEdsResource("B.1", edsResourceWatcher); + xdsClient.watchCdsResource("C", cdsResourceWatcher); + xdsClient.watchEdsResource("C.1", edsResourceWatcher); + DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); + assertThat(call).isNotNull(); + verifyResourceMetadataRequested(CDS, "A"); + verifyResourceMetadataRequested(CDS, "B"); + verifyResourceMetadataRequested(CDS, "C"); + verifyResourceMetadataRequested(EDS, "A.1"); + verifyResourceMetadataRequested(EDS, "B.1"); + verifyResourceMetadataRequested(EDS, "C.1"); + verifySubscribedResourcesMetadataSizes(0, 3, 0, 3); + + // CDS -> {A, B, C}, version 1 + ImmutableMap resourcesV1 = ImmutableMap.of( + "A", Any.pack(mf.buildEdsCluster("A", "A.1", "round_robin", null, false, null, + "envoy.transport_sockets.tls", null + )), + "B", Any.pack(mf.buildEdsCluster("B", "B.1", "round_robin", null, false, null, + "envoy.transport_sockets.tls", null + )), + "C", Any.pack(mf.buildEdsCluster("C", "C.1", "round_robin", null, false, null, + "envoy.transport_sockets.tls", null + ))); + call.sendResponse(CDS, resourcesV1.values().asList(), VERSION_1, "0000"); + // {A, B, C} -> ACK, version 1 + verifyResourceMetadataAcked(CDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT); + verifyResourceMetadataAcked(CDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT); + verifyResourceMetadataAcked(CDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); + call.verifyRequest(CDS, subscribedResourceNames, VERSION_1, "0000", NODE); + + // EDS -> {A.1, B.1, C.1}, version 1 + List dropOverloads = ImmutableList.of(); + List endpointsV1 = ImmutableList.of(lbEndpointHealthy); + ImmutableMap resourcesV11 = ImmutableMap.of( + "A.1", Any.pack(mf.buildClusterLoadAssignment("A.1", endpointsV1, dropOverloads)), + "B.1", Any.pack(mf.buildClusterLoadAssignment("B.1", endpointsV1, dropOverloads)), + "C.1", Any.pack(mf.buildClusterLoadAssignment("C.1", endpointsV1, dropOverloads))); + call.sendResponse(EDS, resourcesV11.values().asList(), VERSION_1, "0000"); + // {A.1, B.1, C.1} -> ACK, version 1 + verifyResourceMetadataAcked(EDS, "A.1", resourcesV11.get("A.1"), VERSION_1, TIME_INCREMENT * 2); + verifyResourceMetadataAcked(EDS, "B.1", resourcesV11.get("B.1"), VERSION_1, TIME_INCREMENT * 2); + verifyResourceMetadataAcked(EDS, "C.1", resourcesV11.get("C.1"), VERSION_1, TIME_INCREMENT * 2); + + // CDS -> {A, B}, version 2 + // Failed to parse endpoint B + ImmutableMap resourcesV2 = ImmutableMap.of( + "A", Any.pack(mf.buildEdsCluster("A", "A.2", "round_robin", null, false, null, + "envoy.transport_sockets.tls", null + )), + "B", Any.pack(mf.buildClusterInvalid("B"))); + call.sendResponse(CDS, resourcesV2.values().asList(), VERSION_2, "0001"); + // {A} -> ACK, version 2 + // {B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B + // {C} -> does not exist + List errorsV2 = ImmutableList.of("CDS response Cluster 'B' validation error: "); + verifyResourceMetadataAcked(CDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 3); + verifyResourceMetadataNacked( + CDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT, VERSION_2, TIME_INCREMENT * 3, + errorsV2); + verifyResourceMetadataDoesNotExist(CDS, "C"); + call.verifyRequestNack(CDS, subscribedResourceNames, VERSION_1, "0001", NODE, errorsV2); + // {A.1} -> does not exist + // {B.1} -> version 1 + // {C.1} -> does not exist + verifyResourceMetadataDoesNotExist(EDS, "A.1"); + verifyResourceMetadataAcked(EDS, "B.1", resourcesV11.get("B.1"), VERSION_1, TIME_INCREMENT * 2); + verifyResourceMetadataDoesNotExist(EDS, "C.1"); + } + @Test public void cdsResourceFound() { DiscoveryRpcCall call = startResourceWatcher(CDS, CDS_RESOURCE, cdsResourceWatcher); @@ -1666,11 +1809,14 @@ public void edsResponseErrorHandling_someResourcesFailedUnpack() { List errors = ImmutableList.of( "EDS response Resource index 0 - can't decode ClusterLoadAssignment: ", "EDS response Resource index 2 - can't decode ClusterLoadAssignment: "); - verifyResourceMetadataNacked(EDS, EDS_RESOURCE, null, "", 0, VERSION_1, TIME_INCREMENT, errors); + verifyResourceMetadataAcked( + EDS, EDS_RESOURCE, testClusterLoadAssignment, VERSION_1, TIME_INCREMENT); verifySubscribedResourcesMetadataSizes(0, 0, 0, 1); // The response is NACKed with the same error message. call.verifyRequestNack(EDS, EDS_RESOURCE, "", "0000", NODE, errors); - verifyNoInteractions(edsResourceWatcher); + verify(edsResourceWatcher).onChanged(edsUpdateCaptor.capture()); + EdsUpdate edsUpdate = edsUpdateCaptor.getValue(); + assertThat(edsUpdate.clusterName).isEqualTo(EDS_RESOURCE); } /** @@ -1713,12 +1859,12 @@ public void edsResponseErrorHandling_subscribedResourceInvalid() { "A", Any.pack(mf.buildClusterLoadAssignment("A", endpointsV2, dropOverloads)), "B", Any.pack(mf.buildClusterLoadAssignmentInvalid("B"))); call.sendResponse(EDS, resourcesV2.values().asList(), VERSION_2, "0001"); - // {A, B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B + // {A} -> ACK, version 2 + // {B} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B // {C} -> ACK, version 1 List errorsV2 = ImmutableList.of("EDS response ClusterLoadAssignment 'B' validation error: "); - verifyResourceMetadataNacked(EDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT, - VERSION_2, TIME_INCREMENT * 2, errorsV2); + verifyResourceMetadataAcked(EDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); verifyResourceMetadataNacked(EDS, "B", resourcesV1.get("B"), VERSION_1, TIME_INCREMENT, VERSION_2, TIME_INCREMENT * 2, errorsV2); verifyResourceMetadataAcked(EDS, "C", resourcesV1.get("C"), VERSION_1, TIME_INCREMENT); @@ -1731,10 +1877,9 @@ public void edsResponseErrorHandling_subscribedResourceInvalid() { "B", Any.pack(mf.buildClusterLoadAssignment("B", endpointsV3, dropOverloads)), "C", Any.pack(mf.buildClusterLoadAssignment("C", endpointsV3, dropOverloads))); call.sendResponse(EDS, resourcesV3.values().asList(), VERSION_3, "0002"); - // {A} -> NACK, version 1, rejected version 2, rejected reason: Failed to parse B + // {A} -> ACK, version 2 // {B, C} -> ACK, version 3 - verifyResourceMetadataNacked(EDS, "A", resourcesV1.get("A"), VERSION_1, TIME_INCREMENT, - VERSION_2, TIME_INCREMENT * 2, errorsV2); + verifyResourceMetadataAcked(EDS, "A", resourcesV2.get("A"), VERSION_2, TIME_INCREMENT * 2); verifyResourceMetadataAcked(EDS, "B", resourcesV3.get("B"), VERSION_3, TIME_INCREMENT * 3); verifyResourceMetadataAcked(EDS, "C", resourcesV3.get("C"), VERSION_3, TIME_INCREMENT * 3); call.verifyRequest(EDS, subscribedResourceNames, VERSION_3, "0002", NODE); From 9ff54059d8055a3287e241644db853b14ac835c4 Mon Sep 17 00:00:00 2001 From: ZHANG Dapeng Date: Mon, 13 Sep 2021 08:31:00 -0700 Subject: [PATCH 11/76] xds: populate envoy RetryPolicy with no retryOn to resolver (#8511) Envoy RetryPolicy with empty retryOn should not be ignored as no retry config when selecting Route config. Therefore, if xDS update for a route contains a RetryPolicy that has no RetryOn value that we support, but the virtual host config does, xds client should choose the Envoy RetryPolicy from the route (even with no RetryOn), rather than choosing the one from virtual host, and try to convert it into grpc RetryPolicy, and end up with no retry. --- xds/src/main/java/io/grpc/xds/ClientXdsClient.java | 11 ++++------- xds/src/main/java/io/grpc/xds/XdsNameResolver.java | 5 +++-- .../java/io/grpc/xds/ClientXdsClientDataTest.java | 3 ++- .../test/java/io/grpc/xds/XdsNameResolverTest.java | 14 +++++++++++++- 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/ClientXdsClient.java b/xds/src/main/java/io/grpc/xds/ClientXdsClient.java index dcd69e427c3..d490c9861b9 100644 --- a/xds/src/main/java/io/grpc/xds/ClientXdsClient.java +++ b/xds/src/main/java/io/grpc/xds/ClientXdsClient.java @@ -1273,13 +1273,10 @@ private static StructOrError parseRetryPolicy( retryableStatusCodesBuilder.add(code); } List retryableStatusCodes = retryableStatusCodesBuilder.build(); - if (!retryableStatusCodes.isEmpty()) { - return StructOrError.fromStruct( - RetryPolicy.create( - maxAttempts, retryableStatusCodes, initialBackoff, maxBackoff, - /* perAttemptRecvTimeout= */ null)); - } - return null; + return StructOrError.fromStruct( + RetryPolicy.create( + maxAttempts, retryableStatusCodes, initialBackoff, maxBackoff, + /* perAttemptRecvTimeout= */ null)); } @VisibleForTesting diff --git a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java index e2905dee26e..4cd52c8b3f9 100644 --- a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java +++ b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java @@ -182,13 +182,14 @@ public void shutdown() { @VisibleForTesting static Map generateServiceConfigWithMethodConfig( @Nullable Long timeoutNano, @Nullable RetryPolicy retryPolicy) { - if (timeoutNano == null && retryPolicy == null) { + if (timeoutNano == null + && (retryPolicy == null || retryPolicy.retryableStatusCodes().isEmpty())) { return Collections.emptyMap(); } ImmutableMap.Builder methodConfig = ImmutableMap.builder(); methodConfig.put( "name", Collections.singletonList(Collections.emptyMap())); - if (retryPolicy != null) { + if (retryPolicy != null && !retryPolicy.retryableStatusCodes().isEmpty()) { ImmutableMap.Builder rawRetryPolicy = ImmutableMap.builder(); rawRetryPolicy.put("maxAttempts", (double) retryPolicy.maxAttempts()); rawRetryPolicy.put("initialBackoff", Durations.toString(retryPolicy.initialBackoff())); diff --git a/xds/src/test/java/io/grpc/xds/ClientXdsClientDataTest.java b/xds/src/test/java/io/grpc/xds/ClientXdsClientDataTest.java index 807f512b0f4..876615d0b39 100644 --- a/xds/src/test/java/io/grpc/xds/ClientXdsClientDataTest.java +++ b/xds/src/test/java/io/grpc/xds/ClientXdsClientDataTest.java @@ -552,7 +552,8 @@ public void parseRouteAction_withRetryPolicy() { .setRetryPolicy(builder.build()) .build(); struct = ClientXdsClient.parseRouteAction(proto, filterRegistry, false); - assertThat(struct.getStruct().retryPolicy()).isNull(); + assertThat(struct.getStruct().retryPolicy()).isNotNull(); + assertThat(struct.getStruct().retryPolicy().retryableStatusCodes()).isEmpty(); // base_interval unset builder diff --git a/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java b/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java index 7a8fec5f74a..babaa2b3034 100644 --- a/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java @@ -988,6 +988,8 @@ public void generateServiceConfig_forPerMethodConfig() throws IOException { RetryPolicy retryPolicy = RetryPolicy.create( 4, ImmutableList.of(Code.UNAVAILABLE, Code.CANCELLED), Durations.fromMillis(100), Durations.fromMillis(200), null); + RetryPolicy retryPolicyWithEmptyStatusCodes = RetryPolicy.create( + 4, ImmutableList.of(), Durations.fromMillis(100), Durations.fromMillis(200), null); // timeout only String expectedServiceConfigJson = "{\n" @@ -1001,6 +1003,11 @@ public void generateServiceConfig_forPerMethodConfig() throws IOException { assertThat(XdsNameResolver.generateServiceConfigWithMethodConfig(timeoutNano, null)) .isEqualTo(expectedServiceConfig); + // timeout and retry with empty retriable status codes + assertThat(XdsNameResolver.generateServiceConfigWithMethodConfig( + timeoutNano, retryPolicyWithEmptyStatusCodes)) + .isEqualTo(expectedServiceConfig); + // retry only expectedServiceConfigJson = "{\n" + " \"methodConfig\": [{\n" @@ -1021,6 +1028,7 @@ public void generateServiceConfig_forPerMethodConfig() throws IOException { assertThat(XdsNameResolver.generateServiceConfigWithMethodConfig(null, retryPolicy)) .isEqualTo(expectedServiceConfig); + // timeout and retry expectedServiceConfigJson = "{\n" + " \"methodConfig\": [{\n" @@ -1043,12 +1051,16 @@ public void generateServiceConfig_forPerMethodConfig() throws IOException { .isEqualTo(expectedServiceConfig); // no timeout and no retry - // timeout and retry expectedServiceConfigJson = "{}"; expectedServiceConfig = (Map) JsonParser.parse(expectedServiceConfigJson); assertThat(XdsNameResolver.generateServiceConfigWithMethodConfig(null, null)) .isEqualTo(expectedServiceConfig); + + // retry with emtry retriable status codes only + assertThat(XdsNameResolver.generateServiceConfigWithMethodConfig( + null, retryPolicyWithEmptyStatusCodes)) + .isEqualTo(expectedServiceConfig); } @Test From 7c6f53ab79846decde5e624bb6fa5f7a1f8a922c Mon Sep 17 00:00:00 2001 From: ZHANG Dapeng Date: Mon, 13 Sep 2021 09:12:04 -0700 Subject: [PATCH 12/76] all: add internal API to disable retry stats (#8510) Resolves b/197648853 for internal performance regression. Reporting retry stats caused significant amount of performance overhead internally. --- .../java/io/grpc/census/CensusStatsModule.java | 13 +++++++++---- .../census/InternalCensusStatsAccessor.java | 17 +++++++++++------ .../java/io/grpc/census/CensusModulesTest.java | 12 ++++++------ .../grpc/inprocess/InProcessChannelBuilder.java | 1 + .../internal/ManagedChannelImplBuilder.java | 9 ++++++++- .../integration/AbstractInteropTest.java | 3 ++- .../io/grpc/testing/integration/RetryTest.java | 2 +- .../grpc/netty/InternalNettyChannelBuilder.java | 4 ++++ .../java/io/grpc/netty/NettyChannelBuilder.java | 4 ++++ 9 files changed, 46 insertions(+), 19 deletions(-) diff --git a/census/src/main/java/io/grpc/census/CensusStatsModule.java b/census/src/main/java/io/grpc/census/CensusStatsModule.java index de860d0854c..4b2832c5989 100644 --- a/census/src/main/java/io/grpc/census/CensusStatsModule.java +++ b/census/src/main/java/io/grpc/census/CensusStatsModule.java @@ -88,19 +88,21 @@ final class CensusStatsModule { private final boolean recordStartedRpcs; private final boolean recordFinishedRpcs; private final boolean recordRealTimeMetrics; + private final boolean recordRetryMetrics; /** * Creates a {@link CensusStatsModule} with the default OpenCensus implementation. */ CensusStatsModule(Supplier stopwatchSupplier, boolean propagateTags, boolean recordStartedRpcs, boolean recordFinishedRpcs, - boolean recordRealTimeMetrics) { + boolean recordRealTimeMetrics, boolean recordRetryMetrics) { this( Tags.getTagger(), Tags.getTagPropagationComponent().getBinarySerializer(), Stats.getStatsRecorder(), stopwatchSupplier, - propagateTags, recordStartedRpcs, recordFinishedRpcs, recordRealTimeMetrics); + propagateTags, recordStartedRpcs, recordFinishedRpcs, recordRealTimeMetrics, + recordRetryMetrics); } /** @@ -111,7 +113,7 @@ final class CensusStatsModule { final TagContextBinarySerializer tagCtxSerializer, StatsRecorder statsRecorder, Supplier stopwatchSupplier, boolean propagateTags, boolean recordStartedRpcs, boolean recordFinishedRpcs, - boolean recordRealTimeMetrics) { + boolean recordRealTimeMetrics, boolean recordRetryMetrics) { this.tagger = checkNotNull(tagger, "tagger"); this.statsRecorder = checkNotNull(statsRecorder, "statsRecorder"); checkNotNull(tagCtxSerializer, "tagCtxSerializer"); @@ -120,6 +122,7 @@ final class CensusStatsModule { this.recordStartedRpcs = recordStartedRpcs; this.recordFinishedRpcs = recordFinishedRpcs; this.recordRealTimeMetrics = recordRealTimeMetrics; + this.recordRetryMetrics = recordRetryMetrics; this.statsHeader = Metadata.Key.of("grpc-tags-bin", new Metadata.BinaryMarshaller() { @Override @@ -521,7 +524,9 @@ void recordFinishedCall() { } else if (inboundMetricTracer != null) { inboundMetricTracer.recordFinishedAttempt(); } - + if (!module.recordRetryMetrics) { + return; + } long retriesPerCall = 0; long attempts = attemptsPerCall.get(); if (attempts > 0) { diff --git a/census/src/main/java/io/grpc/census/InternalCensusStatsAccessor.java b/census/src/main/java/io/grpc/census/InternalCensusStatsAccessor.java index 96be3258dff..3bbbb7dd0e0 100644 --- a/census/src/main/java/io/grpc/census/InternalCensusStatsAccessor.java +++ b/census/src/main/java/io/grpc/census/InternalCensusStatsAccessor.java @@ -49,14 +49,16 @@ private InternalCensusStatsAccessor() { public static ClientInterceptor getClientInterceptor( boolean recordStartedRpcs, boolean recordFinishedRpcs, - boolean recordRealTimeMetrics) { + boolean recordRealTimeMetrics, + boolean recordRetryMetrics) { CensusStatsModule censusStats = new CensusStatsModule( STOPWATCH_SUPPLIER, true, /* propagateTags */ recordStartedRpcs, recordFinishedRpcs, - recordRealTimeMetrics); + recordRealTimeMetrics, + recordRetryMetrics); return censusStats.getClientInterceptor(); } @@ -71,11 +73,13 @@ public static ClientInterceptor getClientInterceptor( boolean propagateTags, boolean recordStartedRpcs, boolean recordFinishedRpcs, - boolean recordRealTimeMetrics) { + boolean recordRealTimeMetrics, + boolean recordRetryMetrics) { CensusStatsModule censusStats = new CensusStatsModule( tagger, tagCtxSerializer, statsRecorder, stopwatchSupplier, - propagateTags, recordStartedRpcs, recordFinishedRpcs, recordRealTimeMetrics); + propagateTags, recordStartedRpcs, recordFinishedRpcs, recordRealTimeMetrics, + recordRetryMetrics); return censusStats.getClientInterceptor(); } @@ -92,7 +96,8 @@ public static ServerStreamTracer.Factory getServerStreamTracerFactory( true, /* propagateTags */ recordStartedRpcs, recordFinishedRpcs, - recordRealTimeMetrics); + recordRealTimeMetrics, + false); return censusStats.getServerTracerFactory(); } @@ -111,7 +116,7 @@ public static ServerStreamTracer.Factory getServerStreamTracerFactory( CensusStatsModule censusStats = new CensusStatsModule( tagger, tagCtxSerializer, statsRecorder, stopwatchSupplier, - propagateTags, recordStartedRpcs, recordFinishedRpcs, recordRealTimeMetrics); + propagateTags, recordStartedRpcs, recordFinishedRpcs, recordRealTimeMetrics, false); return censusStats.getServerTracerFactory(); } } diff --git a/census/src/test/java/io/grpc/census/CensusModulesTest.java b/census/src/test/java/io/grpc/census/CensusModulesTest.java index d285c8fe8c2..b710d1b4112 100644 --- a/census/src/test/java/io/grpc/census/CensusModulesTest.java +++ b/census/src/test/java/io/grpc/census/CensusModulesTest.java @@ -225,7 +225,7 @@ public void setUp() throws Exception { censusStats = new CensusStatsModule( tagger, tagCtxSerializer, statsRecorder, fakeClock.getStopwatchSupplier(), - true, true, true, false /* real-time */); + true, true, true, false /* real-time */, true); censusTracing = new CensusTracingModule(tracer, mockTracingPropagationHandler); } @@ -400,7 +400,7 @@ private void subtestClientBasicStatsDefaultContext( CensusStatsModule localCensusStats = new CensusStatsModule( tagger, tagCtxSerializer, statsRecorder, fakeClock.getStopwatchSupplier(), - true, recordStarts, recordFinishes, recordRealTime); + true, recordStarts, recordFinishes, recordRealTime, true); CensusStatsModule.CallAttemptsTracerFactory callAttemptsTracerFactory = new CensusStatsModule.CallAttemptsTracerFactory( localCensusStats, tagger.empty(), method.getFullMethodName()); @@ -514,7 +514,7 @@ public void recordRetryStats() { CensusStatsModule localCensusStats = new CensusStatsModule( tagger, tagCtxSerializer, statsRecorder, fakeClock.getStopwatchSupplier(), - true, true, true, true); + true, true, true, true, true); CensusStatsModule.CallAttemptsTracerFactory callAttemptsTracerFactory = new CensusStatsModule.CallAttemptsTracerFactory( localCensusStats, tagger.empty(), method.getFullMethodName()); @@ -908,7 +908,7 @@ private void subtestStatsHeadersPropagateTags(boolean propagate, boolean recordS tagCtxSerializer, statsRecorder, fakeClock.getStopwatchSupplier(), - propagate, recordStats, recordStats, recordStats); + propagate, recordStats, recordStats, recordStats, recordStats); Metadata headers = new Metadata(); CensusStatsModule.CallAttemptsTracerFactory callAttemptsTracerFactory = new CensusStatsModule.CallAttemptsTracerFactory( @@ -1168,7 +1168,7 @@ private void subtestServerBasicStatsNoHeaders( CensusStatsModule localCensusStats = new CensusStatsModule( tagger, tagCtxSerializer, statsRecorder, fakeClock.getStopwatchSupplier(), - true, recordStarts, recordFinishes, recordRealTime); + true, recordStarts, recordFinishes, recordRealTime, true); ServerStreamTracer.Factory tracerFactory = localCensusStats.getServerTracerFactory(); ServerStreamTracer tracer = tracerFactory.newServerStreamTracer(method.getFullMethodName(), new Metadata()); @@ -1429,7 +1429,7 @@ public void newTagsPopulateOldViews() throws InterruptedException { CensusStatsModule localCensusStats = new CensusStatsModule( tagger, tagCtxSerializer, localStats.getStatsRecorder(), fakeClock.getStopwatchSupplier(), - false, false, true, false /* real-time */); + false, false, true, false /* real-time */, true); CensusStatsModule.CallAttemptsTracerFactory callAttemptsTracerFactory = new CensusStatsModule.CallAttemptsTracerFactory( diff --git a/core/src/main/java/io/grpc/inprocess/InProcessChannelBuilder.java b/core/src/main/java/io/grpc/inprocess/InProcessChannelBuilder.java index 812ad5d7861..8a309408a94 100644 --- a/core/src/main/java/io/grpc/inprocess/InProcessChannelBuilder.java +++ b/core/src/main/java/io/grpc/inprocess/InProcessChannelBuilder.java @@ -97,6 +97,7 @@ public ClientTransportFactory buildClientTransportFactory() { // https://github.com/grpc/grpc-java/issues/2284 managedChannelImplBuilder.setStatsRecordStartedRpcs(false); managedChannelImplBuilder.setStatsRecordFinishedRpcs(false); + managedChannelImplBuilder.setStatsRecordRetryMetrics(false); } @Internal diff --git a/core/src/main/java/io/grpc/internal/ManagedChannelImplBuilder.java b/core/src/main/java/io/grpc/internal/ManagedChannelImplBuilder.java index 26c48fc8596..243a555ad1a 100644 --- a/core/src/main/java/io/grpc/internal/ManagedChannelImplBuilder.java +++ b/core/src/main/java/io/grpc/internal/ManagedChannelImplBuilder.java @@ -162,6 +162,7 @@ public static ManagedChannelBuilder forTarget(String target) { private boolean recordStartedRpcs = true; private boolean recordFinishedRpcs = true; private boolean recordRealTimeMetrics = false; + private boolean recordRetryMetrics = true; private boolean tracingEnabled = true; /** @@ -583,6 +584,10 @@ public void setStatsRecordFinishedRpcs(boolean value) { public void setStatsRecordRealTimeMetrics(boolean value) { recordRealTimeMetrics = value; } + + public void setStatsRecordRetryMetrics(boolean value) { + recordRetryMetrics = value; + } /** * Disable or enable tracing features. Enabled by default. @@ -643,6 +648,7 @@ List getEffectiveInterceptors() { "getClientInterceptor", boolean.class, boolean.class, + boolean.class, boolean.class); statsInterceptor = (ClientInterceptor) getClientInterceptorMethod @@ -650,7 +656,8 @@ List getEffectiveInterceptors() { null, recordStartedRpcs, recordFinishedRpcs, - recordRealTimeMetrics); + recordRealTimeMetrics, + recordRetryMetrics); } catch (ClassNotFoundException e) { // Replace these separate catch statements with multicatch when Android min-API >= 19 log.log(Level.FINE, "Unable to apply census stats", e); diff --git a/interop-testing/src/main/java/io/grpc/testing/integration/AbstractInteropTest.java b/interop-testing/src/main/java/io/grpc/testing/integration/AbstractInteropTest.java index 33d263e95de..698aa330ec6 100644 --- a/interop-testing/src/main/java/io/grpc/testing/integration/AbstractInteropTest.java +++ b/interop-testing/src/main/java/io/grpc/testing/integration/AbstractInteropTest.java @@ -389,7 +389,8 @@ protected final ClientInterceptor createCensusStatsClientInterceptor() { tagger, tagContextBinarySerializer, clientStatsRecorder, GrpcUtil.STOPWATCH_SUPPLIER, true, true, true, - /* recordRealTimeMetrics= */ false); + /* recordRealTimeMetrics= */ false, + /* recordRetryMetrics= */ true); } protected final ServerStreamTracer.Factory createCustomCensusTracerFactory() { diff --git a/interop-testing/src/test/java/io/grpc/testing/integration/RetryTest.java b/interop-testing/src/test/java/io/grpc/testing/integration/RetryTest.java index eb815501d5c..045d54ea44e 100644 --- a/interop-testing/src/test/java/io/grpc/testing/integration/RetryTest.java +++ b/interop-testing/src/test/java/io/grpc/testing/integration/RetryTest.java @@ -139,7 +139,7 @@ public void run() {} // no-op InternalCensusStatsAccessor.getClientInterceptor( tagger, tagContextBinarySerializer, clientStatsRecorder, fakeClock.getStopwatchSupplier(), true, true, true, - /* recordRealTimeMetrics= */ true); + /* recordRealTimeMetrics= */ true, /* recordRetryMetrics= */ true); private final MethodDescriptor clientStreamingMethod = MethodDescriptor.newBuilder() .setType(MethodType.CLIENT_STREAMING) diff --git a/netty/src/main/java/io/grpc/netty/InternalNettyChannelBuilder.java b/netty/src/main/java/io/grpc/netty/InternalNettyChannelBuilder.java index 363e0c8ef53..72cb211ecf3 100644 --- a/netty/src/main/java/io/grpc/netty/InternalNettyChannelBuilder.java +++ b/netty/src/main/java/io/grpc/netty/InternalNettyChannelBuilder.java @@ -88,6 +88,10 @@ public static void setStatsRecordRealTimeMetrics(NettyChannelBuilder builder, bo builder.setStatsRecordRealTimeMetrics(value); } + public static void setStatsRecordRetryMetrics(NettyChannelBuilder builder, boolean value) { + builder.setStatsRecordRetryMetrics(value); + } + /** * Sets {@link io.grpc.Channel} and {@link io.netty.channel.EventLoopGroup} to Nio. A major * benefit over using setters is gRPC will manage the life cycle of {@link diff --git a/netty/src/main/java/io/grpc/netty/NettyChannelBuilder.java b/netty/src/main/java/io/grpc/netty/NettyChannelBuilder.java index 25338c4100d..809c94f12ce 100644 --- a/netty/src/main/java/io/grpc/netty/NettyChannelBuilder.java +++ b/netty/src/main/java/io/grpc/netty/NettyChannelBuilder.java @@ -605,6 +605,10 @@ void setStatsRecordRealTimeMetrics(boolean value) { this.managedChannelImplBuilder.setStatsRecordRealTimeMetrics(value); } + void setStatsRecordRetryMetrics(boolean value) { + this.managedChannelImplBuilder.setStatsRecordRetryMetrics(value); + } + @VisibleForTesting NettyChannelBuilder setTransportTracerFactory(TransportTracer.Factory transportTracerFactory) { this.transportTracerFactory = transportTracerFactory; From 6e89919e3265eadd33dc7e154ba291d46563f286 Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Mon, 13 Sep 2021 11:30:19 -0700 Subject: [PATCH 13/76] netty: Requests with Connection header are malformed Although this is part of HTTP/2 and should have already been handled already, it was noticed as part of RBAC work to avoid matching hop-by-hop headers. See gRFC A41. Also add a warning if creating Metadata.Key for "Connection". Use this to try to help diagnose a client if it happens to blindly copy headers from HTTP/1, as PROTOCOL_ERROR is hard to debug. --- api/src/main/java/io/grpc/Metadata.java | 12 ++++++++++++ .../io/grpc/netty/GrpcHttp2HeadersUtils.java | 5 +++++ .../io/grpc/netty/NettyServerHandler.java | 7 +++++++ .../io/grpc/netty/NettyServerHandlerTest.java | 19 +++++++++++++++++++ 4 files changed, 43 insertions(+) diff --git a/api/src/main/java/io/grpc/Metadata.java b/api/src/main/java/io/grpc/Metadata.java index e153fd55691..9c2a2227f8c 100644 --- a/api/src/main/java/io/grpc/Metadata.java +++ b/api/src/main/java/io/grpc/Metadata.java @@ -40,6 +40,8 @@ import java.util.Map; import java.util.NoSuchElementException; import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; import javax.annotation.concurrent.NotThreadSafe; @@ -54,6 +56,7 @@ */ @NotThreadSafe public final class Metadata { + private static final Logger logger = Logger.getLogger(Metadata.class.getName()); /** * All binary headers should have this suffix in their names. Vice versa. @@ -733,6 +736,15 @@ private static BitSet generateValidTChars() { private static String validateName(String n, boolean pseudo) { checkNotNull(n, "name"); checkArgument(!n.isEmpty(), "token must have at least 1 tchar"); + if (n.equals("connection")) { + logger.log( + Level.WARNING, + "Metadata key is 'Connection', which should not be used. That is used by HTTP/1 for " + + "connection-specific headers which are not to be forwarded. There is probably an " + + "HTTP/1 conversion bug. Simply removing the Connection header is not enough; you " + + "should remove all headers it references as well. See RFC 7230 section 6.1", + new RuntimeException("exception to show backtrace")); + } for (int i = 0; i < n.length(); i++) { char tChar = n.charAt(i); if (pseudo && tChar == ':' && i == 0) { diff --git a/netty/src/main/java/io/grpc/netty/GrpcHttp2HeadersUtils.java b/netty/src/main/java/io/grpc/netty/GrpcHttp2HeadersUtils.java index 4bdef93ad04..04cf7f4195f 100644 --- a/netty/src/main/java/io/grpc/netty/GrpcHttp2HeadersUtils.java +++ b/netty/src/main/java/io/grpc/netty/GrpcHttp2HeadersUtils.java @@ -145,6 +145,11 @@ protected CharSequence get(AsciiString name) { return null; } + @Override + public boolean contains(CharSequence name) { + return get(name) != null; + } + @Override public CharSequence status() { return get(Http2Headers.PseudoHeaderName.STATUS.value()); diff --git a/netty/src/main/java/io/grpc/netty/NettyServerHandler.java b/netty/src/main/java/io/grpc/netty/NettyServerHandler.java index 6fca656e795..ee21129b67a 100644 --- a/netty/src/main/java/io/grpc/netty/NettyServerHandler.java +++ b/netty/src/main/java/io/grpc/netty/NettyServerHandler.java @@ -26,6 +26,7 @@ import static io.grpc.netty.Utils.HTTP_METHOD; import static io.grpc.netty.Utils.TE_HEADER; import static io.grpc.netty.Utils.TE_TRAILERS; +import static io.netty.handler.codec.http.HttpHeaderNames.CONNECTION; import static io.netty.handler.codec.http2.DefaultHttp2LocalFlowController.DEFAULT_WINDOW_UPDATE_RATIO; import com.google.common.annotations.VisibleForTesting; @@ -375,6 +376,12 @@ public void run() { private void onHeadersRead(ChannelHandlerContext ctx, int streamId, Http2Headers headers) throws Http2Exception { try { + // Connection-specific header fields makes a request malformed. Ideally this would be handled + // by Netty. RFC 7540 section 8.1.2.2 + if (headers.contains(CONNECTION)) { + resetStream(ctx, streamId, Http2Error.PROTOCOL_ERROR.code(), ctx.newPromise()); + return; + } // Remove the leading slash of the path and get the fully qualified method name CharSequence path = headers.path(); diff --git a/netty/src/test/java/io/grpc/netty/NettyServerHandlerTest.java b/netty/src/test/java/io/grpc/netty/NettyServerHandlerTest.java index 961f983d9cd..efae5c04989 100644 --- a/netty/src/test/java/io/grpc/netty/NettyServerHandlerTest.java +++ b/netty/src/test/java/io/grpc/netty/NettyServerHandlerTest.java @@ -537,6 +537,25 @@ public void headersSupportExtensionContentType() throws Exception { stream = streamCaptor.getValue(); } + @Test + public void headersWithConnectionHeaderShouldFail() throws Exception { + manualSetUp(); + Http2Headers headers = new DefaultHttp2Headers() + .method(HTTP_METHOD) + .set(CONTENT_TYPE_HEADER, CONTENT_TYPE_GRPC) + .set(AsciiString.of("connection"), CONTENT_TYPE_GRPC) + .path(new AsciiString("/foo/bar")); + ByteBuf headersFrame = headersFrame(STREAM_ID, headers); + channelRead(headersFrame); + + verifyWrite() + .writeRstStream( + eq(ctx()), + eq(STREAM_ID), + eq(Http2Error.PROTOCOL_ERROR.code()), + any(ChannelPromise.class)); + } + @Test public void keepAliveManagerOnDataReceived_headersRead() throws Exception { manualSetUp(); From 876f56e2ea32b9dfe13e96dc37111ef216f796ae Mon Sep 17 00:00:00 2001 From: Terry Wilson Date: Mon, 13 Sep 2021 14:54:25 -0700 Subject: [PATCH 14/76] api: Stabilize the Status.asException() call. (#8520) Removes the ExperimentalApi annotation from this call. Contributes to: #4683 --- api/src/main/java/io/grpc/Status.java | 1 - 1 file changed, 1 deletion(-) diff --git a/api/src/main/java/io/grpc/Status.java b/api/src/main/java/io/grpc/Status.java index 963a001d9cd..1ad5abc0539 100644 --- a/api/src/main/java/io/grpc/Status.java +++ b/api/src/main/java/io/grpc/Status.java @@ -546,7 +546,6 @@ public StatusException asException() { /** * Same as {@link #asException()} but includes the provided trailers in the returned exception. */ - @ExperimentalApi("https://github.com/grpc/grpc-java/issues/4683") public StatusException asException(@Nullable Metadata trailers) { return new StatusException(this, trailers); } From 3b237339c7b0b5e60b6331cac9e23650e97b1e8c Mon Sep 17 00:00:00 2001 From: ZHANG Dapeng Date: Mon, 13 Sep 2021 17:15:45 -0700 Subject: [PATCH 15/76] core: discard outbound content-length header (#8522) Since netty version v4.1.67, content-lenght header validation will be enforced. So once grpc upgrades netty to that version or above, RPCs with invalid content-length header will fail. Some libraries such as HTTP to gRPC adapters blindly copy all HTTP headers to gRPC metadata, but the content-length header is one of those that shouldn't be forwarded because gRPC uses different encoding. This mistake has already been in existence for a long time. Discard outbound content-length headers in gRPC, so that users who encounter invalid content-length issue when upgrading grpc-java version on server/client side would be able to workaround by upgrading grpc-java on client/server side as well without fixing the HTTP adapter. --- .../main/java/io/grpc/internal/ClientCallImpl.java | 2 ++ core/src/main/java/io/grpc/internal/GrpcUtil.java | 3 +++ .../main/java/io/grpc/internal/ServerCallImpl.java | 2 ++ .../java/io/grpc/internal/ClientCallImplTest.java | 9 +++++++++ .../java/io/grpc/internal/ServerCallImplTest.java | 11 +++++++++++ 5 files changed, 27 insertions(+) diff --git a/core/src/main/java/io/grpc/internal/ClientCallImpl.java b/core/src/main/java/io/grpc/internal/ClientCallImpl.java index dd17244e2a5..6f850ade667 100644 --- a/core/src/main/java/io/grpc/internal/ClientCallImpl.java +++ b/core/src/main/java/io/grpc/internal/ClientCallImpl.java @@ -24,6 +24,7 @@ import static io.grpc.Status.DEADLINE_EXCEEDED; import static io.grpc.internal.GrpcUtil.CONTENT_ACCEPT_ENCODING_KEY; import static io.grpc.internal.GrpcUtil.CONTENT_ENCODING_KEY; +import static io.grpc.internal.GrpcUtil.CONTENT_LENGTH_KEY; import static io.grpc.internal.GrpcUtil.MESSAGE_ACCEPT_ENCODING_KEY; import static io.grpc.internal.GrpcUtil.MESSAGE_ENCODING_KEY; import static java.lang.Math.max; @@ -163,6 +164,7 @@ static void prepareHeaders( DecompressorRegistry decompressorRegistry, Compressor compressor, boolean fullStreamDecompression) { + headers.discardAll(CONTENT_LENGTH_KEY); headers.discardAll(MESSAGE_ENCODING_KEY); if (compressor != Codec.Identity.NONE) { headers.put(MESSAGE_ENCODING_KEY, compressor.getMessageEncoding()); diff --git a/core/src/main/java/io/grpc/internal/GrpcUtil.java b/core/src/main/java/io/grpc/internal/GrpcUtil.java index 98e1d00585e..55e2cd81530 100644 --- a/core/src/main/java/io/grpc/internal/GrpcUtil.java +++ b/core/src/main/java/io/grpc/internal/GrpcUtil.java @@ -109,6 +109,9 @@ public final class GrpcUtil { public static final Metadata.Key CONTENT_ACCEPT_ENCODING_KEY = InternalMetadata.keyOf(GrpcUtil.CONTENT_ACCEPT_ENCODING, new AcceptEncodingMarshaller()); + static final Metadata.Key CONTENT_LENGTH_KEY = + Metadata.Key.of("content-length", Metadata.ASCII_STRING_MARSHALLER); + private static final class AcceptEncodingMarshaller implements TrustedAsciiMarshaller { @Override public byte[] toAsciiString(byte[] value) { diff --git a/core/src/main/java/io/grpc/internal/ServerCallImpl.java b/core/src/main/java/io/grpc/internal/ServerCallImpl.java index f82d87cade0..deba21c315d 100644 --- a/core/src/main/java/io/grpc/internal/ServerCallImpl.java +++ b/core/src/main/java/io/grpc/internal/ServerCallImpl.java @@ -20,6 +20,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; import static io.grpc.internal.GrpcUtil.ACCEPT_ENCODING_SPLITTER; +import static io.grpc.internal.GrpcUtil.CONTENT_LENGTH_KEY; import static io.grpc.internal.GrpcUtil.MESSAGE_ACCEPT_ENCODING_KEY; import static io.grpc.internal.GrpcUtil.MESSAGE_ENCODING_KEY; @@ -107,6 +108,7 @@ private void sendHeadersInternal(Metadata headers) { checkState(!sendHeadersCalled, "sendHeaders has already been called"); checkState(!closeCalled, "call is closed"); + headers.discardAll(CONTENT_LENGTH_KEY); headers.discardAll(MESSAGE_ENCODING_KEY); if (compressor == null) { compressor = Codec.Identity.NONE; diff --git a/core/src/test/java/io/grpc/internal/ClientCallImplTest.java b/core/src/test/java/io/grpc/internal/ClientCallImplTest.java index 0e5e5f50599..ecf7a90b13e 100644 --- a/core/src/test/java/io/grpc/internal/ClientCallImplTest.java +++ b/core/src/test/java/io/grpc/internal/ClientCallImplTest.java @@ -469,6 +469,15 @@ public void prepareHeaders_ignoreIdentityEncoding() { assertNull(m.get(GrpcUtil.MESSAGE_ENCODING_KEY)); } + @Test + public void prepareHeaders_ignoreContentLength() { + Metadata m = new Metadata(); + m.put(GrpcUtil.CONTENT_LENGTH_KEY, "123"); + ClientCallImpl.prepareHeaders(m, decompressorRegistry, Codec.Identity.NONE, false); + + assertNull(m.get(GrpcUtil.CONTENT_LENGTH_KEY)); + } + @Test public void prepareHeaders_acceptedMessageEncodingsAdded() { Metadata m = new Metadata(); diff --git a/core/src/test/java/io/grpc/internal/ServerCallImplTest.java b/core/src/test/java/io/grpc/internal/ServerCallImplTest.java index ea49b94e8aa..edf303a0bcd 100644 --- a/core/src/test/java/io/grpc/internal/ServerCallImplTest.java +++ b/core/src/test/java/io/grpc/internal/ServerCallImplTest.java @@ -17,6 +17,7 @@ package io.grpc.internal; import static com.google.common.base.Charsets.UTF_8; +import static io.grpc.internal.GrpcUtil.CONTENT_LENGTH_KEY; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; @@ -151,6 +152,16 @@ public void sendHeader_firstCall() { verify(stream).writeHeaders(headers); } + @Test + public void sendHeader_contentLengthDiscarded() { + Metadata headers = new Metadata(); + headers.put(CONTENT_LENGTH_KEY, "123"); + call.sendHeaders(headers); + + verify(stream).writeHeaders(headers); + assertNull(headers.get(CONTENT_LENGTH_KEY)); + } + @Test public void sendHeader_failsOnSecondCall() { call.sendHeaders(new Metadata()); From 122b3b2f7cf2b50fe0a0cebc55a84133441a4348 Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Wed, 15 Sep 2021 09:40:56 -0700 Subject: [PATCH 16/76] netty: Support Host header on server-side We want to know the single, unambiguous authority for the request. If there is no authority, we use host instead. While authority would be most typical for HTTP/2, requests proxied from HTTP/1 may use host instead of authority. This is generally useful, but the impetus is RBAC. See gRFC A41. --- .../io/grpc/netty/GrpcHttp2HeadersUtils.java | 16 +++++ .../io/grpc/netty/NettyServerHandler.java | 16 +++++ .../io/grpc/netty/NettyServerHandlerTest.java | 71 +++++++++++++++++++ 3 files changed, 103 insertions(+) diff --git a/netty/src/main/java/io/grpc/netty/GrpcHttp2HeadersUtils.java b/netty/src/main/java/io/grpc/netty/GrpcHttp2HeadersUtils.java index 04cf7f4195f..df7875fc7ae 100644 --- a/netty/src/main/java/io/grpc/netty/GrpcHttp2HeadersUtils.java +++ b/netty/src/main/java/io/grpc/netty/GrpcHttp2HeadersUtils.java @@ -365,12 +365,28 @@ private void addPseudoHeader(CharSequence csName, CharSequence csValue) { AsciiString value = requireAsciiString(csValue); if (equals(PATH_HEADER, name)) { + if (path != null) { + PlatformDependent.throwException( + connectionError(PROTOCOL_ERROR, "Duplicate :path header")); + } path = value; } else if (equals(AUTHORITY_HEADER, name)) { + if (authority != null) { + PlatformDependent.throwException( + connectionError(PROTOCOL_ERROR, "Duplicate :authority header")); + } authority = value; } else if (equals(METHOD_HEADER, name)) { + if (method != null) { + PlatformDependent.throwException( + connectionError(PROTOCOL_ERROR, "Duplicate :method header")); + } method = value; } else if (equals(SCHEME_HEADER, name)) { + if (scheme != null) { + PlatformDependent.throwException( + connectionError(PROTOCOL_ERROR, "Duplicate :scheme header")); + } scheme = value; } else { PlatformDependent.throwException( diff --git a/netty/src/main/java/io/grpc/netty/NettyServerHandler.java b/netty/src/main/java/io/grpc/netty/NettyServerHandler.java index ee21129b67a..91f8f556f81 100644 --- a/netty/src/main/java/io/grpc/netty/NettyServerHandler.java +++ b/netty/src/main/java/io/grpc/netty/NettyServerHandler.java @@ -27,7 +27,9 @@ import static io.grpc.netty.Utils.TE_HEADER; import static io.grpc.netty.Utils.TE_TRAILERS; import static io.netty.handler.codec.http.HttpHeaderNames.CONNECTION; +import static io.netty.handler.codec.http.HttpHeaderNames.HOST; import static io.netty.handler.codec.http2.DefaultHttp2LocalFlowController.DEFAULT_WINDOW_UPDATE_RATIO; +import static io.netty.handler.codec.http2.Http2Headers.PseudoHeaderName.AUTHORITY; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -383,6 +385,20 @@ private void onHeadersRead(ChannelHandlerContext ctx, int streamId, Http2Headers return; } + if (headers.authority() == null) { + List hosts = headers.getAll(HOST); + if (hosts.size() > 1) { + // RFC 7230 section 5.4 + respondWithHttpError(ctx, streamId, 400, Status.Code.INTERNAL, + "Multiple host headers"); + return; + } + if (!hosts.isEmpty()) { + headers.add(AUTHORITY.value(), hosts.get(0)); + } + } + headers.remove(HOST); + // Remove the leading slash of the path and get the fully qualified method name CharSequence path = headers.path(); diff --git a/netty/src/test/java/io/grpc/netty/NettyServerHandlerTest.java b/netty/src/test/java/io/grpc/netty/NettyServerHandlerTest.java index efae5c04989..2f01ed99282 100644 --- a/netty/src/test/java/io/grpc/netty/NettyServerHandlerTest.java +++ b/netty/src/test/java/io/grpc/netty/NettyServerHandlerTest.java @@ -556,6 +556,77 @@ public void headersWithConnectionHeaderShouldFail() throws Exception { any(ChannelPromise.class)); } + @Test + public void headersWithMultipleHostsShouldFail() throws Exception { + manualSetUp(); + Http2Headers headers = new DefaultHttp2Headers() + .method(HTTP_METHOD) + .set(CONTENT_TYPE_HEADER, CONTENT_TYPE_GRPC) + .add(AsciiString.of("host"), AsciiString.of("example.com")) + .add(AsciiString.of("host"), AsciiString.of("bad.com")) + .path(new AsciiString("/foo/bar")); + ByteBuf headersFrame = headersFrame(STREAM_ID, headers); + channelRead(headersFrame); + Http2Headers responseHeaders = new DefaultHttp2Headers() + .set(InternalStatus.CODE_KEY.name(), String.valueOf(Code.INTERNAL.value())) + .set(InternalStatus.MESSAGE_KEY.name(), "Multiple host headers") + .status("" + 400) + .set(CONTENT_TYPE_HEADER, "text/plain; charset=utf-8"); + + verifyWrite() + .writeHeaders( + eq(ctx()), + eq(STREAM_ID), + eq(responseHeaders), + eq(0), + eq(false), + any(ChannelPromise.class)); + } + + @Test + public void headersWithAuthorityAndHostUsesAuthority() throws Exception { + manualSetUp(); + Http2Headers headers = new DefaultHttp2Headers() + .method(HTTP_METHOD) + .authority("example.com") + .set(CONTENT_TYPE_HEADER, CONTENT_TYPE_GRPC) + .add(AsciiString.of("host"), AsciiString.of("bad.com")) + .path(new AsciiString("/foo/bar")); + ByteBuf headersFrame = headersFrame(STREAM_ID, headers); + channelRead(headersFrame); + Metadata.Key hostKey = Metadata.Key.of("host", Metadata.ASCII_STRING_MARSHALLER); + + ArgumentCaptor streamCaptor = + ArgumentCaptor.forClass(NettyServerStream.class); + ArgumentCaptor metadataCaptor = ArgumentCaptor.forClass(Metadata.class); + verify(transportListener).streamCreated(streamCaptor.capture(), eq("foo/bar"), + metadataCaptor.capture()); + Truth.assertThat(streamCaptor.getValue().getAuthority()).isEqualTo("example.com"); + Truth.assertThat(metadataCaptor.getValue().get(hostKey)).isNull(); + } + + @Test + public void headersWithOnlyHostBecomesAuthority() throws Exception { + manualSetUp(); + // No authority header + Http2Headers headers = new DefaultHttp2Headers() + .method(HTTP_METHOD) + .set(CONTENT_TYPE_HEADER, CONTENT_TYPE_GRPC) + .add(AsciiString.of("host"), AsciiString.of("example.com")) + .path(new AsciiString("/foo/bar")); + ByteBuf headersFrame = headersFrame(STREAM_ID, headers); + channelRead(headersFrame); + Metadata.Key hostKey = Metadata.Key.of("host", Metadata.ASCII_STRING_MARSHALLER); + + ArgumentCaptor streamCaptor = + ArgumentCaptor.forClass(NettyServerStream.class); + ArgumentCaptor metadataCaptor = ArgumentCaptor.forClass(Metadata.class); + verify(transportListener).streamCreated(streamCaptor.capture(), eq("foo/bar"), + metadataCaptor.capture()); + Truth.assertThat(streamCaptor.getValue().getAuthority()).isEqualTo("example.com"); + Truth.assertThat(metadataCaptor.getValue().get(hostKey)).isNull(); + } + @Test public void keepAliveManagerOnDataReceived_headersRead() throws Exception { manualSetUp(); From 5307b69c9e5701bfe8ca5c304f3c44b8a44b50a8 Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Tue, 14 Sep 2021 10:49:48 -0700 Subject: [PATCH 17/76] netty: Allow protocol negotiators to shut down transport, with grace period This will be used for draining old connections when xDS configuration changes. --- .../netty/GracefulServerCloseCommand.java | 53 +++++++++++++++++++ .../InternalGracefulServerCloseCommand.java | 36 +++++++++++++ .../io/grpc/netty/NettyServerHandler.java | 24 +++++++-- .../WriteBufferingAndExceptionHandler.java | 2 + .../io/grpc/netty/NettyServerHandlerTest.java | 50 +++++++++++++++++ 5 files changed, 160 insertions(+), 5 deletions(-) create mode 100644 netty/src/main/java/io/grpc/netty/GracefulServerCloseCommand.java create mode 100644 netty/src/main/java/io/grpc/netty/InternalGracefulServerCloseCommand.java diff --git a/netty/src/main/java/io/grpc/netty/GracefulServerCloseCommand.java b/netty/src/main/java/io/grpc/netty/GracefulServerCloseCommand.java new file mode 100644 index 00000000000..97904687548 --- /dev/null +++ b/netty/src/main/java/io/grpc/netty/GracefulServerCloseCommand.java @@ -0,0 +1,53 @@ +/* + * Copyright 2021 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.netty; + +import com.google.common.base.Preconditions; +import java.util.concurrent.TimeUnit; + +/** + * A command to trigger close and allow streams naturally close. + */ +class GracefulServerCloseCommand extends WriteQueue.AbstractQueuedCommand { + private final String goAwayDebugString; + private final long graceTime; + private final TimeUnit graceTimeUnit; + + public GracefulServerCloseCommand(String goAwayDebugString) { + this(goAwayDebugString, -1, null); + } + + public GracefulServerCloseCommand( + String goAwayDebugString, long graceTime, TimeUnit graceTimeUnit) { + this.goAwayDebugString = Preconditions.checkNotNull(goAwayDebugString, "goAwayDebugString"); + this.graceTime = graceTime; + this.graceTimeUnit = graceTimeUnit; + } + + public String getGoAwayDebugString() { + return goAwayDebugString; + } + + /** Has no meaning if {@code getGraceTimeUnit() == null}. */ + public long getGraceTime() { + return graceTime; + } + + public TimeUnit getGraceTimeUnit() { + return graceTimeUnit; + } +} diff --git a/netty/src/main/java/io/grpc/netty/InternalGracefulServerCloseCommand.java b/netty/src/main/java/io/grpc/netty/InternalGracefulServerCloseCommand.java new file mode 100644 index 00000000000..deb72373ac7 --- /dev/null +++ b/netty/src/main/java/io/grpc/netty/InternalGracefulServerCloseCommand.java @@ -0,0 +1,36 @@ +/* + * Copyright 2021 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.netty; + +import io.grpc.Internal; +import java.util.concurrent.TimeUnit; + +/** + * Internal accessor for {@link GracefulServerCloseCommand}. + */ +@Internal +public final class InternalGracefulServerCloseCommand { + private InternalGracefulServerCloseCommand() {} + + public static Object create(String goAwayDebugString) { + return new GracefulServerCloseCommand(goAwayDebugString); + } + + public static Object create(String goAwayDebugString, long graceTime, TimeUnit graceTimeUnit) { + return new GracefulServerCloseCommand(goAwayDebugString, graceTime, graceTimeUnit); + } +} diff --git a/netty/src/main/java/io/grpc/netty/NettyServerHandler.java b/netty/src/main/java/io/grpc/netty/NettyServerHandler.java index 91f8f556f81..c286c17f640 100644 --- a/netty/src/main/java/io/grpc/netty/NettyServerHandler.java +++ b/netty/src/main/java/io/grpc/netty/NettyServerHandler.java @@ -641,6 +641,8 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) sendResponseHeaders(ctx, (SendResponseHeadersCommand) msg, promise); } else if (msg instanceof CancelServerStreamCommand) { cancelStream(ctx, (CancelServerStreamCommand) msg, promise); + } else if (msg instanceof GracefulServerCloseCommand) { + gracefulClose(ctx, (GracefulServerCloseCommand) msg, promise); } else if (msg instanceof ForcefulCloseCommand) { forcefulClose(ctx, (ForcefulCloseCommand) msg, promise); } else { @@ -654,11 +656,8 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) @Override public void close(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception { - if (gracefulShutdown == null) { - gracefulShutdown = new GracefulShutdown("app_requested", null); - gracefulShutdown.start(ctx); - ctx.flush(); - } + gracefulClose(ctx, new GracefulServerCloseCommand("app_requested"), promise); + ctx.flush(); } /** @@ -739,6 +738,21 @@ private void cancelStream(ChannelHandlerContext ctx, CancelServerStreamCommand c } } + private void gracefulClose(final ChannelHandlerContext ctx, final GracefulServerCloseCommand msg, + ChannelPromise promise) throws Exception { + // Ideally we'd adjust a pre-existing graceful shutdown's grace period to at least what is + // requested here. But that's an edge case and seems bug-prone. + if (gracefulShutdown == null) { + Long graceTimeInNanos = null; + if (msg.getGraceTimeUnit() != null) { + graceTimeInNanos = msg.getGraceTimeUnit().toNanos(msg.getGraceTime()); + } + gracefulShutdown = new GracefulShutdown(msg.getGoAwayDebugString(), graceTimeInNanos); + gracefulShutdown.start(ctx); + } + promise.setSuccess(); + } + private void forcefulClose(final ChannelHandlerContext ctx, final ForcefulCloseCommand msg, ChannelPromise promise) throws Exception { super.close(ctx, promise); diff --git a/netty/src/main/java/io/grpc/netty/WriteBufferingAndExceptionHandler.java b/netty/src/main/java/io/grpc/netty/WriteBufferingAndExceptionHandler.java index 9521fc93889..100367625fa 100644 --- a/netty/src/main/java/io/grpc/netty/WriteBufferingAndExceptionHandler.java +++ b/netty/src/main/java/io/grpc/netty/WriteBufferingAndExceptionHandler.java @@ -124,6 +124,8 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) promise.setFailure(failCause); ReferenceCountUtil.release(msg); } else { + // Do not special case GracefulServerCloseCommand, as we don't want to cause handshake + // failures. if (msg instanceof GracefulCloseCommand || msg instanceof ForcefulCloseCommand) { // No point in continuing negotiation ctx.close(); diff --git a/netty/src/test/java/io/grpc/netty/NettyServerHandlerTest.java b/netty/src/test/java/io/grpc/netty/NettyServerHandlerTest.java index 2f01ed99282..170273e2c60 100644 --- a/netty/src/test/java/io/grpc/netty/NettyServerHandlerTest.java +++ b/netty/src/test/java/io/grpc/netty/NettyServerHandlerTest.java @@ -350,6 +350,56 @@ public void closeShouldGracefullyCloseChannel() throws Exception { assertFalse(channel().isOpen()); } + @Test + public void gracefulCloseShouldGracefullyCloseChannel() throws Exception { + manualSetUp(); + handler() + .write(ctx(), new GracefulServerCloseCommand("test", 1, TimeUnit.MINUTES), newPromise()); + + verifyWrite().writeGoAway(eq(ctx()), eq(Integer.MAX_VALUE), eq(Http2Error.NO_ERROR.code()), + isA(ByteBuf.class), any(ChannelPromise.class)); + verifyWrite().writePing( + eq(ctx()), + eq(false), + eq(NettyServerHandler.GRACEFUL_SHUTDOWN_PING), + isA(ChannelPromise.class)); + channelRead(pingFrame(/*ack=*/ true , NettyServerHandler.GRACEFUL_SHUTDOWN_PING)); + + verifyWrite().writeGoAway(eq(ctx()), eq(0), eq(Http2Error.NO_ERROR.code()), + isA(ByteBuf.class), any(ChannelPromise.class)); + + // Verify that the channel was closed. + assertFalse(channel().isOpen()); + } + + @Test + public void secondGracefulCloseIsSafe() throws Exception { + manualSetUp(); + handler().write(ctx(), new GracefulServerCloseCommand("test"), newPromise()); + + verifyWrite().writeGoAway(eq(ctx()), eq(Integer.MAX_VALUE), eq(Http2Error.NO_ERROR.code()), + isA(ByteBuf.class), any(ChannelPromise.class)); + verifyWrite().writePing( + eq(ctx()), + eq(false), + eq(NettyServerHandler.GRACEFUL_SHUTDOWN_PING), + isA(ChannelPromise.class)); + + handler().write(ctx(), new GracefulServerCloseCommand("test2"), newPromise()); + + channel().runPendingTasks(); + // No additional GOAWAYs. + verifyWrite().writeGoAway(any(ChannelHandlerContext.class), any(Integer.class), any(Long.class), + any(ByteBuf.class), any(ChannelPromise.class)); + channel().checkException(); + assertTrue(channel().isOpen()); + + channelRead(pingFrame(/*ack=*/ true , NettyServerHandler.GRACEFUL_SHUTDOWN_PING)); + verifyWrite().writeGoAway(eq(ctx()), eq(0), eq(Http2Error.NO_ERROR.code()), + isA(ByteBuf.class), any(ChannelPromise.class)); + assertFalse(channel().isOpen()); + } + @Test public void exceptionCaughtShouldCloseConnection() throws Exception { manualSetUp(); From 43b507160fcd4c8582136b42ad9fa60a17be2c05 Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Tue, 14 Sep 2021 16:40:14 -0700 Subject: [PATCH 18/76] xds: Drain old server connections on Listener updates This is necessary to make sure all connections are using the new configuration. --- ...ilterChainMatchingProtocolNegotiators.java | 75 +++++++-- .../grpc/xds/FilterChainSelectorManager.java | 95 +++++++++++ .../io/grpc/xds/InternalXdsAttributes.java | 13 +- .../java/io/grpc/xds/XdsServerBuilder.java | 44 ++++- .../java/io/grpc/xds/XdsServerWrapper.java | 25 +-- ...rChainMatchingProtocolNegotiatorsTest.java | 153 +++++++++++------- .../xds/FilterChainSelectorManagerTest.java | 107 ++++++++++++ .../XdsClientWrapperForServerSdsTestMisc.java | 33 ++-- .../io/grpc/xds/XdsServerBuilderTest.java | 11 ++ .../io/grpc/xds/XdsServerWrapperTest.java | 96 ++++++----- 10 files changed, 508 insertions(+), 144 deletions(-) create mode 100644 xds/src/main/java/io/grpc/xds/FilterChainSelectorManager.java create mode 100644 xds/src/test/java/io/grpc/xds/FilterChainSelectorManagerTest.java diff --git a/xds/src/main/java/io/grpc/xds/FilterChainMatchingProtocolNegotiators.java b/xds/src/main/java/io/grpc/xds/FilterChainMatchingProtocolNegotiators.java index 0c8780fe744..24cd4e9ae7e 100644 --- a/xds/src/main/java/io/grpc/xds/FilterChainMatchingProtocolNegotiators.java +++ b/xds/src/main/java/io/grpc/xds/FilterChainMatchingProtocolNegotiators.java @@ -17,7 +17,8 @@ package io.grpc.xds; import static com.google.common.base.Preconditions.checkNotNull; -import static io.grpc.xds.InternalXdsAttributes.ATTR_FILTER_CHAIN_SELECTOR_REF; +import static io.grpc.xds.InternalXdsAttributes.ATTR_DRAIN_GRACE_NANOS; +import static io.grpc.xds.InternalXdsAttributes.ATTR_FILTER_CHAIN_SELECTOR_MANAGER; import static io.grpc.xds.XdsServerWrapper.ATTR_SERVER_ROUTING_CONFIG; import static io.grpc.xds.internal.sds.SdsProtocolNegotiators.ATTR_SERVER_SSL_CONTEXT_PROVIDER_SUPPLIER; @@ -28,6 +29,7 @@ import io.grpc.Attributes; import io.grpc.internal.ObjectPool; import io.grpc.netty.GrpcHttp2ConnectionHandler; +import io.grpc.netty.InternalGracefulServerCloseCommand; import io.grpc.netty.InternalProtocolNegotiationEvent; import io.grpc.netty.InternalProtocolNegotiator; import io.grpc.netty.InternalProtocolNegotiator.ProtocolNegotiator; @@ -40,6 +42,8 @@ import io.grpc.xds.XdsServerWrapper.ServerRoutingConfig; import io.grpc.xds.internal.Matchers.CidrMatcher; import io.grpc.xds.internal.sds.SslContextProviderSupplier; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelFutureListener; import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; @@ -54,7 +58,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.Executor; -import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; import javax.annotation.Nullable; @@ -77,14 +81,16 @@ private FilterChainMatchingProtocolNegotiators() { static final class FilterChainMatchingHandler extends ChannelInboundHandlerAdapter { private final GrpcHttp2ConnectionHandler grpcHandler; - private final FilterChainSelector selector; + private final FilterChainSelectorManager filterChainSelectorManager; private final ProtocolNegotiator delegate; FilterChainMatchingHandler( - GrpcHttp2ConnectionHandler grpcHandler, FilterChainSelector selector, + GrpcHttp2ConnectionHandler grpcHandler, + FilterChainSelectorManager filterChainSelectorManager, ProtocolNegotiator delegate) { this.grpcHandler = checkNotNull(grpcHandler, "grpcHandler"); - this.selector = checkNotNull(selector, "selector"); + this.filterChainSelectorManager = + checkNotNull(filterChainSelectorManager, "filterChainSelectorManager"); this.delegate = checkNotNull(delegate, "delegate"); } @@ -94,6 +100,19 @@ public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exc super.userEventTriggered(ctx, evt); return; } + long drainGraceTime = 0; + TimeUnit drainGraceTimeUnit = null; + Long drainGraceNanosObj = grpcHandler.getEagAttributes().get(ATTR_DRAIN_GRACE_NANOS); + if (drainGraceNanosObj != null) { + drainGraceTime = drainGraceNanosObj; + drainGraceTimeUnit = TimeUnit.NANOSECONDS; + } + FilterChainSelectorManager.Closer closer = new FilterChainSelectorManager.Closer( + new GracefullyShutdownChannelRunnable(ctx.channel(), drainGraceTime, drainGraceTimeUnit)); + FilterChainSelector selector = filterChainSelectorManager.register(closer); + ctx.channel().closeFuture().addListener( + new FilterChainSelectorManagerDeregister(filterChainSelectorManager, closer)); + checkNotNull(selector, "selector"); SelectedConfig config = selector.select( (InetSocketAddress) ctx.channel().localAddress(), (InetSocketAddress) ctx.channel().remoteAddress()); @@ -354,10 +373,10 @@ public AsciiString scheme() { @Override public ChannelHandler newHandler(GrpcHttp2ConnectionHandler grpcHandler) { - AtomicReference filterChainSelectorRef = - grpcHandler.getEagAttributes().get(ATTR_FILTER_CHAIN_SELECTOR_REF); - checkNotNull(filterChainSelectorRef, "filterChainSelectorRef"); - return new FilterChainMatchingHandler(grpcHandler, filterChainSelectorRef.get(), + FilterChainSelectorManager filterChainSelectorManager = + grpcHandler.getEagAttributes().get(ATTR_FILTER_CHAIN_SELECTOR_MANAGER); + checkNotNull(filterChainSelectorManager, "filterChainSelectorManager"); + return new FilterChainMatchingHandler(grpcHandler, filterChainSelectorManager, delegate.newNegotiator(offloadExecutorPool)); } @@ -384,4 +403,42 @@ private SelectedConfig(ServerRoutingConfig routingConfig, this.sslContextProviderSupplier = sslContextProviderSupplier; } } + + private static class FilterChainSelectorManagerDeregister implements ChannelFutureListener { + private final FilterChainSelectorManager filterChainSelectorManager; + private final FilterChainSelectorManager.Closer closer; + + public FilterChainSelectorManagerDeregister( + FilterChainSelectorManager filterChainSelectorManager, + FilterChainSelectorManager.Closer closer) { + this.filterChainSelectorManager = + checkNotNull(filterChainSelectorManager, "filterChainSelectorManager"); + this.closer = checkNotNull(closer, "closer"); + } + + @Override public void operationComplete(ChannelFuture future) throws Exception { + filterChainSelectorManager.deregister(closer); + } + } + + private static class GracefullyShutdownChannelRunnable implements Runnable { + private final Channel channel; + private final long drainGraceTime; + @Nullable + private final TimeUnit drainGraceTimeUnit; + + public GracefullyShutdownChannelRunnable( + Channel channel, long drainGraceTime, @Nullable TimeUnit drainGraceTimeUnit) { + this.channel = checkNotNull(channel, "channel"); + this.drainGraceTime = drainGraceTime; + this.drainGraceTimeUnit = drainGraceTimeUnit; + } + + @Override public void run() { + Object gracefulCloseCommand = InternalGracefulServerCloseCommand.create( + "xds_drain", drainGraceTime, drainGraceTimeUnit); + channel.writeAndFlush(gracefulCloseCommand) + .addListener(ChannelFutureListener.CLOSE_ON_FAILURE); + } + } } diff --git a/xds/src/main/java/io/grpc/xds/FilterChainSelectorManager.java b/xds/src/main/java/io/grpc/xds/FilterChainSelectorManager.java new file mode 100644 index 00000000000..4295d75f59b --- /dev/null +++ b/xds/src/main/java/io/grpc/xds/FilterChainSelectorManager.java @@ -0,0 +1,95 @@ +/* + * Copyright 2021 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.xds; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import io.grpc.xds.FilterChainMatchingProtocolNegotiators.FilterChainMatchingHandler.FilterChainSelector; +import java.util.Comparator; +import java.util.TreeSet; +import java.util.concurrent.atomic.AtomicLong; +import javax.annotation.concurrent.GuardedBy; + +/** + * Maintains the current xDS selector and any resources using that selector. When the selector + * changes, old resources are closed to avoid old config usages. + */ +final class FilterChainSelectorManager { + private static final AtomicLong closerId = new AtomicLong(); + + private final Object lock = new Object(); + @GuardedBy("lock") + private FilterChainSelector selector; + // Avoid HashSet since it does not decrease in size, forming a high water mark. + @GuardedBy("lock") + private TreeSet closers = new TreeSet(new CloserComparator()); + + public FilterChainSelector register(Closer closer) { + synchronized (lock) { + Preconditions.checkState(closers.add(closer), "closer already registered"); + return selector; + } + } + + public void deregister(Closer closer) { + synchronized (lock) { + closers.remove(closer); + } + } + + /** Only safe to be called by code that is responsible for updating the selector. */ + public FilterChainSelector getSelectorToUpdateSelector() { + synchronized (lock) { + return selector; + } + } + + public void updateSelector(FilterChainSelector newSelector) { + TreeSet oldClosers; + synchronized (lock) { + oldClosers = closers; + closers = new TreeSet(closers.comparator()); + selector = newSelector; + } + for (Closer closer : oldClosers) { + closer.closer.run(); + } + } + + @VisibleForTesting + int getRegisterCount() { + synchronized (lock) { + return closers.size(); + } + } + + public static final class Closer { + private final long id = closerId.getAndIncrement(); + private final Runnable closer; + + /** {@code closer} may be run multiple times. */ + public Closer(Runnable closer) { + this.closer = Preconditions.checkNotNull(closer, "closer"); + } + } + + private static class CloserComparator implements Comparator { + @Override public int compare(Closer c1, Closer c2) { + return Long.compare(c1.id, c2.id); + } + } +} diff --git a/xds/src/main/java/io/grpc/xds/InternalXdsAttributes.java b/xds/src/main/java/io/grpc/xds/InternalXdsAttributes.java index 82eddd355af..410a64df9ca 100644 --- a/xds/src/main/java/io/grpc/xds/InternalXdsAttributes.java +++ b/xds/src/main/java/io/grpc/xds/InternalXdsAttributes.java @@ -22,10 +22,8 @@ import io.grpc.Internal; import io.grpc.NameResolver; import io.grpc.internal.ObjectPool; -import io.grpc.xds.FilterChainMatchingProtocolNegotiators.FilterChainMatchingHandler.FilterChainSelector; import io.grpc.xds.XdsNameResolverProvider.CallCounterProvider; import io.grpc.xds.internal.sds.SslContextProviderSupplier; -import java.util.concurrent.atomic.AtomicReference; /** * Internal attributes used for xDS implementation. Do not use. @@ -81,9 +79,14 @@ public final class InternalXdsAttributes { * Filter chain match for network filters. */ @Grpc.TransportAttr - static final Attributes.Key> - ATTR_FILTER_CHAIN_SELECTOR_REF = Attributes.Key.create( - "io.grpc.xds.InternalXdsAttributes.filterChainSelectorRef"); + static final Attributes.Key + ATTR_FILTER_CHAIN_SELECTOR_MANAGER = Attributes.Key.create( + "io.grpc.xds.InternalXdsAttributes.filterChainSelectorManager"); + + /** Grace time to use when draining. Null for an infinite grace time. */ + @Grpc.TransportAttr + static final Attributes.Key ATTR_DRAIN_GRACE_NANOS = + Attributes.Key.create("io.grpc.xds.InternalXdsAttributes.drainGraceTime"); private InternalXdsAttributes() {} } diff --git a/xds/src/main/java/io/grpc/xds/XdsServerBuilder.java b/xds/src/main/java/io/grpc/xds/XdsServerBuilder.java index 34879fd8cd0..c95c1e6d48f 100644 --- a/xds/src/main/java/io/grpc/xds/XdsServerBuilder.java +++ b/xds/src/main/java/io/grpc/xds/XdsServerBuilder.java @@ -16,9 +16,11 @@ package io.grpc.xds; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; -import static io.grpc.xds.InternalXdsAttributes.ATTR_FILTER_CHAIN_SELECTOR_REF; +import static io.grpc.xds.InternalXdsAttributes.ATTR_DRAIN_GRACE_NANOS; +import static io.grpc.xds.InternalXdsAttributes.ATTR_FILTER_CHAIN_SELECTOR_MANAGER; import com.google.common.annotations.VisibleForTesting; import com.google.errorprone.annotations.DoNotCall; @@ -33,11 +35,10 @@ import io.grpc.netty.InternalNettyServerCredentials; import io.grpc.netty.InternalProtocolNegotiator; import io.grpc.netty.NettyServerBuilder; -import io.grpc.xds.FilterChainMatchingProtocolNegotiators.FilterChainMatchingHandler.FilterChainSelector; import io.grpc.xds.FilterChainMatchingProtocolNegotiators.FilterChainMatchingNegotiatorServerFactory; import io.grpc.xds.XdsNameResolverProvider.XdsClientPoolFactory; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Logger; /** @@ -45,6 +46,8 @@ */ @ExperimentalApi("https://github.com/grpc/grpc-java/issues/7514") public final class XdsServerBuilder extends ForwardingServerBuilder { + private static final long AS_LARGE_AS_INFINITE = TimeUnit.DAYS.toNanos(1000); + private final NettyServerBuilder delegate; private final int port; private XdsServingStatusListener xdsServingStatusListener; @@ -52,6 +55,8 @@ public final class XdsServerBuilder extends ForwardingServerBuilder= 0, "drain grace time must be non-negative: %s", + drainGraceTime); + checkNotNull(drainGraceTimeUnit, "drainGraceTimeUnit"); + if (drainGraceTimeUnit.toNanos(drainGraceTime) >= AS_LARGE_AS_INFINITE) { + drainGraceTimeUnit = null; + } + this.drainGraceTime = drainGraceTime; + this.drainGraceTimeUnit = drainGraceTimeUnit; + return this; + } + @DoNotCall("Unsupported. Use forPort(int, ServerCredentials) instead") public static ServerBuilder forPort(int port) { throw new UnsupportedOperationException( @@ -94,12 +119,15 @@ public static XdsServerBuilder forPort(int port, ServerCredentials serverCredent @Override public Server build() { checkState(isServerBuilt.compareAndSet(false, true), "Server already built!"); - AtomicReference filterChainSelectorRef = new AtomicReference<>(); - InternalNettyServerBuilder.eagAttributes(delegate, Attributes.newBuilder() - .set(ATTR_FILTER_CHAIN_SELECTOR_REF, filterChainSelectorRef) - .build()); + FilterChainSelectorManager filterChainSelectorManager = new FilterChainSelectorManager(); + Attributes.Builder builder = Attributes.newBuilder() + .set(ATTR_FILTER_CHAIN_SELECTOR_MANAGER, filterChainSelectorManager); + if (drainGraceTimeUnit != null) { + builder.set(ATTR_DRAIN_GRACE_NANOS, drainGraceTimeUnit.toNanos(drainGraceTime)); + } + InternalNettyServerBuilder.eagAttributes(delegate, builder.build()); return new XdsServerWrapper("0.0.0.0:" + port, delegate, xdsServingStatusListener, - filterChainSelectorRef, xdsClientPoolFactory, filterRegistry); + filterChainSelectorManager, xdsClientPoolFactory, filterRegistry); } @VisibleForTesting diff --git a/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java b/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java index faa6e9d34b2..29821f2cba8 100644 --- a/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java +++ b/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java @@ -100,7 +100,7 @@ public void uncaughtException(Thread t, Throwable e) { private final ThreadSafeRandom random = ThreadSafeRandomImpl.instance; private final XdsClientPoolFactory xdsClientPoolFactory; private final XdsServingStatusListener listener; - private final AtomicReference filterChainSelectorRef; + private final FilterChainSelectorManager filterChainSelectorManager; private final AtomicBoolean started = new AtomicBoolean(false); private final AtomicBoolean shutdown = new AtomicBoolean(false); private boolean isServing; @@ -117,11 +117,11 @@ public void uncaughtException(Thread t, Throwable e) { String listenerAddress, ServerBuilder delegateBuilder, XdsServingStatusListener listener, - AtomicReference filterChainSelectorRef, + FilterChainSelectorManager filterChainSelectorManager, XdsClientPoolFactory xdsClientPoolFactory, FilterRegistry filterRegistry) { - this(listenerAddress, delegateBuilder, listener, filterChainSelectorRef, xdsClientPoolFactory, - filterRegistry, SharedResourceHolder.get(GrpcUtil.TIMER_SERVICE)); + this(listenerAddress, delegateBuilder, listener, filterChainSelectorManager, + xdsClientPoolFactory, filterRegistry, SharedResourceHolder.get(GrpcUtil.TIMER_SERVICE)); sharedTimeService = true; } @@ -130,7 +130,7 @@ public void uncaughtException(Thread t, Throwable e) { String listenerAddress, ServerBuilder delegateBuilder, XdsServingStatusListener listener, - AtomicReference filterChainSelectorRef, + FilterChainSelectorManager filterChainSelectorManager, XdsClientPoolFactory xdsClientPoolFactory, FilterRegistry filterRegistry, ScheduledExecutorService timeService) { @@ -138,7 +138,8 @@ public void uncaughtException(Thread t, Throwable e) { this.delegateBuilder = checkNotNull(delegateBuilder, "delegateBuilder"); this.delegateBuilder.intercept(new ConfigApplyingInterceptor()); this.listener = checkNotNull(listener, "listener"); - this.filterChainSelectorRef = checkNotNull(filterChainSelectorRef, "filterChainSelectorRef"); + this.filterChainSelectorManager + = checkNotNull(filterChainSelectorManager, "filterChainSelectorManager"); this.xdsClientPoolFactory = checkNotNull(xdsClientPoolFactory, "xdsClientPoolFactory"); this.timeService = checkNotNull(timeService, "timeService"); this.filterRegistry = checkNotNull(filterRegistry,"filterRegistry"); @@ -361,8 +362,8 @@ public void run() { } checkNotNull(update.listener(), "update"); if (!pendingRds.isEmpty()) { - // filter chain state has not yet been applied to filterChainSelectorRef and there are - // two sets of sslContextProviderSuppliers, so we release the old ones. + // filter chain state has not yet been applied to filterChainSelectorManager and there + // are two sets of sslContextProviderSuppliers, so we release the old ones. releaseSuppliersInFlight(); pendingRds.clear(); } @@ -443,7 +444,7 @@ private void shutdown() { logger.log(Level.FINE, "Stop watching LDS resource {0}", resourceName); xdsClient.cancelLdsResourceWatch(resourceName, this); List toRelease = getSuppliersInUse(); - filterChainSelectorRef.set(FilterChainSelector.NO_FILTER_CHAIN); + filterChainSelectorManager.updateSelector(FilterChainSelector.NO_FILTER_CHAIN); for (SslContextProviderSupplier s: toRelease) { s.close(); } @@ -460,7 +461,7 @@ private void updateSelector() { defaultFilterChain == null ? null : defaultFilterChain.getSslContextProviderSupplier(), defaultFilterChain == null ? null : generateRoutingConfig(defaultFilterChain)); List toRelease = getSuppliersInUse(); - filterChainSelectorRef.set(selector); + filterChainSelectorManager.updateSelector(selector); for (SslContextProviderSupplier e: toRelease) { e.close(); } @@ -482,7 +483,7 @@ private ServerRoutingConfig generateRoutingConfig(FilterChain filterChain) { private void handleConfigNotFound(StatusException exception) { cleanUpRouteDiscoveryStates(); List toRelease = getSuppliersInUse(); - filterChainSelectorRef.set(FilterChainSelector.NO_FILTER_CHAIN); + filterChainSelectorManager.updateSelector(FilterChainSelector.NO_FILTER_CHAIN); for (SslContextProviderSupplier s: toRelease) { s.close(); } @@ -511,7 +512,7 @@ private void cleanUpRouteDiscoveryStates() { private List getSuppliersInUse() { List toRelease = new ArrayList<>(); - FilterChainSelector selector = filterChainSelectorRef.get(); + FilterChainSelector selector = filterChainSelectorManager.getSelectorToUpdateSelector(); if (selector != null) { for (FilterChain f: selector.getRoutingConfigs().keySet()) { if (f.getSslContextProviderSupplier() != null) { diff --git a/xds/src/test/java/io/grpc/xds/FilterChainMatchingProtocolNegotiatorsTest.java b/xds/src/test/java/io/grpc/xds/FilterChainMatchingProtocolNegotiatorsTest.java index 167f3f03c6b..891dec322c0 100644 --- a/xds/src/test/java/io/grpc/xds/FilterChainMatchingProtocolNegotiatorsTest.java +++ b/xds/src/test/java/io/grpc/xds/FilterChainMatchingProtocolNegotiatorsTest.java @@ -65,6 +65,7 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicReference; +import org.junit.After; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; @@ -87,6 +88,7 @@ public class FilterChainMatchingProtocolNegotiatorsTest { private ChannelHandlerContext channelHandlerCtx; @Mock private ProtocolNegotiator mockDelegate; + private FilterChainSelectorManager selectorManager = new FilterChainSelectorManager(); private static final HttpConnectionManager HTTP_CONNECTION_MANAGER = createRds("routing-config"); private static final String LOCAL_IP = "10.1.2.3"; // dest private static final String REMOTE_IP = "10.4.2.3"; // source @@ -94,6 +96,16 @@ public class FilterChainMatchingProtocolNegotiatorsTest { private final ServerRoutingConfig noopConfig = ServerRoutingConfig.create( new ArrayList(), new AtomicReference>()); + @After + @SuppressWarnings("FutureReturnValueIgnored") + public void tearDown() { + if (channel.isActive()) { + channel.close(); + channel.runPendingTasks(); + } + assertThat(selectorManager.getRegisterCount()).isEqualTo(0); + } + @Test public void nofilterChainMatch_defaultSslContext() throws Exception { final SettableFuture sslSet = SettableFuture.create(); @@ -103,10 +115,10 @@ public void nofilterChainMatch_defaultSslContext() throws Exception { SslContextProviderSupplier defaultSsl = new SslContextProviderSupplier(createTls(), tlsContextManager); - FilterChainSelector selector = new FilterChainSelector( - new HashMap(), defaultSsl, noopConfig); + selectorManager.updateSelector(new FilterChainSelector( + new HashMap(), defaultSsl, noopConfig)); FilterChainMatchingHandler filterChainMatchingHandler = - new FilterChainMatchingHandler(grpcHandler, selector, mockDelegate); + new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); setupChannel("172.168.1.1", "172.168.1.2", 80, filterChainMatchingHandler); ChannelHandlerContext channelHandlerCtx = pipeline.context(filterChainMatchingHandler); assertThat(channelHandlerCtx).isNotNull(); @@ -125,10 +137,10 @@ public void nofilterChainMatch_defaultSslContext() throws Exception { @Test public void noFilterChainMatch_noDefaultSslContext() { - FilterChainSelector selector = new FilterChainSelector( - new HashMap(), null, null); + selectorManager.updateSelector(new FilterChainSelector( + new HashMap(), null, null)); FilterChainMatchingHandler filterChainMatchingHandler = - new FilterChainMatchingHandler(grpcHandler, selector, mockDelegate); + new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); setupChannel("172.168.1.1", "172.168.2.2", 90, filterChainMatchingHandler); channelHandlerCtx = pipeline.context(filterChainMatchingHandler); assertThat(channelHandlerCtx).isNotNull(); @@ -139,6 +151,33 @@ public void noFilterChainMatch_noDefaultSslContext() { assertThat(channel.closeFuture().isDone()).isTrue(); } + @Test + public void filterSelectorChange_drainsConnection() { + ChannelHandler next = new ChannelInboundHandlerAdapter(); + when(mockDelegate.newHandler(grpcHandler)).thenReturn(next); + selectorManager.updateSelector(new FilterChainSelector( + new HashMap(), null, noopConfig)); + FilterChainMatchingHandler filterChainMatchingHandler = + new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); + setupChannel("172.168.1.1", "172.168.2.2", 90, filterChainMatchingHandler); + channelHandlerCtx = pipeline.context(filterChainMatchingHandler); + assertThat(channelHandlerCtx).isNotNull(); + + pipeline.fireUserEventTriggered(event); + channelHandlerCtx = pipeline.context(filterChainMatchingHandler); + assertThat(channelHandlerCtx).isNull(); + + channel.runPendingTasks(); + channelHandlerCtx = pipeline.context(next); + assertThat(channelHandlerCtx).isNotNull(); + assertThat(channel.readOutbound()).isNull(); + + selectorManager.updateSelector(new FilterChainSelector( + new HashMap(), null, noopConfig)); + assertThat(channel.readOutbound().getClass().getName()) + .isEqualTo("io.grpc.netty.GracefulServerCloseCommand"); + } + @Test public void singleFilterChainWithoutAlpn() throws Exception { EnvoyServerProtoData.FilterChainMatch filterChainMatch = @@ -157,10 +196,10 @@ public void singleFilterChainWithoutAlpn() throws Exception { "filter-chain-foo", filterChainMatch, HTTP_CONNECTION_MANAGER, tlsContext, tlsContextManager); - FilterChainSelector selector = new FilterChainSelector(ImmutableMap.of(filterChain, noopConfig), - null, null); + selectorManager.updateSelector(new FilterChainSelector(ImmutableMap.of(filterChain, noopConfig), + null, null)); FilterChainMatchingHandler filterChainMatchingHandler = - new FilterChainMatchingHandler(grpcHandler, selector, mockDelegate); + new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); final SettableFuture sslSet = SettableFuture.create(); final SettableFuture routingSettable = SettableFuture.create(); ChannelHandler next = captureAttrHandler(sslSet, routingSettable); @@ -196,11 +235,11 @@ public void singleFilterChainWithAlpn() throws Exception { EnvoyServerProtoData.FilterChain defaultFilterChain = new EnvoyServerProtoData.FilterChain( "filter-chain-bar", null, HTTP_CONNECTION_MANAGER, defaultTlsContext, tlsContextManager); - FilterChainSelector selector = new FilterChainSelector( + selectorManager.updateSelector(new FilterChainSelector( ImmutableMap.of(filterChain, randomConfig("no-match")), - defaultFilterChain.getSslContextProviderSupplier(), noopConfig); + defaultFilterChain.getSslContextProviderSupplier(), noopConfig)); FilterChainMatchingHandler filterChainMatchingHandler = - new FilterChainMatchingHandler(grpcHandler, selector, mockDelegate); + new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); final SettableFuture sslSet = SettableFuture.create(); final SettableFuture routingSettable = SettableFuture.create(); @@ -242,12 +281,12 @@ public void destPortFails_returnDefaultFilterChain() throws Exception { ServerRoutingConfig routingConfig = ServerRoutingConfig.create( new ArrayList(), new AtomicReference<>( ImmutableList.of(createVirtualHost("virtual")))); - FilterChainSelector selector = new FilterChainSelector( + selectorManager.updateSelector(new FilterChainSelector( ImmutableMap.of(filterChainWithDestPort, routingConfig), - defaultFilterChain.getSslContextProviderSupplier(), noopConfig); + defaultFilterChain.getSslContextProviderSupplier(), noopConfig)); FilterChainMatchingHandler filterChainMatchingHandler = - new FilterChainMatchingHandler(grpcHandler, selector, mockDelegate); + new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); final SettableFuture sslSet = SettableFuture.create(); final SettableFuture routingSettable = SettableFuture.create(); @@ -285,12 +324,12 @@ public void destPrefixRangeMatch() throws Exception { "filter-chain-bar", null, HTTP_CONNECTION_MANAGER, tlsContextForDefaultFilterChain, tlsContextManager); - FilterChainSelector selector = new FilterChainSelector( + selectorManager.updateSelector(new FilterChainSelector( ImmutableMap.of(filterChainWithMatch, noopConfig), - defaultFilterChain.getSslContextProviderSupplier(), randomConfig("no-match")); + defaultFilterChain.getSslContextProviderSupplier(), randomConfig("no-match"))); FilterChainMatchingHandler filterChainMatchingHandler = - new FilterChainMatchingHandler(grpcHandler, selector, mockDelegate); + new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); final SettableFuture sslSet = SettableFuture.create(); final SettableFuture routingSettable = SettableFuture.create(); @@ -329,12 +368,12 @@ public void destPrefixRangeMismatch_returnDefaultFilterChain() EnvoyServerProtoData.FilterChain defaultFilterChain = new EnvoyServerProtoData.FilterChain( "filter-chain-bar", null, HTTP_CONNECTION_MANAGER, tlsContextForDefaultFilterChain, tlsContextManager); - FilterChainSelector selector = new FilterChainSelector( + selectorManager.updateSelector(new FilterChainSelector( ImmutableMap.of(filterChainWithMismatch, randomConfig("no-match")), - defaultFilterChain.getSslContextProviderSupplier(), noopConfig); + defaultFilterChain.getSslContextProviderSupplier(), noopConfig)); FilterChainMatchingHandler filterChainMatchingHandler = - new FilterChainMatchingHandler(grpcHandler, selector, mockDelegate); + new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); final SettableFuture sslSet = SettableFuture.create(); final SettableFuture routingSettable = SettableFuture.create(); @@ -374,11 +413,11 @@ public void dest0LengthPrefixRange() "filter-chain-bar", null, HTTP_CONNECTION_MANAGER, tlsContextForDefaultFilterChain, tlsContextManager); - FilterChainSelector selector = new FilterChainSelector( + selectorManager.updateSelector(new FilterChainSelector( ImmutableMap.of(filterChain0Length, noopConfig), - defaultFilterChain.getSslContextProviderSupplier(), null); + defaultFilterChain.getSslContextProviderSupplier(), null)); FilterChainMatchingHandler filterChainMatchingHandler = - new FilterChainMatchingHandler(grpcHandler, selector, mockDelegate); + new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); final SettableFuture sslSet = SettableFuture.create(); final SettableFuture routingSettable = SettableFuture.create(); @@ -431,13 +470,13 @@ public void destPrefixRange_moreSpecificWins() tlsContextManager); EnvoyServerProtoData.FilterChain defaultFilterChain = new EnvoyServerProtoData.FilterChain( "filter-chain-baz", null, HTTP_CONNECTION_MANAGER, null, tlsContextManager); - FilterChainSelector selector = new FilterChainSelector( + selectorManager.updateSelector(new FilterChainSelector( ImmutableMap.of(filterChainLessSpecific, randomConfig("no-match"), filterChainMoreSpecific, noopConfig), - defaultFilterChain.getSslContextProviderSupplier(), randomConfig("default")); + defaultFilterChain.getSslContextProviderSupplier(), randomConfig("default"))); FilterChainMatchingHandler filterChainMatchingHandler = - new FilterChainMatchingHandler(grpcHandler, selector, mockDelegate); + new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); final SettableFuture sslSet = SettableFuture.create(); final SettableFuture routingSettable = SettableFuture.create(); @@ -490,12 +529,12 @@ public void destPrefixRange_emptyListLessSpecific() tlsContextManager); EnvoyServerProtoData.FilterChain defaultFilterChain = new EnvoyServerProtoData.FilterChain( "filter-chain-baz", null, HTTP_CONNECTION_MANAGER, null, tlsContextManager); - FilterChainSelector selector = new FilterChainSelector( + selectorManager.updateSelector(new FilterChainSelector( ImmutableMap.of(filterChainLessSpecific, randomConfig("no-match"), filterChainMoreSpecific, noopConfig), - defaultFilterChain.getSslContextProviderSupplier(), randomConfig("default")); + defaultFilterChain.getSslContextProviderSupplier(), randomConfig("default"))); FilterChainMatchingHandler filterChainMatchingHandler = - new FilterChainMatchingHandler(grpcHandler, selector, mockDelegate); + new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); final SettableFuture sslSet = SettableFuture.create(); final SettableFuture routingSettable = SettableFuture.create(); @@ -547,13 +586,13 @@ public void destPrefixRangeIpv6_moreSpecificWins() tlsContextMoreSpecific, tlsContextManager); EnvoyServerProtoData.FilterChain defaultFilterChain = new EnvoyServerProtoData.FilterChain( "filter-chain-baz", null, HTTP_CONNECTION_MANAGER, null, tlsContextManager); - FilterChainSelector selector = new FilterChainSelector( + selectorManager.updateSelector(new FilterChainSelector( ImmutableMap.of(filterChainLessSpecific, randomConfig("no-match"), filterChainMoreSpecific, noopConfig), - defaultFilterChain.getSslContextProviderSupplier(), randomConfig("default")); + defaultFilterChain.getSslContextProviderSupplier(), randomConfig("default"))); FilterChainMatchingHandler filterChainMatchingHandler = - new FilterChainMatchingHandler(grpcHandler, selector, mockDelegate); + new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); final SettableFuture sslSet = SettableFuture.create(); final SettableFuture routingSettable = SettableFuture.create(); @@ -610,12 +649,12 @@ public void destPrefixRange_moreSpecificWith2Wins() EnvoyServerProtoData.FilterChain defaultFilterChain = new EnvoyServerProtoData.FilterChain( "filter-chain-baz", null, HTTP_CONNECTION_MANAGER, null, tlsContextManager); - FilterChainSelector selector = new FilterChainSelector( + selectorManager.updateSelector(new FilterChainSelector( ImmutableMap.of(filterChainMoreSpecificWith2, noopConfig, filterChainLessSpecific, randomConfig("no-match")), - defaultFilterChain.getSslContextProviderSupplier(), randomConfig("default")); + defaultFilterChain.getSslContextProviderSupplier(), randomConfig("default"))); FilterChainMatchingHandler filterChainMatchingHandler = - new FilterChainMatchingHandler(grpcHandler, selector, mockDelegate); + new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); final SettableFuture sslSet = SettableFuture.create(); final SettableFuture routingSettable = SettableFuture.create(); @@ -653,11 +692,11 @@ public void sourceTypeMismatch_returnDefaultFilterChain() throws Exception { EnvoyServerProtoData.FilterChain defaultFilterChain = new EnvoyServerProtoData.FilterChain( "filter-chain-bar", null, HTTP_CONNECTION_MANAGER,tlsContextForDefaultFilterChain, tlsContextManager); - FilterChainSelector selector = new FilterChainSelector( + selectorManager.updateSelector(new FilterChainSelector( ImmutableMap.of(filterChainWithMismatch, randomConfig("no-match")), - defaultFilterChain.getSslContextProviderSupplier(), noopConfig); + defaultFilterChain.getSslContextProviderSupplier(), noopConfig)); FilterChainMatchingHandler filterChainMatchingHandler = - new FilterChainMatchingHandler(grpcHandler, selector, mockDelegate); + new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); final SettableFuture sslSet = SettableFuture.create(); final SettableFuture routingSettable = SettableFuture.create(); @@ -698,11 +737,11 @@ public void sourceTypeLocal() throws Exception { "filter-chain-bar", null, HTTP_CONNECTION_MANAGER, tlsContextForDefaultFilterChain, tlsContextManager); - FilterChainSelector selector = new FilterChainSelector( + selectorManager.updateSelector(new FilterChainSelector( ImmutableMap.of(filterChainWithMatch, noopConfig), - defaultFilterChain.getSslContextProviderSupplier(), randomConfig("default")); + defaultFilterChain.getSslContextProviderSupplier(), randomConfig("default"))); FilterChainMatchingHandler filterChainMatchingHandler = - new FilterChainMatchingHandler(grpcHandler, selector, mockDelegate); + new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); setupChannel(LOCAL_IP, LOCAL_IP, 15000, filterChainMatchingHandler); pipeline.fireUserEventTriggered(event); channel.runPendingTasks(); @@ -757,13 +796,13 @@ public void sourcePrefixRange_moreSpecificWith2Wins() EnvoyServerProtoData.FilterChain defaultFilterChain = new EnvoyServerProtoData.FilterChain( "filter-chain-baz", null, HTTP_CONNECTION_MANAGER, null, tlsContextManager); - FilterChainSelector selector = new FilterChainSelector( + selectorManager.updateSelector(new FilterChainSelector( ImmutableMap.of(filterChainMoreSpecificWith2, noopConfig, filterChainLessSpecific, randomConfig("no-match")), - defaultFilterChain.getSslContextProviderSupplier(), randomConfig("default")); + defaultFilterChain.getSslContextProviderSupplier(), randomConfig("default"))); FilterChainMatchingHandler filterChainMatchingHandler = - new FilterChainMatchingHandler(grpcHandler, selector, mockDelegate); + new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); setupChannel(LOCAL_IP, REMOTE_IP, 15000, filterChainMatchingHandler); pipeline.fireUserEventTriggered(event); channel.runPendingTasks(); @@ -823,12 +862,12 @@ public void userEventTriggered(ChannelHandlerContext ctx, Object evt) { EnvoyServerProtoData.FilterChain defaultFilterChain = new EnvoyServerProtoData.FilterChain( "filter-chain-baz", null, HTTP_CONNECTION_MANAGER, null, null); - FilterChainSelector selector = new FilterChainSelector( + selectorManager.updateSelector(new FilterChainSelector( ImmutableMap.of(filterChain1, noopConfig, filterChain2, noopConfig), - defaultFilterChain.getSslContextProviderSupplier(), noopConfig); + defaultFilterChain.getSslContextProviderSupplier(), noopConfig)); FilterChainMatchingHandler filterChainMatchingHandler = - new FilterChainMatchingHandler(grpcHandler, selector, mockDelegate); + new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); setupChannel(LOCAL_IP, REMOTE_IP, 15000, filterChainMatchingHandler); pipeline.fireUserEventTriggered(event); channel.runPendingTasks(); @@ -884,13 +923,13 @@ public void sourcePortMatch_exactMatchWinsOverEmptyList() throws Exception { EnvoyServerProtoData.FilterChain defaultFilterChain = new EnvoyServerProtoData.FilterChain( "filter-chain-baz", null, HTTP_CONNECTION_MANAGER, null, tlsContextManager); - FilterChainSelector selector = new FilterChainSelector( + selectorManager.updateSelector(new FilterChainSelector( ImmutableMap.of(filterChainEmptySourcePorts, randomConfig("no-match"), filterChainSourcePortMatch, noopConfig), - defaultFilterChain.getSslContextProviderSupplier(), randomConfig("default")); + defaultFilterChain.getSslContextProviderSupplier(), randomConfig("default"))); FilterChainMatchingHandler filterChainMatchingHandler = - new FilterChainMatchingHandler(grpcHandler, selector, mockDelegate); + new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); final SettableFuture sslSet = SettableFuture.create(); final SettableFuture routingSettable = SettableFuture.create(); ChannelHandler next = captureAttrHandler(sslSet, routingSettable); @@ -1040,11 +1079,11 @@ public void filterChain_5stepMatch() throws Exception { map.put(filterChain4, randomConfig("4")); map.put(filterChain5, noopConfig); map.put(filterChain6, randomConfig("6")); - FilterChainSelector selector = new FilterChainSelector( - map, defaultFilterChain.getSslContextProviderSupplier(), randomConfig("default")); + selectorManager.updateSelector(new FilterChainSelector( + map, defaultFilterChain.getSslContextProviderSupplier(), randomConfig("default"))); FilterChainMatchingHandler filterChainMatchingHandler = - new FilterChainMatchingHandler(grpcHandler, selector, mockDelegate); + new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); final SettableFuture sslSet = SettableFuture.create(); final SettableFuture routingSettable = SettableFuture.create(); @@ -1114,12 +1153,12 @@ public void filterChainMatch_unsupportedMatchers() throws Exception { "filter-chain-baz", defaultFilterChainMatch, HTTP_CONNECTION_MANAGER, tlsContext3, mock(TlsContextManager.class)); - FilterChainSelector selector = new FilterChainSelector( + selectorManager.updateSelector(new FilterChainSelector( ImmutableMap.of(filterChain1, randomConfig("1"), filterChain2, randomConfig("2")), - defaultFilterChain.getSslContextProviderSupplier(), noopConfig); + defaultFilterChain.getSslContextProviderSupplier(), noopConfig)); FilterChainMatchingHandler filterChainMatchingHandler = - new FilterChainMatchingHandler(grpcHandler, selector, mockDelegate); + new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); final SettableFuture sslSet = SettableFuture.create(); final SettableFuture routingSettable = SettableFuture.create(); ChannelHandler next = captureAttrHandler(sslSet, routingSettable); diff --git a/xds/src/test/java/io/grpc/xds/FilterChainSelectorManagerTest.java b/xds/src/test/java/io/grpc/xds/FilterChainSelectorManagerTest.java new file mode 100644 index 00000000000..d7b883f1941 --- /dev/null +++ b/xds/src/test/java/io/grpc/xds/FilterChainSelectorManagerTest.java @@ -0,0 +1,107 @@ +/* + * Copyright 2021 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.xds; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.collect.ImmutableList; +import io.grpc.xds.EnvoyServerProtoData.FilterChain; +import io.grpc.xds.Filter.NamedFilterConfig; +import io.grpc.xds.FilterChainMatchingProtocolNegotiators.FilterChainMatchingHandler.FilterChainSelector; +import io.grpc.xds.FilterChainSelectorManager.Closer; +import io.grpc.xds.XdsServerWrapper.ServerRoutingConfig; +import java.util.Collections; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public final class FilterChainSelectorManagerTest { + private FilterChainSelectorManager manager = new FilterChainSelectorManager(); + private ServerRoutingConfig noopConfig = ServerRoutingConfig.create( + Collections.emptyList(), + new AtomicReference>()); + private FilterChainSelector selector1 = new FilterChainSelector( + Collections.emptyMap(), null, null); + private FilterChainSelector selector2 = new FilterChainSelector( + Collections.emptyMap(), null, noopConfig); + private CounterRunnable runnable1 = new CounterRunnable(); + private CounterRunnable runnable2 = new CounterRunnable(); + + @Test + public void updateSelector_changesSelector() { + assertThat(manager.getSelectorToUpdateSelector()).isNull(); + assertThat(manager.register(new Closer(runnable1))).isNull(); + + manager.updateSelector(selector1); + + assertThat(runnable1.counter).isEqualTo(1); + assertThat(manager.getSelectorToUpdateSelector()).isSameInstanceAs(selector1); + assertThat(manager.register(new Closer(runnable2))).isSameInstanceAs(selector1); + assertThat(runnable2.counter).isEqualTo(0); + } + + @Test + public void updateSelector_callsCloserOnce() { + assertThat(manager.register(new Closer(runnable1))).isNull(); + + manager.updateSelector(selector1); + manager.updateSelector(selector2); + + assertThat(runnable1.counter).isEqualTo(1); + } + + @Test + public void deregister_removesCloser() { + Closer closer1 = new Closer(runnable1); + manager.updateSelector(selector1); + assertThat(manager.register(closer1)).isSameInstanceAs(selector1); + assertThat(manager.getRegisterCount()).isEqualTo(1); + + manager.deregister(closer1); + + assertThat(manager.getRegisterCount()).isEqualTo(0); + manager.updateSelector(selector2); + assertThat(runnable1.counter).isEqualTo(0); + } + + @Test + public void deregister_removesCorrectCloser() { + Closer closer1 = new Closer(runnable1); + Closer closer2 = new Closer(runnable2); + manager.updateSelector(selector1); + assertThat(manager.register(closer1)).isSameInstanceAs(selector1); + assertThat(manager.register(closer2)).isSameInstanceAs(selector1); + assertThat(manager.getRegisterCount()).isEqualTo(2); + + manager.deregister(closer1); + + assertThat(manager.getRegisterCount()).isEqualTo(1); + manager.updateSelector(selector2); + assertThat(runnable1.counter).isEqualTo(0); + assertThat(runnable2.counter).isEqualTo(1); + } + + private static class CounterRunnable implements Runnable { + int counter; + + @Override public void run() { + counter++; + } + } +} diff --git a/xds/src/test/java/io/grpc/xds/XdsClientWrapperForServerSdsTestMisc.java b/xds/src/test/java/io/grpc/xds/XdsClientWrapperForServerSdsTestMisc.java index 532cb282b26..1871cb79770 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientWrapperForServerSdsTestMisc.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientWrapperForServerSdsTestMisc.java @@ -72,7 +72,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -101,7 +100,7 @@ public class XdsClientWrapperForServerSdsTestMisc { @Mock private XdsServingStatusListener listener; private FakeXdsClient xdsClient = new FakeXdsClient(); - private AtomicReference selectorRef = new AtomicReference<>(); + private FilterChainSelectorManager selectorManager = new FilterChainSelectorManager(); private XdsServerWrapper xdsServerWrapper; @@ -117,13 +116,14 @@ public void setUp() { when(mockBuilder.build()).thenReturn(mockServer); when(mockServer.isShutdown()).thenReturn(false); xdsServerWrapper = new XdsServerWrapper("0.0.0.0:" + PORT, mockBuilder, listener, - selectorRef, new FakeXdsClientPoolFactory(xdsClient), FilterRegistry.newRegistry()); + selectorManager, new FakeXdsClientPoolFactory(xdsClient), FilterRegistry.newRegistry()); } @Test public void nonInetSocketAddress_expectNull() throws Exception { sendListenerUpdate(new InProcessSocketAddress("test1"), null, null, tlsContextManager); - assertThat(getSslContextProviderSupplier(selectorRef.get())).isNull(); + assertThat(getSslContextProviderSupplier(selectorManager.getSelectorToUpdateSelector())) + .isNull(); } @Test @@ -168,7 +168,7 @@ public void run() { LdsUpdate listenerUpdate = LdsUpdate.forTcpListener(listener); xdsClient.ldsWatcher.onChanged(listenerUpdate); start.get(5, TimeUnit.SECONDS); - FilterChainSelector selector = selectorRef.get(); + FilterChainSelector selector = selectorManager.getSelectorToUpdateSelector(); assertThat(getSslContextProviderSupplier(selector)).isNull(); } @@ -193,7 +193,7 @@ public void run() { } catch (ExecutionException ex) { assertThat(ex.getCause()).isInstanceOf(IOException.class); } - assertThat(selectorRef.get()).isSameInstanceAs(NO_FILTER_CHAIN); + assertThat(selectorManager.getSelectorToUpdateSelector()).isSameInstanceAs(NO_FILTER_CHAIN); } @Test @@ -217,7 +217,7 @@ public void run() { } catch (ExecutionException ex) { assertThat(ex.getCause()).isInstanceOf(IOException.class); } - assertThat(selectorRef.get()).isSameInstanceAs(NO_FILTER_CHAIN); + assertThat(selectorManager.getSelectorToUpdateSelector()).isSameInstanceAs(NO_FILTER_CHAIN); } @Test @@ -241,7 +241,7 @@ public void run() { } catch (ExecutionException ex) { assertThat(ex.getCause()).isInstanceOf(IOException.class); } - assertThat(selectorRef.get()).isSameInstanceAs(NO_FILTER_CHAIN); + assertThat(selectorManager.getSelectorToUpdateSelector()).isSameInstanceAs(NO_FILTER_CHAIN); } @Test @@ -263,13 +263,14 @@ public void releaseOldSupplierOnChangedOnShutdown_verifyClose() throws Exception localAddress = new InetSocketAddress(ipLocalAddress, PORT); sendListenerUpdate(localAddress, tlsContext1, null, tlsContextManager); - SslContextProviderSupplier returnedSupplier = getSslContextProviderSupplier(selectorRef.get()); + SslContextProviderSupplier returnedSupplier = + getSslContextProviderSupplier(selectorManager.getSelectorToUpdateSelector()); assertThat(returnedSupplier.getTlsContext()).isSameInstanceAs(tlsContext1); callUpdateSslContext(returnedSupplier); XdsServerTestHelper .generateListenerUpdate(xdsClient, Arrays.asList(1234), tlsContext2, tlsContext3, tlsContextManager); - returnedSupplier = getSslContextProviderSupplier(selectorRef.get()); + returnedSupplier = getSslContextProviderSupplier(selectorManager.getSelectorToUpdateSelector()); assertThat(returnedSupplier.getTlsContext()).isSameInstanceAs(tlsContext2); verify(tlsContextManager, times(1)).releaseServerSslContextProvider(eq(sslContextProvider1)); reset(tlsContextManager); @@ -294,7 +295,7 @@ public SocketAddress remoteAddress() { } }; pipeline = channel.pipeline(); - returnedSupplier = getSslContextProviderSupplier(selectorRef.get()); + returnedSupplier = getSslContextProviderSupplier(selectorManager.getSelectorToUpdateSelector()); assertThat(returnedSupplier.getTlsContext()).isSameInstanceAs(tlsContext3); callUpdateSslContext(returnedSupplier); xdsServerWrapper.shutdown(); @@ -314,7 +315,7 @@ public void releaseOldSupplierOnNotFound_verifyClose() throws Exception { sendListenerUpdate(localAddress, tlsContext1, null, tlsContextManager); SslContextProviderSupplier returnedSupplier = - getSslContextProviderSupplier(selectorRef.get()); + getSslContextProviderSupplier(selectorManager.getSelectorToUpdateSelector()); assertThat(returnedSupplier.getTlsContext()).isSameInstanceAs(tlsContext1); callUpdateSslContext(returnedSupplier); xdsClient.ldsWatcher.onResourceDoesNotExist("not-found Error"); @@ -331,7 +332,7 @@ public void releaseOldSupplierOnPermDeniedError_verifyClose() throws Exception { sendListenerUpdate(localAddress, tlsContext1, null, tlsContextManager); SslContextProviderSupplier returnedSupplier = - getSslContextProviderSupplier(selectorRef.get()); + getSslContextProviderSupplier(selectorManager.getSelectorToUpdateSelector()); assertThat(returnedSupplier.getTlsContext()).isSameInstanceAs(tlsContext1); callUpdateSslContext(returnedSupplier); xdsClient.ldsWatcher.onError(Status.PERMISSION_DENIED); @@ -348,7 +349,7 @@ public void releaseOldSupplierOnTemporaryError_noClose() throws Exception { sendListenerUpdate(localAddress, tlsContext1, null, tlsContextManager); SslContextProviderSupplier returnedSupplier = - getSslContextProviderSupplier(selectorRef.get()); + getSslContextProviderSupplier(selectorManager.getSelectorToUpdateSelector()); assertThat(returnedSupplier.getTlsContext()).isSameInstanceAs(tlsContext1); callUpdateSslContext(returnedSupplier); xdsClient.ldsWatcher.onError(Status.CANCELLED); @@ -412,8 +413,10 @@ public void userEventTriggered(ChannelHandlerContext ctx, Object evt) { ProtocolNegotiator mockDelegate = mock(ProtocolNegotiator.class); GrpcHttp2ConnectionHandler grpcHandler = FakeGrpcHttp2ConnectionHandler.newHandler(); when(mockDelegate.newHandler(grpcHandler)).thenReturn(next); + FilterChainSelectorManager manager = new FilterChainSelectorManager(); + manager.updateSelector(selector); FilterChainMatchingHandler filterChainMatchingHandler = - new FilterChainMatchingHandler(grpcHandler, selector, mockDelegate); + new FilterChainMatchingHandler(grpcHandler, manager, mockDelegate); pipeline.addLast(filterChainMatchingHandler); ProtocolNegotiationEvent event = InternalProtocolNegotiationEvent.getDefault(); pipeline.fireUserEventTriggered(event); diff --git a/xds/src/test/java/io/grpc/xds/XdsServerBuilderTest.java b/xds/src/test/java/io/grpc/xds/XdsServerBuilderTest.java index 476dc10a16a..0d15c1f660e 100644 --- a/xds/src/test/java/io/grpc/xds/XdsServerBuilderTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsServerBuilderTest.java @@ -281,4 +281,15 @@ public void xdsServer_2ndSetter_expectException() throws IOException { assertThat(expected).hasMessageThat().contains("Server already built!"); } } + + @Test + public void drainGraceTime_negativeThrows() throws IOException { + buildBuilder(null); + try { + builder.drainGraceTime(-1, TimeUnit.SECONDS); + fail("exception expected"); + } catch (IllegalArgumentException expected) { + assertThat(expected).hasMessageThat().contains("drain grace time"); + } + } } diff --git a/xds/src/test/java/io/grpc/xds/XdsServerWrapperTest.java b/xds/src/test/java/io/grpc/xds/XdsServerWrapperTest.java index 876b0913742..d4421361158 100644 --- a/xds/src/test/java/io/grpc/xds/XdsServerWrapperTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsServerWrapperTest.java @@ -99,7 +99,7 @@ public class XdsServerWrapperTest { @Mock private XdsServingStatusListener listener; - private AtomicReference selectorRef = new AtomicReference<>(); + private FilterChainSelectorManager selectorManager = new FilterChainSelectorManager(); private FakeClock executor = new FakeClock(); private FakeXdsClient xdsClient = new FakeXdsClient(); private FilterRegistry filterRegistry = FilterRegistry.getDefaultRegistry(); @@ -109,7 +109,7 @@ public class XdsServerWrapperTest { public void setup() { when(mockBuilder.build()).thenReturn(mockServer); xdsServerWrapper = new XdsServerWrapper("0.0.0.0:1", mockBuilder, listener, - selectorRef, new FakeXdsClientPoolFactory(xdsClient), + selectorManager, new FakeXdsClientPoolFactory(xdsClient), filterRegistry, executor.getScheduledExecutorService()); } @@ -141,7 +141,7 @@ private void verifyBootstrapFail(Bootstrapper.BootstrapInfo b) throws Exception XdsClient xdsClient = mock(XdsClient.class); when(xdsClient.getBootstrapInfo()).thenReturn(b); xdsServerWrapper = new XdsServerWrapper("0.0.0.0:1", mockBuilder, listener, - selectorRef, new FakeXdsClientPoolFactory(xdsClient), filterRegistry); + selectorManager, new FakeXdsClientPoolFactory(xdsClient), filterRegistry); final SettableFuture start = SettableFuture.create(); Executors.newSingleThreadExecutor().execute(new Runnable() { @Override @@ -377,8 +377,10 @@ public void run() { xdsClient.deliverLdsUpdate(Collections.singletonList(filterChain), null); start.get(5000, TimeUnit.MILLISECONDS); assertThat(ldsWatched).isEqualTo("grpc/server?udpa.resource.listening_address=0.0.0.0:1"); - assertThat(selectorRef.get().getRoutingConfigs().size()).isEqualTo(1); - ServerRoutingConfig realConfig = selectorRef.get().getRoutingConfigs().get(filterChain); + assertThat(selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().size()) + .isEqualTo(1); + ServerRoutingConfig realConfig = + selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(filterChain); assertThat(realConfig.virtualHosts().get()).isEqualTo(httpConnectionManager.virtualHosts()); assertThat(realConfig.httpFilterConfigs()).isEqualTo(httpConnectionManager.httpFilterConfigs()); verify(listener).onServing(); @@ -408,7 +410,7 @@ public void run() { xdsClient.rdsCount = new CountDownLatch(3); xdsClient.deliverLdsUpdate(Arrays.asList(f0, f1), null); assertThat(start.isDone()).isFalse(); - assertThat(selectorRef.get()).isNull(); + assertThat(selectorManager.getSelectorToUpdateSelector()).isNull(); verify(mockServer, never()).start(); verify(listener, never()).onServing(); @@ -426,23 +428,26 @@ public void run() { Collections.singletonList(createVirtualHost("virtual-host-2"))); start.get(5000, TimeUnit.MILLISECONDS); verify(mockServer).start(); - ServerRoutingConfig realConfig = selectorRef.get().getRoutingConfigs().get(f0); + ServerRoutingConfig realConfig = + selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f0); assertThat(realConfig.virtualHosts().get()).isEqualTo( Collections.singletonList(createVirtualHost("virtual-host-0"))); assertThat(realConfig.httpFilterConfigs()).isEqualTo( f0.getHttpConnectionManager().httpFilterConfigs()); - assertThat(selectorRef.get().getRoutingConfigs().size()).isEqualTo(2); - realConfig = selectorRef.get().getRoutingConfigs().get(f2); + assertThat(selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().size()) + .isEqualTo(2); + realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f2); assertThat(realConfig.virtualHosts().get()).isEqualTo( Collections.singletonList(createVirtualHost("virtual-host-1"))); assertThat(realConfig.httpFilterConfigs()).isEqualTo( f2.getHttpConnectionManager().httpFilterConfigs()); - realConfig = selectorRef.get().getDefaultRoutingConfig(); + realConfig = selectorManager.getSelectorToUpdateSelector().getDefaultRoutingConfig(); assertThat(realConfig.virtualHosts().get()).isEqualTo( Collections.singletonList(createVirtualHost("virtual-host-2"))); assertThat(realConfig.httpFilterConfigs()).isEqualTo( f3.getHttpConnectionManager().httpFilterConfigs()); - assertThat(selectorRef.get().getDefaultSslContextProviderSupplier()).isEqualTo( + assertThat(selectorManager.getSelectorToUpdateSelector().getDefaultSslContextProviderSupplier()) + .isEqualTo( f3.getSslContextProviderSupplier()); } @@ -468,31 +473,32 @@ public void run() { xdsClient.rdsCount = new CountDownLatch(1); xdsClient.deliverLdsUpdate(Arrays.asList(f0, f1), f2); assertThat(start.isDone()).isFalse(); - assertThat(selectorRef.get()).isNull(); + assertThat(selectorManager.getSelectorToUpdateSelector()).isNull(); xdsClient.rdsCount.await(5, TimeUnit.SECONDS); xdsClient.deliverRdsUpdate("r0", Collections.singletonList(createVirtualHost("virtual-host-0"))); start.get(5000, TimeUnit.MILLISECONDS); verify(mockServer, times(1)).start(); - ServerRoutingConfig realConfig = selectorRef.get().getRoutingConfigs().get(f0); + ServerRoutingConfig realConfig = + selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f0); assertThat(realConfig.virtualHosts().get()).isEqualTo( Collections.singletonList(createVirtualHost("virtual-host-0"))); assertThat(realConfig.httpFilterConfigs()).isEqualTo( f0.getHttpConnectionManager().httpFilterConfigs()); - realConfig = selectorRef.get().getRoutingConfigs().get(f1); + realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f1); assertThat(realConfig.virtualHosts().get()).isEqualTo( Collections.singletonList(createVirtualHost("virtual-host-0"))); assertThat(realConfig.httpFilterConfigs()).isEqualTo( f1.getHttpConnectionManager().httpFilterConfigs()); - realConfig = selectorRef.get().getDefaultRoutingConfig(); + realConfig = selectorManager.getSelectorToUpdateSelector().getDefaultRoutingConfig(); assertThat(realConfig.virtualHosts().get()).isEqualTo( Collections.singletonList(createVirtualHost("virtual-host-0"))); assertThat(realConfig.httpFilterConfigs()).isEqualTo( f2.getHttpConnectionManager().httpFilterConfigs()); - assertThat(selectorRef.get().getDefaultSslContextProviderSupplier()).isSameInstanceAs( - f2.getSslContextProviderSupplier()); + assertThat(selectorManager.getSelectorToUpdateSelector().getDefaultSslContextProviderSupplier()) + .isSameInstanceAs(f2.getSslContextProviderSupplier()); EnvoyServerProtoData.FilterChain f3 = createFilterChain("filter-chain-3", createRds("r0")); EnvoyServerProtoData.FilterChain f4 = createFilterChain("filter-chain-4", createRds("r1")); @@ -505,24 +511,26 @@ public void run() { xdsClient.deliverRdsUpdate("r0", Collections.singletonList(createVirtualHost("virtual-host-0"))); - assertThat(selectorRef.get().getRoutingConfigs().size()).isEqualTo(2); - realConfig = selectorRef.get().getRoutingConfigs().get(f5); + assertThat(selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().size()) + .isEqualTo(2); + realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f5); assertThat(realConfig.virtualHosts().get()).isEqualTo( Collections.singletonList(createVirtualHost("virtual-host-1"))); assertThat(realConfig.httpFilterConfigs()).isEqualTo( f5.getHttpConnectionManager().httpFilterConfigs()); - realConfig = selectorRef.get().getRoutingConfigs().get(f3); + realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f3); assertThat(realConfig.virtualHosts().get()).isEqualTo( Collections.singletonList(createVirtualHost("virtual-host-0"))); assertThat(realConfig.httpFilterConfigs()).isEqualTo( f3.getHttpConnectionManager().httpFilterConfigs()); - realConfig = selectorRef.get().getDefaultRoutingConfig(); + realConfig = selectorManager.getSelectorToUpdateSelector().getDefaultRoutingConfig(); assertThat(realConfig.virtualHosts().get()).isEqualTo( Collections.singletonList(createVirtualHost("virtual-host-1"))); assertThat(realConfig.httpFilterConfigs()).isEqualTo( f4.getHttpConnectionManager().httpFilterConfigs()); - assertThat(selectorRef.get().getDefaultSslContextProviderSupplier()).isSameInstanceAs( + assertThat(selectorManager.getSelectorToUpdateSelector().getDefaultSslContextProviderSupplier()) + .isSameInstanceAs( f4.getSslContextProviderSupplier()); verify(mockServer, times(1)).start(); xdsServerWrapper.shutdown(); @@ -556,33 +564,35 @@ public void run() { xdsClient.rdsCount.await(); xdsClient.rdsWatchers.get("r0").onError(Status.CANCELLED); start.get(5000, TimeUnit.MILLISECONDS); - assertThat(selectorRef.get().getRoutingConfigs().size()).isEqualTo(2); - ServerRoutingConfig realConfig = selectorRef.get().getRoutingConfigs().get(f1); + assertThat(selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().size()) + .isEqualTo(2); + ServerRoutingConfig realConfig = + selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f1); assertThat(realConfig.virtualHosts().get()).isNull(); assertThat(realConfig.httpFilterConfigs()).isEqualTo( f1.getHttpConnectionManager().httpFilterConfigs()); - realConfig = selectorRef.get().getRoutingConfigs().get(f0); + realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f0); assertThat(realConfig.virtualHosts().get()).isEqualTo(hcmVirtual.virtualHosts()); assertThat(realConfig.httpFilterConfigs()).isEqualTo( f0.getHttpConnectionManager().httpFilterConfigs()); xdsClient.deliverRdsUpdate("r0", Collections.singletonList(createVirtualHost("virtual-host-1"))); - realConfig = selectorRef.get().getRoutingConfigs().get(f1); + realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f1); assertThat(realConfig.virtualHosts().get()).isEqualTo( Collections.singletonList(createVirtualHost("virtual-host-1"))); assertThat(realConfig.httpFilterConfigs()).isEqualTo( f1.getHttpConnectionManager().httpFilterConfigs()); xdsClient.rdsWatchers.get("r0").onError(Status.CANCELLED); - realConfig = selectorRef.get().getRoutingConfigs().get(f1); + realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f1); assertThat(realConfig.virtualHosts().get()).isEqualTo( Collections.singletonList(createVirtualHost("virtual-host-1"))); assertThat(realConfig.httpFilterConfigs()).isEqualTo( f1.getHttpConnectionManager().httpFilterConfigs()); xdsClient.rdsWatchers.get("r0").onResourceDoesNotExist("r0"); - realConfig = selectorRef.get().getRoutingConfigs().get(f1); + realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f1); assertThat(realConfig.virtualHosts().get()).isNull(); assertThat(realConfig.httpFilterConfigs()).isEqualTo( f1.getHttpConnectionManager().httpFilterConfigs()); @@ -615,7 +625,8 @@ public void run() { SslContextProviderSupplier sslSupplier0 = filterChain0.getSslContextProviderSupplier(); xdsClient.deliverLdsUpdate(Collections.singletonList(filterChain0), null); xdsClient.ldsWatcher.onError(Status.INTERNAL); - assertThat(selectorRef.get()).isSameInstanceAs(FilterChainSelector.NO_FILTER_CHAIN); + assertThat(selectorManager.getSelectorToUpdateSelector()) + .isSameInstanceAs(FilterChainSelector.NO_FILTER_CHAIN); assertThat(xdsClient.rdsWatchers).isEmpty(); verify(mockBuilder, times(1)).build(); verify(listener, times(2)).onNotServing(any(StatusException.class)); @@ -634,8 +645,10 @@ public void run() { verify(mockBuilder, times(1)).build(); verify(mockServer, times(2)).start(); verify(listener, times(1)).onServing(); - assertThat(selectorRef.get().getRoutingConfigs().size()).isEqualTo(1); - ServerRoutingConfig realConfig = selectorRef.get().getRoutingConfigs().get(filterChain1); + assertThat(selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().size()) + .isEqualTo(1); + ServerRoutingConfig realConfig = + selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(filterChain1); assertThat(realConfig.virtualHosts().get()).isEqualTo( Collections.singletonList(createVirtualHost("virtual-host-1"))); assertThat(realConfig.httpFilterConfigs()).isEqualTo( @@ -648,8 +661,10 @@ public void run() { verify(mockBuilder, times(1)).build(); verify(mockServer, times(2)).start(); verify(listener, times(2)).onNotServing(any(StatusException.class)); - assertThat(selectorRef.get().getRoutingConfigs().size()).isEqualTo(1); - realConfig = selectorRef.get().getRoutingConfigs().get(filterChain1); + assertThat(selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().size()) + .isEqualTo(1); + realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs() + .get(filterChain1); assertThat(realConfig.virtualHosts().get()).isEqualTo( Collections.singletonList(createVirtualHost("virtual-host-2"))); assertThat(realConfig.httpFilterConfigs()).isEqualTo( @@ -661,7 +676,8 @@ public void run() { assertThat(xdsClient.rdsWatchers).isEmpty(); verify(mockServer, times(3)).shutdown(); when(mockServer.isShutdown()).thenReturn(true); - assertThat(selectorRef.get()).isSameInstanceAs(FilterChainSelector.NO_FILTER_CHAIN); + assertThat(selectorManager.getSelectorToUpdateSelector()) + .isSameInstanceAs(FilterChainSelector.NO_FILTER_CHAIN); verify(listener, times(3)).onNotServing(any(StatusException.class)); assertThat(sslSupplier1.isShutdown()).isTrue(); // no op @@ -686,8 +702,10 @@ public void run() { verify(mockServer, times(3)).start(); verify(listener, times(1)).onServing(); verify(listener, times(3)).onNotServing(any(StatusException.class)); - assertThat(selectorRef.get().getRoutingConfigs().size()).isEqualTo(1); - realConfig = selectorRef.get().getRoutingConfigs().get(filterChain2); + assertThat(selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().size()) + .isEqualTo(1); + realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs() + .get(filterChain2); assertThat(realConfig.virtualHosts().get()).isEqualTo( Collections.singletonList(createVirtualHost("virtual-host-1"))); assertThat(realConfig.httpFilterConfigs()).isEqualTo( @@ -712,8 +730,10 @@ public void run() { when(mockServer.isShutdown()).thenReturn(false); verify(listener, times(4)).onNotServing(any(StatusException.class)); - assertThat(selectorRef.get().getRoutingConfigs().size()).isEqualTo(1); - realConfig = selectorRef.get().getRoutingConfigs().get(filterChain3); + assertThat(selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().size()) + .isEqualTo(1); + realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs() + .get(filterChain3); assertThat(realConfig.virtualHosts().get()).isEqualTo( Collections.singletonList(createVirtualHost("virtual-host-1"))); assertThat(realConfig.httpFilterConfigs()).isEqualTo( From 76696567259c0ed3c0a64a989e03ed140894a0a4 Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Wed, 15 Sep 2021 13:23:21 -0700 Subject: [PATCH 19/76] Revert "netty: Requests with Connection header are malformed" This reverts commit 6e89919e3265eadd33dc7e154ba291d46563f286. This was found to break a test proxy. We'll work on fixing the proxy and then roll this forward again. --- api/src/main/java/io/grpc/Metadata.java | 12 ------------ .../io/grpc/netty/GrpcHttp2HeadersUtils.java | 5 ----- .../io/grpc/netty/NettyServerHandler.java | 7 ------- .../io/grpc/netty/NettyServerHandlerTest.java | 19 ------------------- 4 files changed, 43 deletions(-) diff --git a/api/src/main/java/io/grpc/Metadata.java b/api/src/main/java/io/grpc/Metadata.java index 9c2a2227f8c..e153fd55691 100644 --- a/api/src/main/java/io/grpc/Metadata.java +++ b/api/src/main/java/io/grpc/Metadata.java @@ -40,8 +40,6 @@ import java.util.Map; import java.util.NoSuchElementException; import java.util.Set; -import java.util.logging.Level; -import java.util.logging.Logger; import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; import javax.annotation.concurrent.NotThreadSafe; @@ -56,7 +54,6 @@ */ @NotThreadSafe public final class Metadata { - private static final Logger logger = Logger.getLogger(Metadata.class.getName()); /** * All binary headers should have this suffix in their names. Vice versa. @@ -736,15 +733,6 @@ private static BitSet generateValidTChars() { private static String validateName(String n, boolean pseudo) { checkNotNull(n, "name"); checkArgument(!n.isEmpty(), "token must have at least 1 tchar"); - if (n.equals("connection")) { - logger.log( - Level.WARNING, - "Metadata key is 'Connection', which should not be used. That is used by HTTP/1 for " - + "connection-specific headers which are not to be forwarded. There is probably an " - + "HTTP/1 conversion bug. Simply removing the Connection header is not enough; you " - + "should remove all headers it references as well. See RFC 7230 section 6.1", - new RuntimeException("exception to show backtrace")); - } for (int i = 0; i < n.length(); i++) { char tChar = n.charAt(i); if (pseudo && tChar == ':' && i == 0) { diff --git a/netty/src/main/java/io/grpc/netty/GrpcHttp2HeadersUtils.java b/netty/src/main/java/io/grpc/netty/GrpcHttp2HeadersUtils.java index df7875fc7ae..70c0a6f041a 100644 --- a/netty/src/main/java/io/grpc/netty/GrpcHttp2HeadersUtils.java +++ b/netty/src/main/java/io/grpc/netty/GrpcHttp2HeadersUtils.java @@ -145,11 +145,6 @@ protected CharSequence get(AsciiString name) { return null; } - @Override - public boolean contains(CharSequence name) { - return get(name) != null; - } - @Override public CharSequence status() { return get(Http2Headers.PseudoHeaderName.STATUS.value()); diff --git a/netty/src/main/java/io/grpc/netty/NettyServerHandler.java b/netty/src/main/java/io/grpc/netty/NettyServerHandler.java index c286c17f640..c9e0762d292 100644 --- a/netty/src/main/java/io/grpc/netty/NettyServerHandler.java +++ b/netty/src/main/java/io/grpc/netty/NettyServerHandler.java @@ -26,7 +26,6 @@ import static io.grpc.netty.Utils.HTTP_METHOD; import static io.grpc.netty.Utils.TE_HEADER; import static io.grpc.netty.Utils.TE_TRAILERS; -import static io.netty.handler.codec.http.HttpHeaderNames.CONNECTION; import static io.netty.handler.codec.http.HttpHeaderNames.HOST; import static io.netty.handler.codec.http2.DefaultHttp2LocalFlowController.DEFAULT_WINDOW_UPDATE_RATIO; import static io.netty.handler.codec.http2.Http2Headers.PseudoHeaderName.AUTHORITY; @@ -378,12 +377,6 @@ public void run() { private void onHeadersRead(ChannelHandlerContext ctx, int streamId, Http2Headers headers) throws Http2Exception { try { - // Connection-specific header fields makes a request malformed. Ideally this would be handled - // by Netty. RFC 7540 section 8.1.2.2 - if (headers.contains(CONNECTION)) { - resetStream(ctx, streamId, Http2Error.PROTOCOL_ERROR.code(), ctx.newPromise()); - return; - } if (headers.authority() == null) { List hosts = headers.getAll(HOST); diff --git a/netty/src/test/java/io/grpc/netty/NettyServerHandlerTest.java b/netty/src/test/java/io/grpc/netty/NettyServerHandlerTest.java index 170273e2c60..5db8413f60b 100644 --- a/netty/src/test/java/io/grpc/netty/NettyServerHandlerTest.java +++ b/netty/src/test/java/io/grpc/netty/NettyServerHandlerTest.java @@ -587,25 +587,6 @@ public void headersSupportExtensionContentType() throws Exception { stream = streamCaptor.getValue(); } - @Test - public void headersWithConnectionHeaderShouldFail() throws Exception { - manualSetUp(); - Http2Headers headers = new DefaultHttp2Headers() - .method(HTTP_METHOD) - .set(CONTENT_TYPE_HEADER, CONTENT_TYPE_GRPC) - .set(AsciiString.of("connection"), CONTENT_TYPE_GRPC) - .path(new AsciiString("/foo/bar")); - ByteBuf headersFrame = headersFrame(STREAM_ID, headers); - channelRead(headersFrame); - - verifyWrite() - .writeRstStream( - eq(ctx()), - eq(STREAM_ID), - eq(Http2Error.PROTOCOL_ERROR.code()), - any(ChannelPromise.class)); - } - @Test public void headersWithMultipleHostsShouldFail() throws Exception { manualSetUp(); From 49842d2af14d1b3c0ed4dc6a908823628aca181c Mon Sep 17 00:00:00 2001 From: sanjaypujare Date: Wed, 15 Sep 2021 15:46:22 -0700 Subject: [PATCH 20/76] xds: add hashCode and equals back to SslContextProviderSupplier (#8528) --- .../sds/SslContextProviderSupplier.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/xds/src/main/java/io/grpc/xds/internal/sds/SslContextProviderSupplier.java b/xds/src/main/java/io/grpc/xds/internal/sds/SslContextProviderSupplier.java index 17fc442e7da..664b4881bc2 100644 --- a/xds/src/main/java/io/grpc/xds/internal/sds/SslContextProviderSupplier.java +++ b/xds/src/main/java/io/grpc/xds/internal/sds/SslContextProviderSupplier.java @@ -25,6 +25,7 @@ import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; import io.grpc.xds.TlsContextManager; import io.netty.handler.ssl.SslContext; +import java.util.Objects; /** * Enables Client or server side to initialize this object with the received {@link BaseTlsContext} @@ -118,6 +119,24 @@ public synchronized void close() { shutdown = true; } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + SslContextProviderSupplier that = (SslContextProviderSupplier) o; + return Objects.equals(tlsContext, that.tlsContext) + && Objects.equals(tlsContextManager, that.tlsContextManager); + } + + @Override + public int hashCode() { + return Objects.hash(tlsContext, tlsContextManager); + } + @Override public String toString() { return MoreObjects.toStringHelper(this) From 9d9d8ec66b50d855682fd774359259a0f0025239 Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Thu, 16 Sep 2021 12:09:15 -0700 Subject: [PATCH 21/76] xds: Fix test compilation for confused javac The internal build fails with "reference to assertThat is ambiguous". It isn't clear why the internal build fails while the external one is okay, but it is clear that the wildcard T return of readOutbound() is probably confusing things as javac is considering assertThat(BigDecimal) as a possible match. The T return type is a hidden, convenience cast. We force the type passed to assertThat() to be Object to avoid any ambiguity. --- .../grpc/xds/FilterChainMatchingProtocolNegotiatorsTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/xds/src/test/java/io/grpc/xds/FilterChainMatchingProtocolNegotiatorsTest.java b/xds/src/test/java/io/grpc/xds/FilterChainMatchingProtocolNegotiatorsTest.java index 891dec322c0..5b7b1bf3681 100644 --- a/xds/src/test/java/io/grpc/xds/FilterChainMatchingProtocolNegotiatorsTest.java +++ b/xds/src/test/java/io/grpc/xds/FilterChainMatchingProtocolNegotiatorsTest.java @@ -170,7 +170,9 @@ public void filterSelectorChange_drainsConnection() { channel.runPendingTasks(); channelHandlerCtx = pipeline.context(next); assertThat(channelHandlerCtx).isNotNull(); - assertThat(channel.readOutbound()).isNull(); + // Force return value to Object, to avoid confusing javac of the type passed to assertThat() + Object msg = channel.readOutbound(); + assertThat(msg).isNull(); selectorManager.updateSelector(new FilterChainSelector( new HashMap(), null, noopConfig)); From fcf13952bb2e2e5e18c20e71735ed0ffd894a2e1 Mon Sep 17 00:00:00 2001 From: yifeizhuang Date: Thu, 16 Sep 2021 12:35:09 -0700 Subject: [PATCH 22/76] xds, rbac: build per route serverInterceptor for httpConfig (#8524) --- ...ilterChainMatchingProtocolNegotiators.java | 24 +- .../java/io/grpc/xds/XdsServerWrapper.java | 209 ++++++---- ...rChainMatchingProtocolNegotiatorsTest.java | 80 ++-- .../xds/FilterChainSelectorManagerTest.java | 15 +- .../io/grpc/xds/XdsServerWrapperTest.java | 383 +++++++++++------- 5 files changed, 414 insertions(+), 297 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/FilterChainMatchingProtocolNegotiators.java b/xds/src/main/java/io/grpc/xds/FilterChainMatchingProtocolNegotiators.java index 24cd4e9ae7e..b828b862454 100644 --- a/xds/src/main/java/io/grpc/xds/FilterChainMatchingProtocolNegotiators.java +++ b/xds/src/main/java/io/grpc/xds/FilterChainMatchingProtocolNegotiators.java @@ -59,6 +59,7 @@ import java.util.Map; import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Level; import java.util.logging.Logger; import javax.annotation.Nullable; @@ -135,28 +136,29 @@ public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exc static final class FilterChainSelector { public static final FilterChainSelector NO_FILTER_CHAIN = new FilterChainSelector( - Collections.emptyMap(), null, null); - private final Map routingConfigs; + Collections.>emptyMap(), + null, new AtomicReference()); + private final Map> routingConfigs; @Nullable private final SslContextProviderSupplier defaultSslContextProviderSupplier; @Nullable - private final ServerRoutingConfig defaultRoutingConfig; + private final AtomicReference defaultRoutingConfig; - FilterChainSelector(Map routingConfigs, + FilterChainSelector(Map> routingConfigs, @Nullable SslContextProviderSupplier defaultSslContextProviderSupplier, - @Nullable ServerRoutingConfig defaultRoutingConfig) { + @Nullable AtomicReference defaultRoutingConfig) { this.routingConfigs = checkNotNull(routingConfigs, "routingConfigs"); this.defaultSslContextProviderSupplier = defaultSslContextProviderSupplier; - this.defaultRoutingConfig = defaultRoutingConfig; + this.defaultRoutingConfig = checkNotNull(defaultRoutingConfig, "defaultRoutingConfig"); } @VisibleForTesting - Map getRoutingConfigs() { + Map> getRoutingConfigs() { return routingConfigs; } @VisibleForTesting - ServerRoutingConfig getDefaultRoutingConfig() { + AtomicReference getDefaultRoutingConfig() { return defaultRoutingConfig; } @@ -189,7 +191,7 @@ SelectedConfig select(InetSocketAddress localAddr, InetSocketAddress remoteAddr) return new SelectedConfig( routingConfigs.get(selected), selected.getSslContextProviderSupplier()); } - if (defaultRoutingConfig != null) { + if (defaultRoutingConfig.get() != null) { return new SelectedConfig(defaultRoutingConfig, defaultSslContextProviderSupplier); } return null; @@ -393,11 +395,11 @@ public void close() { * The FilterChain level configuration. */ private static final class SelectedConfig { - private final ServerRoutingConfig routingConfig; + private final AtomicReference routingConfig; @Nullable private final SslContextProviderSupplier sslContextProviderSupplier; - private SelectedConfig(ServerRoutingConfig routingConfig, + private SelectedConfig(AtomicReference routingConfig, @Nullable SslContextProviderSupplier sslContextProviderSupplier) { this.routingConfig = checkNotNull(routingConfig, "routingConfig"); this.sslContextProviderSupplier = sslContextProviderSupplier; diff --git a/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java b/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java index 29821f2cba8..e7301500e0e 100644 --- a/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java +++ b/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java @@ -22,6 +22,7 @@ import com.google.auto.value.AutoValue; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.SettableFuture; import io.grpc.Attributes; import io.grpc.InternalServerInterceptors; @@ -87,8 +88,9 @@ public void uncaughtException(Thread t, Throwable e) { } }); - public static final Attributes.Key ATTR_SERVER_ROUTING_CONFIG = - Attributes.Key.create("io.grpc.xds.ServerWrapper.serverRoutingConfig"); + public static final Attributes.Key> + ATTR_SERVER_ROUTING_CONFIG = + Attributes.Key.create("io.grpc.xds.ServerWrapper.serverRoutingConfig"); @VisibleForTesting static final long RETRY_DELAY_NANOS = TimeUnit.MINUTES.toNanos(1); @@ -346,6 +348,15 @@ private final class DiscoveryState implements LdsResourceWatcher { @Nullable private FilterChain defaultFilterChain; private boolean stopped; + private final Map> savedRdsRoutingConfigRef + = new HashMap<>(); + private final ServerInterceptor noopInterceptor = new ServerInterceptor() { + @Override + public Listener interceptCall(ServerCall call, + Metadata headers, ServerCallHandler next) { + return next.startCall(call, headers); + } + }; private DiscoveryState(String resourceName) { this.resourceName = checkNotNull(resourceName, "resourceName"); @@ -452,14 +463,16 @@ private void shutdown() { } private void updateSelector() { - Map filterChainRouting = new HashMap<>(); + Map> filterChainRouting = new HashMap<>(); + savedRdsRoutingConfigRef.clear(); for (FilterChain filterChain: filterChains) { filterChainRouting.put(filterChain, generateRoutingConfig(filterChain)); } FilterChainSelector selector = new FilterChainSelector( Collections.unmodifiableMap(filterChainRouting), defaultFilterChain == null ? null : defaultFilterChain.getSslContextProviderSupplier(), - defaultFilterChain == null ? null : generateRoutingConfig(defaultFilterChain)); + defaultFilterChain == null ? new AtomicReference() : + generateRoutingConfig(defaultFilterChain)); List toRelease = getSuppliersInUse(); filterChainSelectorManager.updateSelector(selector); for (SslContextProviderSupplier e: toRelease) { @@ -468,18 +481,84 @@ private void updateSelector() { startDelegateServer(); } - private ServerRoutingConfig generateRoutingConfig(FilterChain filterChain) { + private AtomicReference generateRoutingConfig(FilterChain filterChain) { HttpConnectionManager hcm = filterChain.getHttpConnectionManager(); if (hcm.virtualHosts() != null) { - return ServerRoutingConfig.create(hcm.httpFilterConfigs(), - new AtomicReference<>(hcm.virtualHosts())); + ImmutableMap interceptors = generatePerRouteInterceptors( + hcm.httpFilterConfigs(), hcm.virtualHosts()); + return new AtomicReference<>(ServerRoutingConfig.create(hcm.virtualHosts(),interceptors)); } else { RouteDiscoveryState rds = routeDiscoveryStates.get(hcm.rdsName()); checkNotNull(rds, "rds"); - return ServerRoutingConfig.create(hcm.httpFilterConfigs(), rds.savedVirtualHosts); + AtomicReference serverRoutingConfigRef = new AtomicReference<>(); + if (rds.savedVirtualHosts != null) { + ImmutableMap interceptors = generatePerRouteInterceptors( + hcm.httpFilterConfigs(), rds.savedVirtualHosts); + ServerRoutingConfig serverRoutingConfig = + ServerRoutingConfig.create(rds.savedVirtualHosts, interceptors); + serverRoutingConfigRef.set(serverRoutingConfig); + } else { + serverRoutingConfigRef.set(ServerRoutingConfig.FAILING_ROUTING_CONFIG); + } + savedRdsRoutingConfigRef.put(filterChain, serverRoutingConfigRef); + return serverRoutingConfigRef; } } + private ImmutableMap generatePerRouteInterceptors( + List namedFilterConfigs, List virtualHosts) { + ImmutableMap.Builder perRouteInterceptors = + new ImmutableMap.Builder<>(); + for (VirtualHost virtualHost : virtualHosts) { + for (Route route : virtualHost.routes()) { + List filterInterceptors = new ArrayList<>(); + Map selectedOverrideConfigs = + new HashMap<>(virtualHost.filterConfigOverrides()); + selectedOverrideConfigs.putAll(route.filterConfigOverrides()); + for (NamedFilterConfig namedFilterConfig : namedFilterConfigs) { + FilterConfig filterConfig = namedFilterConfig.filterConfig; + Filter filter = filterRegistry.get(filterConfig.typeUrl()); + if (filter instanceof ServerInterceptorBuilder) { + ServerInterceptor interceptor = + ((ServerInterceptorBuilder) filter).buildServerInterceptor( + filterConfig, selectedOverrideConfigs.get(namedFilterConfig.name)); + if (interceptor != null) { + filterInterceptors.add(interceptor); + } + } else { + logger.log(Level.WARNING, "HttpFilterConfig(type URL: " + + filterConfig.typeUrl() + ") is not supported on server-side. " + + "Probably a bug at ClientXdsClient verification."); + } + } + ServerInterceptor interceptor = combineInterceptors(filterInterceptors); + perRouteInterceptors.put(route, interceptor); + } + } + return perRouteInterceptors.build(); + } + + private ServerInterceptor combineInterceptors(final List interceptors) { + if (interceptors.isEmpty()) { + return noopInterceptor; + } + if (interceptors.size() == 1) { + return interceptors.get(0); + } + return new ServerInterceptor() { + @Override + public Listener interceptCall(ServerCall call, + Metadata headers, ServerCallHandler next) { + // intercept forward + for (int i = interceptors.size() - 1; i >= 0; i--) { + next = InternalServerInterceptors.interceptCallHandlerCreate( + interceptors.get(i), next); + } + return next.startCall(call, headers); + } + }; + } + private void handleConfigNotFound(StatusException exception) { cleanUpRouteDiscoveryStates(); List toRelease = getSuppliersInUse(); @@ -508,6 +587,7 @@ private void cleanUpRouteDiscoveryStates() { xdsClient.cancelRdsResourceWatch(rdsName, rdsState); } routeDiscoveryStates.clear(); + savedRdsRoutingConfigRef.clear(); } private List getSuppliersInUse() { @@ -544,8 +624,7 @@ private void releaseSuppliersInFlight() { private final class RouteDiscoveryState implements RdsResourceWatcher { private final String resourceName; - private AtomicReference> savedVirtualHosts = - new AtomicReference<>(); + private ImmutableList savedVirtualHosts; private boolean isPending = true; private RouteDiscoveryState(String resourceName) { @@ -560,7 +639,8 @@ public void run() { if (!routeDiscoveryStates.containsKey(resourceName)) { return; } - savedVirtualHosts.set(ImmutableList.copyOf(update.virtualHosts)); + savedVirtualHosts = ImmutableList.copyOf(update.virtualHosts); + updateRdsRoutingConfig(); maybeUpdateSelector(); } }); @@ -575,7 +655,8 @@ public void run() { return; } logger.log(Level.WARNING, "Rds {0} unavailable", resourceName); - savedVirtualHosts.set(null); + savedVirtualHosts = null; + updateRdsRoutingConfig(); maybeUpdateSelector(); } }); @@ -596,6 +677,25 @@ public void run() { }); } + private void updateRdsRoutingConfig() { + for (FilterChain filterChain : savedRdsRoutingConfigRef.keySet()) { + if (resourceName.equals(filterChain.getHttpConnectionManager().rdsName())) { + ServerRoutingConfig updatedRoutingConfig; + if (savedVirtualHosts == null) { + updatedRoutingConfig = ServerRoutingConfig.FAILING_ROUTING_CONFIG; + } else { + ImmutableMap updatedInterceptors = + generatePerRouteInterceptors( + filterChain.getHttpConnectionManager().httpFilterConfigs(), + savedVirtualHosts); + updatedRoutingConfig = ServerRoutingConfig.create(savedVirtualHosts, + updatedInterceptors); + } + savedRdsRoutingConfigRef.get(filterChain).set(updatedRoutingConfig); + } + } + } + // Update the selector to use the most recently updated configs only after all rds have been // discovered for the first time. Later changes on rds will be applied through virtual host // list atomic ref. @@ -632,18 +732,16 @@ public Listener interceptCall(ServerCall call, @Override public Listener interceptCall(ServerCall call, Metadata headers, ServerCallHandler next) { - ServerRoutingConfig routingConfig = call.getAttributes().get(ATTR_SERVER_ROUTING_CONFIG); - if (routingConfig == null) { - String errorMsg = "Missing xDS routing config."; - call.close(Status.UNAVAILABLE.withDescription(errorMsg), new Metadata()); - return new Listener() {}; - } - List virtualHosts = routingConfig.virtualHosts().get(); - if (virtualHosts == null) { - String errorMsg = "Missing xDS routing config VirtualHosts due to RDS config unavailable."; + AtomicReference routingConfigRef = + call.getAttributes().get(ATTR_SERVER_ROUTING_CONFIG); + ServerRoutingConfig routingConfig = routingConfigRef == null ? null : + routingConfigRef.get(); + if (routingConfig == null || routingConfig == ServerRoutingConfig.FAILING_ROUTING_CONFIG) { + String errorMsg = "Missing or broken xDS routing config: RDS config unavailable."; call.close(Status.UNAVAILABLE.withDescription(errorMsg), new Metadata()); return new Listener() {}; } + List virtualHosts = routingConfig.virtualHosts(); VirtualHost virtualHost = RoutingUtils.findVirtualHostForHostName( virtualHosts, call.getAuthority()); if (virtualHost == null) { @@ -653,14 +751,11 @@ public Listener interceptCall(ServerCall call, return new Listener() {}; } Route selectedRoute = null; - Map selectedOverrideConfigs = - new HashMap<>(virtualHost.filterConfigOverrides()); MethodDescriptor method = call.getMethodDescriptor(); for (Route route : virtualHost.routes()) { if (RoutingUtils.matchRoute( route.routeMatch(), "/" + method.getFullMethodName(), headers, random)) { selectedRoute = route; - selectedOverrideConfigs.putAll(route.filterConfigOverrides()); break; } } @@ -670,48 +765,12 @@ public Listener interceptCall(ServerCall call, new Metadata()); return new ServerCall.Listener() {}; } - List filterInterceptors = new ArrayList<>(); - for (NamedFilterConfig namedFilterConfig : routingConfig.httpFilterConfigs()) { - FilterConfig filterConfig = namedFilterConfig.filterConfig; - Filter filter = filterRegistry.get(filterConfig.typeUrl()); - if (filter instanceof ServerInterceptorBuilder) { - ServerInterceptor interceptor = - ((ServerInterceptorBuilder) filter).buildServerInterceptor( - filterConfig, selectedOverrideConfigs.get(namedFilterConfig.name)); - if (interceptor != null) { - filterInterceptors.add(interceptor); - } - } else { - call.close( - Status.UNAVAILABLE.withDescription("HttpFilterConfig(type URL: " - + filterConfig.typeUrl() + ") is not supported on server-side."), - new Metadata()); - return new Listener() {}; - } + ServerInterceptor routeInterceptor = noopInterceptor; + Map perRouteInterceptors = routingConfig.interceptors(); + if (perRouteInterceptors != null && perRouteInterceptors.get(selectedRoute) != null) { + routeInterceptor = perRouteInterceptors.get(selectedRoute); } - ServerInterceptor interceptor = combineInterceptors(filterInterceptors); - return interceptor.interceptCall(call, headers, next); - } - - private ServerInterceptor combineInterceptors(final List interceptors) { - if (interceptors.isEmpty()) { - return noopInterceptor; - } - if (interceptors.size() == 1) { - return interceptors.get(0); - } - return new ServerInterceptor() { - @Override - public Listener interceptCall(ServerCall call, - Metadata headers, ServerCallHandler next) { - // intercept forward - for (int i = interceptors.size() - 1; i >= 0; i--) { - next = InternalServerInterceptors.interceptCallHandlerCreate( - interceptors.get(i), next); - } - return next.startCall(call, headers); - } - }; + return routeInterceptor.interceptCall(call, headers, next); } } @@ -720,20 +779,24 @@ public Listener interceptCall(ServerCall call, */ @AutoValue abstract static class ServerRoutingConfig { - // Top level http filter configs. - abstract ImmutableList httpFilterConfigs(); + @VisibleForTesting + static final ServerRoutingConfig FAILING_ROUTING_CONFIG = ServerRoutingConfig.create( + ImmutableList.of(), ImmutableMap.of()); + + abstract ImmutableList virtualHosts(); - abstract AtomicReference> virtualHosts(); + // Prebuilt per route server interceptors from http filter configs. + abstract ImmutableMap interceptors(); /** * Server routing configuration. * */ - public static ServerRoutingConfig create(List httpFilterConfigs, - AtomicReference> virtualHosts) { - checkNotNull(httpFilterConfigs, "httpFilterConfigs"); + public static ServerRoutingConfig create( + ImmutableList virtualHosts, + ImmutableMap interceptors) { checkNotNull(virtualHosts, "virtualHosts"); - return new AutoValue_XdsServerWrapper_ServerRoutingConfig( - ImmutableList.copyOf(httpFilterConfigs), virtualHosts); + checkNotNull(interceptors, "interceptors"); + return new AutoValue_XdsServerWrapper_ServerRoutingConfig(virtualHosts, interceptors); } } } diff --git a/xds/src/test/java/io/grpc/xds/FilterChainMatchingProtocolNegotiatorsTest.java b/xds/src/test/java/io/grpc/xds/FilterChainMatchingProtocolNegotiatorsTest.java index 5b7b1bf3681..32a7bc19b32 100644 --- a/xds/src/test/java/io/grpc/xds/FilterChainMatchingProtocolNegotiatorsTest.java +++ b/xds/src/test/java/io/grpc/xds/FilterChainMatchingProtocolNegotiatorsTest.java @@ -26,6 +26,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.SettableFuture; +import io.grpc.ServerInterceptor; import io.grpc.internal.TestUtils.NoopChannelLogger; import io.grpc.netty.GrpcHttp2ConnectionHandler; import io.grpc.netty.InternalProtocolNegotiationEvent; @@ -93,8 +94,12 @@ public class FilterChainMatchingProtocolNegotiatorsTest { private static final String LOCAL_IP = "10.1.2.3"; // dest private static final String REMOTE_IP = "10.4.2.3"; // source private static final int PORT = 7000; - private final ServerRoutingConfig noopConfig = ServerRoutingConfig.create( - new ArrayList(), new AtomicReference>()); + private final AtomicReference noopConfig = new AtomicReference<>( + ServerRoutingConfig.create(ImmutableList.of(), + ImmutableMap.of())); + final SettableFuture sslSet = SettableFuture.create(); + final SettableFuture> routingSettable = + SettableFuture.create(); @After @SuppressWarnings("FutureReturnValueIgnored") @@ -108,15 +113,14 @@ public void tearDown() { @Test public void nofilterChainMatch_defaultSslContext() throws Exception { - final SettableFuture sslSet = SettableFuture.create(); - final SettableFuture routingSettable = SettableFuture.create(); ChannelHandler next = captureAttrHandler(sslSet, routingSettable); when(mockDelegate.newHandler(grpcHandler)).thenReturn(next); SslContextProviderSupplier defaultSsl = new SslContextProviderSupplier(createTls(), tlsContextManager); selectorManager.updateSelector(new FilterChainSelector( - new HashMap(), defaultSsl, noopConfig)); + new HashMap>(), + defaultSsl, noopConfig)); FilterChainMatchingHandler filterChainMatchingHandler = new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); setupChannel("172.168.1.1", "172.168.1.2", 80, filterChainMatchingHandler); @@ -138,7 +142,8 @@ public void nofilterChainMatch_defaultSslContext() throws Exception { @Test public void noFilterChainMatch_noDefaultSslContext() { selectorManager.updateSelector(new FilterChainSelector( - new HashMap(), null, null)); + new HashMap>(), + null, new AtomicReference())); FilterChainMatchingHandler filterChainMatchingHandler = new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); setupChannel("172.168.1.1", "172.168.2.2", 90, filterChainMatchingHandler); @@ -156,7 +161,7 @@ public void filterSelectorChange_drainsConnection() { ChannelHandler next = new ChannelInboundHandlerAdapter(); when(mockDelegate.newHandler(grpcHandler)).thenReturn(next); selectorManager.updateSelector(new FilterChainSelector( - new HashMap(), null, noopConfig)); + new HashMap>(), null, noopConfig)); FilterChainMatchingHandler filterChainMatchingHandler = new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); setupChannel("172.168.1.1", "172.168.2.2", 90, filterChainMatchingHandler); @@ -175,7 +180,7 @@ public void filterSelectorChange_drainsConnection() { assertThat(msg).isNull(); selectorManager.updateSelector(new FilterChainSelector( - new HashMap(), null, noopConfig)); + new HashMap>(), null, noopConfig)); assertThat(channel.readOutbound().getClass().getName()) .isEqualTo("io.grpc.netty.GracefulServerCloseCommand"); } @@ -199,11 +204,9 @@ public void singleFilterChainWithoutAlpn() throws Exception { tlsContextManager); selectorManager.updateSelector(new FilterChainSelector(ImmutableMap.of(filterChain, noopConfig), - null, null)); + null, new AtomicReference())); FilterChainMatchingHandler filterChainMatchingHandler = new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); - final SettableFuture sslSet = SettableFuture.create(); - final SettableFuture routingSettable = SettableFuture.create(); ChannelHandler next = captureAttrHandler(sslSet, routingSettable); when(mockDelegate.newHandler(grpcHandler)).thenReturn(next); setupChannel(LOCAL_IP, REMOTE_IP, 15000, filterChainMatchingHandler); @@ -243,8 +246,6 @@ public void singleFilterChainWithAlpn() throws Exception { FilterChainMatchingHandler filterChainMatchingHandler = new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); - final SettableFuture sslSet = SettableFuture.create(); - final SettableFuture routingSettable = SettableFuture.create(); ChannelHandler next = captureAttrHandler(sslSet, routingSettable); when(mockDelegate.newHandler(grpcHandler)).thenReturn(next); setupChannel(LOCAL_IP, REMOTE_IP, 15000, filterChainMatchingHandler); @@ -281,17 +282,16 @@ public void destPortFails_returnDefaultFilterChain() throws Exception { tlsContextForDefaultFilterChain, tlsContextManager); ServerRoutingConfig routingConfig = ServerRoutingConfig.create( - new ArrayList(), new AtomicReference<>( - ImmutableList.of(createVirtualHost("virtual")))); + ImmutableList.of(createVirtualHost("virtual")), + ImmutableMap.of()); selectorManager.updateSelector(new FilterChainSelector( - ImmutableMap.of(filterChainWithDestPort, routingConfig), + ImmutableMap.of(filterChainWithDestPort, + new AtomicReference(routingConfig)), defaultFilterChain.getSslContextProviderSupplier(), noopConfig)); FilterChainMatchingHandler filterChainMatchingHandler = new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); - final SettableFuture sslSet = SettableFuture.create(); - final SettableFuture routingSettable = SettableFuture.create(); ChannelHandler next = captureAttrHandler(sslSet, routingSettable); when(mockDelegate.newHandler(grpcHandler)).thenReturn(next); setupChannel(LOCAL_IP, REMOTE_IP, 15000, filterChainMatchingHandler); @@ -333,8 +333,6 @@ public void destPrefixRangeMatch() throws Exception { FilterChainMatchingHandler filterChainMatchingHandler = new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); - final SettableFuture sslSet = SettableFuture.create(); - final SettableFuture routingSettable = SettableFuture.create(); ChannelHandler next = captureAttrHandler(sslSet, routingSettable); when(mockDelegate.newHandler(grpcHandler)).thenReturn(next); setupChannel(LOCAL_IP, REMOTE_IP, 15000, filterChainMatchingHandler); @@ -377,8 +375,6 @@ public void destPrefixRangeMismatch_returnDefaultFilterChain() FilterChainMatchingHandler filterChainMatchingHandler = new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); - final SettableFuture sslSet = SettableFuture.create(); - final SettableFuture routingSettable = SettableFuture.create(); ChannelHandler next = captureAttrHandler(sslSet, routingSettable); when(mockDelegate.newHandler(grpcHandler)).thenReturn(next); setupChannel(LOCAL_IP, REMOTE_IP, 15000, filterChainMatchingHandler); @@ -417,12 +413,11 @@ public void dest0LengthPrefixRange() selectorManager.updateSelector(new FilterChainSelector( ImmutableMap.of(filterChain0Length, noopConfig), - defaultFilterChain.getSslContextProviderSupplier(), null)); + defaultFilterChain.getSslContextProviderSupplier(), + new AtomicReference())); FilterChainMatchingHandler filterChainMatchingHandler = new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); - final SettableFuture sslSet = SettableFuture.create(); - final SettableFuture routingSettable = SettableFuture.create(); ChannelHandler next = captureAttrHandler(sslSet, routingSettable); when(mockDelegate.newHandler(grpcHandler)).thenReturn(next); setupChannel(LOCAL_IP, REMOTE_IP, 15000, filterChainMatchingHandler); @@ -480,8 +475,6 @@ public void destPrefixRange_moreSpecificWins() FilterChainMatchingHandler filterChainMatchingHandler = new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); - final SettableFuture sslSet = SettableFuture.create(); - final SettableFuture routingSettable = SettableFuture.create(); ChannelHandler next = captureAttrHandler(sslSet, routingSettable); when(mockDelegate.newHandler(grpcHandler)).thenReturn(next); setupChannel(LOCAL_IP, REMOTE_IP, 15000, filterChainMatchingHandler); @@ -538,8 +531,6 @@ public void destPrefixRange_emptyListLessSpecific() FilterChainMatchingHandler filterChainMatchingHandler = new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); - final SettableFuture sslSet = SettableFuture.create(); - final SettableFuture routingSettable = SettableFuture.create(); ChannelHandler next = captureAttrHandler(sslSet, routingSettable); when(mockDelegate.newHandler(grpcHandler)).thenReturn(next); setupChannel(LOCAL_IP, REMOTE_IP, 15000, filterChainMatchingHandler); @@ -596,8 +587,6 @@ public void destPrefixRangeIpv6_moreSpecificWins() FilterChainMatchingHandler filterChainMatchingHandler = new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); - final SettableFuture sslSet = SettableFuture.create(); - final SettableFuture routingSettable = SettableFuture.create(); ChannelHandler next = captureAttrHandler(sslSet, routingSettable); when(mockDelegate.newHandler(grpcHandler)).thenReturn(next); @@ -658,8 +647,6 @@ filterChainLessSpecific, randomConfig("no-match")), FilterChainMatchingHandler filterChainMatchingHandler = new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); - final SettableFuture sslSet = SettableFuture.create(); - final SettableFuture routingSettable = SettableFuture.create(); ChannelHandler next = captureAttrHandler(sslSet, routingSettable); when(mockDelegate.newHandler(grpcHandler)).thenReturn(next); setupChannel(LOCAL_IP, REMOTE_IP, 15000, filterChainMatchingHandler); @@ -700,8 +687,7 @@ public void sourceTypeMismatch_returnDefaultFilterChain() throws Exception { FilterChainMatchingHandler filterChainMatchingHandler = new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); - final SettableFuture sslSet = SettableFuture.create(); - final SettableFuture routingSettable = SettableFuture.create(); + ChannelHandler next = captureAttrHandler(sslSet, routingSettable); when(mockDelegate.newHandler(grpcHandler)).thenReturn(next); setupChannel(LOCAL_IP, REMOTE_IP, 15000, filterChainMatchingHandler); @@ -714,8 +700,6 @@ public void sourceTypeMismatch_returnDefaultFilterChain() throws Exception { @Test public void sourceTypeLocal() throws Exception { - final SettableFuture sslSet = SettableFuture.create(); - final SettableFuture routingSettable = SettableFuture.create(); ChannelHandler next = captureAttrHandler(sslSet, routingSettable); when(mockDelegate.newHandler(grpcHandler)).thenReturn(next); EnvoyServerProtoData.DownstreamTlsContext tlsContextMatch = @@ -755,8 +739,6 @@ public void sourceTypeLocal() throws Exception { @Test public void sourcePrefixRange_moreSpecificWith2Wins() throws Exception { - final SettableFuture sslSet = SettableFuture.create(); - final SettableFuture routingSettable = SettableFuture.create(); ChannelHandler next = captureAttrHandler(sslSet, routingSettable); when(mockDelegate.newHandler(grpcHandler)).thenReturn(next); @@ -817,7 +799,6 @@ filterChainLessSpecific, randomConfig("no-match")), @Test public void sourcePrefixRange_2Matchers_expectException() throws UnknownHostException { - final SettableFuture sslSet = SettableFuture.create(); ChannelHandler next = new ChannelInboundHandlerAdapter() { @Override public void userEventTriggered(ChannelHandlerContext ctx, Object evt) { @@ -932,8 +913,6 @@ public void sourcePortMatch_exactMatchWinsOverEmptyList() throws Exception { FilterChainMatchingHandler filterChainMatchingHandler = new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); - final SettableFuture sslSet = SettableFuture.create(); - final SettableFuture routingSettable = SettableFuture.create(); ChannelHandler next = captureAttrHandler(sslSet, routingSettable); when(mockDelegate.newHandler(grpcHandler)).thenReturn(next); setupChannel(LOCAL_IP, REMOTE_IP, 15000, filterChainMatchingHandler); @@ -1074,7 +1053,7 @@ public void filterChain_5stepMatch() throws Exception { EnvoyServerProtoData.FilterChain defaultFilterChain = new EnvoyServerProtoData.FilterChain( "filter-chain-7", null, HTTP_CONNECTION_MANAGER, null, tlsContextManager); - Map map = new HashMap<>(); + Map> map = new HashMap<>(); map.put(filterChain1, randomConfig("1")); map.put(filterChain2, randomConfig("2")); map.put(filterChain3, randomConfig("3")); @@ -1087,8 +1066,6 @@ public void filterChain_5stepMatch() throws Exception { FilterChainMatchingHandler filterChainMatchingHandler = new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); - final SettableFuture sslSet = SettableFuture.create(); - final SettableFuture routingSettable = SettableFuture.create(); ChannelHandler next = captureAttrHandler(sslSet, routingSettable); when(mockDelegate.newHandler(grpcHandler)).thenReturn(next); setupChannel(LOCAL_IP, REMOTE_IP, 15000, filterChainMatchingHandler); @@ -1161,8 +1138,6 @@ public void filterChainMatch_unsupportedMatchers() throws Exception { FilterChainMatchingHandler filterChainMatchingHandler = new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate); - final SettableFuture sslSet = SettableFuture.create(); - final SettableFuture routingSettable = SettableFuture.create(); ChannelHandler next = captureAttrHandler(sslSet, routingSettable); when(mockDelegate.newHandler(grpcHandler)).thenReturn(next); setupChannel(LOCAL_IP, REMOTE_IP, 15000, filterChainMatchingHandler); @@ -1186,10 +1161,11 @@ private static VirtualHost createVirtualHost(String name) { ImmutableMap.of()); } - private static ServerRoutingConfig randomConfig(String domain) { - return ServerRoutingConfig.create( - new ArrayList(), new AtomicReference<>( - ImmutableList.of(createVirtualHost(domain)))); + private static AtomicReference randomConfig(String domain) { + return new AtomicReference<>( + ServerRoutingConfig.create(ImmutableList.of(createVirtualHost(domain)), + ImmutableMap.of()) + ); } private EnvoyServerProtoData.DownstreamTlsContext createTls() { @@ -1218,7 +1194,7 @@ public SocketAddress remoteAddress() { private static ChannelHandler captureAttrHandler( final SettableFuture sslSet, - final SettableFuture routingSettable) { + final SettableFuture> routingSettable) { return new ChannelInboundHandlerAdapter() { @Override public void userEventTriggered(ChannelHandlerContext ctx, Object evt) { diff --git a/xds/src/test/java/io/grpc/xds/FilterChainSelectorManagerTest.java b/xds/src/test/java/io/grpc/xds/FilterChainSelectorManagerTest.java index d7b883f1941..a3a2218d4c3 100644 --- a/xds/src/test/java/io/grpc/xds/FilterChainSelectorManagerTest.java +++ b/xds/src/test/java/io/grpc/xds/FilterChainSelectorManagerTest.java @@ -19,8 +19,9 @@ import static com.google.common.truth.Truth.assertThat; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.grpc.ServerInterceptor; import io.grpc.xds.EnvoyServerProtoData.FilterChain; -import io.grpc.xds.Filter.NamedFilterConfig; import io.grpc.xds.FilterChainMatchingProtocolNegotiators.FilterChainMatchingHandler.FilterChainSelector; import io.grpc.xds.FilterChainSelectorManager.Closer; import io.grpc.xds.XdsServerWrapper.ServerRoutingConfig; @@ -33,13 +34,15 @@ @RunWith(JUnit4.class) public final class FilterChainSelectorManagerTest { private FilterChainSelectorManager manager = new FilterChainSelectorManager(); - private ServerRoutingConfig noopConfig = ServerRoutingConfig.create( - Collections.emptyList(), - new AtomicReference>()); + private AtomicReference noopConfig = new AtomicReference<>( + ServerRoutingConfig.create(ImmutableList.of(), + ImmutableMap.of())); private FilterChainSelector selector1 = new FilterChainSelector( - Collections.emptyMap(), null, null); + Collections.>emptyMap(), + null, new AtomicReference()); private FilterChainSelector selector2 = new FilterChainSelector( - Collections.emptyMap(), null, noopConfig); + Collections.>emptyMap(), + null, noopConfig); private CounterRunnable runnable1 = new CounterRunnable(); private CounterRunnable runnable2 = new CounterRunnable(); diff --git a/xds/src/test/java/io/grpc/xds/XdsServerWrapperTest.java b/xds/src/test/java/io/grpc/xds/XdsServerWrapperTest.java index d4421361158..c109bb44a13 100644 --- a/xds/src/test/java/io/grpc/xds/XdsServerWrapperTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsServerWrapperTest.java @@ -104,6 +104,8 @@ public class XdsServerWrapperTest { private FakeXdsClient xdsClient = new FakeXdsClient(); private FilterRegistry filterRegistry = FilterRegistry.getDefaultRegistry(); private XdsServerWrapper xdsServerWrapper; + private ServerRoutingConfig noopConfig = ServerRoutingConfig.create( + ImmutableList.of(), ImmutableMap.of()); @Before public void setup() { @@ -380,9 +382,9 @@ public void run() { assertThat(selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().size()) .isEqualTo(1); ServerRoutingConfig realConfig = - selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(filterChain); - assertThat(realConfig.virtualHosts().get()).isEqualTo(httpConnectionManager.virtualHosts()); - assertThat(realConfig.httpFilterConfigs()).isEqualTo(httpConnectionManager.httpFilterConfigs()); + selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(filterChain).get(); + assertThat(realConfig.virtualHosts()).isEqualTo(httpConnectionManager.virtualHosts()); + assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of()); verify(listener).onServing(); verify(mockServer).start(); } @@ -429,26 +431,21 @@ public void run() { start.get(5000, TimeUnit.MILLISECONDS); verify(mockServer).start(); ServerRoutingConfig realConfig = - selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f0); - assertThat(realConfig.virtualHosts().get()).isEqualTo( + selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f0).get(); + assertThat(realConfig.virtualHosts()).isEqualTo( Collections.singletonList(createVirtualHost("virtual-host-0"))); - assertThat(realConfig.httpFilterConfigs()).isEqualTo( - f0.getHttpConnectionManager().httpFilterConfigs()); + assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of()); assertThat(selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().size()) .isEqualTo(2); - realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f2); - assertThat(realConfig.virtualHosts().get()).isEqualTo( + realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f2).get(); + assertThat(realConfig.virtualHosts()).isEqualTo( Collections.singletonList(createVirtualHost("virtual-host-1"))); - assertThat(realConfig.httpFilterConfigs()).isEqualTo( - f2.getHttpConnectionManager().httpFilterConfigs()); - realConfig = selectorManager.getSelectorToUpdateSelector().getDefaultRoutingConfig(); - assertThat(realConfig.virtualHosts().get()).isEqualTo( + assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of()); + realConfig = selectorManager.getSelectorToUpdateSelector().getDefaultRoutingConfig().get(); + assertThat(realConfig.virtualHosts()).isEqualTo( Collections.singletonList(createVirtualHost("virtual-host-2"))); - assertThat(realConfig.httpFilterConfigs()).isEqualTo( - f3.getHttpConnectionManager().httpFilterConfigs()); assertThat(selectorManager.getSelectorToUpdateSelector().getDefaultSslContextProviderSupplier()) - .isEqualTo( - f3.getSslContextProviderSupplier()); + .isEqualTo(f3.getSslContextProviderSupplier()); } @Test @@ -481,22 +478,20 @@ public void run() { start.get(5000, TimeUnit.MILLISECONDS); verify(mockServer, times(1)).start(); ServerRoutingConfig realConfig = - selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f0); - assertThat(realConfig.virtualHosts().get()).isEqualTo( + selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f0).get(); + assertThat(realConfig.virtualHosts()).isEqualTo( Collections.singletonList(createVirtualHost("virtual-host-0"))); - assertThat(realConfig.httpFilterConfigs()).isEqualTo( - f0.getHttpConnectionManager().httpFilterConfigs()); - realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f1); - assertThat(realConfig.virtualHosts().get()).isEqualTo( + assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of()); + + realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f1).get(); + assertThat(realConfig.virtualHosts()).isEqualTo( Collections.singletonList(createVirtualHost("virtual-host-0"))); - assertThat(realConfig.httpFilterConfigs()).isEqualTo( - f1.getHttpConnectionManager().httpFilterConfigs()); + assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of()); - realConfig = selectorManager.getSelectorToUpdateSelector().getDefaultRoutingConfig(); - assertThat(realConfig.virtualHosts().get()).isEqualTo( + realConfig = selectorManager.getSelectorToUpdateSelector().getDefaultRoutingConfig().get(); + assertThat(realConfig.virtualHosts()).isEqualTo( Collections.singletonList(createVirtualHost("virtual-host-0"))); - assertThat(realConfig.httpFilterConfigs()).isEqualTo( - f2.getHttpConnectionManager().httpFilterConfigs()); + assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of()); assertThat(selectorManager.getSelectorToUpdateSelector().getDefaultSslContextProviderSupplier()) .isSameInstanceAs(f2.getSslContextProviderSupplier()); @@ -513,25 +508,22 @@ public void run() { assertThat(selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().size()) .isEqualTo(2); - realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f5); - assertThat(realConfig.virtualHosts().get()).isEqualTo( + realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f5).get(); + assertThat(realConfig.virtualHosts()).isEqualTo( Collections.singletonList(createVirtualHost("virtual-host-1"))); - assertThat(realConfig.httpFilterConfigs()).isEqualTo( - f5.getHttpConnectionManager().httpFilterConfigs()); - realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f3); - assertThat(realConfig.virtualHosts().get()).isEqualTo( + assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of()); + realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f3).get(); + assertThat(realConfig.virtualHosts()).isEqualTo( Collections.singletonList(createVirtualHost("virtual-host-0"))); - assertThat(realConfig.httpFilterConfigs()).isEqualTo( - f3.getHttpConnectionManager().httpFilterConfigs()); + assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of()); - realConfig = selectorManager.getSelectorToUpdateSelector().getDefaultRoutingConfig(); - assertThat(realConfig.virtualHosts().get()).isEqualTo( + realConfig = selectorManager.getSelectorToUpdateSelector().getDefaultRoutingConfig().get(); + assertThat(realConfig.virtualHosts()).isEqualTo( Collections.singletonList(createVirtualHost("virtual-host-1"))); - assertThat(realConfig.httpFilterConfigs()).isEqualTo( - f4.getHttpConnectionManager().httpFilterConfigs()); + assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of()); + assertThat(selectorManager.getSelectorToUpdateSelector().getDefaultSslContextProviderSupplier()) - .isSameInstanceAs( - f4.getSslContextProviderSupplier()); + .isSameInstanceAs(f4.getSslContextProviderSupplier()); verify(mockServer, times(1)).start(); xdsServerWrapper.shutdown(); verify(mockServer, times(1)).shutdown(); @@ -567,35 +559,31 @@ public void run() { assertThat(selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().size()) .isEqualTo(2); ServerRoutingConfig realConfig = - selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f1); - assertThat(realConfig.virtualHosts().get()).isNull(); - assertThat(realConfig.httpFilterConfigs()).isEqualTo( - f1.getHttpConnectionManager().httpFilterConfigs()); - realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f0); - assertThat(realConfig.virtualHosts().get()).isEqualTo(hcmVirtual.virtualHosts()); - assertThat(realConfig.httpFilterConfigs()).isEqualTo( - f0.getHttpConnectionManager().httpFilterConfigs()); + selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f1).get(); + assertThat(realConfig.virtualHosts()).isEmpty(); + assertThat(realConfig.interceptors()).isEmpty(); + + realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f0).get(); + assertThat(realConfig.virtualHosts()).isEqualTo(hcmVirtual.virtualHosts()); + assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of()); xdsClient.deliverRdsUpdate("r0", Collections.singletonList(createVirtualHost("virtual-host-1"))); - realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f1); - assertThat(realConfig.virtualHosts().get()).isEqualTo( + realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f1).get(); + assertThat(realConfig.virtualHosts()).isEqualTo( Collections.singletonList(createVirtualHost("virtual-host-1"))); - assertThat(realConfig.httpFilterConfigs()).isEqualTo( - f1.getHttpConnectionManager().httpFilterConfigs()); + assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of()); xdsClient.rdsWatchers.get("r0").onError(Status.CANCELLED); - realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f1); - assertThat(realConfig.virtualHosts().get()).isEqualTo( + realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f1).get(); + assertThat(realConfig.virtualHosts()).isEqualTo( Collections.singletonList(createVirtualHost("virtual-host-1"))); - assertThat(realConfig.httpFilterConfigs()).isEqualTo( - f1.getHttpConnectionManager().httpFilterConfigs()); + assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of()); xdsClient.rdsWatchers.get("r0").onResourceDoesNotExist("r0"); - realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f1); - assertThat(realConfig.virtualHosts().get()).isNull(); - assertThat(realConfig.httpFilterConfigs()).isEqualTo( - f1.getHttpConnectionManager().httpFilterConfigs()); + realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f1).get(); + assertThat(realConfig.virtualHosts()).isEmpty(); + assertThat(realConfig.interceptors()).isEmpty(); } @Test @@ -648,11 +636,11 @@ public void run() { assertThat(selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().size()) .isEqualTo(1); ServerRoutingConfig realConfig = - selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(filterChain1); - assertThat(realConfig.virtualHosts().get()).isEqualTo( + selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(filterChain1).get(); + assertThat(realConfig.virtualHosts()).isEqualTo( Collections.singletonList(createVirtualHost("virtual-host-1"))); - assertThat(realConfig.httpFilterConfigs()).isEqualTo( - filterChain1.getHttpConnectionManager().httpFilterConfigs()); + assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of()); + // xds update after start xdsClient.deliverRdsUpdate("rds", Collections.singletonList(createVirtualHost("virtual-host-2"))); @@ -664,11 +652,11 @@ public void run() { assertThat(selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().size()) .isEqualTo(1); realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs() - .get(filterChain1); - assertThat(realConfig.virtualHosts().get()).isEqualTo( + .get(filterChain1).get(); + assertThat(realConfig.virtualHosts()).isEqualTo( Collections.singletonList(createVirtualHost("virtual-host-2"))); - assertThat(realConfig.httpFilterConfigs()).isEqualTo( - filterChain1.getHttpConnectionManager().httpFilterConfigs()); + assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of()); + assertThat(sslSupplier1.isShutdown()).isFalse(); // not serving after serving @@ -705,11 +693,11 @@ public void run() { assertThat(selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().size()) .isEqualTo(1); realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs() - .get(filterChain2); - assertThat(realConfig.virtualHosts().get()).isEqualTo( + .get(filterChain2).get(); + assertThat(realConfig.virtualHosts()).isEqualTo( Collections.singletonList(createVirtualHost("virtual-host-1"))); - assertThat(realConfig.httpFilterConfigs()).isEqualTo( - filterChain2.getHttpConnectionManager().httpFilterConfigs()); + assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of()); + assertThat(executor.numPendingTasks()).isEqualTo(1); xdsClient.ldsWatcher.onResourceDoesNotExist(ldsResource); verify(mockServer, times(4)).shutdown(); @@ -733,11 +721,11 @@ public void run() { assertThat(selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().size()) .isEqualTo(1); realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs() - .get(filterChain3); - assertThat(realConfig.virtualHosts().get()).isEqualTo( + .get(filterChain3).get(); + assertThat(realConfig.virtualHosts()).isEqualTo( Collections.singletonList(createVirtualHost("virtual-host-1"))); - assertThat(realConfig.httpFilterConfigs()).isEqualTo( - filterChain3.getHttpConnectionManager().httpFilterConfigs()); + assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of()); + xdsServerWrapper.shutdown(); verify(mockServer, times(5)).shutdown(); assertThat(sslSupplier3.isShutdown()).isTrue(); @@ -747,9 +735,9 @@ public void run() { @Test @SuppressWarnings("unchecked") - public void interceptor_notServerInterceptor() throws Exception { + public void interceptor_success() throws Exception { ArgumentCaptor interceptorCaptor = - ArgumentCaptor.forClass(ConfigApplyingInterceptor.class); + ArgumentCaptor.forClass(ConfigApplyingInterceptor.class); final SettableFuture start = SettableFuture.create(); Executors.newSingleThreadExecutor().execute(new Runnable() { @Override @@ -764,26 +752,36 @@ public void run() { xdsClient.ldsResource.get(5, TimeUnit.SECONDS); verify(mockBuilder).intercept(interceptorCaptor.capture()); ConfigApplyingInterceptor interceptor = interceptorCaptor.getValue(); - ServerRoutingConfig routingConfig = createRoutingConfig("/FooService/barMethod", - "foo.google.com", "filter-type-url"); + RouteMatch routeMatch = + RouteMatch.create( + PathMatcher.fromPath("/FooService/barMethod", true), + Collections.emptyList(), null); + Route route = Route.forAction(routeMatch, null, + ImmutableMap.of()); + VirtualHost virtualHost = VirtualHost.create( + "v1", Collections.singletonList("foo.google.com"), Arrays.asList(route), + ImmutableMap.of()); + final List interceptorTrace = new ArrayList<>(); + ServerInterceptor interceptor0 = new ServerInterceptor() { + @Override + public ServerCall.Listener interceptCall(ServerCall call, + Metadata headers, ServerCallHandler next) { + interceptorTrace.add(0); + return next.startCall(call, headers); + } + }; + ServerRoutingConfig realConfig = ServerRoutingConfig.create( + ImmutableList.of(virtualHost), ImmutableMap.of(route, interceptor0)); ServerCall serverCall = mock(ServerCall.class); - when(serverCall.getAttributes()).thenReturn( - Attributes.newBuilder().set(ATTR_SERVER_ROUTING_CONFIG, routingConfig).build()); when(serverCall.getMethodDescriptor()).thenReturn(createMethod("FooService/barMethod")); + when(serverCall.getAttributes()).thenReturn( + Attributes.newBuilder().set(ATTR_SERVER_ROUTING_CONFIG, + new AtomicReference<>(realConfig)).build()); when(serverCall.getAuthority()).thenReturn("foo.google.com"); - - Filter filter = mock(Filter.class); - when(filter.typeUrls()).thenReturn(new String[]{"filter-type-url"}); - filterRegistry.register(filter); ServerCallHandler next = mock(ServerCallHandler.class); interceptor.interceptCall(serverCall, new Metadata(), next); - verify(next, never()).startCall(any(ServerCall.class), any(Metadata.class)); - ArgumentCaptor statusCaptor = ArgumentCaptor.forClass(Status.class); - verify(serverCall).close(statusCaptor.capture(), any(Metadata.class)); - Status status = statusCaptor.getValue(); - assertThat(status.getCode()).isEqualTo(Status.UNAVAILABLE.getCode()); - assertThat(status.getDescription()).isEqualTo( - "HttpFilterConfig(type URL: filter-type-url) is not supported on server-side."); + verify(next).startCall(eq(serverCall), any(Metadata.class)); + assertThat(interceptorTrace).isEqualTo(Arrays.asList(0)); } @Test @@ -809,7 +807,8 @@ public void run() { "foo.google.com", "filter-type-url"); ServerCall serverCall = mock(ServerCall.class); when(serverCall.getAttributes()).thenReturn( - Attributes.newBuilder().set(ATTR_SERVER_ROUTING_CONFIG, routingConfig).build()); + Attributes.newBuilder().set(ATTR_SERVER_ROUTING_CONFIG, + new AtomicReference<>(routingConfig)).build()); when(serverCall.getAuthority()).thenReturn("not-match.google.com"); Filter filter = mock(Filter.class); @@ -848,7 +847,8 @@ public void run() { "foo.google.com", "filter-type-url"); ServerCall serverCall = mock(ServerCall.class); when(serverCall.getAttributes()).thenReturn( - Attributes.newBuilder().set(ATTR_SERVER_ROUTING_CONFIG, routingConfig).build()); + Attributes.newBuilder() + .set(ATTR_SERVER_ROUTING_CONFIG, new AtomicReference<>(routingConfig)).build()); when(serverCall.getMethodDescriptor()).thenReturn(createMethod("NotMatchMethod")); when(serverCall.getAuthority()).thenReturn("foo.google.com"); @@ -884,12 +884,11 @@ public void run() { xdsClient.ldsResource.get(5, TimeUnit.SECONDS); verify(mockBuilder).intercept(interceptorCaptor.capture()); ConfigApplyingInterceptor interceptor = interceptorCaptor.getValue(); - ServerRoutingConfig failingConfig = ServerRoutingConfig.create( - ImmutableList.of(), new AtomicReference>() - ); ServerCall serverCall = mock(ServerCall.class); + when(serverCall.getAttributes()).thenReturn( - Attributes.newBuilder().set(ATTR_SERVER_ROUTING_CONFIG, failingConfig).build()); + Attributes.newBuilder().set(ATTR_SERVER_ROUTING_CONFIG, + new AtomicReference<>(ServerRoutingConfig.FAILING_ROUTING_CONFIG)).build()); ServerCallHandler next = mock(ServerCallHandler.class); interceptor.interceptCall(serverCall, new Metadata(), next); @@ -899,14 +898,12 @@ public void run() { Status status = statusCaptor.getValue(); assertThat(status.getCode()).isEqualTo(Status.UNAVAILABLE.getCode()); assertThat(status.getDescription()).isEqualTo( - "Missing xDS routing config VirtualHosts due to RDS config unavailable."); + "Missing or broken xDS routing config: RDS config unavailable."); } @Test @SuppressWarnings("unchecked") - public void interceptors() throws Exception { - ArgumentCaptor interceptorCaptor = - ArgumentCaptor.forClass(ConfigApplyingInterceptor.class); + public void buildInterceptor_inline() throws Exception { final SettableFuture start = SettableFuture.create(); Executors.newSingleThreadExecutor().execute(new Runnable() { @Override @@ -919,14 +916,12 @@ public void run() { } }); xdsClient.ldsResource.get(5, TimeUnit.SECONDS); - verify(mockBuilder).intercept(interceptorCaptor.capture()); - final ConfigApplyingInterceptor interceptor = interceptorCaptor.getValue(); RouteMatch routeMatch = - RouteMatch.create( - PathMatcher.fromPath("/FooService/barMethod", true), - Collections.emptyList(), null); + RouteMatch.create( + PathMatcher.fromPath("/FooService/barMethod", true), + Collections.emptyList(), null); Filter filter = mock(Filter.class, withSettings() - .extraInterfaces(ServerInterceptorBuilder.class)); + .extraInterfaces(ServerInterceptorBuilder.class)); when(filter.typeUrls()).thenReturn(new String[]{"filter-type-url"}); filterRegistry.register(filter); FilterConfig f0 = mock(FilterConfig.class); @@ -936,7 +931,7 @@ public void run() { ServerInterceptor interceptor0 = new ServerInterceptor() { @Override public ServerCall.Listener interceptCall(ServerCall call, - Metadata headers, ServerCallHandler next) { + Metadata headers, ServerCallHandler next) { interceptorTrace.add(0); return next.startCall(call, headers); } @@ -949,55 +944,130 @@ public ServerCall.Listener interceptCall(ServerCallof()); VirtualHost virtualHost = VirtualHost.create( - "v1", Collections.singletonList("foo.google.com"), - Arrays.asList(Route.forAction(routeMatch, null, - ImmutableMap.of())), - ImmutableMap.of("filter-config-name-0", f0Override)); - ServerRoutingConfig routingConfig = ServerRoutingConfig.create( - Arrays.asList(new NamedFilterConfig("filter-config-name-0", f0), - new NamedFilterConfig("filter-config-name-1", f0)), - new AtomicReference<>(ImmutableList.of(virtualHost)) - ); + "v1", Collections.singletonList("foo.google.com"), Arrays.asList(route), + ImmutableMap.of("filter-config-name-0", f0Override)); + HttpConnectionManager hcmVirtual = HttpConnectionManager.forVirtualHosts( + 0L, Collections.singletonList(virtualHost), + Arrays.asList(new NamedFilterConfig("filter-config-name-0", f0), + new NamedFilterConfig("filter-config-name-1", f0))); + EnvoyServerProtoData.FilterChain filterChain = createFilterChain("filter-chain-0", hcmVirtual); + xdsClient.deliverLdsUpdate(Collections.singletonList(filterChain), null); + start.get(5000, TimeUnit.MILLISECONDS); + verify(mockServer).start(); + assertThat(selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().size()) + .isEqualTo(1); + ServerInterceptor realInterceptor = selectorManager.getSelectorToUpdateSelector() + .getRoutingConfigs().get(filterChain).get().interceptors().get(route); + assertThat(realInterceptor).isNotNull(); + ServerCall serverCall = mock(ServerCall.class); ServerCallHandler mockNext = mock(ServerCallHandler.class); final ServerCall.Listener listener = new ServerCall.Listener() {}; when(mockNext.startCall(any(ServerCall.class), any(Metadata.class))).thenReturn(listener); - when(serverCall.getAttributes()).thenReturn( - Attributes.newBuilder().set(ATTR_SERVER_ROUTING_CONFIG, routingConfig).build()); - when(serverCall.getMethodDescriptor()).thenReturn(createMethod("FooService/barMethod")); - when(serverCall.getAuthority()).thenReturn("foo.google.com"); + realInterceptor.interceptCall(serverCall, new Metadata(), mockNext); + assertThat(interceptorTrace).isEqualTo(Arrays.asList(1, 0)); + verify(mockNext).startCall(eq(serverCall), any(Metadata.class)); + } - when(((ServerInterceptorBuilder)filter).buildServerInterceptor(f0, f0Override)) - .thenReturn(null); + @Test + @SuppressWarnings("unchecked") + public void buildInterceptor_rds() throws Exception { + final SettableFuture start = SettableFuture.create(); + Executors.newSingleThreadExecutor().execute(new Runnable() { + @Override + public void run() { + try { + start.set(xdsServerWrapper.start()); + } catch (Exception ex) { + start.setException(ex); + } + } + }); + xdsClient.ldsResource.get(5, TimeUnit.SECONDS); + + Filter filter = mock(Filter.class, withSettings() + .extraInterfaces(ServerInterceptorBuilder.class)); + when(filter.typeUrls()).thenReturn(new String[]{"filter-type-url"}); + filterRegistry.register(filter); + FilterConfig f0 = mock(FilterConfig.class); + FilterConfig f0Override = mock(FilterConfig.class); + when(f0.typeUrl()).thenReturn("filter-type-url"); + final List interceptorTrace = new ArrayList<>(); + ServerInterceptor interceptor0 = new ServerInterceptor() { + @Override + public ServerCall.Listener interceptCall(ServerCall call, + Metadata headers, ServerCallHandler next) { + interceptorTrace.add(0); + return next.startCall(call, headers); + } + }; + ServerInterceptor interceptor1 = new ServerInterceptor() { + @Override + public ServerCall.Listener interceptCall(ServerCall call, + Metadata headers, ServerCallHandler next) { + interceptorTrace.add(1); + return next.startCall(call, headers); + } + }; when(((ServerInterceptorBuilder)filter).buildServerInterceptor(f0, null)) - .thenReturn(null); - ServerCall.Listener configApplyingInterceptorListener = - interceptor.interceptCall(serverCall, new Metadata(), mockNext); - assertThat(configApplyingInterceptorListener).isSameInstanceAs(listener); + .thenReturn(interceptor0); + when(((ServerInterceptorBuilder)filter).buildServerInterceptor(f0, f0Override)) + .thenReturn(interceptor1); + RouteMatch routeMatch = + RouteMatch.create( + PathMatcher.fromPath("/FooService/barMethod", true), + Collections.emptyList(), null); + + HttpConnectionManager rdsHcm = HttpConnectionManager.forRdsName(0L, "r0", + Arrays.asList(new NamedFilterConfig("filter-config-name-0", f0), + new NamedFilterConfig("filter-config-name-1", f0))); + EnvoyServerProtoData.FilterChain filterChain = createFilterChain("filter-chain-0", rdsHcm); + xdsClient.deliverLdsUpdate(Collections.singletonList(filterChain), null); + Route route = Route.forAction(routeMatch, null, + ImmutableMap.of()); + VirtualHost virtualHost = VirtualHost.create( + "v1", Collections.singletonList("foo.google.com"), Arrays.asList(route), + ImmutableMap.of("filter-config-name-0", f0Override)); + xdsClient.rdsCount.await(5, TimeUnit.SECONDS); + xdsClient.deliverRdsUpdate("r0", Collections.singletonList(virtualHost)); + start.get(5000, TimeUnit.MILLISECONDS); + verify(mockServer).start(); + assertThat(selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().size()) + .isEqualTo(1); + ServerInterceptor realInterceptor = selectorManager.getSelectorToUpdateSelector() + .getRoutingConfigs().get(filterChain).get().interceptors().get(route); + assertThat(realInterceptor).isNotNull(); + + ServerCall serverCall = mock(ServerCall.class); + ServerCallHandler mockNext = mock(ServerCallHandler.class); + final ServerCall.Listener listener = new ServerCall.Listener() {}; + when(mockNext.startCall(any(ServerCall.class), any(Metadata.class))).thenReturn(listener); + realInterceptor.interceptCall(serverCall, new Metadata(), mockNext); + assertThat(interceptorTrace).isEqualTo(Arrays.asList(1, 0)); verify(mockNext).startCall(eq(serverCall), any(Metadata.class)); - assertThat(interceptorTrace).isEqualTo(Arrays.asList()); - when(((ServerInterceptorBuilder)filter).buildServerInterceptor(f0, f0Override)) - .thenReturn(null); - when(((ServerInterceptorBuilder)filter).buildServerInterceptor(f0, null)) - .thenReturn(interceptor0); - configApplyingInterceptorListener = interceptor.interceptCall( - serverCall, new Metadata(), mockNext); - assertThat(configApplyingInterceptorListener).isSameInstanceAs(listener); + virtualHost = VirtualHost.create( + "v1", Collections.singletonList("foo.google.com"), Arrays.asList(route), + ImmutableMap.of()); + xdsClient.deliverRdsUpdate("r0", Collections.singletonList(virtualHost)); + realInterceptor = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs() + .get(filterChain).get().interceptors().get(route); + assertThat(realInterceptor).isNotNull(); + interceptorTrace.clear(); + realInterceptor.interceptCall(serverCall, new Metadata(), mockNext); + assertThat(interceptorTrace).isEqualTo(Arrays.asList(0, 0)); verify(mockNext, times(2)).startCall(eq(serverCall), any(Metadata.class)); - assertThat(interceptorTrace).isEqualTo(Arrays.asList(0)); - when(((ServerInterceptorBuilder)filter).buildServerInterceptor(f0, f0Override)) - .thenReturn(interceptor0); - when(((ServerInterceptorBuilder)filter).buildServerInterceptor(f0, null)) - .thenReturn(interceptor1); - configApplyingInterceptorListener = interceptor.interceptCall( - serverCall, new Metadata(), mockNext); - assertThat(configApplyingInterceptorListener).isSameInstanceAs(listener); - verify(mockNext, times(3)).startCall(eq(serverCall), any(Metadata.class)); - assertThat(interceptorTrace).isEqualTo(Arrays.asList(0, 0, 1)); + xdsClient.rdsWatchers.get("r0").onResourceDoesNotExist("r0"); + assertThat(selectorManager.getSelectorToUpdateSelector().getRoutingConfigs() + .get(filterChain).get()).isEqualTo(noopConfig); } private static FilterChain createFilterChain(String name, HttpConnectionManager hcm) { @@ -1012,8 +1082,12 @@ private static VirtualHost createVirtualHost(String name) { } private static HttpConnectionManager createRds(String name) { + return createRds(name, null); + } + + private static HttpConnectionManager createRds(String name, FilterConfig filterConfig) { return HttpConnectionManager.forRdsName(0L, name, - Arrays.asList(new NamedFilterConfig("named-config-" + name, null))); + Arrays.asList(new NamedFilterConfig("named-config-" + name, filterConfig))); } private static EnvoyServerProtoData.FilterChainMatch createMatch() { @@ -1041,9 +1115,8 @@ private static ServerRoutingConfig createRoutingConfig(String path, String domai Collections.emptyMap()); FilterConfig f0 = mock(FilterConfig.class); when(f0.typeUrl()).thenReturn(filterType); - return ServerRoutingConfig.create( - Arrays.asList(new NamedFilterConfig("filter-config-name-0", f0)), - new AtomicReference<>(ImmutableList.of(virtualHost)) + return ServerRoutingConfig.create(ImmutableList.of(virtualHost), + ImmutableMap.of() ); } From 38a554c23a52b51b99e1341a27be8dd04e96c125 Mon Sep 17 00:00:00 2001 From: yifeizhuang Date: Thu, 16 Sep 2021 16:12:52 -0700 Subject: [PATCH 23/76] xds: implement RBAC gRFC misc cases (#8518) --- .../java/io/grpc/xds/ClientXdsClient.java | 12 +++- xds/src/main/java/io/grpc/xds/RbacFilter.java | 16 +++++ .../java/io/grpc/xds/XdsServerWrapper.java | 15 +++-- .../rbac/engine/GrpcAuthorizationEngine.java | 56 +++++++++++++++- .../io/grpc/xds/ClientXdsClientDataTest.java | 17 +++++ .../test/java/io/grpc/xds/RbacFilterTest.java | 43 ++++++++++++ .../io/grpc/xds/XdsServerWrapperTest.java | 63 +++++++++++++++-- .../engine/GrpcAuthorizationEngineTest.java | 67 +++++++++++++++++++ 8 files changed, 275 insertions(+), 14 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/ClientXdsClient.java b/xds/src/main/java/io/grpc/xds/ClientXdsClient.java index d490c9861b9..3bef4c416e0 100644 --- a/xds/src/main/java/io/grpc/xds/ClientXdsClient.java +++ b/xds/src/main/java/io/grpc/xds/ClientXdsClient.java @@ -136,6 +136,10 @@ final class ClientXdsClient extends AbstractXdsClient { static boolean enableRetry = Strings.isNullOrEmpty(System.getenv("GRPC_XDS_EXPERIMENTAL_ENABLE_RETRY")) || Boolean.parseBoolean(System.getenv("GRPC_XDS_EXPERIMENTAL_ENABLE_RETRY")); + @VisibleForTesting + static boolean enableRbac = + Strings.isNullOrEmpty(System.getenv("GRPC_XDS_EXPERIMENTAL_RBAC")) + || Boolean.parseBoolean(System.getenv("GRPC_XDS_EXPERIMENTAL_RBAC")); private static final String TYPE_URL_HTTP_CONNECTION_MANAGER_V2 = "type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2" @@ -218,7 +222,7 @@ protected void handleLdsResponse(String versionInfo, List resources, String listener, retainedRdsResources, enableFaultInjection && isResourceV3); } else { ldsUpdate = processServerSideListener( - listener, retainedRdsResources, enableFaultInjection && isResourceV3); + listener, retainedRdsResources, enableRbac); } } catch (ResourceInvalidException e) { errors.add( @@ -729,10 +733,14 @@ private static FilterChainMatch parseFilterChainMatch( static io.grpc.xds.HttpConnectionManager parseHttpConnectionManager( HttpConnectionManager proto, Set rdsResources, FilterRegistry filterRegistry, boolean parseHttpFilter, boolean isForClient) throws ResourceInvalidException { - if (proto.getXffNumTrustedHops() != 0) { + if (enableRbac && proto.getXffNumTrustedHops() != 0) { throw new ResourceInvalidException( "HttpConnectionManager with xff_num_trusted_hops unsupported"); } + if (enableRbac && !proto.getOriginalIpDetectionExtensionsList().isEmpty()) { + throw new ResourceInvalidException("HttpConnectionManager with " + + "original_ip_detection_extensions unsupported"); + } // Obtain max_stream_duration from Http Protocol Options. long maxStreamDuration = 0; if (proto.hasCommonHttpProtocolOptions()) { diff --git a/xds/src/main/java/io/grpc/xds/RbacFilter.java b/xds/src/main/java/io/grpc/xds/RbacFilter.java index 48b4954767a..39f91b475ae 100644 --- a/xds/src/main/java/io/grpc/xds/RbacFilter.java +++ b/xds/src/main/java/io/grpc/xds/RbacFilter.java @@ -28,6 +28,7 @@ import io.envoyproxy.envoy.config.rbac.v3.Principal; import io.envoyproxy.envoy.extensions.filters.http.rbac.v3.RBAC; import io.envoyproxy.envoy.extensions.filters.http.rbac.v3.RBACPerRoute; +import io.envoyproxy.envoy.type.v3.Int32Range; import io.grpc.Metadata; import io.grpc.ServerCall; import io.grpc.ServerCallHandler; @@ -45,6 +46,7 @@ import io.grpc.xds.internal.rbac.engine.GrpcAuthorizationEngine.AuthenticatedMatcher; import io.grpc.xds.internal.rbac.engine.GrpcAuthorizationEngine.DestinationIpMatcher; import io.grpc.xds.internal.rbac.engine.GrpcAuthorizationEngine.DestinationPortMatcher; +import io.grpc.xds.internal.rbac.engine.GrpcAuthorizationEngine.DestinationPortRangeMatcher; import io.grpc.xds.internal.rbac.engine.GrpcAuthorizationEngine.InvertMatcher; import io.grpc.xds.internal.rbac.engine.GrpcAuthorizationEngine.Matcher; import io.grpc.xds.internal.rbac.engine.GrpcAuthorizationEngine.OrMatcher; @@ -216,6 +218,8 @@ private static Matcher parsePermission(Permission permission) { return createDestinationIpMatcher(permission.getDestinationIp()); case DESTINATION_PORT: return createDestinationPortMatcher(permission.getDestinationPort()); + case DESTINATION_PORT_RANGE: + return parseDestinationPortRangeMatcher(permission.getDestinationPortRange()); case NOT_RULE: return new InvertMatcher(parsePermission(permission.getNotRule())); case METADATA: // hard coded, never match. @@ -291,6 +295,14 @@ private static RequestedServerNameMatcher parseRequestedServerNameMatcher( private static AuthHeaderMatcher parseHeaderMatcher( io.envoyproxy.envoy.config.route.v3.HeaderMatcher proto) { + if (proto.getName().startsWith("grpc-")) { + throw new IllegalArgumentException("Invalid header matcher config: [grpc-] prefixed " + + "header name is not allowed."); + } + if (":scheme".equals(proto.getName())) { + throw new IllegalArgumentException("Invalid header matcher config: header name [:scheme] " + + "is not allowed."); + } return new AuthHeaderMatcher(MatcherParser.parseHeaderMatcher(proto)); } @@ -304,6 +316,10 @@ private static DestinationPortMatcher createDestinationPortMatcher(int port) { return new DestinationPortMatcher(port); } + private static DestinationPortRangeMatcher parseDestinationPortRangeMatcher(Int32Range range) { + return new DestinationPortRangeMatcher(range.getStart(), range.getEnd()); + } + private static DestinationIpMatcher createDestinationIpMatcher(CidrRange cidrRange) { return new DestinationIpMatcher(Matchers.CidrMatcher.create( resolve(cidrRange), cidrRange.getPrefixLen().getValue())); diff --git a/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java b/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java index e7301500e0e..fdc5f099bfe 100644 --- a/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java +++ b/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java @@ -639,6 +639,9 @@ public void run() { if (!routeDiscoveryStates.containsKey(resourceName)) { return; } + if (savedVirtualHosts == null && !isPending) { + logger.log(Level.WARNING, "Received valid Rds {0} configuration.", resourceName); + } savedVirtualHosts = ImmutableList.copyOf(update.virtualHosts); updateRdsRoutingConfig(); maybeUpdateSelector(); @@ -746,8 +749,8 @@ public Listener interceptCall(ServerCall call, virtualHosts, call.getAuthority()); if (virtualHost == null) { call.close( - Status.UNAVAILABLE.withDescription("Could not find xDS virtual host matching RPC"), - new Metadata()); + Status.UNAVAILABLE.withDescription("Could not find xDS virtual host matching RPC"), + new Metadata()); return new Listener() {}; } Route selectedRoute = null; @@ -760,11 +763,15 @@ public Listener interceptCall(ServerCall call, } } if (selectedRoute == null) { - call.close( - Status.UNAVAILABLE.withDescription("Could not find xDS route matching RPC"), + call.close(Status.UNAVAILABLE.withDescription("Could not find xDS route matching RPC"), new Metadata()); return new ServerCall.Listener() {}; } + if (selectedRoute.routeAction() != null) { + call.close(Status.UNAVAILABLE.withDescription("Invalid xDS route action for matching " + + "route: only Route.non_forwarding_action should be allowed."), new Metadata()); + return new ServerCall.Listener() {}; + } ServerInterceptor routeInterceptor = noopInterceptor; Map perRouteInterceptors = routingConfig.interceptors(); if (perRouteInterceptors != null && perRouteInterceptors.get(selectedRoute) != null) { diff --git a/xds/src/main/java/io/grpc/xds/internal/rbac/engine/GrpcAuthorizationEngine.java b/xds/src/main/java/io/grpc/xds/internal/rbac/engine/GrpcAuthorizationEngine.java index 6d275d322a2..bb911461a27 100644 --- a/xds/src/main/java/io/grpc/xds/internal/rbac/engine/GrpcAuthorizationEngine.java +++ b/xds/src/main/java/io/grpc/xds/internal/rbac/engine/GrpcAuthorizationEngine.java @@ -20,6 +20,7 @@ import com.google.auto.value.AutoValue; import com.google.common.base.Joiner; +import com.google.common.io.BaseEncoding; import io.grpc.Grpc; import io.grpc.Metadata; import io.grpc.ServerCall; @@ -35,6 +36,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.logging.Level; import java.util.logging.Logger; import javax.annotation.Nullable; @@ -234,6 +236,23 @@ public boolean matches(EvaluateArgs args) { } } + public static final class DestinationPortRangeMatcher implements Matcher { + private final int start; + private final int end; + + /** Start of the range is inclusive. End of the range is exclusive.*/ + public DestinationPortRangeMatcher(int start, int end) { + this.start = start; + this.end = end; + } + + @Override + public boolean matches(EvaluateArgs args) { + int port = args.getDestinationPort(); + return port >= start && port < end; + } + } + public static final class RequestedServerNameMatcher implements Matcher { private final Matchers.StringMatcher delegate; @@ -316,9 +335,44 @@ private Collection getPrincipalNames() { @Nullable private String getHeader(String headerName) { - if (headerName.endsWith(Metadata.BINARY_HEADER_SUFFIX)) { + headerName = headerName.toLowerCase(Locale.ROOT); + if ("te".equals(headerName)) { return null; } + if (":authority".equals(headerName)) { + headerName = "host"; + } + if ("host".equals(headerName)) { + return serverCall.getAuthority(); + } + if (":path".equals(headerName)) { + return getPath(); + } + if (":method".equals(headerName)) { + return "POST"; + } + return deserializeHeader(headerName); + } + + @Nullable + private String deserializeHeader(String headerName) { + if (headerName.endsWith(Metadata.BINARY_HEADER_SUFFIX)) { + Metadata.Key key; + try { + key = Metadata.Key.of(headerName, Metadata.BINARY_BYTE_MARSHALLER); + } catch (IllegalArgumentException e) { + return null; + } + Iterable values = metadata.getAll(key); + if (values == null) { + return null; + } + List encoded = new ArrayList<>(); + for (byte[] v : values) { + encoded.add(BaseEncoding.base64().omitPadding().encode(v)); + } + return Joiner.on(",").join(encoded); + } Metadata.Key key; try { key = Metadata.Key.of(headerName, Metadata.ASCII_STRING_MARSHALLER); diff --git a/xds/src/test/java/io/grpc/xds/ClientXdsClientDataTest.java b/xds/src/test/java/io/grpc/xds/ClientXdsClientDataTest.java index 876615d0b39..597ca7df2d9 100644 --- a/xds/src/test/java/io/grpc/xds/ClientXdsClientDataTest.java +++ b/xds/src/test/java/io/grpc/xds/ClientXdsClientDataTest.java @@ -131,16 +131,20 @@ public class ClientXdsClientDataTest { public final ExpectedException thrown = ExpectedException.none(); private final FilterRegistry filterRegistry = FilterRegistry.getDefaultRegistry(); private boolean originalEnableRetry; + private boolean originalEnableRbac; @Before public void setUp() { originalEnableRetry = ClientXdsClient.enableRetry; assertThat(originalEnableRetry).isTrue(); + originalEnableRbac = ClientXdsClient.enableRbac; + assertThat(originalEnableRbac).isTrue(); } @After public void tearDown() { ClientXdsClient.enableRetry = originalEnableRetry; + ClientXdsClient.enableRbac = originalEnableRbac; } @Test @@ -1108,6 +1112,19 @@ public void parseHttpConnectionManager_xffNumTrustedHopsUnsupported() hcm, new HashSet(), filterRegistry, false /* does not matter */, true /* does not matter */); } + + @Test + public void parseHttpConnectionManager_OriginalIpDetectionExtensionsMustEmpty() + throws ResourceInvalidException { + @SuppressWarnings("deprecation") + HttpConnectionManager hcm = HttpConnectionManager.newBuilder() + .addOriginalIpDetectionExtensions(TypedExtensionConfig.newBuilder().build()) + .build(); + thrown.expect(ResourceInvalidException.class); + thrown.expectMessage("HttpConnectionManager with original_ip_detection_extensions unsupported"); + ClientXdsClient.parseHttpConnectionManager( + hcm, new HashSet(), filterRegistry, false /* does not matter */, false); + } @Test public void parseHttpConnectionManager_missingRdsAndInlinedRouteConfiguration() diff --git a/xds/src/test/java/io/grpc/xds/RbacFilterTest.java b/xds/src/test/java/io/grpc/xds/RbacFilterTest.java index d8f1d8aa825..082c49ef665 100644 --- a/xds/src/test/java/io/grpc/xds/RbacFilterTest.java +++ b/xds/src/test/java/io/grpc/xds/RbacFilterTest.java @@ -41,6 +41,7 @@ import io.envoyproxy.envoy.type.matcher.v3.MetadataMatcher; import io.envoyproxy.envoy.type.matcher.v3.PathMatcher; import io.envoyproxy.envoy.type.matcher.v3.StringMatcher; +import io.envoyproxy.envoy.type.v3.Int32Range; import io.grpc.Attributes; import io.grpc.Grpc; import io.grpc.Metadata; @@ -109,6 +110,33 @@ public void ipPortParser() { assertThat(decision.decision()).isEqualTo(GrpcAuthorizationEngine.Action.DENY); } + @Test + @SuppressWarnings({"unchecked", "deprecation"}) + public void portRangeParser() { + List permissionList = Arrays.asList( + Permission.newBuilder().setDestinationPortRange( + Int32Range.newBuilder().setStart(1010).setEnd(65535).build() + ).build()); + List principalList = Arrays.asList( + Principal.newBuilder().setRemoteIp( + CidrRange.newBuilder().setAddressPrefix("10.10.10.0") + .setPrefixLen(UInt32Value.of(24)).build() + ).build()); + ConfigOrError result = parse(permissionList, principalList); + assertThat(result.errorDetail).isNull(); + ServerCall serverCall = mock(ServerCall.class); + Attributes attributes = Attributes.newBuilder() + .set(Grpc.TRANSPORT_ATTR_REMOTE_ADDR, new InetSocketAddress("10.10.10.0", 1)) + .set(Grpc.TRANSPORT_ATTR_LOCAL_ADDR, new InetSocketAddress("10.10.10.0",9090)) + .build(); + when(serverCall.getAttributes()).thenReturn(attributes); + when(serverCall.getMethodDescriptor()).thenReturn(method().build()); + GrpcAuthorizationEngine engine = + new GrpcAuthorizationEngine(((RbacConfig)result.config).authConfig()); + AuthDecision decision = engine.evaluate(new Metadata(), serverCall); + assertThat(decision.decision()).isEqualTo(GrpcAuthorizationEngine.Action.DENY); + } + @Test @SuppressWarnings("unchecked") public void pathParser() { @@ -172,6 +200,21 @@ public void headerParser() { assertThat(decision.decision()).isEqualTo(GrpcAuthorizationEngine.Action.DENY); } + @Test + @SuppressWarnings("deprecation") + public void headerParser_headerName() { + HeaderMatcher headerMatcher = HeaderMatcher.newBuilder() + .setName("grpc--feature").setExactMatch("win").build(); + List permissionList = Arrays.asList( + Permission.newBuilder().setHeader(headerMatcher).build()); + HeaderMatcher headerMatcher2 = HeaderMatcher.newBuilder() + .setName(":scheme").setExactMatch("win").build(); + List principalList = Arrays.asList( + Principal.newBuilder().setHeader(headerMatcher2).build()); + ConfigOrError result = parseOverride(permissionList, principalList); + assertThat(result.errorDetail).isNotNull(); + } + @Test @SuppressWarnings("unchecked") public void compositeRules() { diff --git a/xds/src/test/java/io/grpc/xds/XdsServerWrapperTest.java b/xds/src/test/java/io/grpc/xds/XdsServerWrapperTest.java index c109bb44a13..f2b6e9e4790 100644 --- a/xds/src/test/java/io/grpc/xds/XdsServerWrapperTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsServerWrapperTest.java @@ -865,6 +865,50 @@ public void run() { assertThat(status.getDescription()).isEqualTo("Could not find xDS route matching RPC"); } + @Test + @SuppressWarnings("unchecked") + public void interceptor_invalidRouteAction() throws Exception { + ArgumentCaptor interceptorCaptor = + ArgumentCaptor.forClass(ConfigApplyingInterceptor.class); + final SettableFuture start = SettableFuture.create(); + Executors.newSingleThreadExecutor().execute(new Runnable() { + @Override + public void run() { + try { + start.set(xdsServerWrapper.start()); + } catch (Exception ex) { + start.setException(ex); + } + } + }); + xdsClient.ldsResource.get(5, TimeUnit.SECONDS); + verify(mockBuilder).intercept(interceptorCaptor.capture()); + ConfigApplyingInterceptor interceptor = interceptorCaptor.getValue(); + ServerRoutingConfig routingConfig = createRoutingConfig("/FooService/barMethod", + "foo.google.com", "filter-type-url", Route.RouteAction.forCluster( + "cluster", Collections.emptyList(), null, null + )); + ServerCall serverCall = mock(ServerCall.class); + when(serverCall.getAttributes()).thenReturn( + Attributes.newBuilder() + .set(ATTR_SERVER_ROUTING_CONFIG, new AtomicReference<>(routingConfig)).build()); + when(serverCall.getMethodDescriptor()).thenReturn(createMethod("FooService/barMethod")); + when(serverCall.getAuthority()).thenReturn("foo.google.com"); + + Filter filter = mock(Filter.class); + when(filter.typeUrls()).thenReturn(new String[]{"filter-type-url"}); + filterRegistry.register(filter); + ServerCallHandler next = mock(ServerCallHandler.class); + interceptor.interceptCall(serverCall, new Metadata(), next); + verify(next, never()).startCall(any(ServerCall.class), any(Metadata.class)); + ArgumentCaptor statusCaptor = ArgumentCaptor.forClass(Status.class); + verify(serverCall).close(statusCaptor.capture(), any(Metadata.class)); + Status status = statusCaptor.getValue(); + assertThat(status.getCode()).isEqualTo(Status.UNAVAILABLE.getCode()); + assertThat(status.getDescription()).isEqualTo("Invalid xDS route action for matching " + + "route: only Route.non_forwarding_action should be allowed."); + } + @Test @SuppressWarnings("unchecked") public void interceptor_failingRouterConfig() throws Exception { @@ -1104,15 +1148,20 @@ private static EnvoyServerProtoData.FilterChainMatch createMatch() { private static ServerRoutingConfig createRoutingConfig(String path, String domain, String filterType) { + return createRoutingConfig(path, domain, filterType, null); + } + + private static ServerRoutingConfig createRoutingConfig(String path, String domain, + String filterType, Route.RouteAction action) { RouteMatch routeMatch = - RouteMatch.create( - PathMatcher.fromPath(path, true), - Collections.emptyList(), null); + RouteMatch.create( + PathMatcher.fromPath(path, true), + Collections.emptyList(), null); VirtualHost virtualHost = VirtualHost.create( - "v1", Collections.singletonList(domain), - Arrays.asList(Route.forAction(routeMatch, null, - ImmutableMap.of())), - Collections.emptyMap()); + "v1", Collections.singletonList(domain), + Arrays.asList(Route.forAction(routeMatch, action, + ImmutableMap.of())), + Collections.emptyMap()); FilterConfig f0 = mock(FilterConfig.class); when(f0.typeUrl()).thenReturn(filterType); return ServerRoutingConfig.create(ImmutableList.of(virtualHost), diff --git a/xds/src/test/java/io/grpc/xds/internal/rbac/engine/GrpcAuthorizationEngineTest.java b/xds/src/test/java/io/grpc/xds/internal/rbac/engine/GrpcAuthorizationEngineTest.java index 504c9e8df2a..626a4cfc275 100644 --- a/xds/src/test/java/io/grpc/xds/internal/rbac/engine/GrpcAuthorizationEngineTest.java +++ b/xds/src/test/java/io/grpc/xds/internal/rbac/engine/GrpcAuthorizationEngineTest.java @@ -16,12 +16,14 @@ package io.grpc.xds.internal.rbac.engine; +import static com.google.common.base.Charsets.US_ASCII; import static com.google.common.truth.Truth.assertThat; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableList; +import com.google.common.io.BaseEncoding; import io.grpc.Attributes; import io.grpc.Grpc; import io.grpc.Metadata; @@ -177,6 +179,71 @@ public void headerMatcher() { assertThat(decision.decision()).isEqualTo(Action.DENY); } + @Test + public void headerMatcher_binaryHeader() { + AuthHeaderMatcher headerMatcher = new AuthHeaderMatcher(Matchers.HeaderMatcher + .forExactValue(HEADER_KEY + Metadata.BINARY_HEADER_SUFFIX, + BaseEncoding.base64().omitPadding().encode(HEADER_VALUE.getBytes(US_ASCII)), false)); + OrMatcher principal = OrMatcher.create(headerMatcher); + OrMatcher permission = OrMatcher.create( + new InvertMatcher(new DestinationPortMatcher(PORT + 1))); + PolicyMatcher policyMatcher = new PolicyMatcher(POLICY_NAME, permission, principal); + GrpcAuthorizationEngine engine = new GrpcAuthorizationEngine( + new AuthConfig(Collections.singletonList(policyMatcher), Action.ALLOW)); + Metadata metadata = new Metadata(); + metadata.put(Metadata.Key.of(HEADER_KEY + Metadata.BINARY_HEADER_SUFFIX, + Metadata.BINARY_BYTE_MARSHALLER), HEADER_VALUE.getBytes(US_ASCII)); + AuthDecision decision = engine.evaluate(metadata, serverCall); + assertThat(decision.decision()).isEqualTo(Action.ALLOW); + assertThat(decision.matchingPolicyName()).isEqualTo(POLICY_NAME); + } + + @Test + public void headerMatcher_hardcodePostMethod() { + AuthHeaderMatcher headerMatcher = new AuthHeaderMatcher(Matchers.HeaderMatcher + .forExactValue(":method", "POST", false)); + OrMatcher principal = OrMatcher.create(headerMatcher); + OrMatcher permission = OrMatcher.create( + new InvertMatcher(new DestinationPortMatcher(PORT + 1))); + PolicyMatcher policyMatcher = new PolicyMatcher(POLICY_NAME, permission, principal); + GrpcAuthorizationEngine engine = new GrpcAuthorizationEngine( + new AuthConfig(Collections.singletonList(policyMatcher), Action.ALLOW)); + AuthDecision decision = engine.evaluate(new Metadata(), serverCall); + assertThat(decision.decision()).isEqualTo(Action.ALLOW); + assertThat(decision.matchingPolicyName()).isEqualTo(POLICY_NAME); + } + + @Test + public void headerMatcher_pathHeader() { + AuthHeaderMatcher headerMatcher = new AuthHeaderMatcher(Matchers.HeaderMatcher + .forExactValue(":path", "/" + PATH, false)); + OrMatcher principal = OrMatcher.create(headerMatcher); + OrMatcher permission = OrMatcher.create( + new InvertMatcher(new DestinationPortMatcher(PORT + 1))); + PolicyMatcher policyMatcher = new PolicyMatcher(POLICY_NAME, permission, principal); + GrpcAuthorizationEngine engine = new GrpcAuthorizationEngine( + new AuthConfig(Collections.singletonList(policyMatcher), Action.ALLOW)); + AuthDecision decision = engine.evaluate(HEADER, serverCall); + assertThat(decision.decision()).isEqualTo(Action.ALLOW); + assertThat(decision.matchingPolicyName()).isEqualTo(POLICY_NAME); + } + + @Test + public void headerMatcher_aliasAuthorityAndHost() { + AuthHeaderMatcher headerMatcher = new AuthHeaderMatcher(Matchers.HeaderMatcher + .forExactValue("Host", "google.com", false)); + OrMatcher principal = OrMatcher.create(headerMatcher); + OrMatcher permission = OrMatcher.create( + new InvertMatcher(new DestinationPortMatcher(PORT + 1))); + PolicyMatcher policyMatcher = new PolicyMatcher(POLICY_NAME, permission, principal); + GrpcAuthorizationEngine engine = new GrpcAuthorizationEngine( + new AuthConfig(Collections.singletonList(policyMatcher), Action.ALLOW)); + when(serverCall.getAuthority()).thenReturn("google.com"); + AuthDecision decision = engine.evaluate(new Metadata(), serverCall); + assertThat(decision.decision()).isEqualTo(Action.ALLOW); + assertThat(decision.matchingPolicyName()).isEqualTo(POLICY_NAME); + } + @Test public void pathMatcher() { PathMatcher pathMatcher = new PathMatcher(STRING_MATCHER); From 838438cedbb0a71d268402501fd4f8dd10daa23d Mon Sep 17 00:00:00 2001 From: ZhenLian Date: Fri, 17 Sep 2021 09:45:41 -0700 Subject: [PATCH 24/76] AdvancedTls: add functions to load credentials from static files (#8525) * AdvancedTls: add functions to load credentials from static files --- .../grpc/util/AdvancedTlsX509KeyManager.java | 15 ++++++++ .../util/AdvancedTlsX509TrustManager.java | 14 +++++++ .../java/io/grpc/netty/AdvancedTlsTest.java | 38 +++++++++++++++++++ 3 files changed, 67 insertions(+) diff --git a/core/src/main/java/io/grpc/util/AdvancedTlsX509KeyManager.java b/core/src/main/java/io/grpc/util/AdvancedTlsX509KeyManager.java index 8541c6b5280..9c9102b12cb 100644 --- a/core/src/main/java/io/grpc/util/AdvancedTlsX509KeyManager.java +++ b/core/src/main/java/io/grpc/util/AdvancedTlsX509KeyManager.java @@ -141,6 +141,21 @@ public Closeable updateIdentityCredentialsFromFile(File keyFile, File certFile, }; } + /** + * Updates the private key and certificate chains from the local file paths. + * + * @param keyFile the file on disk holding the private key + * @param certFile the file on disk holding the certificate chain + */ + public void updateIdentityCredentialsFromFile(File keyFile, File certFile) throws IOException, + GeneralSecurityException { + UpdateResult newResult = readAndUpdate(keyFile, certFile, 0, 0); + if (!newResult.success) { + throw new GeneralSecurityException( + "Files were unmodified before their initial update. Probably a bug."); + } + } + private static class KeyInfo { // The private key and the cert chain we will use to send to peers to prove our identity. final PrivateKey key; diff --git a/core/src/main/java/io/grpc/util/AdvancedTlsX509TrustManager.java b/core/src/main/java/io/grpc/util/AdvancedTlsX509TrustManager.java index ad69fe4abfa..51bf57aeb34 100644 --- a/core/src/main/java/io/grpc/util/AdvancedTlsX509TrustManager.java +++ b/core/src/main/java/io/grpc/util/AdvancedTlsX509TrustManager.java @@ -255,6 +255,20 @@ public void run() { } } + /** + * Updates the trust certificates from a local file path. + * + * @param trustCertFile the file on disk holding the trust certificates + */ + public void updateTrustCredentialsFromFile(File trustCertFile) throws IOException, + GeneralSecurityException { + long updatedTime = readAndUpdate(trustCertFile, 0); + if (updatedTime == 0) { + throw new GeneralSecurityException( + "Files were unmodified before their initial update. Probably a bug."); + } + } + /** * Reads the trust certificates specified in the path location, and update the key store if the * modified time has changed since last read. diff --git a/netty/src/test/java/io/grpc/netty/AdvancedTlsTest.java b/netty/src/test/java/io/grpc/netty/AdvancedTlsTest.java index 9e0d8170c40..6b5a96b45ab 100644 --- a/netty/src/test/java/io/grpc/netty/AdvancedTlsTest.java +++ b/netty/src/test/java/io/grpc/netty/AdvancedTlsTest.java @@ -388,6 +388,44 @@ public void onFileReloadingKeyManagerTrustManagerTest() throws Exception { clientTrustShutdown.close(); } + @Test + public void onFileLoadingKeyManagerTrustManagerTest() throws Exception { + // Create & start a server. + AdvancedTlsX509KeyManager serverKeyManager = new AdvancedTlsX509KeyManager(); + serverKeyManager.updateIdentityCredentialsFromFile(serverKey0File, serverCert0File); + AdvancedTlsX509TrustManager serverTrustManager = AdvancedTlsX509TrustManager.newBuilder() + .setVerification(Verification.CERTIFICATE_ONLY_VERIFICATION) + .build(); + serverTrustManager.updateTrustCredentialsFromFile(caCertFile); + ServerCredentials serverCredentials = TlsServerCredentials.newBuilder() + .keyManager(serverKeyManager).trustManager(serverTrustManager) + .clientAuth(ClientAuth.REQUIRE).build(); + server = Grpc.newServerBuilderForPort(0, serverCredentials).addService( + new SimpleServiceImpl()).build().start(); + // Create a client to connect. + AdvancedTlsX509KeyManager clientKeyManager = new AdvancedTlsX509KeyManager(); + clientKeyManager.updateIdentityCredentialsFromFile(clientKey0File, clientCert0File); + AdvancedTlsX509TrustManager clientTrustManager = AdvancedTlsX509TrustManager.newBuilder() + .setVerification(Verification.CERTIFICATE_AND_HOST_NAME_VERIFICATION) + .build(); + clientTrustManager.updateTrustCredentialsFromFile(caCertFile); + ChannelCredentials channelCredentials = TlsChannelCredentials.newBuilder() + .keyManager(clientKeyManager).trustManager(clientTrustManager).build(); + channel = Grpc.newChannelBuilderForAddress("localhost", server.getPort(), channelCredentials) + .overrideAuthority("foo.test.google.com.au").build(); + // Start the connection. + try { + SimpleServiceGrpc.SimpleServiceBlockingStub client = + SimpleServiceGrpc.newBlockingStub(channel); + // Send an actual request, via the full GRPC & network stack, and check that a proper + // response comes back. + client.unaryRpc(SimpleRequest.getDefaultInstance()); + } catch (StatusRuntimeException e) { + e.printStackTrace(); + fail("Find error: " + e.getMessage()); + } + } + @Test public void onFileReloadingKeyManagerBadInitialContentTest() throws Exception { exceptionRule.expect(GeneralSecurityException.class); From e4a13778e00aa03d1a9ee5cd2cfd6023d1b4f778 Mon Sep 17 00:00:00 2001 From: yifeizhuang Date: Mon, 20 Sep 2021 13:46:36 -0700 Subject: [PATCH 25/76] xds: disable rbac by default (#8537) --- .../java/io/grpc/xds/ClientXdsClient.java | 6 ++-- .../java/io/grpc/xds/XdsServerWrapper.java | 28 ++++++++++--------- .../io/grpc/xds/ClientXdsClientDataTest.java | 3 +- .../io/grpc/xds/ClientXdsClientTestBase.java | 5 ++++ 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/ClientXdsClient.java b/xds/src/main/java/io/grpc/xds/ClientXdsClient.java index 3bef4c416e0..f39992c24ac 100644 --- a/xds/src/main/java/io/grpc/xds/ClientXdsClient.java +++ b/xds/src/main/java/io/grpc/xds/ClientXdsClient.java @@ -138,8 +138,8 @@ final class ClientXdsClient extends AbstractXdsClient { || Boolean.parseBoolean(System.getenv("GRPC_XDS_EXPERIMENTAL_ENABLE_RETRY")); @VisibleForTesting static boolean enableRbac = - Strings.isNullOrEmpty(System.getenv("GRPC_XDS_EXPERIMENTAL_RBAC")) - || Boolean.parseBoolean(System.getenv("GRPC_XDS_EXPERIMENTAL_RBAC")); + !Strings.isNullOrEmpty(System.getenv("GRPC_XDS_EXPERIMENTAL_RBAC")) + && Boolean.parseBoolean(System.getenv("GRPC_XDS_EXPERIMENTAL_RBAC")); private static final String TYPE_URL_HTTP_CONNECTION_MANAGER_V2 = "type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2" @@ -222,7 +222,7 @@ protected void handleLdsResponse(String versionInfo, List resources, String listener, retainedRdsResources, enableFaultInjection && isResourceV3); } else { ldsUpdate = processServerSideListener( - listener, retainedRdsResources, enableRbac); + listener, retainedRdsResources, enableRbac && isResourceV3); } } catch (ResourceInvalidException e) { errors.add( diff --git a/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java b/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java index fdc5f099bfe..5f7cc43d670 100644 --- a/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java +++ b/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java @@ -515,20 +515,22 @@ private ImmutableMap generatePerRouteInterceptors( Map selectedOverrideConfigs = new HashMap<>(virtualHost.filterConfigOverrides()); selectedOverrideConfigs.putAll(route.filterConfigOverrides()); - for (NamedFilterConfig namedFilterConfig : namedFilterConfigs) { - FilterConfig filterConfig = namedFilterConfig.filterConfig; - Filter filter = filterRegistry.get(filterConfig.typeUrl()); - if (filter instanceof ServerInterceptorBuilder) { - ServerInterceptor interceptor = - ((ServerInterceptorBuilder) filter).buildServerInterceptor( - filterConfig, selectedOverrideConfigs.get(namedFilterConfig.name)); - if (interceptor != null) { - filterInterceptors.add(interceptor); + if (namedFilterConfigs != null) { + for (NamedFilterConfig namedFilterConfig : namedFilterConfigs) { + FilterConfig filterConfig = namedFilterConfig.filterConfig; + Filter filter = filterRegistry.get(filterConfig.typeUrl()); + if (filter instanceof ServerInterceptorBuilder) { + ServerInterceptor interceptor = + ((ServerInterceptorBuilder) filter).buildServerInterceptor( + filterConfig, selectedOverrideConfigs.get(namedFilterConfig.name)); + if (interceptor != null) { + filterInterceptors.add(interceptor); + } + } else { + logger.log(Level.WARNING, "HttpFilterConfig(type URL: " + + filterConfig.typeUrl() + ") is not supported on server-side. " + + "Probably a bug at ClientXdsClient verification."); } - } else { - logger.log(Level.WARNING, "HttpFilterConfig(type URL: " - + filterConfig.typeUrl() + ") is not supported on server-side. " - + "Probably a bug at ClientXdsClient verification."); } } ServerInterceptor interceptor = combineInterceptors(filterInterceptors); diff --git a/xds/src/test/java/io/grpc/xds/ClientXdsClientDataTest.java b/xds/src/test/java/io/grpc/xds/ClientXdsClientDataTest.java index 597ca7df2d9..17ca907da42 100644 --- a/xds/src/test/java/io/grpc/xds/ClientXdsClientDataTest.java +++ b/xds/src/test/java/io/grpc/xds/ClientXdsClientDataTest.java @@ -138,7 +138,8 @@ public void setUp() { originalEnableRetry = ClientXdsClient.enableRetry; assertThat(originalEnableRetry).isTrue(); originalEnableRbac = ClientXdsClient.enableRbac; - assertThat(originalEnableRbac).isTrue(); + assertThat(originalEnableRbac).isFalse(); + ClientXdsClient.enableRbac = true; } @After diff --git a/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java b/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java index e66c73163be..3a9ab23aa74 100644 --- a/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java +++ b/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java @@ -246,6 +246,7 @@ public long currentTimeNanos() { private ManagedChannel channel; private ClientXdsClient xdsClient; private boolean originalEnableFaultInjection; + private boolean originalEnableRbac; @Before public void setUp() throws IOException { @@ -258,6 +259,9 @@ public void setUp() throws IOException { // Start the server and the client. originalEnableFaultInjection = ClientXdsClient.enableFaultInjection; ClientXdsClient.enableFaultInjection = true; + originalEnableRbac = ClientXdsClient.enableRbac; + assertThat(originalEnableRbac).isFalse(); + ClientXdsClient.enableRbac = true; final String serverName = InProcessServerBuilder.generateName(); cleanupRule.register( InProcessServerBuilder @@ -297,6 +301,7 @@ public void setUp() throws IOException { @After public void tearDown() { ClientXdsClient.enableFaultInjection = originalEnableFaultInjection; + ClientXdsClient.enableRbac = originalEnableRbac; xdsClient.shutdown(); channel.shutdown(); // channel not owned by XdsClient assertThat(adsEnded.get()).isTrue(); From 25022f6846ec1fdd8003789cba32d4ca981c3171 Mon Sep 17 00:00:00 2001 From: ZHANG Dapeng Date: Tue, 21 Sep 2021 09:14:54 -0700 Subject: [PATCH 26/76] dep: bump netty to 4.1.63.Final and tcnative to 2.0.38.Final (#8167) Upgrade Netty. This should also resolve #7830. --- SECURITY.md | 3 ++- build.gradle | 4 ++-- repositories.bzl | 24 ++++++++++++------------ 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index d2482e18cbd..df9061eab8f 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -408,7 +408,8 @@ grpc-netty version | netty-handler version | netty-tcnative-boringssl-static ver 1.28.x | 4.1.45.Final | 2.0.28.Final 1.29.x-1.31.x | 4.1.48.Final | 2.0.30.Final 1.32.x-1.34.x | 4.1.51.Final | 2.0.31.Final -1.35.x- | 4.1.52.Final | 2.0.34.Final +1.35.x-1.41.x | 4.1.52.Final | 2.0.34.Final +1.42.x- | 4.1.63.Final | 2.0.38.Final _(grpc-netty-shaded avoids issues with keeping these versions in sync.)_ diff --git a/build.gradle b/build.gradle index fbd55703aea..3c746a9dab7 100644 --- a/build.gradle +++ b/build.gradle @@ -54,7 +54,7 @@ subprojects { protocPluginBaseName = 'protoc-gen-grpc-java' javaPluginPath = "$rootDir/compiler/build/exe/java_plugin/$protocPluginBaseName$exeSuffix" - nettyVersion = '4.1.52.Final' + nettyVersion = '4.1.63.Final' guavaVersion = '30.1-android' googleauthVersion = '0.22.2' protobufVersion = '3.17.2' @@ -176,7 +176,7 @@ subprojects { // SECURITY.md (multiple occurrences) // examples/example-tls/build.gradle // examples/example-tls/pom.xml - netty_tcnative: 'io.netty:netty-tcnative-boringssl-static:2.0.34.Final', + netty_tcnative: 'io.netty:netty-tcnative-boringssl-static:2.0.38.Final', conscrypt: 'org.conscrypt:conscrypt-openjdk-uber:2.5.1', re2j: 'com.google.re2j:re2j:1.5', diff --git a/repositories.bzl b/repositories.bzl index 0d6e9ab2f74..3222e15ff65 100644 --- a/repositories.bzl +++ b/repositories.bzl @@ -25,18 +25,18 @@ IO_GRPC_GRPC_JAVA_ARTIFACTS = [ "com.google.truth:truth:1.0.1", "com.squareup.okhttp:okhttp:2.7.4", "com.squareup.okio:okio:1.17.5", - "io.netty:netty-buffer:4.1.52.Final", - "io.netty:netty-codec-http2:4.1.52.Final", - "io.netty:netty-codec-http:4.1.52.Final", - "io.netty:netty-codec-socks:4.1.52.Final", - "io.netty:netty-codec:4.1.52.Final", - "io.netty:netty-common:4.1.52.Final", - "io.netty:netty-handler-proxy:4.1.52.Final", - "io.netty:netty-handler:4.1.52.Final", - "io.netty:netty-resolver:4.1.52.Final", - "io.netty:netty-tcnative-boringssl-static:2.0.34.Final", - "io.netty:netty-transport-native-epoll:jar:linux-x86_64:4.1.52.Final", - "io.netty:netty-transport:4.1.52.Final", + "io.netty:netty-buffer:4.1.63.Final", + "io.netty:netty-codec-http2:4.1.63.Final", + "io.netty:netty-codec-http:4.1.63.Final", + "io.netty:netty-codec-socks:4.1.63.Final", + "io.netty:netty-codec:4.1.63.Final", + "io.netty:netty-common:4.1.63.Final", + "io.netty:netty-handler-proxy:4.1.63.Final", + "io.netty:netty-handler:4.1.63.Final", + "io.netty:netty-resolver:4.1.63.Final", + "io.netty:netty-tcnative-boringssl-static:2.0.38.Final", + "io.netty:netty-transport-native-epoll:jar:linux-x86_64:4.1.63.Final", + "io.netty:netty-transport:4.1.63.Final", "io.opencensus:opencensus-api:0.24.0", "io.opencensus:opencensus-contrib-grpc-metrics:0.24.0", "io.perfmark:perfmark-api:0.23.0", From 29d238afca83a73ac5c9aeee1539ac1dd7066b0c Mon Sep 17 00:00:00 2001 From: ZHANG Dapeng Date: Tue, 21 Sep 2021 10:38:41 -0700 Subject: [PATCH 27/76] api,stub: clarify StreamObserver and Listener param type (#8544) --- api/src/main/java/io/grpc/ClientCall.java | 2 ++ stub/src/main/java/io/grpc/stub/CallStreamObserver.java | 2 ++ .../main/java/io/grpc/stub/ClientCallStreamObserver.java | 6 ++---- stub/src/main/java/io/grpc/stub/ClientCalls.java | 9 +++++---- .../main/java/io/grpc/stub/ServerCallStreamObserver.java | 7 +++---- stub/src/main/java/io/grpc/stub/ServerCalls.java | 2 +- 6 files changed, 15 insertions(+), 13 deletions(-) diff --git a/api/src/main/java/io/grpc/ClientCall.java b/api/src/main/java/io/grpc/ClientCall.java index b572f1ee55b..2a8716a9249 100644 --- a/api/src/main/java/io/grpc/ClientCall.java +++ b/api/src/main/java/io/grpc/ClientCall.java @@ -108,6 +108,8 @@ public abstract class ClientCall { * an instance from multiple threads, but only one call simultaneously. A single thread may * interleave calls to multiple instances, so implementations using ThreadLocals must be careful * to avoid leaking inappropriate state (e.g., clearing the ThreadLocal before returning). + * + * @param type of message received. */ public abstract static class Listener { diff --git a/stub/src/main/java/io/grpc/stub/CallStreamObserver.java b/stub/src/main/java/io/grpc/stub/CallStreamObserver.java index b014e9cfc25..52dd046831d 100644 --- a/stub/src/main/java/io/grpc/stub/CallStreamObserver.java +++ b/stub/src/main/java/io/grpc/stub/CallStreamObserver.java @@ -49,6 +49,8 @@ * *

DO NOT MOCK: The API is too complex to reliably mock. Use InProcessChannelBuilder to create * "real" RPCs suitable for testing. + * + * @param type of outbound message. */ @ExperimentalApi("https://github.com/grpc/grpc-java/issues/8499") public abstract class CallStreamObserver implements StreamObserver { diff --git a/stub/src/main/java/io/grpc/stub/ClientCallStreamObserver.java b/stub/src/main/java/io/grpc/stub/ClientCallStreamObserver.java index 5fb70c76de3..8f420fa77e4 100644 --- a/stub/src/main/java/io/grpc/stub/ClientCallStreamObserver.java +++ b/stub/src/main/java/io/grpc/stub/ClientCallStreamObserver.java @@ -29,7 +29,7 @@ *

DO NOT MOCK: The API is too complex to reliably mock. Use InProcessChannelBuilder to create * "real" RPCs suitable for testing and make a fake for the server-side. */ -public abstract class ClientCallStreamObserver extends CallStreamObserver { +public abstract class ClientCallStreamObserver extends CallStreamObserver { /** * Prevent any further processing for this {@code ClientCallStreamObserver}. No further messages * will be received. The server is informed of cancellations, but may not stop processing the @@ -78,9 +78,7 @@ public void disableAutoRequestWithInitial(int request) { * thread will always be used to execute the {@link Runnable}, it is guaranteed that executions * are serialized with calls to the 'inbound' {@link StreamObserver}. * - *

On client-side this method may only be called during {@link - * ClientResponseObserver#beforeStart}. On server-side it may only be called during the initial - * call to the application, before the service returns its {@code StreamObserver}. + *

May only be called during {@link ClientResponseObserver#beforeStart}. * *

Because there is a processing delay to deliver this notification, it is possible for * concurrent writes to cause {@code isReady() == false} within this callback. Handle "spurious" diff --git a/stub/src/main/java/io/grpc/stub/ClientCalls.java b/stub/src/main/java/io/grpc/stub/ClientCalls.java index 0266cb7d9af..7456948ddf1 100644 --- a/stub/src/main/java/io/grpc/stub/ClientCalls.java +++ b/stub/src/main/java/io/grpc/stub/ClientCalls.java @@ -337,9 +337,10 @@ private abstract static class StartableListener extends ClientCall.Listener extends ClientCallStreamObserver { + private static final class CallToStreamObserverAdapter + extends ClientCallStreamObserver { private boolean frozen; - private final ClientCall call; + private final ClientCall call; private final boolean streamingResponse; private Runnable onReadyHandler; private int initialRequest = 1; @@ -348,7 +349,7 @@ private static final class CallToStreamObserverAdapter extends ClientCallStre private boolean completed = false; // Non private to avoid synthetic class - CallToStreamObserverAdapter(ClientCall call, boolean streamingResponse) { + CallToStreamObserverAdapter(ClientCall call, boolean streamingResponse) { this.call = call; this.streamingResponse = streamingResponse; } @@ -358,7 +359,7 @@ private void freeze() { } @Override - public void onNext(T value) { + public void onNext(ReqT value) { checkState(!aborted, "Stream was terminated by error, no further calls are allowed"); checkState(!completed, "Stream is already completed, no further calls are allowed"); call.sendMessage(value); diff --git a/stub/src/main/java/io/grpc/stub/ServerCallStreamObserver.java b/stub/src/main/java/io/grpc/stub/ServerCallStreamObserver.java index 2ac008b269a..00d6f5d3c7c 100644 --- a/stub/src/main/java/io/grpc/stub/ServerCallStreamObserver.java +++ b/stub/src/main/java/io/grpc/stub/ServerCallStreamObserver.java @@ -27,7 +27,7 @@ *

DO NOT MOCK: The API is too complex to reliably mock. Use InProcessChannelBuilder to create * "real" RPCs suitable for testing and interact with the server using a normal client stub. */ -public abstract class ServerCallStreamObserver extends CallStreamObserver { +public abstract class ServerCallStreamObserver extends CallStreamObserver { /** * Returns {@code true} when the call is cancelled and the server is encouraged to abort @@ -113,9 +113,8 @@ public void disableAutoRequest() { * thread will always be used to execute the {@link Runnable}, it is guaranteed that executions * are serialized with calls to the 'inbound' {@link StreamObserver}. * - *

On client-side this method may only be called during {@link - * ClientResponseObserver#beforeStart}. On server-side it may only be called during the initial - * call to the application, before the service returns its {@code StreamObserver}. + *

May only be called during the initial call to the application, before the service returns + * its {@code StreamObserver}. * *

Because there is a processing delay to deliver this notification, it is possible for * concurrent writes to cause {@code isReady() == false} within this callback. Handle "spurious" diff --git a/stub/src/main/java/io/grpc/stub/ServerCalls.java b/stub/src/main/java/io/grpc/stub/ServerCalls.java index ba08139b716..0f7d6d09ab1 100644 --- a/stub/src/main/java/io/grpc/stub/ServerCalls.java +++ b/stub/src/main/java/io/grpc/stub/ServerCalls.java @@ -447,7 +447,7 @@ public static void asyncUnimplementedUnaryCall( * @param methodDescriptor of method for which error will be thrown. * @param responseObserver on which error will be set. */ - public static StreamObserver asyncUnimplementedStreamingCall( + public static StreamObserver asyncUnimplementedStreamingCall( MethodDescriptor methodDescriptor, StreamObserver responseObserver) { // NB: For streaming call we want to do the same as for unary call. Fail-fast by setting error // on responseObserver and then return no-op observer. From a6abb1b8d97b3dda354673a4e10fab6a1472fc0c Mon Sep 17 00:00:00 2001 From: Piotr Morgwai Kotarbinski Date: Wed, 22 Sep 2021 01:31:04 +0700 Subject: [PATCH 28/76] stub: add ServerCallStreamObserver.setOnCloseHandler(...) (#8452) This allows for user code to be notified when the messages are actually put on the wire and the stream is closed. Fixes #5895 --- .../grpc/stub/ServerCallStreamObserver.java | 24 +++++++ .../main/java/io/grpc/stub/ServerCalls.java | 23 ++++++ .../java/io/grpc/stub/ServerCallsTest.java | 72 +++++++++++++++++++ 3 files changed, 119 insertions(+) diff --git a/stub/src/main/java/io/grpc/stub/ServerCallStreamObserver.java b/stub/src/main/java/io/grpc/stub/ServerCallStreamObserver.java index 00d6f5d3c7c..e31cae1fad4 100644 --- a/stub/src/main/java/io/grpc/stub/ServerCallStreamObserver.java +++ b/stub/src/main/java/io/grpc/stub/ServerCallStreamObserver.java @@ -16,6 +16,8 @@ package io.grpc.stub; +import io.grpc.ExperimentalApi; + /** * A refinement of {@link CallStreamObserver} to allows for interaction with call * cancellation events on the server side. An instance of this class is obtained by casting the @@ -145,4 +147,26 @@ public void disableAutoRequest() { */ @Override public abstract void setMessageCompression(boolean enable); + + /** + * Sets a {@link Runnable} to be executed when the call is closed cleanly from the server's + * point of view: either {@link #onCompleted()} or {@link #onError(Throwable)} has been called, + * all the messages and trailing metadata have been sent and the stream has been closed. Note + * however that the client still may have not received all the messages due to network delay, + * client crashes, and cancellation races. + * + *

Exactly one of {@code onCloseHandler} and {@code onCancelHandler} is guaranteed to be called + * when the RPC terminates.

+ * + *

It is guaranteed that execution of {@code onCloseHandler} is serialized with calls to + * the 'inbound' {@link StreamObserver}. That also means that the callback will be delayed if + * other callbacks are running.

+ * + *

This method may only be called during the initial call to the application, before the + * service returns its {@link StreamObserver request observer}.

+ * + * @param onCloseHandler to execute when the call has been closed cleanly. + */ + @ExperimentalApi("https://github.com/grpc/grpc-java/issues/8467") + public abstract void setOnCloseHandler(Runnable onCloseHandler); } diff --git a/stub/src/main/java/io/grpc/stub/ServerCalls.java b/stub/src/main/java/io/grpc/stub/ServerCalls.java index 0f7d6d09ab1..09f86d0364c 100644 --- a/stub/src/main/java/io/grpc/stub/ServerCalls.java +++ b/stub/src/main/java/io/grpc/stub/ServerCalls.java @@ -206,6 +206,13 @@ public void onReady() { responseObserver.onReadyHandler.run(); } } + + @Override + public void onComplete() { + if (responseObserver.onCloseHandler != null) { + responseObserver.onCloseHandler.run(); + } + } } } @@ -291,6 +298,13 @@ public void onReady() { responseObserver.onReadyHandler.run(); } } + + @Override + public void onComplete() { + if (responseObserver.onCloseHandler != null) { + responseObserver.onCloseHandler.run(); + } + } } } @@ -320,6 +334,7 @@ private static final class ServerCallStreamObserverImpl private Runnable onCancelHandler; private boolean aborted = false; private boolean completed = false; + private Runnable onCloseHandler; // Non private to avoid synthetic class ServerCallStreamObserverImpl(ServerCall call, boolean serverStreamingOrBidi) { @@ -423,6 +438,14 @@ public void disableAutoRequest() { public void request(int count) { call.request(count); } + + @Override + public void setOnCloseHandler(Runnable onCloseHandler) { + checkState(!frozen, "Cannot alter onCloseHandler after initialization. May only be called " + + "during the initial call to the application, before the service returns its " + + "StreamObserver"); + this.onCloseHandler = onCloseHandler; + } } /** diff --git a/stub/src/test/java/io/grpc/stub/ServerCallsTest.java b/stub/src/test/java/io/grpc/stub/ServerCallsTest.java index a2a1ef93961..7227d26c5b8 100644 --- a/stub/src/test/java/io/grpc/stub/ServerCallsTest.java +++ b/stub/src/test/java/io/grpc/stub/ServerCallsTest.java @@ -199,6 +199,53 @@ public StreamObserver invoke(StreamObserver responseObserver) callObserver.get().onCompleted(); } + @Test + public void onCloseHandlerCalledIfSetInStreamingClientCall() throws Exception { + final AtomicBoolean onCloseHandlerCalled = new AtomicBoolean(); + ServerCallHandler callHandler = ServerCalls.asyncBidiStreamingCall( + new ServerCalls.BidiStreamingMethod() { + @Override + public StreamObserver invoke(StreamObserver responseObserver) { + ServerCallStreamObserver serverCallObserver = + (ServerCallStreamObserver) responseObserver; + serverCallObserver.setOnCloseHandler(new Runnable() { + @Override + public void run() { + onCloseHandlerCalled.set(true); + } + }); + return new ServerCalls.NoopStreamObserver<>(); + } + }); + ServerCall.Listener callListener = callHandler.startCall(serverCall, new Metadata()); + callListener.onComplete(); + assertTrue(onCloseHandlerCalled.get()); + } + + @Test + public void onCloseHandlerCalledIfSetInUnaryClientCall() throws Exception { + final AtomicBoolean onCloseHandlerCalled = new AtomicBoolean(); + ServerCallHandler callHandler = ServerCalls.asyncServerStreamingCall( + new ServerCalls.ServerStreamingMethod() { + @Override + public void invoke(Integer request, StreamObserver responseObserver) { + ServerCallStreamObserver serverCallObserver = + (ServerCallStreamObserver) responseObserver; + serverCallObserver.setOnCloseHandler(new Runnable() { + @Override + public void run() { + onCloseHandlerCalled.set(true); + } + }); + } + }); + ServerCall.Listener callListener = callHandler.startCall(serverCall, new Metadata()); + callListener.onMessage(0); + callListener.onHalfClose(); + callListener.onComplete(); + assertTrue(onCloseHandlerCalled.get()); + } + @Test public void cannotSetOnCancelHandlerAfterServiceInvocation() throws Exception { final AtomicReference> callObserver = @@ -255,6 +302,31 @@ public void run() { } } + @Test + public void cannotSetOnCloseHandlerAfterServiceInvocation() throws Exception { + final AtomicReference> callObserver = new AtomicReference<>(); + ServerCallHandler callHandler = ServerCalls.asyncBidiStreamingCall( + new ServerCalls.BidiStreamingMethod() { + @Override + public StreamObserver invoke(StreamObserver responseObserver) { + callObserver.set((ServerCallStreamObserver) responseObserver); + return new ServerCalls.NoopStreamObserver<>(); + } + }); + ServerCall.Listener callListener = callHandler.startCall(serverCall, new Metadata()); + callListener.onMessage(1); + try { + callObserver.get().setOnCloseHandler(new Runnable() { + @Override + public void run() { + } + }); + fail("Cannot set onReady after service invocation"); + } catch (IllegalStateException expected) { + // Expected + } + } + @Test public void cannotDisableAutoRequestAfterServiceInvocation() throws Exception { final AtomicReference> callObserver = From f33daf0d9e0c2dbcd530d594c2cc5697c55df1df Mon Sep 17 00:00:00 2001 From: yifeizhuang Date: Tue, 21 Sep 2021 16:29:07 -0700 Subject: [PATCH 29/76] xds: implement equals hashcode in rbac matcher tree (#8546) --- xds/src/main/java/io/grpc/xds/RbacFilter.java | 37 ++-- .../rbac/engine/GrpcAuthorizationEngine.java | 178 ++++++++++-------- .../test/java/io/grpc/xds/RbacFilterTest.java | 20 +- .../engine/GrpcAuthorizationEngineTest.java | 149 ++++++++++----- 4 files changed, 225 insertions(+), 159 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/RbacFilter.java b/xds/src/main/java/io/grpc/xds/RbacFilter.java index 39f91b475ae..be49df31b14 100644 --- a/xds/src/main/java/io/grpc/xds/RbacFilter.java +++ b/xds/src/main/java/io/grpc/xds/RbacFilter.java @@ -126,14 +126,15 @@ static ConfigOrError parseRbacConfig(RBAC rbac) { return ConfigOrError.fromError( "Policy.condition and Policy.checked_condition must not set: " + entry.getKey()); } - policyMatchers.add(new PolicyMatcher(entry.getKey(), + policyMatchers.add(PolicyMatcher.create(entry.getKey(), parsePermissionList(policy.getPermissionsList()), parsePrincipalList(policy.getPrincipalsList()))); } catch (Exception e) { return ConfigOrError.fromError("Encountered error parsing policy: " + e); } } - return ConfigOrError.fromConfig(RbacConfig.create(new AuthConfig(policyMatchers, authAction))); + return ConfigOrError.fromConfig(RbacConfig.create( + AuthConfig.create(policyMatchers, authAction))); } @Override @@ -195,7 +196,7 @@ private static OrMatcher parsePermissionList(List permissions) { for (Permission permission : permissions) { anyMatch.add(parsePermission(permission)); } - return new OrMatcher(anyMatch); + return OrMatcher.create(anyMatch); } private static Matcher parsePermission(Permission permission) { @@ -205,7 +206,7 @@ private static Matcher parsePermission(Permission permission) { for (Permission p : permission.getAndRules().getRulesList()) { andMatch.add(parsePermission(p)); } - return new AndMatcher(andMatch); + return AndMatcher.create(andMatch); case OR_RULES: return parsePermissionList(permission.getOrRules().getRulesList()); case ANY: @@ -221,9 +222,9 @@ private static Matcher parsePermission(Permission permission) { case DESTINATION_PORT_RANGE: return parseDestinationPortRangeMatcher(permission.getDestinationPortRange()); case NOT_RULE: - return new InvertMatcher(parsePermission(permission.getNotRule())); + return InvertMatcher.create(parsePermission(permission.getNotRule())); case METADATA: // hard coded, never match. - return new InvertMatcher(AlwaysTrueMatcher.INSTANCE); + return InvertMatcher.create(AlwaysTrueMatcher.INSTANCE); case REQUESTED_SERVER_NAME: return parseRequestedServerNameMatcher(permission.getRequestedServerName()); case RULE_NOT_SET: @@ -238,7 +239,7 @@ private static OrMatcher parsePrincipalList(List principals) { for (Principal principal: principals) { anyMatch.add(parsePrincipal(principal)); } - return new OrMatcher(anyMatch); + return OrMatcher.create(anyMatch); } private static Matcher parsePrincipal(Principal principal) { @@ -250,7 +251,7 @@ private static Matcher parsePrincipal(Principal principal) { for (Principal next : principal.getAndIds().getIdsList()) { nextMatchers.add(parsePrincipal(next)); } - return new AndMatcher(nextMatchers); + return AndMatcher.create(nextMatchers); case ANY: return AlwaysTrueMatcher.INSTANCE; case AUTHENTICATED: @@ -264,11 +265,11 @@ private static Matcher parsePrincipal(Principal principal) { case HEADER: return parseHeaderMatcher(principal.getHeader()); case NOT_ID: - return new InvertMatcher(parsePrincipal(principal.getNotId())); + return InvertMatcher.create(parsePrincipal(principal.getNotId())); case URL_PATH: return parsePathMatcher(principal.getUrlPath()); case METADATA: // hard coded, never match. - return new InvertMatcher(AlwaysTrueMatcher.INSTANCE); + return InvertMatcher.create(AlwaysTrueMatcher.INSTANCE); case IDENTIFIER_NOT_SET: default: throw new IllegalArgumentException( @@ -280,7 +281,7 @@ private static PathMatcher parsePathMatcher( io.envoyproxy.envoy.type.matcher.v3.PathMatcher proto) { switch (proto.getRuleCase()) { case PATH: - return new PathMatcher(MatcherParser.parseStringMatcher(proto.getPath())); + return PathMatcher.create(MatcherParser.parseStringMatcher(proto.getPath())); case RULE_NOT_SET: default: throw new IllegalArgumentException( @@ -290,7 +291,7 @@ private static PathMatcher parsePathMatcher( private static RequestedServerNameMatcher parseRequestedServerNameMatcher( io.envoyproxy.envoy.type.matcher.v3.StringMatcher proto) { - return new RequestedServerNameMatcher(MatcherParser.parseStringMatcher(proto)); + return RequestedServerNameMatcher.create(MatcherParser.parseStringMatcher(proto)); } private static AuthHeaderMatcher parseHeaderMatcher( @@ -303,30 +304,30 @@ private static AuthHeaderMatcher parseHeaderMatcher( throw new IllegalArgumentException("Invalid header matcher config: header name [:scheme] " + "is not allowed."); } - return new AuthHeaderMatcher(MatcherParser.parseHeaderMatcher(proto)); + return AuthHeaderMatcher.create(MatcherParser.parseHeaderMatcher(proto)); } private static AuthenticatedMatcher parseAuthenticatedMatcher( Principal.Authenticated proto) { Matchers.StringMatcher matcher = MatcherParser.parseStringMatcher(proto.getPrincipalName()); - return new AuthenticatedMatcher(matcher); + return AuthenticatedMatcher.create(matcher); } private static DestinationPortMatcher createDestinationPortMatcher(int port) { - return new DestinationPortMatcher(port); + return DestinationPortMatcher.create(port); } private static DestinationPortRangeMatcher parseDestinationPortRangeMatcher(Int32Range range) { - return new DestinationPortRangeMatcher(range.getStart(), range.getEnd()); + return DestinationPortRangeMatcher.create(range.getStart(), range.getEnd()); } private static DestinationIpMatcher createDestinationIpMatcher(CidrRange cidrRange) { - return new DestinationIpMatcher(Matchers.CidrMatcher.create( + return DestinationIpMatcher.create(Matchers.CidrMatcher.create( resolve(cidrRange), cidrRange.getPrefixLen().getValue())); } private static SourceIpMatcher createSourceIpMatcher(CidrRange cidrRange) { - return new SourceIpMatcher(Matchers.CidrMatcher.create( + return SourceIpMatcher.create(Matchers.CidrMatcher.create( resolve(cidrRange), cidrRange.getPrefixLen().getValue())); } diff --git a/xds/src/main/java/io/grpc/xds/internal/rbac/engine/GrpcAuthorizationEngine.java b/xds/src/main/java/io/grpc/xds/internal/rbac/engine/GrpcAuthorizationEngine.java index bb911461a27..385a055daef 100644 --- a/xds/src/main/java/io/grpc/xds/internal/rbac/engine/GrpcAuthorizationEngine.java +++ b/xds/src/main/java/io/grpc/xds/internal/rbac/engine/GrpcAuthorizationEngine.java @@ -20,6 +20,7 @@ import com.google.auto.value.AutoValue; import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; import com.google.common.io.BaseEncoding; import io.grpc.Grpc; import io.grpc.Metadata; @@ -67,14 +68,14 @@ public AuthDecision evaluate(Metadata metadata, ServerCall serverCall) { checkNotNull(serverCall, "serverCall"); String firstMatch = null; EvaluateArgs args = new EvaluateArgs(metadata, serverCall); - for (PolicyMatcher policyMatcher : authConfig.policies) { + for (PolicyMatcher policyMatcher : authConfig.policies()) { if (policyMatcher.matches(args)) { - firstMatch = policyMatcher.name; + firstMatch = policyMatcher.name(); break; } } Action decisionType = Action.DENY; - if (Action.DENY.equals(authConfig.action) == (firstMatch == null)) { + if (Action.DENY.equals(authConfig.action()) == (firstMatch == null)) { decisionType = Action.ALLOW; } log.log(Level.FINER, "RBAC decision: {0}, policy match: {1}.", @@ -103,13 +104,15 @@ static AuthDecision create(Action decisionType, @Nullable String matchingPolicy) } /** Represents authorization config policy that the engine will evaluate against. */ - public static final class AuthConfig { - private final List policies; - private final Action action; + @AutoValue + public abstract static class AuthConfig { + public abstract ImmutableList policies(); + + public abstract Action action(); - public AuthConfig(List policies, Action action) { - this.policies = Collections.unmodifiableList(new ArrayList<>(policies)); - this.action = action; + public static AuthConfig create(List policies, Action action) { + return new AutoValue_GrpcAuthorizationEngine_AuthConfig( + ImmutableList.copyOf(policies), action); } } @@ -121,33 +124,36 @@ public AuthConfig(List policies, Action action) { *

Currently we only support matching some of the request fields. Those unsupported fields are * considered not match until we stop ignoring them. */ - public static final class PolicyMatcher implements Matcher { - private final OrMatcher permissions; - private final OrMatcher principals; - private final String name; + @AutoValue + public abstract static class PolicyMatcher implements Matcher { + public abstract String name(); + + public abstract OrMatcher permissions(); + + public abstract OrMatcher principals(); /** Constructs a matcher for one RBAC policy. */ - public PolicyMatcher(String name, OrMatcher permissions, OrMatcher principals) { - this.name = name; - this.permissions = permissions; - this.principals = principals; + public static PolicyMatcher create(String name, OrMatcher permissions, OrMatcher principals) { + return new AutoValue_GrpcAuthorizationEngine_PolicyMatcher(name, permissions, principals); } @Override public boolean matches(EvaluateArgs args) { - return permissions.matches(args) && principals.matches(args); + return permissions().matches(args) && principals().matches(args); } } - public static final class AuthenticatedMatcher implements Matcher { - private final Matchers.StringMatcher delegate; + @AutoValue + public abstract static class AuthenticatedMatcher implements Matcher { + @Nullable + public abstract Matchers.StringMatcher delegate(); /** * Passing in null will match all authenticated user, i.e. SSL session is present. * https://github.com/envoyproxy/envoy/blob/main/api/envoy/config/rbac/v3/rbac.proto#L240 * */ - public AuthenticatedMatcher(@Nullable Matchers.StringMatcher delegate) { - this.delegate = delegate; + public static AuthenticatedMatcher create(@Nullable Matchers.StringMatcher delegate) { + return new AutoValue_GrpcAuthorizationEngine_AuthenticatedMatcher(delegate); } @Override @@ -159,11 +165,11 @@ public boolean matches(EvaluateArgs args) { return false; } // Connection is authenticated, so returns match when delegated string matcher is not present. - if (delegate == null) { + if (delegate() == null) { return true; } for (String name : principalNames) { - if (delegate.matches(name)) { + if (delegate().matches(name)) { return true; } } @@ -171,98 +177,105 @@ public boolean matches(EvaluateArgs args) { } } - public static final class DestinationIpMatcher implements Matcher { - private final Matchers.CidrMatcher delegate; + @AutoValue + public abstract static class DestinationIpMatcher implements Matcher { + public abstract Matchers.CidrMatcher delegate(); - public DestinationIpMatcher(Matchers.CidrMatcher delegate) { - this.delegate = checkNotNull(delegate, "delegate"); + public static DestinationIpMatcher create(Matchers.CidrMatcher delegate) { + return new AutoValue_GrpcAuthorizationEngine_DestinationIpMatcher(delegate); } @Override public boolean matches(EvaluateArgs args) { - return delegate.matches(args.getDestinationIp()); + return delegate().matches(args.getDestinationIp()); } } - public static final class SourceIpMatcher implements Matcher { - private final Matchers.CidrMatcher delegate; + @AutoValue + public abstract static class SourceIpMatcher implements Matcher { + public abstract Matchers.CidrMatcher delegate(); - public SourceIpMatcher(Matchers.CidrMatcher delegate) { - this.delegate = checkNotNull(delegate, "delegate"); + public static SourceIpMatcher create(Matchers.CidrMatcher delegate) { + return new AutoValue_GrpcAuthorizationEngine_SourceIpMatcher(delegate); } @Override public boolean matches(EvaluateArgs args) { - return delegate.matches(args.getSourceIp()); + return delegate().matches(args.getSourceIp()); } } - public static final class PathMatcher implements Matcher { - private final Matchers.StringMatcher delegate; + @AutoValue + public abstract static class PathMatcher implements Matcher { + public abstract Matchers.StringMatcher delegate(); - public PathMatcher(Matchers.StringMatcher delegate) { - this.delegate = checkNotNull(delegate, "delegate"); + public static PathMatcher create(Matchers.StringMatcher delegate) { + return new AutoValue_GrpcAuthorizationEngine_PathMatcher(delegate); } @Override public boolean matches(EvaluateArgs args) { - return delegate.matches(args.getPath()); + return delegate().matches(args.getPath()); } } - public static final class AuthHeaderMatcher implements Matcher { - private final Matchers.HeaderMatcher delegate; + @AutoValue + public abstract static class AuthHeaderMatcher implements Matcher { + public abstract Matchers.HeaderMatcher delegate(); - public AuthHeaderMatcher(Matchers.HeaderMatcher delegate) { - this.delegate = checkNotNull(delegate, "delegate"); + public static AuthHeaderMatcher create(Matchers.HeaderMatcher delegate) { + return new AutoValue_GrpcAuthorizationEngine_AuthHeaderMatcher(delegate); } @Override public boolean matches(EvaluateArgs args) { - return delegate.matches(args.getHeader(delegate.name())); + return delegate().matches(args.getHeader(delegate().name())); } } - public static final class DestinationPortMatcher implements Matcher { - private final int port; + @AutoValue + public abstract static class DestinationPortMatcher implements Matcher { + public abstract int port(); - public DestinationPortMatcher(int port) { - this.port = port; + public static DestinationPortMatcher create(int port) { + return new AutoValue_GrpcAuthorizationEngine_DestinationPortMatcher(port); } @Override public boolean matches(EvaluateArgs args) { - return port == args.getDestinationPort(); + return port() == args.getDestinationPort(); } } - public static final class DestinationPortRangeMatcher implements Matcher { - private final int start; - private final int end; + @AutoValue + public abstract static class DestinationPortRangeMatcher implements Matcher { + public abstract int start(); + + public abstract int end(); /** Start of the range is inclusive. End of the range is exclusive.*/ - public DestinationPortRangeMatcher(int start, int end) { - this.start = start; - this.end = end; + public static DestinationPortRangeMatcher create(int start, int end) { + return new AutoValue_GrpcAuthorizationEngine_DestinationPortRangeMatcher(start, end); } @Override public boolean matches(EvaluateArgs args) { int port = args.getDestinationPort(); - return port >= start && port < end; + return port >= start() && port < end(); } } - public static final class RequestedServerNameMatcher implements Matcher { - private final Matchers.StringMatcher delegate; + @AutoValue + public abstract static class RequestedServerNameMatcher implements Matcher { + public abstract Matchers.StringMatcher delegate(); - public RequestedServerNameMatcher(Matchers.StringMatcher delegate) { - this.delegate = checkNotNull(delegate, "delegate"); + public static RequestedServerNameMatcher create(Matchers.StringMatcher delegate) { + return new AutoValue_GrpcAuthorizationEngine_RequestedServerNameMatcher(delegate); } @Override public boolean matches(EvaluateArgs args) { - return delegate.matches(args.getRequestedServerName()); + return delegate().matches(args.getRequestedServerName()); } } @@ -407,25 +420,26 @@ public interface Matcher { boolean matches(EvaluateArgs args); } - public static final class OrMatcher implements Matcher { - private final List anyMatch; + @AutoValue + public abstract static class OrMatcher implements Matcher { + public abstract ImmutableList anyMatch(); /** Matches when any of the matcher matches. */ - public OrMatcher(List matchers) { + public static OrMatcher create(List matchers) { checkNotNull(matchers, "matchers"); for (Matcher matcher : matchers) { checkNotNull(matcher, "matcher"); } - this.anyMatch = Collections.unmodifiableList(new ArrayList<>(matchers)); + return new AutoValue_GrpcAuthorizationEngine_OrMatcher(ImmutableList.copyOf(matchers)); } public static OrMatcher create(Matcher...matchers) { - return new OrMatcher(Arrays.asList(matchers)); + return OrMatcher.create(Arrays.asList(matchers)); } @Override public boolean matches(EvaluateArgs args) { - for (Matcher m : anyMatch) { + for (Matcher m : anyMatch()) { if (m.matches(args)) { return true; } @@ -434,25 +448,26 @@ public boolean matches(EvaluateArgs args) { } } - public static final class AndMatcher implements Matcher { - private final List allMatch; + @AutoValue + public abstract static class AndMatcher implements Matcher { + public abstract ImmutableList allMatch(); /** Matches when all of the matchers match. */ - public AndMatcher(List matchers) { + public static AndMatcher create(List matchers) { checkNotNull(matchers, "matchers"); for (Matcher matcher : matchers) { checkNotNull(matcher, "matcher"); } - this.allMatch = Collections.unmodifiableList(new ArrayList<>(matchers)); + return new AutoValue_GrpcAuthorizationEngine_AndMatcher(ImmutableList.copyOf(matchers)); } public static AndMatcher create(Matcher...matchers) { - return new AndMatcher(Arrays.asList(matchers)); + return AndMatcher.create(Arrays.asList(matchers)); } @Override public boolean matches(EvaluateArgs args) { - for (Matcher m : allMatch) { + for (Matcher m : allMatch()) { if (!m.matches(args)) { return false; } @@ -462,8 +477,10 @@ public boolean matches(EvaluateArgs args) { } /** Always true matcher.*/ - public static final class AlwaysTrueMatcher implements Matcher { - public static AlwaysTrueMatcher INSTANCE = new AlwaysTrueMatcher(); + @AutoValue + public abstract static class AlwaysTrueMatcher implements Matcher { + public static AlwaysTrueMatcher INSTANCE = + new AutoValue_GrpcAuthorizationEngine_AlwaysTrueMatcher(); @Override public boolean matches(EvaluateArgs args) { @@ -472,16 +489,17 @@ public boolean matches(EvaluateArgs args) { } /** Negate matcher.*/ - public static final class InvertMatcher implements Matcher { - private final Matcher toInvertMatcher; + @AutoValue + public abstract static class InvertMatcher implements Matcher { + public abstract Matcher toInvertMatcher(); - public InvertMatcher(Matcher matcher) { - this.toInvertMatcher = checkNotNull(matcher, "matcher"); + public static InvertMatcher create(Matcher matcher) { + return new AutoValue_GrpcAuthorizationEngine_InvertMatcher(matcher); } @Override public boolean matches(EvaluateArgs args) { - return !toInvertMatcher.matches(args); + return !toInvertMatcher().matches(args); } } } diff --git a/xds/src/test/java/io/grpc/xds/RbacFilterTest.java b/xds/src/test/java/io/grpc/xds/RbacFilterTest.java index 082c49ef665..c97ca5c52d9 100644 --- a/xds/src/test/java/io/grpc/xds/RbacFilterTest.java +++ b/xds/src/test/java/io/grpc/xds/RbacFilterTest.java @@ -246,10 +246,10 @@ public void testAuthorizationInterceptor() { .set(Grpc.TRANSPORT_ATTR_LOCAL_ADDR, new InetSocketAddress("1::", 20)) .build(); when(mockServerCall.getAttributes()).thenReturn(attr); - PolicyMatcher policyMatcher = new PolicyMatcher("policy-matcher", - OrMatcher.create(new DestinationPortMatcher(99999)), + PolicyMatcher policyMatcher = PolicyMatcher.create("policy-matcher", + OrMatcher.create(DestinationPortMatcher.create(99999)), OrMatcher.create(AlwaysTrueMatcher.INSTANCE)); - AuthConfig authconfig = new AuthConfig(Collections.singletonList(policyMatcher), + AuthConfig authconfig = AuthConfig.create(Collections.singletonList(policyMatcher), GrpcAuthorizationEngine.Action.ALLOW); new RbacFilter().buildServerInterceptor(RbacConfig.create(authconfig), null) .interceptCall(mockServerCall, new Metadata(), mockHandler); @@ -260,7 +260,7 @@ public void testAuthorizationInterceptor() { verify(mockServerCall).getAttributes(); verifyNoMoreInteractions(mockServerCall); - authconfig = new AuthConfig(Collections.singletonList(policyMatcher), + authconfig = AuthConfig.create(Collections.singletonList(policyMatcher), GrpcAuthorizationEngine.Action.DENY); new RbacFilter().buildServerInterceptor(RbacConfig.create(authconfig), null) .interceptCall(mockServerCall, new Metadata(), mockHandler); @@ -302,10 +302,10 @@ public void overrideConfig() { .build(); when(mockServerCall.getAttributes()).thenReturn(attr); - PolicyMatcher policyMatcher = new PolicyMatcher("policy-matcher", - OrMatcher.create(new DestinationPortMatcher(99999)), + PolicyMatcher policyMatcher = PolicyMatcher.create("policy-matcher", + OrMatcher.create(DestinationPortMatcher.create(99999)), OrMatcher.create(AlwaysTrueMatcher.INSTANCE)); - AuthConfig authconfig = new AuthConfig(Collections.singletonList(policyMatcher), + AuthConfig authconfig = AuthConfig.create(Collections.singletonList(policyMatcher), GrpcAuthorizationEngine.Action.ALLOW); RbacConfig original = RbacConfig.create(authconfig); @@ -316,10 +316,10 @@ public void overrideConfig() { ServerInterceptor interceptor = new RbacFilter().buildServerInterceptor(original, override); assertThat(interceptor).isNull(); - policyMatcher = new PolicyMatcher("policy-matcher-override", - OrMatcher.create(new DestinationPortMatcher(20)), + policyMatcher = PolicyMatcher.create("policy-matcher-override", + OrMatcher.create(DestinationPortMatcher.create(20)), OrMatcher.create(AlwaysTrueMatcher.INSTANCE)); - authconfig = new AuthConfig(Collections.singletonList(policyMatcher), + authconfig = AuthConfig.create(Collections.singletonList(policyMatcher), GrpcAuthorizationEngine.Action.ALLOW); override = RbacConfig.create(authconfig); diff --git a/xds/src/test/java/io/grpc/xds/internal/rbac/engine/GrpcAuthorizationEngineTest.java b/xds/src/test/java/io/grpc/xds/internal/rbac/engine/GrpcAuthorizationEngineTest.java index 626a4cfc275..410ffb1b462 100644 --- a/xds/src/test/java/io/grpc/xds/internal/rbac/engine/GrpcAuthorizationEngineTest.java +++ b/xds/src/test/java/io/grpc/xds/internal/rbac/engine/GrpcAuthorizationEngineTest.java @@ -104,16 +104,16 @@ public void setUp() throws Exception { @Test public void ipMatcher() throws Exception { CidrMatcher ip1 = CidrMatcher.create(InetAddress.getByName(IP_ADDR1), 24); - DestinationIpMatcher destIpMatcher = new DestinationIpMatcher(ip1); + DestinationIpMatcher destIpMatcher = DestinationIpMatcher.create(ip1); CidrMatcher ip2 = CidrMatcher.create(InetAddress.getByName(IP_ADDR2), 24); - SourceIpMatcher sourceIpMatcher = new SourceIpMatcher(ip2); - DestinationPortMatcher portMatcher = new DestinationPortMatcher(PORT); + SourceIpMatcher sourceIpMatcher = SourceIpMatcher.create(ip2); + DestinationPortMatcher portMatcher = DestinationPortMatcher.create(PORT); OrMatcher permission = OrMatcher.create(AndMatcher.create(portMatcher, destIpMatcher)); OrMatcher principal = OrMatcher.create(sourceIpMatcher); - PolicyMatcher policyMatcher = new PolicyMatcher(POLICY_NAME, permission, principal); + PolicyMatcher policyMatcher = PolicyMatcher.create(POLICY_NAME, permission, principal); GrpcAuthorizationEngine engine = new GrpcAuthorizationEngine( - new AuthConfig(Collections.singletonList(policyMatcher), Action.ALLOW)); + AuthConfig.create(Collections.singletonList(policyMatcher), Action.ALLOW)); AuthDecision decision = engine.evaluate(HEADER, serverCall); assertThat(decision.decision()).isEqualTo(Action.ALLOW); assertThat(decision.matchingPolicyName()).isEqualTo(POLICY_NAME); @@ -137,7 +137,7 @@ public void ipMatcher() throws Exception { assertThat(decision.matchingPolicyName()).isEqualTo(null); engine = new GrpcAuthorizationEngine( - new AuthConfig(Collections.singletonList(policyMatcher), Action.DENY)); + AuthConfig.create(Collections.singletonList(policyMatcher), Action.DENY)); decision = engine.evaluate(HEADER, serverCall); assertThat(decision.decision()).isEqualTo(Action.ALLOW); assertThat(decision.matchingPolicyName()).isEqualTo(null); @@ -145,51 +145,51 @@ public void ipMatcher() throws Exception { @Test public void headerMatcher() { - AuthHeaderMatcher headerMatcher = new AuthHeaderMatcher(Matchers.HeaderMatcher + AuthHeaderMatcher headerMatcher = AuthHeaderMatcher.create(Matchers.HeaderMatcher .forExactValue(HEADER_KEY, HEADER_VALUE, false)); OrMatcher principal = OrMatcher.create(headerMatcher); OrMatcher permission = OrMatcher.create( - new InvertMatcher(new DestinationPortMatcher(PORT + 1))); - PolicyMatcher policyMatcher = new PolicyMatcher(POLICY_NAME, permission, principal); + InvertMatcher.create(DestinationPortMatcher.create(PORT + 1))); + PolicyMatcher policyMatcher = PolicyMatcher.create(POLICY_NAME, permission, principal); GrpcAuthorizationEngine engine = new GrpcAuthorizationEngine( - new AuthConfig(Collections.singletonList(policyMatcher), Action.ALLOW)); + AuthConfig.create(Collections.singletonList(policyMatcher), Action.ALLOW)); AuthDecision decision = engine.evaluate(HEADER, serverCall); assertThat(decision.decision()).isEqualTo(Action.ALLOW); assertThat(decision.matchingPolicyName()).isEqualTo(POLICY_NAME); HEADER.put(Metadata.Key.of(HEADER_KEY, Metadata.ASCII_STRING_MARSHALLER), HEADER_VALUE); - headerMatcher = new AuthHeaderMatcher(Matchers.HeaderMatcher + headerMatcher = AuthHeaderMatcher.create(Matchers.HeaderMatcher .forExactValue(HEADER_KEY, HEADER_VALUE + "," + HEADER_VALUE, false)); principal = OrMatcher.create(headerMatcher); - policyMatcher = new PolicyMatcher(POLICY_NAME, + policyMatcher = PolicyMatcher.create(POLICY_NAME, OrMatcher.create(AlwaysTrueMatcher.INSTANCE), principal); engine = new GrpcAuthorizationEngine( - new AuthConfig(Collections.singletonList(policyMatcher), Action.ALLOW)); + AuthConfig.create(Collections.singletonList(policyMatcher), Action.ALLOW)); decision = engine.evaluate(HEADER, serverCall); assertThat(decision.decision()).isEqualTo(Action.ALLOW); - headerMatcher = new AuthHeaderMatcher(Matchers.HeaderMatcher + headerMatcher = AuthHeaderMatcher.create(Matchers.HeaderMatcher .forExactValue(HEADER_KEY + Metadata.BINARY_HEADER_SUFFIX, HEADER_VALUE, false)); principal = OrMatcher.create(headerMatcher); - policyMatcher = new PolicyMatcher(POLICY_NAME, + policyMatcher = PolicyMatcher.create(POLICY_NAME, OrMatcher.create(AlwaysTrueMatcher.INSTANCE), principal); engine = new GrpcAuthorizationEngine( - new AuthConfig(Collections.singletonList(policyMatcher), Action.ALLOW)); + AuthConfig.create(Collections.singletonList(policyMatcher), Action.ALLOW)); decision = engine.evaluate(HEADER, serverCall); assertThat(decision.decision()).isEqualTo(Action.DENY); } @Test public void headerMatcher_binaryHeader() { - AuthHeaderMatcher headerMatcher = new AuthHeaderMatcher(Matchers.HeaderMatcher + AuthHeaderMatcher headerMatcher = AuthHeaderMatcher.create(Matchers.HeaderMatcher .forExactValue(HEADER_KEY + Metadata.BINARY_HEADER_SUFFIX, BaseEncoding.base64().omitPadding().encode(HEADER_VALUE.getBytes(US_ASCII)), false)); OrMatcher principal = OrMatcher.create(headerMatcher); OrMatcher permission = OrMatcher.create( - new InvertMatcher(new DestinationPortMatcher(PORT + 1))); - PolicyMatcher policyMatcher = new PolicyMatcher(POLICY_NAME, permission, principal); + InvertMatcher.create(DestinationPortMatcher.create(PORT + 1))); + PolicyMatcher policyMatcher = PolicyMatcher.create(POLICY_NAME, permission, principal); GrpcAuthorizationEngine engine = new GrpcAuthorizationEngine( - new AuthConfig(Collections.singletonList(policyMatcher), Action.ALLOW)); + AuthConfig.create(Collections.singletonList(policyMatcher), Action.ALLOW)); Metadata metadata = new Metadata(); metadata.put(Metadata.Key.of(HEADER_KEY + Metadata.BINARY_HEADER_SUFFIX, Metadata.BINARY_BYTE_MARSHALLER), HEADER_VALUE.getBytes(US_ASCII)); @@ -200,14 +200,14 @@ public void headerMatcher_binaryHeader() { @Test public void headerMatcher_hardcodePostMethod() { - AuthHeaderMatcher headerMatcher = new AuthHeaderMatcher(Matchers.HeaderMatcher + AuthHeaderMatcher headerMatcher = AuthHeaderMatcher.create(Matchers.HeaderMatcher .forExactValue(":method", "POST", false)); OrMatcher principal = OrMatcher.create(headerMatcher); OrMatcher permission = OrMatcher.create( - new InvertMatcher(new DestinationPortMatcher(PORT + 1))); - PolicyMatcher policyMatcher = new PolicyMatcher(POLICY_NAME, permission, principal); + InvertMatcher.create(DestinationPortMatcher.create(PORT + 1))); + PolicyMatcher policyMatcher = PolicyMatcher.create(POLICY_NAME, permission, principal); GrpcAuthorizationEngine engine = new GrpcAuthorizationEngine( - new AuthConfig(Collections.singletonList(policyMatcher), Action.ALLOW)); + AuthConfig.create(Collections.singletonList(policyMatcher), Action.ALLOW)); AuthDecision decision = engine.evaluate(new Metadata(), serverCall); assertThat(decision.decision()).isEqualTo(Action.ALLOW); assertThat(decision.matchingPolicyName()).isEqualTo(POLICY_NAME); @@ -215,14 +215,14 @@ public void headerMatcher_hardcodePostMethod() { @Test public void headerMatcher_pathHeader() { - AuthHeaderMatcher headerMatcher = new AuthHeaderMatcher(Matchers.HeaderMatcher + AuthHeaderMatcher headerMatcher = AuthHeaderMatcher.create(Matchers.HeaderMatcher .forExactValue(":path", "/" + PATH, false)); OrMatcher principal = OrMatcher.create(headerMatcher); OrMatcher permission = OrMatcher.create( - new InvertMatcher(new DestinationPortMatcher(PORT + 1))); - PolicyMatcher policyMatcher = new PolicyMatcher(POLICY_NAME, permission, principal); + InvertMatcher.create(DestinationPortMatcher.create(PORT + 1))); + PolicyMatcher policyMatcher = PolicyMatcher.create(POLICY_NAME, permission, principal); GrpcAuthorizationEngine engine = new GrpcAuthorizationEngine( - new AuthConfig(Collections.singletonList(policyMatcher), Action.ALLOW)); + AuthConfig.create(Collections.singletonList(policyMatcher), Action.ALLOW)); AuthDecision decision = engine.evaluate(HEADER, serverCall); assertThat(decision.decision()).isEqualTo(Action.ALLOW); assertThat(decision.matchingPolicyName()).isEqualTo(POLICY_NAME); @@ -230,14 +230,14 @@ public void headerMatcher_pathHeader() { @Test public void headerMatcher_aliasAuthorityAndHost() { - AuthHeaderMatcher headerMatcher = new AuthHeaderMatcher(Matchers.HeaderMatcher + AuthHeaderMatcher headerMatcher = AuthHeaderMatcher.create(Matchers.HeaderMatcher .forExactValue("Host", "google.com", false)); OrMatcher principal = OrMatcher.create(headerMatcher); OrMatcher permission = OrMatcher.create( - new InvertMatcher(new DestinationPortMatcher(PORT + 1))); - PolicyMatcher policyMatcher = new PolicyMatcher(POLICY_NAME, permission, principal); + InvertMatcher.create(DestinationPortMatcher.create(PORT + 1))); + PolicyMatcher policyMatcher = PolicyMatcher.create(POLICY_NAME, permission, principal); GrpcAuthorizationEngine engine = new GrpcAuthorizationEngine( - new AuthConfig(Collections.singletonList(policyMatcher), Action.ALLOW)); + AuthConfig.create(Collections.singletonList(policyMatcher), Action.ALLOW)); when(serverCall.getAuthority()).thenReturn("google.com"); AuthDecision decision = engine.evaluate(new Metadata(), serverCall); assertThat(decision.decision()).isEqualTo(Action.ALLOW); @@ -246,12 +246,12 @@ public void headerMatcher_aliasAuthorityAndHost() { @Test public void pathMatcher() { - PathMatcher pathMatcher = new PathMatcher(STRING_MATCHER); + PathMatcher pathMatcher = PathMatcher.create(STRING_MATCHER); OrMatcher permission = OrMatcher.create(AlwaysTrueMatcher.INSTANCE); OrMatcher principal = OrMatcher.create(pathMatcher); - PolicyMatcher policyMatcher = new PolicyMatcher(POLICY_NAME, permission, principal); + PolicyMatcher policyMatcher = PolicyMatcher.create(POLICY_NAME, permission, principal); GrpcAuthorizationEngine engine = new GrpcAuthorizationEngine( - new AuthConfig(Collections.singletonList(policyMatcher), Action.DENY)); + AuthConfig.create(Collections.singletonList(policyMatcher), Action.DENY)); AuthDecision decision = engine.evaluate(HEADER, serverCall); assertThat(decision.decision()).isEqualTo(Action.DENY); assertThat(decision.matchingPolicyName()).isEqualTo(POLICY_NAME); @@ -259,14 +259,14 @@ public void pathMatcher() { @Test public void authenticatedMatcher() throws Exception { - AuthenticatedMatcher authMatcher = new AuthenticatedMatcher( + AuthenticatedMatcher authMatcher = AuthenticatedMatcher.create( StringMatcher.forExact("*.test.google.fr", false)); - PathMatcher pathMatcher = new PathMatcher(STRING_MATCHER); + PathMatcher pathMatcher = PathMatcher.create(STRING_MATCHER); OrMatcher permission = OrMatcher.create(authMatcher); OrMatcher principal = OrMatcher.create(pathMatcher); - PolicyMatcher policyMatcher = new PolicyMatcher(POLICY_NAME, permission, principal); + PolicyMatcher policyMatcher = PolicyMatcher.create(POLICY_NAME, permission, principal); GrpcAuthorizationEngine engine = new GrpcAuthorizationEngine( - new AuthConfig(Collections.singletonList(policyMatcher), Action.ALLOW)); + AuthConfig.create(Collections.singletonList(policyMatcher), Action.ALLOW)); AuthDecision decision = engine.evaluate(HEADER, serverCall); assertThat(decision.decision()).isEqualTo(Action.ALLOW); assertThat(decision.matchingPolicyName()).isEqualTo(POLICY_NAME); @@ -306,11 +306,11 @@ public void authenticatedMatcher() throws Exception { assertThat(engine.evaluate(HEADER, serverCall).decision()).isEqualTo(Action.ALLOW); // match any authenticated connection if StringMatcher not set in AuthenticatedMatcher - permission = OrMatcher.create(new AuthenticatedMatcher(null)); - policyMatcher = new PolicyMatcher(POLICY_NAME, permission, principal); + permission = OrMatcher.create(AuthenticatedMatcher.create(null)); + policyMatcher = PolicyMatcher.create(POLICY_NAME, permission, principal); when(mockCert.getSubjectAlternativeNames()).thenReturn( Arrays.>asList(Arrays.asList(6, "random"))); - engine = new GrpcAuthorizationEngine(new AuthConfig(Collections.singletonList(policyMatcher), + engine = new GrpcAuthorizationEngine(AuthConfig.create(Collections.singletonList(policyMatcher), Action.ALLOW)); assertThat(engine.evaluate(HEADER, serverCall).decision()).isEqualTo(Action.ALLOW); @@ -330,31 +330,78 @@ public void authenticatedMatcher() throws Exception { @Test public void multiplePolicies() throws Exception { - AuthenticatedMatcher authMatcher = new AuthenticatedMatcher( + AuthenticatedMatcher authMatcher = AuthenticatedMatcher.create( StringMatcher.forSuffix("TEST.google.fr", true)); - PathMatcher pathMatcher = new PathMatcher(STRING_MATCHER); + PathMatcher pathMatcher = PathMatcher.create(STRING_MATCHER); OrMatcher principal = OrMatcher.create(AndMatcher.create(authMatcher, pathMatcher)); OrMatcher permission = OrMatcher.create(AndMatcher.create(pathMatcher, - new InvertMatcher(new DestinationPortMatcher(PORT + 1)))); - PolicyMatcher policyMatcher1 = new PolicyMatcher(POLICY_NAME, permission, principal); + InvertMatcher.create(DestinationPortMatcher.create(PORT + 1)))); + PolicyMatcher policyMatcher1 = PolicyMatcher.create(POLICY_NAME, permission, principal); - AuthHeaderMatcher headerMatcher = new AuthHeaderMatcher(Matchers.HeaderMatcher + AuthHeaderMatcher headerMatcher = AuthHeaderMatcher.create(Matchers.HeaderMatcher .forExactValue(HEADER_KEY, HEADER_VALUE + 1, false)); - authMatcher = new AuthenticatedMatcher( + authMatcher = AuthenticatedMatcher.create( StringMatcher.forContains("TEST.google.fr")); principal = OrMatcher.create(headerMatcher, authMatcher); CidrMatcher ip1 = CidrMatcher.create(InetAddress.getByName(IP_ADDR1), 24); - DestinationIpMatcher destIpMatcher = new DestinationIpMatcher(ip1); + DestinationIpMatcher destIpMatcher = DestinationIpMatcher.create(ip1); permission = OrMatcher.create(destIpMatcher, pathMatcher); - PolicyMatcher policyMatcher2 = new PolicyMatcher(POLICY_NAME + "-2", permission, principal); + PolicyMatcher policyMatcher2 = PolicyMatcher.create(POLICY_NAME + "-2", permission, principal); GrpcAuthorizationEngine engine = new GrpcAuthorizationEngine( - new AuthConfig(ImmutableList.of(policyMatcher1, policyMatcher2), Action.DENY)); + AuthConfig.create(ImmutableList.of(policyMatcher1, policyMatcher2), Action.DENY)); AuthDecision decision = engine.evaluate(HEADER, serverCall); assertThat(decision.decision()).isEqualTo(Action.DENY); assertThat(decision.matchingPolicyName()).isEqualTo(POLICY_NAME); } + @Test + public void matchersEqualHashcode() throws Exception { + PathMatcher pathMatcher = PathMatcher.create(STRING_MATCHER); + AuthHeaderMatcher headerMatcher = AuthHeaderMatcher.create( + Matchers.HeaderMatcher.forExactValue("foo", "bar", true)); + DestinationIpMatcher destinationIpMatcher = DestinationIpMatcher.create( + CidrMatcher.create(InetAddress.getByName(IP_ADDR1), 24)); + DestinationPortMatcher destinationPortMatcher = DestinationPortMatcher.create(PORT); + GrpcAuthorizationEngine.DestinationPortRangeMatcher portRangeMatcher = + GrpcAuthorizationEngine.DestinationPortRangeMatcher.create(PORT, PORT + 1); + InvertMatcher invertMatcher = InvertMatcher.create(portRangeMatcher); + GrpcAuthorizationEngine.RequestedServerNameMatcher requestedServerNameMatcher = + GrpcAuthorizationEngine.RequestedServerNameMatcher.create(STRING_MATCHER); + OrMatcher permission = OrMatcher.create(pathMatcher, headerMatcher, destinationIpMatcher, + destinationPortMatcher, invertMatcher, requestedServerNameMatcher); + AuthenticatedMatcher authenticatedMatcher = AuthenticatedMatcher.create(STRING_MATCHER); + SourceIpMatcher sourceIpMatcher1 = SourceIpMatcher.create( + CidrMatcher.create(InetAddress.getByName(IP_ADDR1), 24)); + OrMatcher principal = OrMatcher.create(authenticatedMatcher, + AndMatcher.create(sourceIpMatcher1, AlwaysTrueMatcher.INSTANCE)); + PolicyMatcher policyMatcher1 = PolicyMatcher.create("match", permission, principal); + AuthConfig config1 = AuthConfig.create(Collections.singletonList(policyMatcher1), Action.ALLOW); + + PathMatcher pathMatcher2 = PathMatcher.create(STRING_MATCHER); + AuthHeaderMatcher headerMatcher2 = AuthHeaderMatcher.create( + Matchers.HeaderMatcher.forExactValue("foo", "bar", true)); + DestinationIpMatcher destinationIpMatcher2 = DestinationIpMatcher.create( + CidrMatcher.create(InetAddress.getByName(IP_ADDR1), 24)); + DestinationPortMatcher destinationPortMatcher2 = DestinationPortMatcher.create(PORT); + GrpcAuthorizationEngine.DestinationPortRangeMatcher portRangeMatcher2 = + GrpcAuthorizationEngine.DestinationPortRangeMatcher.create(PORT, PORT + 1); + InvertMatcher invertMatcher2 = InvertMatcher.create(portRangeMatcher2); + GrpcAuthorizationEngine.RequestedServerNameMatcher requestedServerNameMatcher2 = + GrpcAuthorizationEngine.RequestedServerNameMatcher.create(STRING_MATCHER); + OrMatcher permission2 = OrMatcher.create(pathMatcher2, headerMatcher2, destinationIpMatcher2, + destinationPortMatcher2, invertMatcher2, requestedServerNameMatcher2); + AuthenticatedMatcher authenticatedMatcher2 = AuthenticatedMatcher.create(STRING_MATCHER); + SourceIpMatcher sourceIpMatcher2 = SourceIpMatcher.create( + CidrMatcher.create(InetAddress.getByName(IP_ADDR1), 24)); + OrMatcher principal2 = OrMatcher.create(authenticatedMatcher2, + AndMatcher.create(sourceIpMatcher2, AlwaysTrueMatcher.INSTANCE)); + PolicyMatcher policyMatcher2 = PolicyMatcher.create("match", permission2, principal2); + AuthConfig config2 = AuthConfig.create(Collections.singletonList(policyMatcher2), Action.ALLOW); + assertThat(config1).isEqualTo(config2); + assertThat(config1.hashCode()).isEqualTo(config2.hashCode()); + } + private MethodDescriptor.Builder method() { return MethodDescriptor.newBuilder() .setType(MethodType.BIDI_STREAMING) From e41df60bea276e55d698be37d18fc2a02a209676 Mon Sep 17 00:00:00 2001 From: Zhouyihai Ding Date: Wed, 22 Sep 2021 09:43:08 -0700 Subject: [PATCH 30/76] core: change the mapping from ChannelLogLevel to java.util.logging.Level Instead of `ChannelLogLevel.{DEBUG,INFO}` mapping to the same java level, `ChannelLogLevel.{WARNING,ERROR}` will shame the same java level. This allows us to be able to independently control the visibility of `ChannelLogLevel.DEBUG` logs which are the most verbose. --- api/src/main/java/io/grpc/ChannelLogger.java | 4 ++-- .../src/main/java/io/grpc/internal/ChannelLoggerImpl.java | 3 ++- .../test/java/io/grpc/internal/ChannelLoggerImplTest.java | 8 ++++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/api/src/main/java/io/grpc/ChannelLogger.java b/api/src/main/java/io/grpc/ChannelLogger.java index a0f6114fbc3..ce654ec9d5b 100644 --- a/api/src/main/java/io/grpc/ChannelLogger.java +++ b/api/src/main/java/io/grpc/ChannelLogger.java @@ -36,8 +36,8 @@ public abstract class ChannelLogger { * | ChannelLogger Level | Channelz Severity | Java Logger Level | * +---------------------+-------------------+-------------------+ * | DEBUG | N/A | FINEST | - * | INFO | CT_INFO | FINEST | - * | WARNING | CT_WARNING | FINER | + * | INFO | CT_INFO | FINER | + * | WARNING | CT_WARNING | FINE | * | ERROR | CT_ERROR | FINE | * +---------------------+-------------------+-------------------+ * diff --git a/core/src/main/java/io/grpc/internal/ChannelLoggerImpl.java b/core/src/main/java/io/grpc/internal/ChannelLoggerImpl.java index e000d872e8a..e8504498cdb 100644 --- a/core/src/main/java/io/grpc/internal/ChannelLoggerImpl.java +++ b/core/src/main/java/io/grpc/internal/ChannelLoggerImpl.java @@ -97,8 +97,9 @@ private static Severity toTracerSeverity(ChannelLogLevel level) { private static Level toJavaLogLevel(ChannelLogLevel level) { switch (level) { case ERROR: - return Level.FINE; case WARNING: + return Level.FINE; + case INFO: return Level.FINER; default: return Level.FINEST; diff --git a/core/src/test/java/io/grpc/internal/ChannelLoggerImplTest.java b/core/src/test/java/io/grpc/internal/ChannelLoggerImplTest.java index 486d96fd7b9..afe8e9828c9 100644 --- a/core/src/test/java/io/grpc/internal/ChannelLoggerImplTest.java +++ b/core/src/test/java/io/grpc/internal/ChannelLoggerImplTest.java @@ -100,7 +100,7 @@ public void logging() { .setTimestampNanos(200) .build(); assertThat(stats.channelTrace.events).containsExactly(event); - assertThat(logs).contains("FINER: " + logPrefix + "Warning message"); + assertThat(logs).contains("FINE: " + logPrefix + "Warning message"); clock.forwardNanos(100); logger.log(ChannelLogLevel.INFO, "Info message"); @@ -112,7 +112,7 @@ public void logging() { .setTimestampNanos(300) .build(); assertThat(stats.channelTrace.events).containsExactly(event); - assertThat(logs).contains("FINEST: " + logPrefix + "Info message"); + assertThat(logs).contains("FINER: " + logPrefix + "Info message"); clock.forwardNanos(100); logger.log(ChannelLogLevel.DEBUG, "Debug message"); @@ -154,7 +154,7 @@ public void formatLogging() { .setTimestampNanos(200) .build(); assertThat(stats.channelTrace.events).containsExactly(event); - assertThat(logs).contains("FINER: " + logPrefix + "Warning message foo, bar"); + assertThat(logs).contains("FINE: " + logPrefix + "Warning message foo, bar"); clock.forwardNanos(100); logger.log(ChannelLogLevel.INFO, "Info message {0}", "bar"); @@ -166,7 +166,7 @@ public void formatLogging() { .setTimestampNanos(300) .build(); assertThat(stats.channelTrace.events).containsExactly(event); - assertThat(logs).contains("FINEST: " + logPrefix + "Info message bar"); + assertThat(logs).contains("FINER: " + logPrefix + "Info message bar"); clock.forwardNanos(100); logger.log(ChannelLogLevel.DEBUG, "Debug message {0}", "foo"); From 5396a1de3d2bb9c51402127f72b923dbd7462dcb Mon Sep 17 00:00:00 2001 From: Zhouyihai Ding Date: Wed, 22 Sep 2021 10:13:42 -0700 Subject: [PATCH 31/76] grpclb: remove redundant logs and add a system property to hide server lists in logs The server list updates are very verbose and currently logged every second, causing a huge log spam if `ChannelLogger` is completely enabled. For debugging an internal issue, we need to turn on `ChannelLogger` but hide the server list updates from the logs to keep the log size reasonable. --- .../main/java/io/grpc/grpclb/GrpclbState.java | 38 +++--- .../grpc/grpclb/GrpclbLoadBalancerTest.java | 122 +++--------------- 2 files changed, 34 insertions(+), 126 deletions(-) diff --git a/grpclb/src/main/java/io/grpc/grpclb/GrpclbState.java b/grpclb/src/main/java/io/grpc/grpclb/GrpclbState.java index 8793a44c7c7..8607d3996a5 100644 --- a/grpclb/src/main/java/io/grpc/grpclb/GrpclbState.java +++ b/grpclb/src/main/java/io/grpc/grpclb/GrpclbState.java @@ -89,6 +89,11 @@ final class GrpclbState { private static final Attributes LB_PROVIDED_BACKEND_ATTRS = Attributes.newBuilder().set(GrpclbConstants.ATTR_LB_PROVIDED_BACKEND, true).build(); + // Temporary workaround to reduce log spam for a grpclb server that incessantly sends updates + // Tracked by b/198440401 + static final boolean SHOULD_LOG_SERVER_LISTS = + Boolean.parseBoolean(System.getProperty("io.grpc.grpclb.LogServerLists", "true")); + @VisibleForTesting static final PickResult DROP_PICK_RESULT = PickResult.withDrop(Status.UNAVAILABLE.withDescription("Dropped as requested by balancer")); @@ -469,12 +474,6 @@ GrpclbClientLoadRecorder getLoadRecorder() { private void updateServerList( List newDropList, List newBackendAddrList, @Nullable GrpclbClientLoadRecorder loadRecorder) { - logger.log( - ChannelLogLevel.INFO, - "[grpclb-<{0}>] Using RR list={1}, drop={2}", - serviceName, - newBackendAddrList, - newDropList); HashMap, Subchannel> newSubchannelMap = new HashMap<>(); List newBackendList = new ArrayList<>(); @@ -710,9 +709,12 @@ private void handleResponse(LoadBalanceResponse response) { scheduleNextLoadReport(); return; } - - logger.log( - ChannelLogLevel.DEBUG, "[grpclb-<{0}>] Got an LB response: {1}", serviceName, response); + if (SHOULD_LOG_SERVER_LISTS) { + logger.log( + ChannelLogLevel.DEBUG, "[grpclb-<{0}>] Got an LB response: {1}", serviceName, response); + } else { + logger.log(ChannelLogLevel.DEBUG, "[grpclb-<{0}>] Got an LB response", serviceName); + } if (typeCase == LoadBalanceResponseTypeCase.FALLBACK_RESPONSE) { // Force entering fallback requested by balancer. @@ -926,13 +928,6 @@ private void maybeUpdatePicker(ConnectivityState state, RoundRobinPicker picker) return; } currentPicker = picker; - logger.log( - ChannelLogLevel.INFO, - "[grpclb-<{0}>] Update balancing state to {1}: picks={2}, drops={3}", - serviceName, - state, - picker.pickList, - picker.dropList); helper.updateBalancingState(state, picker); } @@ -1179,10 +1174,13 @@ public PickResult pickSubchannel(PickSubchannelArgs args) { @Override public String toString() { - return MoreObjects.toStringHelper(RoundRobinPicker.class) - .add("dropList", dropList) - .add("pickList", pickList) - .toString(); + if (SHOULD_LOG_SERVER_LISTS) { + return MoreObjects.toStringHelper(RoundRobinPicker.class) + .add("dropList", dropList) + .add("pickList", pickList) + .toString(); + } + return MoreObjects.toStringHelper(RoundRobinPicker.class).toString(); } } } diff --git a/grpclb/src/test/java/io/grpc/grpclb/GrpclbLoadBalancerTest.java b/grpclb/src/test/java/io/grpc/grpclb/GrpclbLoadBalancerTest.java index a68962ad7d9..cb231c6c055 100644 --- a/grpclb/src/test/java/io/grpc/grpclb/GrpclbLoadBalancerTest.java +++ b/grpclb/src/test/java/io/grpc/grpclb/GrpclbLoadBalancerTest.java @@ -794,12 +794,10 @@ public void nameResolutionFailsThenRecover() { Status error = Status.NOT_FOUND.withDescription("www.google.com not found"); deliverNameResolutionError(error); verify(helper).updateBalancingState(eq(TRANSIENT_FAILURE), pickerCaptor.capture()); - assertThat(logs).containsExactly( - "INFO: [grpclb-] Created", - "DEBUG: [grpclb-] Error: " + error, - "INFO: [grpclb-] Update balancing state to TRANSIENT_FAILURE: picks=" - + "[Status{code=UNAVAILABLE, description=www.google.com not found, cause=null}]," - + " drops=[]") + assertThat(logs) + .containsExactly( + "INFO: [grpclb-] Created", + "DEBUG: [grpclb-] Error: " + error) .inOrder(); logs.clear(); @@ -964,24 +962,13 @@ public void grpclbWorking() { inOrder.verifyNoMoreInteractions(); assertThat(logs).containsExactly( - "DEBUG: [grpclb-] Got an LB response: " + buildLbResponse(backends1), - "INFO: [grpclb-] Using RR list=" - + "[[[/127.0.0.1:2000]/{io.grpc.grpclb.lbProvidedBackend=true}](token0001)," - + " [[/127.0.0.1:2010]/{io.grpc.grpclb.lbProvidedBackend=true}](token0002)]," - + " drop=[null, null]", - "INFO: [grpclb-]" - + " Update balancing state to CONNECTING: picks=[BUFFER_ENTRY], drops=[null, null]") + "DEBUG: [grpclb-] Got an LB response: " + buildLbResponse(backends1)) .inOrder(); logs.clear(); // Let subchannels be connected deliverSubchannelState(subchannel2, ConnectivityStateInfo.forNonError(READY)); inOrder.verify(helper).updateBalancingState(eq(READY), pickerCaptor.capture()); - assertThat(logs).containsExactly( - "INFO: [grpclb-] Update balancing state to READY: picks=" - + "[[[[[/127.0.0.1:2010]/{io.grpc.grpclb.lbProvidedBackend=true}]](token0002)]]," - + " drops=[null, null]"); - logs.clear(); RoundRobinPicker picker1 = (RoundRobinPicker) pickerCaptor.getValue(); @@ -991,12 +978,6 @@ public void grpclbWorking() { deliverSubchannelState(subchannel1, ConnectivityStateInfo.forNonError(READY)); inOrder.verify(helper).updateBalancingState(eq(READY), pickerCaptor.capture()); - assertThat(logs).containsExactly( - "INFO: [grpclb-] Update balancing state to READY: picks=" - + "[[[[[/127.0.0.1:2000]/{io.grpc.grpclb.lbProvidedBackend=true}]](token0001)]," - + " [[[[/127.0.0.1:2010]/{io.grpc.grpclb.lbProvidedBackend=true}]](token0002)]]," - + " drops=[null, null]"); - logs.clear(); RoundRobinPicker picker2 = (RoundRobinPicker) pickerCaptor.getValue(); assertThat(picker2.dropList).containsExactly(null, null); @@ -1011,11 +992,6 @@ public void grpclbWorking() { verify(subchannel1, times(2)).requestConnection(); inOrder.verify(helper).refreshNameResolution(); inOrder.verify(helper).updateBalancingState(eq(READY), pickerCaptor.capture()); - assertThat(logs).containsExactly( - "INFO: [grpclb-] Update balancing state to READY: picks=" - + "[[[[[/127.0.0.1:2010]/{io.grpc.grpclb.lbProvidedBackend=true}]](token0002)]]," - + " drops=[null, null]"); - logs.clear(); RoundRobinPicker picker3 = (RoundRobinPicker) pickerCaptor.getValue(); assertThat(picker3.dropList).containsExactly(null, null); @@ -1037,10 +1013,6 @@ public void grpclbWorking() { deliverSubchannelState(subchannel2, ConnectivityStateInfo.forNonError(IDLE)); verify(subchannel2, times(2)).requestConnection(); inOrder.verify(helper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture()); - assertThat(logs).containsExactly( - "INFO: [grpclb-] " - + "Update balancing state to CONNECTING: picks=[BUFFER_ENTRY], drops=[null, null]"); - logs.clear(); RoundRobinPicker picker4 = (RoundRobinPicker) pickerCaptor.getValue(); assertThat(picker4.dropList).containsExactly(null, null); @@ -1059,15 +1031,7 @@ public void grpclbWorking() { lbResponseObserver.onNext(buildLbResponse(backends2)); assertThat(logs).containsExactly( - "DEBUG: [grpclb-] Got an LB response: " + buildLbResponse(backends2), - "INFO: [grpclb-] Using RR list=" - + "[[[/127.0.0.1:2030]/{io.grpc.grpclb.lbProvidedBackend=true}](token0003)," - + " [[/127.0.0.1:2010]/{io.grpc.grpclb.lbProvidedBackend=true}](token0004)," - + " [[/127.0.0.1:2030]/{io.grpc.grpclb.lbProvidedBackend=true}](token0005)]," - + " drop=[null, drop(token0003), null, null, drop(token0006)]", - "INFO: [grpclb-]" - + " Update balancing state to CONNECTING: picks=[BUFFER_ENTRY]," - + " drops=[null, drop(token0003), null, null, drop(token0006)]") + "DEBUG: [grpclb-] Got an LB response: " + buildLbResponse(backends2)) .inOrder(); logs.clear(); @@ -1267,14 +1231,8 @@ private void subtestGrpclbFallbackInitialTimeout(boolean timerExpires) { fakeClock.forwardTime(1, TimeUnit.MILLISECONDS); assertEquals(0, fakeClock.numPendingTasks(FALLBACK_MODE_TASK_FILTER)); - assertThat(logs).containsExactly( - "INFO: [grpclb-] Using fallback backends", - "INFO: [grpclb-]" - + " Using RR list=[[[FakeSocketAddress-fake-address-0]/{}], " - + "[[FakeSocketAddress-fake-address-1]/{}]], drop=[null, null]", - "INFO: [grpclb-] " - + "Update balancing state to CONNECTING: picks=[BUFFER_ENTRY], " - + "drops=[null, null]") + assertThat(logs) + .containsExactly("INFO: [grpclb-] Using fallback backends") .inOrder(); // Fall back to the backends from resolver @@ -1462,14 +1420,7 @@ public void grpclbFallback_noBalancerAddress() { List backendList = backends.subList(0, 2); deliverResolvedAddresses(backendList, Collections.emptyList()); - assertThat(logs).containsAtLeast( - "INFO: [grpclb-] Using fallback backends", - "INFO: [grpclb-] " - + "Using RR list=[[[FakeSocketAddress-fake-address-0]/{}], " - + "[[FakeSocketAddress-fake-address-1]/{}]], drop=[null, null]", - "INFO: [grpclb-] " - + "Update balancing state to CONNECTING: picks=[BUFFER_ENTRY], drops=[null, null]") - .inOrder(); + assertThat(logs).contains("INFO: [grpclb-] Using fallback backends"); // Fall back to the backends from resolver fallbackTestVerifyUseOfFallbackBackendLists(inOrder, backendList); @@ -1487,16 +1438,7 @@ public void grpclbFallback_noBalancerAddress() { backendList = backends.subList(2, 5); deliverResolvedAddresses(backendList, Collections.emptyList()); - assertThat(logs).containsAtLeast( - "INFO: [grpclb-] Using fallback backends", - "INFO: [grpclb-] " - + "Using RR list=[[[FakeSocketAddress-fake-address-2]/{}], " - + "[[FakeSocketAddress-fake-address-3]/{}], " - + "[[FakeSocketAddress-fake-address-4]/{}]], drop=[null, null, null]", - "INFO: [grpclb-] " - + "Update balancing state to CONNECTING: picks=[BUFFER_ENTRY], " - + "drops=[null, null, null]") - .inOrder(); + assertThat(logs).contains("INFO: [grpclb-] Using fallback backends"); // Shift to use updated backends fallbackTestVerifyUseOfFallbackBackendLists(inOrder, backendList); @@ -2558,25 +2500,15 @@ public void grpclbWorking_lbSendsFallbackMessage() { assertThat(picker0.pickList).containsExactly(BUFFER_ENTRY); inOrder.verifyNoMoreInteractions(); - assertThat(logs).containsExactly( - "DEBUG: [grpclb-] Got an LB response: " + buildLbResponse(backends1), - "INFO: [grpclb-] Using RR list=" - + "[[[/127.0.0.1:2000]/{io.grpc.grpclb.lbProvidedBackend=true}](token0001)," - + " [[/127.0.0.1:2010]/{io.grpc.grpclb.lbProvidedBackend=true}](token0002)]," - + " drop=[null, null]", - "INFO: [grpclb-] " - + "Update balancing state to CONNECTING: picks=[BUFFER_ENTRY], drops=[null, null]") + assertThat(logs) + .containsExactly( + "DEBUG: [grpclb-] Got an LB response: " + buildLbResponse(backends1)) .inOrder(); logs.clear(); // Let subchannels be connected deliverSubchannelState(subchannel2, ConnectivityStateInfo.forNonError(READY)); inOrder.verify(helper).updateBalancingState(eq(READY), pickerCaptor.capture()); - assertThat(logs).containsExactly( - "INFO: [grpclb-] Update balancing state to READY: picks=" - + "[[[[[/127.0.0.1:2010]/{io.grpc.grpclb.lbProvidedBackend=true}]](token0002)]]," - + " drops=[null, null]"); - logs.clear(); RoundRobinPicker picker1 = (RoundRobinPicker) pickerCaptor.getValue(); @@ -2586,12 +2518,6 @@ public void grpclbWorking_lbSendsFallbackMessage() { deliverSubchannelState(subchannel1, ConnectivityStateInfo.forNonError(READY)); inOrder.verify(helper).updateBalancingState(eq(READY), pickerCaptor.capture()); - assertThat(logs).containsExactly( - "INFO: [grpclb-] Update balancing state to READY: picks=" - + "[[[[[/127.0.0.1:2000]/{io.grpc.grpclb.lbProvidedBackend=true}]](token0001)]," - + " [[[[/127.0.0.1:2010]/{io.grpc.grpclb.lbProvidedBackend=true}]](token0002)]]," - + " drops=[null, null]"); - logs.clear(); RoundRobinPicker picker2 = (RoundRobinPicker) pickerCaptor.getValue(); assertThat(picker2.dropList).containsExactly(null, null); @@ -2672,25 +2598,15 @@ public void grpclbWorking_lbSendsFallbackMessage() { assertThat(picker6.pickList).containsExactly(BUFFER_ENTRY); inOrder.verifyNoMoreInteractions(); - assertThat(logs).containsExactly( - "DEBUG: [grpclb-] Got an LB response: " + buildLbResponse(backends2), - "INFO: [grpclb-] Using RR list=" - + "[[[/127.0.0.1:8000]/{io.grpc.grpclb.lbProvidedBackend=true}](token1001)," - + " [[/127.0.0.1:8010]/{io.grpc.grpclb.lbProvidedBackend=true}](token1002)]," - + " drop=[null, null]", - "INFO: [grpclb-] " - + "Update balancing state to CONNECTING: picks=[BUFFER_ENTRY], drops=[null, null]") + assertThat(logs) + .containsExactly( + "DEBUG: [grpclb-] Got an LB response: " + buildLbResponse(backends2)) .inOrder(); logs.clear(); // Let new subchannels be connected deliverSubchannelState(subchannel3, ConnectivityStateInfo.forNonError(READY)); inOrder.verify(helper).updateBalancingState(eq(READY), pickerCaptor.capture()); - assertThat(logs).containsExactly( - "INFO: [grpclb-] Update balancing state to READY: picks=" - + "[[[[[/127.0.0.1:8000]/{io.grpc.grpclb.lbProvidedBackend=true}]](token1001)]]," - + " drops=[null, null]"); - logs.clear(); RoundRobinPicker picker3 = (RoundRobinPicker) pickerCaptor.getValue(); assertThat(picker3.dropList).containsExactly(null, null); @@ -2699,12 +2615,6 @@ public void grpclbWorking_lbSendsFallbackMessage() { deliverSubchannelState(subchannel4, ConnectivityStateInfo.forNonError(READY)); inOrder.verify(helper).updateBalancingState(eq(READY), pickerCaptor.capture()); - assertThat(logs).containsExactly( - "INFO: [grpclb-] Update balancing state to READY: picks=" - + "[[[[[/127.0.0.1:8000]/{io.grpc.grpclb.lbProvidedBackend=true}]](token1001)]," - + " [[[[/127.0.0.1:8010]/{io.grpc.grpclb.lbProvidedBackend=true}]](token1002)]]," - + " drops=[null, null]"); - logs.clear(); RoundRobinPicker picker4 = (RoundRobinPicker) pickerCaptor.getValue(); assertThat(picker4.dropList).containsExactly(null, null); From e244065b0cbbab37c0be1a7fbd6799f7167e3e3c Mon Sep 17 00:00:00 2001 From: markb74 <57717302+markb74@users.noreply.github.com> Date: Wed, 22 Sep 2021 19:19:10 +0200 Subject: [PATCH 32/76] Fix flakey security policy tests. (#8550) Using ShadowProcess to set the processes uID doesn't help since SecurityPolicies class fetches the ID in a static initializer, and it may have already been loaded. Instead, just rely on whatever the uID is already, and ensure the other UIDs we test with are offset from that first value. --- binder/build.gradle | 2 -- .../test/java/io/grpc/binder/SecurityPoliciesTest.java | 10 ++-------- .../java/io/grpc/binder/ServerSecurityPolicyTest.java | 10 ++-------- 3 files changed, 4 insertions(+), 18 deletions(-) diff --git a/binder/build.gradle b/binder/build.gradle index c5bb9885623..537c23a0092 100644 --- a/binder/build.gradle +++ b/binder/build.gradle @@ -13,8 +13,6 @@ android { srcDirs += "${projectDir}/../core/src/test/java/" setIncludes(["io/grpc/internal/FakeClock.java", "io/grpc/binder/**"]) - exclude 'io/grpc/binder/ServerSecurityPolicyTest.java' - exclude 'io/grpc/binder/SecurityPoliciesTest.java' } } androidTest { diff --git a/binder/src/test/java/io/grpc/binder/SecurityPoliciesTest.java b/binder/src/test/java/io/grpc/binder/SecurityPoliciesTest.java index 604addb3b09..6fd9e22ebaa 100644 --- a/binder/src/test/java/io/grpc/binder/SecurityPoliciesTest.java +++ b/binder/src/test/java/io/grpc/binder/SecurityPoliciesTest.java @@ -18,27 +18,21 @@ import static com.google.common.truth.Truth.assertThat; +import android.os.Process; import io.grpc.Status; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.robolectric.RobolectricTestRunner; -import org.robolectric.shadows.ShadowProcess; @RunWith(RobolectricTestRunner.class) public final class SecurityPoliciesTest { - private static final int MY_UID = 1234; + private static final int MY_UID = Process.myUid(); private static final int OTHER_UID = MY_UID + 1; private static final String PERMISSION_DENIED_REASONS = "some reasons"; private SecurityPolicy policy; - @Before - public void setUp() { - ShadowProcess.setUid(MY_UID); - } - @Test public void testInternalOnly() throws Exception { policy = SecurityPolicies.internalOnly(); diff --git a/binder/src/test/java/io/grpc/binder/ServerSecurityPolicyTest.java b/binder/src/test/java/io/grpc/binder/ServerSecurityPolicyTest.java index 8c61b6119b1..95a4fe5a6e8 100644 --- a/binder/src/test/java/io/grpc/binder/ServerSecurityPolicyTest.java +++ b/binder/src/test/java/io/grpc/binder/ServerSecurityPolicyTest.java @@ -18,13 +18,12 @@ import static com.google.common.truth.Truth.assertThat; +import android.os.Process; import com.google.common.base.Function; import io.grpc.Status; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.robolectric.RobolectricTestRunner; -import org.robolectric.shadows.ShadowProcess; @RunWith(RobolectricTestRunner.class) public final class ServerSecurityPolicyTest { @@ -33,16 +32,11 @@ public final class ServerSecurityPolicyTest { private static final String SERVICE2 = "service_two"; private static final String SERVICE3 = "service_three"; - private static final int MY_UID = 1234; + private static final int MY_UID = Process.myUid(); private static final int OTHER_UID = MY_UID + 1; ServerSecurityPolicy policy; - @Before - public void setUp() { - ShadowProcess.setUid(MY_UID); - } - @Test public void testDefaultInternalOnly() { policy = new ServerSecurityPolicy(); From 3049c2c147047efdc1512e7caa772de28257f19e Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 29 Apr 2021 11:29:53 +0200 Subject: [PATCH 33/76] reenable previously disabled aarch64 tests --- buildscripts/run_arm64_tests_in_docker.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/buildscripts/run_arm64_tests_in_docker.sh b/buildscripts/run_arm64_tests_in_docker.sh index a47fb102a0c..e7bf82022b7 100755 --- a/buildscripts/run_arm64_tests_in_docker.sh +++ b/buildscripts/run_arm64_tests_in_docker.sh @@ -32,11 +32,8 @@ docker run $DOCKER_ARGS --rm=true -v "${grpc_java_dir}":/grpc-java -w /grpc-java # A note on the "docker run" args used: # - run docker container under current user's UID to avoid polluting the workspace # - set the user.home property to avoid creating a "?" directory under grpc-java -# TODO(jtattermusch): avoid skipping ":grpc-netty:test" once https://github.com/grpc/grpc-java/issues/7830 is fixed. -# TODO(jtattermusch): avoid skipping ":grpc-xds:test" once https://github.com/grpc/grpc-java/issues/8118 is fixed. -# TODO(jtattermusch): avoid skipping "grpc-netty-shaded:testShadow" once it's stable docker run $DOCKER_ARGS --rm=true -v "${grpc_java_dir}":/grpc-java -w /grpc-java \ --user "$(id -u):$(id -g)" \ -e "JAVA_OPTS=-Duser.home=/grpc-java/.current-user-home -Djava.util.prefs.userRoot=/grpc-java/.current-user-home/.java/.userPrefs" \ arm64v8/openjdk:11-jdk-slim-buster \ - ./gradlew build -PskipAndroid=true -PskipCodegen=true -x :grpc-netty:test -x :grpc-netty-shaded:testShadow -x :grpc-xds:test + ./gradlew build -PskipAndroid=true -PskipCodegen=true From 46dbac3eb63988e6a3e930fe6d85e14ec0e69bee Mon Sep 17 00:00:00 2001 From: John Cormie Date: Wed, 22 Sep 2021 18:53:16 +0000 Subject: [PATCH 34/76] Make manifest usable with android_instrumentation_test()s in google3 (#8545) --- binder/src/androidTest/AndroidManifest.xml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/binder/src/androidTest/AndroidManifest.xml b/binder/src/androidTest/AndroidManifest.xml index da2cdcb75ea..f53575029de 100644 --- a/binder/src/androidTest/AndroidManifest.xml +++ b/binder/src/androidTest/AndroidManifest.xml @@ -1,8 +1,15 @@ + - - - - + + + + + + + + From 3ff23d36848b26a1adf11c1d9a5b1b2c2ba00376 Mon Sep 17 00:00:00 2001 From: John Cormie Date: Wed, 22 Sep 2021 19:19:48 +0000 Subject: [PATCH 35/76] Synchronize access to acknowledgedOutgoingBytes/transmitWindowFull. (#8547) Fixes #8536 --- .../grpc/binder/internal/BinderTransport.java | 40 ++------ .../grpc/binder/internal/FlowController.java | 84 +++++++++++++++++ .../binder/internal/FlowControllerTest.java | 93 +++++++++++++++++++ 3 files changed, 187 insertions(+), 30 deletions(-) create mode 100644 binder/src/main/java/io/grpc/binder/internal/FlowController.java create mode 100644 binder/src/test/java/io/grpc/binder/internal/FlowControllerTest.java diff --git a/binder/src/main/java/io/grpc/binder/internal/BinderTransport.java b/binder/src/main/java/io/grpc/binder/internal/BinderTransport.java index b132844069c..9e97b7795ab 100644 --- a/binder/src/main/java/io/grpc/binder/internal/BinderTransport.java +++ b/binder/src/main/java/io/grpc/binder/internal/BinderTransport.java @@ -111,7 +111,8 @@ public abstract class BinderTransport /** The authority of the server. */ @Internal - public static final Attributes.Key SERVER_AUTHORITY = Attributes.Key.create("server-authority"); + public static final Attributes.Key SERVER_AUTHORITY = + Attributes.Key.create("server-authority"); /** A transport attribute to hold the {@link InboundParcelablePolicy}. */ @Internal @@ -197,24 +198,14 @@ protected enum TransportState { @Nullable private IBinder outgoingBinder; - /** The number of outgoing bytes we've transmitted. */ - private final AtomicLong numOutgoingBytes; + private final FlowController flowController; /** The number of incoming bytes we've received. */ private final AtomicLong numIncomingBytes; - /** The number of our outgoing bytes our peer has told us it received. */ - private long acknowledgedOutgoingBytes; - /** The number of incoming bytes we've told our peer we've received. */ private long acknowledgedIncomingBytes; - /** - * Whether there are too many unacknowledged outgoing bytes to allow more RPCs right now. This is - * volatile because it'll be read without holding the lock. - */ - private volatile boolean transmitWindowFull; - private BinderTransport( ObjectPool executorServicePool, Attributes attributes, @@ -225,7 +216,7 @@ private BinderTransport( scheduledExecutorService = executorServicePool.getObject(); incomingBinder = new LeakSafeOneWayBinder(this); ongoingCalls = new ConcurrentHashMap<>(); - numOutgoingBytes = new AtomicLong(); + flowController = new FlowController(TRANSACTION_BYTES_WINDOW); numIncomingBytes = new AtomicLong(); } @@ -254,7 +245,7 @@ public final synchronized Attributes getAttributes() { * since this will be called while Outbound is held. */ final boolean isReady() { - return !transmitWindowFull; + return !flowController.isTransmitWindowFull(); } abstract void notifyShutdown(Status shutdownStatus); @@ -410,11 +401,8 @@ final void sendTransaction(int callId, Parcel parcel) throws StatusException { } catch (RemoteException re) { throw statusFromRemoteException(re).asException(); } - long nob = numOutgoingBytes.addAndGet(dataSize); - if ((nob - acknowledgedOutgoingBytes) > TRANSACTION_BYTES_WINDOW) { - logger.log(Level.FINE, "transmist window full. Outgoing=" + nob + " Ack'd Outgoing=" + - acknowledgedOutgoingBytes + " " + this); - transmitWindowFull = true; + if (flowController.notifyBytesSent(dataSize)) { + logger.log(Level.FINE, "transmit window now full " + this); } } @@ -531,25 +519,17 @@ private void sendAcknowledgeBytes(IBinder iBinder) { @GuardedBy("this") final void handleAcknowledgedBytes(long numBytes) { - // The remote side has acknowledged reception of rpc data. - // (update with Math.max in case transactions are delivered out of order). - acknowledgedOutgoingBytes = wrapAwareMax(acknowledgedOutgoingBytes, numBytes); - if ((numOutgoingBytes.get() - acknowledgedOutgoingBytes) < TRANSACTION_BYTES_WINDOW - && transmitWindowFull) { - logger.log(Level.FINE, + if (flowController.handleAcknowledgedBytes(numBytes)) { + logger.log( + Level.FINE, "handleAcknowledgedBytes: Transmit Window No-Longer Full. Unblock calls: " + this); // We're ready again, and need to poke any waiting transactions. - transmitWindowFull = false; for (Inbound inbound : ongoingCalls.values()) { inbound.onTransportReady(); } } } - private static final long wrapAwareMax(long a, long b) { - return a - b < 0 ? b : a; - } - /** Concrete client-side transport implementation. */ @ThreadSafe @Internal diff --git a/binder/src/main/java/io/grpc/binder/internal/FlowController.java b/binder/src/main/java/io/grpc/binder/internal/FlowController.java new file mode 100644 index 00000000000..1972ea00e6c --- /dev/null +++ b/binder/src/main/java/io/grpc/binder/internal/FlowController.java @@ -0,0 +1,84 @@ +/* + * Copyright 2021 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.grpc.binder.internal; + +import javax.annotation.concurrent.GuardedBy; + +/** Keeps track of the number of bytes on the wire in a single direction. */ +final class FlowController { + private final int maxUnackedBytes; + + @GuardedBy("this") + private long totalBytesSent; + + @GuardedBy("this") + private long totalBytesAckedByPeer; + + // @GuardedBy("this") for writes but not reads. + private volatile boolean transmitWindowFull; + + /** + * Creates a new instance of {@link FlowController}. + * + * @param maxUnackedBytes a weak limit on the number of bytes sent but not yet acknowledged + */ + public FlowController(int maxUnackedBytes) { + this.maxUnackedBytes = maxUnackedBytes; + } + + /** + * Returns true iff the number of reported bytes sent but not yet acknowledged is greater than or + * equal to {@code maxUnackedBytes}. + */ + public boolean isTransmitWindowFull() { + return transmitWindowFull; + } + + /** + * Informs this flow controller that more data has been sent. + * + * @param numBytesSent a non-negative number of additional bytes sent + * @return true iff this report caused {@link #isTransmitWindowFull()} to transition to true + */ + public synchronized boolean notifyBytesSent(long numBytesSent) { + totalBytesSent += numBytesSent; + if ((totalBytesSent - totalBytesAckedByPeer) >= maxUnackedBytes && !transmitWindowFull) { + transmitWindowFull = true; + return true; + } + return false; + } + + /** + * Processes an acknowledgement from our peer. + * + * @param numBytesAcked the total number of bytes ever received over this connection modulo 2^64, + * with the most significant bit of this value encoded in this argument's sign bit + * @return true iff this report caused {@link #isTransmitWindowFull()} to transition to false + */ + public synchronized boolean handleAcknowledgedBytes(long numBytesAcked) { + totalBytesAckedByPeer = wrapAwareMax(totalBytesAckedByPeer, numBytesAcked); + if ((totalBytesSent - totalBytesAckedByPeer) < maxUnackedBytes && transmitWindowFull) { + transmitWindowFull = false; + return true; + } + return false; + } + + private static final long wrapAwareMax(long a, long b) { + return a - b < 0 ? b : a; + } +} diff --git a/binder/src/test/java/io/grpc/binder/internal/FlowControllerTest.java b/binder/src/test/java/io/grpc/binder/internal/FlowControllerTest.java new file mode 100644 index 00000000000..f2ba83d9bc1 --- /dev/null +++ b/binder/src/test/java/io/grpc/binder/internal/FlowControllerTest.java @@ -0,0 +1,93 @@ +/* + * Copyright 2021 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.binder.internal; + +import static com.google.common.truth.Truth.assertThat; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public final class FlowControllerTest { + + @Test + public void shouldReportTransmitWindowInitiallyNonFull() { + FlowController flowController = new FlowController(100); + assertThat(flowController.isTransmitWindowFull()).isFalse(); + } + + @Test + public void shouldReportTransmitWindowFull() { + FlowController flowController = new FlowController(100); + boolean transition = flowController.notifyBytesSent(99); + assertThat(transition).isFalse(); + assertThat(flowController.isTransmitWindowFull()).isFalse(); + transition = flowController.notifyBytesSent(1); + assertThat(transition).isTrue(); + assertThat(flowController.isTransmitWindowFull()).isTrue(); + } + + @Test + public void shouldHandleAck() { + FlowController flowController = new FlowController(100); + assertThat(flowController.isTransmitWindowFull()).isFalse(); + flowController.notifyBytesSent(1); + flowController.notifyBytesSent(100); + assertThat(flowController.isTransmitWindowFull()).isTrue(); + boolean transition = flowController.handleAcknowledgedBytes(100); + assertThat(transition).isTrue(); + assertThat(flowController.isTransmitWindowFull()).isFalse(); + transition = flowController.handleAcknowledgedBytes(1); + assertThat(transition).isFalse(); + } + + @Test + public void shouldHandleAcksOutOfOrder() { + FlowController flowController = new FlowController(100); + flowController.notifyBytesSent(49); + flowController.notifyBytesSent(51); + assertThat(flowController.isTransmitWindowFull()).isTrue(); + flowController.handleAcknowledgedBytes(100); + assertThat(flowController.isTransmitWindowFull()).isFalse(); + flowController.handleAcknowledgedBytes(49); + assertThat(flowController.isTransmitWindowFull()).isFalse(); + } + + @Test + public void shouldHandleAckOverflow() { + FlowController flowController = new FlowController(2); + + // Signed long overflow + flowController.notifyBytesSent(Long.MAX_VALUE); // totalBytesSent = 0b011..1 + flowController.handleAcknowledgedBytes(Long.MAX_VALUE); // totalBytesAckedByPeer = 0b011..1 + assertThat(flowController.isTransmitWindowFull()).isFalse(); + flowController.notifyBytesSent(1); // totalBytesSent = 0b100..00 + assertThat(flowController.isTransmitWindowFull()).isFalse(); + flowController.notifyBytesSent(1); // totalBytesSent = 0b100..01 + assertThat(flowController.isTransmitWindowFull()).isTrue(); + + // Unsigned long overflow. + flowController.notifyBytesSent(Long.MAX_VALUE - 1); // totalBytesSent = 0b111..11 + flowController.handleAcknowledgedBytes(-1); // totalBytesAckedByPeer = 0b111..1 + assertThat(flowController.isTransmitWindowFull()).isFalse(); + flowController.notifyBytesSent(1); // totalBytesSent = 0 + assertThat(flowController.isTransmitWindowFull()).isFalse(); + flowController.notifyBytesSent(1); // totalBytesSent = 1 + assertThat(flowController.isTransmitWindowFull()).isTrue(); + } +} From 499c51fa3f5839680ac46fd32881d861465863ae Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Wed, 22 Sep 2021 10:15:42 -0700 Subject: [PATCH 36/76] RELEASING.md: Bump protobuf version to match build.gradle For 1.40.0 the protobuf version was bumped to the latest version, which we hadn't tested at all. We want to bump to the version used in the release. --- RELEASING.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/RELEASING.md b/RELEASING.md index 48f3b645144..5c802f3eea5 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -117,7 +117,8 @@ Tagging the Release $ git checkout v$MAJOR.$MINOR.x $ git pull upstream v$MAJOR.$MINOR.x $ git checkout -b release - # Bump documented versions. Don't forget protobuf version + # Bump documented gRPC versions. + # Also update protoc version to match protocVersion in build.gradle. $ ${EDITOR:-nano -w} README.md $ ${EDITOR:-nano -w} documentation/android-channel-builder.md $ ${EDITOR:-nano -w} cronet/README.md From e76efbb5da4a9fa347432a6987e7bcd64f1de21a Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Wed, 22 Sep 2021 10:07:29 -0700 Subject: [PATCH 37/76] Update README etc to reference 1.41.0 --- README.md | 30 ++++++++++++------------ cronet/README.md | 2 +- documentation/android-channel-builder.md | 4 ++-- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 6611b0ef1af..e6c06bde236 100644 --- a/README.md +++ b/README.md @@ -31,8 +31,8 @@ For a guided tour, take a look at the [quick start guide](https://grpc.io/docs/languages/java/quickstart) or the more explanatory [gRPC basics](https://grpc.io/docs/languages/java/basics). -The [examples](https://github.com/grpc/grpc-java/tree/v1.40.1/examples) and the -[Android example](https://github.com/grpc/grpc-java/tree/v1.40.1/examples/android) +The [examples](https://github.com/grpc/grpc-java/tree/v1.41.0/examples) and the +[Android example](https://github.com/grpc/grpc-java/tree/v1.41.0/examples/android) are standalone projects that showcase the usage of gRPC. Download @@ -43,17 +43,17 @@ Download [the JARs][]. Or for Maven with non-Android, add to your `pom.xml`: io.grpc grpc-netty-shaded - 1.40.1 + 1.41.0 io.grpc grpc-protobuf - 1.40.1 + 1.41.0 io.grpc grpc-stub - 1.40.1 + 1.41.0 org.apache.tomcat @@ -65,23 +65,23 @@ Download [the JARs][]. Or for Maven with non-Android, add to your `pom.xml`: Or for Gradle with non-Android, add to your dependencies: ```gradle -implementation 'io.grpc:grpc-netty-shaded:1.40.1' -implementation 'io.grpc:grpc-protobuf:1.40.1' -implementation 'io.grpc:grpc-stub:1.40.1' +implementation 'io.grpc:grpc-netty-shaded:1.41.0' +implementation 'io.grpc:grpc-protobuf:1.41.0' +implementation 'io.grpc:grpc-stub:1.41.0' compileOnly 'org.apache.tomcat:annotations-api:6.0.53' // necessary for Java 9+ ``` For Android client, use `grpc-okhttp` instead of `grpc-netty-shaded` and `grpc-protobuf-lite` instead of `grpc-protobuf`: ```gradle -implementation 'io.grpc:grpc-okhttp:1.40.1' -implementation 'io.grpc:grpc-protobuf-lite:1.40.1' -implementation 'io.grpc:grpc-stub:1.40.1' +implementation 'io.grpc:grpc-okhttp:1.41.0' +implementation 'io.grpc:grpc-protobuf-lite:1.41.0' +implementation 'io.grpc:grpc-stub:1.41.0' compileOnly 'org.apache.tomcat:annotations-api:6.0.53' // necessary for Java 9+ ``` [the JARs]: -https://search.maven.org/search?q=g:io.grpc%20AND%20v:1.40.1 +https://search.maven.org/search?q=g:io.grpc%20AND%20v:1.41.0 Development snapshots are available in [Sonatypes's snapshot repository](https://oss.sonatype.org/content/repositories/snapshots/). @@ -113,7 +113,7 @@ For protobuf-based codegen integrated with the Maven build system, you can use com.google.protobuf:protoc:3.17.3:exe:${os.detected.classifier} grpc-java - io.grpc:protoc-gen-grpc-java:1.40.1:exe:${os.detected.classifier} + io.grpc:protoc-gen-grpc-java:1.41.0:exe:${os.detected.classifier} @@ -143,7 +143,7 @@ protobuf { } plugins { grpc { - artifact = 'io.grpc:protoc-gen-grpc-java:1.40.1' + artifact = 'io.grpc:protoc-gen-grpc-java:1.41.0' } } generateProtoTasks { @@ -176,7 +176,7 @@ protobuf { } plugins { grpc { - artifact = 'io.grpc:protoc-gen-grpc-java:1.40.1' + artifact = 'io.grpc:protoc-gen-grpc-java:1.41.0' } } generateProtoTasks { diff --git a/cronet/README.md b/cronet/README.md index c982604bdac..8b220bd606d 100644 --- a/cronet/README.md +++ b/cronet/README.md @@ -26,7 +26,7 @@ In your app module's `build.gradle` file, include a dependency on both `grpc-cro Google Play Services Client Library for Cronet ``` -implementation 'io.grpc:grpc-cronet:1.40.1' +implementation 'io.grpc:grpc-cronet:1.41.0' implementation 'com.google.android.gms:play-services-cronet:16.0.0' ``` diff --git a/documentation/android-channel-builder.md b/documentation/android-channel-builder.md index 113b20159b9..60e3bb35a85 100644 --- a/documentation/android-channel-builder.md +++ b/documentation/android-channel-builder.md @@ -36,8 +36,8 @@ In your `build.gradle` file, include a dependency on both `grpc-android` and `grpc-okhttp`: ``` -implementation 'io.grpc:grpc-android:1.40.1' -implementation 'io.grpc:grpc-okhttp:1.40.1' +implementation 'io.grpc:grpc-android:1.41.0' +implementation 'io.grpc:grpc-okhttp:1.41.0' ``` You also need permission to access the device's network state in your From cf41181c487c1b5856c40242f148fcfabfeff1a0 Mon Sep 17 00:00:00 2001 From: Zhouyihai Ding Date: Wed, 22 Sep 2021 21:40:41 -0700 Subject: [PATCH 38/76] alts: add channel logs in handshake The logs are to help with debugging issues for an internal customer. --- .../alts/internal/AltsHandshakerClient.java | 21 ++++++++---- .../alts/internal/AltsProtocolNegotiator.java | 22 +++++++++---- .../grpc/alts/internal/AltsTsiHandshaker.java | 32 ++++++++++++------- .../alts/internal/TsiHandshakerFactory.java | 3 +- .../internal/AltsHandshakerClientTest.java | 7 ++-- .../internal/AltsProtocolNegotiatorTest.java | 9 +++--- .../alts/internal/AltsTsiHandshakerTest.java | 6 ++-- .../io/grpc/alts/internal/AltsTsiTest.java | 11 ++++--- .../grpc/alts/internal/FakeTsiHandshaker.java | 12 ++++--- 9 files changed, 81 insertions(+), 42 deletions(-) diff --git a/alts/src/main/java/io/grpc/alts/internal/AltsHandshakerClient.java b/alts/src/main/java/io/grpc/alts/internal/AltsHandshakerClient.java index 3b6a0c69616..9eb07f3e86d 100644 --- a/alts/src/main/java/io/grpc/alts/internal/AltsHandshakerClient.java +++ b/alts/src/main/java/io/grpc/alts/internal/AltsHandshakerClient.java @@ -20,19 +20,17 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.protobuf.ByteString; +import io.grpc.ChannelLogger; +import io.grpc.ChannelLogger.ChannelLogLevel; import io.grpc.Status; import io.grpc.alts.internal.HandshakerServiceGrpc.HandshakerServiceStub; import java.io.IOException; import java.nio.Buffer; import java.nio.ByteBuffer; import java.security.GeneralSecurityException; -import java.util.logging.Level; -import java.util.logging.Logger; /** An API for conducting handshakes via ALTS handshaker service. */ class AltsHandshakerClient { - private static final Logger logger = Logger.getLogger(AltsHandshakerClient.class.getName()); - private static final String APPLICATION_PROTOCOL = "grpc"; private static final String RECORD_PROTOCOL = "ALTSRP_GCM_AES128_REKEY"; private static final int KEY_LENGTH = AltsChannelCrypter.getKeyLength(); @@ -41,17 +39,22 @@ class AltsHandshakerClient { private final AltsHandshakerOptions handshakerOptions; private HandshakerResult result; private HandshakerStatus status; + private final ChannelLogger logger; /** Starts a new handshake interacting with the handshaker service. */ - AltsHandshakerClient(HandshakerServiceStub stub, AltsHandshakerOptions options) { + AltsHandshakerClient( + HandshakerServiceStub stub, AltsHandshakerOptions options, ChannelLogger logger) { handshakerStub = new AltsHandshakerStub(stub); handshakerOptions = options; + this.logger = logger; } @VisibleForTesting - AltsHandshakerClient(AltsHandshakerStub handshakerStub, AltsHandshakerOptions options) { + AltsHandshakerClient( + AltsHandshakerStub handshakerStub, AltsHandshakerOptions options, ChannelLogger logger) { this.handshakerStub = handshakerStub; handshakerOptions = options; + this.logger = logger; } static String getApplicationProtocol() { @@ -154,7 +157,7 @@ private void handleResponse(HandshakerResp resp) throws GeneralSecurityException } if (status.getCode() != Status.Code.OK.value()) { String error = "Handshaker service error: " + status.getDetails(); - logger.log(Level.INFO, error); + logger.log(ChannelLogLevel.DEBUG, error); close(); throw new GeneralSecurityException(error); } @@ -173,7 +176,9 @@ public ByteBuffer startClientHandshake() throws GeneralSecurityException { setStartClientFields(req); HandshakerResp resp; try { + logger.log(ChannelLogLevel.DEBUG, "Send ALTS handshake request to upstream"); resp = handshakerStub.send(req.build()); + logger.log(ChannelLogLevel.DEBUG, "Receive ALTS handshake response from upstream"); } catch (IOException | InterruptedException e) { throw new GeneralSecurityException(e); } @@ -223,7 +228,9 @@ public ByteBuffer next(ByteBuffer inBytes) throws GeneralSecurityException { .build()); HandshakerResp resp; try { + logger.log(ChannelLogLevel.DEBUG, "Send ALTS handshake request to upstream"); resp = handshakerStub.send(req.build()); + logger.log(ChannelLogLevel.DEBUG, "Receive ALTS handshake response from upstream"); } catch (IOException | InterruptedException e) { throw new GeneralSecurityException(e); } diff --git a/alts/src/main/java/io/grpc/alts/internal/AltsProtocolNegotiator.java b/alts/src/main/java/io/grpc/alts/internal/AltsProtocolNegotiator.java index ad2edd4e988..edfff2b481f 100644 --- a/alts/src/main/java/io/grpc/alts/internal/AltsProtocolNegotiator.java +++ b/alts/src/main/java/io/grpc/alts/internal/AltsProtocolNegotiator.java @@ -115,8 +115,9 @@ public AsciiString scheme() { @Override public ChannelHandler newHandler(GrpcHttp2ConnectionHandler grpcHandler) { - TsiHandshaker handshaker = handshakerFactory.newHandshaker(grpcHandler.getAuthority()); ChannelLogger negotiationLogger = grpcHandler.getNegotiationLogger(); + TsiHandshaker handshaker = + handshakerFactory.newHandshaker(grpcHandler.getAuthority(), negotiationLogger); NettyTsiHandshaker nettyHandshaker = new NettyTsiHandshaker(handshaker); ChannelHandler gnh = InternalProtocolNegotiators.grpcNegotiationHandler(grpcHandler); ChannelHandler thh = new TsiHandshakeHandler( @@ -142,11 +143,13 @@ public static ProtocolNegotiator serverAltsProtocolNegotiator( final class ServerTsiHandshakerFactory implements TsiHandshakerFactory { @Override - public TsiHandshaker newHandshaker(@Nullable String authority) { + public TsiHandshaker newHandshaker( + @Nullable String authority, ChannelLogger negotiationLogger) { assert authority == null; return AltsTsiHandshaker.newServer( HandshakerServiceGrpc.newStub(lazyHandshakerChannel.get()), - new AltsHandshakerOptions(RpcProtocolVersionsUtil.getRpcProtocolVersions())); + new AltsHandshakerOptions(RpcProtocolVersionsUtil.getRpcProtocolVersions()), + negotiationLogger); } } @@ -174,7 +177,8 @@ public AsciiString scheme() { @Override public ChannelHandler newHandler(GrpcHttp2ConnectionHandler grpcHandler) { ChannelLogger negotiationLogger = grpcHandler.getNegotiationLogger(); - TsiHandshaker handshaker = handshakerFactory.newHandshaker(/* authority= */ null); + TsiHandshaker handshaker = + handshakerFactory.newHandshaker(/* authority= */ null, negotiationLogger); NettyTsiHandshaker nettyHandshaker = new NettyTsiHandshaker(handshaker); ChannelHandler gnh = InternalProtocolNegotiators.grpcNegotiationHandler(grpcHandler); ChannelHandler thh = new TsiHandshakeHandler( @@ -292,7 +296,8 @@ public ChannelHandler newHandler(GrpcHttp2ConnectionHandler grpcHandler) { if (grpcHandler.getEagAttributes().get(GrpclbConstants.ATTR_LB_ADDR_AUTHORITY) != null || grpcHandler.getEagAttributes().get(GrpclbConstants.ATTR_LB_PROVIDED_BACKEND) != null || isXdsDirectPath) { - TsiHandshaker handshaker = handshakerFactory.newHandshaker(grpcHandler.getAuthority()); + TsiHandshaker handshaker = + handshakerFactory.newHandshaker(grpcHandler.getAuthority(), negotiationLogger); NettyTsiHandshaker nettyHandshaker = new NettyTsiHandshaker(handshaker); securityHandler = new TsiHandshakeHandler( gnh, nettyHandshaker, new AltsHandshakeValidator(), handshakeSemaphore, @@ -325,7 +330,8 @@ private static final class ClientTsiHandshakerFactory implements TsiHandshakerFa } @Override - public TsiHandshaker newHandshaker(@Nullable String authority) { + public TsiHandshaker newHandshaker( + @Nullable String authority, ChannelLogger negotiationLogger) { AltsClientOptions handshakerOptions = new AltsClientOptions.Builder() .setRpcProtocolVersions(RpcProtocolVersionsUtil.getRpcProtocolVersions()) @@ -333,7 +339,9 @@ public TsiHandshaker newHandshaker(@Nullable String authority) { .setTargetName(authority) .build(); return AltsTsiHandshaker.newClient( - HandshakerServiceGrpc.newStub(lazyHandshakerChannel.get()), handshakerOptions); + HandshakerServiceGrpc.newStub(lazyHandshakerChannel.get()), + handshakerOptions, + negotiationLogger); } } diff --git a/alts/src/main/java/io/grpc/alts/internal/AltsTsiHandshaker.java b/alts/src/main/java/io/grpc/alts/internal/AltsTsiHandshaker.java index cc750a72a7a..2269f0a0fa9 100644 --- a/alts/src/main/java/io/grpc/alts/internal/AltsTsiHandshaker.java +++ b/alts/src/main/java/io/grpc/alts/internal/AltsTsiHandshaker.java @@ -20,6 +20,8 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import io.grpc.ChannelLogger; +import io.grpc.ChannelLogger.ChannelLogLevel; import io.grpc.alts.internal.HandshakerServiceGrpc.HandshakerServiceStub; import io.netty.buffer.ByteBufAllocator; import java.nio.Buffer; @@ -27,14 +29,12 @@ import java.security.GeneralSecurityException; import java.util.ArrayList; import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; /** * Negotiates a grpc channel key to be used by the TsiFrameProtector, using ALTs handshaker service. */ public final class AltsTsiHandshaker implements TsiHandshaker { - private static final Logger logger = Logger.getLogger(AltsTsiHandshaker.class.getName()); + private final ChannelLogger logger; public static final String TSI_SERVICE_ACCOUNT_PEER_PROPERTY = "service_account"; @@ -45,15 +45,20 @@ public final class AltsTsiHandshaker implements TsiHandshaker { /** Starts a new TSI handshaker with client options. */ private AltsTsiHandshaker( - boolean isClient, HandshakerServiceStub stub, AltsHandshakerOptions options) { + boolean isClient, + HandshakerServiceStub stub, + AltsHandshakerOptions options, + ChannelLogger logger) { this.isClient = isClient; - handshaker = new AltsHandshakerClient(stub, options); + this.logger = logger; + handshaker = new AltsHandshakerClient(stub, options, logger); } @VisibleForTesting - AltsTsiHandshaker(boolean isClient, AltsHandshakerClient handshaker) { + AltsTsiHandshaker(boolean isClient, AltsHandshakerClient handshaker, ChannelLogger logger) { this.isClient = isClient; this.handshaker = handshaker; + this.logger = logger; } /** @@ -80,6 +85,7 @@ public boolean processBytesFromPeer(ByteBuffer bytes) throws GeneralSecurityExce checkState(!isClient, "Client handshaker should not process any frame at the beginning."); outputFrame = handshaker.startServerHandshake(bytes); } else { + logger.log(ChannelLogLevel.DEBUG, "Receive ALTS handshake from downstream"); outputFrame = handshaker.next(bytes); } // If handshake has finished or we already have bytes to write, just return true. @@ -124,13 +130,15 @@ public Object extractPeerObject() throws GeneralSecurityException { } /** Creates a new TsiHandshaker for use by the client. */ - public static TsiHandshaker newClient(HandshakerServiceStub stub, AltsHandshakerOptions options) { - return new AltsTsiHandshaker(true, stub, options); + public static TsiHandshaker newClient( + HandshakerServiceStub stub, AltsHandshakerOptions options, ChannelLogger logger) { + return new AltsTsiHandshaker(true, stub, options, logger); } /** Creates a new TsiHandshaker for use by the server. */ - public static TsiHandshaker newServer(HandshakerServiceStub stub, AltsHandshakerOptions options) { - return new AltsTsiHandshaker(false, stub, options); + public static TsiHandshaker newServer( + HandshakerServiceStub stub, AltsHandshakerOptions options, ChannelLogger logger) { + return new AltsTsiHandshaker(false, stub, options, logger); } /** @@ -142,12 +150,14 @@ public static TsiHandshaker newServer(HandshakerServiceStub stub, AltsHandshaker public void getBytesToSendToPeer(ByteBuffer bytes) throws GeneralSecurityException { if (outputFrame == null) { // A null outputFrame indicates we haven't started the handshake. if (isClient) { + logger.log(ChannelLogLevel.DEBUG, "Initial ALTS handshake to downstream"); outputFrame = handshaker.startClientHandshake(); } else { // The server needs bytes to process before it can start the handshake. return; } } + logger.log(ChannelLogLevel.DEBUG, "Send ALTS request to downstream"); // Write as many bytes as we are able. ByteBuffer outputFrameAlias = outputFrame; if (outputFrame.remaining() > bytes.remaining()) { @@ -190,7 +200,7 @@ public TsiFrameProtector createFrameProtector(int maxFrameSize, ByteBufAllocator maxFrameSize = Math.min(peerMaxFrameSize, AltsTsiFrameProtector.getMaxFrameSize()); maxFrameSize = Math.max(AltsTsiFrameProtector.getMinFrameSize(), maxFrameSize); } - logger.log(Level.FINE, "Maximum frame size value is {0}.", maxFrameSize); + logger.log(ChannelLogLevel.INFO, "Maximum frame size value is {0}.", maxFrameSize); return new AltsTsiFrameProtector(maxFrameSize, new AltsChannelCrypter(key, isClient), alloc); } diff --git a/alts/src/main/java/io/grpc/alts/internal/TsiHandshakerFactory.java b/alts/src/main/java/io/grpc/alts/internal/TsiHandshakerFactory.java index 996bd003654..7d17a3954c8 100644 --- a/alts/src/main/java/io/grpc/alts/internal/TsiHandshakerFactory.java +++ b/alts/src/main/java/io/grpc/alts/internal/TsiHandshakerFactory.java @@ -16,11 +16,12 @@ package io.grpc.alts.internal; +import io.grpc.ChannelLogger; import javax.annotation.Nullable; /** Factory that manufactures instances of {@link TsiHandshaker}. */ public interface TsiHandshakerFactory { /** Creates a new handshaker. */ - TsiHandshaker newHandshaker(@Nullable String authority); + TsiHandshaker newHandshaker(@Nullable String authority, ChannelLogger logger); } diff --git a/alts/src/test/java/io/grpc/alts/internal/AltsHandshakerClientTest.java b/alts/src/test/java/io/grpc/alts/internal/AltsHandshakerClientTest.java index 27ad16ee2d3..5a41fc0fc4f 100644 --- a/alts/src/test/java/io/grpc/alts/internal/AltsHandshakerClientTest.java +++ b/alts/src/test/java/io/grpc/alts/internal/AltsHandshakerClientTest.java @@ -29,6 +29,7 @@ import com.google.common.collect.ImmutableList; import com.google.protobuf.ByteString; +import io.grpc.internal.TestUtils.NoopChannelLogger; import java.nio.Buffer; import java.nio.ByteBuffer; import java.security.GeneralSecurityException; @@ -60,7 +61,8 @@ public void setUp() { .setTargetName(TEST_TARGET_NAME) .setTargetServiceAccounts(ImmutableList.of(TEST_TARGET_SERVICE_ACCOUNT)) .build(); - handshaker = new AltsHandshakerClient(mockStub, clientOptions); + NoopChannelLogger channelLogger = new NoopChannelLogger(); + handshaker = new AltsHandshakerClient(mockStub, clientOptions, channelLogger); } @Test @@ -266,7 +268,8 @@ public void setRpcVersions() throws Exception { .setTargetServiceAccounts(ImmutableList.of(TEST_TARGET_SERVICE_ACCOUNT)) .setRpcProtocolVersions(rpcVersions) .build(); - handshaker = new AltsHandshakerClient(mockStub, clientOptions); + NoopChannelLogger channelLogger = new NoopChannelLogger(); + handshaker = new AltsHandshakerClient(mockStub, clientOptions, channelLogger); handshaker.startClientHandshake(); diff --git a/alts/src/test/java/io/grpc/alts/internal/AltsProtocolNegotiatorTest.java b/alts/src/test/java/io/grpc/alts/internal/AltsProtocolNegotiatorTest.java index a981bf8db27..b2506288efc 100644 --- a/alts/src/test/java/io/grpc/alts/internal/AltsProtocolNegotiatorTest.java +++ b/alts/src/test/java/io/grpc/alts/internal/AltsProtocolNegotiatorTest.java @@ -26,6 +26,7 @@ import io.grpc.Attributes; import io.grpc.Channel; +import io.grpc.ChannelLogger; import io.grpc.Grpc; import io.grpc.InternalChannelz; import io.grpc.ManagedChannel; @@ -131,8 +132,8 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws E TsiHandshakerFactory handshakerFactory = new DelegatingTsiHandshakerFactory(FakeTsiHandshaker.clientHandshakerFactory()) { @Override - public TsiHandshaker newHandshaker(String authority) { - return new DelegatingTsiHandshaker(super.newHandshaker(authority)) { + public TsiHandshaker newHandshaker(String authority, ChannelLogger logger) { + return new DelegatingTsiHandshaker(super.newHandshaker(authority, logger)) { @Override public TsiPeer extractPeer() throws GeneralSecurityException { return mockedTsiPeer; @@ -427,8 +428,8 @@ private static class DelegatingTsiHandshakerFactory implements TsiHandshakerFact } @Override - public TsiHandshaker newHandshaker(String authority) { - return delegate.newHandshaker(authority); + public TsiHandshaker newHandshaker(String authority, ChannelLogger logger) { + return delegate.newHandshaker(authority, logger); } } diff --git a/alts/src/test/java/io/grpc/alts/internal/AltsTsiHandshakerTest.java b/alts/src/test/java/io/grpc/alts/internal/AltsTsiHandshakerTest.java index b0b2f8f9faa..ae1696401be 100644 --- a/alts/src/test/java/io/grpc/alts/internal/AltsTsiHandshakerTest.java +++ b/alts/src/test/java/io/grpc/alts/internal/AltsTsiHandshakerTest.java @@ -26,6 +26,7 @@ import static org.mockito.Mockito.when; import com.google.protobuf.ByteString; +import io.grpc.internal.TestUtils.NoopChannelLogger; import java.nio.Buffer; import java.nio.ByteBuffer; import org.junit.Before; @@ -71,8 +72,9 @@ public class AltsTsiHandshakerTest { public void setUp() throws Exception { mockClient = mock(AltsHandshakerClient.class); mockServer = mock(AltsHandshakerClient.class); - handshakerClient = new AltsTsiHandshaker(true, mockClient); - handshakerServer = new AltsTsiHandshaker(false, mockServer); + NoopChannelLogger channelLogger = new NoopChannelLogger(); + handshakerClient = new AltsTsiHandshaker(true, mockClient, channelLogger); + handshakerServer = new AltsTsiHandshaker(false, mockServer, channelLogger); } private HandshakerResult getHandshakerResult(boolean isClient) { diff --git a/alts/src/test/java/io/grpc/alts/internal/AltsTsiTest.java b/alts/src/test/java/io/grpc/alts/internal/AltsTsiTest.java index a6832ad8de2..cb39abb9ddc 100644 --- a/alts/src/test/java/io/grpc/alts/internal/AltsTsiTest.java +++ b/alts/src/test/java/io/grpc/alts/internal/AltsTsiTest.java @@ -21,6 +21,7 @@ import com.google.common.testing.GcFinalization; import io.grpc.alts.internal.ByteBufTestUtils.RegisterRef; import io.grpc.alts.internal.TsiTest.Handshakers; +import io.grpc.internal.TestUtils.NoopChannelLogger; import io.netty.buffer.ByteBuf; import io.netty.util.ReferenceCounted; import io.netty.util.ResourceLeakDetector; @@ -61,8 +62,9 @@ public void setUp() throws Exception { AltsHandshakerOptions handshakerOptions = new AltsHandshakerOptions(null); MockAltsHandshakerStub clientStub = new MockAltsHandshakerStub(); MockAltsHandshakerStub serverStub = new MockAltsHandshakerStub(); - client = new AltsHandshakerClient(clientStub, handshakerOptions); - server = new AltsHandshakerClient(serverStub, handshakerOptions); + NoopChannelLogger channelLogger = new NoopChannelLogger(); + client = new AltsHandshakerClient(clientStub, handshakerOptions, channelLogger); + server = new AltsHandshakerClient(serverStub, handshakerOptions, channelLogger); } @After @@ -76,8 +78,9 @@ public void tearDown() { } private Handshakers newHandshakers() { - TsiHandshaker clientHandshaker = new AltsTsiHandshaker(true, client); - TsiHandshaker serverHandshaker = new AltsTsiHandshaker(false, server); + NoopChannelLogger channelLogger = new NoopChannelLogger(); + TsiHandshaker clientHandshaker = new AltsTsiHandshaker(true, client, channelLogger); + TsiHandshaker serverHandshaker = new AltsTsiHandshaker(false, server, channelLogger); return new Handshakers(clientHandshaker, serverHandshaker); } diff --git a/alts/src/test/java/io/grpc/alts/internal/FakeTsiHandshaker.java b/alts/src/test/java/io/grpc/alts/internal/FakeTsiHandshaker.java index fcc33681732..a68f842a98e 100644 --- a/alts/src/test/java/io/grpc/alts/internal/FakeTsiHandshaker.java +++ b/alts/src/test/java/io/grpc/alts/internal/FakeTsiHandshaker.java @@ -19,7 +19,9 @@ import static java.nio.charset.StandardCharsets.UTF_8; import com.google.common.base.Preconditions; +import io.grpc.ChannelLogger; import io.grpc.alts.internal.TsiPeer.Property; +import io.grpc.internal.TestUtils.NoopChannelLogger; import io.netty.buffer.ByteBufAllocator; import java.nio.ByteBuffer; import java.security.GeneralSecurityException; @@ -37,7 +39,7 @@ public class FakeTsiHandshaker implements TsiHandshaker { private static final TsiHandshakerFactory clientHandshakerFactory = new TsiHandshakerFactory() { @Override - public TsiHandshaker newHandshaker(String authority) { + public TsiHandshaker newHandshaker(String authority, ChannelLogger logger) { return new FakeTsiHandshaker(true); } }; @@ -45,7 +47,7 @@ public TsiHandshaker newHandshaker(String authority) { private static final TsiHandshakerFactory serverHandshakerFactory = new TsiHandshakerFactory() { @Override - public TsiHandshaker newHandshaker(String authority) { + public TsiHandshaker newHandshaker(String authority, ChannelLogger logger) { return new FakeTsiHandshaker(false); } }; @@ -83,11 +85,13 @@ public static TsiHandshakerFactory serverHandshakerFactory() { } public static TsiHandshaker newFakeHandshakerClient() { - return clientHandshakerFactory.newHandshaker(null); + NoopChannelLogger channelLogger = new NoopChannelLogger(); + return clientHandshakerFactory.newHandshaker(null, channelLogger); } public static TsiHandshaker newFakeHandshakerServer() { - return serverHandshakerFactory.newHandshaker(null); + NoopChannelLogger channelLogger = new NoopChannelLogger(); + return serverHandshakerFactory.newHandshaker(null, channelLogger); } protected FakeTsiHandshaker(boolean isClient) { From ce311bdfd8fae4bc29f734834fb8d82e955622bd Mon Sep 17 00:00:00 2001 From: yifeizhuang Date: Thu, 23 Sep 2021 16:25:38 -0700 Subject: [PATCH 39/76] tsan: fix SdsProtocolNegotiatorsTest tsan failure due to thread unsafeness (#8374) --- .../CommonCertProviderTestUtils.java | 25 +++++++++++++++++++ .../sds/SdsProtocolNegotiatorsTest.java | 14 +++++++++++ 2 files changed, 39 insertions(+) diff --git a/xds/src/test/java/io/grpc/xds/internal/certprovider/CommonCertProviderTestUtils.java b/xds/src/test/java/io/grpc/xds/internal/certprovider/CommonCertProviderTestUtils.java index 2933d630068..0e60c4c6716 100644 --- a/xds/src/test/java/io/grpc/xds/internal/certprovider/CommonCertProviderTestUtils.java +++ b/xds/src/test/java/io/grpc/xds/internal/certprovider/CommonCertProviderTestUtils.java @@ -19,7 +19,10 @@ import static java.nio.charset.StandardCharsets.UTF_8; import com.google.common.io.CharStreams; +import io.grpc.internal.FakeClock; +import io.grpc.internal.TimeProvider; import io.grpc.internal.testing.TestUtils; +import io.grpc.xds.internal.certprovider.FileWatcherCertificateProviderProvider.ScheduledExecutorServiceFactory; import io.grpc.xds.internal.sds.trust.CertificateUtils; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -29,6 +32,7 @@ import java.security.PrivateKey; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; +import java.util.concurrent.ScheduledExecutorService; public class CommonCertProviderTestUtils { @@ -52,4 +56,25 @@ private static String getResourceContents(String resourceName) throws IOExceptio } return text; } + + /** Allow tests to register a provider using test clock. + */ + public static void register(final FakeClock fakeClock) { + FileWatcherCertificateProviderProvider tmp = new FileWatcherCertificateProviderProvider( + FileWatcherCertificateProvider.Factory.getInstance(), + new ScheduledExecutorServiceFactory() { + + @Override + ScheduledExecutorService create() { + return fakeClock.getScheduledExecutorService(); + } + }, + TimeProvider.SYSTEM_TIME_PROVIDER); + CertificateProviderRegistry.getInstance().register(tmp); + } + + public static void register0() { + CertificateProviderRegistry.getInstance().register( + new FileWatcherCertificateProviderProvider()); + } } diff --git a/xds/src/test/java/io/grpc/xds/internal/sds/SdsProtocolNegotiatorsTest.java b/xds/src/test/java/io/grpc/xds/internal/sds/SdsProtocolNegotiatorsTest.java index 4c89aa4b79a..502d2185a82 100644 --- a/xds/src/test/java/io/grpc/xds/internal/sds/SdsProtocolNegotiatorsTest.java +++ b/xds/src/test/java/io/grpc/xds/internal/sds/SdsProtocolNegotiatorsTest.java @@ -38,6 +38,7 @@ import io.grpc.Attributes; import io.grpc.ChannelLogger; import io.grpc.ChannelLogger.ChannelLogLevel; +import io.grpc.internal.FakeClock; import io.grpc.internal.TestUtils.NoopChannelLogger; import io.grpc.netty.GrpcHttp2ConnectionHandler; import io.grpc.netty.InternalProtocolNegotiationEvent; @@ -50,6 +51,7 @@ import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; import io.grpc.xds.InternalXdsAttributes; import io.grpc.xds.TlsContextManager; +import io.grpc.xds.internal.certprovider.CommonCertProviderTestUtils; import io.grpc.xds.internal.sds.SdsProtocolNegotiators.ClientSdsHandler; import io.grpc.xds.internal.sds.SdsProtocolNegotiators.ClientSdsProtocolNegotiator; import io.netty.channel.ChannelHandler; @@ -142,6 +144,8 @@ public void clientSdsProtocolNegotiatorNewHandler_withTlsContextAttribute() { @Test public void clientSdsHandler_addLast() throws InterruptedException, TimeoutException, ExecutionException { + FakeClock executor = new FakeClock(); + CommonCertProviderTestUtils.register(executor); Bootstrapper.BootstrapInfo bootstrapInfoForClient = CommonBootstrapperTestUtils .buildBootstrapInfo("google_cloud_private_spiffe-client", CLIENT_KEY_FILE, CLIENT_PEM_FILE, CA_PEM_FILE, null, null, null, null); @@ -173,6 +177,7 @@ protected void onException(Throwable throwable) { future.set(throwable); } }); + assertThat(executor.runDueTasks()).isEqualTo(1); channel.runPendingTasks(); Object fromFuture = future.get(2, TimeUnit.SECONDS); assertThat(fromFuture).isInstanceOf(SslContext.class); @@ -186,11 +191,14 @@ protected void onException(Throwable throwable) { // ProtocolNegotiators.ClientTlsHandler.class not accessible, get canonical name assertThat(iterator.next().getValue().getClass().getCanonicalName()) .contains("ProtocolNegotiators.ClientTlsHandler"); + CommonCertProviderTestUtils.register0(); } @Test public void serverSdsHandler_addLast() throws InterruptedException, TimeoutException, ExecutionException { + FakeClock executor = new FakeClock(); + CommonCertProviderTestUtils.register(executor); // we need InetSocketAddress instead of EmbeddedSocketAddress as localAddress for this test channel = new EmbeddedChannel() { @@ -247,6 +255,7 @@ protected void onException(Throwable throwable) { } }); channel.runPendingTasks(); // need this for tasks to execute on eventLoop + assertThat(executor.runDueTasks()).isEqualTo(1); Object fromFuture = future.get(2, TimeUnit.SECONDS); assertThat(fromFuture).isInstanceOf(SslContext.class); channel.runPendingTasks(); @@ -259,6 +268,7 @@ protected void onException(Throwable throwable) { // ProtocolNegotiators.ServerTlsHandler.class is not accessible, get canonical name assertThat(iterator.next().getValue().getClass().getCanonicalName()) .contains("ProtocolNegotiators.ServerTlsHandler"); + CommonCertProviderTestUtils.register0(); } @Test @@ -346,6 +356,8 @@ public void nullTlsContext_nullFallbackProtocolNegotiator_expectException() { @Test public void clientSdsProtocolNegotiatorNewHandler_fireProtocolNegotiationEvent() throws InterruptedException, TimeoutException, ExecutionException { + FakeClock executor = new FakeClock(); + CommonCertProviderTestUtils.register(executor); Bootstrapper.BootstrapInfo bootstrapInfoForClient = CommonBootstrapperTestUtils .buildBootstrapInfo("google_cloud_private_spiffe-client", CLIENT_KEY_FILE, CLIENT_PEM_FILE, CA_PEM_FILE, null, null, null, null); @@ -378,6 +390,7 @@ protected void onException(Throwable throwable) { future.set(throwable); } }); + executor.runDueTasks(); channel.runPendingTasks(); // need this for tasks to execute on eventLoop Object fromFuture = future.get(5, TimeUnit.SECONDS); assertThat(fromFuture).isInstanceOf(SslContext.class); @@ -389,6 +402,7 @@ protected void onException(Throwable throwable) { pipeline.fireUserEventTriggered(sslEvent); channel.runPendingTasks(); // need this for tasks to execute on eventLoop assertTrue(channel.isOpen()); + CommonCertProviderTestUtils.register0(); } private static final class FakeGrpcHttp2ConnectionHandler extends GrpcHttp2ConnectionHandler { From 0245a7292690907c623d2cef508491b9e036ea67 Mon Sep 17 00:00:00 2001 From: yifeizhuang Date: Fri, 24 Sep 2021 10:36:00 -0700 Subject: [PATCH 40/76] xds: error descriptions improvements(#8554) --- .../java/io/grpc/xds/CdsLoadBalancer2.java | 3 ++- .../main/java/io/grpc/xds/ClientXdsClient.java | 3 ++- .../io/grpc/xds/ClusterImplLoadBalancer.java | 4 +++- .../grpc/xds/ClusterManagerLoadBalancer.java | 3 ++- .../grpc/xds/ClusterResolverLoadBalancer.java | 7 +++++-- ...FilterChainMatchingProtocolNegotiators.java | 13 +++++++++++-- xds/src/main/java/io/grpc/xds/RbacFilter.java | 8 +++++--- .../java/io/grpc/xds/RingHashLoadBalancer.java | 6 ++++-- .../java/io/grpc/xds/XdsServerWrapper.java | 5 +++++ .../rbac/engine/GrpcAuthorizationEngine.java | 2 -- .../internal/sds/SdsProtocolNegotiators.java | 7 ++++++- .../java/io/grpc/xds/CdsLoadBalancer2Test.java | 18 ++++++++++++------ .../io/grpc/xds/ClientXdsClientDataTest.java | 4 ++-- .../xds/ClusterManagerLoadBalancerTest.java | 3 ++- 14 files changed, 61 insertions(+), 25 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/CdsLoadBalancer2.java b/xds/src/main/java/io/grpc/xds/CdsLoadBalancer2.java index 036f77f7cd1..4af187bf1dd 100644 --- a/xds/src/main/java/io/grpc/xds/CdsLoadBalancer2.java +++ b/xds/src/main/java/io/grpc/xds/CdsLoadBalancer2.java @@ -179,7 +179,8 @@ private void handleClusterDiscovered() { childLb = null; } Status unavailable = - Status.UNAVAILABLE.withDescription("Cluster " + root.name + " unusable"); + Status.UNAVAILABLE.withDescription("CDS error: found 0 leaf (logical DNS or EDS) " + + "clusters for root cluster " + root.name); helper.updateBalancingState(TRANSIENT_FAILURE, new ErrorPicker(unavailable)); return; } diff --git a/xds/src/main/java/io/grpc/xds/ClientXdsClient.java b/xds/src/main/java/io/grpc/xds/ClientXdsClient.java index f39992c24ac..848e6691732 100644 --- a/xds/src/main/java/io/grpc/xds/ClientXdsClient.java +++ b/xds/src/main/java/io/grpc/xds/ClientXdsClient.java @@ -566,7 +566,8 @@ private static void checkForUniqueness(Set uniqueSet, List crossProduct = getCrossProduct(filterChainMatch); for (FilterChainMatch cur : crossProduct) { if (!uniqueSet.add(cur)) { - throw new ResourceInvalidException("Found duplicate matcher: " + cur); + throw new ResourceInvalidException("FilterChainMatch must be unique. " + + "Found duplicate: " + cur); } } } diff --git a/xds/src/main/java/io/grpc/xds/ClusterImplLoadBalancer.java b/xds/src/main/java/io/grpc/xds/ClusterImplLoadBalancer.java index d95361935a7..330c4e2f7a5 100644 --- a/xds/src/main/java/io/grpc/xds/ClusterImplLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/ClusterImplLoadBalancer.java @@ -124,7 +124,9 @@ public void handleResolvedAddresses(ResolvedAddresses resolvedAddresses) { if (config.lrsServerName.isEmpty()) { dropStats = xdsClient.addClusterDropStats(cluster, edsServiceName); } else { - logger.log(XdsLogLevel.WARNING, "Can only report load to the same management server"); + logger.log(XdsLogLevel.WARNING, "Cluster {0} config error: can only report load " + + "to the same management server. Config lrsServerName {1} should be empty. ", + cluster, config.lrsServerName); } } } diff --git a/xds/src/main/java/io/grpc/xds/ClusterManagerLoadBalancer.java b/xds/src/main/java/io/grpc/xds/ClusterManagerLoadBalancer.java index e91fbe2fa1b..0557f3a6a8c 100644 --- a/xds/src/main/java/io/grpc/xds/ClusterManagerLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/ClusterManagerLoadBalancer.java @@ -149,7 +149,8 @@ public PickResult pickSubchannel(PickSubchannelArgs args) { if (delegate == null) { return PickResult.withError( - Status.UNAVAILABLE.withDescription("Unable to find cluster " + clusterName)); + Status.UNAVAILABLE.withDescription("CDS encountered error: unable to find " + + "available subchannel for cluster " + clusterName)); } return delegate.pickSubchannel(args); } diff --git a/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancer.java b/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancer.java index 9cfbe3a753b..9ba7541e314 100644 --- a/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancer.java @@ -501,7 +501,8 @@ void start() { } resolver = nameResolverFactory.newNameResolver(uri, nameResolverArgs); if (resolver == null) { - status = Status.INTERNAL.withDescription("Cannot find DNS resolver"); + status = Status.INTERNAL.withDescription("Xds cluster resolver lb for logical DNS " + + "cluster [" + name + "] cannot find DNS resolver with uri:" + uri); handleEndpointResolutionError(); return; } @@ -607,7 +608,9 @@ public void run() { } long delayNanos = backoffPolicy.nextBackoffNanos(); logger.log(XdsLogLevel.DEBUG, - "Scheduling DNS resolution backoff for {0} ns", delayNanos); + "Logical DNS resolver for cluster {0} encountered name resolution " + + "error: {1}, scheduling DNS resolution backoff for {2} ns", + name, error, delayNanos); scheduledRefresh = syncContext.schedule( new DelayedNameResolverRefresh(), delayNanos, TimeUnit.NANOSECONDS, diff --git a/xds/src/main/java/io/grpc/xds/FilterChainMatchingProtocolNegotiators.java b/xds/src/main/java/io/grpc/xds/FilterChainMatchingProtocolNegotiators.java index b828b862454..5d72151db50 100644 --- a/xds/src/main/java/io/grpc/xds/FilterChainMatchingProtocolNegotiators.java +++ b/xds/src/main/java/io/grpc/xds/FilterChainMatchingProtocolNegotiators.java @@ -23,6 +23,7 @@ import static io.grpc.xds.internal.sds.SdsProtocolNegotiators.ATTR_SERVER_SSL_CONTEXT_PROVIDER_SUPPLIER; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.MoreObjects; import com.google.common.base.Strings; import com.google.common.collect.Iterables; import com.google.protobuf.UInt32Value; @@ -141,12 +142,11 @@ static final class FilterChainSelector { private final Map> routingConfigs; @Nullable private final SslContextProviderSupplier defaultSslContextProviderSupplier; - @Nullable private final AtomicReference defaultRoutingConfig; FilterChainSelector(Map> routingConfigs, @Nullable SslContextProviderSupplier defaultSslContextProviderSupplier, - @Nullable AtomicReference defaultRoutingConfig) { + AtomicReference defaultRoutingConfig) { this.routingConfigs = checkNotNull(routingConfigs, "routingConfigs"); this.defaultSslContextProviderSupplier = defaultSslContextProviderSupplier; this.defaultRoutingConfig = checkNotNull(defaultRoutingConfig, "defaultRoutingConfig"); @@ -350,6 +350,15 @@ private static Collection filterOnIpAddress( } return topOnes; } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("routingConfigs", routingConfigs) + .add("defaultSslContextProviderSupplier", defaultSslContextProviderSupplier) + .add("defaultRoutingConfig", defaultRoutingConfig) + .toString(); + } } } diff --git a/xds/src/main/java/io/grpc/xds/RbacFilter.java b/xds/src/main/java/io/grpc/xds/RbacFilter.java index be49df31b14..387afac82cc 100644 --- a/xds/src/main/java/io/grpc/xds/RbacFilter.java +++ b/xds/src/main/java/io/grpc/xds/RbacFilter.java @@ -177,9 +177,11 @@ public ServerCall.Listener interceptCall( final ServerCall call, final Metadata headers, ServerCallHandler next) { AuthDecision authResult = authEngine.evaluate(headers, call); - logger.log(Level.FINE, - "Authorization result for serverCall {0}: {1}, matching policy: {2}.", - new Object[]{call, authResult.decision(), authResult.matchingPolicyName()}); + if (logger.isLoggable(Level.FINER)) { + logger.log(Level.FINER, + "Authorization result for serverCall {0}: {1}, matching policy: {2}.", + new Object[]{call, authResult.decision(), authResult.matchingPolicyName()}); + } if (GrpcAuthorizationEngine.Action.DENY.equals(authResult.decision())) { Status status = Status.UNAUTHENTICATED.withDescription( "Access Denied, matching policy: " + authResult.matchingPolicyName()); diff --git a/xds/src/main/java/io/grpc/xds/RingHashLoadBalancer.java b/xds/src/main/java/io/grpc/xds/RingHashLoadBalancer.java index 05f29e20112..a8f517a8967 100644 --- a/xds/src/main/java/io/grpc/xds/RingHashLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/RingHashLoadBalancer.java @@ -56,7 +56,8 @@ final class RingHashLoadBalancer extends LoadBalancer { private static final Attributes.Key> STATE_INFO = Attributes.Key.create("state-info"); private static final Status RPC_HASH_NOT_FOUND = - Status.INTERNAL.withDescription("RPC hash not found"); + Status.INTERNAL.withDescription("RPC hash not found. Probably a bug because xds resolver" + + " config selector always generates a hash."); private static final XxHash64 hashFunc = XxHash64.INSTANCE; private final XdsLogger logger; @@ -79,7 +80,8 @@ public void handleResolvedAddresses(ResolvedAddresses resolvedAddresses) { logger.log(XdsLogLevel.DEBUG, "Received resolution result: {0}", resolvedAddresses); List addrList = resolvedAddresses.getAddresses(); if (addrList.isEmpty()) { - handleNameResolutionError(Status.UNAVAILABLE.withDescription("No server addresses found")); + handleNameResolutionError(Status.UNAVAILABLE.withDescription("Ring hash lb error: EDS " + + "resolution was successful, but returned server addresses are empty.")); return; } Map latestAddrs = stripAttrs(addrList); diff --git a/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java b/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java index 5f7cc43d670..a80398dedcd 100644 --- a/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java +++ b/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java @@ -318,6 +318,7 @@ private void startDelegateServer() { initialStarted = true; initialStartFuture.set(null); } + logger.log(Level.FINER, "Delegate server started."); } catch (IOException e) { logger.log(Level.FINE, "Fail to start delegate server: {0}", e); if (!initialStarted) { @@ -371,6 +372,7 @@ public void run() { if (stopped) { return; } + logger.log(Level.FINEST, "Received Lds update {0}", update); checkNotNull(update.listener(), "update"); if (!pendingRds.isEmpty()) { // filter chain state has not yet been applied to filterChainSelectorManager and there @@ -474,6 +476,7 @@ private void updateSelector() { defaultFilterChain == null ? new AtomicReference() : generateRoutingConfig(defaultFilterChain)); List toRelease = getSuppliersInUse(); + logger.log(Level.FINEST, "Updating selector {0}", selector); filterChainSelectorManager.updateSelector(selector); for (SslContextProviderSupplier e: toRelease) { e.close(); @@ -696,6 +699,8 @@ private void updateRdsRoutingConfig() { updatedRoutingConfig = ServerRoutingConfig.create(savedVirtualHosts, updatedInterceptors); } + logger.log(Level.FINEST, "Updating filter chain {0} rds routing config: {1}", + new Object[]{filterChain.getName(), updatedRoutingConfig}); savedRdsRoutingConfigRef.get(filterChain).set(updatedRoutingConfig); } } diff --git a/xds/src/main/java/io/grpc/xds/internal/rbac/engine/GrpcAuthorizationEngine.java b/xds/src/main/java/io/grpc/xds/internal/rbac/engine/GrpcAuthorizationEngine.java index 385a055daef..49574f9018c 100644 --- a/xds/src/main/java/io/grpc/xds/internal/rbac/engine/GrpcAuthorizationEngine.java +++ b/xds/src/main/java/io/grpc/xds/internal/rbac/engine/GrpcAuthorizationEngine.java @@ -78,8 +78,6 @@ public AuthDecision evaluate(Metadata metadata, ServerCall serverCall) { if (Action.DENY.equals(authConfig.action()) == (firstMatch == null)) { decisionType = Action.ALLOW; } - log.log(Level.FINER, "RBAC decision: {0}, policy match: {1}.", - new Object[]{decisionType, firstMatch}); return AuthDecision.create(decisionType, firstMatch); } diff --git a/xds/src/main/java/io/grpc/xds/internal/sds/SdsProtocolNegotiators.java b/xds/src/main/java/io/grpc/xds/internal/sds/SdsProtocolNegotiators.java index 0128fa53106..fefc9d406b2 100644 --- a/xds/src/main/java/io/grpc/xds/internal/sds/SdsProtocolNegotiators.java +++ b/xds/src/main/java/io/grpc/xds/internal/sds/SdsProtocolNegotiators.java @@ -281,11 +281,16 @@ public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exc SslContextProviderSupplier sslContextProviderSupplier = InternalProtocolNegotiationEvent .getAttributes(pne).get(ATTR_SERVER_SSL_CONTEXT_PROVIDER_SUPPLIER); if (sslContextProviderSupplier == null) { + logger.log(Level.FINE, "No sslContextProviderSupplier found in filterChainMatch " + + "for connection from {0} to {1}", + new Object[]{ctx.channel().remoteAddress(), ctx.channel().localAddress()}); if (fallbackProtocolNegotiator == null) { ctx.fireExceptionCaught(new CertStoreException("No certificate source found!")); return; } - logger.log(Level.INFO, "Using fallback for {0}", ctx.channel().localAddress()); + logger.log(Level.FINE, "Using fallback sslContextProviderSupplier for connection " + + "from {0} to {1}", + new Object[]{ctx.channel().remoteAddress(), ctx.channel().localAddress()}); ctx.pipeline() .replace( this, diff --git a/xds/src/test/java/io/grpc/xds/CdsLoadBalancer2Test.java b/xds/src/test/java/io/grpc/xds/CdsLoadBalancer2Test.java index 6953ecca649..24586c70e91 100644 --- a/xds/src/test/java/io/grpc/xds/CdsLoadBalancer2Test.java +++ b/xds/src/test/java/io/grpc/xds/CdsLoadBalancer2Test.java @@ -179,7 +179,8 @@ public void nonAggregateCluster_resourceNotExist_returnErrorPicker() { xdsClient.deliverResourceNotExist(CLUSTER); verify(helper).updateBalancingState( eq(ConnectivityState.TRANSIENT_FAILURE), pickerCaptor.capture()); - Status unavailable = Status.UNAVAILABLE.withDescription("Cluster " + CLUSTER + " unusable"); + Status unavailable = Status.UNAVAILABLE.withDescription( + "CDS error: found 0 leaf (logical DNS or EDS) clusters for root cluster " + CLUSTER); assertPicker(pickerCaptor.getValue(), unavailable, null); assertThat(childBalancers).isEmpty(); } @@ -221,7 +222,8 @@ public void nonAggregateCluster_resourceRevoked() { xdsClient.deliverResourceNotExist(CLUSTER); assertThat(childBalancer.shutdown).isTrue(); - Status unavailable = Status.UNAVAILABLE.withDescription("Cluster " + CLUSTER + " unusable"); + Status unavailable = Status.UNAVAILABLE.withDescription( + "CDS error: found 0 leaf (logical DNS or EDS) clusters for root cluster " + CLUSTER); verify(helper).updateBalancingState( eq(ConnectivityState.TRANSIENT_FAILURE), pickerCaptor.capture()); assertPicker(pickerCaptor.getValue(), unavailable, null); @@ -295,7 +297,8 @@ public void aggregateCluster_noNonAggregateClusterExits_returnErrorPicker() { xdsClient.deliverResourceNotExist(cluster1); verify(helper).updateBalancingState( eq(ConnectivityState.TRANSIENT_FAILURE), pickerCaptor.capture()); - Status unavailable = Status.UNAVAILABLE.withDescription("Cluster " + CLUSTER + " unusable"); + Status unavailable = Status.UNAVAILABLE.withDescription( + "CDS error: found 0 leaf (logical DNS or EDS) clusters for root cluster " + CLUSTER); assertPicker(pickerCaptor.getValue(), unavailable, null); assertThat(childBalancers).isEmpty(); } @@ -341,7 +344,8 @@ public void aggregateCluster_descendantClustersRevoked() { xdsClient.deliverResourceNotExist(cluster2); verify(helper).updateBalancingState( eq(ConnectivityState.TRANSIENT_FAILURE), pickerCaptor.capture()); - Status unavailable = Status.UNAVAILABLE.withDescription("Cluster " + CLUSTER + " unusable"); + Status unavailable = Status.UNAVAILABLE.withDescription( + "CDS error: found 0 leaf (logical DNS or EDS) clusters for root cluster " + CLUSTER); assertPicker(pickerCaptor.getValue(), unavailable, null); assertThat(childBalancer.shutdown).isTrue(); assertThat(childBalancers).isEmpty(); @@ -379,7 +383,8 @@ public void aggregateCluster_rootClusterRevoked() { .containsExactly(CLUSTER); // subscription to all descendant clusters cancelled verify(helper).updateBalancingState( eq(ConnectivityState.TRANSIENT_FAILURE), pickerCaptor.capture()); - Status unavailable = Status.UNAVAILABLE.withDescription("Cluster " + CLUSTER + " unusable"); + Status unavailable = Status.UNAVAILABLE.withDescription( + "CDS error: found 0 leaf (logical DNS or EDS) clusters for root cluster " + CLUSTER); assertPicker(pickerCaptor.getValue(), unavailable, null); assertThat(childBalancer.shutdown).isTrue(); assertThat(childBalancers).isEmpty(); @@ -427,7 +432,8 @@ public void aggregateCluster_intermediateClusterChanges() { .containsExactly(CLUSTER, cluster2); // cancelled subscription to cluster3 verify(helper).updateBalancingState( eq(ConnectivityState.TRANSIENT_FAILURE), pickerCaptor.capture()); - Status unavailable = Status.UNAVAILABLE.withDescription("Cluster " + CLUSTER + " unusable"); + Status unavailable = Status.UNAVAILABLE.withDescription( + "CDS error: found 0 leaf (logical DNS or EDS) clusters for root cluster " + CLUSTER); assertPicker(pickerCaptor.getValue(), unavailable, null); assertThat(childBalancer.shutdown).isTrue(); assertThat(childBalancers).isEmpty(); diff --git a/xds/src/test/java/io/grpc/xds/ClientXdsClientDataTest.java b/xds/src/test/java/io/grpc/xds/ClientXdsClientDataTest.java index 17ca907da42..ac8c3fb44e7 100644 --- a/xds/src/test/java/io/grpc/xds/ClientXdsClientDataTest.java +++ b/xds/src/test/java/io/grpc/xds/ClientXdsClientDataTest.java @@ -1407,7 +1407,7 @@ public void parseServerSideListener_nonUniqueFilterChainMatch() throws ResourceI .addAllFilterChains(Arrays.asList(filterChain1, filterChain2)) .build(); thrown.expect(ResourceInvalidException.class); - thrown.expectMessage("Found duplicate matcher:"); + thrown.expectMessage("FilterChainMatch must be unique. Found duplicate:"); ClientXdsClient.parseServerSideListener( listener, new HashSet(), null, filterRegistry, null, true /* does not matter */); } @@ -1456,7 +1456,7 @@ public void parseServerSideListener_nonUniqueFilterChainMatch_sameFilter() .addAllFilterChains(Arrays.asList(filterChain1, filterChain2)) .build(); thrown.expect(ResourceInvalidException.class); - thrown.expectMessage("Found duplicate matcher:"); + thrown.expectMessage("FilterChainMatch must be unique. Found duplicate:"); ClientXdsClient.parseServerSideListener( listener, new HashSet(), null, filterRegistry, null, true /* does not matter */); } diff --git a/xds/src/test/java/io/grpc/xds/ClusterManagerLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/ClusterManagerLoadBalancerTest.java index 38aed01a234..5890f6f9abb 100644 --- a/xds/src/test/java/io/grpc/xds/ClusterManagerLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/ClusterManagerLoadBalancerTest.java @@ -138,7 +138,8 @@ public void handleResolvedAddressesUpdatesChannelPicker() { assertThat(pickSubchannel(picker, "childC")).isEqualTo(PickResult.withNoResult()); Status status = pickSubchannel(picker, "childB").getStatus(); assertThat(status.getCode()).isEqualTo(Code.UNAVAILABLE); - assertThat(status.getDescription()).isEqualTo("Unable to find cluster childB"); + assertThat(status.getDescription()).isEqualTo( + "CDS encountered error: unable to find available subchannel for cluster childB"); assertThat(fakeClock.numPendingTasks()) .isEqualTo(1); // (delayed) shutdown because "childB" is removed assertThat(childBalancer1.shutdown).isFalse(); From 60475de2044250d8d9f3d787bdb04ef8d33575c0 Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Fri, 24 Sep 2021 12:18:17 -0700 Subject: [PATCH 41/76] xds: Log about fallback credentials, not supplier The sslContextProviderSupplier is used by the xds creds themselves when the control plane has security configured. But the fallback credentials don't use such a supplier and may not even be using TLS. Language tweak following #8554. --- .../java/io/grpc/xds/internal/sds/SdsProtocolNegotiators.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/internal/sds/SdsProtocolNegotiators.java b/xds/src/main/java/io/grpc/xds/internal/sds/SdsProtocolNegotiators.java index fefc9d406b2..a032737e647 100644 --- a/xds/src/main/java/io/grpc/xds/internal/sds/SdsProtocolNegotiators.java +++ b/xds/src/main/java/io/grpc/xds/internal/sds/SdsProtocolNegotiators.java @@ -288,8 +288,7 @@ public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exc ctx.fireExceptionCaught(new CertStoreException("No certificate source found!")); return; } - logger.log(Level.FINE, "Using fallback sslContextProviderSupplier for connection " - + "from {0} to {1}", + logger.log(Level.FINE, "Using fallback credentials for connection from {0} to {1}", new Object[]{ctx.channel().remoteAddress(), ctx.channel().localAddress()}); ctx.pipeline() .replace( From 192688f1f22c93632aaf598f895cf33ca84eeec0 Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Fri, 24 Sep 2021 13:41:33 -0700 Subject: [PATCH 42/76] netty: Requests with Connection header are malformed Although this is part of HTTP/2 and should have already been handled already, it was noticed as part of RBAC work to avoid matching hop-by-hop headers. See gRFC A41. Also add a warning if creating Metadata.Key for "Connection". Use this to try to help diagnose a client if it happens to blindly copy headers from HTTP/1, as PROTOCOL_ERROR is hard to debug. This rolls-forward 6e89919 after it was reverted in 7669656, now that the test proxy has been fixed. --- api/src/main/java/io/grpc/Metadata.java | 12 ++++++++++++ .../io/grpc/netty/GrpcHttp2HeadersUtils.java | 5 +++++ .../io/grpc/netty/NettyServerHandler.java | 7 +++++++ .../io/grpc/netty/NettyServerHandlerTest.java | 19 +++++++++++++++++++ 4 files changed, 43 insertions(+) diff --git a/api/src/main/java/io/grpc/Metadata.java b/api/src/main/java/io/grpc/Metadata.java index e153fd55691..9c2a2227f8c 100644 --- a/api/src/main/java/io/grpc/Metadata.java +++ b/api/src/main/java/io/grpc/Metadata.java @@ -40,6 +40,8 @@ import java.util.Map; import java.util.NoSuchElementException; import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; import javax.annotation.Nullable; import javax.annotation.concurrent.Immutable; import javax.annotation.concurrent.NotThreadSafe; @@ -54,6 +56,7 @@ */ @NotThreadSafe public final class Metadata { + private static final Logger logger = Logger.getLogger(Metadata.class.getName()); /** * All binary headers should have this suffix in their names. Vice versa. @@ -733,6 +736,15 @@ private static BitSet generateValidTChars() { private static String validateName(String n, boolean pseudo) { checkNotNull(n, "name"); checkArgument(!n.isEmpty(), "token must have at least 1 tchar"); + if (n.equals("connection")) { + logger.log( + Level.WARNING, + "Metadata key is 'Connection', which should not be used. That is used by HTTP/1 for " + + "connection-specific headers which are not to be forwarded. There is probably an " + + "HTTP/1 conversion bug. Simply removing the Connection header is not enough; you " + + "should remove all headers it references as well. See RFC 7230 section 6.1", + new RuntimeException("exception to show backtrace")); + } for (int i = 0; i < n.length(); i++) { char tChar = n.charAt(i); if (pseudo && tChar == ':' && i == 0) { diff --git a/netty/src/main/java/io/grpc/netty/GrpcHttp2HeadersUtils.java b/netty/src/main/java/io/grpc/netty/GrpcHttp2HeadersUtils.java index 70c0a6f041a..df7875fc7ae 100644 --- a/netty/src/main/java/io/grpc/netty/GrpcHttp2HeadersUtils.java +++ b/netty/src/main/java/io/grpc/netty/GrpcHttp2HeadersUtils.java @@ -145,6 +145,11 @@ protected CharSequence get(AsciiString name) { return null; } + @Override + public boolean contains(CharSequence name) { + return get(name) != null; + } + @Override public CharSequence status() { return get(Http2Headers.PseudoHeaderName.STATUS.value()); diff --git a/netty/src/main/java/io/grpc/netty/NettyServerHandler.java b/netty/src/main/java/io/grpc/netty/NettyServerHandler.java index c9e0762d292..c286c17f640 100644 --- a/netty/src/main/java/io/grpc/netty/NettyServerHandler.java +++ b/netty/src/main/java/io/grpc/netty/NettyServerHandler.java @@ -26,6 +26,7 @@ import static io.grpc.netty.Utils.HTTP_METHOD; import static io.grpc.netty.Utils.TE_HEADER; import static io.grpc.netty.Utils.TE_TRAILERS; +import static io.netty.handler.codec.http.HttpHeaderNames.CONNECTION; import static io.netty.handler.codec.http.HttpHeaderNames.HOST; import static io.netty.handler.codec.http2.DefaultHttp2LocalFlowController.DEFAULT_WINDOW_UPDATE_RATIO; import static io.netty.handler.codec.http2.Http2Headers.PseudoHeaderName.AUTHORITY; @@ -377,6 +378,12 @@ public void run() { private void onHeadersRead(ChannelHandlerContext ctx, int streamId, Http2Headers headers) throws Http2Exception { try { + // Connection-specific header fields makes a request malformed. Ideally this would be handled + // by Netty. RFC 7540 section 8.1.2.2 + if (headers.contains(CONNECTION)) { + resetStream(ctx, streamId, Http2Error.PROTOCOL_ERROR.code(), ctx.newPromise()); + return; + } if (headers.authority() == null) { List hosts = headers.getAll(HOST); diff --git a/netty/src/test/java/io/grpc/netty/NettyServerHandlerTest.java b/netty/src/test/java/io/grpc/netty/NettyServerHandlerTest.java index 5db8413f60b..170273e2c60 100644 --- a/netty/src/test/java/io/grpc/netty/NettyServerHandlerTest.java +++ b/netty/src/test/java/io/grpc/netty/NettyServerHandlerTest.java @@ -587,6 +587,25 @@ public void headersSupportExtensionContentType() throws Exception { stream = streamCaptor.getValue(); } + @Test + public void headersWithConnectionHeaderShouldFail() throws Exception { + manualSetUp(); + Http2Headers headers = new DefaultHttp2Headers() + .method(HTTP_METHOD) + .set(CONTENT_TYPE_HEADER, CONTENT_TYPE_GRPC) + .set(AsciiString.of("connection"), CONTENT_TYPE_GRPC) + .path(new AsciiString("/foo/bar")); + ByteBuf headersFrame = headersFrame(STREAM_ID, headers); + channelRead(headersFrame); + + verifyWrite() + .writeRstStream( + eq(ctx()), + eq(STREAM_ID), + eq(Http2Error.PROTOCOL_ERROR.code()), + any(ChannelPromise.class)); + } + @Test public void headersWithMultipleHostsShouldFail() throws Exception { manualSetUp(); From 816a54a83b354cad169ce83b72a431093c764469 Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Mon, 27 Sep 2021 13:26:04 -0700 Subject: [PATCH 43/76] api: Add doc snippet to convert types for defaultServiceConfig() Tested with Jackson's `new ObjectMapper().readValue(json, Map.class)`. Fixes #8300 --- .../main/java/io/grpc/ManagedChannelBuilder.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/api/src/main/java/io/grpc/ManagedChannelBuilder.java b/api/src/main/java/io/grpc/ManagedChannelBuilder.java index 98b22807ccc..726f2e1ba53 100644 --- a/api/src/main/java/io/grpc/ManagedChannelBuilder.java +++ b/api/src/main/java/io/grpc/ManagedChannelBuilder.java @@ -555,6 +555,22 @@ public T proxyDetector(ProxyDetector proxyDetector) { * *

If null is passed, then there will be no default service config. * + *

Your preferred JSON parser may not produce results in the format expected. For such cases, + * you can convert its output. For example, if your parser produces Integers and other Numbers + * in addition to Double: + * + * 

{@code @SuppressWarnings("unchecked")
+   * private static Object convertNumbers(Object o) {
+   *   if (o instanceof Map) {
+   *     ((Map) o).replaceAll((k,v) -> convertNumbers(v));
+   *   } else if (o instanceof List) {
+   *     ((List) o).replaceAll(YourClass::convertNumbers);
+   *   } else if (o instanceof Number && !(o instanceof Double)) {
+   *     o = ((Number) o).doubleValue();
+   *   }
+   *   return o;
+   * }}
+ * * @throws IllegalArgumentException When the given serviceConfig is invalid or the current version * of grpc library can not parse it gracefully. The state of the builder is unchanged if * an exception is thrown. From 0287d83182493a5c50f09edfa3486c8e514b1c95 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Fri, 17 Sep 2021 11:29:11 -0700 Subject: [PATCH 44/76] Add testing_version flag --- buildscripts/kokoro/xds_url_map.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/buildscripts/kokoro/xds_url_map.sh b/buildscripts/kokoro/xds_url_map.sh index d8487582980..87268ea5ea2 100755 --- a/buildscripts/kokoro/xds_url_map.sh +++ b/buildscripts/kokoro/xds_url_map.sh @@ -108,6 +108,7 @@ run_test() { --flagfile="${TEST_DRIVER_FLAGFILE}" \ --kube_context="${KUBE_CONTEXT}" \ --client_image="${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" \ + --testing_version="$(echo "$KOKORO_JOB_NAME" | sed -E 's|^grpc/java/([^/]+)/.*|\1|')" \ --xml_output_file="${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml" \ --flagfile="config/url-map.cfg" set +x From fbded2a05f283fdb185fabe4e0a37627407681a2 Mon Sep 17 00:00:00 2001 From: yifeizhuang Date: Tue, 28 Sep 2021 13:31:40 -0700 Subject: [PATCH 45/76] default throw ServerCallStreamObserver.setOnCloseHandler (#8564) --- stub/src/main/java/io/grpc/stub/ServerCallStreamObserver.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/stub/src/main/java/io/grpc/stub/ServerCallStreamObserver.java b/stub/src/main/java/io/grpc/stub/ServerCallStreamObserver.java index e31cae1fad4..8201a230546 100644 --- a/stub/src/main/java/io/grpc/stub/ServerCallStreamObserver.java +++ b/stub/src/main/java/io/grpc/stub/ServerCallStreamObserver.java @@ -168,5 +168,7 @@ public void disableAutoRequest() { * @param onCloseHandler to execute when the call has been closed cleanly. */ @ExperimentalApi("https://github.com/grpc/grpc-java/issues/8467") - public abstract void setOnCloseHandler(Runnable onCloseHandler); + public void setOnCloseHandler(Runnable onCloseHandler) { + throw new UnsupportedOperationException(); + } } From 9209c1eaf5fcceff49d13e96c168529ec73df197 Mon Sep 17 00:00:00 2001 From: Liam Miller-Cushon Date: Tue, 28 Sep 2021 14:18:53 -0700 Subject: [PATCH 46/76] Migrate off deprecated mockito method (#8562) See: https://javadoc.io/doc/org.mockito/mockito-core/latest/org/mockito/ArgumentMatchers.html#anyListOf-java.lang.Class- --- .../certprovider/CertificateProviderStoreTest.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/xds/src/test/java/io/grpc/xds/internal/certprovider/CertificateProviderStoreTest.java b/xds/src/test/java/io/grpc/xds/internal/certprovider/CertificateProviderStoreTest.java index 53144c2d48e..33ec6b291ed 100644 --- a/xds/src/test/java/io/grpc/xds/internal/certprovider/CertificateProviderStoreTest.java +++ b/xds/src/test/java/io/grpc/xds/internal/certprovider/CertificateProviderStoreTest.java @@ -20,7 +20,6 @@ import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyListOf; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -38,6 +37,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; +import org.mockito.ArgumentMatchers; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -158,7 +158,7 @@ public void onePluginSameConfig_sameInstance() { assertThat(distWatcher.downstreamWatchers).hasSize(1); testCertificateProvider.getWatcher().updateCertificate(testKey, testList); verify(mockWatcher1, never()) - .updateCertificate(any(PrivateKey.class), anyListOf(X509Certificate.class)); + .updateCertificate(any(PrivateKey.class), ArgumentMatchers.anyList()); verify(mockWatcher2, times(1)).updateCertificate(eq(testKey), eq(testList)); testCertificateProvider.getWatcher().updateTrustedRoots(testList); verify(mockWatcher2, times(1)).updateTrustedRoots(eq(testList)); @@ -199,8 +199,8 @@ public void onePluginSameConfig_secondWatcherAfterFirstNotify() { verify(mockWatcher2, never()).onError(eq(Status.CANCELLED)); // and none to first one verify(mockWatcher1, never()) - .updateCertificate(any(PrivateKey.class), anyListOf(X509Certificate.class)); - verify(mockWatcher1, never()).updateTrustedRoots(anyListOf(X509Certificate.class)); + .updateCertificate(any(PrivateKey.class), ArgumentMatchers.anyList()); + verify(mockWatcher1, never()).updateTrustedRoots(ArgumentMatchers.anyList()); verify(mockWatcher1, never()).onError(any(Status.class)); } @@ -302,7 +302,7 @@ private static void checkDifferentInstances( testCertificateProvider1.getWatcher().updateCertificate(testKey1, testList1); verify(mockWatcher1, times(1)).updateCertificate(eq(testKey1), eq(testList1)); verify(mockWatcher2, never()) - .updateCertificate(any(PrivateKey.class), anyListOf(X509Certificate.class)); + .updateCertificate(any(PrivateKey.class), ArgumentMatchers.anyList()); reset(mockWatcher1); PrivateKey testKey2 = mock(PrivateKey.class); @@ -311,7 +311,7 @@ private static void checkDifferentInstances( testCertificateProvider2.getWatcher().updateCertificate(testKey2, testList2); verify(mockWatcher2, times(1)).updateCertificate(eq(testKey2), eq(testList2)); verify(mockWatcher1, never()) - .updateCertificate(any(PrivateKey.class), anyListOf(X509Certificate.class)); + .updateCertificate(any(PrivateKey.class), ArgumentMatchers.anyList()); assertThat(testCertificateProvider1.startCalled).isEqualTo(1); assertThat(testCertificateProvider2.startCalled).isEqualTo(1); handle2.close(); From f57de6bd030bc45549744107ba64c8f53daabc96 Mon Sep 17 00:00:00 2001 From: markb74 <57717302+markb74@users.noreply.github.com> Date: Wed, 29 Sep 2021 11:28:14 +0200 Subject: [PATCH 47/76] Make binder instrumentation tests run on kokoro. (#8563) The tests run as part of the existing android-interop-testing job. We needed to modify the manifest of the apk built under android-interop-testing to declare Android Services used by the binder tests. --- .../src/main/AndroidManifest.xml | 3 +++ buildscripts/kokoro/android-interop.sh | 15 +++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/android-interop-testing/src/main/AndroidManifest.xml b/android-interop-testing/src/main/AndroidManifest.xml index c3f35131ad4..2e1e2c696d8 100644 --- a/android-interop-testing/src/main/AndroidManifest.xml +++ b/android-interop-testing/src/main/AndroidManifest.xml @@ -19,6 +19,9 @@ + + + diff --git a/buildscripts/kokoro/android-interop.sh b/buildscripts/kokoro/android-interop.sh index 5d9774bb12f..50523c8e88e 100755 --- a/buildscripts/kokoro/android-interop.sh +++ b/buildscripts/kokoro/android-interop.sh @@ -38,3 +38,18 @@ gcloud firebase test android run \ --device model=Nexus6P,version=23,locale=en,orientation=portrait \ --device model=Nexus6,version=22,locale=en,orientation=portrait \ --device model=Nexus6,version=21,locale=en,orientation=portrait + +# Build and run binderchannel instrumentation tests on Firebase Test Lab +cd ../binder +../gradlew assembleDebugAndroidTest +gcloud firebase test android run \ + --type instrumentation \ + --app ../android-interop-testing/build/outputs/apk/debug/grpc-android-interop-testing-debug.apk \ + --test build/outputs/apk/androidTest/debug/grpc-binder-debug-androidTest.apk \ + --device model=Nexus6P,version=27,locale=en,orientation=portrait \ + --device model=Nexus6P,version=26,locale=en,orientation=portrait \ + --device model=Nexus6P,version=25,locale=en,orientation=portrait \ + --device model=Nexus6P,version=24,locale=en,orientation=portrait \ + --device model=Nexus6P,version=23,locale=en,orientation=portrait \ + --device model=Nexus6,version=22,locale=en,orientation=portrait \ + --device model=Nexus6,version=21,locale=en,orientation=portrait From 28f2647aaf3aea8484354324787f48b523238994 Mon Sep 17 00:00:00 2001 From: ZHANG Dapeng Date: Wed, 29 Sep 2021 09:42:59 -0700 Subject: [PATCH 48/76] core: move closed check from Stream.isReady() to Call.isReady() (#8566) This fixes data race described in #8565. We are doubtful whether checking closed in isReady() is necessary (#3201 might be a requirement), but it was easier to just maintain the existing behavior than think heavily about it. --- .../java/io/grpc/internal/AbstractStream.java | 3 --- .../java/io/grpc/internal/ClientCallImpl.java | 3 +++ .../java/io/grpc/internal/ServerCallImpl.java | 3 +++ .../io/grpc/internal/ClientCallImplTest.java | 18 ++++++++++++++++++ .../io/grpc/internal/ServerCallImplTest.java | 2 ++ .../io/grpc/netty/NettyStreamTestBase.java | 7 ------- 6 files changed, 26 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/io/grpc/internal/AbstractStream.java b/core/src/main/java/io/grpc/internal/AbstractStream.java index e0660182493..69df1eee8e4 100644 --- a/core/src/main/java/io/grpc/internal/AbstractStream.java +++ b/core/src/main/java/io/grpc/internal/AbstractStream.java @@ -91,9 +91,6 @@ public final void setCompressor(Compressor compressor) { @Override public boolean isReady() { - if (framer().isClosed()) { - return false; - } return transportState().isReady(); } diff --git a/core/src/main/java/io/grpc/internal/ClientCallImpl.java b/core/src/main/java/io/grpc/internal/ClientCallImpl.java index 6f850ade667..db1a992b968 100644 --- a/core/src/main/java/io/grpc/internal/ClientCallImpl.java +++ b/core/src/main/java/io/grpc/internal/ClientCallImpl.java @@ -544,6 +544,9 @@ public void setMessageCompression(boolean enabled) { @Override public boolean isReady() { + if (halfCloseCalled) { + return false; + } return stream.isReady(); } diff --git a/core/src/main/java/io/grpc/internal/ServerCallImpl.java b/core/src/main/java/io/grpc/internal/ServerCallImpl.java index deba21c315d..b31aadd08a9 100644 --- a/core/src/main/java/io/grpc/internal/ServerCallImpl.java +++ b/core/src/main/java/io/grpc/internal/ServerCallImpl.java @@ -194,6 +194,9 @@ public void setCompression(String compressorName) { @Override public boolean isReady() { + if (closeCalled) { + return false; + } return stream.isReady(); } diff --git a/core/src/test/java/io/grpc/internal/ClientCallImplTest.java b/core/src/test/java/io/grpc/internal/ClientCallImplTest.java index ecf7a90b13e..e409f2f9df4 100644 --- a/core/src/test/java/io/grpc/internal/ClientCallImplTest.java +++ b/core/src/test/java/io/grpc/internal/ClientCallImplTest.java @@ -1022,6 +1022,24 @@ public void onMessage(Void message) { assertSame(cause, status.getCause()); } + @Test + public void halfClosedShouldNotBeReady() { + when(stream.isReady()).thenReturn(true); + ClientCallImpl call = new ClientCallImpl<>( + method, + MoreExecutors.directExecutor(), + baseCallOptions, + clientStreamProvider, + deadlineCancellationExecutor, + channelCallTracer, configSelector); + + call.start(callListener, new Metadata()); + assertThat(call.isReady()).isTrue(); + + call.halfClose(); + assertThat(call.isReady()).isFalse(); + } + @Test public void startAddsMaxSize() { CallOptions callOptions = diff --git a/core/src/test/java/io/grpc/internal/ServerCallImplTest.java b/core/src/test/java/io/grpc/internal/ServerCallImplTest.java index edf303a0bcd..9c25f474804 100644 --- a/core/src/test/java/io/grpc/internal/ServerCallImplTest.java +++ b/core/src/test/java/io/grpc/internal/ServerCallImplTest.java @@ -334,6 +334,8 @@ public void isReady() { when(stream.isReady()).thenReturn(true); assertTrue(call.isReady()); + call.close(Status.OK, new Metadata()); + assertFalse(call.isReady()); } @Test diff --git a/netty/src/test/java/io/grpc/netty/NettyStreamTestBase.java b/netty/src/test/java/io/grpc/netty/NettyStreamTestBase.java index 0f7d54e7942..dc245b3f505 100644 --- a/netty/src/test/java/io/grpc/netty/NettyStreamTestBase.java +++ b/netty/src/test/java/io/grpc/netty/NettyStreamTestBase.java @@ -138,13 +138,6 @@ public void shouldBeImmediatelyReadyForData() { assertTrue(stream.isReady()); } - @Test - public void closedShouldNotBeReady() throws IOException { - assertTrue(stream.isReady()); - closeStream(); - assertFalse(stream.isReady()); - } - @Test public void notifiedOnReadyAfterWriteCompletes() throws IOException { sendHeadersIfServer(); From fcc7b9694ee1fef20f12d21bdea193362fa1874b Mon Sep 17 00:00:00 2001 From: markb74 <57717302+markb74@users.noreply.github.com> Date: Wed, 29 Sep 2021 20:04:47 +0200 Subject: [PATCH 49/76] Add LifecycleOnDestroyHelper to support shutdown of channel/server on Android lifecycle changes (#8568) --- binder/build.gradle | 3 + .../grpc/binder/LifecycleOnDestroyHelper.java | 85 ++++++++++++++++ .../binder/LifecycleOnDestroyHelperTest.java | 96 +++++++++++++++++++ build.gradle | 1 + 4 files changed, 185 insertions(+) create mode 100644 binder/src/main/java/io/grpc/binder/LifecycleOnDestroyHelper.java create mode 100644 binder/src/test/java/io/grpc/binder/LifecycleOnDestroyHelperTest.java diff --git a/binder/build.gradle b/binder/build.gradle index 537c23a0092..81606d47263 100644 --- a/binder/build.gradle +++ b/binder/build.gradle @@ -49,9 +49,12 @@ dependencies { implementation libraries.androidx_annotation implementation libraries.androidx_core + implementation libraries.androidx_lifecycle_common implementation libraries.guava testImplementation libraries.androidx_core testImplementation libraries.androidx_test + testImplementation libraries.androidx_lifecycle_common + testImplementation libraries.androidx_lifecycle_service testImplementation libraries.junit testImplementation libraries.mockito testImplementation (libraries.robolectric) { diff --git a/binder/src/main/java/io/grpc/binder/LifecycleOnDestroyHelper.java b/binder/src/main/java/io/grpc/binder/LifecycleOnDestroyHelper.java new file mode 100644 index 00000000000..8631bac0df3 --- /dev/null +++ b/binder/src/main/java/io/grpc/binder/LifecycleOnDestroyHelper.java @@ -0,0 +1,85 @@ +/* + * Copyright 2021 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.binder; + +import androidx.annotation.MainThread; +import androidx.lifecycle.Lifecycle; +import androidx.lifecycle.Lifecycle.State; +import androidx.lifecycle.LifecycleEventObserver; +import androidx.lifecycle.LifecycleObserver; +import androidx.lifecycle.LifecycleOwner; +import io.grpc.ManagedChannel; +import io.grpc.Server; + +/** + * Helps work around certain quirks of {@link Lifecycle#addObserver} and {@link State#DESTROYED}. + * + *

In particular, calls to {@link Lifecycle#addObserver(LifecycleObserver)} are silently ignored + * if the owner is already destroyed. + */ +public final class LifecycleOnDestroyHelper { + + private LifecycleOnDestroyHelper() {} + + /** + * Arranges for {@link ManagedChannel#shutdownNow()} to be called on {@code channel} just before + * {@code lifecycle} is destroyed, or immediately if {@code lifecycle} is already destroyed. + * + *

Must only be called on the application's main thread. + */ + @MainThread + public static void shutdownUponDestruction(Lifecycle lifecycle, ManagedChannel channel) { + if (lifecycle.getCurrentState() == State.DESTROYED) { + channel.shutdownNow(); + } else { + lifecycle.addObserver( + new LifecycleEventObserver() { + @Override + public void onStateChanged(LifecycleOwner source, Lifecycle.Event event) { + if (event == Lifecycle.Event.ON_DESTROY) { + source.getLifecycle().removeObserver(this); + channel.shutdownNow(); + } + } + }); + } + } + + /** + * Arranges for {@link Server#shutdownNow()} to be called on {@code server} just before {@code + * lifecycle} is destroyed, or immediately if {@code lifecycle} is already destroyed. + * + *

Must only be called on the application's main thread. + */ + @MainThread + public static void shutdownUponDestruction(Lifecycle lifecycle, Server server) { + if (lifecycle.getCurrentState() == State.DESTROYED) { + server.shutdownNow(); + } else { + lifecycle.addObserver( + new LifecycleEventObserver() { + @Override + public void onStateChanged(LifecycleOwner source, Lifecycle.Event event) { + if (event == Lifecycle.Event.ON_DESTROY) { + source.getLifecycle().removeObserver(this); + server.shutdownNow(); + } + } + }); + } + } +} diff --git a/binder/src/test/java/io/grpc/binder/LifecycleOnDestroyHelperTest.java b/binder/src/test/java/io/grpc/binder/LifecycleOnDestroyHelperTest.java new file mode 100644 index 00000000000..48365204f73 --- /dev/null +++ b/binder/src/test/java/io/grpc/binder/LifecycleOnDestroyHelperTest.java @@ -0,0 +1,96 @@ +/* + * Copyright 2021 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.binder; + +import static android.os.Looper.getMainLooper; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.robolectric.Shadows.shadowOf; + +import androidx.lifecycle.LifecycleService; +import io.grpc.ManagedChannel; +import io.grpc.Server; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.robolectric.Robolectric; +import org.robolectric.RobolectricTestRunner; +import org.robolectric.android.controller.ServiceController; + +@RunWith(RobolectricTestRunner.class) +public final class LifecycleOnDestroyHelperTest { + + @Rule public final MockitoRule mocks = MockitoJUnit.rule(); + + private ServiceController sourceController; + private MyService sourceService; + + @Mock ManagedChannel mockChannel; + @Mock Server mockServer; + + @Before + public void setup() { + sourceController = Robolectric.buildService(MyService.class); + sourceService = sourceController.create().get(); + } + + @Test + public void shouldShutdownChannelUponSourceDestruction() { + LifecycleOnDestroyHelper.shutdownUponDestruction(sourceService.getLifecycle(), mockChannel); + shadowOf(getMainLooper()).idle(); + verifyNoInteractions(mockChannel); + + sourceController.destroy(); + shadowOf(getMainLooper()).idle(); + verify(mockChannel).shutdownNow(); + } + + @Test + public void shouldShutdownChannelForInitiallyDestroyedSource() { + sourceController.destroy(); + shadowOf(getMainLooper()).idle(); + + LifecycleOnDestroyHelper.shutdownUponDestruction(sourceService.getLifecycle(), mockChannel); + verify(mockChannel).shutdownNow(); + } + + @Test + public void shouldShutdownServerUponServiceDestruction() { + LifecycleOnDestroyHelper.shutdownUponDestruction(sourceService.getLifecycle(), mockServer); + shadowOf(getMainLooper()).idle(); + verifyNoInteractions(mockServer); + + sourceController.destroy(); + shadowOf(getMainLooper()).idle(); + verify(mockServer).shutdownNow(); + } + + @Test + public void shouldShutdownServerForInitiallyDestroyedSource() { + sourceController.destroy(); + shadowOf(getMainLooper()).idle(); + + LifecycleOnDestroyHelper.shutdownUponDestruction(sourceService.getLifecycle(), mockServer); + verify(mockServer).shutdownNow(); + } + + private static class MyService extends LifecycleService {} +} diff --git a/build.gradle b/build.gradle index 3c746a9dab7..df487de8fc4 100644 --- a/build.gradle +++ b/build.gradle @@ -191,6 +191,7 @@ subprojects { guava_testlib: "com.google.guava:guava-testlib:${guavaVersion}", androidx_annotation: "androidx.annotation:annotation:1.1.0", androidx_core: "androidx.core:core:1.3.0", + androidx_lifecycle_common: "androidx.lifecycle:lifecycle-common:2.3.0", androidx_lifecycle_service: "androidx.lifecycle:lifecycle-service:2.3.0", androidx_test: "androidx.test:core:1.3.0", androidx_test_rules: "androidx.test:rules:1.3.0", From dc4a41498ef35ed45c8aa01a66d23f8ab1b120c5 Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Fri, 24 Sep 2021 15:55:02 -0700 Subject: [PATCH 50/76] xds: Register RBAC with pretty-printer Ideally we should plumb this through Filter, but FilterRegistry will need to be plumbed to XdsClient and it started becoming non-trivial compared to the "just add two lines." Expediency is helpful as the XDS logs are pretty hard to read without the pretty-printing. --- xds/src/main/java/io/grpc/xds/MessagePrinter.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/xds/src/main/java/io/grpc/xds/MessagePrinter.java b/xds/src/main/java/io/grpc/xds/MessagePrinter.java index 8d7a9d43058..edddcb7a465 100644 --- a/xds/src/main/java/io/grpc/xds/MessagePrinter.java +++ b/xds/src/main/java/io/grpc/xds/MessagePrinter.java @@ -26,6 +26,8 @@ import io.envoyproxy.envoy.config.route.v3.RouteConfiguration; import io.envoyproxy.envoy.extensions.clusters.aggregate.v3.ClusterConfig; import io.envoyproxy.envoy.extensions.filters.http.fault.v3.HTTPFault; +import io.envoyproxy.envoy.extensions.filters.http.rbac.v3.RBAC; +import io.envoyproxy.envoy.extensions.filters.http.rbac.v3.RBACPerRoute; import io.envoyproxy.envoy.extensions.filters.http.router.v3.Router; import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext; @@ -48,6 +50,8 @@ final class MessagePrinter { .HttpConnectionManager.getDescriptor()) .add(HTTPFault.getDescriptor()) .add(io.envoyproxy.envoy.config.filter.http.fault.v2.HTTPFault.getDescriptor()) + .add(RBAC.getDescriptor()) + .add(RBACPerRoute.getDescriptor()) .add(Router.getDescriptor()) .add(io.envoyproxy.envoy.config.filter.http.router.v2.Router.getDescriptor()) // UpstreamTlsContext and DownstreamTlsContext in v3 are not transitively imported From 2b4a4747593355af27d070b89fcf5986d3dbfd21 Mon Sep 17 00:00:00 2001 From: Sumit Bhagwani Date: Wed, 29 Sep 2021 16:41:32 -0400 Subject: [PATCH 51/76] Fix javadoc (#8570) --- .../src/main/java/io/grpc/android/AndroidChannelBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/android/src/main/java/io/grpc/android/AndroidChannelBuilder.java b/android/src/main/java/io/grpc/android/AndroidChannelBuilder.java index e09a58ae428..8c69ca68b5f 100644 --- a/android/src/main/java/io/grpc/android/AndroidChannelBuilder.java +++ b/android/src/main/java/io/grpc/android/AndroidChannelBuilder.java @@ -150,7 +150,7 @@ public ManagedChannel build() { /** * Wraps an OkHttp channel and handles invoking the appropriate methods (e.g., {@link - * ManagedChannel#enterIdle) when the device network state changes. + * ManagedChannel#enterIdle}) when the device network state changes. */ @VisibleForTesting static final class AndroidChannel extends ManagedChannel { From 0e857376365d2080348f368bc8daac035f0a9035 Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Wed, 29 Sep 2021 13:04:47 -0700 Subject: [PATCH 52/76] .github/workflows: Swap from adoptopenjdk to temurin Adopt OpenJDK is longer seeing new releases, and instead has moved under the Eclipse umbrella with Temurin releases. https://blog.adoptopenjdk.net/2021/08/goodbye-adoptopenjdk-hello-adoptium/ https://github.com/actions/setup-java#supported-distributions The adopt binaries still work, but won't see new versions. --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 0bcced9889c..99177462ef8 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -25,7 +25,7 @@ jobs: - uses: actions/setup-java@v2 with: java-version: ${{ matrix.jre }} - distribution: 'adopt' + distribution: 'temurin' - name: Gradle cache uses: actions/cache@v2 From 979508ea442c9ec7563fea3fc88914c82245cb08 Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Mon, 4 Oct 2021 12:29:28 -0700 Subject: [PATCH 53/76] context: Remove misleading example that leaks CancellableContext The example should unconditionally cancel the context, but fails to. And it is really unclear what situation the example is demonstrating as the Runnable looping on isCancelled() is guaranteed to have returned when the Throwable catch is run. It is non-trivial to fix up this example such that it is concise, useful, and correct as it essentially needs a rewrite. We have other examples demonstrating CancellableContext usage. We can just rely on them instead. --- context/src/main/java/io/grpc/Context.java | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/context/src/main/java/io/grpc/Context.java b/context/src/main/java/io/grpc/Context.java index 142cd3cadbb..ad47ca9cb4d 100644 --- a/context/src/main/java/io/grpc/Context.java +++ b/context/src/main/java/io/grpc/Context.java @@ -57,26 +57,13 @@ * * *

Contexts are also used to represent a scoped unit of work. When the unit of work is done the - * context can be cancelled. This cancellation will also cascade to all descendant contexts. You can + * context must be cancelled. This cancellation will cascade to all descendant contexts. You can * add a {@link CancellationListener} to a context to be notified when it or one of its ancestors * has been cancelled. Cancellation does not release the state stored by a context and it's * perfectly valid to {@link #attach()} an already cancelled context to make it current. To cancel a * context (and its descendants) you first create a {@link CancellableContext} and when you need to * signal cancellation call {@link CancellableContext#cancel} or {@link - * CancellableContext#detachAndCancel}. For example: - * 

- *   CancellableContext withCancellation = Context.current().withCancellation();
- *   try {
- *     withCancellation.run(new Runnable() {
- *       public void run() {
- *         while (waitingForData() && !Context.current().isCancelled()) {}
- *       }
- *     });
- *     doSomeWork();
- *   } catch (Throwable t) {
- *      withCancellation.cancel(t);
- *   }
- *
+ * CancellableContext#detachAndCancel}. * *

Contexts can also be created with a timeout relative to the system nano clock which will * cause it to automatically cancel at the desired time. From fc57cad4ec7774eefd9ae4990f57805da84f611f Mon Sep 17 00:00:00 2001 From: Terry Wilson Date: Tue, 5 Oct 2021 10:34:44 -0700 Subject: [PATCH 54/76] Revert "Revert "core/auth: Remove CallCredentials2 (#8464)"" (#8572) This reverts commit a91cc85dfd2f1f8cb749f6e8b5984a61da294902. --- .../main/java/io/grpc/CallCredentials2.java | 73 ---- .../GoogleAuthLibraryCallCredentials.java | 6 +- .../CallCredentials2ApplyingTest.java | 351 ------------------ .../internal/CallCredentialsApplyingTest.java | 45 +++ 4 files changed, 47 insertions(+), 428 deletions(-) delete mode 100644 api/src/main/java/io/grpc/CallCredentials2.java delete mode 100644 core/src/test/java/io/grpc/internal/CallCredentials2ApplyingTest.java diff --git a/api/src/main/java/io/grpc/CallCredentials2.java b/api/src/main/java/io/grpc/CallCredentials2.java deleted file mode 100644 index fdb7f51070a..00000000000 --- a/api/src/main/java/io/grpc/CallCredentials2.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright 2016 The gRPC Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.grpc; - -import java.util.concurrent.Executor; - -/** - * The new interface for {@link CallCredentials}. - * - *

THIS CLASS NAME IS TEMPORARY and is part of a migration. This class will BE DELETED as it - * replaces {@link CallCredentials} in short-term. THIS CLASS IS ONLY REFERENCED BY IMPLEMENTIONS. - * All consumers should be always referencing {@link CallCredentials}. - * - * @deprecated the new interface has been promoted into {@link CallCredentials}. Implementations - * should switch back to "{@code extends CallCredentials}". - */ -@Deprecated -@ExperimentalApi("https://github.com/grpc/grpc-java/issues/4901") -public abstract class CallCredentials2 extends CallCredentials { - /** - * Pass the credential data to the given {@link MetadataApplier}, which will propagate it to the - * request metadata. - * - *

It is called for each individual RPC, within the {@link Context} of the call, before the - * stream is about to be created on a transport. Implementations should not block in this - * method. If metadata is not immediately available, e.g., needs to be fetched from network, the - * implementation may give the {@code applier} to an asynchronous task which will eventually call - * the {@code applier}. The RPC proceeds only after the {@code applier} is called. - * - * @param requestInfo request-related information - * @param appExecutor The application thread-pool. It is provided to the implementation in case it - * needs to perform blocking operations. - * @param applier The outlet of the produced headers. It can be called either before or after this - * method returns. - */ - @ExperimentalApi("https://github.com/grpc/grpc-java/issues/1914") - public abstract void applyRequestMetadata( - RequestInfo requestInfo, Executor appExecutor, MetadataApplier applier); - - @Override - public final void applyRequestMetadata( - RequestInfo requestInfo, Executor appExecutor, - final CallCredentials.MetadataApplier applier) { - applyRequestMetadata(requestInfo, appExecutor, new MetadataApplier() { - @Override - public void apply(Metadata headers) { - applier.apply(headers); - } - - @Override - public void fail(Status status) { - applier.fail(status); - } - }); - } - - @ExperimentalApi("https://github.com/grpc/grpc-java/issues/1914") - public abstract static class MetadataApplier extends CallCredentials.MetadataApplier {} -} diff --git a/auth/src/main/java/io/grpc/auth/GoogleAuthLibraryCallCredentials.java b/auth/src/main/java/io/grpc/auth/GoogleAuthLibraryCallCredentials.java index 852fba73b20..4b95a6c7f4d 100644 --- a/auth/src/main/java/io/grpc/auth/GoogleAuthLibraryCallCredentials.java +++ b/auth/src/main/java/io/grpc/auth/GoogleAuthLibraryCallCredentials.java @@ -42,11 +42,9 @@ import javax.annotation.Nullable; /** - * Wraps {@link Credentials} as a {@link CallCredentials}. + * Wraps {@link Credentials} as a {@link io.grpc.CallCredentials}. */ -// TODO(zhangkun83): remove the suppression after we change the base class to CallCredential -@SuppressWarnings("deprecation") -final class GoogleAuthLibraryCallCredentials extends io.grpc.CallCredentials2 { +final class GoogleAuthLibraryCallCredentials extends io.grpc.CallCredentials { private static final Logger log = Logger.getLogger(GoogleAuthLibraryCallCredentials.class.getName()); private static final JwtHelper jwtHelper diff --git a/core/src/test/java/io/grpc/internal/CallCredentials2ApplyingTest.java b/core/src/test/java/io/grpc/internal/CallCredentials2ApplyingTest.java deleted file mode 100644 index 963a586319b..00000000000 --- a/core/src/test/java/io/grpc/internal/CallCredentials2ApplyingTest.java +++ /dev/null @@ -1,351 +0,0 @@ -/* - * Copyright 2016 The gRPC Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.grpc.internal; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.same; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import io.grpc.Attributes; -import io.grpc.CallCredentials.MetadataApplier; -import io.grpc.CallCredentials.RequestInfo; -import io.grpc.CallOptions; -import io.grpc.ChannelLogger; -import io.grpc.ClientStreamTracer; -import io.grpc.IntegerMarshaller; -import io.grpc.Metadata; -import io.grpc.MethodDescriptor; -import io.grpc.SecurityLevel; -import io.grpc.Status; -import io.grpc.StringMarshaller; -import java.net.SocketAddress; -import java.util.concurrent.Executor; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; -import org.mockito.ArgumentCaptor; -import org.mockito.ArgumentMatchers; -import org.mockito.Mock; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.junit.MockitoJUnit; -import org.mockito.junit.MockitoRule; -import org.mockito.stubbing.Answer; - -/** - * Unit test for {@link CallCredentials2} applying functionality implemented by {@link - * CallCredentialsApplyingTransportFactory} and {@link MetadataApplierImpl}. - */ -@SuppressWarnings("deprecation") -@RunWith(JUnit4.class) -public class CallCredentials2ApplyingTest { - @Rule - public final MockitoRule mocks = MockitoJUnit.rule(); - - @Mock - private ClientTransportFactory mockTransportFactory; - - @Mock - private ConnectionClientTransport mockTransport; - - @Mock - private ClientStream mockStream; - - @Mock - private io.grpc.CallCredentials2 mockCreds; - - @Mock - private Executor mockExecutor; - - @Mock - private SocketAddress address; - - // Noop logger; - @Mock - private ChannelLogger channelLogger; - - private static final String AUTHORITY = "testauthority"; - private static final String USER_AGENT = "testuseragent"; - private static final Attributes.Key ATTR_KEY = Attributes.Key.create("somekey"); - private static final String ATTR_VALUE = "somevalue"; - private static final MethodDescriptor method = - MethodDescriptor.newBuilder() - .setType(MethodDescriptor.MethodType.UNKNOWN) - .setFullMethodName("service/method") - .setRequestMarshaller(new StringMarshaller()) - .setResponseMarshaller(new IntegerMarshaller()) - .build(); - private static final Metadata.Key ORIG_HEADER_KEY = - Metadata.Key.of("header1", Metadata.ASCII_STRING_MARSHALLER); - private static final String ORIG_HEADER_VALUE = "some original header value"; - private static final Metadata.Key CREDS_KEY = - Metadata.Key.of("test-creds", Metadata.ASCII_STRING_MARSHALLER); - private static final String CREDS_VALUE = "some credentials"; - private static final ClientStreamTracer[] tracers = new ClientStreamTracer[] { - new ClientStreamTracer() {} - }; - - private final Metadata origHeaders = new Metadata(); - private ForwardingConnectionClientTransport transport; - private CallOptions callOptions; - - @Before - public void setUp() { - ClientTransportFactory.ClientTransportOptions clientTransportOptions = - new ClientTransportFactory.ClientTransportOptions() - .setAuthority(AUTHORITY) - .setUserAgent(USER_AGENT); - - origHeaders.put(ORIG_HEADER_KEY, ORIG_HEADER_VALUE); - when(mockTransportFactory.newClientTransport(address, clientTransportOptions, channelLogger)) - .thenReturn(mockTransport); - when(mockTransport.newStream( - same(method), any(Metadata.class), any(CallOptions.class), - ArgumentMatchers.any())) - .thenReturn(mockStream); - ClientTransportFactory transportFactory = new CallCredentialsApplyingTransportFactory( - mockTransportFactory, null, mockExecutor); - transport = (ForwardingConnectionClientTransport) - transportFactory.newClientTransport(address, clientTransportOptions, channelLogger); - callOptions = CallOptions.DEFAULT.withCallCredentials(mockCreds); - verify(mockTransportFactory).newClientTransport(address, clientTransportOptions, channelLogger); - assertSame(mockTransport, transport.delegate()); - } - - @Test - public void parameterPropagation_base() { - Attributes transportAttrs = Attributes.newBuilder().set(ATTR_KEY, ATTR_VALUE).build(); - when(mockTransport.getAttributes()).thenReturn(transportAttrs); - - transport.newStream(method, origHeaders, callOptions, tracers); - - ArgumentCaptor infoCaptor = ArgumentCaptor.forClass(null); - verify(mockCreds).applyRequestMetadata( - infoCaptor.capture(), same(mockExecutor), - any(io.grpc.CallCredentials2.MetadataApplier.class)); - RequestInfo info = infoCaptor.getValue(); - assertSame(method, info.getMethodDescriptor()); - assertSame(ATTR_VALUE, info.getTransportAttrs().get(ATTR_KEY)); - assertSame(AUTHORITY, info.getAuthority()); - assertSame(SecurityLevel.NONE, info.getSecurityLevel()); - } - - @Test - public void parameterPropagation_transportSetSecurityLevel() { - Attributes transportAttrs = Attributes.newBuilder() - .set(ATTR_KEY, ATTR_VALUE) - .set(GrpcAttributes.ATTR_SECURITY_LEVEL, SecurityLevel.INTEGRITY) - .build(); - when(mockTransport.getAttributes()).thenReturn(transportAttrs); - - transport.newStream(method, origHeaders, callOptions, tracers); - - ArgumentCaptor infoCaptor = ArgumentCaptor.forClass(null); - verify(mockCreds).applyRequestMetadata( - infoCaptor.capture(), same(mockExecutor), - any(io.grpc.CallCredentials2.MetadataApplier.class)); - RequestInfo info = infoCaptor.getValue(); - assertSame(method, info.getMethodDescriptor()); - assertSame(ATTR_VALUE, info.getTransportAttrs().get(ATTR_KEY)); - assertSame(AUTHORITY, info.getAuthority()); - assertSame(SecurityLevel.INTEGRITY, info.getSecurityLevel()); - } - - @Test - public void parameterPropagation_callOptionsSetAuthority() { - Attributes transportAttrs = Attributes.newBuilder() - .set(ATTR_KEY, ATTR_VALUE) - .build(); - when(mockTransport.getAttributes()).thenReturn(transportAttrs); - Executor anotherExecutor = mock(Executor.class); - - transport.newStream( - method, origHeaders, - callOptions.withAuthority("calloptions-authority").withExecutor(anotherExecutor), - tracers); - - ArgumentCaptor infoCaptor = ArgumentCaptor.forClass(null); - verify(mockCreds).applyRequestMetadata( - infoCaptor.capture(), same(anotherExecutor), - any(io.grpc.CallCredentials2.MetadataApplier.class)); - RequestInfo info = infoCaptor.getValue(); - assertSame(method, info.getMethodDescriptor()); - assertSame(ATTR_VALUE, info.getTransportAttrs().get(ATTR_KEY)); - assertEquals("calloptions-authority", info.getAuthority()); - assertSame(SecurityLevel.NONE, info.getSecurityLevel()); - } - - @Test - public void credentialThrows() { - final RuntimeException ex = new RuntimeException(); - when(mockTransport.getAttributes()).thenReturn(Attributes.EMPTY); - doThrow(ex).when(mockCreds).applyRequestMetadata( - any(RequestInfo.class), same(mockExecutor), - any(io.grpc.CallCredentials2.MetadataApplier.class)); - - FailingClientStream stream = - (FailingClientStream) transport.newStream(method, origHeaders, callOptions, tracers); - - verify(mockTransport, never()).newStream( - any(MethodDescriptor.class), any(Metadata.class), any(CallOptions.class), - ArgumentMatchers.any()); - assertEquals(Status.Code.UNAUTHENTICATED, stream.getError().getCode()); - assertSame(ex, stream.getError().getCause()); - transport.shutdown(Status.UNAVAILABLE); - assertTrue(transport.newStream(method, origHeaders, callOptions, tracers) - instanceof FailingClientStream); - verify(mockTransport).shutdown(Status.UNAVAILABLE); - } - - @Test - public void applyMetadata_inline() { - when(mockTransport.getAttributes()).thenReturn(Attributes.EMPTY); - doAnswer(new Answer() { - @Override - public Void answer(InvocationOnMock invocation) throws Throwable { - MetadataApplier applier = (MetadataApplier) invocation.getArguments()[2]; - Metadata headers = new Metadata(); - headers.put(CREDS_KEY, CREDS_VALUE); - applier.apply(headers); - return null; - } - }).when(mockCreds).applyRequestMetadata( - any(RequestInfo.class), same(mockExecutor), - any(io.grpc.CallCredentials2.MetadataApplier.class)); - - ClientStream stream = transport.newStream(method, origHeaders, callOptions, tracers); - - verify(mockTransport).newStream(method, origHeaders, callOptions, tracers); - assertSame(mockStream, stream); - assertEquals(CREDS_VALUE, origHeaders.get(CREDS_KEY)); - assertEquals(ORIG_HEADER_VALUE, origHeaders.get(ORIG_HEADER_KEY)); - transport.shutdown(Status.UNAVAILABLE); - assertTrue(transport.newStream(method, origHeaders, callOptions, tracers) - instanceof FailingClientStream); - verify(mockTransport).shutdown(Status.UNAVAILABLE); - } - - @Test - public void fail_inline() { - final Status error = Status.FAILED_PRECONDITION.withDescription("channel not secure for creds"); - when(mockTransport.getAttributes()).thenReturn(Attributes.EMPTY); - doAnswer(new Answer() { - @Override - public Void answer(InvocationOnMock invocation) throws Throwable { - MetadataApplier applier = (MetadataApplier) invocation.getArguments()[2]; - applier.fail(error); - return null; - } - }).when(mockCreds).applyRequestMetadata( - any(RequestInfo.class), same(mockExecutor), - any(io.grpc.CallCredentials2.MetadataApplier.class)); - - FailingClientStream stream = - (FailingClientStream) transport.newStream(method, origHeaders, callOptions, tracers); - - verify(mockTransport, never()).newStream( - any(MethodDescriptor.class), any(Metadata.class), any(CallOptions.class), - ArgumentMatchers.any()); - assertSame(error, stream.getError()); - transport.shutdownNow(Status.UNAVAILABLE); - assertTrue(transport.newStream(method, origHeaders, callOptions, tracers) - instanceof FailingClientStream); - verify(mockTransport).shutdownNow(Status.UNAVAILABLE); - } - - @Test - public void applyMetadata_delayed() { - when(mockTransport.getAttributes()).thenReturn(Attributes.EMPTY); - - // Will call applyRequestMetadata(), which is no-op. - DelayedStream stream = (DelayedStream) transport.newStream( - method, origHeaders, callOptions, tracers); - - ArgumentCaptor applierCaptor = ArgumentCaptor.forClass(null); - verify(mockCreds).applyRequestMetadata( - any(RequestInfo.class), same(mockExecutor), applierCaptor.capture()); - verify(mockTransport, never()).newStream( - any(MethodDescriptor.class), any(Metadata.class), any(CallOptions.class), - ArgumentMatchers.any()); - - transport.shutdown(Status.UNAVAILABLE); - verify(mockTransport, never()).shutdown(Status.UNAVAILABLE); - - Metadata headers = new Metadata(); - headers.put(CREDS_KEY, CREDS_VALUE); - applierCaptor.getValue().apply(headers); - - verify(mockTransport).newStream(method, origHeaders, callOptions, tracers); - assertSame(mockStream, stream.getRealStream()); - assertEquals(CREDS_VALUE, origHeaders.get(CREDS_KEY)); - assertEquals(ORIG_HEADER_VALUE, origHeaders.get(ORIG_HEADER_KEY)); - assertTrue(transport.newStream(method, origHeaders, callOptions, tracers) - instanceof FailingClientStream); - verify(mockTransport).shutdown(Status.UNAVAILABLE); - } - - @Test - public void fail_delayed() { - when(mockTransport.getAttributes()).thenReturn(Attributes.EMPTY); - - // Will call applyRequestMetadata(), which is no-op. - DelayedStream stream = (DelayedStream) transport.newStream( - method, origHeaders, callOptions, tracers); - - ArgumentCaptor applierCaptor = ArgumentCaptor.forClass(null); - verify(mockCreds).applyRequestMetadata( - any(RequestInfo.class), same(mockExecutor), applierCaptor.capture()); - - Status error = Status.FAILED_PRECONDITION.withDescription("channel not secure for creds"); - applierCaptor.getValue().fail(error); - - verify(mockTransport, never()).newStream( - any(MethodDescriptor.class), any(Metadata.class), any(CallOptions.class), - ArgumentMatchers.any()); - FailingClientStream failingStream = (FailingClientStream) stream.getRealStream(); - assertSame(error, failingStream.getError()); - transport.shutdown(Status.UNAVAILABLE); - assertTrue(transport.newStream(method, origHeaders, callOptions, tracers) - instanceof FailingClientStream); - verify(mockTransport).shutdown(Status.UNAVAILABLE); - } - - @Test - public void noCreds() { - callOptions = callOptions.withCallCredentials(null); - ClientStream stream = transport.newStream(method, origHeaders, callOptions, tracers); - - verify(mockTransport).newStream(method, origHeaders, callOptions, tracers); - assertSame(mockStream, stream); - assertNull(origHeaders.get(CREDS_KEY)); - assertEquals(ORIG_HEADER_VALUE, origHeaders.get(ORIG_HEADER_KEY)); - transport.shutdown(Status.UNAVAILABLE); - assertTrue(transport.newStream(method, origHeaders, callOptions, tracers) - instanceof FailingClientStream); - verify(mockTransport).shutdown(Status.UNAVAILABLE); - } -} diff --git a/core/src/test/java/io/grpc/internal/CallCredentialsApplyingTest.java b/core/src/test/java/io/grpc/internal/CallCredentialsApplyingTest.java index ef49e66bf2d..2f0ce1070b1 100644 --- a/core/src/test/java/io/grpc/internal/CallCredentialsApplyingTest.java +++ b/core/src/test/java/io/grpc/internal/CallCredentialsApplyingTest.java @@ -176,6 +176,51 @@ public void parameterPropagation_overrideByCallOptions() { assertSame(SecurityLevel.INTEGRITY, info.getSecurityLevel()); } + @Test + public void parameterPropagation_transportSetSecurityLevel() { + Attributes transportAttrs = Attributes.newBuilder() + .set(ATTR_KEY, ATTR_VALUE) + .set(GrpcAttributes.ATTR_SECURITY_LEVEL, SecurityLevel.INTEGRITY) + .build(); + when(mockTransport.getAttributes()).thenReturn(transportAttrs); + + transport.newStream(method, origHeaders, callOptions, tracers); + + ArgumentCaptor infoCaptor = ArgumentCaptor.forClass(null); + verify(mockCreds).applyRequestMetadata( + infoCaptor.capture(), same(mockExecutor), + any(io.grpc.CallCredentials.MetadataApplier.class)); + RequestInfo info = infoCaptor.getValue(); + assertSame(method, info.getMethodDescriptor()); + assertSame(ATTR_VALUE, info.getTransportAttrs().get(ATTR_KEY)); + assertSame(AUTHORITY, info.getAuthority()); + assertSame(SecurityLevel.INTEGRITY, info.getSecurityLevel()); + } + + @Test + public void parameterPropagation_callOptionsSetAuthority() { + Attributes transportAttrs = Attributes.newBuilder() + .set(ATTR_KEY, ATTR_VALUE) + .build(); + when(mockTransport.getAttributes()).thenReturn(transportAttrs); + Executor anotherExecutor = mock(Executor.class); + + transport.newStream( + method, origHeaders, + callOptions.withAuthority("calloptions-authority").withExecutor(anotherExecutor), + tracers); + + ArgumentCaptor infoCaptor = ArgumentCaptor.forClass(null); + verify(mockCreds).applyRequestMetadata( + infoCaptor.capture(), same(anotherExecutor), + any(io.grpc.CallCredentials.MetadataApplier.class)); + RequestInfo info = infoCaptor.getValue(); + assertSame(method, info.getMethodDescriptor()); + assertSame(ATTR_VALUE, info.getTransportAttrs().get(ATTR_KEY)); + assertEquals("calloptions-authority", info.getAuthority()); + assertSame(SecurityLevel.NONE, info.getSecurityLevel()); + } + @Test public void credentialThrows() { final RuntimeException ex = new RuntimeException(); From 2e84b0f20ae68d75007e5a8ea7293c2360793c29 Mon Sep 17 00:00:00 2001 From: ZHANG Dapeng Date: Wed, 6 Oct 2021 10:02:32 -0700 Subject: [PATCH 55/76] android: bump min Android SDK version to 19 (#8583) As Google Play Service [discontinued updates for Jelly Bean (API levels 16, 17 & 18)](https://android-developers.googleblog.com/2021/07/google-play-services-discontinuing-jelly-bean.html). --- android-interop-testing/build.gradle | 2 +- android/build.gradle | 2 +- binder/build.gradle | 2 +- cronet/build.gradle | 2 +- examples/android/helloworld/app/build.gradle | 2 +- examples/android/routeguide/app/build.gradle | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/android-interop-testing/build.gradle b/android-interop-testing/build.gradle index bbf1fcfe99e..8f1c0849e9e 100644 --- a/android-interop-testing/build.gradle +++ b/android-interop-testing/build.gradle @@ -33,7 +33,7 @@ android { defaultConfig { applicationId "io.grpc.android.integrationtest" - minSdkVersion 16 + minSdkVersion 19 targetSdkVersion 26 versionCode 1 versionName "1.0" diff --git a/android/build.gradle b/android/build.gradle index a50f7b85592..b0916f9e04c 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -15,7 +15,7 @@ android { compileSdkVersion 29 defaultConfig { consumerProguardFiles "proguard-rules.txt" - minSdkVersion 16 + minSdkVersion 19 targetSdkVersion 29 versionCode 1 versionName "1.0" diff --git a/binder/build.gradle b/binder/build.gradle index 81606d47263..c0e0b66fd61 100644 --- a/binder/build.gradle +++ b/binder/build.gradle @@ -29,7 +29,7 @@ android { targetCompatibility 1.8 } defaultConfig { - minSdkVersion 16 + minSdkVersion 19 targetSdkVersion 29 versionCode 1 versionName "1.0" diff --git a/cronet/build.gradle b/cronet/build.gradle index 7a6d58a7e4b..e0a0b7b53a7 100644 --- a/cronet/build.gradle +++ b/cronet/build.gradle @@ -15,7 +15,7 @@ repositories { android { compileSdkVersion 29 defaultConfig { - minSdkVersion 16 + minSdkVersion 19 targetSdkVersion 29 versionCode 1 versionName "1.0" diff --git a/examples/android/helloworld/app/build.gradle b/examples/android/helloworld/app/build.gradle index f93118f8c66..29b7bc61c0f 100644 --- a/examples/android/helloworld/app/build.gradle +++ b/examples/android/helloworld/app/build.gradle @@ -10,7 +10,7 @@ android { defaultConfig { applicationId "io.grpc.helloworldexample" - minSdkVersion 16 + minSdkVersion 19 targetSdkVersion 27 versionCode 1 versionName "1.0" diff --git a/examples/android/routeguide/app/build.gradle b/examples/android/routeguide/app/build.gradle index ef9f5593895..665730306a9 100644 --- a/examples/android/routeguide/app/build.gradle +++ b/examples/android/routeguide/app/build.gradle @@ -10,7 +10,7 @@ android { defaultConfig { applicationId "io.grpc.routeguideexample" - minSdkVersion 16 + minSdkVersion 19 targetSdkVersion 27 versionCode 1 versionName "1.0" From e939bf6fb89a52e2afc7f830a787f73e58af3807 Mon Sep 17 00:00:00 2001 From: yifeizhuang Date: Wed, 6 Oct 2021 11:02:42 -0700 Subject: [PATCH 56/76] rbac: fix status code PERMISSION_DENIED (#8578) RBAC should fail with PERMISSION_DENIED, fix https://github.com/grpc/grpc-java/issues/8576 --- xds/src/main/java/io/grpc/xds/RbacFilter.java | 7 +++---- xds/src/test/java/io/grpc/xds/RbacFilterTest.java | 3 ++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/RbacFilter.java b/xds/src/main/java/io/grpc/xds/RbacFilter.java index 387afac82cc..0f5ac1025ba 100644 --- a/xds/src/main/java/io/grpc/xds/RbacFilter.java +++ b/xds/src/main/java/io/grpc/xds/RbacFilter.java @@ -177,14 +177,13 @@ public ServerCall.Listener interceptCall( final ServerCall call, final Metadata headers, ServerCallHandler next) { AuthDecision authResult = authEngine.evaluate(headers, call); - if (logger.isLoggable(Level.FINER)) { - logger.log(Level.FINER, + if (logger.isLoggable(Level.FINE)) { + logger.log(Level.FINE, "Authorization result for serverCall {0}: {1}, matching policy: {2}.", new Object[]{call, authResult.decision(), authResult.matchingPolicyName()}); } if (GrpcAuthorizationEngine.Action.DENY.equals(authResult.decision())) { - Status status = Status.UNAUTHENTICATED.withDescription( - "Access Denied, matching policy: " + authResult.matchingPolicyName()); + Status status = Status.PERMISSION_DENIED.withDescription("Access Denied"); call.close(status, new Metadata()); return new ServerCall.Listener(){}; } diff --git a/xds/src/test/java/io/grpc/xds/RbacFilterTest.java b/xds/src/test/java/io/grpc/xds/RbacFilterTest.java index c97ca5c52d9..da42ec1a2f6 100644 --- a/xds/src/test/java/io/grpc/xds/RbacFilterTest.java +++ b/xds/src/test/java/io/grpc/xds/RbacFilterTest.java @@ -256,7 +256,8 @@ public void testAuthorizationInterceptor() { verify(mockHandler, never()).startCall(eq(mockServerCall), any(Metadata.class)); ArgumentCaptor captor = ArgumentCaptor.forClass(Status.class); verify(mockServerCall).close(captor.capture(), any(Metadata.class)); - assertThat(captor.getValue().getCode()).isEqualTo(Status.UNAUTHENTICATED.getCode()); + assertThat(captor.getValue().getCode()).isEqualTo(Status.PERMISSION_DENIED.getCode()); + assertThat(captor.getValue().getDescription()).isEqualTo("Access Denied"); verify(mockServerCall).getAttributes(); verifyNoMoreInteractions(mockServerCall); From 8ac9a4e7bd651f1492d4376f4fc78643613bf8aa Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Mon, 4 Oct 2021 16:25:46 -0700 Subject: [PATCH 57/76] [xDS interop] add Docker tagging logic to the xds_url_map job --- buildscripts/kokoro/xds_url_map.sh | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/buildscripts/kokoro/xds_url_map.sh b/buildscripts/kokoro/xds_url_map.sh index 87268ea5ea2..99d54da99c5 100755 --- a/buildscripts/kokoro/xds_url_map.sh +++ b/buildscripts/kokoro/xds_url_map.sh @@ -7,6 +7,7 @@ readonly GITHUB_REPOSITORY_NAME="grpc-java" readonly GKE_CLUSTER_NAME="interop-test-psm-basic" readonly GKE_CLUSTER_ZONE="us-central1-c" ## xDS test client Docker images +readonly SERVER_IMAGE_NAME="gcr.io/grpc-testing/xds-interop/java-server" readonly CLIENT_IMAGE_NAME="gcr.io/grpc-testing/xds-interop/java-client" readonly FORCE_IMAGE_BUILD="${FORCE_IMAGE_BUILD:-0}" readonly BUILD_APP_PATH="interop-testing/build/install/grpc-interop-testing" @@ -19,7 +20,7 @@ readonly BUILD_APP_PATH="interop-testing/build/install/grpc-interop-testing" # Arguments: # None # Outputs: -# Writes the output of xds-test-client --help to stderr +# Writes the output of xds-test-client and xds-test-server --help to stderr ####################################### build_java_test_app() { echo "Building Java test app" @@ -29,12 +30,14 @@ build_java_test_app() { # Test-run binaries run_ignore_exit_code "${SRC_DIR}/${BUILD_APP_PATH}/bin/xds-test-client" --help + run_ignore_exit_code "${SRC_DIR}/${BUILD_APP_PATH}/bin/xds-test-server" --help } ####################################### # Builds test app Docker images and pushes them to GCR # Globals: # BUILD_APP_PATH +# SERVER_IMAGE_NAME: Test server Docker image name # CLIENT_IMAGE_NAME: Test client Docker image name # GIT_COMMIT: SHA-1 of git commit being built # Arguments: @@ -51,10 +54,16 @@ build_test_app_docker_images() { cp -v "${docker_dir}/"*.Dockerfile "${build_dir}" cp -v "${docker_dir}/"*.properties "${build_dir}" cp -rv "${SRC_DIR}/${BUILD_APP_PATH}" "${build_dir}" + # Pick a branch name for the built image + if [[ -n $KOKORO_JOB_NAME ]]; then + branch_name="$(echo "$KOKORO_JOB_NAME" | sed -E 's|^grpc/java/([^/]+)/.*|\1|')" + else + branch_name='experimental' + fi # Run Google Cloud Build gcloud builds submit "${build_dir}" \ --config "${docker_dir}/cloudbuild.yaml" \ - --substitutions "_CLIENT_IMAGE_NAME=${CLIENT_IMAGE_NAME},COMMIT_SHA=${GIT_COMMIT},BRANCH_NAME=experimental" + --substitutions "_SERVER_IMAGE_NAME=${SERVER_IMAGE_NAME},_CLIENT_IMAGE_NAME=${CLIENT_IMAGE_NAME},COMMIT_SHA=${GIT_COMMIT},BRANCH_NAME=${branch_name}" # TODO(sergiitk): extra "cosmetic" tags for versioned branches, e.g. v1.34.x # TODO(sergiitk): do this when adding support for custom configs per version } @@ -62,6 +71,7 @@ build_test_app_docker_images() { ####################################### # Builds test app and its docker images unless they already exist # Globals: +# SERVER_IMAGE_NAME: Test server Docker image name # CLIENT_IMAGE_NAME: Test client Docker image name # GIT_COMMIT: SHA-1 of git commit being built # FORCE_IMAGE_BUILD @@ -72,12 +82,16 @@ build_test_app_docker_images() { ####################################### build_docker_images_if_needed() { # Check if images already exist + server_tags="$(gcloud_gcr_list_image_tags "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}")" + printf "Server image: %s:%s\n" "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}" + echo "${server_tags:-Server image not found}" + client_tags="$(gcloud_gcr_list_image_tags "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}")" printf "Client image: %s:%s\n" "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}" echo "${client_tags:-Client image not found}" # Build if any of the images are missing, or FORCE_IMAGE_BUILD=1 - if [[ "${FORCE_IMAGE_BUILD}" == "1" || -z "${client_tags}" ]]; then + if [[ "${FORCE_IMAGE_BUILD}" == "1" || -z "${server_tags}" || -z "${client_tags}" ]]; then build_java_test_app build_test_app_docker_images else From 83d36104e1f4f928d36d5b9ad3eb2b4700c0ffce Mon Sep 17 00:00:00 2001 From: Terry Wilson Date: Thu, 7 Oct 2021 13:05:03 -0700 Subject: [PATCH 58/76] Source k8s test driver install script from core repo (#8573) The test driver install script is read directly from the core repo master branch and the copy in the Java repo is deleted. --- .../kokoro/xds-k8s-install-test-driver.sh | 347 ------------------ buildscripts/kokoro/xds-k8s.sh | 13 +- buildscripts/kokoro/xds_url_map.sh | 13 +- 3 files changed, 16 insertions(+), 357 deletions(-) delete mode 100755 buildscripts/kokoro/xds-k8s-install-test-driver.sh diff --git a/buildscripts/kokoro/xds-k8s-install-test-driver.sh b/buildscripts/kokoro/xds-k8s-install-test-driver.sh deleted file mode 100755 index 3edfccf8439..00000000000 --- a/buildscripts/kokoro/xds-k8s-install-test-driver.sh +++ /dev/null @@ -1,347 +0,0 @@ -#!/usr/bin/env bash -# TODO(sergiitk): move to grpc/grpc when implementing support of other languages -set -eo pipefail - -# Constants -readonly PYTHON_VERSION="3.6" -# Test driver -readonly TEST_DRIVER_REPO_NAME="grpc" -readonly TEST_DRIVER_REPO_URL="https://github.com/${TEST_DRIVER_REPO_OWNER:-grpc}/grpc.git" -readonly TEST_DRIVER_BRANCH="${TEST_DRIVER_BRANCH:-master}" -readonly TEST_DRIVER_PATH="tools/run_tests/xds_k8s_test_driver" -readonly TEST_DRIVER_PROTOS_PATH="src/proto/grpc/testing" - -####################################### -# Run command end report its exit code. Doesn't exit on non-zero exit code. -# Globals: -# None -# Arguments: -# Command to execute -# Outputs: -# Writes the output of given command to stdout, stderr -####################################### -run_ignore_exit_code() { - local exit_code=-1 - "$@" || exit_code=$? - echo "Exit code: ${exit_code}" -} - -####################################### -# Parses information about git repository at given path to global variables. -# Globals: -# GIT_ORIGIN_URL: Populated with the origin URL of git repo used for the build -# GIT_COMMIT: Populated with the SHA-1 of git commit being built -# GIT_COMMIT_SHORT: Populated with the short SHA-1 of git commit being built -# Arguments: -# Git source dir -####################################### -parse_src_repo_git_info() { - local src_dir="${SRC_DIR:?SRC_DIR must be set}" - readonly GIT_ORIGIN_URL=$(git -C "${src_dir}" remote get-url origin) - readonly GIT_COMMIT=$(git -C "${src_dir}" rev-parse HEAD) - readonly GIT_COMMIT_SHORT=$(git -C "${src_dir}" rev-parse --short HEAD) -} - -####################################### -# List GCR image tags matching given tag name. -# Arguments: -# Image name -# Tag name -# Outputs: -# Writes the table with the list of found tags to stdout. -# If no tags found, the output is an empty string. -####################################### -gcloud_gcr_list_image_tags() { - gcloud container images list-tags --format="table[box](tags,digest,timestamp.date())" --filter="tags:$2" "$1" -} - -####################################### -# A helper to execute `gcloud -q components update`. -# Arguments: -# None -# Outputs: -# Writes the output of `gcloud` command to stdout, stderr -####################################### -gcloud_update() { - echo "Update gcloud components:" - gcloud -q components update -} - -####################################### -# Create kube context authenticated with GKE cluster, saves context name. -# to KUBE_CONTEXT -# Globals: -# GKE_CLUSTER_NAME -# GKE_CLUSTER_ZONE -# KUBE_CONTEXT: Populated with name of kubectl context with GKE cluster access -# Arguments: -# None -# Outputs: -# Writes the output of `gcloud` command to stdout, stderr -# Writes authorization info $HOME/.kube/config -####################################### -gcloud_get_cluster_credentials() { - gcloud container clusters get-credentials "${GKE_CLUSTER_NAME}" --zone "${GKE_CLUSTER_ZONE}" - readonly KUBE_CONTEXT="$(kubectl config current-context)" -} - -####################################### -# Clone the source code of the test driver to $TEST_DRIVER_REPO_DIR, unless -# given folder exists. -# Globals: -# TEST_DRIVER_REPO_URL -# TEST_DRIVER_BRANCH -# TEST_DRIVER_REPO_DIR: path to the repo containing the test driver -# TEST_DRIVER_REPO_DIR_USE_EXISTING: set non-empty value to use exiting -# clone of the driver repo located at $TEST_DRIVER_REPO_DIR. -# Useful for debugging the build script locally. -# Arguments: -# None -# Outputs: -# Writes the output of `git` command to stdout, stderr -# Writes driver source code to $TEST_DRIVER_REPO_DIR -####################################### -test_driver_get_source() { - if [[ -n "${TEST_DRIVER_REPO_DIR_USE_EXISTING}" && -d "${TEST_DRIVER_REPO_DIR}" ]]; then - echo "Using exiting driver directory: ${TEST_DRIVER_REPO_DIR}." - else - echo "Cloning driver to ${TEST_DRIVER_REPO_URL} branch ${TEST_DRIVER_BRANCH} to ${TEST_DRIVER_REPO_DIR}" - git clone -b "${TEST_DRIVER_BRANCH}" --depth=1 "${TEST_DRIVER_REPO_URL}" "${TEST_DRIVER_REPO_DIR}" - fi -} - -####################################### -# Install Python modules from required in $TEST_DRIVER_FULL_DIR/requirements.txt -# to Python virtual environment. Creates and activates Python venv if necessary. -# Globals: -# TEST_DRIVER_FULL_DIR -# PYTHON_VERSION -# Arguments: -# None -# Outputs: -# Writes the output of `python`, `pip` commands to stdout, stderr -# Writes the list of installed modules to stdout -####################################### -test_driver_pip_install() { - echo "Install python dependencies" - cd "${TEST_DRIVER_FULL_DIR}" - - # Create and activate virtual environment unless already using one - if [[ -z "${VIRTUAL_ENV}" ]]; then - local venv_dir="${TEST_DRIVER_FULL_DIR}/venv" - if [[ -d "${venv_dir}" ]]; then - echo "Found python virtual environment directory: ${venv_dir}" - else - echo "Creating python virtual environment: ${venv_dir}" - "python${PYTHON_VERSION} -m venv ${venv_dir}" - fi - # Intentional: No need to check python venv activate script. - # shellcheck source=/dev/null - source "${venv_dir}/bin/activate" - fi - - pip install -r requirements.txt - echo "Installed Python packages:" - pip list -} - -####################################### -# Compile proto-files needed for the test driver -# Globals: -# TEST_DRIVER_REPO_DIR -# TEST_DRIVER_FULL_DIR -# TEST_DRIVER_PROTOS_PATH -# Arguments: -# None -# Outputs: -# Writes the output of `python -m grpc_tools.protoc` to stdout, stderr -# Writes the list if compiled python code to stdout -# Writes compiled python code with proto messages and grpc services to -# $TEST_DRIVER_FULL_DIR/src/proto -####################################### -test_driver_compile_protos() { - declare -a protos - protos=( - "${TEST_DRIVER_PROTOS_PATH}/test.proto" - "${TEST_DRIVER_PROTOS_PATH}/messages.proto" - "${TEST_DRIVER_PROTOS_PATH}/empty.proto" - ) - echo "Generate python code from grpc.testing protos: ${protos[*]}" - cd "${TEST_DRIVER_REPO_DIR}" - python -m grpc_tools.protoc \ - --proto_path=. \ - --python_out="${TEST_DRIVER_FULL_DIR}" \ - --grpc_python_out="${TEST_DRIVER_FULL_DIR}" \ - "${protos[@]}" - local protos_out_dir="${TEST_DRIVER_FULL_DIR}/${TEST_DRIVER_PROTOS_PATH}" - echo "Generated files ${protos_out_dir}:" - ls -Fl "${protos_out_dir}" -} - -####################################### -# Installs the test driver and it's requirements. -# https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#installation -# Globals: -# TEST_DRIVER_REPO_DIR: Populated with the path to the repo containing -# the test driver -# TEST_DRIVER_FULL_DIR: Populated with the path to the test driver source code -# Arguments: -# The directory for test driver's source code -# Outputs: -# Writes the output to stdout, stderr -####################################### -test_driver_install() { - readonly TEST_DRIVER_REPO_DIR="${1:?Usage test_driver_install TEST_DRIVER_REPO_DIR}" - readonly TEST_DRIVER_FULL_DIR="${TEST_DRIVER_REPO_DIR}/${TEST_DRIVER_PATH}" - test_driver_get_source - test_driver_pip_install - test_driver_compile_protos -} - -####################################### -# Outputs Kokoro image version and Ubuntu's lsb_release -# Arguments: -# None -# Outputs: -# Writes the output to stdout -####################################### -kokoro_print_version() { - echo "Kokoro VM version:" - if [[ -f /VERSION ]]; then - cat /VERSION - fi - run_ignore_exit_code lsb_release -a -} - -####################################### -# Report extra information about the job via sponge properties. -# Globals: -# KOKORO_ARTIFACTS_DIR -# GIT_ORIGIN_URL -# GIT_COMMIT_SHORT -# TESTGRID_EXCLUDE -# Arguments: -# None -# Outputs: -# Writes the output to stdout -# Writes job properties to $KOKORO_ARTIFACTS_DIR/custom_sponge_config.csv -####################################### -kokoro_write_sponge_properties() { - # CSV format: "property_name","property_value" - # Bump TESTS_FORMAT_VERSION when reported test name changed enough to when it - # makes more sense to discard previous test results from a testgrid board. - # Use GIT_ORIGIN_URL to exclude test runs executed against repo forks from - # testgrid reports. - cat >"${KOKORO_ARTIFACTS_DIR}/custom_sponge_config.csv" < Date: Thu, 7 Oct 2021 16:17:08 -0700 Subject: [PATCH 59/76] xds: override bootstrap for xds server (#8575) added xdsServerBuilder method `overrideBootstrapForTest()`. Fix issue https://github.com/grpc/grpc-java/issues/7819 --- xds/src/main/java/io/grpc/xds/XdsServerBuilder.java | 13 +++++++++++++ .../test/java/io/grpc/xds/XdsServerBuilderTest.java | 13 ++++++++++++- .../test/java/io/grpc/xds/XdsServerTestHelper.java | 3 ++- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/XdsServerBuilder.java b/xds/src/main/java/io/grpc/xds/XdsServerBuilder.java index c95c1e6d48f..d4df317a7e9 100644 --- a/xds/src/main/java/io/grpc/xds/XdsServerBuilder.java +++ b/xds/src/main/java/io/grpc/xds/XdsServerBuilder.java @@ -37,6 +37,7 @@ import io.grpc.netty.NettyServerBuilder; import io.grpc.xds.FilterChainMatchingProtocolNegotiators.FilterChainMatchingNegotiatorServerFactory; import io.grpc.xds.XdsNameResolverProvider.XdsClientPoolFactory; +import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Logger; @@ -136,6 +137,18 @@ XdsServerBuilder xdsClientPoolFactory(XdsClientPoolFactory xdsClientPoolFactory) return this; } + /** + * Allows providing bootstrap override, useful for testing. + */ + public XdsServerBuilder overrideBootstrapForTest(Map bootstrapOverride) { + checkNotNull(bootstrapOverride, "bootstrapOverride"); + if (this.xdsClientPoolFactory == SharedXdsClientPoolProvider.getDefaultProvider()) { + this.xdsClientPoolFactory = new SharedXdsClientPoolProvider(); + } + this.xdsClientPoolFactory.setBootstrapOverride(bootstrapOverride); + return this; + } + /** * Returns the delegate {@link NettyServerBuilder} to allow experimental level * transport-specific configuration. Note this API will always be experimental. diff --git a/xds/src/test/java/io/grpc/xds/XdsServerBuilderTest.java b/xds/src/test/java/io/grpc/xds/XdsServerBuilderTest.java index 0d15c1f660e..d67ed9d09fc 100644 --- a/xds/src/test/java/io/grpc/xds/XdsServerBuilderTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsServerBuilderTest.java @@ -40,7 +40,9 @@ import java.net.InetSocketAddress; import java.net.ServerSocket; import java.net.SocketAddress; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -65,6 +67,7 @@ public class XdsServerBuilderTest { private int port; private TlsContextManager tlsContextManager; private FakeXdsClient xdsClient = new FakeXdsClient(); + private FakeXdsClientPoolFactory xdsClientPoolFactory = new FakeXdsClientPoolFactory(xdsClient); private void buildServer(XdsServerBuilder.XdsServingStatusListener xdsServingStatusListener) throws IOException { @@ -77,7 +80,7 @@ private void buildBuilder(XdsServerBuilder.XdsServingStatusListener xdsServingSt builder = XdsServerBuilder.forPort( port, XdsServerCredentials.create(InsecureServerCredentials.create())); - builder.xdsClientPoolFactory(new FakeXdsClientPoolFactory(xdsClient)); + builder.xdsClientPoolFactory(xdsClientPoolFactory); if (xdsServingStatusListener != null) { builder.xdsServingStatusListener(xdsServingStatusListener); } @@ -292,4 +295,12 @@ public void drainGraceTime_negativeThrows() throws IOException { assertThat(expected).hasMessageThat().contains("drain grace time"); } } + + @Test + public void testOverrideBootstrap() throws Exception { + Map b = new HashMap<>(); + buildBuilder(null); + builder.overrideBootstrapForTest(b); + assertThat(xdsClientPoolFactory.savedBootstrap).isEqualTo(b); + } } diff --git a/xds/src/test/java/io/grpc/xds/XdsServerTestHelper.java b/xds/src/test/java/io/grpc/xds/XdsServerTestHelper.java index f289c4726fb..ffe4a72f522 100644 --- a/xds/src/test/java/io/grpc/xds/XdsServerTestHelper.java +++ b/xds/src/test/java/io/grpc/xds/XdsServerTestHelper.java @@ -113,6 +113,7 @@ static final class FakeXdsClientPoolFactory implements XdsNameResolverProvider.XdsClientPoolFactory { private XdsClient xdsClient; + Map savedBootstrap; FakeXdsClientPoolFactory(XdsClient xdsClient) { this.xdsClient = xdsClient; @@ -120,7 +121,7 @@ static final class FakeXdsClientPoolFactory @Override public void setBootstrapOverride(Map bootstrap) { - throw new UnsupportedOperationException("Should not be called"); + this.savedBootstrap = bootstrap; } @Override From 0d25d8f7d6aef41d47f3734704b2b868baea651f Mon Sep 17 00:00:00 2001 From: markb74 <57717302+markb74@users.noreply.github.com> Date: Fri, 8 Oct 2021 12:03:44 +0200 Subject: [PATCH 60/76] Publish binder in releases. (#8585) --- binder/build.gradle | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/binder/build.gradle b/binder/build.gradle index c0e0b66fd61..04f8444ff01 100644 --- a/binder/build.gradle +++ b/binder/build.gradle @@ -2,6 +2,7 @@ plugins { id "maven-publish" id "com.android.library" id "ru.vyarus.animalsniffer" + id "digital.wup.android-maven-publish" } description = 'gRPC BinderChannel' @@ -94,4 +95,39 @@ tasks.withType(JavaCompile) { options.errorprone.check("UnnecessaryAnonymousClass", CheckSeverity.OFF) } -[publishMavenPublicationToMavenRepository]*.onlyIf { false } +task javadocs(type: Javadoc) { + source = android.sourceSets.main.java.srcDirs + classpath += files(android.getBootClasspath()) + classpath += files({ + android.libraryVariants.collect { variant -> + variant.javaCompileProvider.get().classpath + } + }) + options { + // Disable JavaDoc doclint on Java 8. + if (JavaVersion.current().isJava8Compatible()) { + addStringOption('Xdoclint:none', '-quiet') + } + } +} + +task javadocJar(type: Jar, dependsOn: javadocs) { + classifier = 'javadoc' + from javadocs.destinationDir +} + +task sourcesJar(type: Jar) { + classifier = 'sources' + from android.sourceSets.main.java.srcDirs +} + +publishing { + publications { + maven { + from components.android + + artifact javadocJar + artifact sourcesJar + } + } +} From bb51bb6dfa90fcd9c4336e14cdd2af360aed8145 Mon Sep 17 00:00:00 2001 From: Ivo List Date: Fri, 8 Oct 2021 22:48:22 +0200 Subject: [PATCH 61/76] java_grpc_library.bzl: Fix parameters of java_common.compile (#7598) Parameter host_javabase is removed. This is preparation for flipping incompatible_java_common_parameters in Bazel 5. See https://github.com/bazelbuild/bazel/issues/12373 Bazel versions prior to 4 require host_javabase, so are no longer supported. --- buildscripts/kokoro/bazel.sh | 2 +- java_grpc_library.bzl | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/buildscripts/kokoro/bazel.sh b/buildscripts/kokoro/bazel.sh index f63c05f300b..107a305bbf9 100755 --- a/buildscripts/kokoro/bazel.sh +++ b/buildscripts/kokoro/bazel.sh @@ -3,7 +3,7 @@ set -exu -o pipefail cat /VERSION -use_bazel.sh 1.0.1 +use_bazel.sh 4.0.0 bazel version cd github/grpc-java diff --git a/java_grpc_library.bzl b/java_grpc_library.bzl index cd385998d27..913e905a711 100644 --- a/java_grpc_library.bzl +++ b/java_grpc_library.bzl @@ -2,7 +2,6 @@ _JavaRpcToolchainInfo = provider( fields = [ - "host_javabase", "java_toolchain", "plugin", "plugin_arg", @@ -14,7 +13,6 @@ _JavaRpcToolchainInfo = provider( def _java_rpc_toolchain_impl(ctx): return [ _JavaRpcToolchainInfo( - host_javabase = ctx.attr._host_javabase, java_toolchain = ctx.attr._java_toolchain, plugin = ctx.executable.plugin, plugin_arg = ctx.attr.plugin_arg, @@ -44,10 +42,6 @@ java_rpc_toolchain = rule( "_java_toolchain": attr.label( default = Label("@bazel_tools//tools/jdk:current_java_toolchain"), ), - "_host_javabase": attr.label( - cfg = "host", - default = Label("@bazel_tools//tools/jdk:current_java_runtime"), - ), }, provides = [ _JavaRpcToolchainInfo, @@ -106,7 +100,6 @@ def _java_rpc_library_impl(ctx): java_info = java_common.compile( ctx, java_toolchain = toolchain.java_toolchain[java_common.JavaToolchainInfo], - host_javabase = toolchain.host_javabase[java_common.JavaRuntimeInfo], source_jars = [srcjar], output = ctx.outputs.jar, output_source_jar = ctx.outputs.srcjar, From 9266174812de1e6723dbdae3ae9c9a3f70f83705 Mon Sep 17 00:00:00 2001 From: markb74 <57717302+markb74@users.noreply.github.com> Date: Sat, 9 Oct 2021 12:27:01 +0200 Subject: [PATCH 62/76] Fix code & javadoc warnings in the binder package. (#8588) Note: I didn't fix all javadoc warnings mentioned in #8585, since they're not generated with a modern java version, and the fix feels worse than the warning. Specifically, {@link X.Y} generates a warning if only X is imported, and {@link Z} generates a warning if Z is declared later in the class. In particular, attempting to fix the first issue by importing X.Y results in a code-readability warning suggesting I shouldn't do that. --- binder/src/main/java/io/grpc/binder/SecurityPolicy.java | 2 +- .../src/main/java/io/grpc/binder/internal/BinderTransport.java | 2 +- .../src/main/java/io/grpc/binder/internal/MetadataHelper.java | 2 +- binder/src/main/java/io/grpc/binder/internal/Outbound.java | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/binder/src/main/java/io/grpc/binder/SecurityPolicy.java b/binder/src/main/java/io/grpc/binder/SecurityPolicy.java index 55aa33e0216..d7dad53fdc8 100644 --- a/binder/src/main/java/io/grpc/binder/SecurityPolicy.java +++ b/binder/src/main/java/io/grpc/binder/SecurityPolicy.java @@ -35,7 +35,7 @@ @ExperimentalApi("https://github.com/grpc/grpc-java/issues/8022") public abstract class SecurityPolicy { - public SecurityPolicy() {} + protected SecurityPolicy() {} /** * Decides whether the given Android UID is authorized. (Validity is implementation dependent). diff --git a/binder/src/main/java/io/grpc/binder/internal/BinderTransport.java b/binder/src/main/java/io/grpc/binder/internal/BinderTransport.java index 9e97b7795ab..b3bc488b64a 100644 --- a/binder/src/main/java/io/grpc/binder/internal/BinderTransport.java +++ b/binder/src/main/java/io/grpc/binder/internal/BinderTransport.java @@ -87,7 +87,7 @@ * need to call into this class to send a transaction (possibly waiting for the transport to become * ready). * - *

The split between Outbound & Inbound helps reduce this risk, but not entirely remove it. + *

The split between Outbound & Inbound helps reduce this risk, but not entirely remove it. * *

For this reason, while most state within this class is guarded by this instance, methods * exposed to individual stream instances need to use atomic or volatile types, since those calls diff --git a/binder/src/main/java/io/grpc/binder/internal/MetadataHelper.java b/binder/src/main/java/io/grpc/binder/internal/MetadataHelper.java index 211768f2948..bd473780788 100644 --- a/binder/src/main/java/io/grpc/binder/internal/MetadataHelper.java +++ b/binder/src/main/java/io/grpc/binder/internal/MetadataHelper.java @@ -31,7 +31,7 @@ import javax.annotation.Nullable; /** - * Helper class for reading & writing metadata to parcels. + * Helper class for reading & writing metadata to parcels. * *

Metadata is written to a parcel as a single int for the number of name/value pairs, followed * by the following pattern for each pair. diff --git a/binder/src/main/java/io/grpc/binder/internal/Outbound.java b/binder/src/main/java/io/grpc/binder/internal/Outbound.java index f629a05a447..2a4e968b1e8 100644 --- a/binder/src/main/java/io/grpc/binder/internal/Outbound.java +++ b/binder/src/main/java/io/grpc/binder/internal/Outbound.java @@ -219,6 +219,7 @@ final void send() throws StatusException { } @GuardedBy("this") + @SuppressWarnings("fallthrough") protected final void sendInternal() throws StatusException { Parcel parcel = Parcel.obtain(); int flags = 0; From 7cf05781767a5c069e5a621b544909d68c41a0b1 Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Wed, 13 Oct 2021 09:54:51 -0700 Subject: [PATCH 63/76] .github/workflows: Bump codecov-action to v2 The codecov bash uploader is being replaced (supposedly partially for security reasons, but it seems maintenance reasons are the real goal). https://about.codecov.io/blog/codecov-uploader-deprecation-plan/ v1 uses the bash uploader. v2 uses the new uploader. The bash uploader will begin seeing brownouts soon. --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 99177462ef8..ea883a77b04 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -60,4 +60,4 @@ jobs: if: matrix.jre == 8 # Upload once, instead of for each job in the matrix run: ./gradlew :grpc-all:coveralls -x compileJava - name: Codecov - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v2 From 48e3bafb11cda8ba03eb4f062e94193682568fb8 Mon Sep 17 00:00:00 2001 From: ZHANG Dapeng Date: Thu, 14 Oct 2021 10:01:56 -0700 Subject: [PATCH 64/76] rls: limit cache_size in rls config to 5M (#8603) In the latest grpc-rls-lb-policy-design, if the value of cache_size_bytes is greater than 5M, we cap it at 5M. --- .../main/java/io/grpc/rls/RlsProtoData.java | 3 +- .../java/io/grpc/rls/RlsProtoDataTest.java | 56 +++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 rls/src/test/java/io/grpc/rls/RlsProtoDataTest.java diff --git a/rls/src/main/java/io/grpc/rls/RlsProtoData.java b/rls/src/main/java/io/grpc/rls/RlsProtoData.java index 3556ca609be..f13f0369d89 100644 --- a/rls/src/main/java/io/grpc/rls/RlsProtoData.java +++ b/rls/src/main/java/io/grpc/rls/RlsProtoData.java @@ -141,6 +141,7 @@ public String toString() { static final class RouteLookupConfig { private static final long MAX_AGE_MILLIS = TimeUnit.MINUTES.toMillis(5); + private static final long MAX_CACHE_SIZE = 5 * 1024 * 1024; private final ImmutableList grpcKeyBuilders; @@ -194,7 +195,7 @@ static final class RouteLookupConfig { this.maxAgeInMillis = Math.min(maxAgeInMillis, MAX_AGE_MILLIS); this.staleAgeInMillis = Math.min(staleAgeInMillis, this.maxAgeInMillis); checkArgument(cacheSizeBytes > 0, "cacheSize must be positive"); - this.cacheSizeBytes = cacheSizeBytes; + this.cacheSizeBytes = Math.min(cacheSizeBytes, MAX_CACHE_SIZE); this.validTargets = ImmutableList.copyOf(checkNotNull(validTargets, "validTargets")); this.defaultTarget = defaultTarget; } diff --git a/rls/src/test/java/io/grpc/rls/RlsProtoDataTest.java b/rls/src/test/java/io/grpc/rls/RlsProtoDataTest.java new file mode 100644 index 00000000000..9cfb6c753fb --- /dev/null +++ b/rls/src/test/java/io/grpc/rls/RlsProtoDataTest.java @@ -0,0 +1,56 @@ +/* + * Copyright 2021 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.rls; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.grpc.rls.RlsProtoData.ExtraKeys; +import io.grpc.rls.RlsProtoData.GrpcKeyBuilder; +import io.grpc.rls.RlsProtoData.GrpcKeyBuilder.Name; +import io.grpc.rls.RlsProtoData.NameMatcher; +import io.grpc.rls.RlsProtoData.RouteLookupConfig; +import java.util.concurrent.TimeUnit; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests for {@link RlsProtoData}. */ +@RunWith(JUnit4.class) +public class RlsProtoDataTest { + @Test + public void maxCacheSize() { + RouteLookupConfig config = new RouteLookupConfig( + ImmutableList.of( + new GrpcKeyBuilder( + ImmutableList.of(new Name("service1", "create")), + ImmutableList.of( + new NameMatcher("user", ImmutableList.of("User", "Parent"), true), + new NameMatcher("id", ImmutableList.of("X-Google-Id"), true)), + ExtraKeys.create("server", "service-key", "method-key"), + ImmutableMap.of())), + "service1", + /* lookupServiceTimeoutInMillis= */ TimeUnit.SECONDS.toMillis(2), + /* maxAgeInMillis= */ TimeUnit.SECONDS.toMillis(300), + /* staleAgeInMillis= */ TimeUnit.SECONDS.toMillis(240), + /* cacheSizeBytes= */ 20 * 1000 * 1000, + ImmutableList.of("a-valid-target"), + "default-target"); + assertThat(config.getCacheSizeBytes()).isEqualTo(5 * 1024 * 1024); + } +} From 8e5c18819c729b7f2a3285df0ea468cdb4c2fff6 Mon Sep 17 00:00:00 2001 From: yifeizhuang Date: Thu, 14 Oct 2021 11:14:48 -0700 Subject: [PATCH 65/76] enable rbac by default (#8604) --- xds/src/main/java/io/grpc/xds/ClientXdsClient.java | 4 ++-- xds/src/test/java/io/grpc/xds/ClientXdsClientDataTest.java | 3 +-- xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java | 3 +-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/ClientXdsClient.java b/xds/src/main/java/io/grpc/xds/ClientXdsClient.java index 848e6691732..aaea10580d2 100644 --- a/xds/src/main/java/io/grpc/xds/ClientXdsClient.java +++ b/xds/src/main/java/io/grpc/xds/ClientXdsClient.java @@ -138,8 +138,8 @@ final class ClientXdsClient extends AbstractXdsClient { || Boolean.parseBoolean(System.getenv("GRPC_XDS_EXPERIMENTAL_ENABLE_RETRY")); @VisibleForTesting static boolean enableRbac = - !Strings.isNullOrEmpty(System.getenv("GRPC_XDS_EXPERIMENTAL_RBAC")) - && Boolean.parseBoolean(System.getenv("GRPC_XDS_EXPERIMENTAL_RBAC")); + Strings.isNullOrEmpty(System.getenv("GRPC_XDS_EXPERIMENTAL_RBAC")) + || Boolean.parseBoolean(System.getenv("GRPC_XDS_EXPERIMENTAL_RBAC")); private static final String TYPE_URL_HTTP_CONNECTION_MANAGER_V2 = "type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2" diff --git a/xds/src/test/java/io/grpc/xds/ClientXdsClientDataTest.java b/xds/src/test/java/io/grpc/xds/ClientXdsClientDataTest.java index ac8c3fb44e7..0207c2b94e5 100644 --- a/xds/src/test/java/io/grpc/xds/ClientXdsClientDataTest.java +++ b/xds/src/test/java/io/grpc/xds/ClientXdsClientDataTest.java @@ -138,8 +138,7 @@ public void setUp() { originalEnableRetry = ClientXdsClient.enableRetry; assertThat(originalEnableRetry).isTrue(); originalEnableRbac = ClientXdsClient.enableRbac; - assertThat(originalEnableRbac).isFalse(); - ClientXdsClient.enableRbac = true; + assertThat(originalEnableRbac).isTrue(); } @After diff --git a/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java b/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java index 3a9ab23aa74..a488175b835 100644 --- a/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java +++ b/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java @@ -260,8 +260,7 @@ public void setUp() throws IOException { originalEnableFaultInjection = ClientXdsClient.enableFaultInjection; ClientXdsClient.enableFaultInjection = true; originalEnableRbac = ClientXdsClient.enableRbac; - assertThat(originalEnableRbac).isFalse(); - ClientXdsClient.enableRbac = true; + assertThat(originalEnableRbac).isTrue(); final String serverName = InProcessServerBuilder.generateName(); cleanupRule.register( InProcessServerBuilder From 9f644a086168ebd72e5eabbc5abdafedb2df08c6 Mon Sep 17 00:00:00 2001 From: ZHANG Dapeng Date: Thu, 14 Oct 2021 11:55:29 -0700 Subject: [PATCH 66/76] xds: migrate Bootstrapper data classes to use AutoValue (#8594) As many new fields will be added to `BootstrapInfo` for xds federation support, refactor `Bootstrapper.java` to use `AutoValue`. All the other files are just mechanical changes due to the refactoring. --- .../java/io/grpc/xds/AbstractXdsClient.java | 6 +- .../main/java/io/grpc/xds/Bootstrapper.java | 116 +++++++----------- .../java/io/grpc/xds/BootstrapperImpl.java | 11 +- .../java/io/grpc/xds/ClientXdsClient.java | 10 +- .../main/java/io/grpc/xds/CsdsService.java | 2 +- .../grpc/xds/SharedXdsClientPoolProvider.java | 8 +- .../java/io/grpc/xds/XdsServerWrapper.java | 4 +- .../CertProviderSslContextProvider.java | 8 +- .../sds/ClientSslContextProviderFactory.java | 4 +- .../sds/ServerSslContextProviderFactory.java | 4 +- .../io/grpc/xds/BootstrapperImplTest.java | 78 ++++++------ .../io/grpc/xds/ClientXdsClientTestBase.java | 16 +-- .../grpc/xds/CommonBootstrapperTestUtils.java | 27 ++-- .../java/io/grpc/xds/CsdsServiceTest.java | 22 ++-- .../xds/SharedXdsClientPoolProviderTest.java | 18 +-- .../grpc/xds/XdsNameResolverProviderTest.java | 2 +- .../java/io/grpc/xds/XdsServerTestHelper.java | 13 +- .../io/grpc/xds/XdsServerWrapperTest.java | 23 ++-- ...tProviderClientSslContextProviderTest.java | 8 +- ...tProviderServerSslContextProviderTest.java | 8 +- 20 files changed, 188 insertions(+), 200 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/AbstractXdsClient.java b/xds/src/main/java/io/grpc/xds/AbstractXdsClient.java index 357534233b0..a6c3c2feb99 100644 --- a/xds/src/main/java/io/grpc/xds/AbstractXdsClient.java +++ b/xds/src/main/java/io/grpc/xds/AbstractXdsClient.java @@ -303,7 +303,7 @@ protected final boolean isInBackoff() { // Must be synchronized. private void startRpcStream() { checkState(adsStream == null, "Previous adsStream has not been cleared yet"); - if (bootstrapInfo.getServers().get(0).isUseProtocolV3()) { + if (bootstrapInfo.servers().get(0).useProtocolV3()) { adsStream = new AdsStreamV3(); } else { adsStream = new AdsStreamV2(); @@ -619,7 +619,7 @@ void sendDiscoveryRequest(ResourceType type, String versionInfo, Collection rawData) throws XdsInitializationExceptio * Data class containing xDS server information, such as server URI and channel credentials * to be used for communication. */ + @AutoValue @Internal - static class ServerInfo { - private final String target; - private final ChannelCredentials channelCredentials; - private final boolean useProtocolV3; + abstract static class ServerInfo { + abstract String target(); - @VisibleForTesting - ServerInfo(String target, ChannelCredentials channelCredentials, boolean useProtocolV3) { - this.target = checkNotNull(target, "target"); - this.channelCredentials = checkNotNull(channelCredentials, "channelCredentials"); - this.useProtocolV3 = useProtocolV3; - } + abstract ChannelCredentials channelCredentials(); - String getTarget() { - return target; - } + abstract boolean useProtocolV3(); - ChannelCredentials getChannelCredentials() { - return channelCredentials; - } - - boolean isUseProtocolV3() { - return useProtocolV3; + @VisibleForTesting + static ServerInfo create( + String target, ChannelCredentials channelCredentials, boolean useProtocolV3) { + return new AutoValue_Bootstrapper_ServerInfo(target, channelCredentials, useProtocolV3); } } @@ -79,71 +69,57 @@ boolean isUseProtocolV3() { * Data class containing Certificate provider information: the plugin-name and an opaque * Map that represents the config for that plugin. */ + @AutoValue @Internal - public static class CertificateProviderInfo { - private final String pluginName; - private final Map config; + public abstract static class CertificateProviderInfo { + public abstract String pluginName(); - @VisibleForTesting - public CertificateProviderInfo(String pluginName, Map config) { - this.pluginName = checkNotNull(pluginName, "pluginName"); - this.config = checkNotNull(config, "config"); - } - - public String getPluginName() { - return pluginName; - } + public abstract ImmutableMap config(); - public Map getConfig() { - return config; + @VisibleForTesting + public static CertificateProviderInfo create(String pluginName, Map config) { + return new AutoValue_Bootstrapper_CertificateProviderInfo( + pluginName, ImmutableMap.copyOf(config)); } } /** * Data class containing the results of reading bootstrap. */ + @AutoValue @Internal - public static class BootstrapInfo { - private List servers; - private final Node node; - @Nullable private final Map certProviders; - @Nullable private final String serverListenerResourceNameTemplate; + public abstract static class BootstrapInfo { + /** Returns the list of xDS servers to be connected to. */ + abstract ImmutableList servers(); + + /** Returns the node identifier to be included in xDS requests. */ + public abstract Node node(); + + /** Returns the cert-providers config map. */ + @Nullable + public abstract ImmutableMap certProviders(); + + @Nullable + public abstract String serverListenerResourceNameTemplate(); @VisibleForTesting - BootstrapInfo( - List servers, - Node node, - Map certProviders, - String serverListenerResourceNameTemplate) { - this.servers = servers; - this.node = node; - this.certProviders = certProviders; - this.serverListenerResourceNameTemplate = serverListenerResourceNameTemplate; + static Builder builder() { + return new AutoValue_Bootstrapper_BootstrapInfo.Builder(); } - /** - * Returns the list of xDS servers to be connected to. - */ - List getServers() { - return Collections.unmodifiableList(servers); - } + @AutoValue.Builder + @VisibleForTesting + abstract static class Builder { + abstract Builder servers(List servers); - /** - * Returns the node identifier to be included in xDS requests. - */ - public Node getNode() { - return node; - } + abstract Builder node(Node node); - /** Returns the cert-providers config map. */ - @Nullable - public Map getCertProviders() { - return certProviders == null ? null : Collections.unmodifiableMap(certProviders); - } + abstract Builder certProviders(@Nullable Map certProviders); - @Nullable - public String getServerListenerResourceNameTemplate() { - return serverListenerResourceNameTemplate; + abstract Builder serverListenerResourceNameTemplate( + @Nullable String serverListenerResourceNameTemplate); + + abstract BootstrapInfo build(); } } } diff --git a/xds/src/main/java/io/grpc/xds/BootstrapperImpl.java b/xds/src/main/java/io/grpc/xds/BootstrapperImpl.java index 00a8aa06453..0a044da5290 100644 --- a/xds/src/main/java/io/grpc/xds/BootstrapperImpl.java +++ b/xds/src/main/java/io/grpc/xds/BootstrapperImpl.java @@ -154,7 +154,7 @@ BootstrapInfo bootstrap(Map rawData) throws XdsInitializationExceptio logger.log(XdsLogLevel.INFO, "Server features: {0}", serverFeatures); useProtocolV3 = serverFeatures.contains(XDS_V3_SERVER_FEATURE); } - servers.add(new ServerInfo(serverUri, channelCredentials, useProtocolV3)); + servers.add(ServerInfo.create(serverUri, channelCredentials, useProtocolV3)); } Node.Builder nodeBuilder = Node.newBuilder(); @@ -211,13 +211,18 @@ BootstrapInfo bootstrap(Map rawData) throws XdsInitializationExceptio checkForNull(JsonUtil.getString(valueMap, "plugin_name"), "plugin_name"); Map config = checkForNull(JsonUtil.getObject(valueMap, "config"), "config"); CertificateProviderInfo certificateProviderInfo = - new CertificateProviderInfo(pluginName, config); + CertificateProviderInfo.create(pluginName, config); certProviders.put(name, certificateProviderInfo); } } String grpcServerResourceId = JsonUtil.getString(rawData, "server_listener_resource_name_template"); - return new BootstrapInfo(servers, nodeBuilder.build(), certProviders, grpcServerResourceId); + return BootstrapInfo.builder() + .servers(servers) + .node(nodeBuilder.build()) + .certProviders(certProviders) + .serverListenerResourceNameTemplate(grpcServerResourceId) + .build(); } @VisibleForTesting diff --git a/xds/src/main/java/io/grpc/xds/ClientXdsClient.java b/xds/src/main/java/io/grpc/xds/ClientXdsClient.java index aaea10580d2..8a43717f97e 100644 --- a/xds/src/main/java/io/grpc/xds/ClientXdsClient.java +++ b/xds/src/main/java/io/grpc/xds/ClientXdsClient.java @@ -186,7 +186,7 @@ final class ClientXdsClient extends AbstractXdsClient { this.timeProvider = timeProvider; this.tlsContextManager = checkNotNull(tlsContextManager, "tlsContextManager"); lrsClient = new LoadReportClient(loadStatsManager, channel, context, - bootstrapInfo.getServers().get(0).isUseProtocolV3(), bootstrapInfo.getNode(), + bootstrapInfo.servers().get(0).useProtocolV3(), bootstrapInfo.node(), getSyncContext(), timeService, backoffPolicyProvider, stopwatchSupplier); } @@ -263,8 +263,8 @@ private LdsUpdate processServerSideListener( Listener proto, Set rdsResources, boolean parseHttpFilter) throws ResourceInvalidException { Set certProviderInstances = null; - if (getBootstrapInfo() != null && getBootstrapInfo().getCertProviders() != null) { - certProviderInstances = getBootstrapInfo().getCertProviders().keySet(); + if (getBootstrapInfo() != null && getBootstrapInfo().certProviders() != null) { + certProviderInstances = getBootstrapInfo().certProviders().keySet(); } return LdsUpdate.forTcpListener(parseServerSideListener( proto, rdsResources, tlsContextManager, filterRegistry, certProviderInstances, @@ -1404,8 +1404,8 @@ protected void handleCdsResponse(String versionInfo, List resources, String CdsUpdate cdsUpdate; try { Set certProviderInstances = null; - if (getBootstrapInfo() != null && getBootstrapInfo().getCertProviders() != null) { - certProviderInstances = getBootstrapInfo().getCertProviders().keySet(); + if (getBootstrapInfo() != null && getBootstrapInfo().certProviders() != null) { + certProviderInstances = getBootstrapInfo().certProviders().keySet(); } cdsUpdate = parseCluster(cluster, retainedEdsResources, certProviderInstances); } catch (ResourceInvalidException e) { diff --git a/xds/src/main/java/io/grpc/xds/CsdsService.java b/xds/src/main/java/io/grpc/xds/CsdsService.java index 7fc6d68b96f..5146930859d 100644 --- a/xds/src/main/java/io/grpc/xds/CsdsService.java +++ b/xds/src/main/java/io/grpc/xds/CsdsService.java @@ -162,7 +162,7 @@ static ClientConfig getClientConfigForXdsClient(XdsClient xdsClient) { xdsClient.getSubscribedResourcesMetadata(ResourceType.EDS)); return ClientConfig.newBuilder() - .setNode(xdsClient.getBootstrapInfo().getNode().toEnvoyProtoNode()) + .setNode(xdsClient.getBootstrapInfo().node().toEnvoyProtoNode()) .addXdsConfig(PerXdsConfig.newBuilder().setListenerConfig(ldsConfig)) .addXdsConfig(PerXdsConfig.newBuilder().setRouteConfig(rdsConfig)) .addXdsConfig(PerXdsConfig.newBuilder().setClusterConfig(cdsConfig)) diff --git a/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java b/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java index 95eef3e3d80..14bdced5dac 100644 --- a/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java +++ b/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java @@ -90,7 +90,7 @@ public ObjectPool getOrCreate() throws XdsInitializationException { } else { bootstrapInfo = bootstrapper.bootstrap(); } - if (bootstrapInfo.getServers().isEmpty()) { + if (bootstrapInfo.servers().isEmpty()) { throw new XdsInitializationException("No xDS server provided"); } ref = xdsClientPool = new RefCountedXdsClientObjectPool(bootstrapInfo); @@ -128,9 +128,9 @@ static class RefCountedXdsClientObjectPool implements ObjectPool { public XdsClient getObject() { synchronized (lock) { if (refCount == 0) { - ServerInfo serverInfo = bootstrapInfo.getServers().get(0); // use first server - String target = serverInfo.getTarget(); - ChannelCredentials channelCredentials = serverInfo.getChannelCredentials(); + ServerInfo serverInfo = bootstrapInfo.servers().get(0); // use first server + String target = serverInfo.target(); + ChannelCredentials channelCredentials = serverInfo.channelCredentials(); channel = Grpc.newChannelBuilder(target, channelCredentials) .keepAliveTime(5, TimeUnit.MINUTES) .build(); diff --git a/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java b/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java index a80398dedcd..ed51ccc9edf 100644 --- a/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java +++ b/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java @@ -181,8 +181,8 @@ private void internalStart() { return; } xdsClient = xdsClientPool.getObject(); - boolean useProtocolV3 = xdsClient.getBootstrapInfo().getServers().get(0).isUseProtocolV3(); - String listenerTemplate = xdsClient.getBootstrapInfo().getServerListenerResourceNameTemplate(); + boolean useProtocolV3 = xdsClient.getBootstrapInfo().servers().get(0).useProtocolV3(); + String listenerTemplate = xdsClient.getBootstrapInfo().serverListenerResourceNameTemplate(); if (!useProtocolV3 || listenerTemplate == null) { StatusException statusException = Status.UNAVAILABLE.withDescription( diff --git a/xds/src/main/java/io/grpc/xds/internal/certprovider/CertProviderSslContextProvider.java b/xds/src/main/java/io/grpc/xds/internal/certprovider/CertProviderSslContextProvider.java index 1ec58764196..5c4dba99dc3 100644 --- a/xds/src/main/java/io/grpc/xds/internal/certprovider/CertProviderSslContextProvider.java +++ b/xds/src/main/java/io/grpc/xds/internal/certprovider/CertProviderSslContextProvider.java @@ -61,8 +61,8 @@ protected CertProviderSslContextProvider( certHandle = certProviderInstanceConfig == null ? null : certificateProviderStore.createOrGetProvider( certInstance.getCertificateName(), - certProviderInstanceConfig.getPluginName(), - certProviderInstanceConfig.getConfig(), + certProviderInstanceConfig.pluginName(), + certProviderInstanceConfig.config(), this, true); } else { @@ -76,8 +76,8 @@ protected CertProviderSslContextProvider( rootCertHandle = certProviderInstanceConfig == null ? null : certificateProviderStore.createOrGetProvider( rootCertInstance.getCertificateName(), - certProviderInstanceConfig.getPluginName(), - certProviderInstanceConfig.getConfig(), + certProviderInstanceConfig.pluginName(), + certProviderInstanceConfig.config(), this, true); } else { diff --git a/xds/src/main/java/io/grpc/xds/internal/sds/ClientSslContextProviderFactory.java b/xds/src/main/java/io/grpc/xds/internal/sds/ClientSslContextProviderFactory.java index 3fff30ed649..bb7c0314bb4 100644 --- a/xds/src/main/java/io/grpc/xds/internal/sds/ClientSslContextProviderFactory.java +++ b/xds/src/main/java/io/grpc/xds/internal/sds/ClientSslContextProviderFactory.java @@ -52,8 +52,8 @@ public SslContextProvider create(UpstreamTlsContext upstreamTlsContext) { upstreamTlsContext.getCommonTlsContext())) { return certProviderClientSslContextProviderFactory.getProvider( upstreamTlsContext, - bootstrapInfo.getNode().toEnvoyProtoNode(), - bootstrapInfo.getCertProviders()); + bootstrapInfo.node().toEnvoyProtoNode(), + bootstrapInfo.certProviders()); } throw new UnsupportedOperationException("Unsupported configurations in UpstreamTlsContext!"); } diff --git a/xds/src/main/java/io/grpc/xds/internal/sds/ServerSslContextProviderFactory.java b/xds/src/main/java/io/grpc/xds/internal/sds/ServerSslContextProviderFactory.java index 6db96e2235e..590ffdb47c5 100644 --- a/xds/src/main/java/io/grpc/xds/internal/sds/ServerSslContextProviderFactory.java +++ b/xds/src/main/java/io/grpc/xds/internal/sds/ServerSslContextProviderFactory.java @@ -53,8 +53,8 @@ public SslContextProvider create( downstreamTlsContext.getCommonTlsContext())) { return certProviderServerSslContextProviderFactory.getProvider( downstreamTlsContext, - bootstrapInfo.getNode().toEnvoyProtoNode(), - bootstrapInfo.getCertProviders()); + bootstrapInfo.node().toEnvoyProtoNode(), + bootstrapInfo.certProviders()); } throw new UnsupportedOperationException("Unsupported configurations in DownstreamTlsContext!"); } diff --git a/xds/src/test/java/io/grpc/xds/BootstrapperImplTest.java b/xds/src/test/java/io/grpc/xds/BootstrapperImplTest.java index 7f5e74b9308..283efcea852 100644 --- a/xds/src/test/java/io/grpc/xds/BootstrapperImplTest.java +++ b/xds/src/test/java/io/grpc/xds/BootstrapperImplTest.java @@ -106,11 +106,11 @@ public void parseBootstrap_singleXdsServer() throws XdsInitializationException { bootstrapper.setFileReader(createFileReader(BOOTSTRAP_FILE_PATH, rawData)); BootstrapInfo info = bootstrapper.bootstrap(); - assertThat(info.getServers()).hasSize(1); - ServerInfo serverInfo = Iterables.getOnlyElement(info.getServers()); - assertThat(serverInfo.getTarget()).isEqualTo(SERVER_URI); - assertThat(serverInfo.getChannelCredentials()).isInstanceOf(InsecureChannelCredentials.class); - assertThat(info.getNode()).isEqualTo( + assertThat(info.servers()).hasSize(1); + ServerInfo serverInfo = Iterables.getOnlyElement(info.servers()); + assertThat(serverInfo.target()).isEqualTo(SERVER_URI); + assertThat(serverInfo.channelCredentials()).isInstanceOf(InsecureChannelCredentials.class); + assertThat(info.node()).isEqualTo( getNodeBuilder() .setId("ENVOY_NODE_ID") .setCluster("ENVOY_CLUSTER") @@ -158,17 +158,17 @@ public void parseBootstrap_multipleXdsServers() throws XdsInitializationExceptio bootstrapper.setFileReader(createFileReader(BOOTSTRAP_FILE_PATH, rawData)); BootstrapInfo info = bootstrapper.bootstrap(); - assertThat(info.getServers()).hasSize(2); - List serverInfoList = info.getServers(); - assertThat(serverInfoList.get(0).getTarget()) + assertThat(info.servers()).hasSize(2); + List serverInfoList = info.servers(); + assertThat(serverInfoList.get(0).target()) .isEqualTo("trafficdirector-foo.googleapis.com:443"); - assertThat(serverInfoList.get(0).getChannelCredentials()) + assertThat(serverInfoList.get(0).channelCredentials()) .isInstanceOf(TlsChannelCredentials.class); - assertThat(serverInfoList.get(1).getTarget()) + assertThat(serverInfoList.get(1).target()) .isEqualTo("trafficdirector-bar.googleapis.com:443"); - assertThat(serverInfoList.get(1).getChannelCredentials()) + assertThat(serverInfoList.get(1).channelCredentials()) .isInstanceOf(InsecureChannelCredentials.class); - assertThat(info.getNode()).isEqualTo( + assertThat(info.node()).isEqualTo( getNodeBuilder() .setId("ENVOY_NODE_ID") .setCluster("ENVOY_CLUSTER") @@ -208,11 +208,11 @@ public void parseBootstrap_IgnoreIrrelevantFields() throws XdsInitializationExce bootstrapper.setFileReader(createFileReader(BOOTSTRAP_FILE_PATH, rawData)); BootstrapInfo info = bootstrapper.bootstrap(); - assertThat(info.getServers()).hasSize(1); - ServerInfo serverInfo = Iterables.getOnlyElement(info.getServers()); - assertThat(serverInfo.getTarget()).isEqualTo(SERVER_URI); - assertThat(serverInfo.getChannelCredentials()).isInstanceOf(InsecureChannelCredentials.class); - assertThat(info.getNode()).isEqualTo( + assertThat(info.servers()).hasSize(1); + ServerInfo serverInfo = Iterables.getOnlyElement(info.servers()); + assertThat(serverInfo.target()).isEqualTo(SERVER_URI); + assertThat(serverInfo.channelCredentials()).isInstanceOf(InsecureChannelCredentials.class); + assertThat(info.node()).isEqualTo( getNodeBuilder() .setId("ENVOY_NODE_ID") .setCluster("ENVOY_CLUSTER") @@ -277,11 +277,11 @@ public void parseBootstrap_useFirstSupportedChannelCredentials() bootstrapper.setFileReader(createFileReader(BOOTSTRAP_FILE_PATH, rawData)); BootstrapInfo info = bootstrapper.bootstrap(); - assertThat(info.getServers()).hasSize(1); - ServerInfo serverInfo = Iterables.getOnlyElement(info.getServers()); - assertThat(serverInfo.getTarget()).isEqualTo(SERVER_URI); - assertThat(serverInfo.getChannelCredentials()).isInstanceOf(InsecureChannelCredentials.class); - assertThat(info.getNode()).isEqualTo(getNodeBuilder().build()); + assertThat(info.servers()).hasSize(1); + ServerInfo serverInfo = Iterables.getOnlyElement(info.servers()); + assertThat(serverInfo.target()).isEqualTo(SERVER_URI); + assertThat(serverInfo.channelCredentials()).isInstanceOf(InsecureChannelCredentials.class); + assertThat(info.node()).isEqualTo(getNodeBuilder().build()); } @Test @@ -379,17 +379,17 @@ public void parseBootstrap_certProviderInstances() throws XdsInitializationExcep bootstrapper.setFileReader(createFileReader(BOOTSTRAP_FILE_PATH, rawData)); BootstrapInfo info = bootstrapper.bootstrap(); - assertThat(info.getServers()).isEmpty(); - assertThat(info.getNode()).isEqualTo(getNodeBuilder().build()); - Map certProviders = info.getCertProviders(); + assertThat(info.servers()).isEmpty(); + assertThat(info.node()).isEqualTo(getNodeBuilder().build()); + Map certProviders = info.certProviders(); assertThat(certProviders).isNotNull(); Bootstrapper.CertificateProviderInfo gcpId = certProviders.get("gcp_id"); Bootstrapper.CertificateProviderInfo fileProvider = certProviders.get("file_provider"); - assertThat(gcpId.getPluginName()).isEqualTo("meshca"); - assertThat(gcpId.getConfig()).isInstanceOf(Map.class); - assertThat(fileProvider.getPluginName()).isEqualTo("file_watcher"); - assertThat(fileProvider.getConfig()).isInstanceOf(Map.class); - Map meshCaConfig = (Map)gcpId.getConfig(); + assertThat(gcpId.pluginName()).isEqualTo("meshca"); + assertThat(gcpId.config()).isInstanceOf(Map.class); + assertThat(fileProvider.pluginName()).isEqualTo("file_watcher"); + assertThat(fileProvider.config()).isInstanceOf(Map.class); + Map meshCaConfig = gcpId.config(); assertThat(meshCaConfig.get("key_size")).isEqualTo(2048); } @@ -552,7 +552,7 @@ public void parseBootstrap_grpcServerResourceId() throws XdsInitializationExcept bootstrapper.setFileReader(createFileReader(BOOTSTRAP_FILE_PATH, rawData)); BootstrapInfo info = bootstrapper.bootstrap(); - assertThat(info.getServerListenerResourceNameTemplate()).isEqualTo("grpc/serverx=%s"); + assertThat(info.serverListenerResourceNameTemplate()).isEqualTo("grpc/serverx=%s"); } @Test @@ -571,10 +571,10 @@ public void useV2ProtocolByDefault() throws XdsInitializationException { bootstrapper.setFileReader(createFileReader(BOOTSTRAP_FILE_PATH, rawData)); BootstrapInfo info = bootstrapper.bootstrap(); - ServerInfo serverInfo = Iterables.getOnlyElement(info.getServers()); - assertThat(serverInfo.getTarget()).isEqualTo(SERVER_URI); - assertThat(serverInfo.getChannelCredentials()).isInstanceOf(InsecureChannelCredentials.class); - assertThat(serverInfo.isUseProtocolV3()).isFalse(); + ServerInfo serverInfo = Iterables.getOnlyElement(info.servers()); + assertThat(serverInfo.target()).isEqualTo(SERVER_URI); + assertThat(serverInfo.channelCredentials()).isInstanceOf(InsecureChannelCredentials.class); + assertThat(serverInfo.useProtocolV3()).isFalse(); } @Test @@ -593,10 +593,10 @@ public void useV3ProtocolIfV3FeaturePresent() throws XdsInitializationException bootstrapper.setFileReader(createFileReader(BOOTSTRAP_FILE_PATH, rawData)); BootstrapInfo info = bootstrapper.bootstrap(); - ServerInfo serverInfo = Iterables.getOnlyElement(info.getServers()); - assertThat(serverInfo.getTarget()).isEqualTo(SERVER_URI); - assertThat(serverInfo.getChannelCredentials()).isInstanceOf(InsecureChannelCredentials.class); - assertThat(serverInfo.isUseProtocolV3()).isTrue(); + ServerInfo serverInfo = Iterables.getOnlyElement(info.servers()); + assertThat(serverInfo.target()).isEqualTo(SERVER_URI); + assertThat(serverInfo.channelCredentials()).isInstanceOf(InsecureChannelCredentials.class); + assertThat(serverInfo.useProtocolV3()).isTrue(); } @Test diff --git a/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java b/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java index a488175b835..5fa9e3da734 100644 --- a/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java +++ b/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java @@ -274,14 +274,14 @@ public void setUp() throws IOException { cleanupRule.register(InProcessChannelBuilder.forName(serverName).directExecutor().build()); Bootstrapper.BootstrapInfo bootstrapInfo = - new Bootstrapper.BootstrapInfo( - Arrays.asList( - new Bootstrapper.ServerInfo( - SERVER_URI, InsecureChannelCredentials.create(), useProtocolV3())), - EnvoyProtoData.Node.newBuilder().build(), - ImmutableMap.of("cert-instance-name", - new CertificateProviderInfo("file-watcher", ImmutableMap.of())), - null); + Bootstrapper.BootstrapInfo.builder() + .servers(Arrays.asList( + Bootstrapper.ServerInfo.create( + SERVER_URI, InsecureChannelCredentials.create(), useProtocolV3()))) + .node(EnvoyProtoData.Node.newBuilder().build()) + .certProviders(ImmutableMap.of("cert-instance-name", + CertificateProviderInfo.create("file-watcher", ImmutableMap.of()))) + .build(); xdsClient = new ClientXdsClient( channel, diff --git a/xds/src/test/java/io/grpc/xds/CommonBootstrapperTestUtils.java b/xds/src/test/java/io/grpc/xds/CommonBootstrapperTestUtils.java index d7c6dbc787d..73632c8addb 100644 --- a/xds/src/test/java/io/grpc/xds/CommonBootstrapperTestUtils.java +++ b/xds/src/test/java/io/grpc/xds/CommonBootstrapperTestUtils.java @@ -19,6 +19,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import io.grpc.internal.JsonParser; +import io.grpc.xds.Bootstrapper.ServerInfo; import io.grpc.xds.internal.sds.CommonTlsContextTestsUtil; import java.io.IOException; import java.util.HashMap; @@ -57,19 +58,20 @@ public class CommonBootstrapperTestUtils { public static Bootstrapper.BootstrapInfo getTestBootstrapInfo() { try { Bootstrapper.CertificateProviderInfo gcpId = - new Bootstrapper.CertificateProviderInfo( + Bootstrapper.CertificateProviderInfo.create( "testca", (Map) JsonParser.parse(MESHCA_CONFIG)); Bootstrapper.CertificateProviderInfo fileProvider = - new Bootstrapper.CertificateProviderInfo( + Bootstrapper.CertificateProviderInfo.create( "file_watcher", (Map) JsonParser.parse(FILE_WATCHER_CONFIG)); Map certProviders = ImmutableMap.of("gcp_id", gcpId, "file_provider", fileProvider); Bootstrapper.BootstrapInfo bootstrapInfo = - new Bootstrapper.BootstrapInfo( - ImmutableList.of(), - EnvoyProtoData.Node.newBuilder().build(), - certProviders, - "grpc/server"); + Bootstrapper.BootstrapInfo.builder() + .servers(ImmutableList.of()) + .node(EnvoyProtoData.Node.newBuilder().build()) + .certProviders(certProviders) + .serverListenerResourceNameTemplate("grpc/server") + .build(); return bootstrapInfo; } catch (IOException e) { throw new AssertionError(e); @@ -113,7 +115,7 @@ public static Bootstrapper.BootstrapInfo buildBootstrapInfo( config.put("private_key_file", privateKey1); config.put("ca_certificate_file", trustCa1); Bootstrapper.CertificateProviderInfo certificateProviderInfo = - new Bootstrapper.CertificateProviderInfo("file_watcher", config); + Bootstrapper.CertificateProviderInfo.create("file_watcher", config); HashMap certProviders = new HashMap<>(); certProviders.put(certInstanceName1, certificateProviderInfo); @@ -123,10 +125,13 @@ public static Bootstrapper.BootstrapInfo buildBootstrapInfo( config.put("private_key_file", privateKey2); config.put("ca_certificate_file", trustCa2); certificateProviderInfo = - new Bootstrapper.CertificateProviderInfo("file_watcher", config); + Bootstrapper.CertificateProviderInfo.create("file_watcher", config); certProviders.put(certInstanceName2, certificateProviderInfo); } - return new Bootstrapper.BootstrapInfo(null, EnvoyProtoData.Node.newBuilder().build(), - certProviders, null); + return Bootstrapper.BootstrapInfo.builder() + .servers(ImmutableList.of()) + .node(EnvoyProtoData.Node.newBuilder().build()) + .certProviders(certProviders) + .build(); } } diff --git a/xds/src/test/java/io/grpc/xds/CsdsServiceTest.java b/xds/src/test/java/io/grpc/xds/CsdsServiceTest.java index 8a6e36d635c..9a50e2e0599 100644 --- a/xds/src/test/java/io/grpc/xds/CsdsServiceTest.java +++ b/xds/src/test/java/io/grpc/xds/CsdsServiceTest.java @@ -84,12 +84,12 @@ public class CsdsServiceTest { private static final XdsClient XDS_CLIENT_NO_RESOURCES = new XdsClient() { @Override Bootstrapper.BootstrapInfo getBootstrapInfo() { - return new Bootstrapper.BootstrapInfo( - Arrays.asList( - new Bootstrapper.ServerInfo(SERVER_URI, InsecureChannelCredentials.create(), false)), - BOOTSTRAP_NODE, - null, - null); + return Bootstrapper.BootstrapInfo.builder() + .servers(Arrays.asList( + Bootstrapper.ServerInfo.create( + SERVER_URI, InsecureChannelCredentials.create(), false))) + .node(BOOTSTRAP_NODE) + .build(); } @Override @@ -695,10 +695,12 @@ public void getClientConfigForXdsClient_subscribedResourcesToPerXdsConfig() { ClientConfig clientConfig = CsdsService.getClientConfigForXdsClient(new XdsClient() { @Override Bootstrapper.BootstrapInfo getBootstrapInfo() { - return new Bootstrapper.BootstrapInfo(Arrays.asList( - new Bootstrapper.ServerInfo( - SERVER_URI, InsecureChannelCredentials.create(), false)), - BOOTSTRAP_NODE, null,null); + return Bootstrapper.BootstrapInfo.builder() + .servers(Arrays.asList( + Bootstrapper.ServerInfo.create( + SERVER_URI, InsecureChannelCredentials.create(), false))) + .node(BOOTSTRAP_NODE) + .build(); } @Override diff --git a/xds/src/test/java/io/grpc/xds/SharedXdsClientPoolProviderTest.java b/xds/src/test/java/io/grpc/xds/SharedXdsClientPoolProviderTest.java index 5167dfae9ba..6a3cba4ac35 100644 --- a/xds/src/test/java/io/grpc/xds/SharedXdsClientPoolProviderTest.java +++ b/xds/src/test/java/io/grpc/xds/SharedXdsClientPoolProviderTest.java @@ -57,7 +57,7 @@ public class SharedXdsClientPoolProviderTest { @Test public void noServer() throws XdsInitializationException { BootstrapInfo bootstrapInfo = - new BootstrapInfo(Collections.emptyList(), node, null, null); + BootstrapInfo.builder().servers(Collections.emptyList()).node(node).build(); when(bootstrapper.bootstrap()).thenReturn(bootstrapInfo); SharedXdsClientPoolProvider provider = new SharedXdsClientPoolProvider(bootstrapper); thrown.expect(XdsInitializationException.class); @@ -68,9 +68,9 @@ public void noServer() throws XdsInitializationException { @Test public void sharedXdsClientObjectPool() throws XdsInitializationException { - ServerInfo server = new ServerInfo(SERVER_URI, InsecureChannelCredentials.create(), false); + ServerInfo server = ServerInfo.create(SERVER_URI, InsecureChannelCredentials.create(), false); BootstrapInfo bootstrapInfo = - new BootstrapInfo(Collections.singletonList(server), node, null, null); + BootstrapInfo.builder().servers(Collections.singletonList(server)).node(node).build(); when(bootstrapper.bootstrap()).thenReturn(bootstrapInfo); SharedXdsClientPoolProvider provider = new SharedXdsClientPoolProvider(bootstrapper); @@ -85,9 +85,9 @@ public void sharedXdsClientObjectPool() throws XdsInitializationException { @Test public void refCountedXdsClientObjectPool_delayedCreation() { - ServerInfo server = new ServerInfo(SERVER_URI, InsecureChannelCredentials.create(), false); + ServerInfo server = ServerInfo.create(SERVER_URI, InsecureChannelCredentials.create(), false); BootstrapInfo bootstrapInfo = - new BootstrapInfo(Collections.singletonList(server), node, null, null); + BootstrapInfo.builder().servers(Collections.singletonList(server)).node(node).build(); RefCountedXdsClientObjectPool xdsClientPool = new RefCountedXdsClientObjectPool(bootstrapInfo); assertThat(xdsClientPool.getXdsClientForTest()).isNull(); assertThat(xdsClientPool.getChannelForTest()).isNull(); @@ -98,9 +98,9 @@ public void refCountedXdsClientObjectPool_delayedCreation() { @Test public void refCountedXdsClientObjectPool_refCounted() { - ServerInfo server = new ServerInfo(SERVER_URI, InsecureChannelCredentials.create(), false); + ServerInfo server = ServerInfo.create(SERVER_URI, InsecureChannelCredentials.create(), false); BootstrapInfo bootstrapInfo = - new BootstrapInfo(Collections.singletonList(server), node, null, null); + BootstrapInfo.builder().servers(Collections.singletonList(server)).node(node).build(); RefCountedXdsClientObjectPool xdsClientPool = new RefCountedXdsClientObjectPool(bootstrapInfo); // getObject once XdsClient xdsClient = xdsClientPool.getObject(); @@ -118,9 +118,9 @@ public void refCountedXdsClientObjectPool_refCounted() { @Test public void refCountedXdsClientObjectPool_getObjectCreatesNewInstanceIfAlreadyShutdown() { - ServerInfo server = new ServerInfo(SERVER_URI, InsecureChannelCredentials.create(), false); + ServerInfo server = ServerInfo.create(SERVER_URI, InsecureChannelCredentials.create(), false); BootstrapInfo bootstrapInfo = - new BootstrapInfo(Collections.singletonList(server), node, null, null); + BootstrapInfo.builder().servers(Collections.singletonList(server)).node(node).build(); RefCountedXdsClientObjectPool xdsClientPool = new RefCountedXdsClientObjectPool(bootstrapInfo); XdsClient xdsClient1 = xdsClientPool.getObject(); ManagedChannel channel1 = xdsClientPool.getChannelForTest(); diff --git a/xds/src/test/java/io/grpc/xds/XdsNameResolverProviderTest.java b/xds/src/test/java/io/grpc/xds/XdsNameResolverProviderTest.java index 32850b441d7..95e3f2f997f 100644 --- a/xds/src/test/java/io/grpc/xds/XdsNameResolverProviderTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsNameResolverProviderTest.java @@ -164,7 +164,7 @@ public void newProvider_overrideBootstrap() { .newNameResolver(URI.create("no-scheme:///localhost"), args); resolver.start(mock(NameResolver.Listener2.class)); assertThat(resolver).isInstanceOf(XdsNameResolver.class); - assertThat(((XdsNameResolver)resolver).getXdsClient().getBootstrapInfo().getNode().getId()) + assertThat(((XdsNameResolver)resolver).getXdsClient().getBootstrapInfo().node().getId()) .isEqualTo("ENVOY_NODE_ID"); resolver.shutdown(); registry.deregister(provider); diff --git a/xds/src/test/java/io/grpc/xds/XdsServerTestHelper.java b/xds/src/test/java/io/grpc/xds/XdsServerTestHelper.java index ffe4a72f522..66b3d00a84b 100644 --- a/xds/src/test/java/io/grpc/xds/XdsServerTestHelper.java +++ b/xds/src/test/java/io/grpc/xds/XdsServerTestHelper.java @@ -49,12 +49,13 @@ public class XdsServerTestHelper { private static final EnvoyProtoData.Node BOOTSTRAP_NODE = EnvoyProtoData.Node.newBuilder().setId(NODE_ID).build(); static final Bootstrapper.BootstrapInfo BOOTSTRAP_INFO = - new Bootstrapper.BootstrapInfo( - Arrays.asList( - new Bootstrapper.ServerInfo(SERVER_URI, InsecureChannelCredentials.create(), true)), - BOOTSTRAP_NODE, - null, - "grpc/server?udpa.resource.listening_address=%s"); + Bootstrapper.BootstrapInfo.builder() + .servers(Arrays.asList( + Bootstrapper.ServerInfo.create( + SERVER_URI, InsecureChannelCredentials.create(), true))) + .node(BOOTSTRAP_NODE) + .serverListenerResourceNameTemplate("grpc/server?udpa.resource.listening_address=%s") + .build(); static void generateListenerUpdate(FakeXdsClient xdsClient, EnvoyServerProtoData.DownstreamTlsContext tlsContext, diff --git a/xds/src/test/java/io/grpc/xds/XdsServerWrapperTest.java b/xds/src/test/java/io/grpc/xds/XdsServerWrapperTest.java index f2b6e9e4790..e68d0f5175c 100644 --- a/xds/src/test/java/io/grpc/xds/XdsServerWrapperTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsServerWrapperTest.java @@ -118,24 +118,23 @@ selectorManager, new FakeXdsClientPoolFactory(xdsClient), @Test public void testBootstrap_notV3() throws Exception { Bootstrapper.BootstrapInfo b = - new Bootstrapper.BootstrapInfo( - Arrays.asList( - new Bootstrapper.ServerInfo("uri", InsecureChannelCredentials.create(), false)), - EnvoyProtoData.Node.newBuilder().setId("id").build(), - null, - "grpc/server?udpa.resource.listening_address=%s"); + Bootstrapper.BootstrapInfo.builder() + .servers(Arrays.asList( + Bootstrapper.ServerInfo.create("uri", InsecureChannelCredentials.create(), false))) + .node(EnvoyProtoData.Node.newBuilder().setId("id").build()) + .serverListenerResourceNameTemplate("grpc/server?udpa.resource.listening_address=%s") + .build(); verifyBootstrapFail(b); } @Test public void testBootstrap_noTemplate() throws Exception { Bootstrapper.BootstrapInfo b = - new Bootstrapper.BootstrapInfo( - Arrays.asList( - new Bootstrapper.ServerInfo("uri", InsecureChannelCredentials.create(), true)), - EnvoyProtoData.Node.newBuilder().setId("id").build(), - null, - null); + Bootstrapper.BootstrapInfo.builder() + .servers(Arrays.asList( + Bootstrapper.ServerInfo.create("uri", InsecureChannelCredentials.create(), true))) + .node(EnvoyProtoData.Node.newBuilder().setId("id").build()) + .build(); verifyBootstrapFail(b); } diff --git a/xds/src/test/java/io/grpc/xds/internal/certprovider/CertProviderClientSslContextProviderTest.java b/xds/src/test/java/io/grpc/xds/internal/certprovider/CertProviderClientSslContextProviderTest.java index 1eed5488aa0..111a44e3224 100644 --- a/xds/src/test/java/io/grpc/xds/internal/certprovider/CertProviderClientSslContextProviderTest.java +++ b/xds/src/test/java/io/grpc/xds/internal/certprovider/CertProviderClientSslContextProviderTest.java @@ -83,8 +83,8 @@ private CertProviderClientSslContextProvider getSslContextProvider( staticCertValidationContext); return certProviderClientSslContextProviderFactory.getProvider( upstreamTlsContext, - bootstrapInfo.getNode().toEnvoyProtoNode(), - bootstrapInfo.getCertProviders()); + bootstrapInfo.node().toEnvoyProtoNode(), + bootstrapInfo.certProviders()); } /** Helper method to build CertProviderClientSslContextProvider. */ @@ -104,8 +104,8 @@ private CertProviderClientSslContextProvider getNewSslContextProvider( staticCertValidationContext); return certProviderClientSslContextProviderFactory.getProvider( upstreamTlsContext, - bootstrapInfo.getNode().toEnvoyProtoNode(), - bootstrapInfo.getCertProviders()); + bootstrapInfo.node().toEnvoyProtoNode(), + bootstrapInfo.certProviders()); } @Test diff --git a/xds/src/test/java/io/grpc/xds/internal/certprovider/CertProviderServerSslContextProviderTest.java b/xds/src/test/java/io/grpc/xds/internal/certprovider/CertProviderServerSslContextProviderTest.java index 783ce2b11f7..7cd3cd2a793 100644 --- a/xds/src/test/java/io/grpc/xds/internal/certprovider/CertProviderServerSslContextProviderTest.java +++ b/xds/src/test/java/io/grpc/xds/internal/certprovider/CertProviderServerSslContextProviderTest.java @@ -79,8 +79,8 @@ private CertProviderServerSslContextProvider getSslContextProvider( requireClientCert); return certProviderServerSslContextProviderFactory.getProvider( downstreamTlsContext, - bootstrapInfo.getNode().toEnvoyProtoNode(), - bootstrapInfo.getCertProviders()); + bootstrapInfo.node().toEnvoyProtoNode(), + bootstrapInfo.certProviders()); } /** Helper method to build CertProviderServerSslContextProvider. */ @@ -102,8 +102,8 @@ private CertProviderServerSslContextProvider getNewSslContextProvider( requireClientCert); return certProviderServerSslContextProviderFactory.getProvider( downstreamTlsContext, - bootstrapInfo.getNode().toEnvoyProtoNode(), - bootstrapInfo.getCertProviders()); + bootstrapInfo.node().toEnvoyProtoNode(), + bootstrapInfo.certProviders()); } From 0376de15b8ba2dfebc24bc8fa0e60370d3eae003 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Thu, 14 Oct 2021 20:25:06 -0400 Subject: [PATCH 67/76] Fix AbstractManagedChannelImplBuilder#maxInboundMessageSize(int) ABI (#8607) In refactoring described in #7211, the implementation of #maxInboundMessageSize(int) (and its corresponding field) were pulled down from internal AbstractManagedChannelImplBuilder to concrete classes that actually enforce this setting. For the same reason, it wasn't ported to ManagedChannelImplBuilder (the #delegate()). Then AbstractManagedChannelImplBuilder was brought back to fix ABI backward compatibility, and temporarily turned into a ForwardingChannelBuilder, ref PR #7564. Eventually it will be deleted, after a period with "bridge" ABI solution introduced in #7834. However, restoring AbstractManagedChannelImplBuilder unintentionally made ABI of pre-refactoring builds expect it to be a method of AbstractManagedChannelImplBuilder, and not concrete classes, ref #8313. The end goal is to keep #maxInboundMessageSize(int) only in concrete classes that enforce it. To fix method's ABI, we temporary reintroduce it to the original layer it was removed from: AbstractManagedChannelImplBuilder. This class' only intention is to provide short-term ABI compatibility. Once we move forward with dropping the ABI, both fixes are no longer necessary, and both will perish with removing AbstractManagedChannelImplBuilder. --- .../AbstractManagedChannelImplBuilder.java | 35 ++++++++++++++++++- ...AbstractManagedChannelImplBuilderTest.java | 14 ++++++-- .../io/grpc/netty/NettyChannelBuilder.java | 1 - .../io/grpc/okhttp/OkHttpChannelBuilder.java | 1 - 4 files changed, 46 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/io/grpc/internal/AbstractManagedChannelImplBuilder.java b/core/src/main/java/io/grpc/internal/AbstractManagedChannelImplBuilder.java index 98bbfcc7b1e..dd8328ee09c 100644 --- a/core/src/main/java/io/grpc/internal/AbstractManagedChannelImplBuilder.java +++ b/core/src/main/java/io/grpc/internal/AbstractManagedChannelImplBuilder.java @@ -17,6 +17,7 @@ package io.grpc.internal; import com.google.common.base.MoreObjects; +import com.google.common.base.Preconditions; import com.google.errorprone.annotations.DoNotCall; import io.grpc.BinaryLog; import io.grpc.ClientInterceptor; @@ -42,6 +43,14 @@ public abstract class AbstractManagedChannelImplBuilder > extends ManagedChannelBuilder { + /** + * Added for ABI compatibility. + * + *

See details in {@link #maxInboundMessageSize(int)}. + * TODO(sergiitk): move back to concrete classes as a private field, when this class is removed. + */ + protected int maxInboundMessageSize = GrpcUtil.DEFAULT_MAX_MESSAGE_SIZE; + /** * The default constructor. */ @@ -161,7 +170,31 @@ public T idleTimeout(long value, TimeUnit unit) { @Override public T maxInboundMessageSize(int max) { - delegate().maxInboundMessageSize(max); + /* + Why this method is not delegating, as the rest of the methods? + + In refactoring described in #7211, the implementation of #maxInboundMessageSize(int) + (and its corresponding field) was pulled down from internal AbstractManagedChannelImplBuilder + to concrete classes that actually enforce this setting. For the same reason, it wasn't ported + to ManagedChannelImplBuilder (the #delegate()). + + Then AbstractManagedChannelImplBuilder was brought back to fix ABI backward compatibility, + and temporarily turned into a ForwardingChannelBuilder, ref PR #7564. Eventually it will + be deleted, after a period with "bridge" ABI solution introduced in #7834. + + However, restoring AbstractManagedChannelImplBuilder unintentionally made ABI of + #maxInboundMessageSize(int) implemented by the concrete classes backward incompatible: + pre-refactoring builds expect it to be a method of AbstractManagedChannelImplBuilder, + and not concrete classes, ref #8313. + + The end goal is to keep #maxInboundMessageSize(int) only in concrete classes that enforce it. + To fix method's ABI, we temporary reintroduce it to the original layer it was removed from: + AbstractManagedChannelImplBuilder. This class' only intention is to provide short-term + ABI compatibility. Once we move forward with dropping the ABI, both fixes are no longer + necessary, and both will perish with removing AbstractManagedChannelImplBuilder. + */ + Preconditions.checkArgument(max >= 0, "negative max"); + maxInboundMessageSize = max; return thisT(); } diff --git a/core/src/test/java/io/grpc/internal/AbstractManagedChannelImplBuilderTest.java b/core/src/test/java/io/grpc/internal/AbstractManagedChannelImplBuilderTest.java index 8643165eec6..81c50b19fbf 100644 --- a/core/src/test/java/io/grpc/internal/AbstractManagedChannelImplBuilderTest.java +++ b/core/src/test/java/io/grpc/internal/AbstractManagedChannelImplBuilderTest.java @@ -21,12 +21,12 @@ import static org.mockito.Mockito.mock; import com.google.common.base.Defaults; +import com.google.common.collect.ImmutableSet; import io.grpc.ForwardingTestUtil; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import java.lang.reflect.Method; import java.lang.reflect.Modifier; -import java.util.Collections; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -53,7 +53,8 @@ public void allMethodsForwarded() throws Exception { ManagedChannelBuilder.class, mockDelegate, testChannelBuilder, - Collections.emptyList(), + // maxInboundMessageSize is the only method that shouldn't forward. + ImmutableSet.of(ManagedChannelBuilder.class.getMethod("maxInboundMessageSize", int.class)), new ForwardingTestUtil.ArgumentProvider() { @Override public Object get(Method method, int argPos, Class clazz) { @@ -66,6 +67,15 @@ public Object get(Method method, int argPos, Class clazz) { }); } + @Test + public void testMaxInboundMessageSize() { + assertThat(testChannelBuilder.maxInboundMessageSize) + .isEqualTo(GrpcUtil.DEFAULT_MAX_MESSAGE_SIZE); + + testChannelBuilder.maxInboundMessageSize(42); + assertThat(testChannelBuilder.maxInboundMessageSize).isEqualTo(42); + } + @Test public void allBuilderMethodsReturnThis() throws Exception { for (Method method : ManagedChannelBuilder.class.getDeclaredMethods()) { diff --git a/netty/src/main/java/io/grpc/netty/NettyChannelBuilder.java b/netty/src/main/java/io/grpc/netty/NettyChannelBuilder.java index 809c94f12ce..17801afa382 100644 --- a/netty/src/main/java/io/grpc/netty/NettyChannelBuilder.java +++ b/netty/src/main/java/io/grpc/netty/NettyChannelBuilder.java @@ -99,7 +99,6 @@ public final class NettyChannelBuilder extends private ObjectPool eventLoopGroupPool = DEFAULT_EVENT_LOOP_GROUP_POOL; private boolean autoFlowControl = DEFAULT_AUTO_FLOW_CONTROL; private int flowControlWindow = DEFAULT_FLOW_CONTROL_WINDOW; - private int maxInboundMessageSize = GrpcUtil.DEFAULT_MAX_MESSAGE_SIZE; private int maxHeaderListSize = GrpcUtil.DEFAULT_MAX_HEADER_LIST_SIZE; private long keepAliveTimeNanos = KEEPALIVE_TIME_NANOS_DISABLED; private long keepAliveTimeoutNanos = DEFAULT_KEEPALIVE_TIMEOUT_NANOS; diff --git a/okhttp/src/main/java/io/grpc/okhttp/OkHttpChannelBuilder.java b/okhttp/src/main/java/io/grpc/okhttp/OkHttpChannelBuilder.java index f7d0d973802..af5ebe2886c 100644 --- a/okhttp/src/main/java/io/grpc/okhttp/OkHttpChannelBuilder.java +++ b/okhttp/src/main/java/io/grpc/okhttp/OkHttpChannelBuilder.java @@ -174,7 +174,6 @@ public static OkHttpChannelBuilder forTarget(String target, ChannelCredentials c private long keepAliveTimeoutNanos = DEFAULT_KEEPALIVE_TIMEOUT_NANOS; private int flowControlWindow = DEFAULT_FLOW_CONTROL_WINDOW; private boolean keepAliveWithoutCalls; - private int maxInboundMessageSize = GrpcUtil.DEFAULT_MAX_MESSAGE_SIZE; private int maxInboundMetadataSize = Integer.MAX_VALUE; /** From e9b0c2e85178e459510c501cae0b28401a1e306c Mon Sep 17 00:00:00 2001 From: ZhenLian Date: Fri, 15 Oct 2021 14:42:14 -0700 Subject: [PATCH 68/76] Make CertificateUtils to use other key algorithms (#8609) --- .../java/io/grpc/util/CertificateUtils.java | 18 +++++++++++++++--- .../io/grpc/util/CertificateUtilsTest.java | 10 ++++++++++ testing/src/main/resources/certs/README | 5 +++++ testing/src/main/resources/certs/ecdsa.key | 5 +++++ 4 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 testing/src/main/resources/certs/ecdsa.key diff --git a/core/src/main/java/io/grpc/util/CertificateUtils.java b/core/src/main/java/io/grpc/util/CertificateUtils.java index 980862d3836..a886ff1f6e1 100644 --- a/core/src/main/java/io/grpc/util/CertificateUtils.java +++ b/core/src/main/java/io/grpc/util/CertificateUtils.java @@ -56,7 +56,8 @@ public static X509Certificate[] getX509Certificates(InputStream inputStream) /** * Generates a {@link PrivateKey} from a PEM file. - * The key should be PKCS #8 formatted. + * The key should be PKCS #8 formatted. The key algorithm should be "RSA", "DiffieHellman", + * "DSA", or "EC". * The PEM file should contain one item in Base64 encoding, with plain-text headers and footers * (e.g. -----BEGIN PRIVATE KEY----- and -----END PRIVATE KEY-----). * @@ -80,9 +81,20 @@ public static PrivateKey getPrivateKey(InputStream inputStream) keyContent.append(line); } byte[] decodedKeyBytes = BaseEncoding.base64().decode(keyContent.toString()); - KeyFactory keyFactory = KeyFactory.getInstance("RSA"); PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(decodedKeyBytes); - return keyFactory.generatePrivate(keySpec); + try { + return KeyFactory.getInstance("RSA").generatePrivate(keySpec); + } catch (InvalidKeySpecException ignore) { + try { + return KeyFactory.getInstance("DSA").generatePrivate(keySpec); + } catch (InvalidKeySpecException ignore2) { + try { + return KeyFactory.getInstance("EC").generatePrivate(keySpec); + } catch (InvalidKeySpecException e) { + throw new InvalidKeySpecException("Neither RSA, DSA nor EC worked", e); + } + } + } } } diff --git a/core/src/test/java/io/grpc/util/CertificateUtilsTest.java b/core/src/test/java/io/grpc/util/CertificateUtilsTest.java index 5fa93d5b85b..35923994483 100644 --- a/core/src/test/java/io/grpc/util/CertificateUtilsTest.java +++ b/core/src/test/java/io/grpc/util/CertificateUtilsTest.java @@ -43,6 +43,7 @@ public class CertificateUtilsTest { public static final String BAD_PEM_CONTENT = "----BEGIN PRIVATE KEY-----\n" + "MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDvdzKDTYvRgjBO\n" + "-----END PRIVATE KEY-----"; + public static final String ECDSA_KEY_FILE = "ecdsa.key"; @Test public void readPemCertFile() throws CertificateException, IOException { @@ -101,4 +102,13 @@ public void readBadContentKeyFile() { } } + @Test + public void readEcdsaKeyFile() throws Exception { + InputStream in = TestUtils.class.getResourceAsStream("/certs/" + ECDSA_KEY_FILE); + PrivateKey key = CertificateUtils.getPrivateKey(in); + // Checks some information on the test key. + assertThat(key.getAlgorithm()).isEqualTo("EC"); + assertThat(key.getFormat()).isEqualTo("PKCS#8"); + } + } diff --git a/testing/src/main/resources/certs/README b/testing/src/main/resources/certs/README index ab0d851a18b..1fa6b733950 100644 --- a/testing/src/main/resources/certs/README +++ b/testing/src/main/resources/certs/README @@ -62,6 +62,11 @@ common name which is set to *.test.google.com. $ openssl x509 -req -CA ca.pem -CAkey ca.key -CAcreateserial -in server1.csr \ -out server1.pem -extensions req_ext -extfile server1-openssl.cnf -days 3650 +ecdsa.key is used to test keys with algorithm other than RSA: +---------------------------------------------------------------------------- +$ openssl ecparam -name secp256k1 -genkey -noout -out ecdsa.pem +$ openssl pkcs8 -topk8 -in ecdsa.pem -out ecdsa.key -nocrypt + Clean up: --------- $ rm *.rsa diff --git a/testing/src/main/resources/certs/ecdsa.key b/testing/src/main/resources/certs/ecdsa.key new file mode 100644 index 00000000000..62b42fd038f --- /dev/null +++ b/testing/src/main/resources/certs/ecdsa.key @@ -0,0 +1,5 @@ +-----BEGIN PRIVATE KEY----- +MIGEAgEAMBAGByqGSM49AgEGBSuBBAAKBG0wawIBAQQgS0hDYghCuDnBobsToMW6 +vGqwulbAGUX8Oku4ysWMa4qhRANCAAThAMij1tkl4/7RQpZg3w7z1McGSS9q01+4 +3bDcF/Ge2gATx/SNYT5TqaSx7Rka/sJAGaX47ExWLca4gz9KGHih +-----END PRIVATE KEY----- From 1f90e0e28d5628195cb1f861b73e45ed003f2973 Mon Sep 17 00:00:00 2001 From: ZHANG Dapeng Date: Mon, 18 Oct 2021 16:19:34 -0700 Subject: [PATCH 69/76] xds: add and parse new bootstrap fields for federation (#8608) Made changes as per "Bootstrap File Changes" section in go/grpc-xds-federation and implemented bootstrap file parsing logic for the change. --- .../main/java/io/grpc/xds/Bootstrapper.java | 93 +++++++++++- .../java/io/grpc/xds/BootstrapperImpl.java | 135 ++++++++++++------ .../io/grpc/xds/BootstrapperImplTest.java | 96 +++++++++++++ 3 files changed, 283 insertions(+), 41 deletions(-) diff --git a/xds/src/main/java/io/grpc/xds/Bootstrapper.java b/xds/src/main/java/io/grpc/xds/Bootstrapper.java index e1246701295..862f8691080 100644 --- a/xds/src/main/java/io/grpc/xds/Bootstrapper.java +++ b/xds/src/main/java/io/grpc/xds/Bootstrapper.java @@ -83,6 +83,38 @@ public static CertificateProviderInfo create(String pluginName, Map c } } + @AutoValue + abstract static class AuthorityInfo { + + /** + * A template for the name of the Listener resource to subscribe to for a gRPC client + * channel. Used only when the channel is created using an "xds:" URI with this authority + * name. + * + *

The token "%s", if present in this string, will be replaced with %-encoded + * service authority (i.e., the path part of the target URI used to create the gRPC channel). + * + *

Return value must start with {@code "xdstp:///"}. + */ + abstract String clientListenerResourceNameTemplate(); + + /** + * Ordered list of xDS servers to contact for this authority. + * + *

If the same server is listed in multiple authorities, the entries will be de-duped (i.e., + * resources for both authorities will be fetched on the same ADS stream). + * + *

If empty, the top-level server list {@link BootstrapInfo#servers()} will be used. + */ + abstract ImmutableList xdsServers(); + + static AuthorityInfo create( + String clientListenerResourceNameTemplate, List xdsServers) { + return new AutoValue_Bootstrapper_AuthorityInfo( + clientListenerResourceNameTemplate, ImmutableList.copyOf(xdsServers)); + } + } + /** * Data class containing the results of reading bootstrap. */ @@ -99,17 +131,71 @@ public abstract static class BootstrapInfo { @Nullable public abstract ImmutableMap certProviders(); + /** + * A template for the name of the Listener resource to subscribe to for a gRPC server. + * + *

If starts with "xdstp:", will be interpreted as a new-style name, in which case the + * authority of the URI will be used to select the relevant configuration in the + * "authorities" map. The token "%s", if present in this string, will be replaced with + * the IP and port on which the server is listening. If the template starts with "xdstp:", + * the replaced string will be %-encoded. + * + *

There is no default; if unset, xDS-based server creation fails. + */ @Nullable public abstract String serverListenerResourceNameTemplate(); + /** + * A template for the name of the Listener resource to subscribe to for a gRPC client channel. + * Used only when the channel is created with an "xds:" URI with no authority. + * + *

If starts with "xdstp:", will be interpreted as a new-style name, in which case the + * authority of the URI will be used to select the relevant configuration in the "authorities" + * map. + * + *

The token "%s", if present in this string, will be replaced with the service authority + * (i.e., the path part of the target URI used to create the gRPC channel). If the template + * starts with "xdstp:", the replaced string will be %-encoded. + * + *

Defaults to {@code "%s"}. + */ + abstract String clientDefaultListenerResourceNameTemplate(); + + /** + * A map of authority name to corresponding configuration. + * + *

This is used in the following cases: + * + *

    + *
  • A gRPC client channel is created using an "xds:" URI that includes an + * authority.
    • + * + *
  • A gRPC client channel is created using an "xds:" URI with no authority, + * but the "client_default_listener_resource_name_template" field above turns it into an + * "xdstp:" URI.
    • + * + *
  • A gRPC server is created and the "server_listener_resource_name_template" field is an + * "xdstp:" URI.
    • + *
+ * + *

In any of those cases, it is an error if the specified authority is not present in this + * map. + * + *

Defaults to an empty map. + */ + abstract ImmutableMap authorities(); + @VisibleForTesting static Builder builder() { - return new AutoValue_Bootstrapper_BootstrapInfo.Builder(); + return new AutoValue_Bootstrapper_BootstrapInfo.Builder() + .clientDefaultListenerResourceNameTemplate("%s") + .authorities(ImmutableMap.of()); } @AutoValue.Builder @VisibleForTesting abstract static class Builder { + abstract Builder servers(List servers); abstract Builder node(Node node); @@ -119,6 +205,11 @@ abstract static class Builder { abstract Builder serverListenerResourceNameTemplate( @Nullable String serverListenerResourceNameTemplate); + abstract Builder clientDefaultListenerResourceNameTemplate( + String clientDefaultListenerResourceNameTemplate); + + abstract Builder authorities(Map authorities); + abstract BootstrapInfo build(); } } diff --git a/xds/src/main/java/io/grpc/xds/BootstrapperImpl.java b/xds/src/main/java/io/grpc/xds/BootstrapperImpl.java index 0a044da5290..200ded3c1c4 100644 --- a/xds/src/main/java/io/grpc/xds/BootstrapperImpl.java +++ b/xds/src/main/java/io/grpc/xds/BootstrapperImpl.java @@ -17,6 +17,8 @@ package io.grpc.xds; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import io.grpc.ChannelCredentials; import io.grpc.InsecureChannelCredentials; import io.grpc.Internal; @@ -33,7 +35,6 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -121,41 +122,14 @@ public BootstrapInfo bootstrap() throws XdsInitializationException { @Override BootstrapInfo bootstrap(Map rawData) throws XdsInitializationException { - List servers = new ArrayList<>(); + BootstrapInfo.Builder builder = BootstrapInfo.builder(); + List rawServerConfigs = JsonUtil.getList(rawData, "xds_servers"); if (rawServerConfigs == null) { throw new XdsInitializationException("Invalid bootstrap: 'xds_servers' does not exist."); } - logger.log(XdsLogLevel.INFO, "Configured with {0} xDS servers", rawServerConfigs.size()); - // TODO(chengyuanzhang): require at least one server URI. - List> serverConfigList = JsonUtil.checkObjectList(rawServerConfigs); - for (Map serverConfig : serverConfigList) { - String serverUri = JsonUtil.getString(serverConfig, "server_uri"); - if (serverUri == null) { - throw new XdsInitializationException("Invalid bootstrap: missing 'server_uri'"); - } - logger.log(XdsLogLevel.INFO, "xDS server URI: {0}", serverUri); - - List rawChannelCredsList = JsonUtil.getList(serverConfig, "channel_creds"); - if (rawChannelCredsList == null || rawChannelCredsList.isEmpty()) { - throw new XdsInitializationException( - "Invalid bootstrap: server " + serverUri + " 'channel_creds' required"); - } - ChannelCredentials channelCredentials = - parseChannelCredentials(JsonUtil.checkObjectList(rawChannelCredsList), serverUri); - if (channelCredentials == null) { - throw new XdsInitializationException( - "Server " + serverUri + ": no supported channel credentials found"); - } - - boolean useProtocolV3 = false; - List serverFeatures = JsonUtil.getListOfStrings(serverConfig, "server_features"); - if (serverFeatures != null) { - logger.log(XdsLogLevel.INFO, "Server features: {0}", serverFeatures); - useProtocolV3 = serverFeatures.contains(XDS_V3_SERVER_FEATURE); - } - servers.add(ServerInfo.create(serverUri, channelCredentials, useProtocolV3)); - } + List servers = parseServerInfos(rawServerConfigs, logger); + builder.servers(servers); Node.Builder nodeBuilder = Node.newBuilder(); Map rawNode = JsonUtil.getObject(rawData, "node"); @@ -200,29 +174,110 @@ BootstrapInfo bootstrap(Map rawData) throws XdsInitializationExceptio nodeBuilder.setUserAgentName(buildVersion.getUserAgent()); nodeBuilder.setUserAgentVersion(buildVersion.getImplementationVersion()); nodeBuilder.addClientFeatures(CLIENT_FEATURE_DISABLE_OVERPROVISIONING); + builder.node(nodeBuilder.build()); Map certProvidersBlob = JsonUtil.getObject(rawData, "certificate_providers"); - Map certProviders = null; if (certProvidersBlob != null) { - certProviders = new HashMap<>(certProvidersBlob.size()); + logger.log(XdsLogLevel.INFO, "Configured with {0} cert providers", certProvidersBlob.size()); + Map certProviders = new HashMap<>(certProvidersBlob.size()); for (String name : certProvidersBlob.keySet()) { Map valueMap = JsonUtil.getObject(certProvidersBlob, name); String pluginName = checkForNull(JsonUtil.getString(valueMap, "plugin_name"), "plugin_name"); + logger.log(XdsLogLevel.INFO, "cert provider: {0}, plugin name: {1}", name, pluginName); Map config = checkForNull(JsonUtil.getObject(valueMap, "config"), "config"); CertificateProviderInfo certificateProviderInfo = CertificateProviderInfo.create(pluginName, config); certProviders.put(name, certificateProviderInfo); } + builder.certProviders(certProviders); } + String grpcServerResourceId = JsonUtil.getString(rawData, "server_listener_resource_name_template"); - return BootstrapInfo.builder() - .servers(servers) - .node(nodeBuilder.build()) - .certProviders(certProviders) - .serverListenerResourceNameTemplate(grpcServerResourceId) - .build(); + logger.log( + XdsLogLevel.INFO, "server_listener_resource_name_template: {0}", grpcServerResourceId); + builder.serverListenerResourceNameTemplate(grpcServerResourceId); + + String grpcClientDefaultListener = + JsonUtil.getString(rawData, "client_default_listener_resource_name_template"); + logger.log( + XdsLogLevel.INFO, "client_default_listener_resource_name_template: {0}", + grpcClientDefaultListener); + if (grpcClientDefaultListener != null) { + builder.clientDefaultListenerResourceNameTemplate(grpcClientDefaultListener); + } + + Map rawAuthoritiesMap = + JsonUtil.getObject(rawData, "authorities"); + ImmutableMap.Builder authorityInfoMapBuilder = ImmutableMap.builder(); + if (rawAuthoritiesMap != null) { + logger.log( + XdsLogLevel.INFO, "Configured with {0} xDS server authorities", rawAuthoritiesMap.size()); + for (String authorityName : rawAuthoritiesMap.keySet()) { + logger.log(XdsLogLevel.INFO, "xDS server authority: {0}", authorityName); + Map rawAuthority = JsonUtil.getObject(rawAuthoritiesMap, authorityName); + String clientListnerTemplate = + JsonUtil.getString(rawAuthority, "client_listener_resource_name_template"); + logger.log( + XdsLogLevel.INFO, "client_listener_resource_name_template: {0}", clientListnerTemplate); + String prefix = "xdstp://" + authorityName + "/"; + if (clientListnerTemplate == null) { + clientListnerTemplate = prefix + "envoy.config.listener.v3.Listener/%s"; + } else if (!clientListnerTemplate.startsWith(prefix)) { + throw new XdsInitializationException( + "client_listener_resource_name_template: '" + clientListnerTemplate + + "' does not start with " + prefix); + } + List rawAuthorityServers = JsonUtil.getList(rawAuthority, "xds_servers"); + List authorityServers; + if (rawAuthorityServers == null || rawAuthorityServers.isEmpty()) { + authorityServers = servers; + } else { + authorityServers = parseServerInfos(rawAuthorityServers, logger); + } + authorityInfoMapBuilder.put( + authorityName, AuthorityInfo.create(clientListnerTemplate, authorityServers)); + } + builder.authorities(authorityInfoMapBuilder.build()); + } + + return builder.build(); + } + + private static List parseServerInfos(List rawServerConfigs, XdsLogger logger) + throws XdsInitializationException { + logger.log(XdsLogLevel.INFO, "Configured with {0} xDS servers", rawServerConfigs.size()); + ImmutableList.Builder servers = ImmutableList.builder(); + List> serverConfigList = JsonUtil.checkObjectList(rawServerConfigs); + for (Map serverConfig : serverConfigList) { + String serverUri = JsonUtil.getString(serverConfig, "server_uri"); + if (serverUri == null) { + throw new XdsInitializationException("Invalid bootstrap: missing 'server_uri'"); + } + logger.log(XdsLogLevel.INFO, "xDS server URI: {0}", serverUri); + + List rawChannelCredsList = JsonUtil.getList(serverConfig, "channel_creds"); + if (rawChannelCredsList == null || rawChannelCredsList.isEmpty()) { + throw new XdsInitializationException( + "Invalid bootstrap: server " + serverUri + " 'channel_creds' required"); + } + ChannelCredentials channelCredentials = + parseChannelCredentials(JsonUtil.checkObjectList(rawChannelCredsList), serverUri); + if (channelCredentials == null) { + throw new XdsInitializationException( + "Server " + serverUri + ": no supported channel credentials found"); + } + + boolean useProtocolV3 = false; + List serverFeatures = JsonUtil.getListOfStrings(serverConfig, "server_features"); + if (serverFeatures != null) { + logger.log(XdsLogLevel.INFO, "Server features: {0}", serverFeatures); + useProtocolV3 = serverFeatures.contains(XDS_V3_SERVER_FEATURE); + } + servers.add(ServerInfo.create(serverUri, channelCredentials, useProtocolV3)); + } + return servers.build(); } @VisibleForTesting diff --git a/xds/src/test/java/io/grpc/xds/BootstrapperImplTest.java b/xds/src/test/java/io/grpc/xds/BootstrapperImplTest.java index 283efcea852..183d67018dc 100644 --- a/xds/src/test/java/io/grpc/xds/BootstrapperImplTest.java +++ b/xds/src/test/java/io/grpc/xds/BootstrapperImplTest.java @@ -27,6 +27,7 @@ import io.grpc.TlsChannelCredentials; import io.grpc.internal.GrpcUtil; import io.grpc.internal.GrpcUtil.GrpcBuildVersion; +import io.grpc.xds.Bootstrapper.AuthorityInfo; import io.grpc.xds.Bootstrapper.BootstrapInfo; import io.grpc.xds.Bootstrapper.ServerInfo; import io.grpc.xds.EnvoyProtoData.Node; @@ -677,6 +678,101 @@ public void fallbackToConfigFromSysProp() throws XdsInitializationException { bootstrapper.bootstrap(); } + @Test + public void parseClientDefaultListenerResourceNameTemplate() throws Exception { + String rawData = "{\n" + + " \"xds_servers\": [\n" + + " ]\n" + + "}"; + bootstrapper.setFileReader(createFileReader(BOOTSTRAP_FILE_PATH, rawData)); + BootstrapInfo info = bootstrapper.bootstrap(); + assertThat(info.clientDefaultListenerResourceNameTemplate()).isEqualTo("%s"); + + rawData = "{\n" + + " \"client_default_listener_resource_name_template\": \"xdstp://a.com/faketype/%s\",\n" + + " \"xds_servers\": [\n" + + " ]\n" + + "}"; + bootstrapper.setFileReader(createFileReader(BOOTSTRAP_FILE_PATH, rawData)); + info = bootstrapper.bootstrap(); + assertThat(info.clientDefaultListenerResourceNameTemplate()) + .isEqualTo("xdstp://a.com/faketype/%s"); + } + + @Test + public void parseAuthorities() throws Exception { + String rawData = "{\n" + + " \"xds_servers\": [\n" + + " {\n" + + " \"server_uri\": \"" + SERVER_URI + "\",\n" + + " \"channel_creds\": [\n" + + " {\"type\": \"insecure\"}\n" + + " ]\n" + + " }\n" + + " ]\n" + + "}"; + bootstrapper.setFileReader(createFileReader(BOOTSTRAP_FILE_PATH, rawData)); + BootstrapInfo info = bootstrapper.bootstrap(); + assertThat(info.authorities()).isEmpty(); + + rawData = "{\n" + + " \"authorities\": {\n" + + " \"a.com\": {\n" + + " \"client_listener_resource_name_template\": \"xdstp://a.com/v1.Listener/id-%s\"\n" + + " }\n" + + " },\n" + + " \"xds_servers\": [\n" + + " {\n" + + " \"server_uri\": \"" + SERVER_URI + "\",\n" + + " \"channel_creds\": [\n" + + " {\"type\": \"insecure\"}\n" + + " ]\n" + + " }\n" + + " ]\n" + + "}"; + bootstrapper.setFileReader(createFileReader(BOOTSTRAP_FILE_PATH, rawData)); + info = bootstrapper.bootstrap(); + assertThat(info.authorities()).hasSize(1); + AuthorityInfo authorityInfo = info.authorities().get("a.com"); + assertThat(authorityInfo.clientListenerResourceNameTemplate()) + .isEqualTo("xdstp://a.com/v1.Listener/id-%s"); + // Defaults to top-level servers. + assertThat(authorityInfo.xdsServers()).hasSize(1); + assertThat(authorityInfo.xdsServers().get(0).target()).isEqualTo(SERVER_URI); + + rawData = "{\n" + + " \"authorities\": {\n" + + " \"a.com\": {\n" + + " \"xds_servers\": [\n" + + " {\n" + + " \"server_uri\": \"td2.googleapis.com:443\",\n" + + " \"channel_creds\": [\n" + + " {\"type\": \"insecure\"}\n" + + " ]\n" + + " }\n" + + " ]\n" + + " }\n" + + " },\n" + + " \"xds_servers\": [\n" + + " {\n" + + " \"server_uri\": \"" + SERVER_URI + "\",\n" + + " \"channel_creds\": [\n" + + " {\"type\": \"insecure\"}\n" + + " ]\n" + + " }\n" + + " ]\n" + + "}"; + bootstrapper.setFileReader(createFileReader(BOOTSTRAP_FILE_PATH, rawData)); + info = bootstrapper.bootstrap(); + assertThat(info.authorities()).hasSize(1); + authorityInfo = info.authorities().get("a.com"); + // Defaults to "xdstp://>/envoy.config.listener.v3.Listener/%s" + assertThat(authorityInfo.clientListenerResourceNameTemplate()) + .isEqualTo("xdstp://a.com/envoy.config.listener.v3.Listener/%s"); + assertThat(authorityInfo.xdsServers()).hasSize(1); + assertThat(authorityInfo.xdsServers().get(0).target()).isEqualTo("td2.googleapis.com:443"); + } + private static BootstrapperImpl.FileReader createFileReader( final String expectedPath, final String rawData) { return new BootstrapperImpl.FileReader() { From d2b9151e7bbdc6346ea624731326e3e538d9155f Mon Sep 17 00:00:00 2001 From: ZhenLian Date: Tue, 19 Oct 2021 16:50:33 -0700 Subject: [PATCH 70/76] core: remove DSA check in CertificateUtils --- .../main/java/io/grpc/util/CertificateUtils.java | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/io/grpc/util/CertificateUtils.java b/core/src/main/java/io/grpc/util/CertificateUtils.java index a886ff1f6e1..e7082f177ab 100644 --- a/core/src/main/java/io/grpc/util/CertificateUtils.java +++ b/core/src/main/java/io/grpc/util/CertificateUtils.java @@ -56,8 +56,7 @@ public static X509Certificate[] getX509Certificates(InputStream inputStream) /** * Generates a {@link PrivateKey} from a PEM file. - * The key should be PKCS #8 formatted. The key algorithm should be "RSA", "DiffieHellman", - * "DSA", or "EC". + * The key should be PKCS #8 formatted. The key algorithm should be "RSA" or "EC". * The PEM file should contain one item in Base64 encoding, with plain-text headers and footers * (e.g. -----BEGIN PRIVATE KEY----- and -----END PRIVATE KEY-----). * @@ -86,13 +85,9 @@ public static PrivateKey getPrivateKey(InputStream inputStream) return KeyFactory.getInstance("RSA").generatePrivate(keySpec); } catch (InvalidKeySpecException ignore) { try { - return KeyFactory.getInstance("DSA").generatePrivate(keySpec); - } catch (InvalidKeySpecException ignore2) { - try { - return KeyFactory.getInstance("EC").generatePrivate(keySpec); - } catch (InvalidKeySpecException e) { - throw new InvalidKeySpecException("Neither RSA, DSA nor EC worked", e); - } + return KeyFactory.getInstance("EC").generatePrivate(keySpec); + } catch (InvalidKeySpecException e) { + throw new InvalidKeySpecException("Neither RSA nor EC worked", e); } } } From d7454ed968c26c9c8447a1e005dc7bb7dd94b682 Mon Sep 17 00:00:00 2001 From: ZHANG Dapeng Date: Wed, 20 Oct 2021 17:59:21 -0700 Subject: [PATCH 71/76] xds: add protection flag for federation (#8619) See https://github.com/grpc/proposal/pull/268/files#diff-e68147af61f13db5bd497e86ffd970fef6af29b88f4f23fb486deefdb35dfea3R659 for detail. --- .../java/io/grpc/xds/BootstrapperImpl.java | 8 +++++ .../io/grpc/xds/BootstrapperImplTest.java | 36 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/xds/src/main/java/io/grpc/xds/BootstrapperImpl.java b/xds/src/main/java/io/grpc/xds/BootstrapperImpl.java index 200ded3c1c4..c11dbbf7659 100644 --- a/xds/src/main/java/io/grpc/xds/BootstrapperImpl.java +++ b/xds/src/main/java/io/grpc/xds/BootstrapperImpl.java @@ -17,6 +17,7 @@ package io.grpc.xds; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import io.grpc.ChannelCredentials; @@ -62,6 +63,10 @@ public class BootstrapperImpl extends Bootstrapper { @VisibleForTesting static final String CLIENT_FEATURE_DISABLE_OVERPROVISIONING = "envoy.lb.does_not_support_overprovisioning"; + @VisibleForTesting + static boolean enableFederation = + !Strings.isNullOrEmpty(System.getenv("GRPC_EXPERIMENTAL_XDS_FEDERATION")) + && Boolean.parseBoolean(System.getenv("GRPC_EXPERIMENTAL_XDS_FEDERATION")); private final XdsLogger logger; private FileReader reader = LocalFileReader.INSTANCE; @@ -199,6 +204,9 @@ BootstrapInfo bootstrap(Map rawData) throws XdsInitializationExceptio XdsLogLevel.INFO, "server_listener_resource_name_template: {0}", grpcServerResourceId); builder.serverListenerResourceNameTemplate(grpcServerResourceId); + if (!enableFederation) { + return builder.build(); + } String grpcClientDefaultListener = JsonUtil.getString(rawData, "client_default_listener_resource_name_template"); logger.log( diff --git a/xds/src/test/java/io/grpc/xds/BootstrapperImplTest.java b/xds/src/test/java/io/grpc/xds/BootstrapperImplTest.java index 183d67018dc..53b52a7bc02 100644 --- a/xds/src/test/java/io/grpc/xds/BootstrapperImplTest.java +++ b/xds/src/test/java/io/grpc/xds/BootstrapperImplTest.java @@ -57,6 +57,7 @@ public class BootstrapperImplTest { private String originalBootstrapPathFromSysProp; private String originalBootstrapConfigFromEnvVar; private String originalBootstrapConfigFromSysProp; + private boolean originalEnableFederation; @Before public void setUp() { @@ -69,6 +70,7 @@ private void saveEnvironment() { originalBootstrapPathFromSysProp = BootstrapperImpl.bootstrapPathFromSysProp; originalBootstrapConfigFromEnvVar = BootstrapperImpl.bootstrapConfigFromEnvVar; originalBootstrapConfigFromSysProp = BootstrapperImpl.bootstrapConfigFromSysProp; + originalEnableFederation = BootstrapperImpl.enableFederation; } @After @@ -77,6 +79,7 @@ public void restoreEnvironment() { BootstrapperImpl.bootstrapPathFromSysProp = originalBootstrapPathFromSysProp; BootstrapperImpl.bootstrapConfigFromEnvVar = originalBootstrapConfigFromEnvVar; BootstrapperImpl.bootstrapConfigFromSysProp = originalBootstrapConfigFromSysProp; + BootstrapperImpl.enableFederation = originalEnableFederation; } @Test @@ -680,6 +683,7 @@ public void fallbackToConfigFromSysProp() throws XdsInitializationException { @Test public void parseClientDefaultListenerResourceNameTemplate() throws Exception { + BootstrapperImpl.enableFederation = true; String rawData = "{\n" + " \"xds_servers\": [\n" + " ]\n" @@ -701,6 +705,7 @@ public void parseClientDefaultListenerResourceNameTemplate() throws Exception { @Test public void parseAuthorities() throws Exception { + BootstrapperImpl.enableFederation = true; String rawData = "{\n" + " \"xds_servers\": [\n" + " {\n" @@ -773,6 +778,37 @@ public void parseAuthorities() throws Exception { assertThat(authorityInfo.xdsServers().get(0).target()).isEqualTo("td2.googleapis.com:443"); } + @Test + public void badFederationConfig() throws Exception { + BootstrapperImpl.enableFederation = true; + String rawData = "{\n" + + " \"authorities\": {\n" + + " \"a.com\": {\n" + + " \"client_listener_resource_name_template\": \"xdstp://wrong/\"\n" + + " }\n" + + " },\n" + + " \"xds_servers\": [\n" + + " {\n" + + " \"server_uri\": \"" + SERVER_URI + "\",\n" + + " \"channel_creds\": [\n" + + " {\"type\": \"insecure\"}\n" + + " ]\n" + + " }\n" + + " ]\n" + + "}"; + bootstrapper.setFileReader(createFileReader(BOOTSTRAP_FILE_PATH, rawData)); + try { + bootstrapper.bootstrap(); + fail("should fail"); + } catch (XdsInitializationException e) { + assertThat(e).hasMessageThat().isEqualTo( + "client_listener_resource_name_template: 'xdstp://wrong/' does not start with " + + "xdstp://a.com/"); + } + BootstrapperImpl.enableFederation = false; + bootstrapper.bootstrap(); + } + private static BootstrapperImpl.FileReader createFileReader( final String expectedPath, final String rawData) { return new BootstrapperImpl.FileReader() { From 900b68f378b2094c8767810556ab86053849ff8b Mon Sep 17 00:00:00 2001 From: ZHANG Dapeng Date: Wed, 20 Oct 2021 17:56:53 -0700 Subject: [PATCH 72/76] Update README for Android API level (#8620) We dropped support for Android API levels <19 in #8583 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e6c06bde236..ea11cbde260 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ gRPC-Java - An RPC library and framework ======================================== gRPC-Java works with JDK 7. gRPC-Java clients are supported on Android API -levels 16 and up (Jelly Bean and later). Deploying gRPC servers on an Android +levels 19 and up (KitKat and later). Deploying gRPC servers on an Android device is not supported. TLS usage typically requires using Java 8, or Play Services Dynamic Security From c5fc08d62dbd09dd2bb06032e2faa60bbbca1467 Mon Sep 17 00:00:00 2001 From: markb74 <57717302+markb74@users.noreply.github.com> Date: Mon, 1 Nov 2021 18:57:30 +0100 Subject: [PATCH 73/76] binder: SecurityPolicy updates (take 2). (#8637) The previous attempt at this CL relied on guava's Hashing class which is still in beta. This update compares Signature objects directly instead of SHA256 hashs, removing the need for the Hashing class. Add additional comments to the security policy class, to mention that implementing new policies requires significant care. With that in mind, add security policies to check the peer app's signature, so people can create cross-app communication without having to implement their own policy. Finally, add the UntrustedSecurityPolicies class, since that's inevitably a policy which is sometimes needed. --- .../java/io/grpc/binder/SecurityPolicies.java | 132 ++++++++++++++++++ .../java/io/grpc/binder/SecurityPolicy.java | 5 + .../binder/UntrustedSecurityPolicies.java | 47 +++++++ .../io/grpc/binder/SecurityPoliciesTest.java | 118 ++++++++++++++++ 4 files changed, 302 insertions(+) create mode 100644 binder/src/main/java/io/grpc/binder/UntrustedSecurityPolicies.java diff --git a/binder/src/main/java/io/grpc/binder/SecurityPolicies.java b/binder/src/main/java/io/grpc/binder/SecurityPolicies.java index be46b9e3e54..dcf36be00ca 100644 --- a/binder/src/main/java/io/grpc/binder/SecurityPolicies.java +++ b/binder/src/main/java/io/grpc/binder/SecurityPolicies.java @@ -16,9 +16,20 @@ package io.grpc.binder; +import android.annotation.SuppressLint; +import android.content.pm.PackageInfo; +import android.content.pm.PackageManager; +import android.content.pm.PackageManager.NameNotFoundException; +import android.content.pm.Signature; +import android.os.Build; import android.os.Process; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import io.grpc.ExperimentalApi; import io.grpc.Status; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; import javax.annotation.CheckReturnValue; /** Static factory methods for creating standard security policies. */ @@ -55,4 +66,125 @@ public Status checkAuthorization(int uid) { } }; } + + /** + * Creates a {@link SecurityPolicy} which checks if the package signature + * matches {@code requiredSignature}. + * + * @param packageName the package name of the allowed package. + * @param requiredSignature the allowed signature of the allowed package. + * @throws NullPointerException if any of the inputs are {@code null}. + */ + public static SecurityPolicy hasSignature( + PackageManager packageManager, String packageName, Signature requiredSignature) { + return oneOfSignatures( + packageManager, packageName, ImmutableList.of(requiredSignature)); + } + + /** + * Creates a {@link SecurityPolicy} which checks if the package signature + * matches any of {@code requiredSignatures}. + * + * @param packageName the package name of the allowed package. + * @param requiredSignatures the allowed signatures of the allowed package. + * @throws NullPointerException if any of the inputs are {@code null}. + * @throws IllegalArgumentException if {@code requiredSignatures} is empty. + */ + public static SecurityPolicy oneOfSignatures( + PackageManager packageManager, + String packageName, + Collection requiredSignatures) { + Preconditions.checkNotNull(packageManager, "packageManager"); + Preconditions.checkNotNull(packageName, "packageName"); + Preconditions.checkNotNull(requiredSignatures, "requiredSignatures"); + Preconditions.checkArgument(!requiredSignatures.isEmpty(), + "requiredSignatures"); + ImmutableList requiredSignaturesImmutable = ImmutableList.copyOf(requiredSignatures); + + for (Signature requiredSignature : requiredSignaturesImmutable) { + Preconditions.checkNotNull(requiredSignature); + } + + return new SecurityPolicy() { + @Override + public Status checkAuthorization(int uid) { + return checkUidSignature( + packageManager, uid, packageName, requiredSignaturesImmutable); + } + }; + } + + private static Status checkUidSignature( + PackageManager packageManager, + int uid, + String packageName, + ImmutableList requiredSignatures) { + String[] packages = packageManager.getPackagesForUid(uid); + if (packages == null) { + return Status.UNAUTHENTICATED.withDescription( + "Rejected by signature check security policy"); + } + boolean packageNameMatched = false; + for (String pkg : packages) { + if (!packageName.equals(pkg)) { + continue; + } + packageNameMatched = true; + if (checkPackageSignature(packageManager, pkg, requiredSignatures)) { + return Status.OK; + } + } + return Status.PERMISSION_DENIED.withDescription( + "Rejected by signature check security policy. Package name matched: " + + packageNameMatched); + } + + /** + * Checks if the signature of {@code packageName} matches one of the given signatures. + * + * @param packageName the package to be checked + * @param requiredSignatures list of signatures. + * @return {@code true} if {@code packageName} has a matching signature. + */ + @SuppressWarnings("deprecation") // For PackageInfo.signatures + @SuppressLint("PackageManagerGetSignatures") // We only allow 1 signature. + private static boolean checkPackageSignature( + PackageManager packageManager, + String packageName, + ImmutableList requiredSignatures) { + PackageInfo packageInfo; + try { + if (Build.VERSION.SDK_INT >= 28) { + packageInfo = + packageManager.getPackageInfo(packageName, PackageManager.GET_SIGNING_CERTIFICATES); + if (packageInfo.signingInfo == null) { + return false; + } + Signature[] signatures = + packageInfo.signingInfo.hasMultipleSigners() + ? packageInfo.signingInfo.getApkContentsSigners() + : packageInfo.signingInfo.getSigningCertificateHistory(); + + for (Signature signature : signatures) { + if (requiredSignatures.contains(signature)) { + return true; + } + } + } else { + packageInfo = packageManager.getPackageInfo(packageName, PackageManager.GET_SIGNATURES); + if (packageInfo.signatures == null || packageInfo.signatures.length != 1) { + // Reject multiply-signed apks because of b/13678484 + // (See PackageManagerGetSignatures supression above). + return false; + } + + if (requiredSignatures.contains(packageInfo.signatures[0])) { + return true; + } + } + } catch (NameNotFoundException nnfe) { + return false; + } + return false; + } } diff --git a/binder/src/main/java/io/grpc/binder/SecurityPolicy.java b/binder/src/main/java/io/grpc/binder/SecurityPolicy.java index d7dad53fdc8..d13f3a863fd 100644 --- a/binder/src/main/java/io/grpc/binder/SecurityPolicy.java +++ b/binder/src/main/java/io/grpc/binder/SecurityPolicy.java @@ -23,6 +23,11 @@ /** * Decides whether a given Android UID is authorized to access some resource. * + * While it's possible to extend this class to define your own policy, it's strongly + * recommended that you only use the policies provided by the {@link SecurityPolicies} or + * {@link UntrustedSecurityPolicies} classes. Implementing your own security policy requires + * significant care, and an understanding of the details and pitfalls of Android security. + * *

IMPORTANT For any concrete extensions of this class, it's assumed that the * authorization status of a given UID will not change as long as a process with that UID is * alive. diff --git a/binder/src/main/java/io/grpc/binder/UntrustedSecurityPolicies.java b/binder/src/main/java/io/grpc/binder/UntrustedSecurityPolicies.java new file mode 100644 index 00000000000..7c842b025ac --- /dev/null +++ b/binder/src/main/java/io/grpc/binder/UntrustedSecurityPolicies.java @@ -0,0 +1,47 @@ +/* + * Copyright 2021 The gRPC Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.grpc.binder; + +import io.grpc.ExperimentalApi; +import io.grpc.Status; +import javax.annotation.CheckReturnValue; + +/** + * Static factory methods for creating untrusted security policies. + */ +@CheckReturnValue +@ExperimentalApi("https://github.com/grpc/grpc-java/issues/8022") +public final class UntrustedSecurityPolicies { + + private UntrustedSecurityPolicies() {} + + /** + * Return a security policy which allows any peer on device. + * Servers should only use this policy if they intend to expose + * a service to all applications on device. + * Clients should only use this policy if they don't need to trust the + * application they're connecting to. + */ + public static SecurityPolicy untrustedPublic() { + return new SecurityPolicy() { + @Override + public Status checkAuthorization(int uid) { + return Status.OK; + } + }; + } +} diff --git a/binder/src/test/java/io/grpc/binder/SecurityPoliciesTest.java b/binder/src/test/java/io/grpc/binder/SecurityPoliciesTest.java index 6fd9e22ebaa..86edb5ad7df 100644 --- a/binder/src/test/java/io/grpc/binder/SecurityPoliciesTest.java +++ b/binder/src/test/java/io/grpc/binder/SecurityPoliciesTest.java @@ -17,22 +17,64 @@ package io.grpc.binder; import static com.google.common.truth.Truth.assertThat; +import static org.robolectric.Shadows.shadowOf; +import android.content.Context; +import android.content.pm.PackageInfo; +import android.content.pm.PackageManager; +import android.content.pm.Signature; import android.os.Process; +import androidx.test.core.app.ApplicationProvider; +import com.google.common.collect.ImmutableList; import io.grpc.Status; +import io.grpc.binder.SecurityPolicy; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.robolectric.RobolectricTestRunner; @RunWith(RobolectricTestRunner.class) public final class SecurityPoliciesTest { + private static final int MY_UID = Process.myUid(); private static final int OTHER_UID = MY_UID + 1; + private static final int OTHER_UID_SAME_SIGNATURE = MY_UID + 2; + private static final int OTHER_UID_NO_SIGNATURE = MY_UID + 3; + private static final int OTHER_UID_UNKNOWN = MY_UID + 4; private static final String PERMISSION_DENIED_REASONS = "some reasons"; + private static final Signature SIG1 = new Signature("1234"); + private static final Signature SIG2 = new Signature("4321"); + + private static final String OTHER_UID_PACKAGE_NAME = "other.package"; + private static final String OTHER_UID_SAME_SIGNATURE_PACKAGE_NAME = "other.package.samesignature"; + private static final String OTHER_UID_NO_SIGNATURE_PACKAGE_NAME = "other.package.nosignature"; + + private Context appContext; + private PackageManager packageManager; + private SecurityPolicy policy; + @Before + public void setUp() { + appContext = ApplicationProvider.getApplicationContext(); + packageManager = appContext.getPackageManager(); + installPackage(MY_UID, appContext.getPackageName(), SIG1); + installPackage(OTHER_UID, OTHER_UID_PACKAGE_NAME, SIG2); + installPackage(OTHER_UID_SAME_SIGNATURE, OTHER_UID_SAME_SIGNATURE_PACKAGE_NAME, SIG1); + installPackage(OTHER_UID_NO_SIGNATURE, OTHER_UID_NO_SIGNATURE_PACKAGE_NAME); + } + + @SuppressWarnings("deprecation") + private void installPackage(int uid, String packageName, Signature... signatures) { + PackageInfo info = new PackageInfo(); + info.packageName = packageName; + info.signatures = signatures; + shadowOf(packageManager).installPackage(info); + shadowOf(packageManager).setPackagesForUid(uid, packageName); + } + @Test public void testInternalOnly() throws Exception { policy = SecurityPolicies.internalOnly(); @@ -53,4 +95,80 @@ public void testPermissionDenied() throws Exception { assertThat(policy.checkAuthorization(OTHER_UID).getDescription()) .isEqualTo(PERMISSION_DENIED_REASONS); } + + @Test + public void testHasSignature_succeedsIfPackageNameAndSignaturesMatch() + throws Exception { + policy = SecurityPolicies.hasSignature(packageManager, OTHER_UID_PACKAGE_NAME, SIG2); + + // THEN UID for package that has SIG2 will be authorized + assertThat(policy.checkAuthorization(OTHER_UID).getCode()).isEqualTo(Status.OK.getCode()); + } + + @Test + public void testHasSignature_failsIfPackageNameDoesNotMatch() throws Exception { + policy = SecurityPolicies.hasSignature(packageManager, appContext.getPackageName(), SIG1); + + // THEN UID for package that has SIG1 but different package name will not be authorized + assertThat(policy.checkAuthorization(OTHER_UID_SAME_SIGNATURE).getCode()) + .isEqualTo(Status.PERMISSION_DENIED.getCode()); + } + + @Test + public void testHasSignature_failsIfSignatureDoesNotMatch() throws Exception { + policy = SecurityPolicies.hasSignature(packageManager, OTHER_UID_PACKAGE_NAME, SIG1); + + // THEN UID for package that doesn't have SIG1 will not be authorized + assertThat(policy.checkAuthorization(OTHER_UID).getCode()) + .isEqualTo(Status.PERMISSION_DENIED.getCode()); + } + + @Test + public void testOneOfSignatures_succeedsIfPackageNameAndSignaturesMatch() + throws Exception { + policy = + SecurityPolicies.oneOfSignatures( + packageManager, OTHER_UID_PACKAGE_NAME, ImmutableList.of(SIG2)); + + // THEN UID for package that has SIG2 will be authorized + assertThat(policy.checkAuthorization(OTHER_UID).getCode()).isEqualTo(Status.OK.getCode()); + } + + @Test + public void testOneOfSignature_failsIfAllSignaturesDoNotMatch() throws Exception { + policy = + SecurityPolicies.oneOfSignatures( + packageManager, + appContext.getPackageName(), + ImmutableList.of(SIG1, new Signature("1314"))); + + // THEN UID for package that has SIG1 but different package name will not be authorized + assertThat(policy.checkAuthorization(OTHER_UID_SAME_SIGNATURE).getCode()) + .isEqualTo(Status.PERMISSION_DENIED.getCode()); + } + + @Test + public void testOneOfSignature_succeedsIfPackageNameAndOneOfSignaturesMatch() + throws Exception { + policy = + SecurityPolicies.oneOfSignatures( + packageManager, + OTHER_UID_PACKAGE_NAME, + ImmutableList.of(SIG1, SIG2)); + + // THEN UID for package that has SIG2 will be authorized + assertThat(policy.checkAuthorization(OTHER_UID).getCode()).isEqualTo(Status.OK.getCode()); + } + + @Test + public void testHasSignature_failsIfUidUnknown() throws Exception { + policy = + SecurityPolicies.hasSignature( + packageManager, + appContext.getPackageName(), + SIG1); + + assertThat(policy.checkAuthorization(OTHER_UID_UNKNOWN).getCode()) + .isEqualTo(Status.UNAUTHENTICATED.getCode()); + } } From 03c49f7255b4ac38a05aea7330c54a36fae32c8c Mon Sep 17 00:00:00 2001 From: Terry Wilson Date: Tue, 2 Nov 2021 16:04:03 -0700 Subject: [PATCH 74/76] grpclb: fallback timer only when not already using fallback backends. (#8646) (#8648) Addresses a problem where we initially only resolve addresses to the backends, but not the load balancer and then later resolve addresses to both. In this situation the fallback timer was started during the second instance even if it resulted in the timer later failing as we were already using fallback backends. This change assures that a fallback time is only ever started if we are not already using the fallback backends. This is a follow-up fix to #8253. --- .../main/java/io/grpc/grpclb/GrpclbState.java | 5 ++-- .../grpc/grpclb/GrpclbLoadBalancerTest.java | 27 +++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/grpclb/src/main/java/io/grpc/grpclb/GrpclbState.java b/grpclb/src/main/java/io/grpc/grpclb/GrpclbState.java index 8607d3996a5..1eebaa63a8e 100644 --- a/grpclb/src/main/java/io/grpc/grpclb/GrpclbState.java +++ b/grpclb/src/main/java/io/grpc/grpclb/GrpclbState.java @@ -287,8 +287,9 @@ void handleAddresses( cancelLbRpcRetryTimer(); startLbRpc(); } - // Start the fallback timer if it's never started - if (fallbackTimer == null) { + // Start the fallback timer if it's never started and we are not already using fallback + // backends. + if (fallbackTimer == null && !usingFallbackBackends) { fallbackTimer = syncContext.schedule( new FallbackModeTask(BALANCER_TIMEOUT_STATUS), FALLBACK_TIMEOUT_MS, TimeUnit.MILLISECONDS, timerService); diff --git a/grpclb/src/test/java/io/grpc/grpclb/GrpclbLoadBalancerTest.java b/grpclb/src/test/java/io/grpc/grpclb/GrpclbLoadBalancerTest.java index cb231c6c055..293c0aa0b82 100644 --- a/grpclb/src/test/java/io/grpc/grpclb/GrpclbLoadBalancerTest.java +++ b/grpclb/src/test/java/io/grpc/grpclb/GrpclbLoadBalancerTest.java @@ -1462,6 +1462,33 @@ public void grpclbFallback_noBalancerAddress() { .updateBalancingState(eq(TRANSIENT_FAILURE), any(SubchannelPicker.class)); } + /** + * A test for a situation where we first only get backend addresses resolved and then in a + * later name resolution get both backend and load balancer addresses. The first instance + * will switch us to using fallback backends and it is important that in the second instance + * we do not start a fallback timer as it will fail when it triggers if the fallback backends + * are already in use. + */ + @Test + public void grpclbFallback_noTimerWhenAlreadyInFallback() { + // Initially we only get backend addresses without any LB ones. This should get us to use + // fallback backends from the start as we won't be able to even talk to the load balancer. + // No fallback timer would be started as we already started to use fallback backends. + deliverResolvedAddresses(createResolvedBalancerAddresses(1), + Collections.emptyList()); + assertEquals(0, fakeClock.numPendingTasks(FALLBACK_MODE_TASK_FILTER)); + + // Later a new name resolution call happens and we get both backend and LB addresses. Since we + // are already operating with fallback backends a fallback timer should not be started to move + // us to fallback mode. + deliverResolvedAddresses(Collections.emptyList(), + createResolvedBalancerAddresses(1)); + + // If a fallback timer is started it will eventually throw an exception when it tries to switch + // us to using fallback backends when we already are using them. + assertEquals(0, fakeClock.numPendingTasks(FALLBACK_MODE_TASK_FILTER)); + } + @Test public void grpclbFallback_balancerLost() { subtestGrpclbFallbackConnectionLost(true, false); From c6bcabf837a5a3f0261d3199985f7b9ea7b2b9e8 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Wed, 3 Nov 2021 17:04:11 -0700 Subject: [PATCH 75/76] Update README etc to reference 1.42.0 --- README.md | 30 ++++++++++++------------ cronet/README.md | 2 +- documentation/android-channel-builder.md | 4 ++-- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index ea11cbde260..aa3c61b9c47 100644 --- a/README.md +++ b/README.md @@ -31,8 +31,8 @@ For a guided tour, take a look at the [quick start guide](https://grpc.io/docs/languages/java/quickstart) or the more explanatory [gRPC basics](https://grpc.io/docs/languages/java/basics). -The [examples](https://github.com/grpc/grpc-java/tree/v1.41.0/examples) and the -[Android example](https://github.com/grpc/grpc-java/tree/v1.41.0/examples/android) +The [examples](https://github.com/grpc/grpc-java/tree/v1.42.0/examples) and the +[Android example](https://github.com/grpc/grpc-java/tree/v1.42.0/examples/android) are standalone projects that showcase the usage of gRPC. Download @@ -43,17 +43,17 @@ Download [the JARs][]. Or for Maven with non-Android, add to your `pom.xml`: io.grpc grpc-netty-shaded - 1.41.0 + 1.42.0 io.grpc grpc-protobuf - 1.41.0 + 1.42.0 io.grpc grpc-stub - 1.41.0 + 1.42.0 org.apache.tomcat @@ -65,23 +65,23 @@ Download [the JARs][]. Or for Maven with non-Android, add to your `pom.xml`: Or for Gradle with non-Android, add to your dependencies: ```gradle -implementation 'io.grpc:grpc-netty-shaded:1.41.0' -implementation 'io.grpc:grpc-protobuf:1.41.0' -implementation 'io.grpc:grpc-stub:1.41.0' +implementation 'io.grpc:grpc-netty-shaded:1.42.0' +implementation 'io.grpc:grpc-protobuf:1.42.0' +implementation 'io.grpc:grpc-stub:1.42.0' compileOnly 'org.apache.tomcat:annotations-api:6.0.53' // necessary for Java 9+ ``` For Android client, use `grpc-okhttp` instead of `grpc-netty-shaded` and `grpc-protobuf-lite` instead of `grpc-protobuf`: ```gradle -implementation 'io.grpc:grpc-okhttp:1.41.0' -implementation 'io.grpc:grpc-protobuf-lite:1.41.0' -implementation 'io.grpc:grpc-stub:1.41.0' +implementation 'io.grpc:grpc-okhttp:1.42.0' +implementation 'io.grpc:grpc-protobuf-lite:1.42.0' +implementation 'io.grpc:grpc-stub:1.42.0' compileOnly 'org.apache.tomcat:annotations-api:6.0.53' // necessary for Java 9+ ``` [the JARs]: -https://search.maven.org/search?q=g:io.grpc%20AND%20v:1.41.0 +https://search.maven.org/search?q=g:io.grpc%20AND%20v:1.42.0 Development snapshots are available in [Sonatypes's snapshot repository](https://oss.sonatype.org/content/repositories/snapshots/). @@ -113,7 +113,7 @@ For protobuf-based codegen integrated with the Maven build system, you can use com.google.protobuf:protoc:3.17.3:exe:${os.detected.classifier} grpc-java - io.grpc:protoc-gen-grpc-java:1.41.0:exe:${os.detected.classifier} + io.grpc:protoc-gen-grpc-java:1.42.0:exe:${os.detected.classifier} @@ -143,7 +143,7 @@ protobuf { } plugins { grpc { - artifact = 'io.grpc:protoc-gen-grpc-java:1.41.0' + artifact = 'io.grpc:protoc-gen-grpc-java:1.42.0' } } generateProtoTasks { @@ -176,7 +176,7 @@ protobuf { } plugins { grpc { - artifact = 'io.grpc:protoc-gen-grpc-java:1.41.0' + artifact = 'io.grpc:protoc-gen-grpc-java:1.42.0' } } generateProtoTasks { diff --git a/cronet/README.md b/cronet/README.md index 8b220bd606d..9a9f8fbe6c4 100644 --- a/cronet/README.md +++ b/cronet/README.md @@ -26,7 +26,7 @@ In your app module's `build.gradle` file, include a dependency on both `grpc-cro Google Play Services Client Library for Cronet ``` -implementation 'io.grpc:grpc-cronet:1.41.0' +implementation 'io.grpc:grpc-cronet:1.42.0' implementation 'com.google.android.gms:play-services-cronet:16.0.0' ``` diff --git a/documentation/android-channel-builder.md b/documentation/android-channel-builder.md index 60e3bb35a85..7e56e391038 100644 --- a/documentation/android-channel-builder.md +++ b/documentation/android-channel-builder.md @@ -36,8 +36,8 @@ In your `build.gradle` file, include a dependency on both `grpc-android` and `grpc-okhttp`: ``` -implementation 'io.grpc:grpc-android:1.41.0' -implementation 'io.grpc:grpc-okhttp:1.41.0' +implementation 'io.grpc:grpc-android:1.42.0' +implementation 'io.grpc:grpc-okhttp:1.42.0' ``` You also need permission to access the device's network state in your From b9b370652254fc4add89a457bd05be9ef3dbd235 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Wed, 3 Nov 2021 17:09:56 -0700 Subject: [PATCH 76/76] Bump version to 1.42.0 --- build.gradle | 2 +- .../src/test/golden/TestDeprecatedService.java.txt | 2 +- compiler/src/test/golden/TestService.java.txt | 2 +- .../src/testLite/golden/TestDeprecatedService.java.txt | 2 +- compiler/src/testLite/golden/TestService.java.txt | 2 +- core/src/main/java/io/grpc/internal/GrpcUtil.java | 2 +- examples/android/clientcache/app/build.gradle | 10 +++++----- examples/android/helloworld/app/build.gradle | 8 ++++---- examples/android/routeguide/app/build.gradle | 8 ++++---- examples/android/strictmode/app/build.gradle | 8 ++++---- examples/build.gradle | 2 +- examples/example-alts/build.gradle | 2 +- examples/example-gauth/build.gradle | 2 +- examples/example-gauth/pom.xml | 4 ++-- examples/example-hostname/build.gradle | 2 +- examples/example-hostname/pom.xml | 4 ++-- examples/example-jwt-auth/build.gradle | 2 +- examples/example-jwt-auth/pom.xml | 4 ++-- examples/example-tls/build.gradle | 2 +- examples/example-tls/pom.xml | 4 ++-- examples/example-xds/build.gradle | 2 +- examples/pom.xml | 4 ++-- 22 files changed, 40 insertions(+), 40 deletions(-) diff --git a/build.gradle b/build.gradle index df487de8fc4..d4c64eead21 100644 --- a/build.gradle +++ b/build.gradle @@ -18,7 +18,7 @@ subprojects { apply plugin: "net.ltgt.errorprone" group = "io.grpc" - version = "1.42.0-SNAPSHOT" // CURRENT_GRPC_VERSION + version = "1.42.0" // CURRENT_GRPC_VERSION repositories { maven { // The google mirror is less flaky than mavenCentral() diff --git a/compiler/src/test/golden/TestDeprecatedService.java.txt b/compiler/src/test/golden/TestDeprecatedService.java.txt index 3d1476a5bee..c89cae3b8e0 100644 --- a/compiler/src/test/golden/TestDeprecatedService.java.txt +++ b/compiler/src/test/golden/TestDeprecatedService.java.txt @@ -8,7 +8,7 @@ import static io.grpc.MethodDescriptor.generateFullMethodName; * */ @javax.annotation.Generated( - value = "by gRPC proto compiler (version 1.42.0-SNAPSHOT)", + value = "by gRPC proto compiler (version 1.42.0)", comments = "Source: grpc/testing/compiler/test.proto") @io.grpc.stub.annotations.GrpcGenerated @java.lang.Deprecated diff --git a/compiler/src/test/golden/TestService.java.txt b/compiler/src/test/golden/TestService.java.txt index 929ab5af817..e2a3bb6b767 100644 --- a/compiler/src/test/golden/TestService.java.txt +++ b/compiler/src/test/golden/TestService.java.txt @@ -8,7 +8,7 @@ import static io.grpc.MethodDescriptor.generateFullMethodName; * */ @javax.annotation.Generated( - value = "by gRPC proto compiler (version 1.42.0-SNAPSHOT)", + value = "by gRPC proto compiler (version 1.42.0)", comments = "Source: grpc/testing/compiler/test.proto") @io.grpc.stub.annotations.GrpcGenerated public final class TestServiceGrpc { diff --git a/compiler/src/testLite/golden/TestDeprecatedService.java.txt b/compiler/src/testLite/golden/TestDeprecatedService.java.txt index e088d7c0ede..7cdc246e16d 100644 --- a/compiler/src/testLite/golden/TestDeprecatedService.java.txt +++ b/compiler/src/testLite/golden/TestDeprecatedService.java.txt @@ -8,7 +8,7 @@ import static io.grpc.MethodDescriptor.generateFullMethodName; * */ @javax.annotation.Generated( - value = "by gRPC proto compiler (version 1.42.0-SNAPSHOT)", + value = "by gRPC proto compiler (version 1.42.0)", comments = "Source: grpc/testing/compiler/test.proto") @io.grpc.stub.annotations.GrpcGenerated @java.lang.Deprecated diff --git a/compiler/src/testLite/golden/TestService.java.txt b/compiler/src/testLite/golden/TestService.java.txt index 17005100271..02a0687291b 100644 --- a/compiler/src/testLite/golden/TestService.java.txt +++ b/compiler/src/testLite/golden/TestService.java.txt @@ -8,7 +8,7 @@ import static io.grpc.MethodDescriptor.generateFullMethodName; * */ @javax.annotation.Generated( - value = "by gRPC proto compiler (version 1.42.0-SNAPSHOT)", + value = "by gRPC proto compiler (version 1.42.0)", comments = "Source: grpc/testing/compiler/test.proto") @io.grpc.stub.annotations.GrpcGenerated public final class TestServiceGrpc { diff --git a/core/src/main/java/io/grpc/internal/GrpcUtil.java b/core/src/main/java/io/grpc/internal/GrpcUtil.java index 55e2cd81530..c165580aeff 100644 --- a/core/src/main/java/io/grpc/internal/GrpcUtil.java +++ b/core/src/main/java/io/grpc/internal/GrpcUtil.java @@ -205,7 +205,7 @@ public byte[] parseAsciiString(byte[] serialized) { public static final Splitter ACCEPT_ENCODING_SPLITTER = Splitter.on(',').trimResults(); - private static final String IMPLEMENTATION_VERSION = "1.42.0-SNAPSHOT"; // CURRENT_GRPC_VERSION + private static final String IMPLEMENTATION_VERSION = "1.42.0"; // CURRENT_GRPC_VERSION /** * The default timeout in nanos for a keepalive ping request. diff --git a/examples/android/clientcache/app/build.gradle b/examples/android/clientcache/app/build.gradle index 99ac5e5db1c..f63a5c4fb89 100644 --- a/examples/android/clientcache/app/build.gradle +++ b/examples/android/clientcache/app/build.gradle @@ -34,7 +34,7 @@ android { protobuf { protoc { artifact = 'com.google.protobuf:protoc:3.17.2' } plugins { - grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION + grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.42.0' // CURRENT_GRPC_VERSION } } generateProtoTasks { @@ -54,12 +54,12 @@ dependencies { implementation 'com.android.support:appcompat-v7:27.0.2' // You need to build grpc-java to obtain these libraries below. - implementation 'io.grpc:grpc-okhttp:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-protobuf-lite:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-stub:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-okhttp:1.42.0' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-protobuf-lite:1.42.0' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-stub:1.42.0' // CURRENT_GRPC_VERSION implementation 'org.apache.tomcat:annotations-api:6.0.53' testImplementation 'junit:junit:4.12' testImplementation 'com.google.truth:truth:1.0.1' - testImplementation 'io.grpc:grpc-testing:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION + testImplementation 'io.grpc:grpc-testing:1.42.0' // CURRENT_GRPC_VERSION } diff --git a/examples/android/helloworld/app/build.gradle b/examples/android/helloworld/app/build.gradle index 29b7bc61c0f..cc5e1b0c990 100644 --- a/examples/android/helloworld/app/build.gradle +++ b/examples/android/helloworld/app/build.gradle @@ -32,7 +32,7 @@ android { protobuf { protoc { artifact = 'com.google.protobuf:protoc:3.17.2' } plugins { - grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION + grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.42.0' // CURRENT_GRPC_VERSION } } generateProtoTasks { @@ -52,8 +52,8 @@ dependencies { implementation 'com.android.support:appcompat-v7:27.0.2' // You need to build grpc-java to obtain these libraries below. - implementation 'io.grpc:grpc-okhttp:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-protobuf-lite:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-stub:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-okhttp:1.42.0' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-protobuf-lite:1.42.0' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-stub:1.42.0' // CURRENT_GRPC_VERSION implementation 'org.apache.tomcat:annotations-api:6.0.53' } diff --git a/examples/android/routeguide/app/build.gradle b/examples/android/routeguide/app/build.gradle index 665730306a9..862d709930f 100644 --- a/examples/android/routeguide/app/build.gradle +++ b/examples/android/routeguide/app/build.gradle @@ -32,7 +32,7 @@ android { protobuf { protoc { artifact = 'com.google.protobuf:protoc:3.17.2' } plugins { - grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION + grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.42.0' // CURRENT_GRPC_VERSION } } generateProtoTasks { @@ -52,8 +52,8 @@ dependencies { implementation 'com.android.support:appcompat-v7:27.0.2' // You need to build grpc-java to obtain these libraries below. - implementation 'io.grpc:grpc-okhttp:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-protobuf-lite:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-stub:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-okhttp:1.42.0' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-protobuf-lite:1.42.0' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-stub:1.42.0' // CURRENT_GRPC_VERSION implementation 'org.apache.tomcat:annotations-api:6.0.53' } diff --git a/examples/android/strictmode/app/build.gradle b/examples/android/strictmode/app/build.gradle index 98374f10b4e..46064290c28 100644 --- a/examples/android/strictmode/app/build.gradle +++ b/examples/android/strictmode/app/build.gradle @@ -33,7 +33,7 @@ android { protobuf { protoc { artifact = 'com.google.protobuf:protoc:3.17.2' } plugins { - grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION + grpc { artifact = 'io.grpc:protoc-gen-grpc-java:1.42.0' // CURRENT_GRPC_VERSION } } generateProtoTasks { @@ -53,8 +53,8 @@ dependencies { implementation 'com.android.support:appcompat-v7:28.0.0' // You need to build grpc-java to obtain these libraries below. - implementation 'io.grpc:grpc-okhttp:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-protobuf-lite:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION - implementation 'io.grpc:grpc-stub:1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-okhttp:1.42.0' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-protobuf-lite:1.42.0' // CURRENT_GRPC_VERSION + implementation 'io.grpc:grpc-stub:1.42.0' // CURRENT_GRPC_VERSION implementation 'org.apache.tomcat:annotations-api:6.0.53' } diff --git a/examples/build.gradle b/examples/build.gradle index db8d9b77c17..3aabec66b09 100644 --- a/examples/build.gradle +++ b/examples/build.gradle @@ -22,7 +22,7 @@ targetCompatibility = 1.7 // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. -def grpcVersion = '1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION +def grpcVersion = '1.42.0' // CURRENT_GRPC_VERSION def protobufVersion = '3.17.2' def protocVersion = protobufVersion diff --git a/examples/example-alts/build.gradle b/examples/example-alts/build.gradle index 2925085f57f..81f862f275e 100644 --- a/examples/example-alts/build.gradle +++ b/examples/example-alts/build.gradle @@ -23,7 +23,7 @@ targetCompatibility = 1.7 // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. -def grpcVersion = '1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION +def grpcVersion = '1.42.0' // CURRENT_GRPC_VERSION def protocVersion = '3.17.2' dependencies { diff --git a/examples/example-gauth/build.gradle b/examples/example-gauth/build.gradle index bef50494821..7f8ee1bb4db 100644 --- a/examples/example-gauth/build.gradle +++ b/examples/example-gauth/build.gradle @@ -23,7 +23,7 @@ targetCompatibility = 1.7 // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. -def grpcVersion = '1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION +def grpcVersion = '1.42.0' // CURRENT_GRPC_VERSION def protobufVersion = '3.17.2' def protocVersion = protobufVersion diff --git a/examples/example-gauth/pom.xml b/examples/example-gauth/pom.xml index ba08911a924..1c96c9ae6e0 100644 --- a/examples/example-gauth/pom.xml +++ b/examples/example-gauth/pom.xml @@ -6,13 +6,13 @@ jar - 1.42.0-SNAPSHOT + 1.42.0 example-gauth https://github.com/grpc/grpc-java UTF-8 - 1.42.0-SNAPSHOT + 1.42.0 3.17.2 1.7 diff --git a/examples/example-hostname/build.gradle b/examples/example-hostname/build.gradle index f6e050d0570..9f949b45552 100644 --- a/examples/example-hostname/build.gradle +++ b/examples/example-hostname/build.gradle @@ -21,7 +21,7 @@ targetCompatibility = 1.7 // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. -def grpcVersion = '1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION +def grpcVersion = '1.42.0' // CURRENT_GRPC_VERSION def protobufVersion = '3.17.2' dependencies { diff --git a/examples/example-hostname/pom.xml b/examples/example-hostname/pom.xml index 0a22488d3e3..0d0fd9f8c02 100644 --- a/examples/example-hostname/pom.xml +++ b/examples/example-hostname/pom.xml @@ -6,13 +6,13 @@ jar - 1.42.0-SNAPSHOT + 1.42.0 example-hostname https://github.com/grpc/grpc-java UTF-8 - 1.42.0-SNAPSHOT + 1.42.0 3.17.2 1.7 diff --git a/examples/example-jwt-auth/build.gradle b/examples/example-jwt-auth/build.gradle index 851fa5ce095..6ec5c9da09c 100644 --- a/examples/example-jwt-auth/build.gradle +++ b/examples/example-jwt-auth/build.gradle @@ -22,7 +22,7 @@ targetCompatibility = 1.7 // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. -def grpcVersion = '1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION +def grpcVersion = '1.42.0' // CURRENT_GRPC_VERSION def protobufVersion = '3.17.2' def protocVersion = protobufVersion diff --git a/examples/example-jwt-auth/pom.xml b/examples/example-jwt-auth/pom.xml index 019d0cf4131..d734f0b0824 100644 --- a/examples/example-jwt-auth/pom.xml +++ b/examples/example-jwt-auth/pom.xml @@ -7,13 +7,13 @@ jar - 1.42.0-SNAPSHOT + 1.42.0 example-jwt-auth https://github.com/grpc/grpc-java UTF-8 - 1.42.0-SNAPSHOT + 1.42.0 3.17.2 3.17.2 diff --git a/examples/example-tls/build.gradle b/examples/example-tls/build.gradle index a1696cfabdd..2836e1144ec 100644 --- a/examples/example-tls/build.gradle +++ b/examples/example-tls/build.gradle @@ -23,7 +23,7 @@ targetCompatibility = 1.7 // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. -def grpcVersion = '1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION +def grpcVersion = '1.42.0' // CURRENT_GRPC_VERSION def protocVersion = '3.17.2' dependencies { diff --git a/examples/example-tls/pom.xml b/examples/example-tls/pom.xml index 4a1dfe1be15..60174b3ccfc 100644 --- a/examples/example-tls/pom.xml +++ b/examples/example-tls/pom.xml @@ -6,13 +6,13 @@ jar - 1.42.0-SNAPSHOT + 1.42.0 example-tls https://github.com/grpc/grpc-java UTF-8 - 1.42.0-SNAPSHOT + 1.42.0 3.17.2 2.0.34.Final diff --git a/examples/example-xds/build.gradle b/examples/example-xds/build.gradle index 9b5a5ee745f..91d7cf42fe9 100644 --- a/examples/example-xds/build.gradle +++ b/examples/example-xds/build.gradle @@ -22,7 +22,7 @@ targetCompatibility = 1.7 // Feel free to delete the comment at the next line. It is just for safely // updating the version in our release process. -def grpcVersion = '1.42.0-SNAPSHOT' // CURRENT_GRPC_VERSION +def grpcVersion = '1.42.0' // CURRENT_GRPC_VERSION def nettyTcNativeVersion = '2.0.31.Final' def protocVersion = '3.17.2' diff --git a/examples/pom.xml b/examples/pom.xml index 93b9e502b90..57c68231b10 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -6,13 +6,13 @@ jar - 1.42.0-SNAPSHOT + 1.42.0 examples https://github.com/grpc/grpc-java UTF-8 - 1.42.0-SNAPSHOT + 1.42.0 3.17.2 3.17.2