diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index a79f06271..68f2b159d 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-java:latest - digest: sha256:2567a120ce90fadb6201999b87d649d9f67459de28815ad239bce9ebfaa18a74 -# created: 2022-05-19T15:12:45.278246753Z + digest: sha256:58ccd4737212f64a7dd4b3063d447447acf71a2b9d409eab19fc7a00b18eadc0 +# created: 2022-06-10T19:20:11.004014696Z diff --git a/CHANGELOG.md b/CHANGELOG.md index 74eff90d3..7b827c74b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## [2.9.0](https://github.com/googleapis/java-datastore/compare/v2.8.0...v2.9.0) (2022-06-22) + + +### Features + +* support readTime in Datastore query splitter. ([#763](https://github.com/googleapis/java-datastore/issues/763)) ([61758e0](https://github.com/googleapis/java-datastore/commit/61758e02c30c8410dd397d0cc77c987332c4f11c)) + + +### Documentation + +* **sample:** clean up README for native image sample ([#771](https://github.com/googleapis/java-datastore/issues/771)) ([7358aa3](https://github.com/googleapis/java-datastore/commit/7358aa34ec9d3d52aae3195fea718ef748ab22b1)) + + +### Dependencies + +* update dependency org.graalvm.buildtools:junit-platform-native to v0.9.12 ([#773](https://github.com/googleapis/java-datastore/issues/773)) ([cab7e54](https://github.com/googleapis/java-datastore/commit/cab7e54359a4fad5fca23b89a9cf52f95e53e19e)) +* update dependency org.graalvm.buildtools:native-maven-plugin to v0.9.12 ([#774](https://github.com/googleapis/java-datastore/issues/774)) ([496c1bc](https://github.com/googleapis/java-datastore/commit/496c1bcb4c7343fd8330629f82ca9f96fb1a9acc)) + ## [2.8.0](https://github.com/googleapis/java-datastore/compare/v2.7.0...v2.8.0) (2022-06-09) diff --git a/README.md b/README.md index 3b3853f06..7781708dd 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ If you are using Maven without BOM, add this to your dependencies: com.google.cloud google-cloud-datastore - 2.7.0 + 2.8.0 ``` @@ -56,13 +56,13 @@ implementation 'com.google.cloud:google-cloud-datastore' If you are using Gradle without BOM, add this to your dependencies ```Groovy -implementation 'com.google.cloud:google-cloud-datastore:2.7.0' +implementation 'com.google.cloud:google-cloud-datastore:2.8.0' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-datastore" % "2.7.0" +libraryDependencies += "com.google.cloud" % "google-cloud-datastore" % "2.8.0" ``` ## Authentication diff --git a/datastore-v1-proto-client/clirr-ignored-differences.xml b/datastore-v1-proto-client/clirr-ignored-differences.xml new file mode 100644 index 000000000..e8c0b27f4 --- /dev/null +++ b/datastore-v1-proto-client/clirr-ignored-differences.xml @@ -0,0 +1,9 @@ + + + + + com/google/datastore/v1/client/QuerySplitter + java.util.List getSplits(com.google.datastore.v1.Query, com.google.datastore.v1.PartitionId, int, com.google.datastore.v1.client.Datastore, com.google.protobuf.Timestamp) + 7012 + + diff --git a/datastore-v1-proto-client/pom.xml b/datastore-v1-proto-client/pom.xml index 8dd942d71..01fd0ee6b 100644 --- a/datastore-v1-proto-client/pom.xml +++ b/datastore-v1-proto-client/pom.xml @@ -19,12 +19,12 @@ 4.0.0 com.google.cloud.datastore datastore-v1-proto-client - 2.8.0 + 2.9.0 com.google.cloud google-cloud-datastore-parent - 2.8.0 + 2.9.0 jar @@ -83,6 +83,11 @@ protobuf-java + + com.google.api + api-common + + junit diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/QuerySplitter.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/QuerySplitter.java index 5286f7842..97268d38d 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/QuerySplitter.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/QuerySplitter.java @@ -15,8 +15,10 @@ */ package com.google.datastore.v1.client; +import com.google.api.core.BetaApi; import com.google.datastore.v1.PartitionId; import com.google.datastore.v1.Query; +import com.google.protobuf.Timestamp; import java.util.List; /** Provides the ability to split a query into multiple shards. */ @@ -39,4 +41,16 @@ public interface QuerySplitter { */ List getSplits(Query query, PartitionId partition, int numSplits, Datastore datastore) throws DatastoreException; + + /** + * Same as {@link #getSplits(Query, PartitionId, int, Datastore)} but the splits are based on + * {@code readTime}, and the returned sharded {@link Query}s should also be executed with {@code + * readTime}. Reading from a timestamp is currently a private preview feature in Datastore. + */ + @BetaApi + default List getSplits( + Query query, PartitionId partition, int numSplits, Datastore datastore, Timestamp readTime) + throws DatastoreException { + throw new UnsupportedOperationException("Not implemented."); + } } diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/QuerySplitterImpl.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/QuerySplitterImpl.java index 92fb38418..6143bdd59 100644 --- a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/QuerySplitterImpl.java +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/QuerySplitterImpl.java @@ -17,6 +17,7 @@ import static com.google.datastore.v1.client.DatastoreHelper.makeAndFilter; +import com.google.api.core.BetaApi; import com.google.datastore.v1.EntityResult; import com.google.datastore.v1.Filter; import com.google.datastore.v1.Key; @@ -29,11 +30,14 @@ import com.google.datastore.v1.Query; import com.google.datastore.v1.QueryResultBatch; import com.google.datastore.v1.QueryResultBatch.MoreResultsType; +import com.google.datastore.v1.ReadOptions; import com.google.datastore.v1.RunQueryRequest; +import com.google.protobuf.Timestamp; import java.util.ArrayList; import java.util.Collections; import java.util.EnumSet; import java.util.List; +import javax.annotation.Nullable; /** * Provides the ability to split a query into multiple shards using Cloud Datastore. @@ -63,7 +67,24 @@ private QuerySplitterImpl() { public List getSplits( Query query, PartitionId partition, int numSplits, Datastore datastore) throws DatastoreException, IllegalArgumentException { + return getSplitsInternal(query, partition, numSplits, datastore, null); + } + @BetaApi + @Override + public List getSplits( + Query query, PartitionId partition, int numSplits, Datastore datastore, Timestamp readTime) + throws DatastoreException, IllegalArgumentException { + return getSplitsInternal(query, partition, numSplits, datastore, readTime); + } + + private List getSplitsInternal( + Query query, + PartitionId partition, + int numSplits, + Datastore datastore, + @Nullable Timestamp readTime) + throws DatastoreException, IllegalArgumentException { List splits = new ArrayList(numSplits); if (numSplits == 1) { splits.add(query); @@ -72,7 +93,7 @@ public List getSplits( validateQuery(query); validateSplitSize(numSplits); - List scatterKeys = getScatterKeys(numSplits, query, partition, datastore); + List scatterKeys = getScatterKeys(numSplits, query, partition, datastore, readTime); Key lastKey = null; for (Key nextKey : getSplitKey(scatterKeys, numSplits)) { splits.add(createSplit(lastKey, nextKey, query)); @@ -182,10 +203,15 @@ private Query createSplit(Key lastKey, Key nextKey, Query query) { * @param query the user query. * @param partition the partition to run the query in. * @param datastore the datastore containing the data. + * @param readTime read time at which to get the split keys from the datastore. * @throws DatastoreException if there was an error when executing the datastore query. */ private List getScatterKeys( - int numSplits, Query query, PartitionId partition, Datastore datastore) + int numSplits, + Query query, + PartitionId partition, + Datastore datastore, + @Nullable Timestamp readTime) throws DatastoreException { Query.Builder scatterPointQuery = createScatterQuery(query, numSplits); @@ -193,12 +219,12 @@ private List getScatterKeys( QueryResultBatch batch; do { - RunQueryRequest scatterRequest = - RunQueryRequest.newBuilder() - .setPartitionId(partition) - .setQuery(scatterPointQuery) - .build(); - batch = datastore.runQuery(scatterRequest).getBatch(); + RunQueryRequest.Builder scatterRequest = + RunQueryRequest.newBuilder().setPartitionId(partition).setQuery(scatterPointQuery); + if (readTime != null) { + scatterRequest.setReadOptions(ReadOptions.newBuilder().setReadTime(readTime).build()); + } + batch = datastore.runQuery(scatterRequest.build()).getBatch(); for (EntityResult result : batch.getEntityResultsList()) { keySplits.add(result.getEntity().getKey()); } diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/testing/MockCredential.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/testing/MockCredential.java new file mode 100644 index 000000000..7579f58b3 --- /dev/null +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/testing/MockCredential.java @@ -0,0 +1,36 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.datastore.v1.client.testing; + +import com.google.api.client.auth.oauth2.Credential; +import com.google.api.client.http.HttpRequest; +import java.io.IOException; + +/** Fake credential used for testing purpose. */ +public class MockCredential extends Credential { + public MockCredential() { + super( + new AccessMethod() { + @Override + public void intercept(HttpRequest request, String accessToken) throws IOException {} + + @Override + public String getAccessTokenFromRequest(HttpRequest request) { + return "MockAccessToken"; + } + }); + } +} diff --git a/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/testing/MockDatastoreFactory.java b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/testing/MockDatastoreFactory.java new file mode 100644 index 000000000..6942a5d79 --- /dev/null +++ b/datastore-v1-proto-client/src/main/java/com/google/datastore/v1/client/testing/MockDatastoreFactory.java @@ -0,0 +1,137 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.datastore.v1.client.testing; + +import static com.google.common.base.Preconditions.checkState; + +import com.google.api.client.auth.oauth2.Credential; +import com.google.api.client.http.GenericUrl; +import com.google.api.client.http.HttpRequestFactory; +import com.google.api.client.http.HttpStatusCodes; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.http.LowLevelHttpRequest; +import com.google.api.client.http.LowLevelHttpResponse; +import com.google.api.client.testing.http.MockHttpTransport; +import com.google.api.client.testing.http.MockLowLevelHttpRequest; +import com.google.api.client.testing.http.MockLowLevelHttpResponse; +import com.google.api.client.testing.util.TestableByteArrayInputStream; +import com.google.common.collect.Iterables; +import com.google.datastore.v1.client.DatastoreFactory; +import com.google.datastore.v1.client.DatastoreOptions; +import com.google.protobuf.Message; +import com.google.rpc.Code; +import com.google.rpc.Status; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.List; + +/** Fake Datastore factory used for testing purposes when a true Datastore service is not needed. */ +public class MockDatastoreFactory extends DatastoreFactory { + private int nextStatus; + private Message nextResponse; + private Status nextError; + private IOException nextException; + + private String lastPath; + private String lastMimeType; + private byte[] lastBody; + private List lastCookies; + private String lastApiFormatHeaderValue; + + public void setNextResponse(Message response) { + nextStatus = HttpStatusCodes.STATUS_CODE_OK; + nextResponse = response; + nextError = null; + nextException = null; + } + + public void setNextError(int status, Code code, String message) { + nextStatus = status; + nextResponse = null; + nextError = makeErrorContent(message, code); + nextException = null; + } + + public void setNextException(IOException exception) { + nextStatus = 0; + nextResponse = null; + nextError = null; + nextException = exception; + } + + @Override + public HttpRequestFactory makeClient(DatastoreOptions options) { + HttpTransport transport = + new MockHttpTransport() { + @Override + public LowLevelHttpRequest buildRequest(String method, String url) { + return new MockLowLevelHttpRequest(url) { + @Override + public LowLevelHttpResponse execute() throws IOException { + lastPath = new GenericUrl(getUrl()).getRawPath(); + lastMimeType = getContentType(); + lastCookies = getHeaderValues("Cookie"); + lastApiFormatHeaderValue = + Iterables.getOnlyElement(getHeaderValues("X-Goog-Api-Format-Version")); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + getStreamingContent().writeTo(out); + lastBody = out.toByteArray(); + if (nextException != null) { + throw nextException; + } + MockLowLevelHttpResponse response = + new MockLowLevelHttpResponse() + .setStatusCode(nextStatus) + .setContentType("application/x-protobuf"); + if (nextError != null) { + checkState(nextResponse == null); + response.setContent(new TestableByteArrayInputStream(nextError.toByteArray())); + } else { + response.setContent(new TestableByteArrayInputStream(nextResponse.toByteArray())); + } + return response; + } + }; + } + }; + Credential credential = options.getCredential(); + return transport.createRequestFactory(credential); + } + + public String getLastPath() { + return lastPath; + } + + public String getLastMimeType() { + return lastMimeType; + } + + public String getLastApiFormatHeaderValue() { + return lastApiFormatHeaderValue; + } + + public byte[] getLastBody() { + return lastBody; + } + + public List getLastCookies() { + return lastCookies; + } + + private static Status makeErrorContent(String message, Code code) { + return Status.newBuilder().setCode(code.getNumber()).setMessage(message).build(); + } +} diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreClientTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreClientTest.java index d8376dc29..2ab2c89f8 100644 --- a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreClientTest.java +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/DatastoreClientTest.java @@ -18,25 +18,12 @@ import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertThrows; import static org.junit.Assert.fail; -import com.google.api.client.auth.oauth2.Credential; -import com.google.api.client.http.GenericUrl; import com.google.api.client.http.HttpRequest; -import com.google.api.client.http.HttpRequestFactory; import com.google.api.client.http.HttpRequestInitializer; -import com.google.api.client.http.HttpStatusCodes; -import com.google.api.client.http.HttpTransport; -import com.google.api.client.http.LowLevelHttpRequest; -import com.google.api.client.http.LowLevelHttpResponse; -import com.google.api.client.testing.http.MockHttpTransport; -import com.google.api.client.testing.http.MockLowLevelHttpRequest; -import com.google.api.client.testing.http.MockLowLevelHttpResponse; -import com.google.api.client.testing.util.TestableByteArrayInputStream; -import com.google.common.collect.Iterables; import com.google.datastore.v1.AllocateIdsRequest; import com.google.datastore.v1.AllocateIdsResponse; import com.google.datastore.v1.BeginTransactionRequest; @@ -53,16 +40,15 @@ import com.google.datastore.v1.RollbackResponse; import com.google.datastore.v1.RunQueryRequest; import com.google.datastore.v1.RunQueryResponse; +import com.google.datastore.v1.client.testing.MockCredential; +import com.google.datastore.v1.client.testing.MockDatastoreFactory; import com.google.protobuf.ByteString; import com.google.protobuf.Message; import com.google.rpc.Code; -import com.google.rpc.Status; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.SocketTimeoutException; -import java.util.List; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -290,7 +276,7 @@ public void initialize(HttpRequest request) { AllocateIdsResponse response = AllocateIdsResponse.newBuilder().build(); mockClient.setNextResponse(response); assertEquals(response, datastore.allocateIds(request)); - assertEquals("magic", mockClient.lastCookies.get(0)); + assertEquals("magic", mockClient.getLastCookies().get(0)); } @Test @@ -361,10 +347,10 @@ private void expectRpc(String methodName, Message request, Message response) thr Object[] callArgs = {request}; assertEquals(response, call.invoke(datastore, callArgs)); - assertEquals("/v1/projects/project-id:" + methodName, mockClient.lastPath); - assertEquals("application/x-protobuf", mockClient.lastMimeType); - assertEquals("2", mockClient.lastApiFormatHeaderValue); - assertArrayEquals(request.toByteArray(), mockClient.lastBody); + assertEquals("/v1/projects/project-id:" + methodName, mockClient.getLastPath()); + assertEquals("application/x-protobuf", mockClient.getLastMimeType()); + assertEquals("2", mockClient.getLastApiFormatHeaderValue()); + assertArrayEquals(request.toByteArray(), mockClient.getLastBody()); assertEquals(1, datastore.getRpcCount()); datastore.resetRpcCount(); @@ -409,97 +395,4 @@ private void expectRpc(String methodName, Message request, Message response) thr assertEquals(3, datastore.getRpcCount()); } - - private static class MockCredential extends Credential { - MockCredential() { - super( - new AccessMethod() { - @Override - public void intercept(HttpRequest request, String accessToken) throws IOException {} - - @Override - public String getAccessTokenFromRequest(HttpRequest request) { - return "MockAccessToken"; - } - }); - } - } - - private static class MockDatastoreFactory extends DatastoreFactory { - int nextStatus; - Message nextResponse; - Status nextError; - IOException nextException; - - String lastPath; - String lastMimeType; - byte[] lastBody; - List lastCookies; - String lastApiFormatHeaderValue; - - void setNextResponse(Message response) { - nextStatus = HttpStatusCodes.STATUS_CODE_OK; - nextResponse = response; - nextError = null; - nextException = null; - } - - void setNextError(int status, Code code, String message) { - nextStatus = status; - nextResponse = null; - nextError = makeErrorContent(message, code); - nextException = null; - } - - void setNextException(IOException exception) { - nextStatus = 0; - nextResponse = null; - nextError = null; - nextException = exception; - } - - @Override - public HttpRequestFactory makeClient(DatastoreOptions options) { - HttpTransport transport = - new MockHttpTransport() { - @Override - public LowLevelHttpRequest buildRequest(String method, String url) { - return new MockLowLevelHttpRequest(url) { - @Override - public LowLevelHttpResponse execute() throws IOException { - lastPath = new GenericUrl(getUrl()).getRawPath(); - lastMimeType = getContentType(); - lastCookies = getHeaderValues("Cookie"); - lastApiFormatHeaderValue = - Iterables.getOnlyElement(getHeaderValues("X-Goog-Api-Format-Version")); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - getStreamingContent().writeTo(out); - lastBody = out.toByteArray(); - if (nextException != null) { - throw nextException; - } - MockLowLevelHttpResponse response = - new MockLowLevelHttpResponse() - .setStatusCode(nextStatus) - .setContentType("application/x-protobuf"); - if (nextError != null) { - assertNull(nextResponse); - response.setContent(new TestableByteArrayInputStream(nextError.toByteArray())); - } else { - response.setContent( - new TestableByteArrayInputStream(nextResponse.toByteArray())); - } - return response; - } - }; - } - }; - Credential credential = options.getCredential(); - return transport.createRequestFactory(credential); - } - } - - private static Status makeErrorContent(String message, Code code) { - return Status.newBuilder().setCode(code.getNumber()).setMessage(message).build(); - } } diff --git a/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/QuerySplitterTest.java b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/QuerySplitterTest.java new file mode 100644 index 000000000..e86943724 --- /dev/null +++ b/datastore-v1-proto-client/src/test/java/com/google/datastore/v1/client/QuerySplitterTest.java @@ -0,0 +1,326 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.datastore.v1.client; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.datastore.v1.client.DatastoreHelper.makeAndFilter; +import static com.google.datastore.v1.client.DatastoreHelper.makeFilter; +import static com.google.datastore.v1.client.DatastoreHelper.makeKey; +import static com.google.datastore.v1.client.DatastoreHelper.makeOrder; +import static com.google.datastore.v1.client.DatastoreHelper.makePropertyReference; +import static com.google.datastore.v1.client.DatastoreHelper.makeValue; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertThrows; + +import com.google.datastore.v1.Entity; +import com.google.datastore.v1.EntityResult; +import com.google.datastore.v1.EntityResult.ResultType; +import com.google.datastore.v1.Filter; +import com.google.datastore.v1.Key; +import com.google.datastore.v1.KindExpression; +import com.google.datastore.v1.PartitionId; +import com.google.datastore.v1.Projection; +import com.google.datastore.v1.PropertyFilter.Operator; +import com.google.datastore.v1.PropertyOrder.Direction; +import com.google.datastore.v1.Query; +import com.google.datastore.v1.QueryResultBatch; +import com.google.datastore.v1.QueryResultBatch.MoreResultsType; +import com.google.datastore.v1.ReadOptions; +import com.google.datastore.v1.RunQueryRequest; +import com.google.datastore.v1.RunQueryResponse; +import com.google.datastore.v1.client.testing.MockCredential; +import com.google.datastore.v1.client.testing.MockDatastoreFactory; +import com.google.protobuf.Int32Value; +import com.google.protobuf.Timestamp; +import java.util.List; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests for {@link QuerySplitterImpl}. */ +@RunWith(JUnit4.class) +public class QuerySplitterTest { + private static final String PROJECT_ID = "project-id"; + private static final PartitionId PARTITION = + PartitionId.newBuilder().setProjectId(PROJECT_ID).build(); + private static final String KIND = "test-kind"; + + private DatastoreFactory factory = new MockDatastoreFactory(); + private DatastoreOptions.Builder options = + new DatastoreOptions.Builder().projectId(PROJECT_ID).credential(new MockCredential()); + + private Filter propertyFilter = makeFilter("foo", Operator.EQUAL, makeValue("value")).build(); + + private Query query = + Query.newBuilder() + .addKind(KindExpression.newBuilder().setName(KIND).build()) + .setFilter(propertyFilter) + .build(); + + private Query splitQuery = + Query.newBuilder() + .addKind(KindExpression.newBuilder().setName(KIND).build()) + .addOrder(makeOrder("__scatter__", Direction.ASCENDING)) + .addProjection(Projection.newBuilder().setProperty(makePropertyReference("__key__"))) + .build(); + + private Key splitKey0 = makeKey(KIND, String.format("%05d", 1)).setPartitionId(PARTITION).build(); + private Key splitKey1 = + makeKey(KIND, String.format("%05d", 101)).setPartitionId(PARTITION).build(); + private Key splitKey2 = + makeKey(KIND, String.format("%05d", 201)).setPartitionId(PARTITION).build(); + private Key splitKey3 = + makeKey(KIND, String.format("%05d", 301)).setPartitionId(PARTITION).build(); + + @Test + public void disallowsSortOrder() { + Datastore datastore = factory.create(options.build()); + Query queryWithOrder = + query.toBuilder().addOrder(makeOrder("bar", Direction.ASCENDING)).build(); + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> QuerySplitterImpl.INSTANCE.getSplits(queryWithOrder, PARTITION, 2, datastore)); + assertThat(exception).hasMessageThat().contains("Query cannot have any sort orders."); + } + + @Test + public void disallowsMultipleKinds() { + Datastore datastore = factory.create(options.build()); + Query queryWithMultipleKinds = + query + .toBuilder() + .addKind(KindExpression.newBuilder().setName("another-kind").build()) + .build(); + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> + QuerySplitterImpl.INSTANCE.getSplits( + queryWithMultipleKinds, PARTITION, 2, datastore)); + assertThat(exception).hasMessageThat().contains("Query must have exactly one kind."); + } + + @Test + public void disallowsKindlessQuery() { + Datastore datastore = factory.create(options.build()); + Query kindlessQuery = query.toBuilder().clearKind().build(); + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> QuerySplitterImpl.INSTANCE.getSplits(kindlessQuery, PARTITION, 2, datastore)); + assertThat(exception).hasMessageThat().contains("Query must have exactly one kind."); + } + + @Test + public void disallowsInequalityFilter() { + Datastore datastore = factory.create(options.build()); + Query queryWithInequality = + query + .toBuilder() + .setFilter(makeFilter("foo", Operator.GREATER_THAN, makeValue("value"))) + .build(); + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> + QuerySplitterImpl.INSTANCE.getSplits(queryWithInequality, PARTITION, 2, datastore)); + assertThat(exception).hasMessageThat().contains("Query cannot have any inequality filters."); + } + + @Test + public void splitsMustBePositive() { + Datastore datastore = factory.create(options.build()); + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> QuerySplitterImpl.INSTANCE.getSplits(query, PARTITION, 0, datastore)); + assertThat(exception).hasMessageThat().contains("The number of splits must be greater than 0."); + } + + @Test + public void getSplits() throws Exception { + Datastore datastore = factory.create(options.build()); + MockDatastoreFactory mockClient = (MockDatastoreFactory) factory; + + RunQueryResponse splitQueryResponse = + RunQueryResponse.newBuilder() + .setQuery(splitQuery) + .setBatch( + QueryResultBatch.newBuilder() + .setEntityResultType(ResultType.KEY_ONLY) + .setMoreResults(MoreResultsType.NO_MORE_RESULTS) + .addEntityResults(makeKeyOnlyEntity(splitKey0)) + .addEntityResults(makeKeyOnlyEntity(splitKey1)) + .addEntityResults(makeKeyOnlyEntity(splitKey2)) + .addEntityResults(makeKeyOnlyEntity(splitKey3)) + .build()) + .build(); + + mockClient.setNextResponse(splitQueryResponse); + + List splittedQueries = + QuerySplitterImpl.INSTANCE.getSplits(query, PARTITION, 3, datastore); + + assertThat(splittedQueries) + .containsExactly( + query + .toBuilder() + .setFilter(makeFilterWithKeyRange(propertyFilter, null, splitKey1)) + .build(), + query + .toBuilder() + .setFilter(makeFilterWithKeyRange(propertyFilter, splitKey1, splitKey3)) + .build(), + query + .toBuilder() + .setFilter(makeFilterWithKeyRange(propertyFilter, splitKey3, null)) + .build()); + + RunQueryRequest expectedSplitQueryRequest = + RunQueryRequest.newBuilder() + .setPartitionId(PARTITION) + .setQuery( + splitQuery.toBuilder().setLimit(Int32Value.newBuilder().setValue(2 * 32).build())) + .build(); + + assertArrayEquals(expectedSplitQueryRequest.toByteArray(), mockClient.getLastBody()); + } + + @Test + public void notEnoughSplits() throws Exception { + Datastore datastore = factory.create(options.build()); + MockDatastoreFactory mockClient = (MockDatastoreFactory) factory; + + RunQueryResponse splitQueryResponse = + RunQueryResponse.newBuilder() + .setQuery(splitQuery) + .setBatch( + QueryResultBatch.newBuilder() + .setEntityResultType(ResultType.KEY_ONLY) + .setMoreResults(MoreResultsType.NO_MORE_RESULTS) + .addEntityResults(makeKeyOnlyEntity(splitKey0)) + .build()) + .build(); + + mockClient.setNextResponse(splitQueryResponse); + + List splittedQueries = + QuerySplitterImpl.INSTANCE.getSplits(query, PARTITION, 100, datastore); + + assertThat(splittedQueries) + .containsExactly( + query + .toBuilder() + .setFilter(makeFilterWithKeyRange(propertyFilter, null, splitKey0)) + .build(), + query + .toBuilder() + .setFilter(makeFilterWithKeyRange(propertyFilter, splitKey0, null)) + .build()); + + RunQueryRequest expectedSplitQueryRequest = + RunQueryRequest.newBuilder() + .setPartitionId(PARTITION) + .setQuery( + splitQuery.toBuilder().setLimit(Int32Value.newBuilder().setValue(99 * 32).build())) + .build(); + + assertArrayEquals(expectedSplitQueryRequest.toByteArray(), mockClient.getLastBody()); + } + + @Test + public void getSplits_withReadTime() throws Exception { + Datastore datastore = factory.create(options.build()); + MockDatastoreFactory mockClient = (MockDatastoreFactory) factory; + + RunQueryResponse splitQueryResponse = + RunQueryResponse.newBuilder() + .setQuery(splitQuery) + .setBatch( + QueryResultBatch.newBuilder() + .setEntityResultType(ResultType.KEY_ONLY) + .setMoreResults(MoreResultsType.NO_MORE_RESULTS) + .addEntityResults(makeKeyOnlyEntity(splitKey0)) + .addEntityResults(makeKeyOnlyEntity(splitKey1)) + .addEntityResults(makeKeyOnlyEntity(splitKey2)) + .addEntityResults(makeKeyOnlyEntity(splitKey3)) + .build()) + .build(); + + mockClient.setNextResponse(splitQueryResponse); + + Timestamp readTime = Timestamp.newBuilder().setSeconds(1654651341L).build(); + + List splittedQueries = + QuerySplitterImpl.INSTANCE.getSplits(query, PARTITION, 3, datastore, readTime); + + assertThat(splittedQueries) + .containsExactly( + query + .toBuilder() + .setFilter(makeFilterWithKeyRange(propertyFilter, null, splitKey1)) + .build(), + query + .toBuilder() + .setFilter(makeFilterWithKeyRange(propertyFilter, splitKey1, splitKey3)) + .build(), + query + .toBuilder() + .setFilter(makeFilterWithKeyRange(propertyFilter, splitKey3, null)) + .build()); + + RunQueryRequest expectedSplitQueryRequest = + RunQueryRequest.newBuilder() + .setPartitionId(PARTITION) + .setQuery( + splitQuery.toBuilder().setLimit(Int32Value.newBuilder().setValue(2 * 32).build())) + .setReadOptions(ReadOptions.newBuilder().setReadTime(readTime)) + .build(); + + assertArrayEquals(expectedSplitQueryRequest.toByteArray(), mockClient.getLastBody()); + } + + private static EntityResult makeKeyOnlyEntity(Key key) { + return EntityResult.newBuilder().setEntity(Entity.newBuilder().setKey(key).build()).build(); + } + + private static Filter makeFilterWithKeyRange(Filter originalFilter, Key startKey, Key endKey) { + Filter startKeyFilter = + startKey == null + ? null + : makeFilter("__key__", Operator.GREATER_THAN_OR_EQUAL, makeValue(startKey)).build(); + + Filter endKeyFilter = + endKey == null + ? null + : makeFilter("__key__", Operator.LESS_THAN, makeValue(endKey)).build(); + + if (startKeyFilter == null && endKeyFilter == null) { + throw new IllegalArgumentException(); + } + + if (startKeyFilter != null && endKeyFilter == null) { + return makeAndFilter(originalFilter, startKeyFilter).build(); + } + + if (startKeyFilter == null && endKeyFilter != null) { + return makeAndFilter(originalFilter, endKeyFilter).build(); + } + + return makeAndFilter(originalFilter, startKeyFilter, endKeyFilter).build(); + } +} diff --git a/google-cloud-datastore-bom/pom.xml b/google-cloud-datastore-bom/pom.xml index f452f3022..0e88ee71e 100644 --- a/google-cloud-datastore-bom/pom.xml +++ b/google-cloud-datastore-bom/pom.xml @@ -3,12 +3,12 @@ 4.0.0 com.google.cloud google-cloud-datastore-bom - 2.8.0 + 2.9.0 pom com.google.cloud google-cloud-shared-config - 1.4.0 + 1.5.0 Google Cloud datastore BOM @@ -52,22 +52,22 @@ com.google.cloud google-cloud-datastore - 2.8.0 + 2.9.0 com.google.api.grpc grpc-google-cloud-datastore-admin-v1 - 2.8.0 + 2.9.0 com.google.api.grpc proto-google-cloud-datastore-v1 - 0.99.0 + 0.100.0 com.google.api.grpc proto-google-cloud-datastore-admin-v1 - 2.8.0 + 2.9.0 diff --git a/google-cloud-datastore/pom.xml b/google-cloud-datastore/pom.xml index 49778e706..c2fc5f547 100644 --- a/google-cloud-datastore/pom.xml +++ b/google-cloud-datastore/pom.xml @@ -2,7 +2,7 @@ 4.0.0 google-cloud-datastore - 2.8.0 + 2.9.0 jar Google Cloud Datastore https://github.com/googleapis/java-datastore @@ -12,7 +12,7 @@ com.google.cloud google-cloud-datastore-parent - 2.8.0 + 2.9.0 google-cloud-datastore diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/admin/v1/DatastoreAdminClient.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/admin/v1/DatastoreAdminClient.java index f29717e5a..ec653b844 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/admin/v1/DatastoreAdminClient.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/admin/v1/DatastoreAdminClient.java @@ -888,7 +888,7 @@ public final ListIndexesPagedResponse listIndexes(ListIndexesRequest request) { * .build(); * while (true) { * ListIndexesResponse response = datastoreAdminClient.listIndexesCallable().call(request); - * for (Index element : response.getResponsesList()) { + * for (Index element : response.getIndexesList()) { * // doThingsWith(element); * } * String nextPageToken = response.getNextPageToken(); diff --git a/grpc-google-cloud-datastore-admin-v1/pom.xml b/grpc-google-cloud-datastore-admin-v1/pom.xml index 6e2eb88fd..333ed662c 100644 --- a/grpc-google-cloud-datastore-admin-v1/pom.xml +++ b/grpc-google-cloud-datastore-admin-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-datastore-admin-v1 - 2.8.0 + 2.9.0 grpc-google-cloud-datastore-admin-v1 GRPC library for google-cloud-datastore com.google.cloud google-cloud-datastore-parent - 2.8.0 + 2.9.0 diff --git a/pom.xml b/pom.xml index b10de9616..26982d78d 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-datastore-parent pom - 2.8.0 + 2.9.0 Google Cloud Datastore Parent https://github.com/googleapis/java-datastore @@ -14,7 +14,7 @@ com.google.cloud google-cloud-shared-config - 1.4.0 + 1.5.0 @@ -159,27 +159,27 @@ com.google.api.grpc proto-google-cloud-datastore-admin-v1 - 2.8.0 + 2.9.0 com.google.api.grpc grpc-google-cloud-datastore-admin-v1 - 2.8.0 + 2.9.0 com.google.cloud google-cloud-datastore - 2.8.0 + 2.9.0 com.google.api.grpc proto-google-cloud-datastore-v1 - 0.99.0 + 0.100.0 com.google.cloud.datastore datastore-v1-proto-client - 2.8.0 + 2.9.0 com.google.api.grpc diff --git a/proto-google-cloud-datastore-admin-v1/pom.xml b/proto-google-cloud-datastore-admin-v1/pom.xml index 81fe70639..c61cc3e4d 100644 --- a/proto-google-cloud-datastore-admin-v1/pom.xml +++ b/proto-google-cloud-datastore-admin-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-datastore-admin-v1 - 2.8.0 + 2.9.0 proto-google-cloud-datastore-admin-v1 Proto library for google-cloud-datastore com.google.cloud google-cloud-datastore-parent - 2.8.0 + 2.9.0 diff --git a/proto-google-cloud-datastore-v1/pom.xml b/proto-google-cloud-datastore-v1/pom.xml index 2cd67b83c..4de0ebb7c 100644 --- a/proto-google-cloud-datastore-v1/pom.xml +++ b/proto-google-cloud-datastore-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-datastore-v1 - 0.99.0 + 0.100.0 proto-google-cloud-datastore-v1 PROTO library for proto-google-cloud-datastore-v1 com.google.cloud google-cloud-datastore-parent - 2.8.0 + 2.9.0 diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index e410952e4..130b6d8ca 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -29,7 +29,7 @@ com.google.cloud google-cloud-datastore - 2.7.0 + 2.8.0 diff --git a/samples/native-image-sample/README.md b/samples/native-image-sample/README.md index 94d053bdc..5f2cfbd27 100644 --- a/samples/native-image-sample/README.md +++ b/samples/native-image-sample/README.md @@ -20,25 +20,25 @@ You will need to follow these prerequisite steps in order to run the samples: **Note:** Authenticating with Application Default Credentials is convenient to use during development, but we recommend [alternate methods of authentication](https://cloud.google.com/docs/authentication/production) during production use. -3. Install the GraalVM compiler. +3. Install the native image compiler. - You can follow the [official installation instructions](https://www.graalvm.org/docs/getting-started/#install-graalvm) from the GraalVM website. + You can follow the [installation instructions](https://www.graalvm.org/docs/getting-started/#install-graalvm). After following the instructions, ensure that you install the native image extension installed by running: ``` gu install native-image ``` - Once you finish following the instructions, verify that the default version of Java is set to the GraalVM version by running `java -version` in a terminal. + Once you finish following the instructions, verify that the default version of Java is set to the correct version by running `java -version` in a terminal. You will see something similar to the below output: ``` $ java -version - openjdk version "11.0.7" 2020-04-14 - OpenJDK Runtime Environment GraalVM CE 20.1.0 (build 11.0.7+10-jvmci-20.1-b02) - OpenJDK 64-Bit Server VM GraalVM CE 20.1.0 (build 11.0.7+10-jvmci-20.1-b02, mixed mode, sharing) + openjdk version "17.0.3" 2022-04-19 + OpenJDK Runtime Environment GraalVM CE 22.1.0 (build 17.0.3+7-jvmci-22.1-b06) + OpenJDK 64-Bit Server VM GraalVM CE 22.1.0 (build 17.0.3+7-jvmci-22.1-b06, mixed mode, sharing) ``` ## Sample 1. **(Optional)** If you wish to run the application against the [Datastore emulator](https://cloud.google.com/sdk/gcloud/reference/beta/emulators/datastore), ensure that you have the [Google Cloud SDK](https://cloud.google.com/sdk) installed. diff --git a/samples/native-image-sample/pom.xml b/samples/native-image-sample/pom.xml index 7e9801d03..f092add3d 100644 --- a/samples/native-image-sample/pom.xml +++ b/samples/native-image-sample/pom.xml @@ -86,7 +86,7 @@ org.graalvm.buildtools junit-platform-native - 0.9.11 + 0.9.12 test @@ -107,7 +107,7 @@ org.graalvm.buildtools native-maven-plugin - 0.9.11 + 0.9.12 true com.example.datastore.NativeImageDatastoreSample diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 0643927f1..cbfeb45e1 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -28,7 +28,7 @@ com.google.cloud google-cloud-datastore - 2.7.0 + 2.8.0 diff --git a/samples/snippets/src/test/java/com/google/datastore/snippets/ConceptsTest.java b/samples/snippets/src/test/java/com/google/datastore/snippets/ConceptsTest.java index 1d10f1f6d..b22efc6fb 100644 --- a/samples/snippets/src/test/java/com/google/datastore/snippets/ConceptsTest.java +++ b/samples/snippets/src/test/java/com/google/datastore/snippets/ConceptsTest.java @@ -27,6 +27,7 @@ import com.google.cloud.datastore.Cursor; import com.google.cloud.datastore.Datastore; import com.google.cloud.datastore.DatastoreException; +import com.google.cloud.datastore.DatastoreOptions; import com.google.cloud.datastore.Entity; import com.google.cloud.datastore.EntityQuery; import com.google.cloud.datastore.FullEntity; @@ -83,6 +84,7 @@ public class ConceptsTest { private static final FullEntity TEST_FULL_ENTITY = FullEntity.newBuilder().build(); private Datastore datastore; + private Datastore datastoreRealBackend; private KeyFactory keyFactory; private Key taskKey; private Entity testEntity; @@ -123,6 +125,8 @@ public void setUp() { endDate = Timestamp.of(calendar.getTime()); calendar.set(1999, DECEMBER, 31); includedDate = Timestamp.of(calendar.getTime()); + // Create a client for tests that require a real backend + datastoreRealBackend = DatastoreOptions.getDefaultInstance().getService(); } /** @@ -384,7 +388,7 @@ private void setUpQueryTests() { .set( "description", StringValue.newBuilder("Learn Cloud Datastore").setExcludeFromIndexes(true).build()) - .set("tag", "fun", "l", "programming") + .set("tag", "fun", "l", "programming", "learn") .build()); } @@ -1044,4 +1048,101 @@ public void testPropertyFilteringRunQuery() { ImmutableMap.of("Task", ImmutableSet.of("priority", "tag")); assertEquals(expected, propertiesByKind); } + + @Test + public void testEqQuerySorted() { + setUpQueryTests(); + // [START datastore_eq_query_sorted] + Query query = + Query.newEntityQueryBuilder() + .setKind("Task") + .setFilter(PropertyFilter.eq("tag", "learn")) + .setOrderBy(OrderBy.asc("tag")) + .build(); + // [END datastore_eq_query_sorted] + assertValidQuery(query); + } + + /** Start tests using a real backend. */ + private V assertValidQueryRealBackend(Query query) { + QueryResults results = datastoreRealBackend.run(query); + V result = results.next(); + // assertFalse(results.hasNext()); + return result; + } + + private void setUpQueryTestsRealBackend() { + Key taskKey = + datastoreRealBackend + .newKeyFactory() + .setKind("Task") + .addAncestors(PathElement.of("TaskList", "default")) + .newKey("someTask"); + datastoreRealBackend.put( + Entity.newBuilder(taskKey) + .set("category", "Personal") + .set("done", false) + .set("completed", false) + .set("priority", 4) + .set("created", includedDate) + .set("percent_complete", 10.0) + .set( + "description", + StringValue.newBuilder("Learn Cloud Datastore").setExcludeFromIndexes(true).build()) + .set("tag", "fun", "l", "programming", "learn") + .build()); + } + + @Test + public void testInQuery() { + setUpQueryTestsRealBackend(); + // [START datastore_in_query] + Query query = + Query.newEntityQueryBuilder() + .setKind("Task") + .setFilter(PropertyFilter.in("tag", ListValue.of("learn", "study"))) + .build(); + // [END datastore_in_query] + assertValidQueryRealBackend(query); + } + + @Test + public void testNotEqualsQuery() { + setUpQueryTestsRealBackend(); + // [START datastore_not_equals_query] + Query query = + Query.newEntityQueryBuilder() + .setKind("Task") + .setFilter(PropertyFilter.neq("category", "Work")) + .build(); + // [END datastore_not_equals_query] + assertValidQueryRealBackend(query); + } + + @Test + public void testNotInQuery() { + setUpQueryTestsRealBackend(); + // [START datastore_not_in_query] + Query query = + Query.newEntityQueryBuilder() + .setKind("Task") + .setFilter(PropertyFilter.not_in("category", ListValue.of("Work", "Chores", "School"))) + .build(); + // [END datastore_not_in_query] + assertValidQueryRealBackend(query); + } + + @Test + public void testInQuerySorted() { + setUpQueryTestsRealBackend(); + // [START datastore_in_query_sorted] + Query query = + Query.newEntityQueryBuilder() + .setKind("Task") + .setFilter(PropertyFilter.in("tag", ListValue.of("learn", "study"))) + .setOrderBy(OrderBy.asc("tag")) + .build(); + // [END datastore_in_query_sorted] + assertValidQueryRealBackend(query); + } } diff --git a/versions.txt b/versions.txt index 9e1dc1b27..1b29554de 100644 --- a/versions.txt +++ b/versions.txt @@ -1,9 +1,9 @@ # Format: # module:released-version:current-version -google-cloud-datastore:2.8.0:2.8.0 -google-cloud-datastore-bom:2.8.0:2.8.0 -proto-google-cloud-datastore-v1:0.99.0:0.99.0 -datastore-v1-proto-client:2.8.0:2.8.0 -proto-google-cloud-datastore-admin-v1:2.8.0:2.8.0 -grpc-google-cloud-datastore-admin-v1:2.8.0:2.8.0 +google-cloud-datastore:2.9.0:2.9.0 +google-cloud-datastore-bom:2.9.0:2.9.0 +proto-google-cloud-datastore-v1:0.100.0:0.100.0 +datastore-v1-proto-client:2.9.0:2.9.0 +proto-google-cloud-datastore-admin-v1:2.9.0:2.9.0 +grpc-google-cloud-datastore-admin-v1:2.9.0:2.9.0