FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

DatastoreIO: add support for querying namespace, and cleanup · spotify/DataflowJavaSDK@4c3fc80 · GitHub

Repository navigation

Commit 4c3fc80

Browse files
authored andcommitted
DatastoreIO: add support for querying namespace, and cleanup
1. Add support for querying within a Datastore namespace to DatastoreIO.Source. (DatastoreIO cannot by itself support writing to a namespace as it receives fully-built Entities.) 2. Add namespace support to the DatastoreWordCount example, showing both reading and writing within a namespace. 3. Deprecate DatastoreIO.read() since it creates a Source, not a Read, and add DatastoreIO.source() to replace it. 4. Clean up code and Javadoc in both DatastoreIO and DatastoreWordCount. Fixes GoogleCloudPlatform#59. ----Release Notes---- - DatastoreIO now supports (parallel) reads within namespaces. Entities can be written to namespaces by setting the namespace in the Entity key. [] ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=103813624
1 parent 531a352 commit 4c3fc80

4 files changed

Lines changed: 576 additions & 361 deletions

File tree

‎examples/src/main/java/com/google/cloud/dataflow/examples/cookbook/DatastoreWordCount.java‎

Lines changed: 52 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616

1717
package com.google.cloud.dataflow.examples.cookbook;
1818

19-
import com.google.api.services.datastore.DatastoreV1;
2019
import com.google.api.services.datastore.DatastoreV1.Entity;
2120
import com.google.api.services.datastore.DatastoreV1.Key;
2221
import com.google.api.services.datastore.DatastoreV1.Property;
@@ -26,6 +25,7 @@
2625
import com.google.cloud.dataflow.examples.WordCount;
2726
import com.google.cloud.dataflow.sdk.Pipeline;
2827
import com.google.cloud.dataflow.sdk.io.DatastoreIO;
28+
import com.google.cloud.dataflow.sdk.io.Read;
2929
import com.google.cloud.dataflow.sdk.io.TextIO;
3030
import com.google.cloud.dataflow.sdk.options.Default;
3131
import com.google.cloud.dataflow.sdk.options.Description;
@@ -38,6 +38,8 @@
3838
import java.util.Map;
3939
import java.util.UUID;
4040

41+
import javax.annotation.Nullable;
42+
4143
/**
4244
* A WordCount example using DatastoreIO.
4345
*
@@ -72,7 +74,7 @@ static class GetContentFn extends DoFn<Entity, String> {
7274
@Override
7375
public void processElement(ProcessContext c) {
7476
Map<String, Value> props = DatastoreHelper.getPropertyMap(c.element());
75-
DatastoreV1.Value value = props.get("content");
77+
Value value = props.get("content");
7678
if (value != null) {
7779
c.output(DatastoreHelper.getString(value));
7880
}
@@ -83,19 +85,36 @@ public void processElement(ProcessContext c) {
8385
* A DoFn that creates entity for every line in Shakespeare.
8486
*/
8587
static class CreateEntityFn extends DoFn<String, Entity> {
86-
private String kind;
88+
private final String namespace;
89+
private final String kind;
90+
private final Key ancestorKey;
8791

88-
CreateEntityFn(String kind) {
92+
CreateEntityFn(String namespace, String kind) {
93+
this.namespace = namespace;
8994
this.kind = kind;
95+
96+
// Build the ancestor key for all created entities once, including the namespace.
97+
Key.Builder keyBuilder = DatastoreHelper.makeKey(kind, "root");
98+
if (namespace != null) {
99+
keyBuilder.getPartitionIdBuilder().setNamespace(namespace);
100+
}
101+
ancestorKey = keyBuilder.build();
90102
}
91103

92104
public Entity makeEntity(String content) {
93105
Entity.Builder entityBuilder = Entity.newBuilder();
94-
// Create entities with same ancestor Key.
95-
Key ancestorKey = DatastoreHelper.makeKey(kind, "root").build();
96-
Key key = DatastoreHelper.makeKey(ancestorKey, kind, UUID.randomUUID().toString()).build();
97106

98-
entityBuilder.setKey(key);
107+
// All created entities have the same ancestor Key.
108+
Key.Builder keyBuilder =
109+
DatastoreHelper.makeKey(ancestorKey, kind, UUID.randomUUID().toString());
110+
// NOTE: Namespace is not inherited between keys created with DatastoreHelper.makeKey, so
111+
// we must set the namespace on keyBuilder. TODO: Once partitionId inheritance is added,
112+
// we can simplify this code.
113+
if (namespace != null) {
114+
keyBuilder.getPartitionIdBuilder().setNamespace(namespace);
115+
}
116+
117+
entityBuilder.setKey(keyBuilder.build());
99118
entityBuilder.addProperty(Property.newBuilder().setName("content")
100119
.setValue(Value.newBuilder().setStringValue(content)));
101120
return entityBuilder.build();
@@ -133,6 +152,10 @@ public static interface Options extends PipelineOptions {
133152
String getKind();
134153
void setKind(String value);
135154

155+
@Description("Dataset namespace")
156+
String getNamespace();
157+
void setNamespace(@Nullable String value);
158+
136159
@Description("Read an existing dataset, do not write first")
137160
boolean isReadOnly();
138161
void setReadOnly(boolean value);
@@ -150,7 +173,7 @@ public static interface Options extends PipelineOptions {
150173
public static void writeDataToDatastore(Options options) {
151174
Pipeline p = Pipeline.create(options);
152175
p.apply(TextIO.Read.named("ReadLines").from(options.getInput()))
153-
.apply(ParDo.of(new CreateEntityFn(options.getKind())))
176+
.apply(ParDo.of(new CreateEntityFn(options.getNamespace(), options.getKind())))
154177
.apply(DatastoreIO.writeTo(options.getDataset()));
155178

156179
p.run();
@@ -165,22 +188,26 @@ public static void readDataFromDatastore(Options options) {
165188
q.addKindBuilder().setName(options.getKind());
166189
Query query = q.build();
167190

191+
// For Datastore sources, the read namespace can be set on the entire query.
192+
DatastoreIO.Source source = DatastoreIO.source()
193+
.withDataset(options.getDataset())
194+
.withQuery(query)
195+
.withNamespace(options.getNamespace());
196+
168197
Pipeline p = Pipeline.create(options);
169-
p.apply(DatastoreIO.readFrom(options.getDataset(), query).named("ReadShakespeareFromDatastore"))
170-
.apply(ParDo.of(new GetContentFn()))
171-
.apply(new WordCount.CountWords())
172-
.apply(ParDo.of(new WordCount.FormatAsTextFn()))
173-
.apply(TextIO.Write.named("WriteLines")
174-
.to(options.getOutput())
175-
.withNumShards(options.getNumShards()));
198+
p.apply("ReadShakespeareFromDatastore", Read.from(source))
199+
.apply("StringifyEntity", ParDo.of(new GetContentFn()))
200+
.apply("CountWords", new WordCount.CountWords())
201+
.apply("PrintWordCount", ParDo.of(new WordCount.FormatAsTextFn()))
202+
.apply("WriteLines", TextIO.Write.to(options.getOutput())
203+
.withNumShards(options.getNumShards()));
176204
p.run();
177205
}
178206

179207
/**
180-
* Main function.
181-
* An example to demo how to use DatastoreIO. The runner here is
182-
* customizable, which means users could pass either DirectPipelineRunner
183-
* or DataflowPipelineRunner in PipelineOptions.
208+
* An example to demo how to use {@link DatastoreIO}. The runner here is
209+
* customizable, which means users could pass either {@code DirectPipelineRunner}
210+
* or {@code DataflowPipelineRunner} in the pipeline options.
184211
*/
185212
public static void main(String args[]) {
186213
// The options are used in two places, for Dataflow service, and
@@ -189,10 +216,11 @@ public static void main(String args[]) {
189216

190217
if (!options.isReadOnly()) {
191218
// First example: write data to Datastore for reading later.
192-
// Note: this will insert new entries with the given kind. Existing entries
193-
// should be cleared first, or the final counts will contain duplicates.
194-
// The Datastore Admin tool in the AppEngine console can be used to erase
195-
// all entries with a particular kind.
219+
//
220+
// NOTE: this write does not delete any existing Entities in the Datastore, so if run
221+
// multiple times with the same output dataset, there may be duplicate entries. The
222+
// Datastore Query tool in the Google Developers Console can be used to inspect or erase all
223+
// entries with a particular namespace and/or kind.
196224
DatastoreWordCount.writeDataToDatastore(options);
197225
}
198226

0 commit comments

Comments
 (0)

Back | FazBrowse Home | New Git URL