| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
1 parent 531a352 commit 4c3fc80
4 files changed
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -16,7 +16,6 @@ | |||
| 16 | 16 | ||
| 17 | 17 | package com.google.cloud.dataflow.examples.cookbook; | |
| 18 | 18 | ||
| 19 | - import com.google.api.services.datastore.DatastoreV1; | ||
| 20 | 19 | import com.google.api.services.datastore.DatastoreV1.Entity; | |
| 21 | 20 | import com.google.api.services.datastore.DatastoreV1.Key; | |
| 22 | 21 | import com.google.api.services.datastore.DatastoreV1.Property; | |
@@ -26,6 +25,7 @@ | |||
| 26 | 25 | import com.google.cloud.dataflow.examples.WordCount; | |
| 27 | 26 | import com.google.cloud.dataflow.sdk.Pipeline; | |
| 28 | 27 | import com.google.cloud.dataflow.sdk.io.DatastoreIO; | |
| 28 | + import com.google.cloud.dataflow.sdk.io.Read; | ||
| 29 | 29 | import com.google.cloud.dataflow.sdk.io.TextIO; | |
| 30 | 30 | import com.google.cloud.dataflow.sdk.options.Default; | |
| 31 | 31 | import com.google.cloud.dataflow.sdk.options.Description; | |
@@ -38,6 +38,8 @@ | |||
| 38 | 38 | import java.util.Map; | |
| 39 | 39 | import java.util.UUID; | |
| 40 | 40 | ||
| 41 | + import javax.annotation.Nullable; | ||
| 42 | + | ||
| 41 | 43 | /** | |
| 42 | 44 | * A WordCount example using DatastoreIO. | |
| 43 | 45 | * | |
@@ -72,7 +74,7 @@ static class GetContentFn extends DoFn<Entity, String> { | |||
| 72 | 74 | @Override | |
| 73 | 75 | public void processElement(ProcessContext c) { | |
| 74 | 76 | Map<String, Value> props = DatastoreHelper.getPropertyMap(c.element()); | |
| 75 | - DatastoreV1.Value value = props.get("content"); | ||
| 77 | + Value value = props.get("content"); | ||
| 76 | 78 | if (value != null) { | |
| 77 | 79 | c.output(DatastoreHelper.getString(value)); | |
| 78 | 80 | } | |
@@ -83,19 +85,36 @@ public void processElement(ProcessContext c) { | |||
| 83 | 85 | * A DoFn that creates entity for every line in Shakespeare. | |
| 84 | 86 | */ | |
| 85 | 87 | static class CreateEntityFn extends DoFn<String, Entity> { | |
| 86 | - private String kind; | ||
| 88 | + private final String namespace; | ||
| 89 | + private final String kind; | ||
| 90 | + private final Key ancestorKey; | ||
| 87 | 91 | ||
| 88 | - CreateEntityFn(String kind) { | ||
| 92 | + CreateEntityFn(String namespace, String kind) { | ||
| 93 | + this.namespace = namespace; | ||
| 89 | 94 | this.kind = kind; | |
| 95 | + | ||
| 96 | + // Build the ancestor key for all created entities once, including the namespace. | ||
| 97 | + Key.Builder keyBuilder = DatastoreHelper.makeKey(kind, "root"); | ||
| 98 | + if (namespace != null) { | ||
| 99 | + keyBuilder.getPartitionIdBuilder().setNamespace(namespace); | ||
| 100 | + } | ||
| 101 | + ancestorKey = keyBuilder.build(); | ||
| 90 | 102 | } | |
| 91 | 103 | ||
| 92 | 104 | public Entity makeEntity(String content) { | |
| 93 | 105 | Entity.Builder entityBuilder = Entity.newBuilder(); | |
| 94 | - // Create entities with same ancestor Key. | ||
| 95 | - Key ancestorKey = DatastoreHelper.makeKey(kind, "root").build(); | ||
| 96 | - Key key = DatastoreHelper.makeKey(ancestorKey, kind, UUID.randomUUID().toString()).build(); | ||
| 97 | 106 | ||
| 98 | - entityBuilder.setKey(key); | ||
| 107 | + // All created entities have the same ancestor Key. | ||
| 108 | + Key.Builder keyBuilder = | ||
| 109 | + DatastoreHelper.makeKey(ancestorKey, kind, UUID.randomUUID().toString()); | ||
| 110 | + // NOTE: Namespace is not inherited between keys created with DatastoreHelper.makeKey, so | ||
| 111 | + // we must set the namespace on keyBuilder. TODO: Once partitionId inheritance is added, | ||
| 112 | + // we can simplify this code. | ||
| 113 | + if (namespace != null) { | ||
| 114 | + keyBuilder.getPartitionIdBuilder().setNamespace(namespace); | ||
| 115 | + } | ||
| 116 | + | ||
| 117 | + entityBuilder.setKey(keyBuilder.build()); | ||
| 99 | 118 | entityBuilder.addProperty(Property.newBuilder().setName("content") | |
| 100 | 119 | .setValue(Value.newBuilder().setStringValue(content))); | |
| 101 | 120 | return entityBuilder.build(); | |
@@ -133,6 +152,10 @@ public static interface Options extends PipelineOptions { | |||
| 133 | 152 | String getKind(); | |
| 134 | 153 | void setKind(String value); | |
| 135 | 154 | ||
| 155 | + @Description("Dataset namespace") | ||
| 156 | + String getNamespace(); | ||
| 157 | + void setNamespace(@Nullable String value); | ||
| 158 | + | ||
| 136 | 159 | @Description("Read an existing dataset, do not write first") | |
| 137 | 160 | boolean isReadOnly(); | |
| 138 | 161 | void setReadOnly(boolean value); | |
@@ -150,7 +173,7 @@ public static interface Options extends PipelineOptions { | |||
| 150 | 173 | public static void writeDataToDatastore(Options options) { | |
| 151 | 174 | Pipeline p = Pipeline.create(options); | |
| 152 | 175 | p.apply(TextIO.Read.named("ReadLines").from(options.getInput())) | |
| 153 | - .apply(ParDo.of(new CreateEntityFn(options.getKind()))) | ||
| 176 | + .apply(ParDo.of(new CreateEntityFn(options.getNamespace(), options.getKind()))) | ||
| 154 | 177 | .apply(DatastoreIO.writeTo(options.getDataset())); | |
| 155 | 178 | ||
| 156 | 179 | p.run(); | |
@@ -165,22 +188,26 @@ public static void readDataFromDatastore(Options options) { | |||
| 165 | 188 | q.addKindBuilder().setName(options.getKind()); | |
| 166 | 189 | Query query = q.build(); | |
| 167 | 190 | ||
| 191 | + // For Datastore sources, the read namespace can be set on the entire query. | ||
| 192 | + DatastoreIO.Source source = DatastoreIO.source() | ||
| 193 | + .withDataset(options.getDataset()) | ||
| 194 | + .withQuery(query) | ||
| 195 | + .withNamespace(options.getNamespace()); | ||
| 196 | + | ||
| 168 | 197 | Pipeline p = Pipeline.create(options); | |
| 169 | - p.apply(DatastoreIO.readFrom(options.getDataset(), query).named("ReadShakespeareFromDatastore")) | ||
| 170 | - .apply(ParDo.of(new GetContentFn())) | ||
| 171 | - .apply(new WordCount.CountWords()) | ||
| 172 | - .apply(ParDo.of(new WordCount.FormatAsTextFn())) | ||
| 173 | - .apply(TextIO.Write.named("WriteLines") | ||
| 174 | - .to(options.getOutput()) | ||
| 175 | - .withNumShards(options.getNumShards())); | ||
| 198 | + p.apply("ReadShakespeareFromDatastore", Read.from(source)) | ||
| 199 | + .apply("StringifyEntity", ParDo.of(new GetContentFn())) | ||
| 200 | + .apply("CountWords", new WordCount.CountWords()) | ||
| 201 | + .apply("PrintWordCount", ParDo.of(new WordCount.FormatAsTextFn())) | ||
| 202 | + .apply("WriteLines", TextIO.Write.to(options.getOutput()) | ||
| 203 | + .withNumShards(options.getNumShards())); | ||
| 176 | 204 | p.run(); | |
| 177 | 205 | } | |
| 178 | 206 | ||
| 179 | 207 | /** | |
| 180 | - * Main function. | ||
| 181 | - * An example to demo how to use DatastoreIO. The runner here is | ||
| 182 | - * customizable, which means users could pass either DirectPipelineRunner | ||
| 183 | - * or DataflowPipelineRunner in PipelineOptions. | ||
| 208 | + * An example to demo how to use {@link DatastoreIO}. The runner here is | ||
| 209 | + * customizable, which means users could pass either {@code DirectPipelineRunner} | ||
| 210 | + * or {@code DataflowPipelineRunner} in the pipeline options. | ||
| 184 | 211 | */ | |
| 185 | 212 | public static void main(String args[]) { | |
| 186 | 213 | // The options are used in two places, for Dataflow service, and | |
@@ -189,10 +216,11 @@ public static void main(String args[]) { | |||
| 189 | 216 | ||
| 190 | 217 | if (!options.isReadOnly()) { | |
| 191 | 218 | // First example: write data to Datastore for reading later. | |
| 192 | - // Note: this will insert new entries with the given kind. Existing entries | ||
| 193 | - // should be cleared first, or the final counts will contain duplicates. | ||
| 194 | - // The Datastore Admin tool in the AppEngine console can be used to erase | ||
| 195 | - // all entries with a particular kind. | ||
| 219 | + // | ||
| 220 | + // NOTE: this write does not delete any existing Entities in the Datastore, so if run | ||
| 221 | + // multiple times with the same output dataset, there may be duplicate entries. The | ||
| 222 | + // Datastore Query tool in the Google Developers Console can be used to inspect or erase all | ||
| 223 | + // entries with a particular namespace and/or kind. | ||
| 196 | 224 | DatastoreWordCount.writeDataToDatastore(options); | |
| 197 | 225 | } | |
| 198 | 226 | ||
| Back | FazBrowse Home | New Git URL |
0 commit comments