FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

Use the new SummaryMetadata field of the Value proto to pass per-plug… · codeplaysoftware/tensorflow@1e8d1fe · GitHub

Repository navigation

Commit 1e8d1fe

Browse files
Use the new SummaryMetadata field of the Value proto to pass per-plugin information.
This will let TensorBoard figure out which plugins should handle which types of summaries. PiperOrigin-RevId: 159202469
1 parent 7fffdb2 commit 1e8d1fe

12 files changed

Lines changed: 215 additions & 22 deletions

‎tensorflow/core/debug/BUILD‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@ tf_cuda_library(
123123
linkstatic = 1,
124124
deps = [
125125
":debug_service_proto_cc",
126+
":debugger_event_metadata_proto_cc",
126127
"//tensorflow/core:core_cpu_internal",
127128
"//tensorflow/core:framework",
128129
"//tensorflow/core:lib",
@@ -144,6 +145,7 @@ tf_cuda_library(
144145
":debug_graph_utils",
145146
":debug_io_utils",
146147
":debug_service_proto_cc",
148+
":debugger_event_metadata_proto_cc",
147149
"//tensorflow/core:framework",
148150
"//tensorflow/core:lib",
149151
"//tensorflow/core:lib_internal",
@@ -194,6 +196,7 @@ tf_cc_test(
194196
deps = [
195197
":debug_grpc_testlib",
196198
":debug_io_utils",
199+
":debugger_event_metadata_proto_cc",
197200
"//tensorflow/core:core_cpu",
198201
"//tensorflow/core:core_cpu_internal",
199202
"//tensorflow/core:framework",
@@ -204,6 +207,7 @@ tf_cc_test(
204207
"//tensorflow/core:test",
205208
"//tensorflow/core:test_main",
206209
"//tensorflow/core:testlib",
210+
"//tensorflow/core/platform/default/build_config:platformlib",
207211
],
208212
)
209213

@@ -247,6 +251,12 @@ tf_cc_test(
247251
],
248252
)
249253

254+
tf_proto_library_cc(
255+
name = "debugger_event_metadata_proto",
256+
srcs = ["debugger_event_metadata.proto"],
257+
cc_api_version = 2,
258+
)
259+
250260
# TODO(cais): Add the following back in when tfdbg is supported on Android.
251261
# filegroup(
252262
# name = "android_srcs",

‎tensorflow/core/debug/debug_grpc_testlib.cc‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,12 @@ limitations under the License.
1616
#include "tensorflow/core/debug/debug_grpc_testlib.h"
1717

1818
#include "tensorflow/core/debug/debug_graph_utils.h"
19+
#include "tensorflow/core/debug/debugger_event_metadata.pb.h"
1920
#include "tensorflow/core/framework/summary.pb.h"
2021
#include "tensorflow/core/lib/io/path.h"
2122
#include "tensorflow/core/lib/strings/str_util.h"
2223
#include "tensorflow/core/platform/env.h"
24+
#include "tensorflow/core/platform/protobuf.h"
2325
#include "tensorflow/core/platform/tracing.h"
2426

2527
namespace tensorflow {
@@ -44,8 +46,6 @@ ::grpc::Status TestEventListenerImpl::SendEvents(
4446
tensorflow::str_util::Split(val.node_name(), ':');
4547

4648
const string node_name = name_items[0];
47-
int32 output_slot = 0;
48-
tensorflow::strings::safe_strto32(name_items[1], &output_slot);
4949
const string debug_op = name_items[2];
5050

5151
const TensorProto& tensor_proto = val.tensor();
@@ -54,9 +54,24 @@ ::grpc::Status TestEventListenerImpl::SendEvents(
5454
return ::grpc::Status::CANCELLED;
5555
}
5656

57-
device_names.push_back(val.tag());
57+
// Obtain the device name, which is encoded in JSON.
58+
third_party::tensorflow::core::debug::DebuggerEventMetadata metadata;
59+
for (int i = 0; i < val.metadata().plugin_data_size(); i++) {
60+
if (val.metadata().plugin_data(i).plugin_name() != "debugger") {
61+
// This plugin data was meant for another plugin.
62+
continue;
63+
}
64+
auto status = tensorflow::protobuf::util::JsonStringToMessage(
65+
val.metadata().plugin_data(i).content(), &metadata);
66+
if (status.ok()) {
67+
// The device name has been determined.
68+
break;
69+
}
70+
}
71+
72+
device_names.push_back(metadata.device());
5873
node_names.push_back(node_name);
59-
output_slots.push_back(output_slot);
74+
output_slots.push_back(metadata.output_slot());
6075
debug_ops.push_back(debug_op);
6176
debug_tensors.push_back(tensor);
6277
}

‎tensorflow/core/debug/debug_io_utils.cc‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,13 @@ limitations under the License.
2626
#pragma comment(lib,"Ws2_32.lib")
2727
#endif
2828

29+
#include "tensorflow/core/debug/debugger_event_metadata.pb.h"
2930
#include "tensorflow/core/framework/summary.pb.h"
3031
#include "tensorflow/core/lib/hash/hash.h"
3132
#include "tensorflow/core/lib/io/path.h"
3233
#include "tensorflow/core/lib/strings/str_util.h"
3334
#include "tensorflow/core/lib/strings/stringprintf.h"
35+
#include "tensorflow/core/platform/protobuf.h"
3436
#include "tensorflow/core/util/event.pb.h"
3537

3638
#define GRPC_OSS_UNIMPLEMENTED_ERROR \
@@ -55,7 +57,32 @@ Event WrapTensorAsEvent(const DebugNodeKey& debug_node_key,
5557
// "DebugIdentity", the debug node_name in the Summary proto will be
5658
// "foo/node_a:0:DebugIdentity".
5759
summ_val->set_node_name(debug_node_key.debug_node_name);
58-
summ_val->set_tag(debug_node_key.device_name);
60+
61+
// Tag by the node name. This allows TensorBoard to quickly fetch data per op.
62+
summ_val->set_tag(debug_node_key.node_name);
63+
64+
// Store data within debugger metadata to be stored for each event.
65+
third_party::tensorflow::core::debug::DebuggerEventMetadata metadata;
66+
metadata.set_device(debug_node_key.device_name);
67+
metadata.set_output_slot(debug_node_key.output_slot);
68+
69+
// Encode the data in JSON.
70+
string json_output;
71+
tensorflow::protobuf::util::JsonPrintOptions json_options;
72+
json_options.always_print_primitive_fields = true;
73+
auto status = tensorflow::protobuf::util::MessageToJsonString(
74+
metadata, &json_output, json_options);
75+
if (status.ok()) {
76+
// Store summary metadata. Set the plugin to use this data as "debugger".
77+
SummaryMetadata::PluginData* plugin_data =
78+
summ_val->mutable_metadata()->add_plugin_data();
79+
plugin_data->set_plugin_name("debugger");
80+
plugin_data->set_content(json_output);
81+
} else {
82+
LOG(WARNING) << "Failed to convert DebuggerEventMetadata proto to JSON. "
83+
<< "The debug_node_name is " << debug_node_key.debug_node_name
84+
<< ".";
85+
}
5986

6087
if (tensor.dtype() == DT_STRING) {
6188
// Treat DT_STRING specially, so that tensor_util.MakeNdarray can convert

‎tensorflow/core/debug/debug_io_utils_test.cc‎

Lines changed: 30 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ limitations under the License.
1515

1616
#include "tensorflow/core/debug/debug_io_utils.h"
1717

18+
#include "tensorflow/core/debug/debugger_event_metadata.pb.h"
1819
#include "tensorflow/core/framework/summary.pb.h"
1920
#include "tensorflow/core/framework/tensor_testutil.h"
2021
#include "tensorflow/core/lib/core/notification.h"
@@ -124,10 +125,18 @@ TEST_F(DebugIOUtilsTest, DumpStringTensorToFileSunnyDay) {
124125

125126
ASSERT_GE(wall_time, event.wall_time());
126127
ASSERT_EQ(1, event.summary().value().size());
127-
ASSERT_EQ(kDebugNodeKey.device_name, event.summary().value(0).tag());
128+
ASSERT_EQ(kDebugNodeKey.node_name, event.summary().value(0).tag());
128129
ASSERT_EQ(kDebugNodeKey.debug_node_name,
129130
event.summary().value(0).node_name());
130131

132+
// Determine and validate some information from the metadata.
133+
third_party::tensorflow::core::debug::DebuggerEventMetadata metadata;
134+
auto status = tensorflow::protobuf::util::JsonStringToMessage(
135+
event.summary().value(0).metadata().plugin_data(0).content(), &metadata);
136+
ASSERT_TRUE(status.ok());
137+
ASSERT_EQ(kDebugNodeKey.device_name, metadata.device());
138+
ASSERT_EQ(kDebugNodeKey.output_slot, metadata.output_slot());
139+
131140
Tensor b_prime(DT_STRING);
132141
ASSERT_TRUE(b_prime.FromProto(event.summary().value(0).tensor()));
133142

@@ -229,10 +238,19 @@ TEST_F(DebugIOUtilsTest, PublishTensorToMultipleFileURLs) {
229238

230239
ASSERT_GE(wall_time, event.wall_time());
231240
ASSERT_EQ(1, event.summary().value().size());
232-
ASSERT_EQ(kDebugNodeKey.device_name, event.summary().value(0).tag());
241+
ASSERT_EQ(kDebugNodeKey.node_name, event.summary().value(0).tag());
233242
ASSERT_EQ(kDebugNodeKey.debug_node_name,
234243
event.summary().value(0).node_name());
235244

245+
// Determine and validate some information from the metadata.
246+
third_party::tensorflow::core::debug::DebuggerEventMetadata metadata;
247+
auto status = tensorflow::protobuf::util::JsonStringToMessage(
248+
event.summary().value(0).metadata().plugin_data(0).content(),
249+
&metadata);
250+
ASSERT_TRUE(status.ok());
251+
ASSERT_EQ(kDebugNodeKey.device_name, metadata.device());
252+
ASSERT_EQ(kDebugNodeKey.output_slot, metadata.output_slot());
253+
236254
Tensor a_prime(DT_FLOAT);
237255
ASSERT_TRUE(a_prime.FromProto(event.summary().value(0).tensor()));
238256

@@ -333,10 +351,19 @@ TEST_F(DebugIOUtilsTest, PublishTensorConcurrentlyToPartiallyOverlappingPaths) {
333351

334352
ASSERT_GE(wall_time, event.wall_time());
335353
ASSERT_EQ(1, event.summary().value().size());
336-
ASSERT_EQ(kDebugNodeKey.device_name, event.summary().value(0).tag());
354+
ASSERT_EQ(kDebugNodeKey.node_name, event.summary().value(0).tag());
337355
ASSERT_EQ(kDebugNodeKey.debug_node_name,
338356
event.summary().value(0).node_name());
339357

358+
// Determine and validate some information from the metadata.
359+
third_party::tensorflow::core::debug::DebuggerEventMetadata metadata;
360+
auto status = tensorflow::protobuf::util::JsonStringToMessage(
361+
event.summary().value(0).metadata().plugin_data(0).content(),
362+
&metadata);
363+
ASSERT_TRUE(status.ok());
364+
ASSERT_EQ(kDebugNodeKey.device_name, metadata.device());
365+
ASSERT_EQ(kDebugNodeKey.output_slot, metadata.output_slot());
366+
340367
Tensor a_prime(DT_FLOAT);
341368
ASSERT_TRUE(a_prime.FromProto(event.summary().value(0).tensor()));
342369

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
syntax = "proto3";
2+
3+
package third_party.tensorflow.core.debug;
4+
5+
// Encapsulates per-event data related to debugging.
6+
message DebuggerEventMetadata {
7+
string device = 1;
8+
int32 output_slot = 2;
9+
};

‎tensorflow/core/kernels/summary_tensor_op_test.cc‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,8 +85,15 @@ TEST_F(SummaryTensorOpV2Test, BasicPluginData) {
8585
ASSERT_EQ(0, out_tensor->dims());
8686
Summary summary;
8787
ParseProtoUnlimited(&summary, out_tensor->scalar<string>()());
88-
8988
ASSERT_EQ(1, summary.value_size());
89+
90+
// Check the content of the tensor stored in the summary.
91+
Tensor string_content_tensor;
92+
CHECK(string_content_tensor.FromProto(summary.value(0).tensor()));
93+
ASSERT_EQ("some string tensor content",
94+
string_content_tensor.scalar<string>()());
95+
96+
// Check plugin-related data.
9097
ASSERT_EQ("tag_foo", summary.value(0).tag());
9198
ASSERT_EQ(2, summary.value(0).metadata().plugin_data_size());
9299
ASSERT_EQ("foo", summary.value(0).metadata().plugin_data(0).plugin_name());

‎tensorflow/python/ops/summary_ops.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,6 @@ def _tensor_summary_v2( # pylint: disable=invalid-name
131131
val = gen_logging_ops._tensor_summary_v2(
132132
tensor=tensor,
133133
tag=tag,
134-
description="",
135134
name=scope,
136135
serialized_summary_metadata=serialized_summary_metadata)
137136
summary_op_util.collect(val, collections, [ops.GraphKeys.SUMMARIES])

‎tensorflow/python/summary/summary.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
@@FileWriter
2121
@@FileWriterCache
2222
@@tensor_summary
23+
@@_tensor_summary_v2
2324
@@scalar
2425
@@histogram
2526
@@audio
@@ -28,6 +29,7 @@
2829
@@merge
2930
@@merge_all
3031
@@get_summary_description
32+
@@PluginAsset
3133
@@get_plugin_asset
3234
@@get_all_plugin_assets
3335
"""

‎tensorflow/python/summary/text_summary.py‎

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,26 @@
2323
from __future__ import division
2424
from __future__ import print_function
2525

26+
from collections import namedtuple
2627
import json
2728

29+
from tensorflow.core.framework import summary_pb2
2830
from tensorflow.python.framework import dtypes
29-
from tensorflow.python.ops.summary_ops import tensor_summary
31+
from tensorflow.python.ops.summary_ops import _tensor_summary_v2
3032
from tensorflow.python.summary import plugin_asset
33+
from tensorflow.python.util import deprecation
3134

35+
PLUGIN_NAME = "text"
3236

37+
# Contains event-related data specific to the text plugin.
38+
_TextPluginData = namedtuple("_TextPluginData", [])
39+
40+
41+
@deprecation.deprecated_args(
42+
"2017-06-13",
43+
"collections is deprecated. Instead of using collections to associate "
44+
"plugins to events, add a PluginData field to the SummaryMetadata of a "
45+
"Value proto.", "collections")
3346
def text_summary(name, tensor, collections=None):
3447
"""Summarizes textual data.
3548
@@ -60,9 +73,16 @@ def text_summary(name, tensor, collections=None):
6073
raise ValueError("Expected tensor %s to have dtype string, got %s" %
6174
(tensor.name, tensor.dtype))
6275

63-
t_summary = tensor_summary(name, tensor, collections=collections)
64-
text_assets = plugin_asset.get_plugin_asset(TextSummaryPluginAsset)
65-
text_assets.register_tensor(t_summary.op.name)
76+
summary_metadata = summary_pb2.SummaryMetadata()
77+
text_plugin_data = _TextPluginData()
78+
data_dict = text_plugin_data._asdict() # pylint: disable=protected-access
79+
summary_metadata.plugin_data.add(
80+
plugin_name=PLUGIN_NAME, content=json.dumps(data_dict))
81+
t_summary = _tensor_summary_v2(
82+
name=name,
83+
tensor=tensor,
84+
summary_metadata=summary_metadata,
85+
collections=collections)
6686
return t_summary
6787

6888

‎tensorflow/python/summary/text_summary_test.py‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
from __future__ import division
1818
from __future__ import print_function
1919

20-
from tensorflow.python.framework import ops as framework_ops
2120
from tensorflow.python.framework import test_util
2221
from tensorflow.python.ops import array_ops
2322
from tensorflow.python.platform import googletest
@@ -43,16 +42,11 @@ def testTextSummaryAPI(self):
4342
# The API accepts vectors.
4443
arr = array_ops.constant(["one", "two", "three"])
4544
summ = text_summary.text_summary("foo", arr)
46-
self.assertEqual(summ.op.type, "TensorSummary")
45+
self.assertEqual(summ.op.type, "TensorSummaryV2")
4746

4847
# the API accepts scalars
4948
summ = text_summary.text_summary("foo", array_ops.constant("one"))
50-
self.assertEqual(summ.op.type, "TensorSummary")
51-
52-
def testTextSummaryCollections(self):
53-
text_summary.text_summary("bar", array_ops.constant("2"), collections=[])
54-
summaries = framework_ops.get_collection(framework_ops.GraphKeys.SUMMARIES)
55-
self.assertEqual(len(summaries), 0)
49+
self.assertEqual(summ.op.type, "TensorSummaryV2")
5650

5751

5852
if __name__ == "__main__":

0 commit comments

Comments
 (0)

Back | FazBrowse Home | New Git URL