FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

Align with tensorflow v2.4.1 · sbdevman/TensorFlow.NET@ad6ad84 · GitHub

/ TensorFlow.NET Public
forked from SciSharp/TensorFlow.NET

Commit ad6ad84

Browse files
Oceania2018
committed
Align with tensorflow v2.4.1
1 parent fb903d8 commit ad6ad84

8 files changed

Lines changed: 62 additions & 101 deletions

File tree

‎src/TensorFlowNET.Console/MemoryBasicTest.cs‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
using Tensorflow.Keras.Engine.DataAdapters;
55
using static Tensorflow.Binding;
66
using static Tensorflow.KerasApi;
7+
using System.Linq;
8+
using System.Collections.Generic;
79

810
namespace Tensorflow
911
{
@@ -35,13 +37,15 @@ public Action<int, int> Constant2x3
3537
public Action<int, int> ConstantString
3638
=> (epoch, iterate) =>
3739
{
38-
var tensor = tf.constant(new string[]
40+
var strList = new string[]
3941
{
4042
"Biden immigration bill would put millions of illegal immigrants on 8-year fast-track to citizenship",
4143
"The Associated Press, which also reported that the eight-year path is in the bill.",
4244
"The bill would also include provisions to stem the flow of migration by addressing root causes of migration from south of the border."
43-
});
44-
var data = tensor.numpy();
45+
};
46+
47+
var tensor = tf.constant(strList, TF_DataType.TF_STRING);
48+
var data = tensor.StringData();
4549
};
4650

4751
public Action<int, int> Variable

‎src/TensorFlowNET.Console/Program.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ static void BasicTest(MemoryMonitor mm)
4747
// explaination of constant
4848
mm.Execute(10, 100 * batchSize, basic.Constant2x3);
4949

50-
mm.Execute(10, 100 * batchSize, basic.ConstantString);
50+
mm.Execute(10, batchSize, basic.ConstantString);
5151

5252
// 100K float variable.
5353
mm.Execute(10, batchSize, basic.Variable);

‎src/TensorFlowNET.Core/APIs/c_api.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ namespace Tensorflow
4343
/// </summary>
4444
public partial class c_api
4545
{
46-
public const string TensorFlowLibName = "tensorflow";
46+
public const string TensorFlowLibName = @"D:\Projects\tensorflow-haiping\bazel-bin\tensorflow\tensorflow";
4747

4848
public static string StringPiece(IntPtr handle)
4949
{

‎src/TensorFlowNET.Core/Tensorflow.Binding.csproj‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
<AssemblyName>TensorFlow.NET</AssemblyName>
66
<RootNamespace>Tensorflow</RootNamespace>
77
<TargetTensorFlow>2.2.0</TargetTensorFlow>
8-
<Version>0.33.0</Version>
8+
<Version>0.40.0</Version>
99
<LangVersion>8.0</LangVersion>
1010
<Authors>Haiping Chen, Meinrad Recheis, Eli Belash</Authors>
1111
<Company>SciSharp STACK</Company>
@@ -19,7 +19,7 @@
1919
<Description>Google's TensorFlow full binding in .NET Standard.
2020
Building, training and infering deep learning models.
2121
https://tensorflownet.readthedocs.io</Description>
22-
<AssemblyVersion>0.33.0.0</AssemblyVersion>
22+
<AssemblyVersion>0.40.0.0</AssemblyVersion>
2323
<PackageReleaseNotes>tf.net 0.20.x and above are based on tensorflow native 2.x.
2424

2525
* Eager Mode is added finally.
@@ -29,8 +29,10 @@ https://tensorflownet.readthedocs.io</Description>
2929
* Improve memory usage.
3030

3131
TensorFlow .NET v0.3x is focused on making more Keras API works.
32-
Keras API is a separate package released as TensorFlow.Keras.</PackageReleaseNotes>
33-
<FileVersion>0.33.0.0</FileVersion>
32+
Keras API is a separate package released as TensorFlow.Keras.
33+
34+
tf.net 0.4x.x aligns with TensorFlow v2.4.1 native library.</PackageReleaseNotes>
35+
<FileVersion>0.40.0.0</FileVersion>
3436
<PackageLicenseFile>LICENSE</PackageLicenseFile>
3537
<PackageRequireLicenseAcceptance>true</PackageRequireLicenseAcceptance>
3638
<SignAssembly>true</SignAssembly>

‎src/TensorFlowNET.Core/Tensors/Tensor.String.cs‎

Lines changed: 22 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -8,27 +8,7 @@ namespace Tensorflow
88
{
99
public partial class Tensor
1010
{
11-
const ulong TF_TSRING_SIZE = 24;
12-
13-
public IntPtr StringTensor25(string[] strings, TensorShape shape)
14-
{
15-
var handle = c_api.TF_AllocateTensor(TF_DataType.TF_STRING,
16-
shape.dims.Select(x => (long)x).ToArray(),
17-
shape.ndim,
18-
(ulong)shape.size * TF_TSRING_SIZE);
19-
20-
var data = c_api.TF_TensorData(handle);
21-
var tstr = c_api.TF_StringInit(handle);
22-
// AllocationHandle = tstr;
23-
// AllocationType = AllocationType.Tensorflow;
24-
for (int i = 0; i< strings.Length; i++)
25-
{
26-
c_api.TF_StringCopy(tstr, strings[i], strings[i].Length);
27-
tstr += (int)TF_TSRING_SIZE;
28-
}
29-
// c_api.TF_StringDealloc(tstr);
30-
return handle;
31-
}
11+
const int TF_TSRING_SIZE = 24;
3212

3313
public IntPtr StringTensor(string[] strings, TensorShape shape)
3414
{
@@ -40,69 +20,28 @@ public IntPtr StringTensor(string[] strings, TensorShape shape)
4020
return StringTensor(buffer, shape);
4121
}
4222

43-
public unsafe IntPtr StringTensor(byte[][] buffer, TensorShape shape)
23+
public IntPtr StringTensor(byte[][] buffer, TensorShape shape)
4424
{
45-
ulong size = 0;
46-
foreach (var b in buffer)
47-
size += c_api.TF_StringEncodedSize((ulong)b.Length);
48-
49-
var src_size = size + (ulong)buffer.Length * sizeof(ulong);
5025
var handle = c_api.TF_AllocateTensor(TF_DataType.TF_STRING,
51-
shape.dims.Select(x => (long)x).ToArray(),
26+
shape.ndim == 0 ? null : shape.dims.Select(x => (long)x).ToArray(),
5227
shape.ndim,
53-
src_size);
54-
AllocationType = AllocationType.Tensorflow;
28+
(ulong)shape.size * TF_TSRING_SIZE);
5529

56-
IntPtr data_start = c_api.TF_TensorData(handle);
57-
IntPtr string_start = data_start + buffer.Length * sizeof(ulong);
58-
IntPtr limit = data_start + (int)src_size;
59-
ulong offset = 0;
30+
var tstr = c_api.TF_TensorData(handle);
31+
#if TRACK_TENSOR_LIFE
32+
print($"New TString 0x{handle.ToString("x16")} {AllocationType} Data: 0x{tstr.ToString("x16")}");
33+
#endif
6034
for (int i = 0; i < buffer.Length; i++)
6135
{
62-
Marshal.WriteInt64(data_start, i * sizeof(ulong), (long)offset);
63-
if (buffer[i].Length == 0)
64-
{
65-
Marshal.WriteByte(string_start, 0);
66-
break;
67-
}
68-
69-
fixed (byte* src = &buffer[i][0])
70-
{
71-
/*Marshal.WriteByte(string_start, Convert.ToByte(buffer[i].Length));
72-
tf.memcpy((string_start + 1).ToPointer(), src, (ulong)buffer[i].Length);
73-
string_start += buffer[i].Length + 1;
74-
offset += buffer[i].Length + 1;*/
75-
76-
var written = c_api.TF_StringEncode(src, (ulong)buffer[i].Length, (byte*)string_start, (ulong)(limit.ToInt64() - string_start.ToInt64()), tf.Status.Handle);
77-
tf.Status.Check(true);
78-
string_start += (int)written;
79-
offset += written;
80-
}
36+
c_api.TF_StringInit(tstr);
37+
c_api.TF_StringCopy(tstr, buffer[i], buffer[i].Length);
38+
var data = c_api.TF_StringGetDataPointer(tstr);
39+
tstr += TF_TSRING_SIZE;
8140
}
8241

8342
return handle;
8443
}
8544

86-
public string[] StringData25()
87-
{
88-
string[] strings = new string[c_api.TF_Dim(_handle, 0)];
89-
var tstrings = TensorDataPointer;
90-
for (int i = 0; i< strings.Length; i++)
91-
{
92-
var tstringData = c_api.TF_StringGetDataPointer(tstrings);
93-
/*var size = c_api.TF_StringGetSize(tstrings);
94-
var capacity = c_api.TF_StringGetCapacity(tstrings);
95-
var type = c_api.TF_StringGetType(tstrings);*/
96-
strings[i] = c_api.StringPiece(tstringData);
97-
tstrings += (int)TF_TSRING_SIZE;
98-
}
99-
return strings;
100-
}
101-
102-
/// <summary>
103-
/// Extracts string array from current Tensor.
104-
/// </summary>
105-
/// <exception cref="InvalidOperationException">When <see cref="dtype"/> != TF_DataType.TF_STRING</exception>
10645
public string[] StringData()
10746
{
10847
var buffer = StringBytes();
@@ -114,7 +53,7 @@ public string[] StringData()
11453
return _str;
11554
}
11655

117-
public unsafe byte[][] StringBytes()
56+
public byte[][] StringBytes()
11857
{
11958
if (dtype != TF_DataType.TF_STRING)
12059
throw new InvalidOperationException($"Unable to call StringData when dtype != TF_DataType.TF_STRING (dtype is {dtype})");
@@ -123,24 +62,22 @@ public unsafe byte[][] StringBytes()
12362
// TF_STRING tensors are encoded with a table of 8-byte offsets followed by TF_StringEncode-encoded bytes.
12463
// [offset1, offset2,...,offsetn, s1size, s1bytes, s2size, s2bytes,...,snsize,snbytes]
12564
//
126-
long size = 1;
65+
int size = 1;
12766
foreach (var s in TensorShape.dims)
12867
size *= s;
12968

13069
var buffer = new byte[size][];
131-
var data_start = c_api.TF_TensorData(_handle);
132-
data_start += (int)(size * sizeof(ulong));
70+
var tstrings = TensorDataPointer;
13371
for (int i = 0; i < buffer.Length; i++)
13472
{
135-
IntPtr dst = IntPtr.Zero;
136-
ulong dstLen = 0;
137-
var read = c_api.TF_StringDecode((byte*)data_start, bytesize, (byte**)&dst, ref dstLen, tf.Status.Handle);
138-
tf.Status.Check(true);
139-
buffer[i] = new byte[(int)dstLen];
140-
Marshal.Copy(dst, buffer[i], 0, buffer[i].Length);
141-
data_start += (int)read;
73+
var data = c_api.TF_StringGetDataPointer(tstrings);
74+
var len = c_api.TF_StringGetSize(tstrings);
75+
buffer[i] = new byte[len];
76+
// var capacity = c_api.TF_StringGetCapacity(tstrings);
77+
// var type = c_api.TF_StringGetType(tstrings);
78+
Marshal.Copy(data, buffer[i], 0, Convert.ToInt32(len));
79+
tstrings += TF_TSRING_SIZE;
14280
}
143-
14481
return buffer;
14582
}
14683
}

‎src/TensorFlowNET.Core/Tensors/Tensor.cs‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ limitations under the License.
1515
******************************************************************************/
1616

1717
using NumSharp;
18-
using NumSharp.Backends.Unmanaged;
1918
using System;
2019
using System.Diagnostics.CodeAnalysis;
2120
using System.Globalization;
@@ -24,7 +23,6 @@ limitations under the License.
2423
using Tensorflow.Eager;
2524
using Tensorflow.Framework;
2625
using Tensorflow.Keras.Engine;
27-
using Tensorflow.Variables;
2826
using static Tensorflow.Binding;
2927

3028
namespace Tensorflow
@@ -287,6 +285,22 @@ protected override void DisposeUnmanagedResources(IntPtr handle)
287285
throw new InvalidOperationException($"Tensor.AllocationHandle is not null ({AllocationHandle}) but AllocationType is not matched to a C# allocation type ({AllocationType}).");
288286
}
289287

288+
if (dtype == TF_DataType.TF_STRING)
289+
{
290+
int size = 1;
291+
foreach (var s in TensorShape.dims)
292+
size *= s;
293+
var tstr = TensorDataPointer;
294+
#if TRACK_TENSOR_LIFE
295+
print($"Delete TString 0x{handle.ToString("x16")} {AllocationType} Data: 0x{tstrings.ToString("x16")}");
296+
#endif
297+
for (int i = 0; i < size; i++)
298+
{
299+
c_api.TF_StringDealloc(tstr);
300+
tstr += TF_TSRING_SIZE;
301+
}
302+
}
303+
290304
c_api.TF_DeleteTensor(handle);
291305
}
292306

‎src/TensorFlowNET.Core/Tensors/c_api.tensor.cs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -182,7 +182,10 @@ public static unsafe IntPtr TF_NewTensor(TF_DataType dataType, long[] dims, int
182182
public static extern unsafe ulong TF_StringEncode(byte* src, ulong src_len, byte* dst, ulong dst_len, SafeStatusHandle status);
183183

184184
[DllImport(TensorFlowLibName)]
185-
public static extern IntPtr TF_StringInit(IntPtr t);
185+
public static extern void TF_StringInit(IntPtr t);
186+
187+
[DllImport(TensorFlowLibName)]
188+
public static extern void TF_StringCopy(IntPtr dst, byte[] text, long size);
186189

187190
[DllImport(TensorFlowLibName)]
188191
public static extern void TF_StringCopy(IntPtr dst, string text, long size);

‎test/TensorFlowNET.Native.UnitTest/Tensors/TensorTest.cs‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ public void Tensor()
111111
/// Port from c_api_test.cc
112112
/// `TEST_F(CApiAttributesTest, StringTensor)`
113113
/// </summary>
114-
[TestMethod, Ignore("Waiting for PR https://github.com/tensorflow/tensorflow/pull/46804")]
114+
[TestMethod]
115115
public void StringTensor()
116116
{
117117
string text = "Hello world!.";
@@ -120,13 +120,14 @@ public void StringTensor()
120120
null,
121121
0,
122122
1 * 24);
123-
var tstr = c_api.TF_StringInit(tensor);
124-
var data = c_api.TF_StringGetDataPointer(tstr);
123+
var tstr = c_api.TF_TensorData(tensor);
124+
c_api.TF_StringInit(tstr);
125125
c_api.TF_StringCopy(tstr, text, text.Length);
126+
var data = c_api.TF_StringGetDataPointer(tstr);
126127

127128
Assert.AreEqual((ulong)text.Length, c_api.TF_StringGetSize(tstr));
128129
Assert.AreEqual(text, c_api.StringPiece(data));
129-
Assert.AreEqual((ulong)text.Length, c_api.TF_TensorByteSize(tensor));
130+
Assert.AreEqual(TF_TString_Type.TF_TSTR_SMALL, c_api.TF_StringGetType(tensor));
130131
Assert.AreEqual(0, c_api.TF_NumDims(tensor));
131132

132133
TF_DeleteTensor(tensor);

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL