TensorFlow.NET/src/TensorFlowNET.Core/Data/MnistDataSet.cs at master · pathway27/TensorFlow.NET · GitHub

History

87 lines (76 loc) · 2.97 KB

Raw

using System;

using System.Collections.Generic;

using System.Diagnostics;

using System.Text;

using NumSharp;

namespace Tensorflow

{

public class MnistDataSet : DataSetBase

{

public int NumOfExamples { get; private set; }

public int EpochsCompleted { get; private set; }

public int IndexInEpoch { get; private set; }

public MnistDataSet(NDArray images, NDArray labels, Type dataType, bool reshape)

{

EpochsCompleted = 0;

IndexInEpoch = 0;

NumOfExamples = images.shape[0];

images = images.reshape(images.shape[0], images.shape[1] * images.shape[2]);

images = images.astype(dataType);

// for debug np.multiply performance

var sw = new Stopwatch();

sw.Start();

images = np.multiply(images, 1.0f / 255.0f);

sw.Stop();

Console.WriteLine($"{sw.ElapsedMilliseconds}ms");

Data = images;

labels = labels.astype(dataType);

Labels = labels;

}

public (NDArray, NDArray) GetNextBatch(int batch_size, bool fake_data = false, bool shuffle = true)

{

if (IndexInEpoch >= NumOfExamples)

IndexInEpoch = 0;

var start = IndexInEpoch;

// Shuffle for the first epoch

if(EpochsCompleted == 0 && start == 0 && shuffle)

{

var perm0 = np.arange(NumOfExamples);

np.random.shuffle(perm0);

Data = Data[perm0];

Labels = Labels[perm0];

}

// Go to the next epoch

if (start + batch_size > NumOfExamples)

{

// Finished epoch

EpochsCompleted += 1;

// Get the rest examples in this epoch

var rest_num_examples = NumOfExamples - start;

var images_rest_part = Data[np.arange(start, NumOfExamples)];

var labels_rest_part = Labels[np.arange(start, NumOfExamples)];

// Shuffle the data

if (shuffle)

{

var perm = np.arange(NumOfExamples);

np.random.shuffle(perm);

Data = Data[perm];

Labels = Labels[perm];

}

start = 0;

IndexInEpoch = batch_size - rest_num_examples;

var end = IndexInEpoch;

var images_new_part = Data[np.arange(start, end)];

var labels_new_part = Labels[np.arange(start, end)];

return (np.concatenate(new[] { images_rest_part, images_new_part }, axis: 0),

np.concatenate(new[] { labels_rest_part, labels_new_part }, axis: 0));

}

else

{

IndexInEpoch += batch_size;

var end = IndexInEpoch;

return (Data[np.arange(start, end)], Labels[np.arange(start, end)]);

}

TensorFlow.NET/src/TensorFlowNET.Core/Data/MnistDataSet.cs at master · pathway27/TensorFlow.NET · GitHub

Expand file tree

Breadcrumbs

MnistDataSet.cs

Latest commit

History

Breadcrumbs

MnistDataSet.cs

File metadata and controls