Tensorflow,无需图形即可读取tfrecord

时间:2017-10-28 01:29:03

标签: tensorflow tensorflow-datasets tfrecord

我尝试用Tensorflow编写一个结构良好的结构化神经网络模型。但我遇到了一个问题,即将数据从tfrecord输入到图表中。代码如下,它挂在以下函数上,我该如何使它工作?

images,labels = network.load_tfrecord_data(1)

此函数无法从我的数据文件,.tfrecords中获取功能(图像)和标签?

任何想法都会受到赞赏吗?

 from __future__ import division
 from __future__ import print_function

 import datetime
 import numpy as np
 import tensorflow as tf
 layers = tf.contrib.layers
 losses = tf.contrib.losses
 metrics = tf.contrib.metrics

 LABELS = 10
 WIDTH = 28
 HEIGHT = 28
 HIDDEN = 100


 def read_and_decode_single_example(filename):
     filename_queue = tf.train.string_input_producer([filename], num_epochs=None)
     reader = tf.TFRecordReader()
     _, serialized_example = reader.read(filename_queue)
     features = tf.parse_single_example(
    serialized_example,
    features={
        'label': tf.FixedLenFeature([], tf.int64),
        'image': tf.FixedLenFeature([50176], tf.int64)
    })
    label = features['label']
    image = features['image']
    image = tf.reshape(image, [-1, 224, 224, 1])
    label = tf.one_hot(label - 1, 11, dtype=tf.int64)
    return label, image

 class Network:
     def __init__(self, logdir, experiment, threads):
         # Construct the graph
         with tf.name_scope("inputs"):
             self.images = tf.placeholder(tf.float32, [None, WIDTH, HEIGHT, 1], name="images")
             self.labels = tf.placeholder(tf.int64, [None], name="labels")
        # self.keep_prob = keep_prob
             self.keep_prob = tf.placeholder(tf.float32, name="keep_prob")
             flattened_images = layers.flatten(self.images)
         hidden_layer = layers.fully_connected(flattened_images, num_outputs=HIDDEN, activation_fn=tf.nn.relu, scope="hidden_layer")
         output_layer = layers.fully_connected(hidden_layer, num_outputs=LABELS, activation_fn=None, scope="output_layer")

         loss = losses.sparse_softmax_cross_entropy(labels=self.labels, logits=output_layer, scope="loss")
         self.training = layers.optimize_loss(loss, None, None, tf.train.AdamOptimizer(), summaries=['loss', 'gradients', 'gradient_norm'], name='training')

         with tf.name_scope("accuracy"):
             predictions = tf.argmax(output_layer, 1, name="predictions")
             accuracy = metrics.accuracy(predictions, self.labels)
             tf.summary.scalar("training/accuracy", accuracy)
         self.accuracy = metrics.accuracy(predictions, self.labels)

         with tf.name_scope("confusion_matrix"):
             confusion_matrix = metrics.confusion_matrix(predictions, self.labels, weights=tf.not_equal(predictions, self.labels), dtype=tf.float32)
             confusion_image = tf.reshape(confusion_matrix, [1, LABELS, LABELS, 1])

         # Summaries
         self.summaries = {'training': tf.summary.merge_all() }
         for dataset in ["dev", "test"]:
             self.summaries[dataset] = tf.summary.scalar(dataset + "/loss", loss)
             self.summaries[dataset] = tf.summary.scalar(dataset + "/accuracy", accuracy)
             self.summaries[dataset] = tf.summary.image(dataset + "/confusion_matrix", confusion_image)

         # Create the session
         self.session = tf.Session(config=tf.ConfigProto(inter_op_parallelism_threads=threads,
                                                    intra_op_parallelism_threads=threads))

         self.session.run(tf.global_variables_initializer())
         timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H%M%S")
         self.summary_writer = tf.summary.FileWriter("{}/{}-{}".format(logdir, timestamp, experiment), graph=self.session.graph, flush_secs=10)
         self.steps = 0

     def train(self, images, labels, keep_prob):
         self.steps += 1
         feed_dict = {self.images: self.session.run(images), self.labels: self.session.run(labels), self.keep_prob: keep_prob}

         if self.steps == 1:
             metadata = tf.RunMetadata()
             self.session.run(self.training, feed_dict, options=tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE), run_metadata=metadata)
             self.summary_writer.add_run_metadata(metadata, 'step1')
         elif self.steps % 100 == 0:
             _, summary = self.session.run([self.training, self.summaries['training']], feed_dict)
             self.summary_writer.add_summary(summary, self.steps)
         else:
             self.session.run(self.training, feed_dict)

     def evaluate(self, dataset, images, labels):
         feed_dict ={self.images: images, self.labels: labels, self.keep_prob: 1}
         summary = self.summaries[dataset].eval({self.images: images, self.labels: labels, self.keep_prob: 1}, self.session)
         self.summary_writer.add_summary(summary, self.steps)

     def load_tfrecord_data(self, training):
         training = training
         if training:
             label, image = read_and_decode_single_example("mhad_Op_train.tfrecords")
             # print(self.session.run(image))
         else:
             label, image = read_and_decode_single_example("mhad_Op_test.tfrecords")

         # image = tf.cast(image, tf.float32) / 255.

         images_batch, labels_batch = tf.train.shuffle_batch(
             [image, label], batch_size=50, num_threads=2,
             capacity=80,
             min_after_dequeue=30)

         return images_batch, labels_batch


 if __name__ == '__main__':
     # Fix random seed
     np.random.seed(42)
     tf.set_random_seed(42)

     # Parse arguments
     import argparse
     parser = argparse.ArgumentParser()
     parser.add_argument('--batch_size', default=256, type=int, help='Batch size.')
     parser.add_argument('--epochs', default=50, type=int, help='Number of epochs.')
     parser.add_argument('--logdir', default="logs", type=str, help='Logdir name.')
     parser.add_argument('--exp', default="mnist-final-confusion_matrix_customized_loss", type=str, help='Experiment name.')
     parser.add_argument('--threads', default=1, type=int, help='Maximum number of threads to use.')
     args = parser.parse_args()

     # Load the data
     keep_prob = 1

     # Construct the network
     network = Network(logdir=args.logdir, experiment=args.exp, threads=args.threads)

     # Train
     for i in range(args.epochs):
         images, labels = network.load_tfrecord_data(1)
         network.train(images, labels, keep_prob)
         print('current epoch', i)

1 个答案:

答案 0 :(得分:0)

您需要在模型中使用images, labels之前启动队列。

with tf.Session() as sess:
    coord = tf.train.Coordinator()
    threads = tf.train.start_queue_runners(coord=coord)
    images, labels = network.load_tfrecord_data(1)
    ...
    coord.request_stop()
    coord.join(threads)

检查this tutorial以获取完整示例