我尝试使用tensorflow 1.4.0对原始记录进行分类。 这个过程如下。
拳头：读取图像和标签，并将“tfrecord”格式输出到文件中。 第二：阅读记录和培训
写tfrecord脚本是
!/usr/bin/env python3
#coding:utf-8
import argparse
import os
import random
import numpy as np
from PIL import Image
import tensorflow as tf
def make_example(label_index, image):
return tf.train.Example(features = tf.train.Features(feature={
'label_index': tf.train.Feature(int64_list=tf.train.Int64List(value=[label_index])),
'image': tf.train.Feature(bytes_list=tf.train.BytesList(value=[image]))
}))
def write_tfrecord(dataset, outputfilepath):
writer = tf.python_io.TFRecordWriter(outputfilepath)
for label_of_one_hot, image in dataset:
ex = make_example(label_of_one_hot, image)
writer.write(ex.SerializeToString())
writer.close()
def importingargs():
parser = argparse.ArgumentParser("tensorflow exampe")
parser.add_argument("--datafolderpath", "-df", help="datafolderpath")
parser.add_argument("--filepath", "-f", help="filepath", required=True)
parser.add_argument("--labelfilepath", "-lf", help="label filepath")
parser.add_argument("--outputfolderpath", "-of", help="outputfolderpath of tf records")
parser.add_argument("--seed", "-s", type=int, required=False, default=0)
args = parser.parse_args()
return args.filepath, args.datafolderpath, args.labelfilepath, args.outputfolderpath, args.seed
def load_data(filepath, datafolderpath, labelfilepath):
with open(labelfilepath, "r") as rf:
labellist = [ line.strip() for line in rf.readlines() ]
with open(filepath, "r") as rf:
filepathlist = [ line.strip() for line in rf.readlines() ]
alldatasets = list()
for filepath in filepathlist:
imagefilepath = os.path.join(datafolderpath, filepath)
# image = open(imagefilepath).read()
img_obj = Image.open(imagefilepath).convert("L")
img = np.array(img_obj)
w, h = img.shape
print(w, h)
print(w*h)
img = img.reshape(w*h).tostring()
print(type(img))
filename = filepath.split(os.path.sep)[-1]
label = filename.split(".")[0].split("_")[1]
index = labellist.index(label) +1
print(index)
alldatasets.append([ index, img ])
return alldatasets
def splitdata(datasets):
random.shuffle(datasets)
train_indexes = [ 0, int(len(datasets) * 0.8 ) ]
valid_indexes = [ train_indexes[-1], int(len(datasets) * 0.9 ) ]
test_indexes = [ valid_indexes[-1], int(len(datasets)) ]
train_data = datasets[train_indexes[0]:train_indexes[1]]
valid_data = datasets[valid_indexes[0]:valid_indexes[1]]
test_data = datasets[test_indexes[0]:test_indexes[1]]
print("train num: %d" % len(train_data))
print("test num: %d" % len(test_data))
print("valid num: %d" % len(valid_data))
return train_data, valid_data, test_data
def main():
filepath, datafolderpath, labelfilepath, outputfolderpath, seed = importingargs()
random.seed(seed)
alldatasets = load_data(filepath, datafolderpath, labelfilepath)
train_data, valid_data, test_data = splitdata(alldatasets)
train_outputfilepath = os.path.join(outputfolderpath, "train.tfrecord")
valid_outputfilepath = os.path.join(outputfolderpath, "valid.tfrecord")
test_outptufilepath = os.path.join(outputfolderpath, "test.tfrecord")
write_tfrecord(train_data, train_outputfilepath)
write_tfrecord(valid_data, valid_outputfilepath)
write_tfrecord(test_data, test_outptufilepath)
if __name__ == "__main__":
main()
load_dataset文件导入train.py
#!/usr/bin/env python3
#coding:utf-8
import argparse
import os
import numpy as np
from PIL import Image
import tensorflow as tf
def read_tfrecord(inputfilepath):
print("read record")
reader = tf.TFRecordReader()
filename_que = tf.train.string_input_producer([inputfilepath])
key, value = reader.read(filename_que)
features = tf.parse_single_example(value,features = {
'label_index': tf.FixedLenFeature([], tf.string),
'image': tf.FixedLenFeature([], tf.string)
})
images = tf.decode_raw(features['image'], tf.float32)
images.set_shape([32*32])
images = tf.cast(images, tf.float32) * (1. / 255)
# images = tf.reshape(images, [-1])
labels = tf.decode_raw(features['label_index'], tf.int32)
# labels = tf.cast(features['label_index'], tf.int64)
# labels.set_shape([5])
print("call one hot")
label_index_one_hot = tf.one_hot(labels, 5)
label_index_one_hot.set_shape([5])
return images, label_index_one_hot
培训脚本
import os
import random
import tensorflow as tf
import load_datasets
import datasets
import make_datasets
print("def input and output")
images = tf.placeholder(tf.float32, shape=[None, 32*32])
labels = tf.placeholder(tf.int32, shape=[None, 5])
print("def layers")
x = tf.placeholder(tf.float32, [ None, 32*32 ])
y_ = tf.placeholder(tf.float32, [None, 5 ])
# W1 = tf.Variable(tf.zeros([ 32*32, 500 ]))
# b1 = tf.Variable(tf.zeros([ 500 ]))
# W2 = tf.Variable(tf.zeros([ 500, 5 ]))
# b2 = tf.Variable(tf.zeros([ 5 ]))
print("def function")
# h1 = tf.matmul(x, W1) + b1
# y = tf.matmul(h1, W2) + b2
W = tf.Variable(tf.zeros([ 32*32, 5 ]))
b = tf.Variable(tf.zeros([ 5 ]))
y = tf.matmul(x, W) + b
print("def leraning model")
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_,logits=y))
train_step = tf.train.AdamOptimizer(0.001).minimize(cross_entropy)
correct_prediction= tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
print("load train dataset")
trainfilepath = "../03tfrecords/train.tfrecord"
images, labels = load_datasets.read_tfrecord(trainfilepath)
input_queue = tf.train.slice_input_producer( [images, labels ], num_epochs=10, shuffle=False )
image_batch, label_batch = tf.train.batch( [images, labels], batch_size=10)
print("load test dataset")
testfilepath = "../03tfrecords/test.tfrecord"
test_image, test_label = load_datasets.read_tfrecord(testfilepath)
img_test_batch, label_test_batch = tf.train.batch([test_image,test_label],batch_size=16)
with tf.Session() as sess:
print("init layer value")
sess.run(tf.global_variables_initializer())
print("start training")
tf.train.start_queue_runners(sess)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
try:
while not coord.should_stop():
for i in range(0, 10):
print("train num %d" % (i+1))
imgs, labels = sess.run([image_batch, label_batch])
sess.run(train_step, feed_dict={x:imgs, y_: labels})
imgs_test, labels_text = sess.run([img_test_batch, label_test_batch])
print(sess.run(accuracy, feed_dict={x:imgs_test, y_:labels_text}))
finally:
coord.request_stop()
coord.join(threads)
制作tfrecords效果很好，但在训练脚本中，会发生错误。
Traceback (most recent call last):
File "/home/omori/.pyenv/versions/tensorflow-py3/lib/python3.5/site-packages/tensorflow/python/framework/tensor_shape.py", line 576, in merge_with
self.assert_same_rank(other)
File "/home/omori/.pyenv/versions/tensorflow-py3/lib/python3.5/site-packages/tensorflow/python/framework/tensor_shape.py", line 621, in assert_same_rank
other))
ValueError: Shapes (?, 5) and (5,) must have the same rank
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "train.py", line 45, in <module>
images, labels = load_datasets.read_tfrecord(trainfilepath)
File "/home/omori/tensorflow_example/01src/load_datasets.py", line 30, in read_tfrecord
label_index_one_hot.set_shape([5])
File "/home/omori/.pyenv/versions/tensorflow-py3/lib/python3.5/site-packages/tensorflow/python/framework/ops.py", line 407, in set_shape
self._shape = self._shape.merge_with(shape)
File "/home/omori/.pyenv/versions/tensorflow-py3/lib/python3.5/site-packages/tensorflow/python/framework/tensor_shape.py", line 582, in merge_with
raise ValueError("Shapes %s and %s are not compatible" % (self, other))
ValueError: Shapes (?, 5) and (5,) are not compatible
我搜索了很多网站，但我无法得到解决方案。 我怎么解决呢？
答案 0 :(得分：0)
Returns:
A Tensor of type out_type. A Tensor with one more dimension than the input bytes. The added
dimension will have size equal to the length of the elements of bytes divided by the number
of bytes to represent out_type.
所以在你的read_tfrecord函数中
labels = tf.decode_raw(features['label_index'], tf.int32)
给labels一个超额维度。您可以使用
label_index_one_hot = tf.one_hot(labels[0], 5)
（请注意添加的[0]）
我必须承认，我不明白添加的维度是什么。