Strange values of training and testing when running my CNN in Tensorflow

Question

I´ve been trying to train and evaluate a convolutional neural network using my own data, which consists in 200 training images and 20 testing images. My complete script is h

Answer 1

sinc I can't follow what your code. here an example a full conv layer script using Tensorflow.

1st If you're working with images it really does make sense to serialize your data convolution operations are tense enough! The following script serializes youe images in TFrecords format. [based on Inception example ].

    '''
Converts image data to TFRecords file format with Example protos.
The image data set is expected to reside in JPEG files located in the
following directory structure.
  trainingset/label_0/image0.jpeg
  trainingset/label_0/image1.jpg
  ...
  testset/label_1/weird-image.jpeg
  testset/label_1/my-image.jpeg
'''

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from datetime import datetime
import os
import random
import sys
import threading

import numpy as np
import tensorflow as tf

tf.app.flags.DEFINE_string('train_directory', '/tmp/',
                           'Training data directory')
tf.app.flags.DEFINE_string('validation_directory', '/tmp/',
                           'Validation data directory')
tf.app.flags.DEFINE_string('output_directory', '/tmp/',
                           'Output data directory')

tf.app.flags.DEFINE_integer('train_shards', 2,
                            'Number of shards in training TFRecord files.')
tf.app.flags.DEFINE_integer('validation_shards', 2,
                            'Number of shards in validation TFRecord files.')

tf.app.flags.DEFINE_integer('num_threads', 2,
                            'Number of threads to preprocess the images.')

# The labels file contains a list of valid labels are held in this file.
# Assumes that the file contains entries as such:
#   dog
#   cat
#   flower
# where each line corresponds to a label. We map each label contained in
# the file to an integer corresponding to the line number starting from 0.
tf.app.flags.DEFINE_string('labels_file', '', 'Labels file')


FLAGS = tf.app.flags.FLAGS


def _int64_feature(value):
  """Wrapper for inserting int64 features into Example proto."""
  if not isinstance(value, list):
    value = [value]
  return tf.train.Feature(int64_list=tf.train.Int64List(value=value))


def _bytes_feature(value):
  """Wrapper for inserting bytes features into Example proto."""
  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))


def _convert_to_example(filename, image_buffer, label, text, height, width):
  """Build an Example proto for an example.
  Args:
    filename: string, path to an image file, e.g., '/path/to/example.JPG'
    image_buffer: string, JPEG encoding of RGB image
    label: integer, identifier for the ground truth for the network
    text: string, unique human-readable, e.g. 'dog'
    height: integer, image height in pixels
    width: integer, image width in pixels
  Returns:
    Example proto
  """

  colorspace = 'RGB'
  channels = 3
  image_format = 'JPEG'

  example = tf.train.Example(features=tf.train.Features(feature={
      'image/height': _int64_feature(height),
      'image/width': _int64_feature(width),
      'image/colorspace': _bytes_feature(tf.compat.as_bytes(colorspace)),
      'image/channels': _int64_feature(channels),
      'image/class/label': _int64_feature(label),
      'image/class/text': _bytes_feature(tf.compat.as_bytes(text)),
      'image/format': _bytes_feature(tf.compat.as_bytes(image_format)),
      'image/filename': _bytes_feature(tf.compat.as_bytes(os.path.basename(filename))),
      'image/encoded': _bytes_feature(tf.compat.as_bytes(image_buffer))}))
  return example


class ImageCoder(object):
  """Helper class that provides TensorFlow image coding utilities."""

  def __init__(self):
    # Create a single Session to run all image coding calls.
    self._sess = tf.Session()

    # Initializes function that converts PNG to JPEG data.
    self._png_data = tf.placeholder(dtype=tf.string)
    image = tf.image.decode_png(self._png_data, channels=3)
    self._png_to_jpeg = tf.image.encode_jpeg(image, format='rgb', quality=100)

    # Initializes function that decodes RGB JPEG data.
    self._decode_jpeg_data = tf.placeholder(dtype=tf.string)
    self._decode_jpeg = tf.image.decode_jpeg(self._decode_jpeg_data, channels=3)

  def png_to_jpeg(self, image_data):
    return self._sess.run(self._png_to_jpeg,
                          feed_dict={self._png_data: image_data})

  def decode_jpeg(self, image_data):
    image = self._sess.run(self._decode_jpeg,
                           feed_dict={self._decode_jpeg_data: image_data})
    assert len(image.shape) == 3
    assert image.shape[2] == 3
    return image


def _is_png(filename):
  """Determine if a file contains a PNG format image.
  Args:
    filename: string, path of the image file.
  Returns:
    boolean indicating if the image is a PNG.
  """
  return '.png' in filename


def _process_image(filename, coder):
  """Process a single image file.
  Args:
    filename: string, path to an image file e.g., '/path/to/example.JPG'.
    coder: instance of ImageCoder to provide TensorFlow image coding utils.
  Returns:
    image_buffer: string, JPEG encoding of RGB image.
    height: integer, image height in pixels.
    width: integer, image width in pixels.
  """
  # Read the image file.
  with tf.gfile.FastGFile(filename, 'rb') as f:
    image_data = f.read()

  # Convert any PNG to JPEG's for consistency.
  if _is_png(filename):
    print('Converting PNG to JPEG for %s' % filename)
    image_data = coder.png_to_jpeg(image_data)

  # Decode the RGB JPEG.
  image = coder.decode_jpeg(image_data)

  # Check that image converted to RGB
  assert len(image.shape) == 3
  height = image.shape[0]
  width = image.shape[1]
  assert image.shape[2] == 3

  return image_data, height, width


def _process_image_files_batch(coder, thread_index, ranges, name, filenames,
                               texts, labels, num_shards):
  """Processes and saves list of images as TFRecord in 1 thread.
  Args:
    coder: instance of ImageCoder to provide TensorFlow image coding utils.
    thread_index: integer, unique batch to run index is within [0, len(ranges)).
    ranges: list of pairs of integers specifying ranges of each batches to
      analyze in parallel.
    name: string, unique identifier specifying the data set
    filenames: list of strings; each string is a path to an image file
    texts: list of strings; each string is human readable, e.g. 'dog'
    labels: list of integer; each integer identifies the ground truth
    num_shards: integer number of shards for this data set.
  """
  # Each thread produces N shards where N = int(num_shards / num_threads).
  # For instance, if num_shards = 128, and the num_threads = 2, then the first
  # thread would produce shards [0, 64).
  num_threads = len(ranges)
  assert not num_shards % num_threads
  num_shards_per_batch = int(num_shards / num_threads)

  shard_ranges = np.linspace(ranges[thread_index][0],
                             ranges[thread_index][1],
                             num_shards_per_batch + 1).astype(int)
  num_files_in_thread = ranges[thread_index][1] - ranges[thread_index][0]

  counter = 0
  for s in range(num_shards_per_batch):
    # Generate a sharded version of the file name, e.g. 'train-00002-of-00010'
    shard = thread_index * num_shards_per_batch + s
    output_filename = '%s-%.5d-of-%.5d' % (name, shard, num_shards)
    output_file = os.path.join(FLAGS.output_directory, output_filename)
    writer = tf.python_io.TFRecordWriter(output_file)

    shard_counter = 0
    files_in_shard = np.arange(shard_ranges[s], shard_ranges[s + 1], dtype=int)
    for i in files_in_shard:
      filename = filenames[i]
      label = labels[i]
      text = texts[i]

      try:
        image_buffer, height, width = _process_image(filename, coder)
      except Exception as e:
        print(e)
        print('SKIPPED: Unexpected eror while decoding %s.' % filename)
        continue

      example = _convert_to_example(filename, image_buffer, label,
                                    text, height, width)
      writer.write(example.SerializeToString())
      shard_counter += 1
      counter += 1

      if not counter % 1000:
        print('%s [thread %d]: Processed %d of %d images in thread batch.' %
              (datetime.now(), thread_index, counter, num_files_in_thread))
        sys.stdout.flush()

    writer.close()
    print('%s [thread %d]: Wrote %d images to %s' %
          (datetime.now(), thread_index, shard_counter, output_file))
    sys.stdout.flush()
    shard_counter = 0
  print('%s [thread %d]: Wrote %d images to %d shards.' %
        (datetime.now(), thread_index, counter, num_files_in_thread))
  sys.stdout.flush()


def _process_image_files(name, filenames, texts, labels, num_shards):
  """Process and save list of images as TFRecord of Example protos.
  Args:
    name: string, unique identifier specifying the data set
    filenames: list of strings; each string is a path to an image file
    texts: list of strings; each string is human readable, e.g. 'dog'
    labels: list of integer; each integer identifies the ground truth
    num_shards: integer number of shards for this data set.
  """
  assert len(filenames) == len(texts)
  assert len(filenames) == len(labels)

  # Break all images into batches with a [ranges[i][0], ranges[i][1]].
  spacing = np.linspace(0, len(filenames), FLAGS.num_threads + 1).astype(np.int)
  ranges = []
  for i in range(len(spacing) - 1):
    ranges.append([spacing[i], spacing[i + 1]])

  # Launch a thread for each batch.
  print('Launching %d threads for spacings: %s' % (FLAGS.num_threads, ranges))
  sys.stdout.flush()

  # Create a mechanism for monitoring when all threads are finished.
  coord = tf.train.Coordinator()

  # Create a generic TensorFlow-based utility for converting all image codings.
  coder = ImageCoder()

  threads = []
  for thread_index in range(len(ranges)):
    args = (coder, thread_index, ranges, name, filenames,
            texts, labels, num_shards)
    t = threading.Thread(target=_process_image_files_batch, args=args)
    t.start()
    threads.append(t)

  # Wait for all the threads to terminate.
  coord.join(threads)
  print('%s: Finished writing all %d images in data set.' %
        (datetime.now(), len(filenames)))
  sys.stdout.flush()


def _find_image_files(data_dir, labels_file):
  """Build a list of all images files and labels in the data set.
  Args:
    data_dir: string, path to the root directory of images.
      Assumes that the image data set resides in JPEG files located in
      the following directory structure.
        data_dir/dog/another-image.JPEG
        data_dir/dog/my-image.jpg
      where 'dog' is the label associated with these images.
    labels_file: string, path to the labels file.
      The list of valid labels are held in this file. Assumes that the file
      contains entries as such:
        dog
        cat
        flower
      where each line corresponds to a label. We map each label contained in
      the file to an integer starting with the integer 0 corresponding to the
      label contained in the first line.
  Returns:
    filenames: list of strings; each string is a path to an image file.
    texts: list of strings; each string is the class, e.g. 'dog'
    labels: list of integer; each integer identifies the ground truth.
  """
  print('Determining list of input files and labels from %s.' % data_dir)
  unique_labels = [l.strip() for l in tf.gfile.FastGFile(
      labels_file, 'r').readlines()]

  labels = []
  filenames = []
  texts = []

  # Leave label index 0 empty as a background class.
  label_index = 1

  # Construct the list of JPEG files and labels.
  for text in unique_labels:
    jpeg_file_path = '%s/%s/*' % (data_dir, text)
    matching_files = tf.gfile.Glob(jpeg_file_path)

    labels.extend([label_index] * len(matching_files))
    texts.extend([text] * len(matching_files))
    filenames.extend(matching_files)

    if not label_index % 100:
      print('Finished finding files in %d of %d classes.' % (
          label_index, len(labels)))
    label_index += 1

  # Shuffle the ordering of all image files in order to guarantee
  # random ordering of the images with respect to label in the
  # saved TFRecord files. Make the randomization repeatable.
  shuffled_index = list(range(len(filenames)))
  random.seed(12345)
  random.shuffle(shuffled_index)

  filenames = [filenames[i] for i in shuffled_index]
  texts = [texts[i] for i in shuffled_index]
  labels = [labels[i] for i in shuffled_index]

  print('Found %d JPEG files across %d labels inside %s.' %
        (len(filenames), len(unique_labels), data_dir))
  return filenames, texts, labels


def _process_dataset(name, directory, num_shards, labels_file):
  """Process a complete data set and save it as a TFRecord.
  Args:
    name: string, unique identifier specifying the data set.
    directory: string, root path to the data set.
    num_shards: integer number of shards for this data set.
    labels_file: string, path to the labels file.
  """
  filenames, texts, labels = _find_image_files(directory, labels_file)
  _process_image_files(name, filenames, texts, labels, num_shards)


def main(unused_argv):
  assert not FLAGS.train_shards % FLAGS.num_threads, (
      'Please make the FLAGS.num_threads commensurate with FLAGS.train_shards')
  assert not FLAGS.validation_shards % FLAGS.num_threads, (
      'Please make the FLAGS.num_threads commensurate with '
      'FLAGS.validation_shards')
  print('Saving results to %s' % FLAGS.output_directory)

  # Run it!
  _process_dataset('validation', FLAGS.validation_directory,
                   FLAGS.validation_shards, FLAGS.labels_file)
  _process_dataset('train', FLAGS.train_directory,
                   FLAGS.train_shards, FLAGS.labels_file)


if __name__ == '__main__':
  tf.app.run()

you need to start the script as followed :

python Building_Set.py --train_directory=TrainingSet --output_directory=TF_Recordsfolder --validation_directory=ReferenceSet --labels_file=labels.txt --train_shards=1 --validation_shards=1 --num_threads=1 

PS: you need a labels.txt where the labels are saved.

After generating both training and test sets serialized files you can now use the data in the following convNN script:

import tensorflow as tf
import sys
import numpy as np
import matplotlib.pyplot as plt
filter_max_dimension = 50
filter_max_depth = 30
filter_h_and_w  = [3,3]
filter_depth    = [3,3]
numberOFclasses = 21
TensorBoard = "TB_conv2NN"
TF_Records   = "TF_Recordsfolder"
learning_rate = 1e-5
max_numberofiteretion =100000
batchSize  = 21
img_height = 128
img_width  = 128


# 1st function to read images form TF_Record
def getImage(filename):
    with tf.device('/cpu:0'):
        # convert filenames to a queue for an input pipeline.
        filenameQ = tf.train.string_input_producer([filename],num_epochs=None)

        # object to read records
        recordReader = tf.TFRecordReader()

        # read the full set of features for a single example
        key, fullExample = recordReader.read(filenameQ)

        # parse the full example into its' component features.
        features = tf.parse_single_example(
            fullExample,
            features={
                'image/height': tf.FixedLenFeature([], tf.int64),
                'image/width': tf.FixedLenFeature([], tf.int64),
                'image/colorspace': tf.FixedLenFeature([], dtype=tf.string,default_value=''),
                'image/channels':  tf.FixedLenFeature([], tf.int64),
                'image/class/label': tf.FixedLenFeature([],tf.int64),
                'image/class/text': tf.FixedLenFeature([], dtype=tf.string,default_value=''),
                'image/format': tf.FixedLenFeature([], dtype=tf.string,default_value=''),
                'image/filename': tf.FixedLenFeature([], dtype=tf.string,default_value=''),
                'image/encoded': tf.FixedLenFeature([], dtype=tf.string, default_value='')
            })

        # now we are going to manipulate the label and image features
        label = features['image/class/label']
        image_buffer = features['image/encoded']
        # Decode the jpeg
        with tf.name_scope('decode_img',[image_buffer], None):
            # decode
            image = tf.image.decode_jpeg(image_buffer, channels=3)

            # and convert to single precision data type
            image = tf.image.convert_image_dtype(image, dtype=tf.float32)
        # cast image into a single array, where each element corresponds to the greyscale
        # value of a single pixel.
        # the "1-.." part inverts the image, so that the background is black.
        image=tf.reshape(1-tf.image.rgb_to_grayscale(image),[img_height*img_width])
        # re-define label as a "one-hot" vector
        # it will be [0,1] or [1,0] here.
        # This approach can easily be extended to more classes.
        label=tf.stack(tf.one_hot(label-1, numberOFclasses))
        return label, image

with tf.device('/cpu:0'):
    train_img,train_label = getImage(TF_Records+"/train-00000-of-00001")
    validation_img,validation_label=getImage(TF_Records+"/validation-00000-of-00001")
    # associate the "label_batch" and "image_batch" objects with a randomly selected batch---
    # of labels and images respectively
    train_imageBatch, train_labelBatch = tf.train.shuffle_batch([train_img, train_label], batch_size=batchSize,capacity=50,min_after_dequeue=10)

    # and similarly for the validation data
    validation_imageBatch, validation_labelBatch = tf.train.shuffle_batch([validation_img, validation_label],
                                                    batch_size=batchSize,capacity=50,min_after_dequeue=10)



def train():
    with tf.device('/gpu:0'):
        config =tf.ConfigProto(log_device_placement=False, allow_soft_placement=True)
        #config.gpu_options.allow_growth = True
        #config.gpu_options.per_process_gpu_memory_fraction=0.9
        sess = tf.InteractiveSession(config = config)
        #defining tensorflow graph :
        with tf.name_scope("input"):
            x = tf.placeholder(tf.float32,[None, img_width*img_height],name ="pixels_values")
            y_= tf.placeholder(tf.float32,[None,numberOFclasses],name='Prediction')
        with tf.name_scope("input_reshape"):
            image_shaped =tf.reshape(x,[-1,img_height,img_width,1])
            tf.summary.image('input_img',image_shaped,numberOFclasses)
        #defining weigths and biases:
        def weights_variable (shape):
            return tf.Variable(tf.truncated_normal(shape,stddev=0.1))
        def bias_variable(shape):
            return tf.Variable(tf.constant(0.1,shape=shape))
        #help function to generates summaries for given variables
        def variable_summaries(var):
            with tf.name_scope('summaries'):
                mean = tf.reduce_mean(var)
                tf.summary.scalar('mean', mean)
                with tf.name_scope('stddev'):
                    stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
                tf.summary.scalar('stddev', stddev)
                tf.summary.scalar('max', tf.reduce_max(var))
                tf.summary.scalar('min', tf.reduce_min(var))
                tf.summary.histogram('histogram', var)

        def conv2d(x, W):
            return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')

        def max_pool_2x2(x):
          return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],strides=[1, 2, 2, 1], padding='SAME')

            with tf.name_scope('1st_conv_layer'):
                W_conv1 = weights_variable([filter_h_and_w[0],filter_h_and_w[0], 1, filter_depth[0]])
                b_conv1 = bias_variable([filter_depth[0]])
                h_conv1 = tf.nn.relu(conv2d(tf.reshape(x,[-1,img_width,img_height,1]), W_conv1) + b_conv1)
            with tf.name_scope('1nd_Pooling_layer'):
                h_conv1 = max_pool_2x2(h_conv1)
            with tf.name_scope('2nd_conv_layer'):
                W_conv2 = weights_variable([filter_h_and_w[1],filter_h_and_w[1], filter_depth[0], filter_depth[1]])
                b_conv2 = bias_variable([filter_depth[1]])
                h_conv2 = tf.nn.relu(conv2d(h_conv1, W_conv2) + b_conv2)

        with tf.name_scope('1st_Full_connected_Layer'):
            W_fc1 = weights_variable([filter_depth[1]*64, 1024])
            b_fc1 = bias_variable([1024])
            h_pool_flat = tf.reshape(h_conv2, [-1,filter_depth[1]*64])
            h_fc1 = tf.nn.relu(tf.matmul(h_pool_flat, W_fc1) + b_fc1)


        with tf.name_scope('Dropout'):
            keep_prob = tf.placeholder("float")
            h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)

        with tf.name_scope('Output_layer'):
            W_fc3 = weights_variable([1024, numberOFclasses])
            b_fc3 = bias_variable([numberOFclasses])
            y_conv=tf.nn.softmax(tf.matmul(h_fc1_drop, W_fc3) + b_fc3)

        with tf.name_scope('cross_entropy'):
        # The raw formulation of cross-entropy,
        #
        # tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.softmax(y)),
        #                               reduction_indices=[1]))
        #
        # can be numerically unstable.
        #
        # So here we use tf.nn.softmax_cross_entropy_with_logits on the
        # raw outputs of the nn_layer above, and then average across
        # the batch.
            diff = tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv)
            with tf.name_scope('total'):
              cross_entropy = tf.reduce_mean(diff)
        tf.summary.scalar('cross_entropy', cross_entropy)

        with tf.name_scope('train'):
          train_step = tf.train.AdamOptimizer(learning_rate).minimize(cross_entropy)

        with tf.name_scope('accuracy'):
            with tf.name_scope('correct_prediction'):
              correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))
            with tf.name_scope('accuracy'):
              accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
        tf.summary.scalar('accuracy', accuracy)
        # Merging Summaries
        merged = tf.summary.merge_all()
        train_writer = tf.summary.FileWriter(TensorBoard + '/train', sess.graph)
        test_writer = tf.summary.FileWriter(TensorBoard + '/test')
        # initialize the variables
        sess.run(tf.global_variables_initializer())

        # start the threads used for reading files
        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(sess=sess,coord=coord)

        # feeding function
        def feed_dict(train):
            if True :
                #img_batch, labels_batch= tf.train.shuffle_batch([train_label,train_img],batch_size=batchSize,capacity=500,min_after_dequeue=200)
                img_batch , labels_batch = sess.run([ train_labelBatch ,train_imageBatch])
                dropoutValue = 0.7
            else:
                #   img_batch,labels_batch = tf.train.shuffle_batch([validation_label,validation_img],batch_size=batchSize,capacity=500,min_after_dequeue=200)
                img_batch,labels_batch = sess.run([ validation_labelBatch,validation_imageBatch])
                dropoutValue = 1
            return {x:img_batch,y_:labels_batch,keep_prob:dropoutValue}

        for i  in range(max_numberofiteretion):
            if i%10 == 0:#Run a Test
                summary, acc = sess.run([merged,accuracy],feed_dict=feed_dict(False))
                #plt.imshow(output[0,:,:,1],cmap='gray')
                #plt.show()
                test_writer.add_summary(summary,i)# Save to TensorBoard
            else: # Training
              if i % 100 == 99:  # Record execution stats
                run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
                run_metadata = tf.RunMetadata()
                summary, _ = sess.run([merged, train_step],
                                      feed_dict=feed_dict(True),
                                      options=run_options,
                                      run_metadata=run_metadata)
                train_writer.add_run_metadata(run_metadata, 'step%03d' % i)
                train_writer.add_summary(summary, i)
              else:  # Record a summary
                output , summary, _ = sess.run([h_conv1,merged, train_step], feed_dict=feed_dict(True))
                train_writer.add_summary(summary, i)
        # finalise
        coord.request_stop()
        coord.join(threads)
        train_writer.close()
        test_writer.close()



filter_h_and_w[0] = np.random.randint(3, filter_max_dimension)
filter_h_and_w[1] = np.random.randint(3, filter_max_dimension)
filter_depth[0] = np.random.randint(3, filter_max_depth)
filter_depth[1] = np.random.randint(3, filter_max_depth)
TensorBoard = "ConV2NN/_filter"+str(filter_h_and_w[0])+"To"+str(filter_h_and_w[1])+"D"+str(filter_depth[0])+"To"+str(filter_depth[1])+"R10e5"

with tf.device('/gpu:0') :
        train()

The script is using both GPU and CPU if you don't have GPU TF is going to use the cpu of your device. The code is self explaining, u need to change the image resolution value and number of class. and you need to start Tensorboard, the script is save a test and train folder for tensorboard you just need to start it in your browser. since you have only 2 classes I think two conv layers are enough, if you think you need more it pretty easy to add ones. I hope this will help