I commented previously, now some idea as an answer:
1) I think that the validation data may include "lucky shots" which match the trained patterns well in some cases. Also a small validation set can be a problem. As far as I know, validation data is the last X% of the data (in Keras). You can set shuffle=True to mix validation data or False to not mix it (as I understand). I would try this.
https://github.com/keras-team/keras/issues/597
2) An option to get a good idea of how the model performs is cross validation. If you can afford it, train 3 or 5 or 10 models with different validation data and have a look at the average accuracy. Should give you also an idea of how volatile the results are.
3) You have a small sample. Thus, adding more validation data can be a problem. However, I would also try with 20%.
4) Since you have a small sample, the NN might have trouble getting all the relevant features. Instead of training a convnet from scratch, you can use a pretrained model. This has already learned a lot of features. Given a small sample, this could be THE way to go. Note that there are a number of different pretrained models which you can try.
https://keras.io/applications/
5) Get more data (if possible).
I recently used a pretrained model in a multiclass setting. Here is my code, which is a slightly modified version of a tutorial code by F. Chollet (https://github.com/fchollet/deep-learning-with-python-notebooks/blob/master/5.3-using-a-pretrained-convnet.ipynb).
from keras.applications import VGG16
import os, datetime, statistics
import numpy as np
from keras.preprocessing.image import ImageDataGenerator
from keras.utils import to_categorical
from keras import models
from keras import layers
from keras import optimizers
from keras.layers.core import Dense, Dropout, Activation
from PIL import ImageFile
ImageFile.LOAD_TRUNCATED_IMAGES = True
###############################################
# DIR with training images
base_dir = 'C:/kerasimages'
# Number training images
ntrain = 2000
# Number validation images
nval = 500
# Batch size
batch_size = 20
# Epochs fine tuning
ep = 600
# Epochs first step
ep_first = 50
# Number of classes (for training, output layer)
nclasses = 22
###############################################
conv_base = VGG16(weights='imagenet', include_top=False, input_shape=(150, 150, 3))
train_dir = os.path.join(base_dir, 'train')
validation_dir = os.path.join(base_dir, 'val')
datagen = ImageDataGenerator(rescale=1./255)
def extract_features(directory, sample_count):
features = np.zeros(shape=(sample_count, 4, 4, 512))
labels = np.zeros(shape=(sample_count))
generator = datagen.flow_from_directory(
directory,
target_size=(150, 150),
batch_size=batch_size,
class_mode='binary')
i = 0
for inputs_batch, labels_batch in generator:
features_batch = conv_base.predict(inputs_batch)
features[i * batch_size : (i + 1) * batch_size] = features_batch
labels[i * batch_size : (i + 1) * batch_size] = labels_batch
i += 1
if i * batch_size >= sample_count:
break
return features, labels
# Lables and features
train_features, train_labels = extract_features(train_dir, ntrain)
validation_features, validation_labels = extract_features(validation_dir, nval)
train_labels = to_categorical(train_labels)
validation_labels = to_categorical(validation_labels)
train_features = np.reshape(train_features, (ntrain, 4 * 4 * 512))
validation_features = np.reshape(validation_features, (nval, 4 * 4 * 512))
#######################################
# Model
model = models.Sequential()
model.add(conv_base)
model.add(layers.Flatten())
model.add(layers.Dense(2048, activation='relu'))#256
model.add(Dropout(0.20))
model.add(layers.Dense(1024, activation='relu'))#256
model.add(Dropout(0.20))
model.add(layers.Dense(512, activation='relu'))#256
model.add(Dropout(0.20))
model.add(layers.Dense(nclasses, activation='softmax'))
conv_base.trainable = False
#######################################
# Data generators
train_datagen = ImageDataGenerator(
rescale=1./255,
rotation_range=40,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True,
fill_mode='nearest')
test_datagen = ImageDataGenerator(rescale=1./255)
train_generator = train_datagen.flow_from_directory(
train_dir,
target_size=(150, 150),
batch_size=batch_size,
class_mode='categorical')
validation_generator = test_datagen.flow_from_directory(
validation_dir,
target_size=(150, 150),
batch_size=batch_size,
class_mode='categorical')
# Model compile / fit
model.compile(loss='categorical_crossentropy',
optimizer=optimizers.RMSprop(lr=2e-5),
metrics=['acc'])
history = model.fit_generator(
train_generator,
steps_per_epoch=100,
epochs=ep_first,
validation_data=validation_generator,
validation_steps=50,
verbose=2)
#######################################
# Fine tuning
conv_base.trainable = True
set_trainable = False
for layer in conv_base.layers:
if layer.name == 'block5_conv1':
set_trainable = True
if set_trainable:
layer.trainable = True
else:
layer.trainable = False
model.compile(loss='categorical_crossentropy',
optimizer=optimizers.RMSprop(lr=1e-5),
metrics=['acc'])
history = model.fit_generator(
train_generator,
steps_per_epoch=100,
epochs=ep,
validation_data=validation_generator,
validation_steps=50)
