机器学习实验（八）：用特征值衰减正则化方法进行深度学习实验_3

from __future__ import print_function
from keras.datasets import cifar10
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation, Flatten
from keras.layers.convolutional import Convolution2D, MaxPooling2D
from keras.optimizers import SGD
from keras.utils import np_utils

#Importing Eigenvalue Decay regularizer:
#from EigenvalueDecay import EigenvalueRegularizer

from keras.models import model_from_json

batch_size = 32
nb_classes = 10
nb_epoch = 10
data_augmentation = False#True

# input image dimensions
img_rows, img_cols = 32, 32
# the CIFAR10 images are RGB
img_channels = 3

# the data, shuffled and split between train and test sets
(X_train, y_train), (X_test, y_test) = cifar10.load_data()
print('X_train shape:', X_train.shape)
print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')

# convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)

model = Sequential()

model.add(Convolution2D(32, 3, 3, border_mode='same',
input_shape=(img_channels, img_rows, img_cols)))
model.add(Activation('relu'))
model.add(Convolution2D(32, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))

model.add(Convolution2D(64, 3, 3, border_mode='same'))
model.add(Activation('relu'))
model.add(Convolution2D(64, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))

model.add(Flatten())
model.add(Dense(512,W_regularizer=EigenvalueRegularizer(0.1))) #Applying Eigenvalue Decay with C=0.1
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(nb_classes,W_regularizer=EigenvalueRegularizer(0.1))) #Applying Eigenvalue Decay with C=0.1
model.add(Activation('softmax'))

# let's train the model using SGD + momentum (how original).
sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])

X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255
X_test /= 255

if not data_augmentation:
print('Not using data augmentation.')
model.fit(X_train, Y_train,
batch_size=batch_size,
nb_epoch=nb_epoch,
validation_data=(X_test, Y_test),
shuffle=True)
else:
print('Using real-time data augmentation.')

# this will do preprocessing and realtime data augmentation
datagen = ImageDataGenerator(
featurewise_center=False,  # set input mean to 0 over the dataset
samplewise_center=False,  # set each sample mean to 0
featurewise_std_normalization=False,  # divide inputs by std of the dataset
samplewise_std_normalization=False,  # divide each input by its std
zca_whitening=False,  # apply ZCA whitening
rotation_range=0,  # randomly rotate images in the range (degrees, 0 to 180)
width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)
height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)
horizontal_flip=True,  # randomly flip images
vertical_flip=False)  # randomly flip images

# compute quantities required for featurewise normalization
# (std, mean, and principal components if ZCA whitening is applied)
datagen.fit(X_train)

# fit the model on the batches generated by datagen.flow()
model.fit_generator(datagen.flow(X_train, Y_train,
batch_size=batch_size),
samples_per_epoch=X_train.shape[0],
nb_epoch=nb_epoch,
validation_data=(X_test, Y_test))

model.save_weights('my_model_weights.h5')
print('model weights trained with Eigenvalue decay saved')

#**********************************  tricking Keras ;-)  ***********************************************************
#Creating a new model, similar but without Eigenvalue Decay, to use with the weights adjusted with Eigenvalue Decay:
#*******************************************************************************************************************

model = Sequential()

model.add(Convolution2D(32, 3, 3, border_mode='same',
input_shape=(img_channels, img_rows, img_cols)))
model.add(Activation('relu'))
model.add(Convolution2D(32, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))

model.add(Convolution2D(64, 3, 3, border_mode='same'))
model.add(Activation('relu'))
model.add(Convolution2D(64, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))

model.add(Flatten())
model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))

json_string = model.to_json()
open('my_model_struct.json', 'w').write(json_string)
print('model structure without Eigenvalue Decay saved')

model = model_from_json(open('my_model_struct.json').read())

sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])

#Loading the weights trained with Eigenvalue Decay:
model.load_weights('my_model_weights.h5')

#Showing the same results as before:
score = model.evaluate(X_test, Y_test, verbose=0)
print('Test score of saved model:', score[0])
print('Test accuracy of saved model:', score[1])

X_train shape: (50000, 3, 32, 32)50000 train samples10000 test samplesNot using data augmentation.Train on 50000 samples, validate on 10000 samplesEpoch 1/1050000/50000 [==============================] - 399s - loss: 2.0288 - acc: 0.3536 - val_loss: 1.3774 - val_acc: 0.5010Epoch 2/1050000/50000 [==============================] - 440s - loss: 1.6359 - acc: 0.5179 - val_loss: 1.1484 - val_acc: 0.6011Epoch 3/1050000/50000 [==============================] - 408s - loss: 1.4912 - acc: 0.5817 - val_loss: 1.1446 - val_acc: 0.6026Epoch 4/1050000/50000 [==============================] - 407s - loss: 1.3987 - acc: 0.6219 - val_loss: 0.9291 - val_acc: 0.6796Epoch 5/1050000/50000 [==============================] - 445s - loss: 1.3316 - acc: 0.6518 - val_loss: 0.8456 - val_acc: 0.7160Epoch 6/1050000/50000 [==============================] - 460s - loss: 1.2835 - acc: 0.6707 - val_loss: 0.8413 - val_acc: 0.7115Epoch 7/1050000/50000 [==============================] - 409s - loss: 1.2471 - acc: 0.6872 - val_loss: 0.7789 - val_acc: 0.7309Epoch 8/1050000/50000 [==============================] - 390s - loss: 1.2094 - acc: 0.7040 - val_loss: 0.7420 - val_acc: 0.7452Epoch 9/1050000/50000 [==============================] - 394s - loss: 1.1906 - acc: 0.7141 - val_loss: 0.7527 - val_acc: 0.7402Epoch 10/1050000/50000 [==============================] - 405s - loss: 1.1706 - acc: 0.7243 - val_loss: 0.7497 - val_acc: 0.7524model weights trained with Eigenvalue decay savedmodel structure without Eigenvalue Decay savedTest score of saved model: 0.749746108055Test accuracy of saved model: 0.7524