commit #22: fix some processing bugs & change learning rate

Author: BIGBALLON

1_Lecun_Network/LeNet_dp_da_keras.py

@@ -8,7 +8,7 @@
 from keras.preprocessing.image import ImageDataGenerator
 
 batch_size    = 128
-epochs        = 180
+epochs        = 200
 iterations    = 391
 num_classes   = 10
 log_filepath  = './lenet_dp_da'
@@ -30,14 +30,13 @@ def build_model():
     return model
 
 def scheduler(epoch):
-    learning_rate_init = 0.02
-    if epoch >= 60:
-        learning_rate_init = 0.01
-    if epoch >= 120:
-        learning_rate_init = 0.004
-    if epoch >= 160:    
-        learning_rate_init = 0.0008
-    return learning_rate_init
+    if epoch <= 60:
+        return 0.05
+    if epoch <= 120:
+        return 0.01
+    if epoch <= 160:    
+        return 0.002
+    return 0.0004
 
 if __name__ == '__main__':
 

1_Lecun_Network/LeNet_dp_da_wd_keras.py

@@ -9,13 +9,12 @@
 from keras.regularizers import l2
 
 batch_size    = 128
-epochs        = 150
+epochs        = 200
 iterations    = 391
 num_classes   = 10
 weight_decay  = 0.0001
 mean          = [125.307, 122.95, 113.865]
 std           = [62.9932, 62.0887, 66.7048]
-lr            = [0.02, 0.004, 0.0008]
 
 log_filepath  = './lenet_dp_da_wd'
 
@@ -34,7 +33,13 @@ def build_model():
     return model
 
 def scheduler(epoch):
-    return lr[epoch//50]
+    if epoch <= 60:
+        return 0.05
+    if epoch <= 120:
+        return 0.01
+    if epoch <= 160:    
+        return 0.002
+    return 0.0004
 
 if __name__ == '__main__':
 

1_Lecun_Network/LeNet_dp_keras.py

@@ -7,7 +7,7 @@
 from keras.callbacks import LearningRateScheduler, TensorBoard
 
 batch_size    = 128
-epochs        = 180
+epochs        = 200
 iterations    = 391
 num_classes   = 10
 log_filepath  = './lenet_dp'
@@ -27,14 +27,13 @@ def build_model():
     return model
 
 def scheduler(epoch):
-    learning_rate_init = 0.02
-    if epoch >= 60:
-        learning_rate_init = 0.01
-    if epoch >= 120:
-        learning_rate_init = 0.004
-    if epoch >= 160:    
-        learning_rate_init = 0.0008
-    return learning_rate_init
+    if epoch <= 60:
+        return 0.05
+    if epoch <= 120:
+        return 0.01
+    if epoch <= 160:    
+        return 0.002
+    return 0.0004
 
 if __name__ == '__main__':
 

1_Lecun_Network/LeNet_keras.py

@@ -6,7 +6,7 @@
 from keras.callbacks import LearningRateScheduler, TensorBoard
 
 batch_size    = 128
-epochs        = 180
+epochs        = 200
 iterations    = 391
 num_classes   = 10
 log_filepath  = './lenet'
@@ -26,12 +26,13 @@ def build_model():
     return model
 
 def scheduler(epoch):
-    learning_rate_init = 0.02
-    if epoch >= 80:
-        learning_rate_init = 0.01
-    if epoch >= 150:
-        learning_rate_init = 0.004
-    return learning_rate_init
+    if epoch <= 60:
+        return 0.05
+    if epoch <= 120:
+        return 0.01
+    if epoch <= 160:    
+        return 0.002
+    return 0.0004
 
 if __name__ == '__main__':
 

2_Network_in_Network/Network_in_Network_bn_keras.py

@@ -11,7 +11,7 @@
 from keras.callbacks import LearningRateScheduler, TensorBoard
 
 batch_size    = 128
-epochs        = 164
+epochs        = 200
 iterations    = 391
 num_classes   = 10
 dropout       = 0.5
@@ -29,12 +29,13 @@ def color_preprocessing(x_train,x_test):
     return x_train, x_test
 
 def scheduler(epoch):
-  learning_rate_init = 0.08
-  if epoch >= 81:
-    learning_rate_init = 0.01
-  if epoch >= 122:
-    learning_rate_init = 0.001
-  return learning_rate_init
+    if epoch <= 60:
+        return 0.05
+    if epoch <= 120:
+        return 0.01
+    if epoch <= 160:    
+        return 0.002
+    return 0.0004
 
 def build_model():
   model = Sequential()

2_Network_in_Network/Network_in_Network_keras.py

@@ -11,7 +11,7 @@
 from keras.layers.normalization import BatchNormalization
 
 batch_size    = 128
-epochs        = 164
+epochs        = 200
 iterations    = 391
 num_classes   = 10
 dropout       = 0.5
@@ -29,12 +29,13 @@ def color_preprocessing(x_train,x_test):
     return x_train, x_test
 
 def scheduler(epoch):
-  learning_rate_init = 0.08
-  if epoch >= 81:
-    learning_rate_init = 0.01
-  if epoch >= 122:
-    learning_rate_init = 0.001
-  return learning_rate_init
+    if epoch <= 60:
+        return 0.05
+    if epoch <= 120:
+        return 0.01
+    if epoch <= 160:    
+        return 0.002
+    return 0.0004
 
 def build_model():
   model = Sequential()

3_Vgg19_Network/Vgg19_keras.py

@@ -22,19 +22,20 @@
 
 num_classes  = 10
 batch_size   = 128
-epochs       = 170
+epochs       = 200
 iterations   = 391
 dropout      = 0.5
 weight_decay = 0.0005
 log_filepath = r'./vgg19_retrain_logs/'
 
 def scheduler(epoch):
-  learning_rate_init = 0.1
-  if epoch > 80:
-    learning_rate_init = 0.01
-  if epoch > 120:
-    learning_rate_init = 0.001
-  return learning_rate_init
+    if epoch <= 60:
+        return 0.05
+    if epoch <= 120:
+        return 0.01
+    if epoch <= 160:    
+        return 0.002
+    return 0.0004
 
 WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg19_weights_tf_dim_ordering_tf_kernels.h5'
 filepath = get_file('vgg19_weights_tf_dim_ordering_tf_kernels.h5', WEIGHTS_PATH, cache_subdir='models')

4_Residual_Network/ResNet_keras.py

@@ -22,23 +22,21 @@
 
 def scheduler(epoch):
     if epoch <= 60:
-        return 0.1
+        return 0.05
     if epoch <= 120:
-        return 0.02
-    if epoch <= 160:
-        return 0.004
-    return 0.0008
+        return 0.01
+    if epoch <= 160:    
+        return 0.002
+    return 0.0004
 
 def color_preprocessing(x_train,x_test):
     x_train = x_train.astype('float32')
     x_test = x_test.astype('float32')
-    x_train[:,:,:,0] = (x_train[:,:,:,0] - np.mean(x_train[:,:,:,0])) / np.std(x_train[:,:,:,0])
-    x_train[:,:,:,1] = (x_train[:,:,:,1] - np.mean(x_train[:,:,:,1])) / np.std(x_train[:,:,:,1])
-    x_train[:,:,:,2] = (x_train[:,:,:,2] - np.mean(x_train[:,:,:,2])) / np.std(x_train[:,:,:,2])
-
-    x_test[:,:,:,0] = (x_test[:,:,:,0] - np.mean(x_test[:,:,:,0])) / np.std(x_test[:,:,:,0])
-    x_test[:,:,:,1] = (x_test[:,:,:,1] - np.mean(x_test[:,:,:,1])) / np.std(x_test[:,:,:,1])
-    x_test[:,:,:,2] = (x_test[:,:,:,2] - np.mean(x_test[:,:,:,2])) / np.std(x_test[:,:,:,2])
+    mean = [125.307, 122.95, 113.865]
+    std  = [62.9932, 62.0887, 66.7048]
+    for i in range(3):
+        x_train[:,:,:,i] = (x_train[:,:,:,i] - mean[i]) / std[i]
+        x_test[:,:,:,i] = (x_test[:,:,:,i] - mean[i]) / std[i]
 
     return x_train, x_test
 
@@ -146,4 +144,4 @@ def residual_block(x,shape,increase_filter=False):
                         epochs=epochs,
                         callbacks=cbks,
                         validation_data=(x_test, y_test))
-    resnet.save('resnet.h5')
+    resnet.save('resnet.h5')

5_Wide_Residual_Network/Wide_ResNet_keras.py

@@ -33,12 +33,11 @@ def scheduler(epoch):
 def color_preprocessing(x_train,x_test):
     x_train = x_train.astype('float32')
     x_test = x_test.astype('float32')
-    x_train[:,:,:,0] = (x_train[:,:,:,0] - np.mean(x_train[:,:,:,0])) / np.std(x_train[:,:,:,0])
-    x_train[:,:,:,1] = (x_train[:,:,:,1] - np.mean(x_train[:,:,:,1])) / np.std(x_train[:,:,:,1])
-    x_train[:,:,:,2] = (x_train[:,:,:,2] - np.mean(x_train[:,:,:,2])) / np.std(x_train[:,:,:,2])
-    x_test[:,:,:,0] = (x_test[:,:,:,0] - np.mean(x_test[:,:,:,0])) / np.std(x_test[:,:,:,0])
-    x_test[:,:,:,1] = (x_test[:,:,:,1] - np.mean(x_test[:,:,:,1])) / np.std(x_test[:,:,:,1])
-    x_test[:,:,:,2] = (x_test[:,:,:,2] - np.mean(x_test[:,:,:,2])) / np.std(x_test[:,:,:,2])
+    mean = [125.307, 122.95, 113.865]
+    std  = [62.9932, 62.0887, 66.7048]
+    for i in range(3):
+        x_train[:,:,:,i] = (x_train[:,:,:,i] - mean[i]) / std[i]
+        x_test[:,:,:,i] = (x_test[:,:,:,i] - mean[i]) / std[i]
 
     return x_train, x_test
 
@@ -127,4 +126,4 @@ def wide_residual_layer(x,out_filters,increase_filter=False):
                         epochs=epochs,
                         callbacks=cbks,
                         validation_data=(x_test, y_test))
-    resnet.save('resnet.h5')
+    resnet.save('resnet.h5')

6_ResNeXt/ResNeXt_keras.py

@@ -12,7 +12,7 @@
 from keras import optimizers
 from keras import regularizers
 
-cardinality        = 8          # 4 or 8 or 16 or 32
+cardinality        = 4          # 4 or 8 or 16 or 32
 base_width         = 64
 inplanes           = 64
 expansion          = 4
@@ -29,11 +29,13 @@
 std  = [62.9932, 62.0887, 66.7048]
 
 def scheduler(epoch):
-    if epoch <= 100:
+    if epoch <= 75:
         return 0.05
-    if epoch <= 180:
+    if epoch <= 150:
         return 0.005
-    return 0.0005
+    if epoch <= 210:
+        return 0.0005
+    return 0.0001
 
 def resnext(img_input,classes_num):
     global inplanes