为什么我的结果仍然无法再现？

Question

为什么我的结果仍然无法再现？

tensorflowkerasconv-neural-networkgoogle-colaboratoryreproducible-research

11

我希望能够获得卷积神经网络的可重复结果。我使用Keras和带有GPU的Google Colab。

除了建议插入某些代码片段以实现可重现性外，我还向层添加了种子。

###### This is the first code snipped to run #####

!pip install -U -q PyDrive
from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
from google.colab import auth
from oauth2client.client import GoogleCredentials

# Authenticate and create the PyDrive client.
# This only needs to be done once per notebook.
auth.authenticate_user()
gauth = GoogleAuth()
gauth.credentials = GoogleCredentials.get_application_default()
drive = GoogleDrive(gauth)

###### This is the second code snipped to run #####

from __future__ import print_function  
import numpy as np 

import tensorflow as tf
print(tf.test.gpu_device_name())

import random as rn 
import os 
os.environ['PYTHONASHSEED'] = '0' 
np.random.seed(1)   
rn.seed(1)   
session_conf = tf.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)


###### This is the third code snipped to run #####

from keras import backend as K

tf.set_random_seed(1) 
sess = tf.Session(graph=tf.get_default_graph(), config=session_conf)  
K.set_session(sess)

###### This is the fourth code snipped to run #####

def model_cnn():
  model = Sequential()
  model.add(Conv2D(32, kernel_size=(3,3), kernel_initializer=initializers.glorot_uniform(seed=1), input_shape=(28,28,1)))
  model.add(BatchNormalization())
  model.add(Activation('relu'))

  model.add(Conv2D(32, kernel_size=(3,3), kernel_initializer=initializers.glorot_uniform(seed=2)))
  model.add(BatchNormalization())
  model.add(Activation('relu'))
  model.add(MaxPooling2D(pool_size=(2,2)))
  model.add(Dropout(0.25, seed=1))  

  model.add(Flatten())

  model.add(Dense(512, kernel_initializer=initializers.glorot_uniform(seed=2)))
  model.add(BatchNormalization())
  model.add(Activation('relu'))
  model.add(Dropout(0.5, seed=1))
  model.add(Dense(10, kernel_initializer=initializers.glorot_uniform(seed=2)))
  model.add(Activation('softmax'))

  model.compile(loss="categorical_crossentropy", optimizer=Adam(lr=0.001), metrics=['accuracy'])
  return model


def split_data(X,y):
  X_train_val, X_val, y_train_val, y_val = train_test_split(X, y, random_state=42, test_size=1/5, stratify=y) 
  return(X_train_val, X_val, y_train_val, y_val) 


def train_model_with_EarlyStopping(model, X, y):
  # make train and validation data
  X_tr, X_val, y_tr, y_val = split_data(X,y)

  es = EarlyStopping(monitor='val_loss', patience=20, mode='min', restore_best_weights=True)

  history = model.fit(X_tr, y_tr,
                      batch_size=64,
                      epochs=200, 
                      verbose=1,
                      validation_data=(X_val,y_val),
                      callbacks=[es])    

  return history

###### This is the fifth code snipped to run #####

train_model_with_EarlyStopping(model_cnn(), X, y)

每次运行上述代码时，我都会得到不同的结果。究其原因是代码本身，还是在Google Colab中使用GPU支持无法获得可重复的结果？

完整的代码（其中有一些不必要的部分，例如未使用的库）：

!pip install -U -q PyDrive
from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
from google.colab import auth
from oauth2client.client import GoogleCredentials
auth.authenticate_user()
gauth = GoogleAuth()
gauth.credentials = GoogleCredentials.get_application_default()
drive = GoogleDrive(gauth)
from __future__ import print_function # NEU 
import numpy as np 

import tensorflow as tf
import random as rn 
import os 
os.environ['PYTHONASHSEED'] = '0' 
np.random.seed(1)   
rn.seed(1)   
session_conf = tf.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1) 
from keras import backend as K

tf.set_random_seed(1)  
sess = tf.Session(graph=tf.get_default_graph(), config=session_conf)   
K.set_session(sess)  

import os
local_root_path = os.path.expanduser("~/data/data")
print(local_root_path)
try:
  os.makedirs(local_root_path, exist_ok=True)  
except: pass

def ListFolder(google_drive_id, destination):
  file_list = drive.ListFile({'q': "'%s' in parents and trashed=false" % google_drive_id}).GetList()
  counter = 0
  for f in file_list:
    # If it is a directory then, create the dicrectory and upload the file inside it
    if f['mimeType']=='application/vnd.google-apps.folder': 
      folder_path = os.path.join(destination, f['title'])
      os.makedirs(folder_path, exist_ok=True)
      print('creating directory {}'.format(folder_path))
      ListFolder(f['id'], folder_path)
    else:
      fname = os.path.join(destination, f['title'])
      f_ = drive.CreateFile({'id': f['id']})
      f_.GetContentFile(fname)
      counter += 1
  print('{} files were uploaded in {}'.format(counter, destination))
ListFolder("1DyM_D2ZJ5UHIXmXq4uHzKqXSkLTH-lSo", local_root_path)

import glob
import h5py
from time import time
from keras import initializers 
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential, model_from_json
from keras.layers import Dense, Dropout, Activation, Flatten, BatchNormalization, merge
from keras.layers import Convolution2D, MaxPooling2D, AveragePooling2D
from keras.optimizers import SGD, Adam, RMSprop, Adagrad, Adadelta, Adamax, Nadam
from keras.utils import np_utils
from keras.callbacks import LearningRateScheduler, ModelCheckpoint, TensorBoard, ReduceLROnPlateau
from keras.regularizers import l2
from keras.layers.advanced_activations import LeakyReLU, ELU
from keras import backend as K
import numpy as np
import pickle as pkl
from matplotlib import pyplot as plt
%matplotlib inline
import gzip
import numpy as np
from keras.utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense, Conv2D, Flatten
from keras.datasets import fashion_mnist
from numpy import mean, std
import matplotlib.pyplot as plt
from sklearn.model_selection import KFold, StratifiedKFold
from keras.datasets import fashion_mnist
from keras.utils import to_categorical
from keras.layers import Conv2D, MaxPooling2D, Dense, Flatten
from keras.optimizers import SGD, Adam
from keras.wrappers.scikit_learn import KerasClassifier
from sklearn.metrics import accuracy_score, confusion_matrix, classification_report
from sklearn.metrics import precision_score
from sklearn.metrics import recall_score
from sklearn.metrics import f1_score
from sklearn.metrics import auc, average_precision_score, f1_score

import time
from keras.callbacks import EarlyStopping
from keras.callbacks import ModelCheckpoint
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, classification_report
from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
from google.colab import files
from PIL import Image 



def model_cnn():
  model = Sequential()
  model.add(Conv2D(32, kernel_size=(3,3), kernel_initializer=initializers.glorot_uniform(seed=1), input_shape=(28,28,1)))
  model.add(BatchNormalization())
  model.add(Activation('relu'))
  model.add(Conv2D(32, kernel_size=(3,3), kernel_initializer=initializers.glorot_uniform(seed=2)))
  model.add(BatchNormalization())
  model.add(Activation('relu'))
  model.add(MaxPooling2D(pool_size=(2,2)))
  model.add(Dropout(0.25, seed=1))  
  model.add(Flatten())
  model.add(Dense(512, kernel_initializer=initializers.glorot_uniform(seed=2)))
  model.add(BatchNormalization())
  model.add(Activation('relu'))
  model.add(Dropout(0.5, seed=1))
  model.add(Dense(10, kernel_initializer=initializers.glorot_uniform(seed=2)))
  model.add(Activation('softmax'))
  model.compile(loss="categorical_crossentropy", optimizer=Adam(lr=0.001), metrics=['accuracy'])
  return model

def train_model_with_EarlyStopping(model, X, y):
  X_tr, X_val, y_tr, y_val = split_train_val_data(X,y)
  es = EarlyStopping(monitor='val_loss', patience=20, mode='min', restore_best_weights=True)      
  history = model.fit(X_tr, y_tr,
                      batch_size=64,
                      epochs=200, 
                      verbose=1,
                      validation_data=(X_val,y_val),
                      callbacks=[es])    
  evaluate_model(model, history, X_tr, y_tr)
  return history 


```

- Code Now

1

你是否得到了明显不同或非完全相等的结果？我发现有些GPU操作并不总是完全可重复的（例如大张量的轻微差异之和约减），这被认为是更好性能的一种权衡。关于这些问题，有一些已经提出的问题，比如这个。虽然我不确定这种情况仍然存在的程度。 - jdehesa

我有0.8905、0.8796、0.8849的测试准确率...问题也在于我在不同的时期获得最高的准确率。因此，EarlyStopping在第15、29、30个时期停止。 - Code Now

可以看到你的完整代码，包括具有正确形状的输入数据（可以随机生成），这将有所帮助。 - OverLordGoldDragon

请在Stackoverflow上检查我的答案：我尝试了网上大多数的解决方案，只有以下代码对我有效... - Ali karimi

请在stackoverflow上检查我的答案：我尝试了网上大多数的解决方案，只有以下代码对我有效... - Ali karimi

显示剩余2条评论

1个回答

网页内容由stack overflow 提供, 点击上面的

可以查看英文原文，
原文链接

- OverLordGoldDragon · Accepted Answer

问题不仅限于Colab，在本地也可以重现。然而，行为可能是不可避免的。

下面的代码是您的代码的最小可再现版本，拟合参数已调整以进行更快的测试。我观察到的是，在每次运行中，对于468个迭代，损失的最大差异仅为0.0144%，跨越5次运行。这很好。使用batch_size=64、60000个样本和20个时期，您将有18750个迭代——这将大大放大这个数字。

不管怎样，GPU并行性很可能是驱动随机性的罪魁祸首，小差异确实会随着时间的推移积累起来，从而产生实质性的差异，如下所示。如果1e-8看起来很小，请尝试向一半的权重添加随机噪声，大小被剪切在1e-8处，并见证其生活哲学的变化。

如果您不使用种子，它们的作用就会显著增强——尝试一下，所有指标都会在前10个迭代内狂飙。此外，损失更适合测量运行时间的差异，因为准确性对数值精度误差更加敏感：在一个由10个样本组成的批次中，60%准确性和70%准确性之间的差异是一个与0.5相比相差0.000001的预测——但损失几乎不会改变。

最后，请注意，您的超参数选择对模型性能的影响要比随机性大得多；无论您投入多少种子，它们都不能让模型成为SOTA。我推荐这段好片段。

您的代码很好。您已经采取了所有实用的措施来确保可重现性，但有一个例外：必须在Python内核启动之前设置PYTHONHASHSEED。

如何减少随机性？ 1. 重复运行，平均结果。可以理解为昂贵，但请注意，即使完全可再现的运行也不是完美的信息，因为关于训练和验证集的模型方差可能比噪声诱导的随机性大得多。 2. K折交叉验证：可以显著减轻数据和噪声方差。 3. 更大的验证集：由于噪声的存在，提取的特征只能有限地不同；验证集越大，权重中的小扰动就应该反映在指标中的相对较小程度。

GPU并行性：放大浮点误差

print(2. * 11. / 9.)  # 2.4444444444444446
print(2. / 9. * 11.)  # 2.444444444444444

操作顺序很重要，通过利用多线程，GPU并行处理不能保证操作以同样的顺序执行。乍一看，这种差异可能看起来无关紧要——但如果迭代足够多次...

one = 1
for _ in range(int(1e8)):
    one *= (2. / 9. * 11.) / (2. * 11. / 9.)
print(one)     # 0.9999999777955395
print(1 - one) # 1.8167285897874308e-08

一个"一(1)"通常是离其原始值1e-08只有极小的权重值，如果1亿次迭代看起来太多了，考虑到这个操作只需要半分钟完成，而您的模型可能需要训练一个小时，而前者完全运行在CPU上。

最小可复现实验：

import tensorflow as tf
import random as rn 
import numpy as np
np.random.seed(1)   
rn.seed(2)   
tf.set_random_seed(3)

from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten, BatchNormalization
from keras.layers import MaxPooling2D, Conv2D
from keras.optimizers import Adam

def model_cnn():
  model = Sequential()
  model.add(Conv2D(32, kernel_size=(3,3), 
                   kernel_initializer='he_uniform', input_shape=(28,28,1)))
  model.add(BatchNormalization())
  model.add(Activation('relu'))
  model.add(Conv2D(32, kernel_size=(3,3), kernel_initializer='he_uniform'))
  model.add(BatchNormalization())
  model.add(Activation('relu'))
  model.add(MaxPooling2D(pool_size=(2,2)))
  model.add(Dropout(0.25))
  model.add(Flatten())
  model.add(Dense(512, kernel_initializer='he_uniform'))
  model.add(BatchNormalization())
  model.add(Activation('relu'))
  model.add(Dropout(0.5))
  model.add(Dense(10, kernel_initializer='he_uniform'))
  model.add(Activation('softmax'))
  model.compile(loss="categorical_crossentropy", optimizer=Adam(lr=0.001), 
                metrics=['accuracy'])
  return model

np.random.seed(1)   
rn.seed(2)     
tf.set_random_seed(3) 

X_train = np.random.randn(30000, 28, 28, 1)
y_train = np.random.randint(0, 2, (30000, 10))
X_val   = np.random.randn(30000, 28, 28, 1)
y_val   = np.random.randint(0, 2, (30000, 10))
model = model_cnn()

np.random.seed(1)   
rn.seed(2)   
tf.set_random_seed(3)

history = model.fit(X_train, y_train, batch_size=64,shuffle=True, 
                    epochs=1, verbose=1, validation_data=(X_val,y_val))

运行差异:

loss: 12.5044 - acc: 0.0971 - val_loss: 11.5389 - val_acc: 0.1051
loss: 12.5047 - acc: 0.0958 - val_loss: 11.5369 - val_acc: 0.1018
loss: 12.5055 - acc: 0.0955 - val_loss: 11.5382 - val_acc: 0.0980
loss: 12.5042 - acc: 0.0961 - val_loss: 11.5382 - val_acc: 0.1179
loss: 12.5062 - acc: 0.0960 - val_loss: 11.5366 - val_acc: 0.1082