从头开始实现Dropout

Question

这段代码尝试使用自定义的dropout实现：

%reset -f

import torch
import torch.nn as nn
# import torchvision
# import torchvision.transforms as transforms
import torch
import torch.nn as nn
import torch.utils.data as data_utils
import numpy as np
import matplotlib.pyplot as plt
import torch.nn.functional as F

num_epochs = 1000

number_samples = 10

from sklearn.datasets import make_moons
from matplotlib import pyplot
from pandas import DataFrame
# generate 2d classification dataset
X, y = make_moons(n_samples=number_samples, noise=0.1)
# scatter plot, dots colored by class value

x_data = [a for a in enumerate(X)]
x_data_train = x_data[:int(len(x_data) * .5)]
x_data_train = [i[1] for i in x_data_train]
x_data_train

y_data = [y[i[0]] for i in x_data]
y_data_train = y_data[:int(len(y_data) * .5)]
y_data_train

x_test = [a[1] for a in x_data[::-1][:int(len(x_data) * .5)]]
y_test = [a for a in y_data[::-1][:int(len(y_data) * .5)]]

x = torch.tensor(x_data_train).float() # <2>
print(x)

y = torch.tensor(y_data_train).long()
print(y)

x_test = torch.tensor(x_test).float()
print(x_test)

y_test = torch.tensor(y_test).long()
print(y_test)

class Dropout(nn.Module):
    def __init__(self, p=0.5, inplace=False):
#         print(p)
        super(Dropout, self).__init__()
        if p < 0 or p > 1:
            raise ValueError("dropout probability has to be between 0 and 1, "
                             "but got {}".format(p))
        self.p = p
        self.inplace = inplace

    def forward(self, input):
        print(list(input.shape))
        return np.random.binomial([np.ones((len(input),np.array(list(input.shape))))],1-dropout_percent)[0] * (1.0/(1-self.p))

    def __repr__(self):
        inplace_str = ', inplace' if self.inplace else ''
        return self.__class__.__name__ + '(' \
            + 'p=' + str(self.p) \
            + inplace_str + ')'

class MyLinear(nn.Linear):
    def __init__(self, in_feats, out_feats, drop_p, bias=True):
        super(MyLinear, self).__init__(in_feats, out_feats, bias=bias)
        self.custom_dropout = Dropout(p=drop_p)

    def forward(self, input):
        dropout_value = self.custom_dropout(self.weight)
        return F.linear(input, dropout_value, self.bias)



my_train = data_utils.TensorDataset(x, y)
train_loader = data_utils.DataLoader(my_train, batch_size=2, shuffle=True)

my_test = data_utils.TensorDataset(x_test, y_test)
test_loader = data_utils.DataLoader(my_train, batch_size=2, shuffle=True)

# Device configuration
device = 'cpu'
print(device)

# Hyper-parameters 
input_size = 2
hidden_size = 100
num_classes = 2

learning_rate = 0.0001

pred = []

# Fully connected neural network with one hidden layer
class NeuralNet(nn.Module):
    def __init__(self, input_size, hidden_size, num_classes, p):
        super(NeuralNet, self).__init__()
#         self.drop_layer = nn.Dropout(p=p)
#         self.drop_layer = MyLinear()
#         self.fc1 = MyLinear(input_size, hidden_size, p)
        self.fc1 = MyLinear(input_size, hidden_size , p) 
        self.relu = nn.ReLU()
        self.fc2 = nn.Linear(hidden_size, num_classes)  

    def forward(self, x):
#         out = self.drop_layer(x)
        out = self.fc1(x)
        out = self.relu(out)
        out = self.fc2(out)
        return out

model = NeuralNet(input_size, hidden_size, num_classes, p=0.9).to(device)

# Loss and optimizer
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)  

# Train the model
total_step = len(train_loader)
for epoch in range(num_epochs):
    for i, (images, labels) in enumerate(train_loader):  
        # Move tensors to the configured device
        images = images.reshape(-1, 2).to(device)
        labels = labels.to(device)

        # Forward pass
        outputs = model(images)
        loss = criterion(outputs, labels)

        # Backward and optimize
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

    if (epoch) % 100 == 0:
        print ('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}'.format(epoch+1, num_epochs, i+1, total_step, loss.item()))

自定义的dropout实现如下：

class Dropout(nn.Module):
    def __init__(self, p=0.5, inplace=False):
#         print(p)
        super(Dropout, self).__init__()
        if p < 0 or p > 1:
            raise ValueError("dropout probability has to be between 0 and 1, "
                             "but got {}".format(p))
        self.p = p
        self.inplace = inplace

    def forward(self, input):
        print(list(input.shape))
        return np.random.binomial([np.ones((len(input),np.array(list(input.shape))))],1-dropout_percent)[0] * (1.0/(1-self.p))

    def __repr__(self):
        inplace_str = ', inplace' if self.inplace else ''
        return self.__class__.__name__ + '(' \
            + 'p=' + str(self.p) \
            + inplace_str + ')'

class MyLinear(nn.Linear):
    def __init__(self, in_feats, out_feats, drop_p, bias=True):
        super(MyLinear, self).__init__(in_feats, out_feats, bias=bias)
        self.custom_dropout = Dropout(p=drop_p)

    def forward(self, input):
        dropout_value = self.custom_dropout(self.weight)
        return F.linear(input, dropout_value, self.bias)

看起来我实现了错误的dropout函数？:

np.random.binomial([np.ones((len(input),np.array(list(input.shape))))],1-dropout_percent)[0] * (1.0/(1-self.p))

如何修改以正确使用dropout？

这些文章有助于理解：

Hinton在Python中用3行代码实现的Dropout： https://iamtrask.github.io/2015/07/28/dropout/ 创建自定义的Dropout函数： https://discuss.pytorch.org/t/making-a-custom-dropout-function/14053/2

- blue-sky

---

您说您的印象是“似乎您不正确地实现了dropout函数”。是什么让您这样想？ - Felix

Answer 1

It seems I've implemented the dropout function incorrectly?

np.random.binomial([np.ones((len(input),np.array(list(input.shape))))],1 dropout_percent)[0] * (1.0/(1-self.p))

实际上，以上实现被称为反向丢弃。 反向丢弃是在各种深度学习框架中实现丢弃的方式。 什么是反向丢弃？ 在深入了解反向丢弃之前，了解一下如何使单个神经元的丢弃起作用可能会有所帮助：

由于在训练阶段，神经元的保持概率为q（=1-p），因此在测试阶段，我们需要模拟在训练阶段使用的网络集合的行为。为此，作者建议在测试阶段将激活函数按q的因子进行缩放，以便使用在训练阶段产生的预期输出作为测试阶段所需的单个输出（第10节，乘性高斯噪声）。因此：

反向丢弃是有所不同的。这种方法在训练阶段对激活进行缩放，保持测试阶段不变。缩放因子是保留概率的倒数1/1-p=1/q，因此：

倒置丢弃（Inverted Dropout）有助于仅更改参数（保留/丢弃概率），就能在同一模型上运行训练和测试。相反，直接丢弃（Direct Dropout）会强制您在测试阶段修改网络，因为如果不乘以q，则神经元的输出将产生比后续神经元期望值更高的值（因此后续神经元可能会饱和或爆炸）：这就是为什么倒置丢弃是更常见的实现方式。

参考文献：

- Andrew NG的coursera授课《Dropout Regularization》 - 什么是倒置丢弃？ - 缩放激活与倒置丢弃的比较 - 丢弃的分析

---

如何在Pytorch中实现反向丢弃？

class MyDropout(nn.Module):
    def __init__(self, p: float = 0.5):
        super(MyDropout, self).__init__()
        if p < 0 or p > 1:
            raise ValueError("dropout probability has to be between 0 and 1, " "but got {}".format(p))
        self.p = p

    def forward(self, X):
        if self.training:
            binomial = torch.distributions.binomial.Binomial(probs=1-self.p)
            return X * binomial.sample(X.size()) * (1.0/(1-self.p))
        return X

如何在Numpy中实现？

import numpy as np

pKeep = 0.8
weights = np.ones([1, 5])
binary_value = np.random.rand(weights.shape[0], weights.shape[1]) < pKeep
res = np.multiply(weights, binary_value)
res /= pKeep  # this line is called inverted dropout technique
print(res)

如何在Tensorflow中实现？

import tensorflow as tf
tf.enable_eager_execution()

weights = tf.ones(shape=[1, 5])
keep_prob = 0.8
random_tensor = keep_prob
random_tensor += tf.random_uniform(weights.shape)
# 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob)
binary_tensor = tf.floor(random_tensor)
ret = tf.div(weights, keep_prob) * binary_tensor
print(ret)

- Amir

---

在Pytorch实现中，为什么使用了二项分布？难道不应该使用伯努利分布吗？ - Alex Deft

二项式由伯努利定义，所以这可能是原因。在pytorch中，您可以使用这两个实现相同的行为。但是似乎二项式可以更快。 - Károly Szabó

在PyTorch实现中，为什么在推理过程中返回权重而不是直接返回激活值X？ - Gilfoyle

Answer 2

使用Torch和伯努利实现...

def forward(self, x):
    output = x @ self.W.t() + self.bias
  if self.training:
    sample = torch.distributions.bernoulli.Bernoulli(self.keep_prob).sample(output.size())
    print(sample)
    return output * sample
  return output

- Sahar Millis