PyTorch_Practice_3

11 Advanced CNN

GooLeNet

减少代码冗余：函数/类

Inception Module

Concatenate:拼接张量

每种方式都使用，通过训练找到最优组合，调整权重。

Average Polling平均池化

What is 1×1 convolution?

可以改变通道数量，降低运算量

Implementation of Inception Module

沿着通道维度拼接

dim=1因为张量维度顺序是B,C,W,H

class Inception(nn.Module):
    # 初始通道可以在构造时指明
    def __init__(self, in_channels):
        super(Incetion, self).__init_()
        self.branch1x1 = nn.Conv2d(in_channels, 16, kernel_size=1)
        
        self.branch5x5_1 = nn.Conv2d(in_channels, 16, kernel_size=1)
        self.branch5x5_2 = nn.Conv2d(16, 24, kernel_size=5, padding=2)
        
        self.branch3x3_1 = nn.Conv2d(in_channels, 16, kernel_size=1)
        self.branch3x3_2 = nn.Conv2d(16, 24, kernel_size=3, padding=1)
        self.branch3x3_3 = nn.Conv2d(24, 24, kernel_size=3, padding=1)
               
        self.branch_pool = nn.Conv2d(in_channels, 24, kernel_size=1)
        
    def forward(self, x):
        branch1x1 = self.branch1x1(x)
        
        branch5x5 = self.branch5x5_1(x)
        branch5x5 = self.branch5x5_2(branch5x5)
        
        branch3x3 = self.branch3x3_1(x)
        branch3x3 = self.branch3x3_2(branch3x3)
        branch3x3 = self.branch3x3_3(branch3x3)
        
        branch_pool = F.avg_pool2d(x, kernel_size=3, stride=1, padding=1)
        branch_pool = self.branch_pool(branch_pool)
        
        outputs = [branch1x1, branch5x5, branch3x3, branch_pool]
        # 张量连接。输出通道是88
        return torch.cat(outputs, dim=1)
          

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)
        self.conv2 = nn.Conv2d(88, 20, kernel_size=5)
        
        self.incep1 = InceptionA(in_channels=10)
        self.incep2 = InceptionA(in_channels=20)
        
        self.mp = nn.MaxPoll2d(2)
        self.fc = nn.Linear(1408, 10)
        
    def forward(self, x):
        in_size = x.size(0)
        x = F.relu(self.mp(self.conv1(x)))
        x = self.incep1(x)
        x = F.relu(self.mp(self.conv2(x)))
        x = self.incep2(x)
        x = x.view(in_size, -1)
        x = self.fc(x)
        return x

Results of using Incuption Module

Can we stack layers to go deeper?

Deep Residual Learning 深度残差学习

增加跳连接，能解决梯度消失问题

Residual Network

虚线表示输入和输出维度不同，需要特殊处理

Implementation of Simple Residual Network

class ResidualBlock(nn.Module):
    def __init__(self, channels):
        super(ResidualBlock, self).__init__()
        self.channels = channels
        self.conv1 = nn.Con2d(channels, channels, kernel_size=3, padding=1)
        self.conv2 = nn.Con2d(channels, channels, kernel_size=3, padding=1)
        
    def forward(self, x):
        y = F.relu(self.conv1(x))
        y = self.conv2(y)
        # 先求和后激活
        return F.relu(x + y)

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(1, 16, kernel_size=5)
        self.conv2 = nn.Conv2d(16, 32, kernel_size=5)
        self.mp = nn.MaxPool2d(2)
        
        self.rblock1 = RessidualBlock(16)
	    self.rblock2 = RessidualBlock(32) 
    
        self.fc = nn.Linear(512, 10)
        
    def forward(self, x):
        in_size = x.size(0)
        x = self.mp(F.relu(self.conv1(x)))
        x = self.rblock1(x)
        x = self.mp(F.relu(self.conv2(x)))
        x = self.rblock2(x)
        x = x.view(in_size, -1)
        x = self.fc(x)
        return x

Results

12 Basic RNN

处理序列关系的数据：自然语言处理

Wht is RNNs?

What is RNN Cell?

RNN Cell in PyTorch

How to use RNNCell

import torch

# Parameters
barch_size = 1
seq_len = 3
input_size = 4
hidden_size = 2

# Construct RNNCell
cell = torch.nn.RNNCell(input_size=input_size, hidden_size=hidden_size)
# Wrapping the sequence into: (seq, batch, features)
dataset = torch.randn(seq_len, batch_size, input_size)
# Initializing the hidden to zero
hidden = torch.zeros(batch_size, hidden_size)

for idx, input in enumerate(dataset):
    # 将字符串 '=' 重复 20 次
    # ==================== idx ====================

    print('=' * 20, idx, '=' * 20)
    # The shape of input is: (batchSize, inputSize)
    # Input size: torch.Size([1, 4])
    print('Input size:', input,shape)
    # The shape of hidden is: (batchSize, hiddenSize)
    # hidden size: torch.Size([1, 2])
    hidden = cell(input, hidden)
    
    print('outputs size:', hidden.shape)
    print(hidden)

How to use RNN

numLayers

RNN的层数，不是RNNCell的个数

import torch

# Parameters
barch_size = 1
seq_len = 3
input_size = 4
hidden_size = 2
# Construction of RNN
cell = torch.nn.RNN(input_size=input_size, hidden_size=hidden_size, num_layers=numlayers)
# Wrapping the sequence into:(seqLen, batchSize, inputSize)
inputs = torch.randn(seq_len, batch_size, input_size)
# Initializing the hidden to zero
hidden = torch.zeros(num_layers, batch_size, hidden_size)

# The shape of output is:(seqSize, batchSize, hiddenSize)
out, hidden = cell(inputs, hidden)

pring('Output size:', out.shape)
pring('Output:', out)
pring('Hidden size:', hidden.shape)
pring('Hidden:', hidden)

Example: Using RNNCell

独热向量：One-Hot Vector

分类问题

Pramenters

import torch

input_size = 4
hidden_size = 4
batch_size = 1

Prepare Data

# The dictionary
idx2char = ['e', 'h', 'l', 'o']
# 'hello' The input sequence
x_data = [1, 0, 2, 2, 3]
# The output sequence
y_data = [3, 1, 2, 3, 2]

one_hot_lookup = [[1, 0, 0, 0],
                 [0, 1, 0, 0],
                 [0, 0, 1, 0],
                 [0, 0, 0, 1]]
# Convert indices into one-hot vector
'''
x会变成
[
  [0, 1, 0, 0],   # 对应索引 1
  [1, 0, 0, 0],   # 对应索引 0
  [0, 0, 1, 0],   # 对应索引 2
  [0, 0, 1, 0],   # 对应索引 2
  [0, 0, 0, 1]    # 对应索引 3
]
'''
x_one_hot = [one_hot_lookup[x] for x x_data]
# Reshape the inputs to (seqLen, batchSize, inputSize)
inputs = torch.Tensor(x_one_hot).view(-1, batch_size, input_size)
# Reshape the label to (seqLen, 1)
labels = torch.LongTensor(y_data).view(-1, 1)

Design Model

class Model(torch.nn.Module):
    def __init__(delf, input_size, hidden_size, barch_size):
        super(Model, self).__init__()
        self.batch_size = batch_size
        self.input_size = input_size
        self.hidden_size = hidden_size
        self.rnncell = torch.nn.RNNCell(input_size=self.input_size, hidden_size = self.hidden_size)
      
    def forward(self, input, hidden):
        hidden = self.rnncell(input, hidden)
        return hidden
    # Provide initial hidden
    def init_hidden(self):
        return torch.zero(self.batch_size, self.hidden_size)
    
net = Model(input_size, hidden_size, batch_size)
    

Loss and Optimizer

criterion = torch.nn.CrossEntroyLoss()
optimizer = torch.optim.Adam(net.parameters(), lr = 0.1)

Training Cycle

for epoch in range(15):
    loss = 0
    optimizer.zero_grad()
    # h0
    hidden = net.init.hidden()
    pring('Predicted string:', end='')
    for input, label in zip(inputs, labels):
        hidden = net(input, hidden)
        # 没有用item()，需要构造计算图
        loss += criterion(hidden, label)
        _, idx = hiddenmax(dim=1)
        pring(idx2char[idx.item()], end='')
    loss.backward()
    optimizer.step()
    pring(', Epoch [%d/15] loss=%.4f' % (epoch+1, loss.item()))

Results

Using Rnn Module

criterion = torch.nn.CrossEntroyLoss()
optimizer = torch.optim.Adam(net.parameters(), lr = 0.1)
for epoch in range(15):
    optimizer.zero_grad()
    outputs = net(inputs)
    loss = criterion(outputs labels)
    loss.backward()
    optimizer.step()
    
    _, idx = outputs.max(dim=1)
    idx = idx.data.numpy()
    pring('Predicted:', ''.join([idx2char[x] for x in idx]), end='')
    pring(', Epoch [%d/15] loss=%.3f' % (epoch+1, loss.item()))

Change Model

Change Data

Result

Associate a vector with a word/character

One-hot encoding of words and characters

• The one-hot vectors are high-dimension.
• The one-hot vectors are sparse.
• The one-hot vectors are hardcoded.

Do we have a way to associate a vector with a word/character
with following specification:

• Lower-dimension
• Dense
• Learned from data

A popular and powerful way is called EMBEDDING.

One-hot vs Embedding

Embedding in Pytorch

Using embedding and linear layer

13 RNN Classifier

Name Claffication

Our Model

Implementation

Main Cycle

if __name__ == '__main__':
    # instantiate the classifier model
    # 字母表大小，隐藏维度，国家，GRU层数
    classifier = RNNClassifier(N_CHARS, HIDDEN_SIZE, N_COUNTRY, N_LAYER)
    # Whether use GPU for training model
    if USE_GPU:
        device = torch.devoce("cuda:0")
        classifier.to(device)
    # Using cross entropy loss as loss function
    # Using Adam optimizer
    criterion = torch.nn.CrossEntropyLoss()
    optimizer = torch.optim.Adam(classifier.parameters(), lr=0.001)
    # For printing elapsed time
    start = time.time()
    pring("Training for %d epochs..." % N_EPOCHS)
    acc_list[]
    # In every epoch , trzining and testing the model once
    for epoch in range(1, N_EPOCHS + 1)
    	# Train cycle
        trainModel()
        acc = testModel()
        acc_list.append(acc)

Preparing Data

将字符串转换成列表，求每个字符对应的ASCII值

padding：使得字符的长度对齐，方便构成张量

将国家名宇索引对应

class NameDataset(Dataset):
    def __init__(self, is_train_set=True):
        # Reading data from .gz file with package gzip and csv
        filename = 'data/names_train.csv.gz' if is_train_set else 'data/names_test.csv.gz'
        with gzip.open(filename, 'rt') as f:
            reader = csv.reader(f)
            rows = list(reader)
        # Save names and countries in list
        self.names = [row[0] for row in rows]
        self.len = len(self.names)
        self.countries = [row[1] for row in rows]
        # Save countries and its index in list and dictionary
        # set去重
        self.country_list = list(sorted(set(self.countries)))
        self.country_dict = self.getCountrDict()
        self.country_num = len(self.country_list)
        
    def __getitem__(self, index):
        # 得到名字和国家索引
        return self.names[index], slef.country_dict[self.countries[index]]
    
    def __len__(self):
        # Return the length of dataset
        return self.len   

class NameDataset(Dataset):
    ......
    # Convert list to dictionary
    def getCountryDict(self):
        # 空字典
        country_dict = dict()
        for idx, country_name in enumerate(self.country_list, 0):
            # 键值对
            country_dict[country_name] = idx
        return country_dict
    # Return country name giving index
    def idx2country(self, index):
        return self.country_list[index]
    # Return the number of country
    def getCountriesNum(self):
        return self.country_num

trainset = NameDataset(is_train_set=True)
trainloader = DataLoader(trainset, batch_size=BATCH_SIZE, shuffle=True)
testset = NameDataset(is_train_set=False)
testloader = DataLoader(testset, batch_size=BATCH_SIZE, shuffle=False)
# output size of out model
N_COUNTRY = trainset.getCountriesNum()

# Paramenters
HIDDEN_SIZE = 100
BATCH_SIZE = 256
N_LAYER = 2
N_EPOCHS = 100
N_CHARS = 128
USE_GPU = False

Model Design

class RNNClassifier(torch.nnModule):
    def __init__(self, input_size, hidden_size, output_size, n_layers=1, bidirectional=True)
    	super(RNNClassifier, self).__init__()
        # Parameters og GRU layers
        self.hidden_size = hidden_size
        self.n_layers = n_layers
        # 双向或单相RNN
        self.n_directions = 2 if bidirectional else 1
        # Parameters of GRU
        self.embedding = torch.nn.Embedding(input_size, hidden_size)
        self.gru = torch.nn.GRU(hidden_size, hidden_size, n_layers, bidirectional = bidirectional)
        self.fc = torch.nn Linear(hidden_size * self.n_directions, output_size)
        
    def _init_hidden(self, batch_size):
        hidden = torch.zeros(self.n_layers * self.n_directions, batch_size, self.hidden_size)
        return creatr_tensor(hidden)

Bi-direction RNN/LSTM/GRU

class RNNClassifier(torch.nn.Module):
    def forward(self, input, seq_lengths):
        # input shape: B × S-> S × B
        # 转秩
        input = input.t()
        # Save batch_size for make intial hidden
        batch_size = input.size(1)
        hidden = self._init_hidden(batch_size)
        embedding = self.embedding(input)
        
        # pack them up
        # returns a PackSquence object
        gru_input = pack_padded_sequence(embedding, seq_lengths)
        
        output, hidden = self.gru(gru_input, hidden)
        if self.n_directions = 2
        	hidden_cat = torch.cat([hidden[-1], hidden[-2]], dim=1)
        else:
            hidden_cat = hidden[-1]
        fc_output = self.fc(hidden_cat)
        return fc_ouput

pack_padded_sequence(embedding, seq_lengths)：

先根据序列长度排序，再只放非0的序列

Convert name to tensor

def make_tensors(names, countries):
    sequences_and_lengths = [name2list(name) for name in names]
    name_sequences = [s1[0] for s1 in sequences_and_lengths]
    seq_lengths = torch.LongTensor([s1[1] for s1 in sequences_and_lengths])
    countries = countries.long()
    
    # make tensor of name, BatchSize x SeqLen
    seq_tensor = torch.zeros(len(name_sequences), seq_lengths.max().long())
    for idx, (seq, seq_len) in enumerate(zip(name_sequences, seq_lengths), 0)
    	seq_tensor[idx, :seq_len] = torch.LongTensor(seq)
        
    # sort by length to use park_padded_sequences
    seq_lengths, perm_idx = seq_lengths.sort(dim=0, descending=True)
    seq_tensor = seq_tensor[perm_idx]
    countries = countries[perm_idx]
    
    return create_tensor(seq_tensor),
		  create_tensor(seq_lengths),
          create_tensor(countries)

One Epoch Training

Testing

Result