PyTorch实践

Veröffentlicht am | Edited on | in | Views:

PyTorch实践

数据集处理

官方数据集

从PyTorch开源项目：vision或者text中加载数据集，对应的包是torchvision、torchtext

1
2
3
4
5
6
7
8
9
10
11
12
13
#torchvision
import torchvision
mnist = torchvision.datasets.MNIST(root='./',download="True")

#torchtext
import torchtext
WORD = torchtext.data.Field(init_token="<bos>",eos_token="<eos>")
UD_TAG = torchtext.data.Field(init_token="<bos>",eos_token="<eos>")

train,val,text = torchtext.datasets.UDPOS.splits(field=(('word',WORD),('udtag',UD_TAG),(None,None)))

print(train.examples[0].word)
print(train.examples[0].udtag)

构建自定义数据集

1
2
3
4
5
6
7
import torch.utils.data as Data

x = torch.tensor([1,2,3])
y = torch.tensor([0,0,1])
#小于2都是0，大于2为1

dataset = Data.TensorDataset(x,y)

细粒度构建自定义数据集

继承torch.utils.data.Dataset ，然后重载函数len，getitem来构建

1
2
3
4
5
6
7
8
9
10
11
12
13
14
import torch.utils.data as Data

class Mydataset(Data.Dataset):
    def __init__(self):
        #读取文件
        pass
    def __getitem__(self):
        #获取数据
        #预处理
        #返回数据对（x,y）
        pass
    def __len__(self):
        #返回数据集大小
        pass
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
import torch.utils.data as Data

person_1 = ['tall','rich','handsome','1']
person_2 = ['n_tall','rich','handsome','1']
person_3 = ['n_tall','n_rich','handsome','1']
person_4 = ['n_tall','n_rich','n_handsome','0']

person = [person_1,person_2,person_3,person_4]

class Person(Data.Dataset):
    def __init__(self,person):
        self.data = person
    def __getitem__(self,index):
        item = self.data[index]
        x = item[:3]
        y = item[-1]
        return x,y
    def __len__(self):
        return(len(self.data))
    
p = Person(person)

for index, (x,y) in enumerate(p):
    print(f"{index} : x:{x},y:{y}")
    
"""
0 : x:['tall', 'rich', 'handsome'],y:1
1 : x:['n_tall', 'rich', 'handsome'],y:1
2 : x:['n_tall', 'n_rich', 'handsome'],y:1
3 : x:['n_tall', 'n_rich', 'n_handsome'],y:0
"""

加载数据集

使用torch.utils.data.Dataloader加载数据集

1
2
3
4
5
tensor_person_data_loader = Data.DataLoader(dataset = person_dataset,
										batch_size = batch_size,
										shuffle = True,
										sampler = sampler,			#怎么抽样
										collate_fn = my_collate_fn)	#抽样的样本怎么处理

加载自定义数据集：

1
2
3
4
5
6
7
8
9
10
11
person_loader = Data.Dataloader(dataset = p,
                               batch_size = 2)

for index,data in enumerate(person_loader):
    x,y = data
    print(f"{index} : x:{x},y:{y}")
    
"""
0 : x:[('tall', 'n_tall'), ('rich', 'rich'), ('handsome', 'handsome')],y:('1', '1')
1 : x:[('n_tall', 'n_tall'), ('n_rich', 'n_rich'), ('handsome', 'n_handsome')],y:('1', '0')
"""

预处理数据集

通过__getitem__方法

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
word2id = {
'tall':1,
'rich':1,
'handsome':1,
'n_tall':0,
'n_rich':0,
'n_handsome':0,
'1':1,
'0':0
}

class TensorPerson(Data.Dataset):
    def __init__(self,person):
        self.data = person
    def __getitem__(self,index):
        item = self.data[index]
        new_item = []
        for feature in item:
            new_item.append(word2id[feature])
        x = new_item[:3]
        y = new_item[-1]
        return x,y
    def __len__(self):
        return(len(self.data))
    
p = TensorPerson(person)
person_loader = Data.DataLoader(dataset = p,
                				batch_size = 2)

for index, data in enumerate(person_loader):
  x,y = data
  print(f"{index} : x:{x},y:{y}")
    
"""
0 : x:[tensor([1, 0]), tensor([1, 1]), tensor([1, 1])],y:tensor([1, 1])
1 : x:[tensor([0, 0]), tensor([0, 0]), tensor([1, 0])],y:tensor([1, 0])
"""

通过collate_fn

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

#定义抽样样本处理方法：
def my_collate_fn(batch_data):
    print("----------")
    x_batch = []
    y_batch = []
    for example in batch_data:
        x,y = example
        x_batch.append(x)
        y_batch.append(y)
    x_batch = torch.tensor(x_batch,dtype = torch.float32)
    y_batch = torch.tensor(y_batch,dtype = torch.float32)
    print("----------")
    return x_batch,y_batch

person_loader = Data.DataLoader(dataset = p,
                				batch_size = 2,
                               collate_fn = my_collate_fn)

for index, data in enumerate(person_loader):
  x,y = data
  print(f"{index} : x:{x},y:{y}")
    
"""
----------
----------
0 : x:tensor([[1., 1., 1.],
        [0., 1., 1.]]),y:tensor([1., 1.])
----------
----------
1 : x:tensor([[0., 0., 1.],
        [0., 0., 0.]]),y:tensor([1., 0.])
"""

模型

TorchVision支持一些经典模型

  • Alexnet
  • VGG
  • ResNet
  • SqueezeNet
  • DenseNet
  • Inception v3

TorchText没有统一模型

经典模型以及可视化

1
2
3
4
5
6
7
8
9
import torch
import torchvision.models
import hiddenlayer as hl
from torchviz import make_dot

resnet = torchvision.models.resnet18()
# hl.build_graph(resnet,torch.randn([1,3,224,224]))
# 由于版本问题，使用hiddenlayer的可视化可能找不到get_trace_graph方法，从而报错
make_dot(resnet(torch.randn([1,3,224,224]))).view()

可以使用torchviz的方法进行可视化，会在目录下生成一个pdf文件

构建自己的神经网络结构

sample

1
2
3
4
5
6
7
8
9
10
11
12
13
14
my_net = torch.nn.Sequential(
    torch.nn.Linear(1,10),
    torch.nn.ReLU(inplace=True),
    torch.nn.Linear(10,1)
)
my_net

"""
Sequential(
  (0): Linear(in_features=1, out_features=10, bias=True)
  (1): ReLU(inplace=True)
  (2): Linear(in_features=10, out_features=1, bias=True)
)
"""

构建多层网络

1
2
3
4
5
6
7
8
hun_layer = [torch.nn.Linear(10,10) for _ in  range(100)]

"""
[Linear(in_features=10, out_features=10, bias=True),
 Linear(in_features=10, out_features=10, bias=True),
 ``````````````````````````
 Linear(in_features=10, out_features=10, bias=True)]
"""

深度自定义

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
import torch.nn as nn

class MyNET(nn.Module):
  def __init__(self):
    super(MyNET,self).__init__()
    pass
  def forward(self,x):
    pass

class SingleDogCls(nn.Module):
  def __init__(self,n_feature,n_hidden,n_output):
    super(SingleDogCls,self).__init__()
    self.hidden = nn.Linear(n_feature,n_hidden)
    self.relu = nn.ReLU()
    self.predict = nn.Linear(n_hidden,n_output)

  def forward(self,x):
    x = self.hidden(x)
    x = self.relu(x)
    x = self.predict(x)
    return x

w = SingleDogCls(n_feature=3, n_hidden=10, n_output=1)
make_dot(w(torch.randn([10,3]))).view()

使用torchviz产生的可视化图不如hiddenlayer的简洁：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
for index,data in enumerate(person_loader):
  print(f"迭代次数{index}")
  print(f"data:{data}")
  input_x,ground_truth = data
  predict = w(input_x)
  print(predict,ground_truth)

"""
----------
----------
迭代次数0
data:(tensor([[1., 1., 1.],
        [0., 1., 1.]]), tensor([1., 1.]))
tensor([[-0.2159],
        [-0.2262]], grad_fn=<AddmmBackward>) tensor([1., 1.])
----------
----------
迭代次数1
data:(tensor([[0., 0., 1.],
        [0., 0., 0.]]), tensor([1., 0.]))
tensor([[-0.0682],
        [-0.2633]], grad_fn=<AddmmBackward>) tensor([1., 0.])
"""

定义损失

官方定义的损失函数在两个地方

  • torch.nn
  • torch.nn.functional

torch.nn下面的loss函数，是作为模型的一部分存在，实际上是torch.nn.functional下loss函数的封装，实际上它还是调用了torch.nn.functional下面的函数

1
2
3
4
5
6
7
8
9
a = torch.tensor(2.)
b = torch.tensor(5.)
print(F.l1_loss(a,b))
print(F.mse_loss(a,b))

"""
tensor(3.)
tensor(9.)
"""

损失函数

torch.nn.functional:

  • binary_cross_entropy
  • binary_cross_entropy_with_logits
  • poisson_nll_loss
  • cross_entropy
  • cosine_embedding_loss
  • ctc_loss
  • hinge_embedding_loss
  • kl_div
  • l1_loss
  • mse_loss
  • margin_ranking_loss
  • multilabel_margin_loss
  • multilabel_soft_margin_loss
  • multi_margin_loss
  • nll_loss
  • smooth_l1_loss
  • soft_margin_loss
  • triplet_margin_loss

如果要定义自己的损失函数，可以：

  • 继承torch.nn.module实现loss（当成模型，计算图中的方块）
  • 继承torch.autograd.function实现loss（当成函数，计算图中的椭圆）

实际区别在于谁去进行反向传播，官方实现中，loss是一个module

继承torch.nn.module实现loss

1
2
3
4
5
6
7
8
9
10
class MyLoss(nn.Module):
    def __init__(self):
        super(MyLoss,self).__init__()
    def forward(self,x,y):
        return x*100-y
    
loss = MyLoss()
loss(a,b)

# tensor(195.)

继承torch.autograd.funcion实现loss

适合损失函数非常难求导时

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
from torch.autograd import Function

class MyLossFunc(torch.autograd.Function):
    def forward(self,a,b):
        return 2*(a-b)
    def backward(self,grad_output):
        print("----------")
        return grad_output*2,grad_output*2 # a_grad, b_grad
    
a = torch.tensor(1.,requires_grad=True)
b = torch.tensor(2.,requires_grad=True)
f_loss = MyLossFunc()
c = f_loss(a,b)

# tensor(-2., grad_fn=<MyLossFunc>)

实现优化算法

pytorch集成了常见的优化算法：

  • torch.optim.Adadelta
  • torch.optim.Adagrad
  • torch.optim.Adam
  • torch.optim.Adamax
  • torch.optim.SparseAdam
  • torch.optim.LBFGS
  • torch.optim..RMSprop
  • torch.optim.Rprop
  • torch.optim.SGD
1
2
3
from torch.optim import Optimizer

optimizer = torch.optim.SGD(w.parameters(),lr=0.05,mometum=0.9)

如果自定义优化算法：

1
2
3
4
class MyOptimizer(Optimizer):
  def step(self,closure=None):
    #每一步是怎么优化的
    pass

调整学习率

pytorch官方，提供了torch.optim.lr_scheduler类来基于动态调整学习率

  • torch.optim.lr.scheduler.LambdaLR
  • torch.optim.lr.scheduler.StepLR
  • torch.optim.lr.scheduler.MultiStepLR
  • torch.optim.lr.scheduler.ExponentialLR
  • torch.optim.lr.scheduler.CosineAnnealingLR
  • torch.optim.lr.scheduler.ReduceLROnPlateau
1
2
3
4
5
from torch.optim.lr_scheduler import StepLR

scheduler = StepLR(optimizer,step_size=30,gamma=0.1)

# 每隔30次迭代就乘gamma,在迭代固定的次数之后以一定的比例降低学习率

迭代训练

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
import torch
import torch.utils.data as Data
import torch.nn as nn
from torch.optim import Optimizer
from torch.optim.lr_scheduler import StepLR

person_1 = ['tall','rich','handsome','1']
person_2 = ['n_tall','rich','handsome','1']
person_3 = ['n_tall','n_rich','handsome','1']
person_4 = ['n_tall','n_rich','n_handsome','0']
person = [person_1,person_2,person_3,person_4]

word2id = {
'tall':1,
'rich':1,
'handsome':1,
'n_tall':0,
'n_rich':0,
'n_handsome':0,
'1':1,
'0':0
}

# 将数据表进行封装
class ManDataset(Data.Dataset):
    def __init__(self,person):
        self.data = person;
    def __getitem__(self,index):
        item = self.data[index]
        new_item = []
        for feature in item:
            new_item.append(word2id[feature])
        x = new_item[:3]
        y = new_item[-1]
        return x,y
    def __len__(self):
        return(len(self.data))

# 对数据进行预处理
def my_collate_fn(batch_data):
    x_batch = []
    y_batch = []
    for example in batch_data:
        x,y = example
        x_batch.append(x)
        y_batch.append(y)
    x_batch = torch.tensor(x_batch,dtype = torch.float32)
    y_batch = torch.tensor(y_batch,dtype = torch.float32)
    return x_batch,y_batch

# 创建数据集实例
person_dataset = ManDataset(person)
# 创建加载器，加载数据集
batch_size = 1
person_data_loader = Data.DataLoader(
    dataset=person_dataset,
    batch_size=batch_size,
    collate_fn=my_collate_fn
)

# 实现模型
class SingleDog(nn.Module):
  def __init__(self,n_feature,n_hidden,n_output):
    super(SingleDog,self).__init__()
    self.hidden = nn.Linear(n_feature,n_hidden)
    self.relu = nn.ReLU()
    self.predict = nn.Linear(n_hidden,n_output)

  def forward(self,x):
    x = self.hidden(x)
    x = self.relu(x)
    x = self.predict(x)
    return x

# 创建模型实例
will_u_be_single = SingleDog(n_feature=3,n_hidden=10,n_output=1)

#########
will_u_be_single.cuda()
#########

# 创建loss函数
single_loss = nn.L1Loss()

# 创建优化器
optimizer = torch.optim.SGD(will_u_be_single.parameters(),lr=0.1,momentum=0.9)

# 根据epoch，自动更新优化器参数
scheduled_optimizer = StepLR(optimizer,step_size=10,gamma=0.1)

# 开始迭代训练
total_epoch = 100
for epoch in range(total_epoch):
  for index,data in enumerate(person_data_loader):
    input_x,ground_truth = data
    
    ##########
    input_x = input_x.cuda()
    ground_truth = ground_truth.cuda()
    ##########
    
    predict = will_u_be_single(input_x)
    loss = single_loss(predict.squeeze(),ground_truth)
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()
  scheduled_optimizer.step()
  print(f"Epoch {epoch},loss: {loss.item()}, current lr:{scheduled_optimizer.get_lr()}")

测试：

1
will_u_be_single(torch.tensor([1.,1.,0.]))

加速计算

1
2
3
4
5
6
7
8
9
10
torch.cuda.is_available()

#########
will_u_be_single.cuda()
#########

##########
input_x = input_x.cuda()
ground_truth = ground_truth.cuda()
##########

储存和加载模型

储存整个模型：

torch.save(will_u_be_single,"./will_u_be_single.pkl")

加载整个模型：

model_from_file = torch.load("./will_u_be_single.pkl")

储存参数：

torch.save(will_u_be_single.state_dict(),"./will_u_be_single_2.pkl")

加载参数

1
2
model_parameter = torch.load("./will_u_be_single_2.pkl")
will_u_be_single.load_state_dict(model_parameter)