# 实验准备

## 基础网络搭建

```class net(pt.nn.Module):

def __init__(self):
super(net,self).__init__()
self.fc1 = pt.nn.Linear(in_features=7*7*256,out_features=512)
self.fc2 = pt.nn.Linear(in_features=512,out_features=10)
self.pool = pt.nn.MaxPool2d(2)

def forward(self,x):
x = self.pool(pt.nn.functional.relu(self.conv1(x)))
x = self.pool(pt.nn.functional.relu(self.conv2(x)))
x = pt.nn.functional.relu(self.fc1(x.view((-1,7*7*256))))
return self.fc2(x)```
```model = net().cuda()
print(model)
print(model(pt.rand(1,1,28,28).cuda()))```
```net(
(conv1): Conv2d(1, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(conv2): Conv2d(64, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(fc1): Linear(in_features=12544, out_features=512, bias=True)
(fc2): Linear(in_features=512, out_features=10, bias=True)
(pool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
)
tensor(1.00000e-02 *
[[-7.7157,  3.0435, -6.5732,  6.5343, -4.2159, -2.8651, -0.6792,
3.9223, -3.7523,  2.4532]], device='cuda:0')```

## 基础网络训练

### 准备数据集

```train_dataset = ptv.datasets.MNIST("./",download=True,transform=ptv.transforms.ToTensor())
test_dataset = ptv.datasets.MNIST("./",train=False,transform=ptv.transforms.ToTensor())

### 代价函数与优化器

```lossfunc = pt.nn.CrossEntropyLoss().cuda()
```def acc(outputs,label):
_,data = pt.max(outputs,dim=1)
return pt.mean((data.float()==label.float()).float()).item()```

### 网络训练

```for _ in range(1):
for i,(data,label) in enumerate(trainloader):
data,label = data.cuda(),label.cuda()
outputs = model(data)
loss = lossfunc(outputs,label)
loss.backward()
optimizer.step()
if i % 100 == 0:
print(i,acc(outputs,label))```
```0 0.1171875
100 0.8984375
200 0.953125
300 0.984375
400 0.96875```

### 测试网络

```def test_model(model,testloader):
result = []
for data,label in testloader:
data,label = data.cuda(),label.cuda()
outputs = model(data)
result.append(acc(outputs,label))
result = sum(result) / len(result)
print(result)
return result
`0.96875`

### 保存网络

`pt.save(model.state_dict(),"./base.ptb")`

# 剪枝实验

## 准备相关工具

### 剪枝函数

```def puring(model,threshold):
for i in model.parameters():
i.data[pt.abs(i) < threshold] = 0
return model```

### 梯度剪枝函数

```def grad_puring(model):
for i in model.parameters():

### 稀疏度评估函数

```def print_sparse(model):
result = []
total_num = 0
total_sparse = 0
print("-----------------------------------")
print("Layer sparse")
for name,f in model.named_parameters():
num = f.view(-1).shape[0]
total_num += num
sparse = pt.nonzero(f).shape[0]
total_sparse+= sparse
print("\t",name,(sparse)/num)
result.append((sparse)/num)
total = total_sparse/total_num
print("Total:",total)

## 剪枝

```model = net().cuda()
`0.96875`
`print_sparse(model)`
```-----------------------------------
Layer sparse
conv1.weight 1.0
conv1.bias 1.0
conv2.weight 1.0
conv2.bias 1.0
fc1.weight 1.0
fc1.bias 1.0
fc2.weight 1.0
fc2.bias 1.0
Total: 1.0```

```model1 = puring(model,0.01)
print_sparse(model1)```
```0.9706289556962026
-----------------------------------
Layer sparse
conv1.weight 0.9739583333333334
conv1.bias 0.90625
conv2.weight 0.7641262478298612
conv2.bias 0.71875
fc1.weight 0.06729390669842156
fc1.bias 0.025390625
fc2.weight 0.7837890625
fc2.bias 0.9
Total: 0.08358673475128647

0.08358673475128647```

```model.load_state_dict(pt.load("./base.ptb"))
model2 = puring(model,0.1)
print_sparse(model2)```
```0.09760680379746836
-----------------------------------
Layer sparse
conv1.weight 0.671875
conv1.bias 0.6875
conv2.weight 0.0
conv2.bias 0.0
fc1.weight 0.0
fc1.bias 0.0
fc2.weight 0.0
fc2.bias 0.0
Total: 6.553616029871108e-05

6.553616029871108e-05```

```sparse_list = []
threshold_list = [x*0.01+0.01 for x in range(10)]
acc_list = []
for i in threshold_list:
model3 = puring(model,i)
sparse_list.append(print_sparse(model3))
threshold_list.append```
```0.9706289556962026
-----------------------------------
Layer sparse
conv1.weight 0.9739583333333334
conv1.bias 0.90625
conv2.weight 0.7641262478298612
conv2.bias 0.71875
fc1.weight 0.06729390669842156
fc1.bias 0.025390625
fc2.weight 0.7837890625
fc2.bias 0.9
Total: 0.08358673475128647
0.47735363924050633
-----------------------------------
Layer sparse
conv1.weight 0.9375
conv1.bias 0.890625
conv2.weight 0.5333726671006944
conv2.bias 0.4765625
fc1.weight 0.0015011222995057398
fc1.bias 0.0
fc2.weight 0.5765625
fc2.bias 0.7
Total: 0.01398429139292775
0.09513449367088607
-----------------------------------
Layer sparse
conv1.weight 0.9045138888888888
conv1.bias 0.890625
conv2.weight 0.3156263563368056
conv2.bias 0.2578125
fc1.weight 1.5414490991709182e-05
fc1.bias 0.0
fc2.weight 0.371875
fc2.bias 0.4
Total: 0.007479941525322959
0.09612341772151899
-----------------------------------
Layer sparse
conv1.weight 0.8732638888888888
conv1.bias 0.875
conv2.weight 0.13545735677083334
conv2.bias 0.0546875
fc1.weight 0.0
fc1.bias 0.0
fc2.weight 0.1615234375
fc2.bias 0.1
Total: 0.003250198205069488
0.09691455696202532
-----------------------------------
Layer sparse
conv1.weight 0.8402777777777778
conv1.bias 0.84375
conv2.weight 0.03839111328125
conv2.bias 0.00390625
fc1.weight 0.0
fc1.bias 0.0
fc2.weight 0.016796875
fc2.bias 0.0
Total: 0.0009558243703890901
0.1003757911392405
-----------------------------------
Layer sparse
conv1.weight 0.8142361111111112
conv1.bias 0.796875
conv2.weight 0.0084228515625
conv2.bias 0.0
fc1.weight 0.0
fc1.bias 0.0
fc2.weight 0.0
fc2.bias 0.0
Total: 0.00026792277133719006
0.09760680379746836
-----------------------------------
Layer sparse
conv1.weight 0.7760416666666666
conv1.bias 0.765625
conv2.weight 0.0014580620659722222
conv2.bias 0.0
fc1.weight 0.0
fc1.bias 0.0
fc2.weight 0.0
fc2.bias 0.0
Total: 0.00010811185608441666
0.09760680379746836
-----------------------------------
Layer sparse
conv1.weight 0.7447916666666666
conv1.bias 0.734375
conv2.weight 0.00014241536458333334
conv2.bias 0.0
fc1.weight 0.0
fc1.bias 0.0
fc2.weight 0.0
fc2.bias 0.0
Total: 7.55718600196274e-05
0.09968354430379747
-----------------------------------
Layer sparse
conv1.weight 0.7065972222222222
conv1.bias 0.71875
conv2.weight 0.0
conv2.bias 0.0
fc1.weight 0.0
fc1.bias 0.0
fc2.weight 0.0
fc2.bias 0.0
Total: 6.888139353901653e-05
0.09760680379746836
-----------------------------------
Layer sparse
conv1.weight 0.671875
conv1.bias 0.6875
conv2.weight 0.0
conv2.bias 0.0
fc1.weight 0.0
fc1.bias 0.0
fc2.weight 0.0
fc2.bias 0.0
Total: 6.553616029871108e-05```
```import matplotlib.pyplot as plt
plt.figure(figsize=(10,3))
plt.subplot(131)
plt.plot(threshold_list,acc_list)
plt.subplot(132)
plt.plot(threshold_list,acc_list)
plt.subplot(133)
plt.plot(sparse_list,acc_list)
plt.show()```

output_30_0.png

## 剪枝后微调

```model = net().cuda()
model1 = puring(model,0.02)
print_sparse(model1)```
```0.4759691455696203
-----------------------------------
Layer sparse
conv1.weight 0.9375
conv1.bias 0.890625
conv2.weight 0.5333726671006944
conv2.bias 0.4765625
fc1.weight 0.0015011222995057398
fc1.bias 0.0
fc2.weight 0.5765625
fc2.bias 0.7
Total: 0.01398429139292775```
```optimizer = pt.optim.Adam(model1.parameters(),1e-5)
lossfunc = pt.nn.CrossEntropyLoss().cuda()
for _ in range(4):
for i,(data,label) in enumerate(trainloader):
data,label = data.cuda(),label.cuda()
outputs = model1(data)
loss = lossfunc(outputs,label)
loss.backward()
optimizer.step()
if i % 100 == 0:
print(i,acc(outputs,label))```
```0 0.4375
100 0.4375
200 0.5625
300 0.6015625
400 0.6875
0 0.7265625
100 0.6953125
200 0.7890625
300 0.8046875
400 0.7734375
0 0.8125
100 0.8046875
200 0.890625
300 0.8515625
400 0.875
0 0.859375
100 0.8515625
200 0.9140625
300 0.890625
400 0.9296875```
```test_model(model1,testloader)
print_sparse(model1)
pt.save(model1.state_dict(),'./puring.pt')```
```0.9367088607594937
-----------------------------------
Layer sparse
conv1.weight 0.9375
conv1.bias 0.890625
conv2.weight 0.5333726671006944
conv2.bias 0.4765625
fc1.weight 0.0015011222995057398
fc1.bias 0.0
fc2.weight 0.5765625
fc2.bias 0.7
Total: 0.01398429139292775```

# 量化实验

## 量化

```model = net().cuda()
`0.9367088607594937`
```from sklearn.cluster import KMeans
import numpy as np
kmean_list = []
bit = 2
for name,i in model.named_parameters():
data = i.data.clone().view(-1).cpu().detach().numpy().reshape(-1)
data = data[data != 0]
if data.size < 2 ** bit:
kmean_list.append(None)
continue
init = [x*(np.max(data)+np.min(data))/(2 ** bit) + np.min(data) for x in range(2 ** bit)]
kmn = KMeans(2 ** bit,init=np.array(init).reshape(2 ** bit,1))
kmn.fit(data.reshape((-1,1)))
kmean_list.append(kmn)
print(name,i.shape)```
```conv1.weight torch.Size([64, 1, 3, 3])
conv1.bias torch.Size([64])
conv2.weight torch.Size([256, 64, 3, 3])
conv2.bias torch.Size([256])
fc1.weight torch.Size([512, 12544])
fc2.weight torch.Size([10, 512])
fc2.bias torch.Size([10])

c:\program files\python35\lib\site-packages\sklearn\cluster\k_means_.py:896: RuntimeWarning: Explicit initial center position passed: performing only one init in k-means instead of n_init=10
return_n_iter=True)```

```for i,(name,f) in enumerate(model.named_parameters()):
data = f.data.clone().view(-1).cpu().detach().numpy().reshape(-1)
data_nozero = data[data != 0].reshape((-1,1))
if data_nozero.size == 0 or data.size < 2 ** bit or kmean_list[i] is None:
f.kmeans_result = None
f.kmeans_label = None
continue
#     print(name)
#     print(data.size)

result = data.copy()
result[result == 0] = -1

#     print(data_nozero)
#     print(kmean_list[i])
label = kmean_list[i].predict(data_nozero).reshape(-1)
#     print(data_nozero)
#     print(label)
new_data = np.array([kmean_list[i].cluster_centers_[x] for x in label])
data[data != 0] = new_data.reshape(-1)
#     print(data,new_data)
f.data = pt.from_numpy(data).view(f.data.shape).cuda()
result[result != -1] = label
f.kmeans_result = pt.from_numpy(result).view(f.data.shape).cuda()
f.kmeans_label = pt.from_numpy(kmean_list[i].cluster_centers_).cuda()```
```test_model(model,testloader)
print_sparse(model)```
```0.8919106012658228
-----------------------------------
Layer sparse
conv1.weight 0.9375
conv1.bias 0.890625
conv2.weight 0.5333726671006944
conv2.bias 0.4765625
fc1.weight 0.0015011222995057398
fc1.bias 0.0
fc2.weight 0.5765625
fc2.bias 0.7
Total: 0.01398429139292775

0.01398429139292775```

## 微调

```lossfunc = pt.nn.CrossEntropyLoss().cuda()
lr = 0.001
for _ in range(1):
for a,(data,label) in enumerate(trainloader):
data,label = data.cuda(),label.cuda()
outputs = model(data)
loss = lossfunc(outputs,label)
loss.backward()

for name,i in model.named_parameters():
#             print(i.data)
#             break
if i.kmeans_result is None:
continue
for x in range(2 ** bit):
i.kmeans_label[x] += -lr * grad.item()
i.data[i.kmeans_result == x] = i.kmeans_label[x].item()
#                 print(i.data)
#                 break
#             print(name)
#             break
if a % 100 == 0:
print(a,acc(outputs,label))
#         break
#     break```
```0 0.8828125
100 0.921875
200 0.9296875
300 0.9296875
400 0.9140625```
```test_model(model,testloader)
print_sparse(model)
pt.save(model.state_dict(),"quantization.pt")```
```0.9384889240506329
-----------------------------------
Layer sparse
conv1.weight 0.9375
conv1.bias 0.890625
conv2.weight 0.5333726671006944
conv2.bias 0.4765625
fc1.weight 0.0015011222995057398
fc1.bias 0.0
fc2.weight 0.5765625
fc2.bias 0.7
Total: 0.01398429139292775```

98 篇文章32 人订阅

0 条评论

4212

4154

3786

961

961

1011

2023

1052

4027

2.1K8