1.FCN网络

1.1 核心思想

  • 不含全连接层的全卷积网络，可适应任意尺寸输入；（可以为不同大小和分辨率的图像生成像素级别的预测）
  • 反卷积层增大图像尺寸，输出精细结果；
  • 结合不同深度层结果的跳级结构，确保鲁棒性和精确性。

1.2 网络结构

1

注：

1.全卷积部分为一些经典的CNN网络（如VGG，ResNet等），用于提取特征。

2.反卷积部分则是通过上采样得到原尺寸的语义分割图像。

3.FCN的输入可以为任意尺寸的彩色图像，输出与输入尺寸相同，通道数为n（目标类别数）+1（背景）。

2.实例

2.1 创建一个全卷积网络

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
%matplotlib inline
import torch
import torchvision
from torch import nn
from torch.nn import functional as F
from d2l import torch as d2l

#加载预训练的ResNet-18模型来提取图像特征，并查看该模型的最后三个子模块的结构和参数
pretrained_net = torchvision.models.resnet18(pretrained=True)
list(pretrained_net.children())[-3:]

out:
[Sequential(
   (0): BasicBlock(
     (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
     (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
     (relu): ReLU(inplace=True)
     (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
     (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
     (downsample): Sequential(
       (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)
       (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
     )
   )
   (1): BasicBlock(
     (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
     (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
     (relu): ReLU(inplace=True)
     (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
     (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
   )
 ),
 AdaptiveAvgPool2d(output_size=(1, 1)),
 Linear(in_features=512, out_features=1000, bias=True)]
1
2
3
4
5
6
7
8
#利用ResNet-18模型创建一个全卷积网络实例net(去除ResNet-18模型的池化层和全连接层)
net = nn.Sequential(*list(pretrained_net.children())[:-2])

X = torch.rand(size=(1, 3, 320, 480))
net(X).shape #对张量X进行前向传播

out:
torch.Size([1, 512, 10, 15])

2.2 添加1x1的卷积层和转置卷积层

1
2
3
4
5
6
7
8
9
10
11
#向网络net中添加final_conv和transpose_conv两个层，特征图的分辨率还原回输入图像的大小
num_classes = 21
net.add_module('final_conv', nn.Conv2d(512, num_classes, kernel_size=1))
net.add_module(
    'transpose_conv',
    nn.ConvTranspose2d(num_classes, num_classes, kernel_size=64, padding=16,
                       stride=32))#kernel_size取stride的两倍，padding取kernel_size的1/4
net(X).shape

out:
torch.Size([1, 21, 320, 480])

2.3 初始化卷积核

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
#初始化转置卷积层
def bilinear_kernel(in_channels, out_channels, kernel_size):
    #找到卷积核的中心位置
    factor = (kernel_size + 1) // 2    
    if kernel_size % 2 == 1:
        center = factor - 1
    else:
        center = factor - 0.5
    og = (torch.arange(kernel_size).reshape(-1, 1),
          torch.arange(kernel_size).reshape(1, -1))
    #权重矩阵 filt,通过将偏移与 factor 相除并从1中减去得到的,确保了中心位置附近的权重最大
    filt = (1 - torch.abs(og[0] - center) / factor) *\
           (1 - torch.abs(og[1] - center) / factor)
    weight = torch.zeros(
        (in_channels, out_channels, kernel_size, kernel_size))
    weight[range(in_channels), range(out_channels), :, :] = filt
    return weight

2.4 双线性插值的上采样的实验

1
2
3
4
5
6
7
8
9
10
11
12
13
conv_trans = nn.ConvTranspose2d(3, 3, kernel_size=4, padding=1, stride=2,
                                bias=False)#kernel_size取stride的两倍，padding取kernel_size的1/4
conv_trans.weight.data.copy_(bilinear_kernel(3, 3, 4));

img = torchvision.transforms.ToTensor()(d2l.Image.open('../data/cat.jpg'))#加载一张图像并将其转换为 PyTorch 张量格式。
X = img.unsqueeze(0)#unsqueeze(0)的作用是在索引0的位置插入一个新的维度，将原始的三维图像张量变成了四维
Y = conv_trans(X)#对X进行转置卷积
out_img = Y[0].permute(1, 2, 0).detach()#删除一个维度，并将其他的三个维度调整为高度（Height）、宽度（Width）和通道数（Channels）的顺序
d2l.set_figsize()
print('input image shape:', img.permute(1, 2, 0).shape)
d2l.plt.imshow(img.permute(1, 2, 0))
print('output image shape:', out_img.shape)
d2l.plt.imshow(out_img);

2

2.5 初始化转置卷积层

1
2
W = bilinear_kernel(num_classes, num_classes, 64)
net.transpose_conv.weight.data.copy_(W);

2.6 读取数据集

1
2
batch_size, crop_size = 32, (320, 480)
train_iter, test_iter = d2l.load_data_voc(batch_size, crop_size)

2.7 训练

1
2
3
4
5
6
def loss(inputs, targets):
    return F.cross_entropy(inputs, targets, reduction='none').mean(1).mean(1)

num_epochs, lr, wd, devices = 5, 0.001, 1e-3, d2l.try_all_gpus()
trainer = torch.optim.SGD(net.parameters(), lr=lr, weight_decay=wd)
d2l.train_ch13(net, train_iter, test_iter, loss, trainer, num_epochs, devices)

3

2.8 可视化预测的类别

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
def predict(img):
    X = test_iter.dataset.normalize_image(img).unsqueeze(0)
    pred = net(X.to(devices[0])).argmax(dim=1)
    return pred.reshape(pred.shape[1], pred.shape[2])
    
def label2image(pred):
    colormap = torch.tensor(d2l.VOC_COLORMAP, device=devices[0])
    X = pred.long()
    return colormap[X, :]

voc_dir = d2l.download_extract('voc2012', 'VOCdevkit/VOC2012')
test_images, test_labels = d2l.read_voc_images(voc_dir, False)
n, imgs = 4, []
for i in range(n):
    crop_rect = (0, 0, 320, 480)
    X = torchvision.transforms.functional.crop(test_images[i], *crop_rect)
    pred = label2image(predict(X))
    imgs += [
        X.permute(1, 2, 0),
        pred.cpu(),
        torchvision.transforms.functional.crop(test_labels[i],
                                               *crop_rect).permute(1, 2, 0)]
d2l.show_images(imgs[::3] + imgs[1::3] + imgs[2::3], 3, n, scale=2);

4