神经网络已经在很多场景下表现出了很好的识别能力，但是缺乏解释性一直所为人诟病。《Grad-CAM:Visual Explanations from Deep Networks via Gradient-based Localization》这篇论文基于梯度为其可解释性做了一些工作，它可以显著描述哪块图片区域对识别起了至关重要的作用，以热度图的方式可视化神经网络的注意力。本博客主要是基于pytorch的简单工程复现。原文见这里，本代码基于这里。

1 import torch
2 import torchvision
3 from torchvision import models
4 from torchvision import transforms
5 from PIL import Image
6 import pylab as plt
7 import numpy as np
8 import cv2
9
10
11 class Extractor():
12 """
13 pytorch在设计时，中间层的梯度完成回传后就释放了
14 这里用hook工具在保存中间参数的梯度
15 """
16 def __init__(self, model, target_layer):
17 self.model = model
18 self.target_layer = target_layer
19 self.gradient = None
20
21 def save_gradient(self, grad):
22 self.gradient=grad
23
24 def __call__(self, x):
25 outputs = []
26 self.gradients = []
27 for name,module in self.model.features._modules.items():
28 x = module(x)
29 if name == self.target_layer:
30 x.register_hook(self.save_gradient)
31 target_activation=x
32 x=x.view(1,-1)
33 for name,module in self.model.classifier._modules.items():
34 x = module(x)
35 # 维度为（1，c, h, w） , (1,class_num)
36 return target_activation, x
37
38
39 def preprocess_image(path):
40 means=[0.485, 0.456, 0.406]
41 stds=[0.229, 0.224, 0.225]
42 m_transform=transforms.Compose([transforms.ToTensor(),transforms.Normalize(means,stds)])
43 img=Image.open(path)
44 return m_transform(img).reshape(1,3,224,224)
45
46
47 class GradCam():
48 def __init__(self, model, target_layer_name, use_cuda):
49 self.model = model
50 self.model.eval()
51 self.cuda = use_cuda
52 if self.cuda:
53 self.model = model.cuda()
54
55 self.extractor = Extractor(self.model, target_layer_name)
56
57
58 def __call__(self, input, index = None):
59 if self.cuda:
60 target_activation, output = self.extractor(input.cuda())
61 else:
62 target_activation, output = self.extractor(input)
63
64 # index是想要查看的类别，未指定时选择网络做出的预测类
65 if index == None:
66 index = np.argmax(output.cpu().data.numpy())
67
68 # batch维为1（我们默认输入的是单张图）
69 one_hot = np.zeros((1, output.size()[-1]), dtype = np.float32)
70 one_hot[0][index] = 1.0
71 one_hot = torch.tensor(one_hot)
72 if self.cuda:
73 one_hot = torch.sum(one_hot.cuda() * output)
74 else:
75 one_hot = torch.sum(one_hot * output)
76
77 self.model.zero_grad()
78 one_hot.backward(retain_graph=True)
79
80 grads_val = self.extractor.gradient.cpu().data.numpy()
81 # 维度为（c, h, w）
82 target = target_activation.cpu().data.numpy()[0]
83 # 维度为（c,）
84 weights = np.mean(grads_val, axis = (2, 3))[0, :]
85 # cam要与target一样大
86 cam = np.zeros(target.shape[1 : ], dtype = np.float32)
87 for i, w in enumerate(weights):
88 cam += w * target[i, :, :]
89
90 # 每个位置选择c个通道上最大的最为输出
91 cam = np.maximum(cam, 0)
92 cam = cv2.resize(cam, (224, 224))
93 cam = cam - np.min(cam)
94 cam = cam / np.max(cam)
95 return cam
96
97
98 def show_cam_on_image(img, mask):
99 heatmap = cv2.applyColorMap(np.uint8(255*mask), cv2.COLORMAP_JET)
100 heatmap = np.float32(heatmap) / 255
101 cam = heatmap + np.float32(img)
102 cam = cam / np.max(cam)
103 cv2.imwrite("cam2.jpg", np.uint8(255 * cam))
104
105
106 #target_layer 越靠近分类层效果越好
107 grad_cam = GradCam(model = models.vgg19(pretrained=True), target_layer_name = "35", use_cuda=True)
108 input = preprocess_image("both.png")
109 mask = grad_cam(input, None)
110 img = cv2.imread("both.png", 1)
111 #热度图是直接resize加到输入图上的
112 img = np.float32(cv2.resize(img, (224, 224))) / 255
113 show_cam_on_image(img, mask)

原图：

可视化图：