How to implement CAM without visualize_cam in this code?

问题

I want to make Class activation map, so I have write the code

from keras.datasets import mnist
from keras.layers import Conv2D, Dense, GlobalAveragePooling2D
from keras.models import Model, Input
from keras.utils import to_categorical

(x_train, y_train), (x_test, y_test) = mnist.load_data()

x_train_resized = x_train.reshape((60000, 28, 28, 1))
x_test_resized = x_test.reshape((10000, 28, 28, 1))
y_train_hot_encoded = to_categorical(y_train)
y_test_hot_encoded = to_categorical(y_test)

inputs = Input(shape=(28,28, 1))

x = Conv2D(64, (3,3), activation='relu')(inputs)
x = Conv2D(64, (3,3), activation='relu')(x)
x = GlobalAveragePooling2D()(x)
predictions = Dense(10, activation='softmax')(x)

model = Model(inputs=inputs, outputs=predictions)
model.compile(optimizer='rmsprop',
              loss='categorical_crossentropy',
              metrics=['accuracy'])
model.fit(x_train_resized, y_train_hot_encoded, epochs=30, batch_size=256, shuffle=True, validation_split=0.3)

works fine, so I have imported visualize_cam module

from vis.visualization import visualize_cam
import matplotlib.pyplot as plt
import numpy as np

for i in range(10):
    ind = np.where(y_test == i)[0][0]
    plt.subplot(141)
    plt.imshow(x_test_resized[ind].reshape((28,28)))
    for j,modifier in enumerate([None, 'guided', 'relu']):
        heat_map = visualize_cam(model, 4, y_test[ind], x_test_resized[ind], backprop_modifier=modifier)
        plt.subplot(1,4,j+2)
        plt.imshow(heat_map)

    plt.show()

but the visualize_cam didn`t work well

I tried many times to fix the module but it doesn`t go well (it depends on scipy which version is below 1.3. but )

so I have to implement cam without that module

Is there any solution to replace visualize_cam into other option to implement CAM?

回答1:

Here is a scipy library independent implementation.

from keras.datasets import mnist
from keras.layers import Conv2D, Dense, GlobalAveragePooling2D
from keras.models import Model, Input
from keras.utils import to_categorical

(x_train, y_train), (x_test, y_test) = mnist.load_data()

x_train_resized = x_train.reshape((60000, 28, 28, 1))
x_test_resized = x_test.reshape((10000, 28, 28, 1))
y_train_hot_encoded = to_categorical(y_train)
y_test_hot_encoded = to_categorical(y_test)

inputs = Input(shape=(28,28, 1))

x = Conv2D(64, (3,3), activation='relu')(inputs)
x = Conv2D(64, (3,3), activation='relu', name='final_conv')(x)
x = GlobalAveragePooling2D()(x)
predictions = Dense(10, activation='softmax')(x)

model = Model(inputs=inputs, outputs=predictions)
model.compile(optimizer='rmsprop',
              loss='categorical_crossentropy',
              metrics=['accuracy'])
model.fit(x_train_resized, y_train_hot_encoded, epochs=1, batch_size=256, shuffle=True, validation_split=0.3)

import numpy as np
import cv2
import io
import requests
from PIL import Image
import matplotlib.pyplot as plt
# Using Keras implementation from tensorflow
from tensorflow.python.keras import applications
from tensorflow.python.keras.preprocessing.image import load_img, img_to_array 
from tensorflow.python.keras import backend as K


# Get the layer of the last conv layer
fianlconv = model.get_layer('final_conv')
# Get the weights matrix of the last layer
weight_softmax = model.layers[-1].get_weights()[0]

# Function to generate Class Activation Mapping
HEIGHT = 28
WIDTH = 28

def returnCAM(feature_conv, weight_softmax, class_idx):
    size_upsample = (WIDTH, HEIGHT)

    # Keras default is channels last, hence nc is in last
    bz, h, w, nc = feature_conv.shape

    output_cam = []
    for idx in class_idx:
        cam = np.dot(weight_softmax[:, idx], np.transpose(feature_conv.reshape(h*w, nc)))
        cam = cam.reshape(h, w)
        cam = cam - np.min(cam)
        cam_img = cam / np.max(cam)
        cam_img = np.uint8(255 * cam_img)

        output_cam.append(cv2.resize(cam_img, size_upsample))

    return output_cam

x = x_test_resized[0,:,:,0]

plt.imshow(x)
plt.show()

classes = {1:'1', 2: '2', 3: '3', 4:'4', 5:'5', 6:'6', 7:'7', 8:'8', 9:'9', 0:'0'}

probs_extractor = K.function([model.input], [model.output])
features_conv_extractor = K.function([model.input], [fianlconv.output])

probs = probs_extractor([np.expand_dims(x, 0).reshape(1,28,28,1)])[0]

features_blob = features_conv_extractor([np.expand_dims(x, 0).reshape(1,28,28,1)])[0]

features_blobs = []
features_blobs.append(features_blob)

idx = np.argsort(probs)
probs = np.sort(probs)

for i in range(-1, -6, -1):
    print('{:.3f} -> {}'.format(probs[0, i], classes[idx[0, i]]))


CAMs = returnCAM(features_blobs[0], weight_softmax, [idx[0, -1]])

heatmap = cv2.applyColorMap(cv2.resize(CAMs[0], (28, 28)), cv2.COLORMAP_JET)

result = heatmap[:,:,0] * 0.3 + x * 0.5

print(result.shape)

plt.imshow(result)
plt.show()

N.B: I'm plotting normalized images so the result isn't great, I also trained only for 1 epoch. For better results, you may try training more, change to appropriate color space.

来源：https://stackoverflow.com/questions/61336309/how-to-implement-cam-without-visualize-cam-in-this-code