Why is tape.gradient returning None?

Question

I am trying to create an adversarial example using this function:

def create_adversarial_pattern(input_image, input_label):
    input_image = tf.cast(input_image, tf.float32)
    
    with tf.GradientTape() as tape:
        tape.watch(input_image)
        predcition = model(input_image)
        loss = loss_object(input_label, prediction)
        
    gradient = tape.gradient(loss, input_image) # <------- this line
    signedgrad = tf.sign(gradient)
    return signed_grad

but the gradient variable has a value of None which is obviously not great. GradientTape.gradient is supposed to return something. This is the implementation of the function

#load image
image = load_img(test_dir + "0/1-30226-A-0.wav.png")
print("Image: " + str(type(image)))

arr = img_to_array(image)
print("Array No. 1: " + str(type(arr)) + " : " + str(arr.shape))

arr /= 255.0

#predict image
arr_ = []
arr_.append(arr)
arr_ = np.array(arr_)
print("Array No. 2: " + str(type(arr_)) + " : " + str(arr_.shape))

prediction = model.predict(arr_)
image_label = prediction.tolist().index(max(prediction.tolist()))

image_label_arr = []
image_label_arr.append(image_label)
image_label_arr = np.array(image_label_arr)

image_label_arr = tf.keras.utils.to_categorical(image_label_arr, len(classes))

print(classes[image_label] + " : " + str(image_label))
print(image_label_arr)

#create example
perturbations = create_adversarial_pattern(arr.reshape((1, 128, 216, 3)), image_label_arr).numpy

this is the model I am using

def create_model():
    model = Sequential()
    model.add(Conv2D(32, (3,3), input_shape = array_image.shape))
    model.add(Activation("relu"))
    model.add(MaxPooling2D())

    model.add(Conv2D(32, (3, 3)))
    model.add(Activation("relu"))
    model.add(MaxPooling2D())

    model.add(Conv2D(64, (3, 3)))
    model.add(Activation("relu"))
    model.add(MaxPooling2D())

    model.add(Flatten())
    model.add(Dense(1024))
    model.add(Activation("relu"))
    model.add(Dropout(0.5))
    model.add(Dense(numberOfClass)) #output
    model.add(Activation("softmax"))
    model.compile(loss = "categorical_crossentropy",
             optimizer = "rmsprop",
             metrics = ["accuracy"])
    return model

Answer 1

I've made some fixes to your code (see the comments inside the code). It should work as expected.

import tensorflow as tf
import numpy as np
from tensorflow.keras.utils import img_to_array
from tensorflow import keras
from tensorflow.keras import Sequential
from keras.layers import Conv2D, Dropout, MaxPooling2D, Flatten, Dense, Activation

# using a loss function
loss_fn = tf.keras.losses.CategoricalCrossentropy(from_logits=True)

def create_adversarial_pattern(input_image, input_label):
    input_image = tf.cast(input_image, tf.float32)
    
    with tf.GradientTape() as tape:
        tape.watch(input_image)
        prediction = model(input_image)  # you had a typo here
        loss = loss_fn(input_label, prediction)
          
    gradient = tape.gradient(loss, input_image) # <------- this line
    print("grad {}".format(gradient))
    signed_grad = tf.sign(gradient)  # another typo here 
    return signed_grad


# dummy image
image = np.ones((960, 640, 3))
print("Image: " + str(type(image)))

arr = img_to_array(image)
print("Array No. 1: " + str(type(arr)) + " : " + str(arr.shape))

arr /= 255.0

#predict image
arr_ = []
arr_.append(arr)
arr_ = np.array(arr_)
print("Array No. 2: " + str(type(arr_)) + " : " + str(arr_.shape))

def create_model():
    model = Sequential()
    model.add(Conv2D(32, (3,3), input_shape = arr.shape))
    model.add(Activation("relu"))
    model.add(MaxPooling2D())

    model.add(Conv2D(32, (3, 3)))
    model.add(Activation("relu"))
    model.add(MaxPooling2D())

    model.add(Conv2D(64, (3, 3)))
    model.add(Activation("relu"))
    model.add(MaxPooling2D())

    model.add(Flatten())
    model.add(Dense(100))
    model.add(Activation("relu"))
    model.add(Dropout(0.5))
    model.add(Dense(3)) #output
    model.add(Activation("softmax"))
    model.compile(loss = "categorical_crossentropy",
             optimizer = "rmsprop",
             metrics = ["accuracy"])
    return model


model = create_model()
prediction = model.predict(arr_)
image_label = prediction.tolist().index(max(prediction.tolist()))

classes = ["0","1","2"]   # I used this classes, you should use yours

image_label_arr = []
image_label_arr.append(image_label)
image_label_arr = np.array(image_label_arr)

image_label_arr = tf.keras.utils.to_categorical(image_label_arr, len(classes))

print(classes[image_label] + " : " + str(image_label))
print(image_label_arr)

#create example

# here I've put arr inside a list, because you have loaded only an image
# and arr has shape (width, height, channels), but model is expecting
# (n_samples_in_batch, width, height, channels)
perturbations = create_adversarial_pattern([arr], image_label_arr).numpy

The line print("grad {}".format(gradient)) gives the following output:

grad [[[[-1.5513586e-05  3.8307126e-06 -5.2626692e-05]
   [-7.3845486e-06 -7.5844241e-06 -1.0758981e-05]
   [ 4.7415083e-06  1.5906917e-06 -2.9444196e-05]
   ...
   [ 0.0000000e+00  0.0000000e+00  0.0000000e+00]
   [ 0.0000000e+00  0.0000000e+00  0.0000000e+00]
   [ 0.0000000e+00  0.0000000e+00  0.0000000e+00]]

  [[ 1.4568100e-06 -5.4016291e-06  6.4816072e-06]
   [-2.6765707e-05 -1.3100296e-05 -1.7976803e-05]
   [-5.2032774e-05 -2.8514034e-05  1.9507925e-05]
   ...
   [ 0.0000000e+00  0.0000000e+00  0.0000000e+00]
   [ 0.0000000e+00  0.0000000e+00  0.0000000e+00]
   [ 0.0000000e+00  0.0000000e+00  0.0000000e+00]]

  [[ 2.9544193e-05 -7.5485059e-06 -6.0243669e-06]
   [ 3.0723353e-05  3.2532912e-06 -4.3765580e-05]
   [-1.6096277e-05  3.5667676e-06  1.3662253e-05]
   ...