Callback gradient norm from scipy.minimise (python)

Question

I am using scipy.minimize with the 'CG' method and I want to callback the gradient norm at each iteration. So far, I have been able to call back the function at each iteration using this:

def min_method(fn, grad, x0):
    all_fn = [fn(x0).item()]
    def store(x): # callback function
    all_fn.append(fn(x).item())

    ans = minimize(fn, x0, method='CG', jac=grad, callback=store,
           options={'disp':True,'gtol': 1e-06})
    return ans, all_fn

How can I add a line to the store() function in order to get the gradient norm at each iteration?

Answer 1

You could use the approx_fprime function. It could be something like:

import numpy as np
from scipy.optimize import minimize
from scipy.optimize.optimize import approx_fprime


def f_(x):   # The rosenbrock function
    return (1 - x[0])**2 + 100 * (x[1] - x[0]**2)**2



def min_method(fn, grad, x0):
    all_fn = [fn(x0).item(),0]

    def store(x, *_):  # callback function
        aprox = approx_fprime(x, fn, 1E-8)
        all_fn.append([fn(x).item(),aprox])


    ans = minimize(fn, x0, method='CG', jac=grad, callback=store,
                   options={'disp': True, 'gtol': 1e-06})
    return ans, all_fn


x0 = np.array([2, 3], dtype=np.double)
ans, all_fn =min_method(f_, '2-point',x0)
print(ans)
print(all_fn)

##Update

If you want the jacobian from the 'CG', I thing you need to modify the function that is called through the scipy.optimize.minimize.

in _minimize.py go to the function minimize

def minimize(fun, x0, args=(), method=None, jac=None, hess=None,
             hessp=None, bounds=None, constraints=(), tol=None,
             callback=None, options=None):

then find the call to the 'CG' methode line 673 _minimize_cg

elif meth == 'cg':
    res = _minimize_cg(fun, x0, args, jac, callback, **options)

in that function in the optimize.py around line 1681 replace

    if callback is not None:
        callback(xk)

with

    if callback is not None: 
        callback(xk,gfk,gnorm)

then you can have access to the jacobian in the callback function

import numpy as np
from scipy.optimize import minimize

def f_(x):   # The rosenbrock function
    return (1 - x[0])**2 + 100 * (x[1] - x[0]**2)**2


def min_method(fn, grad, x0):
    all_fn = [[fn(x0).item(),0]]

    def store(x, gfk,gnorm):  # callback function
        all_fn.append([fn(x).item(),gfk,gnorm])

    ans = minimize(fn, x0, method='CG', jac=grad, callback=store,
                   options={'disp': True, 'gtol': 1e-06})
    return ans, all_fn


x0 = np.array([2, 3], dtype=np.double)
ans, all_fn =min_method(f_, '2-point',x0)
print(ans)
print(all_fn)