I get an error with cv2.fisheye.calibrate and cannot calibrate the fisheye camera

Question

I'm trying to calibrate my PIXPRO SP360 fisheye camera with OpenCV. I acquired seven images of the chessboard with the camera and ran the code, but I get an error in the cv2.fisheye.calibrate part. It seems to be able to detect the corners, but I don't know why it can't be calibrated. Chessboard rows = 9, cols = 7

This is my code:

import yaml
import cv2
import numpy as np
import glob

CHECKERBOARD = (7,9)

subpix_criteria = 
(cv2.TERM_CRITERIA_EPS+cv2.TERM_CRITERIA_MAX_ITER, 30, 0.1)
calibration_flags =cv2.fisheye.CALIB_RECOMPUTE_EXTRINSIC+cv2.fisheye.CALIB_CHECK_COND+cv2.fisheye.CALIB_FIX_SKEW

objp = np.zeros((1, CHECKERBOARD[0]*CHECKERBOARD[1], 3), np.float32)
objp[0,:,:2] = np.mgrid[0:CHECKERBOARD[0], 0:CHECKERBOARD[1]].T.reshape(-1, 2)
_img_shape = None
objpoints = [] # 3d point in real world space
imgpoints = [] # 2d points in image plane.

images = glob.glob('*.jpg')

for fname in images:
    img = cv2.imread(fname)
    if _img_shape == None:
        _img_shape = img.shape[:2]
    else:
        assert _img_shape == img.shape[:2], "All images must share the same size."
    gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
    # Find the chess board corners
    ret, corners = cv2.findChessboardCorners(gray, CHECKERBOARD, cv2.CALIB_CB_ADAPTIVE_THRESH+cv2.CALIB_CB_FAST_CHECK+cv2.CALIB_CB_NORMALIZE_IMAGE)
    # If found, add object points, image points (after refining them)
    if ret == True:
        objpoints.append(objp)
        cv2.cornerSubPix(gray,corners,(3,3), (-1,-1),subpix_criteria)
        imgpoints.append(corners)

N_OK = len(objpoints)
K = np.zeros((3, 3))
D = np.zeros((4, 1))
rvecs = [np.zeros((1, 1, 3), dtype=np.float64) for i in range(N_OK)]
tvecs = [np.zeros((1, 1, 3), dtype=np.float64) for i in range(N_OK)]
rms, _, _, _, _ = \
    cv2.fisheye.calibrate(
        objpoints,
        imgpoints,
        gray.shape[::-1],
        K,
        D,
        rvecs,
        tvecs,
        calibration_flags,
        (cv2.TERM_CRITERIA_EPS+cv2.TERM_CRITERIA_MAX_ITER, 30, 1e-6)
)
print("Found " + str(N_OK) + " valid images for calibration")
print("DIM=" + str(_img_shape[::-1]))
print("K=np.array(" + str(K.tolist()) + ")")
print("D=np.array(" + str(D.tolist()) + ")")

This is error message:

error                                     Traceback (most 
recent call last)
<ipython-input-14-cd3c7fa0f1ad> in <module>
     42         tvecs,
     43         calibration_flags,
---> 44 (cv2.TERM_CRITERIA_EPS+cv2.TERM_CRITERIA_MAX_ITER, 30, 1e-6)
     45     )
     46 print("Found " + str(N_OK) + " valid images for calibration")

error: OpenCV(4.4.0) 
C:\Users\appveyor\AppData\Local\Temp\1\pip-req-build- 
52oirelq\opencv\modules\calib3d\src\fisheye.cpp:1449: 
error: (-3:Internal error) CALIB_CHECK_COND - Ill- 
conditioned matrix for input array 4 in function 
'cv::internal::CalibrateExtrinsics'