opencv undistortPoints doesn't undistort

Question

I have a camera with a fish-eye lens. I successfully undistort my image, but I want to undistort point coordinates. I know the maps contain src image coordinates for each pixel in dst so I cannot directly use those without some iterative algoritm. I thought undistortPoints() would do that, but it incorrectly transforms the first point and it doesn't change the second point. Using undistortPointsIter() and setting the criteria higher doesn't work either.

A similar question was asked before and this answer doesn't work for me either. I get the exact same results with undistortPoints().

So how do I get the pixel points for my undistorted image from pixel points in the distorted image?

My camera parameters:

print(K)
print(D)
print(Dims)

[[338.37324094   0.         319.5       ]
 [  0.         339.059099   239.5       ]
 [  0.           0.           1.        ]]
[[ 0.01794191]
 [-0.12190366]
 [ 0.14111533]
 [-0.09602948]]
(640, 480)

My code:

img = cv2.imread('Chessboards\img_021.jpg')
img_undistorted = cv2.remap(img, unfishmap1, unfishmap2, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT) 

points1=[]
points1.append((165,260))
points1.append((175,410))

print(points1)

img2= img.copy()
for p in points1:
    cv2.circle(img2,p,6,(0,0,255),2)
imgshow(img2)

point_matrix = np.zeros(shape=(len(points1),1,2),dtype=np.float32)

for i in range(0, len(points1)):
    point_matrix[i][0][0] = points1[i][0]
    point_matrix[i][0][1] = points1[i][1]
    
print(point_matrix)
    
points_undistorted = cv2.undistortPoints(point_matrix,K,D,R=None,P=K)
points2=[]
for p in points_undistorted:
    points2.append( (int(p[0][0]),int(p[0][1])) )
  
print(points2)

img2= img_undistorted.copy()
for p in points2:
    cv2.circle(img2,p,6,(0,0,255),2)
imgshow(img2)

#expected
points3=[]
points3.append((155,265))
points3.append((150,443))

print(points3)

img2= img_undistorted.copy()
for p in points3:
    cv2.circle(img2,p,6,(0,0,255),2)
imgshow(img2)

Result:

distorted

undistorted with undistortPoints

undistorted with expected points

Answer 1

It worked thanks to solution provided by micka.

This is what worked for me:

source image

import matplotlib.pyplot as plt
plt.rcParams['figure.dpi'] = 150
import cv2
import copy
import os
import numpy as np

def imgshow(img):
    if len(img.shape) == 3:
        plt.imshow(img[:, :, ::-1])
    else:
        plt.imshow(img,cmap='gray',vmin=0, vmax=255)
    plt.show() 


K = np.array( [[338.37324094,0,319.5],[0,339.059099,239.5],[0,0,1]],dtype=np.float64)
D = np.array( [[ 0.01794191], [-0.12190366],[ 0.14111533],[-0.09602948]],dtype=np.float64)
new_size = (640, 480)

Knew = K.copy()
# alpha = 0.6
# Knew, roi = cv2.getOptimalNewCameraMatrix(K, D, new_size, alpha, new_size,centerPrincipalPoint = True)
    
unfishmap1, unfishmap2 = cv2.fisheye.initUndistortRectifyMap(K, D, np.eye(3), Knew, new_size, cv2.CV_32F)
unfishmap1, unfishmap2 = cv2.convertMaps(unfishmap1, unfishmap2, cv2.CV_16SC2)

img = cv2.imread('3FYUT.jpg')
img_undistorted = cv2.remap(img, unfishmap1, unfishmap2, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT) 

points1=[]
points1.append((165,260))
points1.append((175,410))

print(points1)

img2= img.copy()
for p in points1:
    cv2.circle(img2,p,6,(0,0,255),2)
imgshow(img2)

point_matrix = np.zeros(shape=(len(points1),1,2),dtype=np.float32)

for i in range(0, len(points1)):
    point_matrix[i][0][0] = points1[i][0]
    point_matrix[i][0][1] = points1[i][1]
        
points_undistorted = cv2.fisheye.undistortPoints(point_matrix,K,D,P=Knew)

points2=[]
for p in points_undistorted:
    points2.append( (int(p[0][0]),int(p[0][1])) )

print("fisheye.undistortPoints:")    
print(points2)

img2= img_undistorted.copy()
for p in points2:
    cv2.circle(img2,p,6,(0,0,255),2)
imgshow(img2)

print("expected:")
points3=[]
points3.append((155,265))
points3.append((150,443))

print(points3)

img2= img_undistorted.copy()
for p in points3:
    cv2.circle(img2,p,6,(0,0,255),2)
imgshow(img2)

Result:

fisheye.undistortPoints:
[(152, 261), (147, 441)]
expected:
[(155, 265), (150, 443)]