Affine Stitching Image As Step By Step

Question

#include "header.h"
#include "opencv2/imgproc.hpp"
#include "opencv2/imgcodecs.hpp"
#include "opencv2/highgui.hpp"
#include "opencv2/features2d.hpp"
#include "opencv2/stitching/detail/autocalib.hpp"
#include "opencv2/stitching/detail/blenders.hpp"
#include "opencv2/stitching/detail/timelapsers.hpp"
#include "opencv2/stitching/detail/exposure_compensate.hpp"
#include "opencv2/stitching/detail/matchers.hpp"
#include "opencv2/stitching/detail/motion_estimators.hpp"
#include "opencv2/stitching/detail/seam_finders.hpp"
#include "opencv2/stitching/detail/warpers.hpp"
#include "opencv2/stitching/warpers.hpp"
#include <iostream>

using namespace std;
using namespace cv;
using namespace cv::detail;


void TestAffineStitching()
{
    Mat l_image = imread("./images/input/leftImage.jpg", IMREAD_COLOR);
    Mat r_image = imread("./images/input/rightImage.jpg", IMREAD_COLOR);
    
    // Check if images are loaded successfully
    if (l_image.empty() || r_image.empty()) {
        cerr << "Error loading images." << endl;
        exit(1);
    }

    /* Finding features */
    vector<ImageFeatures> features;
    Ptr<Feature2D> finder = ORB::create();;
    vector<Mat> images = { l_image , r_image };
    features.clear();
    for (int i = 0; i < 2; i++)
    {
        ImageFeatures temp;
        computeImageFeatures(finder, images[i], temp, noArray());
        temp.img_idx = i;
        features.push_back(temp);
    }

    /* Match features */
    bool try_use_gpu = false;  //Should try to use GPU or not
    vector<MatchesInfo> pairwise_matches;
    float match_conf = 0.3f;   //Match distances ratio threshold
    Ptr<FeaturesMatcher> matcher;
    matcher = makePtr<AffineBestOf2NearestMatcher>(try_use_gpu, match_conf);
    (*matcher)(features, pairwise_matches);
    matcher->collectGarbage();

    /* Initial cameras */
    vector<CameraParams> cameras;
    Ptr<Estimator> estimator = makePtr<AffineBasedEstimator>();
    if (!(*estimator)(features, pairwise_matches, cameras))
    {
        cout << "Affine estimation failed." << endl;
        exit(2);
    }
    for (size_t i = 0; i < cameras.size(); ++i)
    {
        Mat R;
        cameras[i].R.convertTo(R, CV_32F);
        cameras[i].R = R;
        cout << "Initial camera intrinsics #" << (i + 1) << ":\nK:\n" << cameras[i].K() << "\nR:\n" << cameras[i].R << endl;
    }


    Ptr<BundleAdjusterBase> adjuster = makePtr<BundleAdjusterAffinePartial>();;
    float conf_thresh = 1.f;
    string ba_refine_mask = "xxxxx";
    adjuster->setConfThresh(conf_thresh);
    Mat_<uchar> refine_mask = Mat::zeros(3, 3, CV_8U);
    if (ba_refine_mask[0] == 'x') refine_mask(0, 0) = 1;
    if (ba_refine_mask[1] == 'x') refine_mask(0, 1) = 1;
    if (ba_refine_mask[2] == 'x') refine_mask(0, 2) = 1;
    if (ba_refine_mask[3] == 'x') refine_mask(1, 1) = 1;
    if (ba_refine_mask[4] == 'x') refine_mask(1, 2) = 1;
    adjuster->setRefinementMask(refine_mask);
    if (!(*adjuster)(features, pairwise_matches, cameras))
    {
        cout << "Camera parameters adjusting failed." << endl;
        exit(3);
    }

    vector<double> focals;
    float warped_image_scale;
    for (size_t i = 0; i < cameras.size(); i++)
    {
        cout << "Camera #" << i + 1 << ":\nK:\n" << cameras[i].K() << "\nR:\n" << cameras[i].R << endl;
        focals.push_back(cameras[i].focal);
    }
    sort(focals.begin(), focals.end());

    if (focals.size() % 2 == 1)
    {
        warped_image_scale = static_cast<float>(focals[focals.size() / 2]);
    }
    else
    {
        warped_image_scale = static_cast<float>(focals[focals.size() / 2 - 1] + focals[focals.size() / 2]) * 0.5f;
    }

    /* Do wave correction for cameras */
    vector<Mat> rmats;
    for (size_t i = 0; i < cameras.size(); ++i)
    {
        rmats.push_back(cameras[i].R.clone());
    }
    waveCorrect(rmats, WAVE_CORRECT_HORIZ);
    for (size_t i = 0; i < cameras.size(); ++i)
    {
        cameras[i].R = rmats[i];
    }
    
    /* Warping images */
    Ptr<WarperCreator> warper_creator = makePtr<cv::AffineWarper>();;
    if (!warper_creator)
    {
        cout << "Can't create the affine warper" << endl;
        exit(4);
    }
    Ptr<RotationWarper> warper = warper_creator->create(warped_image_scale);
    vector<Point> corners;
    vector<UMat> masks_warped;
    vector<UMat> images_warped;
    vector<UMat> images_warped_f;
    vector<Size> sizes;
    corners.clear();
    masks_warped.clear();
    images_warped.clear();
    sizes.clear();
    for (int i = 0; i < 2; i++)
    {
        Point corner;
        UMat mask_warped;
        UMat image_warped;
        UMat mask;
        Mat_<float> K;
        cameras[i].K().convertTo(K, CV_32F);

        corner = warper->warp(images[i], K, cameras[i].R, INTER_LINEAR, BORDER_REFLECT, image_warped);
        corners.push_back(corner);
        images_warped.push_back(image_warped);
        sizes.push_back(image_warped.size());

        mask.create(images[i].size(), CV_8U);
        mask.setTo(Scalar::all(255));
        warper->warp(mask, K, cameras[i].R, INTER_NEAREST, BORDER_CONSTANT, mask_warped);
        masks_warped.push_back(mask);

        string name;
        name = "./images/debug/images_warped" + to_string(i) + ".jpg";
        imwrite(name, image_warped);
        name = "./images/debug/masks_warped" + to_string(i) + ".jpg";
        imwrite(name, mask_warped);
    }
}

I am facing problems at "if (!(*adjuster)(features, pairwise_matches, cameras))" and it jump to

    @param features Features of images
    @param pairwise_matches Pairwise matches of images
    @param cameras Estimated camera parameters
    @return True in case of success, false otherwise
     */
    CV_WRAP_AS(apply) bool operator ()(const std::vector<ImageFeatures> &features,
        const std::vector<MatchesInfo> &pairwise_matches,
        CV_OUT CV_IN_OUT std::vector<CameraParams> &cameras)
    {
        return estimate(features, pairwise_matches, cameras);
    }

If I comment the block code:

        Ptr<BundleAdjusterBase> adjuster = makePtr<BundleAdjusterAffinePartial>();;
    float conf_thresh = 1.f;
    string ba_refine_mask = "xxxxx";
    adjuster->setConfThresh(conf_thresh);
    Mat_<uchar> refine_mask = Mat::zeros(3, 3, CV_8U);
    if (ba_refine_mask[0] == 'x') refine_mask(0, 0) = 1;
    if (ba_refine_mask[1] == 'x') refine_mask(0, 1) = 1;
    if (ba_refine_mask[2] == 'x') refine_mask(0, 2) = 1;
    if (ba_refine_mask[3] == 'x') refine_mask(1, 1) = 1;
    if (ba_refine_mask[4] == 'x') refine_mask(1, 2) = 1;
    adjuster->setRefinementMask(refine_mask);
    if (!(*adjuster)(features, pairwise_matches, cameras))
    {
        cout << "Camera parameters adjusting failed." << endl;
        exit(3);
    }

The output of warping image will be black for all. Could you please help to check and tell me the rootcause of this issue? I am trying to stitch 2 images by affine stitching but I do it step by step as pipeline of openCV, I follow the example of opencv by it is not homography. I change to use affine

Debug code and tell me the rootcause for fixing it.