I get CUFFT_INTERNAL_ERROR when cufftPlanMany

Question

Is there any other reason that CUFFT_INTERNAL_ERROR occurs?

I do cuFFT2D on same size of input and different batch size for every set.
Input array size is 360(rows)x90(cols) and batch size is usually 10(sometimes up to 100).
But I get 'CUFFT_INTERNAL_ERROR' at certain Set(in my case 640..).

There is no particular difference in the input for each set. However, when using the same input data, the above error always occurs in the same set.

I was previously using 12.0, but I thought it was a problem with the Cuda version, so I reinstalled 12.4. However, only the set(640 -> 721) in which the above error occurs has changed, and even if you try again, the error occurs in the set 721.

I stopped at the set where the error occurred in debug mode to check for memory leaks throughout the code.
However, we were able to confirm that there was no continuous increase in memory before stopping as shown in the picture below.

// Environment
RTX3090, Visual Studio 2022, Windows 10, Cuda 12.4v

---

Below is the cufft part of the code I am using.

#define CHECK_CUFFT(cond) check_cufft(cond, __LINE__)
inline void check_cufft(cufftResult err, const int line)
{
    if( CUFFT_SUCCESS != err) {
    fprintf(stderr, "CUFFT error line %d\n %s\nerror %d: %s\nterminating!\n", __LINE__,err, \
                                _cudaGetErrorEnum(err)); \
    cudaDeviceReset(); assert(0); \
}
...

// At Main Code, For Loop below.
int nFFTbatch = 10; //(Random between 1~100)
int nEstCompen[2] = { 360 , 90 };
int nEstCompenNum = 90 * 360;
cufftHandle planPreCompen;
cufftRes = cufftPlanMany(&planPreCompen, 2, nEstCompen, nEstCompen, 1, nEstCompenNum, nEstCompen, 1, nEstCompenNum, CUFFT_C2C, nFFTbatch );
cudaStatus = CHECK_CUFFT(cufftRes); if (cudaStatus != TRUE) return FALSE; // Here gets error 'CUFFT_INTERNAL_ERROR'
cufftRes = cufftExecC2C(planPreCompen, mCompFFT, mCompFFT, CUFFT_FORWARD);
cudaStatus = CHECK_CUFFT(cufftRes); if (cudaStatus != TRUE) return FALSE;
cufftRes = cufftDestroy(planPreCompen);
cudaStatus = CHECK_CUFFT(cufftRes); if (cudaStatus != TRUE) return FALSE;

Answer 1

@Robert Crovella
I found the reason. It was just caused by thrust::sort which placed before the cufft and added most latest to code.
When I delete that sort part and it works just fine....

But it is still curious that I checked error with 'cudaGetLastError()'!!.
I will update this when analyze what the problem is.

#define CHECK_CUFFT(cond) check_cufft(cond, __LINE__)
inline void check_cufft(cufftResult err, const int line)
{
    if( CUFFT_SUCCESS != err) {
    fprintf(stderr, "CUFFT error line %d\n %s\nerror %d: %s\nterminating!\n", __LINE__,err, \
                                _cudaGetErrorEnum(err)); \
    cudaDeviceReset(); assert(0); \
}
struct CompPriority {
    double Mag;
    double Class;
    int Index; // Additional field for indexing
    int OriginIdx;
};

    CompPriority* d_compArray;
    cudaStatus = CHECK_CUDA(cudaMalloc((void**)&d_compArray, nMICD_Num * sizeof(CompPriority)), cu_dir); if (cudaStatus != TRUE) return FALSE;
    // Compen Result Get to Thrust vector
    getCompenPriorInfo << <nbsetPerDetect, threadsPerBlock, 0, streams[nGpuIdx][nThreadIdx] >> > (resMicdFull_d, d_compArray, nMICD_Num, 1);
    cudaStatus = CHECK_CUDA(cudaGetLastError(), cu_dir); if (cudaStatus != TRUE) return FALSE;

    thrust::device_vector<CompPriority> compVector(d_compArray, d_compArray + nMICD_Num);
    thrust::sort(thrust::device, compVector.begin(), compVector.end(), CompCategoryAndMagComparator());
    // Assign indices based on MATLAB logic
    assignIndices << <(nMICD_Num + 255) / 256, 256 >> > (thrust::raw_pointer_cast(compVector.data()), nMICD_Num, MAX_COMPENTOTAL);
    cudaStatus = CHECK_CUDA(cudaGetLastError(), cu_dir); if (cudaStatus != TRUE) return FALSE;
    setCompenPriority << <nbsetPerDetect, threadsPerBlock, 0, streams[nGpuIdx][nThreadIdx] >> > (resMicdFull_d, thrust::raw_pointer_cast(compVector.data()), nMICD_Num, 1);
    cudaStatus = CHECK_CUDA(cudaGetLastError(), cu_dir); if (cudaStatus != TRUE) return FALSE;