CMakeLists linkage error 'magma_opts::parse_opts' using MAGMA testing code

Question

Currently I am using MAGMA 2.5.4 to solve a batched linear solver with tiny sizes of matrices. I want to involve magma_dgesv_batched in my project compiled via a CMakeLists file.

The include_directories and target_link_libraries are shown below.

include_directories( "/usr/local/magma/include" )
include_directories( "/home/research/magma-2.5.4/magma-2.5.4/include" )
include_directories( "/home/research/magma-2.5.4/magma-2.5.4/testing" )
target_link_libraries(minus
-L/usr/local/magma/lib magma_sparse magma
-L/usr/lib/cuda/lib64 cublas cudart cusparse
-L/usr/lib/x86_64-linux-gnu/openblas-pthread/cmake/openblas openblas
pthread
)

However, I met some linkage errors:

tmpxft_00003500_00000000-5_minus_cuda.cudafe1.cpp:(.text+0x506): undefined reference to `magma_opts::magma_opts(magma_opts_t)'
/usr/bin/ld: tmpxft_00003500_00000000-5_minus_cuda.cudafe1.cpp:(.text+0x514): undefined reference to `magma_opts::parse_opts(int, char**)'
/usr/bin/ld: tmpxft_00003500_00000000-5_minus_cuda.cudafe1.cpp:(.text+0xfce): undefined reference to `magma_opts::cleanup()'
collect2: error: ld returned 1 exit status
make[2]: *** [cmd/CMakeFiles/minus-simpleEx.dir/build.make:105: bin/minus-simpleEx] Error 1
make[1]: *** [CMakeFiles/Makefile2:926: cmd/CMakeFiles/minus-simpleEx.dir/all] Error 2
make: *** [Makefile:95: all] Error 2

Obviously, it shows that I am not linking the correct library, yet I have no idea how I should fix it in my CMakeLists file. I had viewed the MAGMA documentation and it seems that there is no additional library I need to link (perhaps I am doing something wrongly).

The MAGMA is installed successfully and I had run its testing code of magma_dgesv_batched perfectly as well. Gcc version is 8 with cuda 10 in Ubuntu 20.04.

Thank you!

Answer 1

Okay, I somehow solve the problem after asking one of the developers of MAGMA. The thing is that, magma_opts::magma_opts(magma_opts_t) is not included in standard MAGMA libraries, but it is kept in the testing folder of the MAGMA. I should not fully copy the testing code from MAGMA and try to run it, but rather, I should mimic its structure. To solve the opt::queue in the testing code, I need to create a magma queue by magma_queue_create and magma_queue_destroy.

Here's the complete code that is run perfectly:

#include <stdio.h>
#include <stdlib.h>

// magma
#include "flops.h"
#include "magma_v2.h"
#include "magma_lapack.h"

int main() {
  magma_init();
  magma_print_environment();

  real_Double_t   gflops, cpu_perf, cpu_time, gpu_perf, gpu_time;
  float          error, Rnorm, Anorm, Xnorm, *work;
  magmaFloatComplex c_one     = MAGMA_C_ONE;
  magmaFloatComplex c_neg_one = MAGMA_C_NEG_ONE;
  magmaFloatComplex *h_A, *h_B, *h_X;
  magmaFloatComplex_ptr d_A, d_B;
  magma_int_t *dipiv, *dinfo_array;
  magma_int_t *ipiv, *cpu_info;
  magma_int_t N, nrhs, lda, ldb, ldda, lddb, info, sizeA, sizeB;
  magma_int_t ione = 1;
  magma_int_t ISEED[4] = {0,0,0,1};
  int status = 0;
  magma_int_t batchCount = 2;
  nrhs = 1;

  magmaFloatComplex **dA_array = NULL;
  magmaFloatComplex **dB_array = NULL;
  magma_int_t     **dipiv_array = NULL;

  bool use_lapack = 1;
  double tol = 0.000001;
  N = 6;

  magma_queue_t my_queue;    // magma queue variable, internally holds a cuda stream and a cublas handle
  magma_device_t cdev;       // variable to indicate current gpu id

  magma_getdevice( &cdev );
  magma_queue_create( cdev, &my_queue );     // create a queue on this cdev

  printf("%% BatchCount   N  NRHS   CPU Gflop/s (msec)   GPU Gflop/s (msec)   ||B - AX|| / N*||A||*||X||\n");
  printf("%%============================================================================================\n");
  lda    = N;
  ldb    = lda;
  ldda   = magma_roundup( N, 32 );  // multiple of 32 by default
  lddb   = ldda;
  gflops = ( FLOPS_DGETRF( N, N ) + FLOPS_DGETRS( N, nrhs ) ) * batchCount / 1e9;

  sizeA = lda*N*batchCount;
  sizeB = ldb*nrhs*batchCount;

  magma_cmalloc_cpu( &h_A, sizeA );
  magma_cmalloc_cpu( &h_B, sizeB );
  magma_cmalloc_cpu( &h_X, sizeB );
  magma_smalloc_cpu( &work, N );
  magma_imalloc_cpu( &ipiv, batchCount*N );
  magma_imalloc_cpu( &cpu_info, batchCount );

  magma_cmalloc( &d_A, ldda*N*batchCount    );
  magma_cmalloc( &d_B, lddb*nrhs*batchCount );
  magma_imalloc( &dipiv, N * batchCount );
  magma_imalloc( &dinfo_array, batchCount );

  magma_malloc( (void**) &dA_array,    batchCount * sizeof(magmaFloatComplex*) );
  magma_malloc( (void**) &dB_array,    batchCount * sizeof(magmaFloatComplex*) );
  magma_malloc( (void**) &dipiv_array, batchCount * sizeof(magma_int_t*) );

  /* Initialize the matrices */
  lapackf77_clarnv( &ione, ISEED, &sizeA, h_A );
  lapackf77_clarnv( &ione, ISEED, &sizeB, h_B );

  magma_csetmatrix( N, N*batchCount,    h_A, lda, d_A, ldda, my_queue );
  magma_csetmatrix( N, nrhs*batchCount, h_B, ldb, d_B, lddb, my_queue );

  /* ====================================================================
     Performs operation using MAGMA
     =================================================================== */
  magma_cset_pointer( dA_array, d_A, ldda, 0, 0, ldda*N, batchCount, my_queue );
  magma_cset_pointer( dB_array, d_B, lddb, 0, 0, lddb*nrhs, batchCount, my_queue );
  magma_iset_pointer( dipiv_array, dipiv, 1, 0, 0, N, batchCount, my_queue );

  gpu_time = magma_sync_wtime( my_queue );
  info = magma_cgesv_batched(N, nrhs, dA_array, ldda, dipiv_array, dB_array, lddb, dinfo_array, batchCount, my_queue);
  gpu_time = magma_sync_wtime( my_queue ) - gpu_time;
  gpu_perf = gflops / gpu_time;

  // check correctness of results throught "dinfo_magma" and correctness of argument throught "info"
  magma_getvector( batchCount, sizeof(magma_int_t), dinfo_array, 1, cpu_info, 1, my_queue );
  for (int i=0; i < batchCount; i++)
  {
      if (cpu_info[i] != 0 ) {
          printf("magma_dgesv_batched matrix %lld returned internal error %lld\n",
                (long long) i, (long long) cpu_info[i] );
      }
  }
  if (info != 0) {
      printf("magma_dgesv_batched returned argument error %lld: %s.\n",
            (long long) info, magma_strerror( info ));
  }

  //=====================================================================
  // Residual
  //=====================================================================
  magma_cgetmatrix( N, nrhs*batchCount, d_B, lddb, h_X, ldb, my_queue );

  error = 0;
  for (magma_int_t s=0; s < batchCount; s++)
  {
      Anorm = lapackf77_clange("I", &N, &N,    h_A + s * lda * N, &lda, work);
      Xnorm = lapackf77_clange("I", &N, &nrhs, h_X + s * ldb * nrhs, &ldb, work);

      blasf77_cgemm( MagmaNoTransStr, MagmaNoTransStr, &N, &nrhs, &N,
                 &c_one,     h_A + s * lda * N, &lda,
                             h_X + s * ldb * nrhs, &ldb,
                 &c_neg_one, h_B + s * ldb * nrhs, &ldb);

      Rnorm = lapackf77_clange("I", &N, &nrhs, h_B + s * ldb * nrhs, &ldb, work);
      float err = Rnorm/(N*Anorm*Xnorm);

      if (std::isnan(err) || std::isinf(err)) {
          error = err;
          break;
      }
      error = max( err, error );
  }
  bool okay = (error < tol);
  status += ! okay;

  /* ====================================================================
     Performs operation using LAPACK
     =================================================================== */
  if ( use_lapack ) {
      cpu_time = magma_wtime();
      // #define BATCHED_DISABLE_PARCPU
      #if !defined (BATCHED_DISABLE_PARCPU) && defined(_OPENMP)
      magma_int_t nthreads = magma_get_lapack_numthreads();
      magma_set_lapack_numthreads(1);
      magma_set_omp_numthreads(nthreads);
      #pragma omp parallel for schedule(dynamic)
      #endif
      for (magma_int_t s=0; s < batchCount; s++)
      {
          magma_int_t locinfo;
          lapackf77_cgesv( &N, &nrhs, h_A + s * lda * N, &lda, ipiv + s * N, h_B + s * ldb * nrhs, &ldb, &locinfo );
          if (locinfo != 0) {
              printf("lapackf77_cgesv matrix %lld returned error %lld: %s.\n",
                      (long long) s, (long long) locinfo, magma_strerror( locinfo ));
          }
      }
      #if !defined (BATCHED_DISABLE_PARCPU) && defined(_OPENMP)
          magma_set_lapack_numthreads(nthreads);
      #endif
      cpu_time = magma_wtime() - cpu_time;
      cpu_perf = gflops / cpu_time;
      printf( "%10lld %5lld %5lld   %7.2f (%7.2f)   %7.2f (%7.2f)   %8.2e   %s\n",
              (long long) batchCount, (long long) N, (long long) nrhs,
              cpu_perf, cpu_time*1000, gpu_perf, gpu_time*1000,
              error, (okay ? "ok" : "failed"));
  }
  else {
      printf( "%10lld %5lld %5lld     ---   (  ---  )   %7.2f (%7.2f)   %8.2e   %s\n",
              (long long) batchCount, (long long) N, (long long) nrhs,
              gpu_perf, gpu_time,
              error, (okay ? "ok" : "failed"));
  }

  magma_queue_destroy( my_queue );

  magma_free_cpu( h_A );
  magma_free_cpu( h_B );
  magma_free_cpu( h_X );
  magma_free_cpu( work );
  magma_free_cpu( ipiv );
  magma_free_cpu( cpu_info );

  magma_free( d_A );
  magma_free( d_B );

  magma_free( dipiv );
  magma_free( dinfo_array );

  magma_free( dA_array );
  magma_free( dB_array );
  magma_free( dipiv_array );

  fflush( stdout );

  printf( "\n" );

  magma_finalize();
}