Creating Python module from Fortan using CMake and make yields no errors but importing module in Python fails due to undefined symbol

Question

I am receiving

Python 3.12.0 | packaged by conda-forge | (main, Oct  3 2023, 08:43:22) [GCC 12.3.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import foo
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
ImportError: /home/.../myproject/build/foo.cpython-312-x86_64-linux-gnu.so: undefined symbol: f2pywrapfoo_

when trying to load a Python module created using CMake from Fortran code using numpy.f2py.

---

I am following the official guide for using numpy.f2py in CMake. I am using the latest version of NumPy (1.26), CMake 3.22 and the gfortran and gcc compilers from the Xubuntu 22.04 repos (11.4 and 12.3 respectively).

My Fortran function is even simpler (since I am very new to the language) than the example in the documentation, namely

function foo(a) result(b)
    implicit none
    
    real(kind=8), intent(in)    :: a(:,:)
    complex(kind=8)             :: b(size(a,1),size(a,2))
    
    b = exp((0,1)*a)
    
end function foo

The part of my CMake that handles the module generation is

if(PYTHON_F2PY)
# https://numpy.org/doc/stable/f2py/buildtools/cmake.html
# https://numpy.org/doc/stable/f2py/usage.html
    message("Creating Python module from Fortran code enabled")
    # Example for interfacing with Python using f2py
    # Check if Python with the required version and components is available
    find_package(Python 3.12 REQUIRED
    COMPONENTS Interpreter Development.Module NumPy)

    # Grab the variables from a local Python installation
    # F2PY headers
    execute_process(
    COMMAND "${Python_EXECUTABLE}"
    -c "import numpy.f2py; print(numpy.f2py.get_include())"
    OUTPUT_VARIABLE F2PY_INCLUDE_DIR
    OUTPUT_STRIP_TRAILING_WHITESPACE
    )
    #message("${F2PY_INCLUDE_DIR}")
    # Print out the discovered paths
    include(CMakePrintHelpers)
    cmake_print_variables(Python_INCLUDE_DIRS)
    cmake_print_variables(F2PY_INCLUDE_DIR)
    cmake_print_variables(Python_NumPy_INCLUDE_DIRS)

    # Common variables
    set(f2py_module_name "foo")
    set(fortran_src_file "${CMAKE_SOURCE_DIR}/src/foo.f90")
    set(f2py_module_c "${f2py_module_name}module.c")

    # Generate sources
    add_custom_target(
    genpyf
    DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_c}"
    )
    add_custom_command(
    OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_c}"
    COMMAND ${Python_EXECUTABLE}  -m "numpy.f2py"
                    "${fortran_src_file}"
                    -m "${f2py_module_name}"
                    --lower # Important
    DEPENDS "src/foo.f90" # Fortran source
    )

    # Set up target
    Python_add_library(foo MODULE WITH_SOABI
    "${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_c}" # Generated
    "${F2PY_INCLUDE_DIR}/fortranobject.c" # From NumPy
    "${fortran_src_file}" # Fortran source(s) #"foo-f2pywrappers2.f90"
    )

    # Depend on sources
    target_link_libraries(foo PRIVATE Python::NumPy)
    add_dependencies(foo genpyf)
    target_include_directories(foo PRIVATE "${F2PY_INCLUDE_DIR}")
endif(PYTHON_F2PY)

Inside my building directory I run

cmake -Wno-dev -DPYTHON_F2PY=1 ..

to generate the project, followed by

make -j10

to build it.

Among others I get the following files:

• foo.cpython-312-x86_64-linux-gnu.so - the shared library that I can load in Python

• foo-f2pywrappers.f - an empty Fortran file

• foo-f2pywrappers2.f90 - Fortran file containing some wrapper code

  !     -*- f90 -*-
  !     This file is autogenerated with f2py (version:1.26.4)
  !     It contains Fortran 90 wrappers to fortran functions.
  subroutine f2pywrapfoo (foof2pywrap, a, f2py_a_d0, f2py_a_d1)
   integer f2py_a_d0
   integer f2py_a_d1
   real(kind=8) a(f2py_a_d0,f2py_a_d1)
   complex(kind=8) foof2pywrap(size(a, 1),size(a, 2))
   interface
     function foo(a) result (b) 
       real(kind=8), intent(in),dimension(:,:) :: a
       complex(kind=8), dimension(size(a,1),size(a,2)) :: b
     end function foo
   end interface
   foof2pywrap = foo(a)
  end
  

• foomodule.c - the C code generated for my module that will is used to build shared the library

The error message

Python 3.12.0 | packaged by conda-forge | (main, Oct  3 2023, 08:43:22) [GCC 12.3.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import foo
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
ImportError: /home/.../myproject/build/foo.cpython-312-x86_64-linux-gnu.so: undefined symbol: f2pywrapfoo_

points as f2pywrapfoo_. As you can see above, the foo-f2pywrappers2.f90 file contains the function (albeit without the _ suffix).

What I did is to add that wrapper to the list of Fortran source files

# Set up target
Python_add_library(foo MODULE WITH_SOABI
"${CMAKE_CURRENT_BINARY_DIR}/${f2py_module_c}" # Generated
"${F2PY_INCLUDE_DIR}/fortranobject.c" # From NumPy
"${fortran_src_file}" "foo-f2pywrappers2.f90" # Fortran source(s)
)

I run CMake and make again. When I repeat the steps for importing the module, now it works:

Python 3.12.0 | packaged by conda-forge | (main, Oct  3 2023, 08:43:22) [GCC 12.3.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> from foo import foo
>>> import numpy as np
>>> a = np.array([[1,2,3,4], [5,6,7,8]], order='F')
>>> foo(a)
array([[ 0.54030231+0.84147098j, -0.41614684+0.90929743j,
        -0.9899925 +0.14112001j, -0.65364362-0.7568025j ],
       [ 0.28366219-0.95892427j,  0.96017029-0.2794155j ,
         0.75390225+0.6569866j , -0.14550003+0.98935825j]])

The problem is that foo-f2pywrappers2.f90 is not available during the first run of CMake and make. It is created only after make is executed. So I cannot really add it as a dependency for the library building stage in the CMakeLists.txt.

Any ideas what to change in order to make this work?

Answer 1

You could do this with a simple Makefile:

f2py_module_name = foo

all: foo.pyf foo.cpython-311-x86_64-linux-gnu.so

clean:
    rm -f *.pyf *.so *wrappers* *module.c

foo.pyf: src/foo.f90
    f2py -m $(f2py_module_name) $< -h $@ --overwrite-signature

foo.cpython-311-x86_64-linux-gnu.so: src/foo.f90
    f2py -m $(f2py_module_name) -c $<

foo-f2pywrappers.f foomodule.c foo-f2pywrappers2.f90: src/foo.f90
    f2py -m $(f2py_module_name) $< --lower

test:
    python3 test.py

Alternatively, a CMakeLists.txt might look like:

cmake_minimum_required(VERSION 3.12)
project(foo_f2py)

find_package(Python 3.12 REQUIRED COMPONENTS Interpreter Development.Module NumPy)

set(f2py_module_name foo)

add_custom_target(generate_foo_pyf
    COMMAND f2py -m ${f2py_module_name} src/foo.f90 -h foo.pyf --overwrite-signature
    COMMENT "Generate foo.pyf."
    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
    VERBATIM
)

add_custom_target(generate_foo_so
    COMMAND f2py -m ${f2py_module_name} -c src/foo.f90
    COMMENT "Generate foo.cpython-311-x86_64-linux-gnu.so."
    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
    VERBATIM
)

add_custom_target(clean_files
    COMMAND rm -f *.pyf *.so *wrappers* *module.c
    COMMENT "Clean files."
    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
    VERBATIM
)

add_custom_target(run_tests
    COMMAND python3 test.py
    COMMENT "Run test."
    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
    VERBATIM
)

add_dependencies(generate_foo_so generate_foo_pyf)

# Set default target.
add_custom_target(default_target ALL DEPENDS generate_foo_so)

To build:

mkdir build
cd build
cmake ..
make
# Return to directory containing built package.
cd ..

Test using Docker (see screenshot below). Of course, you don't need to use Docker and can just build/run this on your localhost. If you don't have Python 3.12 then you can drop the find_package() from CMakeLists.txt and all should still be well.

To evaluate this approach with your version of Python (3.12.0) I made a little Docker image.

Dockerfile

FROM python:3.12.0

RUN apt-get update -qq && \
    apt-get install -y -qq cmake gfortran

WORKDIR /app

COPY requirements.txt .

RUN pip3 install -r requirements.txt

COPY . .

RUN rm -rf build && \
    mkdir build && \
    cd build && \
    cmake .. && \
    # make
    true

requirements.txt

numpy==1.26.4
meson==1.3.2
ninja==1.11.1.1