I'm trying to build on a fresh Linux installation a CFFI-based package I'm maintaining.
However, out of the box, the errors it gives about the dependencies pip couldn't fetch (attached at the end of this post) are useless and inscrutable.
I want to catch, override, or somehow augment these error messages so I can give the naive user trying to execute python -m build or pip install . on a new platform some actually meaningful, actionable error messages suggesting what packages to install (including, for example, the link to Windows' Build Tools).
My project is pyproject.toml-based, with the default backend / no backend specified. It seems to be using build_ext due to its cffi build-time dependency.
How can this be done?
Appendix: Error messages
The solution to this one was dnf groupinstall 'Development Tools'/apt install build-essential:
…
creating build/lib.linux-x86_64-cpython-39/pqc/kem
copying pqc/kem/__init__.py -> build/lib.linux-x86_64-cpython-39/pqc/kem
copying pqc/kem/mceliece348864.py -> build/lib.linux-x86_64-cpython-39/pqc/kem
copying pqc/kem/mceliece460896.py -> build/lib.linux-x86_64-cpython-39/pqc/kem
copying pqc/kem/mceliece6688128.py -> build/lib.linux-x86_64-cpython-39/pqc/kem
copying pqc/kem/mceliece6960119.py -> build/lib.linux-x86_64-cpython-39/pqc/kem
copying pqc/kem/mceliece8192128.py -> build/lib.linux-x86_64-cpython-39/pqc/kem
running build_ext
…
error: command 'gcc' failed: No such file or directory
ERROR Backend subprocess exited when trying to invoke build_wheel
$
The solution to this one was dnf install python3-devel/apt install python3-dev:
…
creating build/lib.linux-x86_64-cpython-39/pqc/kem
copying pqc/kem/__init__.py -> build/lib.linux-x86_64-cpython-39/pqc/kem
copying pqc/kem/mceliece348864.py -> build/lib.linux-x86_64-cpython-39/pqc/kem
copying pqc/kem/mceliece460896.py -> build/lib.linux-x86_64-cpython-39/pqc/kem
copying pqc/kem/mceliece6688128.py -> build/lib.linux-x86_64-cpython-39/pqc/kem
copying pqc/kem/mceliece6960119.py -> build/lib.linux-x86_64-cpython-39/pqc/kem
copying pqc/kem/mceliece8192128.py -> build/lib.linux-x86_64-cpython-39/pqc/kem
running build_ext
…
build/temp.linux-x86_64-cpython-39/pqc._lib.mceliece348864f_clean.c:50:14: fatal error: pyconfig.h: No such file or directory
50 | # include <pyconfig.h>
| ^~~~~~~~~~~~
compilation terminated.
error: command '/usr/bin/gcc' failed with exit code 1
ERROR Backend subprocess exited when trying to invoke build_wheel
$
Appendix 2: MRE
As requested by a guy in the comments, here is a "minimal reproducible example" of a CFFI-based setuptools package which will demonstrate these errors, in case you are unfamiliar with them yet hope to answer this question anyway — this will allow you to reproduce the error on any fresh installation of Windows 10 or Ubuntu LTS (I don't know enough about Python use on Macs to say how the cookie would crumble there, though):
from itertools import chain, repeat, islice
import os
from pathlib import Path
import platform
import shlex
import subprocess
import sys
import tempfile
from textwrap import dedent
import venv
BUILD_CMD = ['python', '-m', 'pip', 'install', '-v', '.']
def main():
with tempfile.TemporaryDirectory() as td:
td = os.path.realpath(td)
venv.create(td, with_pip=True)
root = Path(td) / 'stackoverflow-77686182'
root.mkdir()
# 1. create setup.py
(root / 'setup.py').write_text(dedent('''\
# https://foss.heptapod.net/pypy/cffi/-/issues/441
# https://github.com/python-cffi/cffi/issues/55
from setuptools import setup
setup(
cffi_modules = [
'cffi_module.py:ffi'
]
)
'''))
# 2. create CFFI module
(root / 'cffi_module.py').write_text(dedent('''\
from cffi import FFI
from pathlib import Path
root = Path(__file__).parent
ffibuilder = FFI()
ffibuilder.cdef('unsigned short spam();')
ffibuilder.set_source(
'stackoverflow_77686182._libspam',
'#include "libspam.h"',
sources=[(root / 'libspam.c').as_posix()],
include_dirs=[root.as_posix()]
)
ffi = ffibuilder
'''))
# 3. create C source
(root / 'libspam.c').write_text(dedent('''\
unsigned short spam() {
return 69;
}
'''))
# 4. Create C headers
(root / 'libspam.h').write_text(dedent('''\
unsigned short spam();
'''))
# 5. Create Python package code
(root / 'src' / 'stackoverflow_77686182').mkdir(parents=True, exist_ok=True)
(root / 'src' / 'stackoverflow_77686182' / '__init__.py').write_text(dedent('''\
from . import _libspam
'''))
(root / 'src' / 'stackoverflow_77686182' / '__main__.py').write_text(dedent('''\
from stackoverflow_77686182._libspam import ffi, lib
if lib.spam() == 69:
print('OK')
else:
raise AssertionError('FAIL')
'''))
# 6. Create Python packaging metadata
(root / 'pyproject.toml').write_text(dedent('''\
[project]
name = 'stackoverflow_77686182'
version = '0'
dependencies = [
'cffi >= 1.0.0;platform_python_implementation != "PyPy"',
]
[build-system]
requires = [
'cffi >= 1.14',
'setuptools >= 49.5.0'
]
'''))
(root / 'requirements-dev.txt').write_text(dedent('''\
pip >= 21.1.3
setuptools >= 49.5.0
'''))
# 7. Build and install
_check_call_in_venv(td, ['python', '-m', 'pip', 'install', '-r', root / 'requirements-dev.txt'])
_check_call_in_venv(td, BUILD_CMD, cwd=root)
# 8. Run
_check_call_in_venv(td, ['python', '-m', 'stackoverflow_77686182'])
raise NotImplementedError('This script should be run on a machine that LACKS the necessary items for a CFFI project build, but the build seems to have succeeded.')
def _check_call_in_venv(env_dir, cmd, *a, **k):
script = []
is_win = (platform.system() == 'Windows')
script.append(['.', Path(env_dir) / 'bin' / 'activate'] if not is_win else [Path(env_dir) / 'Scripts' / 'activate.bat'])
script.append(cmd)
_cmd = ['sh', '-c', ';\n'.join(shlex.join(cmd) for cmd in ['set -e'] + script)] if not is_win else f'cmd /C "{" ".join(_intersperse("&&", (" ".join(map(str, cmd)) for cmd in script)))}"'
return subprocess.check_call(_cmd, *a, **k)
def _intersperse(delim, seq):
# https://stackoverflow.com/a/5656097/1874170
return islice(chain.from_iterable(zip(repeat(delim), seq)), 1, None)
if __name__ == '__main__':
main()