'symbol lookup error: undefined symbol' after change of module system on cluster

Question

after the operating system of the cluster I am running my application on with PETSC (Version 3.19) has been changed from CentOS 7 to Rocky Linux 8 and also the module system has been changed to Lmod, I get several error: symbol lookup error: undefined symbol: error messages.

When starting the application with make, nothing happens until the job is aborted per user-direction. When running the application with LD_DEBUGS=files, I get several error messages related to symbols in UCX, e.g.:

/cvmfs/software.hpc.rwth.de/Linux/RH8/x86_64/intel/skylake_avx512/software/UCX/1.12.1-GCCcore-11.3.0/lib/ucx/libuct_cma.so.0: error: symbol lookup error: undefined symbol: ucs_module_global_init (fatal)

Also, these two error messages show up when running with LD_DEBUGS=files

/rwthfs/rz/cluster/home/mh787286/petsc/arch-linux-c-debug/lib/libpetsc.so.3.19: error: symbol lookup error: undefined symbol: MPID_Abort (fatal)

/rwthfs/rz/cluster/home/mh787286/petsc/arch-linux-c-debug/lib/libpetsc.so.3.19: error: symbol lookup error: undefined symbol: ps_tool_initialize (fatal)

I have a custom Makefile, which looks like this:

### VARIABLES

ifndef PETSC_DIR
    PETSC_DIR=petsc/arch-linux-c-debug

    ifdef R_SLURM_ROLES
        module load GCC/11.3.0 OpenMPI/4.1.4
    endif

    UNAME_S := $(shell uname -s)
    ifeq ($(UNAME_S),Darwin)
        PETSC_DIR=/usr/local/Cellar/petsc/3.18.5
    endif
endif



include $(PETSC_DIR)/lib/petsc/conf/petscvariables
CFLAGS=-Werror -ggdb -Wall -D_GNU_SOURCE -Wno-nullability-completeness
INCLUDES=$(PETSC_CC_INCLUDES)

LDFLAGS=-L$(LD_LIBRARY_PATH) -L$(PETSC_DIR)/lib -lm $(PETSC_LIB_BASIC)
ifeq ($(UNAME_S),Darwin)
    LDFLAGS=-L$(PETSC_DIR)/lib -lm $(PETSC_LIB_BASIC)
endif

GMSH = gmsh

export LD_LIBRARY_PATH := $(LD_LIBRARY_PATH):$(PETSC_DIR)/arch-linux-c-debug/lib

### RUNTIME OPTIONS
MPIOPTS=-np 1 # currently no parallel support

SRC := $(wildcard src/*.cpp) $(wildcard src/**/*.cpp) 
OBJ := $(SRC:%.cpp=obj/%.o)


# Meshes
GEO := $(wildcard meshdata/*-1D.geo)
MSH := $(GEO:%.geo=%.msh)


# Systems
SYS := $(wildcard sysdata/*.sys)
DEF := $(SYS:%.sys=%.def)

# PetSc Options
TOL=-snes_atol 1e-8 -snes_rtol 1e-30 -snes_stol 1e-16 -ts_pseudo_fatol 1e-8 -ts_pseudo_frtol 1e-30
MONITOR=-snes_converged_reason #-snes_monitor 
OPTS=$(MONITOR) $(TOL) -snes_max_it 10 # -info

.PHONY: clean all debug valgrind test sys
all: memristor sys

run: sys memristor
    @rm -rf outdata/memristor2
    @mv outdata/memristor outdata/memristor2 || true # cache last result
    @echo "[EX] >>> "$<
    $(LDPATH) $(MPIEXEC) -n 1 ./memristor $(OPTS) 

run-parallel: sys memristor
    $(LDPATH) $(MPIEXEC) -n 2 ./memristor $(OPTS) 

### PATTERNS
obj/%.o: %.cpp
    @echo "[CC] >>> "$<
    @mkdir -p $(dir $@)
    @$(CXX) $(CFLAGS) $(INCLUDES) -std=c++11 -c $< -o $@ 
%1D.msh: %1D.geo
    @echo "[GM] >>> "$<
    $(GMSH) -1 -format msh22 $< -o $@ 
    ### Gmsh -1 -format msh22 input.geo -o output.msh

%.def: %.sys
    @echo "[DG] >>> "$<
    @sysdata/defgen3.py $< > $@

### MESHES
geo: $(MSH)
### SYSTEMS
sys: $(DEF)

### APPLICATIONS

bpm: $(OBJ) obj/bpm.o
    @echo "[LD] >>> "$@
    $(CXX) $(OBJ) obj/bpm.o -o $@  $(LDFLAGS)

bpm-run: sys bpm
    @echo "[EX] >>> "$<
    $(LDPATH) $(MPIEXEC) -n 1 ./bpm $(POTENTIAL)  $(OPTS) 

cathode: $(OBJ) obj/cathode.o
    @echo "[LD] >>> "$@
    $(CXX) $(OBJ) obj/cathode.o -o $@  $(LDFLAGS)

cathode-run: sys cathode
    @echo "[EX] >>> "$<
    $(LDPATH) $(MPIEXEC) -n 1 ./cathode $(OPTS) 

valgrind-cathode: cathode
    @echo "[VG] >>> "$<
    @$(LDPATH) valgrind --leak-check=full --track-origins=yes --show-leak-kinds=all --log-file=valgrind-out.txt ./cathode

callgrind-cathode: cathode
    @echo "[VG] >>> "$<
    @$(LDPATH) valgrind --tool=callgrind ./cathode # --track-origins=yes



clean:
    rm -rf obj/*

I have looked up the shared libraries that need to be linked for running PETSC and it seems they are all within LD_LIBRARY_PATH. So it is hard for me to identify what causes this problem. I am still a beginner in using PETSC, so any help and suggestions are highly appreciated. Thank you!