I am trying to train a BERTopic Model in python. However, I get this error:
RuntimeError: Failed to import transformers.models.auto because of the following error (look up to see its traceback):
libsentencepiece.so.0: cannot open shared object file: No such file or directory
I have tried to uninstall and reinstall transformers, install sentencepiece, and set the LD_LIBRARY_PATH variable to the path to libsentencepiece.so.0. None of these things have worked. Does anyone know why this is? If it matters, I am running this script on a remote server. I think perhaps it may have something to do with conflicts with the dependencies, but I am unsure.
Python Code
from bertopic import BERTopic
model = BERTopic()
Full Traceback
Traceback (most recent call last):
File "/net/vast-storage/scratch/vast/gablab/kmcclenn/bin/mambaforge/envs/item-analysis/lib/python3.8/site-packages/transformers/utils/import_utils.py", line 1086, in _get_module
return importlib.import_module("." + module_name, self.__name__)
File "/net/vast-storage/scratch/vast/gablab/kmcclenn/bin/mambaforge/envs/item-analysis/lib/python3.8/importlib/__init__.py", line 127, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "<frozen importlib._bootstrap>", line 1014, in _gcd_import
File "<frozen importlib._bootstrap>", line 991, in _find_and_load
File "<frozen importlib._bootstrap>", line 961, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "<frozen importlib._bootstrap>", line 1014, in _gcd_import
File "<frozen importlib._bootstrap>", line 991, in _find_and_load
File "<frozen importlib._bootstrap>", line 975, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 671, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 843, in exec_module
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "/net/vast-storage/scratch/vast/gablab/kmcclenn/bin/mambaforge/envs/item-analysis/lib/python3.8/site-packages/transformers/models/__init__.py", line 15, in <module>
from . import (
File "/net/vast-storage/scratch/vast/gablab/kmcclenn/bin/mambaforge/envs/item-analysis/lib/python3.8/site-packages/transformers/models/mt5/__init__.py", line 29, in <module>
from ..t5.tokenization_t5 import T5Tokenizer
File "/net/vast-storage/scratch/vast/gablab/kmcclenn/bin/mambaforge/envs/item-analysis/lib/python3.8/site-packages/transformers/models/t5/tokenization_t5.py", line 24, in <module>
import sentencepiece as spm
File "/net/vast-storage/scratch/vast/gablab/kmcclenn/bin/mambaforge/envs/item-analysis/lib/python3.8/site-packages/sentencepiece/__init__.py", line 13, in <module>
from . import _sentencepiece
ImportError: libsentencepiece.so.0: cannot open shared object file: No such file or directory
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/om2/user/kmcclenn/item_analysis/openmind_python_files/scripts/test_script.py", line 92, in <module>
run()
File "/om2/user/kmcclenn/item_analysis/openmind_python_files/scripts/test_script.py", line 73, in run
from bertopic import BERTopic
File "/net/vast-storage/scratch/vast/gablab/kmcclenn/bin/mambaforge/envs/item-analysis/lib/python3.8/site-packages/bertopic/__init__.py", line 1, in <module>
from bertopic._bertopic import BERTopic
File "/net/vast-storage/scratch/vast/gablab/kmcclenn/bin/mambaforge/envs/item-analysis/lib/python3.8/site-packages/bertopic/_bertopic.py", line 48, in <module>
from bertopic.backend import BaseEmbedder
File "/net/vast-storage/scratch/vast/gablab/kmcclenn/bin/mambaforge/envs/item-analysis/lib/python3.8/site-packages/bertopic/backend/__init__.py", line 22, in <module>
from bertopic.backend._multimodal import MultiModalBackend
File "/net/vast-storage/scratch/vast/gablab/kmcclenn/bin/mambaforge/envs/item-analysis/lib/python3.8/site-packages/bertopic/backend/_multimodal.py", line 6, in <module>
from sentence_transformers import SentenceTransformer
File "/net/vast-storage/scratch/vast/gablab/kmcclenn/bin/mambaforge/envs/item-analysis/lib/python3.8/site-packages/sentence_transformers/__init__.py", line 3, in <module>
from .datasets import SentencesDataset, ParallelSentencesDataset
File "/net/vast-storage/scratch/vast/gablab/kmcclenn/bin/mambaforge/envs/item-analysis/lib/python3.8/site-packages/sentence_transformers/datasets/__init__.py", line 3, in <module>
from .ParallelSentencesDataset import ParallelSentencesDataset
File "/net/vast-storage/scratch/vast/gablab/kmcclenn/bin/mambaforge/envs/item-analysis/lib/python3.8/site-packages/sentence_transformers/datasets/ParallelSentencesDataset.py", line 4, in <module>
from .. import SentenceTransformer
File "/net/vast-storage/scratch/vast/gablab/kmcclenn/bin/mambaforge/envs/item-analysis/lib/python3.8/site-packages/sentence_transformers/SentenceTransformer.py", line 27, in <module>
from .models import Transformer, Pooling, Dense
File "/net/vast-storage/scratch/vast/gablab/kmcclenn/bin/mambaforge/envs/item-analysis/lib/python3.8/site-packages/sentence_transformers/models/__init__.py", line 1, in <module>
from .Transformer import Transformer
File "/net/vast-storage/scratch/vast/gablab/kmcclenn/bin/mambaforge/envs/item-analysis/lib/python3.8/site-packages/sentence_transformers/models/Transformer.py", line 2, in <module>
from transformers import AutoModel, AutoTokenizer, AutoConfig, T5Config
File "<frozen importlib._bootstrap>", line 1039, in _handle_fromlist
File "/net/vast-storage/scratch/vast/gablab/kmcclenn/bin/mambaforge/envs/item-analysis/lib/python3.8/site-packages/transformers/utils/import_utils.py", line 1076, in __getattr__
module = self._get_module(self._class_to_module[name])
File "/net/vast-storage/scratch/vast/gablab/kmcclenn/bin/mambaforge/envs/item-analysis/lib/python3.8/site-packages/transformers/utils/import_utils.py", line 1088, in _get_module
raise RuntimeError(
RuntimeError: Failed to import transformers.models.auto because of the following error (look up to see its traceback):
libsentencepiece.so.0: cannot open shared object file: No such file or directory
Dependencies
# Name Version Build Channel
_libgcc_mutex 0.1 conda_forge conda-forge
_openmp_mutex 4.5 2_kmp_llvm conda-forge
aiohttp 3.8.4 py38h01eb140_1 conda-forge
aiosignal 1.3.1 pyhd8ed1ab_0 conda-forge
appdirs 1.4.4 pyh9f0ad1d_0 conda-forge
asttokens 2.2.1 pyhd8ed1ab_0 conda-forge
async-timeout 4.0.2 pyhd8ed1ab_0 conda-forge
attrs 23.1.0 pyh71513ae_1 conda-forge
aws-c-auth 0.7.0 hf8751d9_2 conda-forge
aws-c-cal 0.6.0 h93469e0_0 conda-forge
aws-c-common 0.8.23 hd590300_0 conda-forge
aws-c-compression 0.2.17 h862ab75_1 conda-forge
aws-c-event-stream 0.3.1 h9599702_1 conda-forge
aws-c-http 0.7.11 hbe98c3e_0 conda-forge
aws-c-io 0.13.28 h3870b5a_0 conda-forge
aws-c-mqtt 0.8.14 h2e270ba_2 conda-forge
aws-c-s3 0.3.13 heb0bb06_2 conda-forge
aws-c-sdkutils 0.1.11 h862ab75_1 conda-forge
aws-checksums 0.1.16 h862ab75_1 conda-forge
aws-crt-cpp 0.20.3 he9c0e7f_4 conda-forge
aws-sdk-cpp 1.10.57 hbc2ea52_17 conda-forge
backcall 0.2.0 pyh9f0ad1d_0 conda-forge
backports 1.0 pyhd8ed1ab_3 conda-forge
backports.functools_lru_cache 1.6.5 pyhd8ed1ab_0 conda-forge
bertopic 0.15.0 pyhd8ed1ab_0 conda-forge
brotli 1.0.9 h166bdaf_9 conda-forge
brotli-bin 1.0.9 h166bdaf_9 conda-forge
brotli-python 1.0.9 py38hfa26641_9 conda-forge
bzip2 1.0.8 h7f98852_4 conda-forge
c-ares 1.19.1 hd590300_0 conda-forge
ca-certificates 2023.5.7 hbcca054_0 conda-forge
certifi 2023.5.7 pyhd8ed1ab_0 conda-forge
charset-normalizer 3.2.0 pyhd8ed1ab_0 conda-forge
click 8.1.5 unix_pyh707e725_0 conda-forge
colorama 0.4.6 pyhd8ed1ab_0 conda-forge
comm 0.1.3 pyhd8ed1ab_0 conda-forge
contourpy 1.1.0 py38h7f3f72f_0 conda-forge
cycler 0.11.0 pyhd8ed1ab_0 conda-forge
cython 0.29.36 py38h17151c0_0 conda-forge
dataclasses 0.8 pyhc8e2a94_3 conda-forge
datasets 2.13.1 pyhd8ed1ab_0 conda-forge
dcor 0.6 pyhd8ed1ab_0 conda-forge
debugpy 1.6.7 py38h8dc9893_0 conda-forge
decorator 5.1.1 pyhd8ed1ab_0 conda-forge
dill 0.3.6 pyhd8ed1ab_1 conda-forge
docker-pycreds 0.4.0 py_0 conda-forge
et_xmlfile 1.1.0 pyhd8ed1ab_0 conda-forge
executing 1.2.0 pyhd8ed1ab_0 conda-forge
filelock 3.12.2 pyhd8ed1ab_0 conda-forge
fonttools 4.41.0 py38h01eb140_0 conda-forge
freetype 2.12.1 hca18f0e_1 conda-forge
frozenlist 1.4.0 py38h01eb140_0 conda-forge
fsspec 2023.6.0 pyh1a96a4e_0 conda-forge
gensim 3.8.3 py38h709712a_4 conda-forge
gflags 2.2.2 he1b5a44_1004 conda-forge
gitdb 4.0.10 pyhd8ed1ab_0 conda-forge
gitpython 3.1.32 pyhd8ed1ab_0 conda-forge
glog 0.6.0 h6f12383_0 conda-forge
gmp 6.2.1 h58526e2_0 conda-forge
gmpy2 2.1.2 py38h793c122_1 conda-forge
hdbscan 0.8.30 py38h31356c5_0 conda-forge
huggingface_hub 0.16.3 pyhd8ed1ab_0 conda-forge
icu 72.1 hcb278e6_0 conda-forge
idna 3.4 pyhd8ed1ab_0 conda-forge
imbalanced-learn 0.11.0 pyhd8ed1ab_0 conda-forge
importlib-metadata 6.8.0 pyha770c72_0 conda-forge
importlib-resources 6.0.0 pyhd8ed1ab_1 conda-forge
importlib_metadata 6.8.0 hd8ed1ab_0 conda-forge
importlib_resources 6.0.0 pyhd8ed1ab_1 conda-forge
ipykernel 6.24.0 pyh71e2992_0 conda-forge
ipython 8.12.2 pyh41d4057_0 conda-forge
jedi 0.18.2 pyhd8ed1ab_0 conda-forge
jinja2 3.1.2 pyhd8ed1ab_1 conda-forge
joblib 1.3.0 pyhd8ed1ab_1 conda-forge
jsonschema 4.18.3 pyhd8ed1ab_0 conda-forge
jsonschema-specifications 2023.6.1 pyhd8ed1ab_0 conda-forge
jupyter_client 8.3.0 pyhd8ed1ab_0 conda-forge
jupyter_core 5.3.0 py38h578d9bd_0 conda-forge
keyutils 1.6.1 h166bdaf_0 conda-forge
kiwisolver 1.4.4 py38h43d8883_1 conda-forge
krb5 1.21.1 h659d440_0 conda-forge
lcms2 2.15 haa2dc70_1 conda-forge
ld_impl_linux-64 2.40 h41732ed_0 conda-forge
lerc 4.0.0 h27087fc_0 conda-forge
libabseil 20230125.3 cxx17_h59595ed_0 conda-forge
libarrow 12.0.1 hd2d78f0_4_cpu conda-forge
libblas 3.9.0 17_linux64_openblas conda-forge
libbrotlicommon 1.0.9 h166bdaf_9 conda-forge
libbrotlidec 1.0.9 h166bdaf_9 conda-forge
libbrotlienc 1.0.9 h166bdaf_9 conda-forge
libcblas 3.9.0 17_linux64_openblas conda-forge
libcrc32c 1.1.2 h9c3ff4c_0 conda-forge
libcurl 8.1.2 hca28451_1 conda-forge
libdeflate 1.18 h0b41bf4_0 conda-forge
libedit 3.1.20191231 he28a2e2_2 conda-forge
libev 4.33 h516909a_1 conda-forge
libevent 2.1.12 hf998b51_1 conda-forge
libffi 3.4.2 h7f98852_5 conda-forge
libgcc-ng 13.1.0 he5830b7_0 conda-forge
libgfortran-ng 13.1.0 h69a702a_0 conda-forge
libgfortran5 13.1.0 h15d22d2_0 conda-forge
libgoogle-cloud 2.12.0 hac9eb74_1 conda-forge
libgrpc 1.54.2 hb20ce57_2 conda-forge
libhwloc 2.9.1 nocuda_h7313eea_6 conda-forge
libiconv 1.17 h166bdaf_0 conda-forge
libjpeg-turbo 2.1.5.1 h0b41bf4_0 conda-forge
liblapack 3.9.0 17_linux64_openblas conda-forge
libllvm10 10.0.1 he513fc3_3 conda-forge
libnghttp2 1.52.0 h61bc06f_0 conda-forge
libnsl 2.0.0 h7f98852_0 conda-forge
libnuma 2.0.16 h0b41bf4_1 conda-forge
libopenblas 0.3.23 pthreads_h80387f5_0 conda-forge
libpng 1.6.39 h753d276_0 conda-forge
libprotobuf 3.21.12 h3eb15da_0 conda-forge
libsentencepiece 0.1.99 h180e1df_0 conda-forge
libsodium 1.0.18 h36c2ea0_1 conda-forge
libsqlite 3.42.0 h2797004_0 conda-forge
libssh2 1.11.0 h0841786_0 conda-forge
libstdcxx-ng 13.1.0 hfd8a6a1_0 conda-forge
libthrift 0.18.1 h8fd135c_2 conda-forge
libtiff 4.5.1 h8b53f26_0 conda-forge
libutf8proc 2.8.0 h166bdaf_0 conda-forge
libuuid 2.38.1 h0b41bf4_0 conda-forge
libwebp-base 1.3.1 hd590300_0 conda-forge
libxcb 1.15 h0b41bf4_0 conda-forge
libxml2 2.11.4 h0d562d8_0 conda-forge
libzlib 1.2.13 hd590300_5 conda-forge
llvm-openmp 16.0.6 h4dfa4b3_0 conda-forge
llvmlite 0.34.0 py38h4f45e52_2 conda-forge
lz4-c 1.9.4 hcb278e6_0 conda-forge
markupsafe 2.1.3 py38h01eb140_0 conda-forge
matplotlib-base 3.7.1 py38hd6c3c57_0 conda-forge
matplotlib-inline 0.1.6 pyhd8ed1ab_0 conda-forge
mkl 2022.2.1 h84fe81f_16997 conda-forge
mpc 1.3.1 hfe3b2da_0 conda-forge
mpfr 4.2.0 hb012696_0 conda-forge
mpmath 1.3.0 pyhd8ed1ab_0 conda-forge
multidict 6.0.4 py38h1de0b5d_0 conda-forge
multiprocess 0.70.14 py38h0a891b7_3 conda-forge
munkres 1.1.4 pyh9f0ad1d_0 conda-forge
nbformat 5.9.1 pyhd8ed1ab_0 conda-forge
ncurses 6.4 hcb278e6_0 conda-forge
nest-asyncio 1.5.6 pyhd8ed1ab_0 conda-forge
networkx 3.1 pyhd8ed1ab_0 conda-forge
nltk 3.8.1 pyhd8ed1ab_0 conda-forge
numba 0.51.2 py38h0573a6f_1
numpy 1.23.5 py38h7042d01_0 conda-forge
openai 0.27.8 pyhd8ed1ab_0 conda-forge
openjpeg 2.5.0 hfec8fc6_2 conda-forge
openpyxl 3.1.2 py38h01eb140_0 conda-forge
openssl 3.1.1 hd590300_1 conda-forge
orc 1.9.0 h2f23424_1 conda-forge
packaging 23.1 pyhd8ed1ab_0 conda-forge
pandas 2.0.3 py38h01efb38_1 conda-forge
pandas-stubs 2.0.1.230501 pyhd8ed1ab_0 conda-forge
parso 0.8.3 pyhd8ed1ab_0 conda-forge
pathtools 0.1.2 py_1 conda-forge
patsy 0.5.3 pyhd8ed1ab_0 conda-forge
pexpect 4.8.0 pyh1a96a4e_2 conda-forge
pickleshare 0.7.5 py_1003 conda-forge
pillow 10.0.0 py38h885162f_0 conda-forge
pip 23.1.2 pyhd8ed1ab_0 conda-forge
pkgutil-resolve-name 1.3.10 pyhd8ed1ab_0 conda-forge
platformdirs 3.8.1 pyhd8ed1ab_0 conda-forge
plotly 5.15.0 pyhd8ed1ab_0 conda-forge
pooch 1.7.0 pyha770c72_3 conda-forge
prompt-toolkit 3.0.39 pyha770c72_0 conda-forge
prompt_toolkit 3.0.39 hd8ed1ab_0 conda-forge
protobuf 4.21.12 py38h8dc9893_0 conda-forge
psutil 5.9.5 py38h1de0b5d_0 conda-forge
pthread-stubs 0.4 h36c2ea0_1001 conda-forge
ptyprocess 0.7.0 pyhd3deb0d_0 conda-forge
pure_eval 0.2.2 pyhd8ed1ab_0 conda-forge
pyarrow 12.0.1 py38h39356d1_4_cpu conda-forge
pygments 2.15.1 pyhd8ed1ab_0 conda-forge
pynndescent 0.5.10 pyh1a96a4e_0 conda-forge
pyparsing 3.1.0 pyhd8ed1ab_0 conda-forge
pysocks 1.7.1 pyha2e5f31_6 conda-forge
python 3.8.17 he550d4f_0_cpython conda-forge
python-dateutil 2.8.2 pyhd8ed1ab_0 conda-forge
python-fastjsonschema 2.17.1 pyhd8ed1ab_0 conda-forge
python-tzdata 2023.3 pyhd8ed1ab_0 conda-forge
python-xxhash 3.2.0 py38h1de0b5d_0 conda-forge
python_abi 3.8 3_cp38 conda-forge
pytorch 2.0.0 cpu_py38h019455c_0 conda-forge
pytz 2023.3 pyhd8ed1ab_0 conda-forge
pyyaml 5.4.1 py38h0a891b7_4 conda-forge
pyzmq 25.1.0 py38h509eb50_0 conda-forge
re2 2023.03.02 h8c504da_0 conda-forge
readline 8.2 h8228510_1 conda-forge
referencing 0.29.1 pyhd8ed1ab_0 conda-forge
regex 2023.6.3 py38h01eb140_0 conda-forge
requests 2.31.0 pyhd8ed1ab_0 conda-forge
responses 0.18.0 pyhd8ed1ab_0 conda-forge
rpds-py 0.8.10 py38h0cc4f7c_0 conda-forge
s2n 1.3.46 h06160fa_0 conda-forge
sacremoses 0.0.53 pyhd8ed1ab_0 conda-forge
safetensors 0.3.1 py38h0cc4f7c_0 conda-forge
scikit-learn 1.3.0 py38hc099248_0 conda-forge
scipy 1.10.1 py38h59b608b_3 conda-forge
seaborn 0.12.2 hd8ed1ab_0 conda-forge
seaborn-base 0.12.2 pyhd8ed1ab_0 conda-forge
sentence-transformers 2.2.2 pyhd8ed1ab_0 conda-forge
sentencepiece 0.1.99 h578d9bd_0 conda-forge
sentencepiece-python 0.1.99 py38h19bc81d_0 conda-forge
sentencepiece-spm 0.1.99 h180e1df_0 conda-forge
sentry-sdk 1.28.1 pyhd8ed1ab_0 conda-forge
setproctitle 1.3.2 py38h0a891b7_1 conda-forge
setuptools 68.0.0 pyhd8ed1ab_0 conda-forge
six 1.16.0 pyh6c4a22f_0 conda-forge
sleef 3.5.1 h9b69904_2 conda-forge
smart_open 6.3.0 pyhd8ed1ab_1 conda-forge
smmap 3.0.5 pyh44b312d_0 conda-forge
snappy 1.1.10 h9fff704_0 conda-forge
stack_data 0.6.2 pyhd8ed1ab_0 conda-forge
statsmodels 0.14.0 py38h31356c5_1 conda-forge
sympy 1.12 pypyh9d50eac_103 conda-forge
tabulate 0.9.0 pyhd8ed1ab_1 conda-forge
tbb 2021.9.0 hf52228f_0 conda-forge
tenacity 8.2.2 pyhd8ed1ab_0 conda-forge
threadpoolctl 3.2.0 pyha21a80b_0 conda-forge
tk 8.6.12 h27826a3_0 conda-forge
tokenizers 0.13.3 py38haefcc50_0 conda-forge
top2vec 1.0.26 pyhd8ed1ab_0 conda-forge
torchvision 0.15.2 cpu_py38h08b8839_1 conda-forge
tornado 6.3.2 py38h01eb140_0 conda-forge
tqdm 4.65.0 pyhd8ed1ab_1 conda-forge
traitlets 5.9.0 pyhd8ed1ab_0 conda-forge
transformers 4.30.2 pyhd8ed1ab_1 conda-forge
types-pytz 2023.3.0.0 pyhd8ed1ab_0 conda-forge
typing-extensions 4.7.1 hd8ed1ab_0 conda-forge
typing_extensions 4.7.1 pyha770c72_0 conda-forge
ucx 1.14.1 h0aa22dc_2 conda-forge
umap-learn 0.5.3 py38h578d9bd_1 conda-forge
unicodedata2 15.0.0 py38h0a891b7_0 conda-forge
urllib3 2.0.3 pyhd8ed1ab_1 conda-forge
wandb 0.15.5 pyhd8ed1ab_0 conda-forge
wcwidth 0.2.6 pyhd8ed1ab_0 conda-forge
wheel 0.40.0 pyhd8ed1ab_0 conda-forge
wordcloud 1.9.2 py38h01eb140_1 conda-forge
xorg-libxau 1.0.11 hd590300_0 conda-forge
xorg-libxdmcp 1.1.3 h7f98852_0 conda-forge
xxhash 0.8.1 h0b41bf4_0 conda-forge
xz 5.2.6 h166bdaf_0 conda-forge
yaml 0.2.5 h7f98852_2 conda-forge
yarl 1.9.2 py38h01eb140_0 conda-forge
zeromq 4.3.4 h9c3ff4c_1 conda-forge
zipp 3.16.0 pyhd8ed1ab_1 conda-forge
zlib 1.2.13 hd590300_5 conda-forge
zstd 1.5.2 hfc55251_7 conda-forge
First I would recommend you to create an environment for example:
After that you can install the package using:
Now we can check the installed packages in the environment using
pip list:Now we can run the model without any error: