I am trying to sync data between Database and Salesforce. For syncing data from SF to on premise DB I am using a REST API which fetches data from SF and calls a DSS service to update data on DB. Similiar set up for syncing data from DB to SF with the help of API and DSS services. I have a cluster set up and see the below error after certain amount of time has passed, if the load is higher, around 1000 records, we see this occurring within 10-15 mins, on a lower load we see the error in an hour or two. I am not using any cache mediator in any of the flows so not entirely sure what is throwing this error. Using EI 6.6.0
TID: [-1234] [] [2023-10-17 07:49:38,916] WARN {org.wso2.carbon.core.clustering.hazelcast.HazelcastDistributedMapProvider} - Cache lookup failed. Falling back to normal path. com.hazelcast.nio.serialization.HazelcastSerializationException: java.lang.ClassNotFoundException: org.wso2.carbon.caching.impl.**CacheEntry cannot be found by axiom_1.2.11.wso2v16**
at com.hazelcast.internal.serialization.impl.JavaDefaultSerializers$JavaSerializer.read(JavaDefaultSerializers.java:88)
at com.hazelcast.internal.serialization.impl.JavaDefaultSerializers$JavaSerializer.read(JavaDefaultSerializers.java:77)
at com.hazelcast.internal.serialization.impl.StreamSerializerAdapter.read(StreamSerializerAdapter.java:48)
at com.hazelcast.internal.serialization.impl.AbstractSerializationService.toObject(AbstractSerializationService.java:187)
at com.hazelcast.map.impl.proxy.MapProxySupport.toObject(MapProxySupport.java:1234)
at com.hazelcast.map.impl.proxy.MapProxyImpl.remove(MapProxyImpl.java:268)
at org.wso2.carbon.core.clustering.hazelcast.HazelcastDistributedMapProvider$DistMap.remove(HazelcastDistributedMapProvider.java:194)
at org.wso2.carbon.caching.impl.CacheImpl.removeLocal(CacheImpl.java:542)
at org.wso2.carbon.caching.impl.CacheImpl.remove(CacheImpl.java:521)
at org.wso2.carbon.registry.core.caching.CachingHandler.removeFromCache(CachingHandler.java:278)
at org.wso2.carbon.registry.core.caching.CachingHandler.clearCache(CachingHandler.java:192)
at org.wso2.carbon.registry.core.caching.CachingHandler.clearCache(CachingHandler.java:98)
at org.wso2.carbon.registry.core.caching.CachingHandler.put(CachingHandler.java:337)
at org.wso2.carbon.registry.core.jdbc.handlers.HandlerManager.put(HandlerManager.java:2505)
at org.wso2.carbon.registry.core.jdbc.handlers.HandlerLifecycleManager.put(HandlerLifecycleManager.java:946)
at org.wso2.carbon.registry.core.jdbc.EmbeddedRegistry.put(EmbeddedRegistry.java:705)
at org.wso2.carbon.registry.core.caching.CacheBackedRegistry.put(CacheBackedRegistry.java:591)
at org.wso2.carbon.registry.core.session.UserRegistry.putInternal(UserRegistry.java:828)
at org.wso2.carbon.registry.core.session.UserRegistry.access$1000(UserRegistry.java:61)
at org.wso2.carbon.registry.core.session.UserRegistry$11.run(UserRegistry.java:804)
at org.wso2.carbon.registry.core.session.UserRegistry$11.run(UserRegistry.java:801)
at java.security.AccessController.doPrivileged(Native Method)
at org.wso2.carbon.registry.core.session.UserRegistry.put(UserRegistry.java:801)
at org.wso2.carbon.mediation.registry.WSO2Registry.updateResource(WSO2Registry.java:402)
at sun.reflect.GeneratedMethodAccessor98.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.mozilla.javascript.MemberBox.invoke(MemberBox.java:126)
at org.mozilla.javascript.NativeJavaMethod.call(NativeJavaMethod.java:225)
at org.mozilla.javascript.optimizer.OptRuntime.call2(OptRuntime.java:42)
at org.mozilla.javascript.gen._Unknown_source__5236._c_updateRegistryTimestamp_1(<Unknown source>:11)
at org.mozilla.javascript.gen._Unknown_source__5236.call(<Unknown source>)
at org.mozilla.javascript.ContextFactory.doTopCall(ContextFactory.java:394)
at org.mozilla.javascript.ScriptRuntime.doTopCall(ScriptRuntime.java:3091)
at org.mozilla.javascript.gen._Unknown_source__5236.call(<Unknown source>)
at com.sun.phobos.script.javascript.RhinoScriptEngine.invokeMethod(RhinoScriptEngine.java:245)
at com.sun.phobos.script.javascript.RhinoScriptEngine.invokeFunction(RhinoScriptEngine.java:216)
at org.apache.synapse.mediators.bsf.ScriptMediator.mediateWithExternalScript(ScriptMediator.java:344)
at org.apache.synapse.mediators.bsf.ScriptMediator.invokeScript(ScriptMediator.java:288)
at org.apache.synapse.mediators.bsf.ScriptMediator.mediate(ScriptMediator.java:258)
at org.apache.synapse.mediators.AbstractListMediator.mediate(AbstractListMediator.java:109)
at org.apache.synapse.mediators.AbstractListMediator.mediate(AbstractListMediator.java:71)
at org.apache.synapse.mediators.filters.FilterMediator.mediate(FilterMediator.java:171)
at org.apache.synapse.mediators.AbstractListMediator.mediate(AbstractListMediator.java:109)
at org.apache.synapse.mediators.AbstractListMediator.mediate(AbstractListMediator.java:71)
at org.apache.synapse.mediators.base.SequenceMediator.mediate(SequenceMediator.java:158)
at org.apache.synapse.mediators.eip.Target.mediateMessage(Target.java:255)
at org.apache.synapse.mediators.eip.Target.mediate(Target.java:110)
at org.apache.synapse.mediators.eip.splitter.IterateMediator.mediate(IterateMediator.java:260)
at org.apache.synapse.mediators.AbstractListMediator.mediate(AbstractListMediator.java:109)
at org.apache.synapse.mediators.AbstractListMediator.mediate(AbstractListMediator.java:71)
at org.apache.synapse.mediators.base.SequenceMediator.mediate(SequenceMediator.java:158)
at org.apache.synapse.mediators.MediatorWorker.run(MediatorWorker.java:86)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.ClassNotFoundException: org.wso2.carbon.caching.impl.CacheEntry cannot be found by axiom_1.2.11.wso2v16
at org.eclipse.osgi.internal.loader.BundleLoader.findClassInternal(BundleLoader.java:512)
at org.eclipse.osgi.internal.loader.BundleLoader.findClass(BundleLoader.java:423)
at org.eclipse.osgi.internal.loader.BundleLoader.findClass(BundleLoader.java:415)
at org.eclipse.osgi.internal.loader.ModuleClassLoader.loadClass(ModuleClassLoader.java:155)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
at com.hazelcast.nio.ClassLoaderUtil.tryLoadClass(ClassLoaderUtil.java:288)
at com.hazelcast.nio.ClassLoaderUtil.loadClass(ClassLoaderUtil.java:252)
at com.hazelcast.nio.IOUtil$ClassLoaderAwareObjectInputStream.resolveClass(IOUtil.java:699)
at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1868)
at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1751)
at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2042)
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1573)
at java.io.ObjectInputStream.readObject(ObjectInputStream.java:431)
at com.hazelcast.internal.serialization.impl.JavaDefaultSerializers$JavaSerializer.read(JavaDefaultSerializers.java:84)
... 55 more
we tried adding axiom_1.2.11.wso2v16 jar file