We have a legacy application written several years ago that seems to be locking up our server. Using VisualVM and FusionReaction (on a trial) we have narrowed it down to this particular application built with Fusebox. However, it isn't a particular action within the application that seems to trigger the hang, but rather any action, at random, after a period of time has elapsed. For some reason, after this thread locks up, all of the other threads seem to stack up behind it very quickly, waiting on something, and the whole server grinds to a halt very shortly.
I've got Fusion Reactor running, but it can't help (there are no slow queries or locks). I've got VisualVM which can at least give me the following thread dump from the locked thread. What is my next step? I found some documentation that indicated it may have been a bug in the JVM, but I went from 1.6 (came with CF) to 1.7.0_65 (Ubuntu apt-get) and it didn't fix the problem.
Thanks in advance.
"catalina-exec-3" - Thread t@176
java.lang.Thread.State: RUNNABLE
at coldfusion.compiler.ClassReader.skipFully(ClassReader.java:79)
at coldfusion.compiler.ClassReader.skipAttrs(ClassReader.java:274)
at coldfusion.compiler.ClassReader.skipMethods(ClassReader.java:284)
at coldfusion.compiler.ClassReader.getSourceModTime(ClassReader.java:196)
at coldfusion.runtime.TemplateClassLoader$TemplateCache$1.fetch(TemplateClassLoader.java:421)
at coldfusion.util.LruCache.get(LruCache.java:180)
at coldfusion.runtime.TemplateClassLoader$TemplateCache.fetchSerial(TemplateClassLoader.java:362)
at coldfusion.util.AbstractCache.fetch(AbstractCache.java:58)
at coldfusion.util.SoftCache.get_statsOff(SoftCache.java:133)
at coldfusion.util.SoftCache.get(SoftCache.java:81)
at coldfusion.runtime.TemplateClassLoader.findClass(TemplateClassLoader.java:609)
at coldfusion.runtime.RuntimeServiceImpl.getFile(RuntimeServiceImpl.java:828)
at coldfusion.runtime.RuntimeServiceImpl.resolveTemplatePath(RuntimeServiceImpl.java:809)
at coldfusion.tagext.lang.IncludeTag.setTemplate(IncludeTag.java:356)
at cffusebox402eruntime2ecfmx2ecfm73232117.runPage(/path/to/our/application/fusebox40.runtime.cfmx.cfm:203)
at coldfusion.runtime.CfJspPage.invoke(CfJspPage.java:244)
at coldfusion.tagext.lang.IncludeTag.doStartTag(IncludeTag.java:444)
at coldfusion.runtime.CfJspPage._emptyTcfTag(CfJspPage.java:2799)
at cfindex2ecfm2109747722.runPage(/path/to/our/application/index.cfm:1)
at coldfusion.runtime.CfJspPage.invoke(CfJspPage.java:244)
at coldfusion.tagext.lang.IncludeTag.doStartTag(IncludeTag.java:444)
at coldfusion.filter.CfincludeFilter.invoke(CfincludeFilter.java:65)
at coldfusion.filter.IpFilter.invoke(IpFilter.java:64)
at coldfusion.filter.ApplicationFilter.invoke(ApplicationFilter.java:449)
at coldfusion.filter.RequestMonitorFilter.invoke(RequestMonitorFilter.java:48)
at coldfusion.filter.MonitoringFilter.invoke(MonitoringFilter.java:40)
at coldfusion.filter.PathFilter.invoke(PathFilter.java:112)
at coldfusion.filter.ExceptionFilter.invoke(ExceptionFilter.java:94)
at coldfusion.filter.BrowserDebugFilter.invoke(BrowserDebugFilter.java:79)
at coldfusion.filter.ClientScopePersistenceFilter.invoke(ClientScopePersistenceFilter.java:28)
at coldfusion.filter.BrowserFilter.invoke(BrowserFilter.java:38)
at coldfusion.filter.NoCacheFilter.invoke(NoCacheFilter.java:58)
at coldfusion.filter.GlobalsFilter.invoke(GlobalsFilter.java:38)
at coldfusion.filter.DatasourceFilter.invoke(DatasourceFilter.java:22)
at coldfusion.filter.CachingFilter.invoke(CachingFilter.java:62)
at coldfusion.CfmServlet.service(CfmServlet.java:219)
at coldfusion.bootstrap.BootstrapServlet.service(BootstrapServlet.java:89)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:305)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210)
at coldfusion.monitor.event.MonitoringServletFilter.doFilter(MonitoringServletFilter.java:42)
at coldfusion.bootstrap.BootstrapFilter.doFilter(BootstrapFilter.java:46)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:243)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:210)
at sun.reflect.GeneratedMethodAccessor87.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at com.intergral.fusionreactor.j2ee.filterchain.WrappedFilterChain.doFilter(WrappedFilterChain.java:97)
at com.intergral.fusionreactor.j2ee.filter.FusionReactorRequestHandler.doNext(FusionReactorRequestHandler.java:472)
at com.intergral.fusionreactor.j2ee.filter.FusionReactorRequestHandler.doHttpServletRequest(FusionReactorRequestHandler.java:312)
at com.intergral.fusionreactor.j2ee.filter.FusionReactorRequestHandler.doFusionRequest(FusionReactorRequestHandler.java:192)
at com.intergral.fusionreactor.j2ee.filter.FusionReactorRequestHandler.handle(FusionReactorRequestHandler.java:507)
at com.intergral.fusionreactor.j2ee.filter.FusionReactorCoreFilter.doFilter(FusionReactorCoreFilter.java:36)
at sun.reflect.GeneratedMethodAccessor86.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at com.intergral.fusionreactor.j2ee.filterchain.WrappedFilterChain.doFilter(WrappedFilterChain.java:79)
at sun.reflect.GeneratedMethodAccessor85.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at com.intergral.fusionreactor.agent.filter.FusionReactorStaticFilter.doFilter(FusionReactorStaticFilter.java:53)
at com.intergral.fusionreactor.agent.pointcuts.NewFilterChainPointCut$1.invoke(NewFilterChainPointCut.java:41)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:224)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:169)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:472)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:168)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:98)
at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:928)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:118)
at org.apache.catalina.valves.RemoteIpValve.invoke(RemoteIpValve.java:680)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:414)
at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:987)
at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:539)
at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:298)
- locked <20d21902> (a org.apache.tomcat.util.net.SocketWrapper)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
Locked ownable synchronizers:
- locked <2b4f2aba> (a java.util.concurrent.ThreadPoolExecutor$Worker)