ActiveMQ timing out waiting for messages (ActiveMQ 5.16.2)

Question

We have an ActiveMQ configured with our Tomcat like so in context.xml of Tomcat:

<Resource name="jms/ConnectionFactory" auth="Container" type="org.apache.activemq.ActiveMQConnectionFactory" description="JMS Connection Factory" factory="org.apache.activemq.jndi.JNDIReferenceFactory" brokerURL="tcp://roapalosa67.gcsc.att.com:7270?wireFormat.maxInactivityDuration=7200000&amp;jms.prefetchPolicy.all=1" brokerName="LocalActiveMQBroker" useEmbeddedBroker="false"/>

Now in the Java code, I am creating the JMS producer like this:

protected void createMessagePublisher() throws ActionTypeException {

    try {

        // Create ConnectionFactory
        Context context = new InitialContext();
        Context environmentContext = (Context) context.lookup("java:comp/env");
        ActiveMQConnectionFactory activeMQConnectionFactory = (ActiveMQConnectionFactory) environmentContext.lookup("jms/ConnectionFactory");

        activeMQConnectionFactory.setTrustAllPackages(true);

        ActiveMQPrefetchPolicy activeMQPrefetchPolicy = new ActiveMQPrefetchPolicy();

        // Set the maximum number messages floating through ActiveMQ JMS (Setting it to 10,000 and Storage of 5,000 on top of that.)
        activeMQPrefetchPolicy.setAll(10000);
        activeMQPrefetchPolicy.setMaximumPendingMessageLimit(5000);

        activeMQConnectionFactory.setAlwaysSyncSend(true);
        activeMQConnectionFactory.setConsumerExpiryCheckEnabled(true);
        activeMQConnectionFactory.setMaxThreadPoolSize(1000);
        activeMQConnectionFactory.setPrefetchPolicy(activeMQPrefetchPolicy);
        activeMQConnectionFactory.setSendTimeout(180000);

        // Create connection
        Connection connection = activeMQConnectionFactory.createConnection();

        connection.start();

        // Create session and producer
        setSession(connection.createSession(false, Session.AUTO_ACKNOWLEDGE));

        String queueName = appStaticProperties.getProperty("jms.destination");

        Queue queue = getSession().createQueue(queueName);

        setProducer(getSession().createProducer(queue));
        setQueueConnection(connection);

        // Set Message "Time to Live" to the request timeout plus 10 minutes
        getProducer().setTimeToLive(getTimeout() + (10 * 60 * 1000L));

    } catch (Exception e) {

        getLogger().logException("Exception creating the JMS Message Publisher", e);

        String errorMessage = "Exception creating the JMS Message Publisher - " + e.getMessage();

        getLogger().error(errorMessage);

        throw new ActionTypeException(errorMessage);
    }
}

My subscriber code looks something like this:

public void createMessageSubscriberJms(String host, int port, String jmsDestination) throws JMSException, UnknownHostException, NamingException {

    String providerEndpoints = "tcp://" + host + ":" + port + "?wireFormat.maxInactivityDuration=7200000";

    Context context = new InitialContext();
    Context environmentContext = (Context) context.lookup("java:comp/env");
    ActiveMQConnectionFactory activeMQConnectionFactory = (ActiveMQConnectionFactory) environmentContext.lookup("jms/ConnectionFactory");

    ArrayList<String> trustedClasses = new ArrayList<>();

    trustedClasses.add("com.att.logicalprovisioning.attcommon.atc.shared.ATCMessage");

    activeMQConnectionFactory.setBrokerURL(providerEndpoints);
    activeMQConnectionFactory.setTrustAllPackages(true);

    setQueueConnection(activeMQConnectionFactory.createQueueConnection());

    getQueueConnection().setClientID(getName());

    setSession(getQueueConnection().createQueueSession(false, Session.AUTO_ACKNOWLEDGE));

    getSession().createQueue(jmsDestination);

    String selector = "JMSCorrelationID = '" + getActionRequest().getOriginId() + "_" + getActionRequest().getRequestId() + "'";

    setConsumer(getSession().createConsumer(getSession().createQueue(jmsDestination), selector));
    getConsumer().setMessageListener(new DefaultMessageListener(this));

    this.atcMessages = new ATCMessageQueue<ATCMessage>();

    getQueueConnection().start();
}

So it works fine when the load is low. Now we pass the messages like:

sendMessage(new ATCMessage(ATCMessage.SCOPE_ACTION, ATCMessage.CODE_OK, output));

And sendMessage is coded like this:

public void sendMessage(ATCMessage atcMessage) {

    if (getProducer() != null) {

        try {

            ObjectMessage objectMessage = getSession().createObjectMessage();

            objectMessage.setObject(atcMessage);

            objectMessage.setJMSCorrelationID(getRequestListName());

            getProducer().send(objectMessage);

        } catch (Exception e) {

            e.printStackTrace();

            getLogger().logException("Message Queue error sending message", e);
        }
    }
}

We are fetching the messages like this:

// Read the return messages 
for (ATCMessage message : requestListClient.waitForAllMessages()) {

    if (message.getScope() == ATCMessage.SCOPE_ACTION && message.getCode() == ATCMessage.CODE_OK) {
        returnConfig = message.getText();
    } else if (message.getScope() == ATCMessage.SCOPE_REQUEST_LOG && message.getCode() == ATCMessage.CODE_OK) {
        getLogger().logRelatedLog("Action log", message.getText());
    }
}

And the waitForAllMessages method looks something like this:

/**
 * Waits for all JMS Message from the ATC until request is complete
 *
 * @param waitTime time to wait for messages
 * @return List of new messages received
 * @throws InterruptedException
 */
public List<ATCMessage> waitForAllMessages(int waitTime) throws InterruptedException {

    List<ATCMessage> newMessages = new ArrayList<ATCMessage>(5);

    long endNow = System.currentTimeMillis() + (waitTime * 1000L);

    // keep looping until we get all messages and request is complete
    do {
        newMessages.addAll(waitForMessage(10 * 1000));
    } while (!isRequestComplete() && (System.currentTimeMillis() < endNow || waitTime == 0));

    // Here if no wait, just return new messages
    return newMessages;
}

And we do see an error like this:

java.lang.Object.wait(Native Method)
com.gtc.logicalprovisioning.atc.shared.RequestListClient.waitForMessage(RequestListClient.java:834)
com.gtc.logicalprovisioning.atc.shared.RequestListClient.waitForAllMessages(RequestListClient.java:871)
com.gtc.logicalprovisioning.atc.shared.RequestListClient.waitForAllMessages(RequestListClient.java:852)
com.gtc.logicalprovisioning.palconfig.base.ActionMonitor.runConfigBuildAction(ActionMonitor.java:121)
com.gtc.logicalprovisioning.palconfig.webservice.PALConfigWS.process(PALConfigWS.java:304)

So occasionally the ActiveMQ messages are getting stuck. And the Webservice is timing out waiting for the JMS messages to come in.

Any pointers would be helpful. This has got our whole team in baffled. The thing is, it happens for a few minutes and then its starts working fine again.

Note: This was very old code that we are using, we will be moving to JSONMessaging soon. Previously it used WebSphere's JMS tools. But since we got rid of WebSphere and moved to Tomcat. We moved to ActiveMQ and re-engineered the code to work with ActiveMQ.

Thread Dump:

Matched threads (matched to currently executing transactions):

/PALConfig/services/ProcessPalConfigSoapHttpPort view trace
Transaction type: Web
Transaction name: /PALConfig/services/ProcessPalConfigSoapHttpPort
Duration: 5,363,547.2 milliseconds
CPU time: 490.6 milliseconds
"https-jsse-nio2-8443-exec-52" #696
   java.lang.Thread.State: TIMED_WAITING
        at java.lang.Object.wait(Native Method)
        - waiting on com.gtt.logicalprovisioning.atc.shared.ATCMessageQueue@68dedef1
        at com.gtt.logicalprovisioning.atc.shared.RequestListClient.waitForMessage(RequestListClient.java:837)
        at com.gtt.logicalprovisioning.atc.shared.RequestListClient.waitForAllMessages(RequestListClient.java:874)
        at com.gtt.logicalprovisioning.atc.shared.RequestListClient.waitForAllMessages(RequestListClient.java:855)
        at com.gtt.logicalprovisioning.palconfig.base.ActionMonitor.runConfigBuildAction(ActionMonitor.java:121)
        at com.gtt.logicalprovisioning.palconfig.webservice.PALConfigWS.process(PALConfigWS.java:304)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179)
        at org.apache.cxf.jaxws.JAXWSMethodInvoker.performInvocation(JAXWSMethodInvoker.java:66)
        at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96)
        at org.apache.cxf.jaxws.AbstractJAXWSMethodInvoker.invoke(AbstractJAXWSMethodInvoker.java:232)
        at org.apache.cxf.jaxws.JAXWSMethodInvoker.invoke(JAXWSMethodInvoker.java:85)
        at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:74)
        at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59)
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.cxf.interceptor.ServiceInvokerInterceptor$2.run(ServiceInvokerInterceptor.java:126)
        - locked on org.apache.cxf.phase.PhaseInterceptorChain@8528fae
        at org.apache.cxf.workqueue.SynchronousExecutor.execute(SynchronousExecutor.java:37)
        at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:131)
        - locked on org.apache.cxf.phase.PhaseInterceptorChain@8528fae
        at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308)
        - locked on org.apache.cxf.phase.PhaseInterceptorChain@8528fae
        at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121)
        at org.apache.cxf.transport.http.AbstractHTTPDestination.invoke(AbstractHTTPDestination.java:267)
        at org.apache.cxf.transport.servlet.ServletController.invokeDestination(ServletController.java:234)
        at org.apache.cxf.transport.servlet.ServletController.invoke(ServletController.java:208)
        at org.apache.cxf.transport.servlet.ServletController.invoke(ServletController.java:160)
        at org.apache.cxf.transport.servlet.CXFNonSpringServlet.invoke(CXFNonSpringServlet.java:216)
        at org.apache.cxf.transport.servlet.AbstractHTTPServlet.handleRequest(AbstractHTTPServlet.java:301)
        at org.apache.cxf.transport.servlet.AbstractHTTPServlet.doPost(AbstractHTTPServlet.java:220)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:681)
        at org.apache.cxf.transport.servlet.AbstractHTTPServlet.service(AbstractHTTPServlet.java:276)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:228)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:163)
        at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:53)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:190)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:163)
        at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:202)
        at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:97)
        at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:542)
        at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:143)
        at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92)
        at org.apache.catalina.valves.AbstractAccessLogValve.invoke(AbstractAccessLogValve.java:687)
        at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:78)
        at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:357)
        at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:382)
        at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65)
        at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:893)
        at org.apache.tomcat.util.net.Nio2Endpoint$SocketProcessor.doRun(Nio2Endpoint.java:1685)
        at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
        - locked on org.apache.tomcat.util.net.Nio2Endpoint$Nio2SocketWrapper@47c90a37
        at org.apache.tomcat.util.net.AbstractEndpoint.processSocket(AbstractEndpoint.java:1171)
        at org.apache.tomcat.util.net.Nio2Endpoint$Nio2SocketWrapper$2.completed(Nio2Endpoint.java:613)
        at org.apache.tomcat.util.net.Nio2Endpoint$Nio2SocketWrapper$2.completed(Nio2Endpoint.java:591)
        at org.apache.tomcat.util.net.SecureNio2Channel$1.completed(SecureNio2Channel.java:1010)
        at org.apache.tomcat.util.net.SecureNio2Channel$1.completed(SecureNio2Channel.java:938)
        at sun.nio.ch.Invoker.invokeUnchecked(Invoker.java:126)
        at sun.nio.ch.Invoker$2.run(Invoker.java:218)
        at sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:112)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
        at java.lang.Thread.run(Thread.java:748)
"ActiveMQ Transport: tcp://some_hostname/some_ip:7270@43712" #6924
   java.lang.Thread.State: RUNNABLE
        at java.net.SocketInputStream.socketRead0(Native Method)
        at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
        at java.net.SocketInputStream.read(SocketInputStream.java:171)
        at java.net.SocketInputStream.read(SocketInputStream.java:141)
        at org.apache.activemq.transport.tcp.TcpBufferedInputStream.fill(TcpBufferedInputStream.java:50)
        at org.apache.activemq.transport.tcp.TcpTransport$2.fill(TcpTransport.java:634)
        at org.apache.activemq.transport.tcp.TcpBufferedInputStream.read(TcpBufferedInputStream.java:59)
        at org.apache.activemq.transport.tcp.TcpTransport$2.read(TcpTransport.java:619)
        at java.io.DataInputStream.readInt(DataInputStream.java:387)
        at org.apache.activemq.openwire.OpenWireFormat.unmarshal(OpenWireFormat.java:268)
        at org.apache.activemq.transport.tcp.TcpTransport.readCommand(TcpTransport.java:240)
        at org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:232)
        at org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:215)
        at java.lang.Thread.run(Thread.java:748)

Unmatched threads (this may possibly include currently executing transactions that just started or just ended):

"http-nio-135.89.97.166-9080-exec-16" #659
   java.lang.Thread.State: RUNNABLE
        at sun.management.ThreadImpl.getThreadAllocatedMemory1(Native Method)
        at sun.management.ThreadImpl.getThreadAllocatedBytes(ThreadImpl.java:353)
        at sun.management.ThreadImpl.getThreadAllocatedBytes(ThreadImpl.java:330)
        at sun.reflect.GeneratedMethodAccessor87.invoke()
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.glowroot.agent.util.ThreadAllocatedBytes.getThreadAllocatedBytesSafely(ThreadAllocatedBytes.java:91)
        at org.glowroot.agent.model.ThreadStatsComponent.<init>(ThreadStatsComponent.java:71)
        at org.glowroot.agent.impl.ThreadContextImpl.<init>(ThreadContextImpl.java:157)
        at org.glowroot.agent.impl.Transaction.<init>(Transaction.java:237)
        at org.glowroot.agent.impl.TransactionService.startTransaction(TransactionService.java:91)
        at org.glowroot.agent.impl.OptionalThreadContextImpl.startTransaction(OptionalThreadContextImpl.java:102)
        at org.glowroot.agent.impl.OptionalThreadContextImpl.startTransaction(OptionalThreadContextImpl.java:77)
        at org.glowroot.agent.plugin.servlet.ServletAspect$ServiceAdvice_.onBeforeCommon(ServletAspect.java:191)
        at org.glowroot.agent.plugin.servlet.ServletAspect$ServiceAdvice_.access$000(ServletAspect.java:64)
        at org.glowroot.agent.plugin.servlet.ServletAspect$DoFilterAdvice_.onBefore(ServletAspect.java:219)
        at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:190)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:163)
        at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:202)
        at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:97)
        at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:542)
        at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:143)
        at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92)
        at org.apache.catalina.valves.AbstractAccessLogValve.invoke(AbstractAccessLogValve.java:687)
        at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:78)
        at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:357)
        at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:382)
        at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65)
        at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:893)
        at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1723)
        at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
        - locked on org.apache.tomcat.util.net.NioEndpoint$NioSocketWrapper@292fed3f
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
        at java.lang.Thread.run(Thread.java:748)

----------------------------------------

"ActiveMQ Transport: tcp://some_hostname/some_ip:7270@46442" #7020
   java.lang.Thread.State: RUNNABLE
        at java.net.SocketInputStream.socketRead0(Native Method)
        at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
        at java.net.SocketInputStream.read(SocketInputStream.java:171)
        at java.net.SocketInputStream.read(SocketInputStream.java:141)
        at org.apache.activemq.transport.tcp.TcpBufferedInputStream.fill(TcpBufferedInputStream.java:50)
        at org.apache.activemq.transport.tcp.TcpTransport$2.fill(TcpTransport.java:634)
        at org.apache.activemq.transport.tcp.TcpBufferedInputStream.read(TcpBufferedInputStream.java:59)
        at org.apache.activemq.transport.tcp.TcpTransport$2.read(TcpTransport.java:619)
        at java.io.DataInputStream.readInt(DataInputStream.java:387)
        at org.apache.activemq.openwire.OpenWireFormat.unmarshal(OpenWireFormat.java:268)
        at org.apache.activemq.transport.tcp.TcpTransport.readCommand(TcpTransport.java:240)
        at org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:232)
        at org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:215)
        at java.lang.Thread.run(Thread.java:748)

----------------------------------------

"main" #1
   java.lang.Thread.State: RUNNABLE
        at java.net.PlainSocketImpl.socketAccept(Native Method)
        at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:535)
        at java.net.ServerSocket.implAccept(ServerSocket.java:545)
        at java.net.ServerSocket.accept(ServerSocket.java:513)
        at org.apache.catalina.core.StandardServer.await(StandardServer.java:609)
        at org.apache.catalina.startup.Catalina.await(Catalina.java:864)
        at org.apache.catalina.startup.Catalina.start(Catalina.java:810)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.catalina.startup.Bootstrap.start(Bootstrap.java:345)
        at org.apache.catalina.startup.Bootstrap.main(Bootstrap.java:476)

----------------------------------------

"https-jsse-nio2-8443-exec-4" #40
   java.lang.Thread.State: TIMED_WAITING
        at sun.misc.Unsafe.park(Native Method)
        - waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@72296732
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
        at org.apache.tomcat.util.threads.TaskQueue.poll(TaskQueue.java:102)
        at org.apache.tomcat.util.threads.TaskQueue.poll(TaskQueue.java:33)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
        at java.lang.Thread.run(Thread.java:748)

"https-jsse-nio2-8443-exec-6" #42
   java.lang.Thread.State: TIMED_WAITING
        at sun.misc.Unsafe.park(Native Method)
        - waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@72296732
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
        at org.apache.tomcat.util.threads.TaskQueue.poll(TaskQueue.java:102)
        at org.apache.tomcat.util.threads.TaskQueue.poll(TaskQueue.java:33)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
        at java.lang.Thread.run(Thread.java:748)

"http-nio-135.89.97.166-9080-exec-3" #612
   java.lang.Thread.State: TIMED_WAITING
        at sun.misc.Unsafe.park(Native Method)
        - waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@67e3425b
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
        at org.apache.tomcat.util.threads.TaskQueue.poll(TaskQueue.java:102)
        at org.apache.tomcat.util.threads.TaskQueue.poll(TaskQueue.java:33)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
        at java.lang.Thread.run(Thread.java:748)

"http-nio-135.89.97.166-9080-exec-8" #619
   java.lang.Thread.State: TIMED_WAITING
        at sun.misc.Unsafe.park(Native Method)
        - waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@67e3425b
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
        at org.apache.tomcat.util.threads.TaskQueue.poll(TaskQueue.java:102)
        at org.apache.tomcat.util.threads.TaskQueue.poll(TaskQueue.java:33)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
        at java.lang.Thread.run(Thread.java:748)

"http-nio-135.89.97.166-9080-exec-10" #621
   java.lang.Thread.State: TIMED_WAITING
        at sun.misc.Unsafe.park(Native Method)
        - waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@67e3425b
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
        at org.apache.tomcat.util.threads.TaskQueue.poll(TaskQueue.java:102)
        at org.apache.tomcat.util.threads.TaskQueue.poll(TaskQueue.java:33)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
        at java.lang.Thread.run(Thread.java:748)

"https-jsse-nio2-8443-exec-16" #627
   java.lang.Thread.State: TIMED_WAITING
        at sun.misc.Unsafe.park(Native Method)
        - waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@72296732
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
        at org.apache.tomcat.util.threads.TaskQueue.poll(TaskQueue.java:102)
        at org.apache.tomcat.util.threads.TaskQueue.poll(TaskQueue.java:33)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
        at java.lang.Thread.run(Thread.java:748)