关于java.sql.SQLRecoverableException: Closed Connection异常的解决方案

在项目中碰到了一个应用异常，从表象来看应用僵死。查看Weblogic状态为Running,内存无溢出，但是出现多次线程堵塞。查看Weblogic日志，发现程序出现多次Time Out。

<Error> <WebLogicServer> <BEA-000337> <[STUCK] ExecuteThread: '1' for queue: 'weblogic.kernel.Default (self-tuning)' has been busy for "640" seconds working on the request "Workmanager: default, Version: 0, Scheduled=true, Started=true, Started time: 640083 ms
", which is more than the configured time (StuckThreadMaxTime) of "600" seconds. Stack trace:
java.net.SocketInputStream.socketRead0(Native Method)
java.net.SocketInputStream.read(SocketInputStream.java:129)
oracle.net.ns.Packet.receive(Packet.java:300)
oracle.net.ns.DataPacket.receive(DataPacket.java:106)
oracle.net.ns.NetInputStream.getNextPacket(NetInputStream.java:315)
oracle.net.ns.NetInputStream.read(NetInputStream.java:260)
oracle.net.ns.NetInputStream.read(NetInputStream.java:185)
oracle.net.ns.NetInputStream.read(NetInputStream.java:102)
oracle.jdbc.driver.T4CSocketInputStreamWrapper.readNextPacket(T4CSocketInputStreamWrapper.java:124)
oracle.jdbc.driver.T4CSocketInputStreamWrapper.read(T4CSocketInputStreamWrapper.java:80)
oracle.jdbc.driver.T4CMAREngine.unmarshalUB1(T4CMAREngine.java:1137)
oracle.jdbc.driver.T4CTTIfun.receive(T4CTTIfun.java:290)
oracle.jdbc.driver.T4CTTIfun.doRPC(T4CTTIfun.java:192)
oracle.jdbc.driver.T4C8Oall.doOALL(T4C8Oall.java:531)
oracle.jdbc.driver.T4CPreparedStatement.doOall8(T4CPreparedStatement.java:207)
oracle.jdbc.driver.T4CPreparedStatement.executeForDescribe(T4CPreparedStatement.java:884)
oracle.jdbc.driver.OracleStatement.executeMaybeDescribe(OracleStatement.java:1167)
oracle.jdbc.driver.OracleStatement.doExecuteWithTimeout(OracleStatement.java:1289)
oracle.jdbc.driver.OraclePreparedStatement.executeInternal(OraclePreparedStatement.java:3593)
oracle.jdbc.driver.OraclePreparedStatement.executeQuery(OraclePreparedStatement.java:3637)
oracle.jdbc.driver.OraclePreparedStatementWrapper.executeQuery(OraclePreparedStatementWrapper.java:1495)
com.trs.infra.persistent.BaseObjs.loadPage(BaseObjs.java:615)
com.trs.infra.persistent.BaseObjs.ensurePageLoaded(BaseObjs.java:552)
com.trs.infra.persistent.BaseObjs.getAt(BaseObjs.java:1500)
com.trs.infra.support.log.LogServer.initOperTypeMap(LogServer.java:570)
com.trs.infra.support.log.LogServer.getOperTypeAsInt(LogServer.java:105)
com.trs.infra.support.log.LogServer.save(LogServer.java:81)
com.trs.infra.support.log.LogServer.startRecord(LogServer.java:630)
com.trs.infra.support.log.LogServer.startRecord(LogServer.java:646)
com.trs.presentation.util.LoginHelper.login(LoginHelper.java:342)
jsp_servlet._app.__login_dowith._jspService(__login_dowith.java:446)
weblogic.servlet.jsp.JspBase.service(JspBase.java:34)
weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:242)
weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:216)
weblogic.servlet.internal.StubSecurityHelper.invokeServlet(StubSecurityHelper.java:132)
weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:338)
weblogic.servlet.internal.ServletStubImpl.onAddToMapException(ServletStubImpl.java:453)
weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:364)
weblogic.servlet.internal.TailFilter.doFilter(TailFilter.java:25)
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:74)
com.trs.deployer.domain.filter.DeployerControllerFilter.doFilter(DeployerControllerFilter.java:88)
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:74)
com.trs.servlet.AclFilter.doFilter(AclFilter.java:54)
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:74)
com.trs.servlet.LoginCheckFilter.doFilter(LoginCheckFilter.java:126)
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:74)
com.trs.cluster.ext.wcm.ClusterProxyFilter.doFilter(ClusterProxyFilter.java:65)
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:74)
com.trs.infra.session.util.SessionFilterBase.doFilter(SessionFilterBase.java:128)
weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:74)
weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.wrapRun(WebAppServletContext.java:3296)
weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.run(WebAppServletContext.java:3262)
weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:321)
weblogic.security.service.SecurityManager.runAs(SecurityManager.java:120)
weblogic.servlet.provider.WlsSubjectHandle.run(WlsSubjectHandle.java:57)
weblogic.servlet.internal.WebAppServletContext.doSecuredExecute(WebAppServletContext.java:2171)
weblogic.servlet.internal.WebAppServletContext.securedExecute(WebAppServletContext.java:2097)
weblogic.servlet.internal.WebAppServletContext.execute(WebAppServletContext.java:2075)
weblogic.servlet.internal.ServletRequestImpl.run(ServletRequestImpl.java:1514)
weblogic.servlet.provider.ContainerSupportProviderImpl$WlsRequestExecutor.run(ContainerSupportProviderImpl.java:254)
weblogic.work.ExecuteThread.execute(ExecuteThread.java:256)
weblogic.work.ExecuteThread.run(ExecuteThread.java:221)

我们知道，Weblogic会自动检测线程运行超时，当超过特点时间（默认600S），即认为此线程为堵塞线程。在日志中发现多次堵塞线程，通过查找资料，发现Weblogic在发生多次线程堵塞后，会自动把应用挂起。默认次数为15次。

是什么造成了线程堵塞呢？通过进一步分析日志，我们发现在线程堵塞之前，发生了多次java.sql.SQLRecoverableException: Closed Connection异常。异常情况：

java.sql.SQLRecoverableException: Closed Connection
at oracle.jdbc.driver.PhysicalConnection.needLine(PhysicalConnection.java:5389)
at oracle.jdbc.driver.OracleStatement.closeOrCache(OracleStatement.java:1578)
at oracle.jdbc.driver.OracleStatement.close(OracleStatement.java:1563)
at oracle.jdbc.driver.OracleStatementWrapper.close(OracleStatementWrapper.java:94)
at oracle.jdbc.driver.OraclePreparedStatementWrapper.close(OraclePreparedStatementWrapper.java:82)
at com.trs.infra.persistent.BaseObj.update(BaseObj.java:1733)
at com.trs.cms.content.CMSObj.update(CMSObj.java:402)
at com.trs.cms.auth.persistent.User.update(User.java:798)
at com.trs.cms.auth.persistent.User.login(User.java:890)
at com.trs.presentation.util.LoginHelper.login(LoginHelper.java:371)
at jsp_servlet._app.__login_dowith._jspService(__login_dowith.java:446)
at weblogic.servlet.jsp.JspBase.service(JspBase.java:34)
at weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:242)
at weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:216)
at weblogic.servlet.internal.StubSecurityHelper.invokeServlet(StubSecurityHelper.java:132)
at weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:338)
at weblogic.servlet.internal.TailFilter.doFilter(TailFilter.java:25)
at weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:74)
at com.trs.deployer.domain.filter.DeployerControllerFilter.doFilter(DeployerControllerFilter.java:88)
at weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:74)
at com.trs.servlet.AclFilter.doFilter(AclFilter.java:54)
at weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:74)
at com.trs.servlet.LoginCheckFilter.doFilter(LoginCheckFilter.java:126)
at weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:74)
at com.trs.cluster.ext.wcm.ClusterProxyFilter.doFilter(ClusterProxyFilter.java:65)
at weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:74)
at com.trs.infra.session.util.SessionFilterBase.doFilter(SessionFilterBase.java:128)
at weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:74)
at weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.wrapRun(WebAppServletContext.java:3296)
at weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.run(WebAppServletContext.java:3262)
at weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:321)
at weblogic.security.service.SecurityManager.runAs(SecurityManager.java:120)
at weblogic.servlet.provider.WlsSubjectHandle.run(WlsSubjectHandle.java:57)
at weblogic.servlet.internal.WebAppServletContext.doSecuredExecute(WebAppServletContext.java:2171)
at weblogic.servlet.internal.WebAppServletContext.securedExecute(WebAppServletContext.java:2097)
at weblogic.servlet.internal.WebAppServletContext.execute(WebAppServletContext.java:2075)
at weblogic.servlet.internal.ServletRequestImpl.run(ServletRequestImpl.java:1514)
at weblogic.servlet.provider.ContainerSupportProviderImpl$WlsRequestExecutor.run(ContainerSupportProviderImpl.java:254)
at weblogic.work.ExecuteThread.execute(ExecuteThread.java:256)
at weblogic.work.ExecuteThread.run(ExecuteThread.java:221)

从表现来看是数据库连接出了异常。我们对数据库和网络进行了分析，确定数据库和网络都无异常。我们的另外一个应用在Weblogic运行没有类似问题。

最后在Oracle的论坛上找到了问题的根结，由于我们的应用是自己开发的数据库连接池，应用和数据库之间有一层防火墙。防火墙策略是对于1800s未使用的Socket连接将自动关闭。Oracle的日志中也发现Socket异常关闭的异常。我们对应用进行了调整，当连接池中的连接15分钟不用时，自动回收，问题解决。