Camunda engine fails to recover after loss of database connection

ENGINE-16004 Exception while closing command context: ; ### Error querying database. Cause: java.sql.SQLRecoverableException: Closed Connection;

2019-10-24 16:04:59.955 5bb01d51 error org.camunda.bpm.engine.context ENGINE-16004 Exception while closing command context: ; ### Error querying database. Cause: java.sql.SQLRecoverableException: No more data to read from socket; ### The error may exist in org/camunda/bpm/engine/impl/mapping/entity/Job.xml; ### The error may involve org.camunda.bpm.engine.impl.persistence.entity.JobEntity.selectNextJobsToExecute-Inline; ### The error occurred while setting parameters; ### SQL: select * from ( select a.*, ROWNUM rnum from ( select RES.* from ACT_RU_JOB RES where (RES.RETRIES_ > 0) and (RES.DUEDATE_ is null or RES.DUEDATE_ <= ?) and (RES.LOCK_OWNER_ is null or RES.LOCK_EXP_TIME_ < ?) and RES.SUSPENSION_STATE_ = 1 and ( ( RES.EXCLUSIVE_ = 1 and not exists( select J2.* from ACT_RU_JOB J2 where J2.PROCESS_INSTANCE_ID_ = RES.PROCESS_INSTANCE_ID_ -- from the same proc. inst. and (J2.EXCLUSIVE_ = 1) -- also exclusive and (J2.LOCK_OWNER_ is not null and J2.LOCK_EXP_TIME_ >= ?) -- in progress ) ) or RES.EXCLUSIVE_ = 0 ) ) a where ROWNUM < ?) where rnum >= ?; ### Cause: java.sql.SQLRecoverableException: No more data to read from socket process-engine BaseLogger.java:156 |-| {"context":{"isoTime":"2019-10-24T16:04:59.955+02:00","unixTime/s":1571925899.955,"thread":"5bb01d51","threadName":"Thread-4","level":"error","title":"org.camunda.bpm.engine.context","text":"ENGINE-16004 Exception while closing command context: \n### Error querying database. Cause: java.sql.SQLRecoverableException: No more data to read from socket\n### The error may exist in org/camunda/bpm/engine/impl/mapping/entity/Job.xml\n### The error may involve org.camunda.bpm.engine.impl.persistence.entity.JobEntity.selectNextJobsToExecute-Inline\n### The error occurred while setting parameters\n### SQL: select * from ( select a.*, ROWNUM rnum from ( select RES.* from ACT_RU_JOB RES where (RES.RETRIES_ > 0) and (RES.DUEDATE_ is null or RES.DUEDATE_ <= ?) and (RES.LOCK_OWNER_ is null or RES.LOCK_EXP_TIME_ < ?) and RES.SUSPENSION_STATE_ = 1 and ( ( RES.EXCLUSIVE_ = 1 and not exists( select J2.* from ACT_RU_JOB J2 where J2.PROCESS_INSTANCE_ID_ = RES.PROCESS_INSTANCE_ID_ -- from the same proc. inst. and (J2.EXCLUSIVE_ = 1) -- also exclusive and (J2.LOCK_OWNER_ is not null and J2.LOCK_EXP_TIME_ >= ?) -- in progress ) ) or RES.EXCLUSIVE_ = 0 ) ) a where ROWNUM < ?) where rnum >= ?\n### Cause: java.sql.SQLRecoverableException: No more data to read from socket","host":"lsaix02a","pid":"60315","component":"process-engine","componentType":"service","id":"process-engine","source":{"class":"org.camunda.commons.logging.BaseLogger","method":"logError","file":"BaseLogger.java","line":156},"tags":[],"map":{},"stacktrace":"org.apache.ibatis.exceptions.PersistenceException: \n### Error querying database. Cause: java.sql.SQLRecoverableException: No more data to read from socket\n### The error may exist in org/camunda/bpm/engine/impl/mapping/entity/Job.xml\n### The error may involve org.camunda.bpm.engine.impl.persistence.entity.JobEntity.selectNextJobsToExecute-Inline\n### The error occurred while setting parameters\n### SQL: select * from ( select a.*, ROWNUM rnum from ( select RES.* from ACT_RU_JOB RES where (RES.RETRIES_ > 0) and (RES.DUEDATE_ is null or RES.DUEDATE_ <= ?) and (RES.LOCK_OWNER_ is null or RES.LOCK_EXP_TIME_ < ?) and RES.SUSPENSION_STATE_ = 1 and ( ( RES.EXCLUSIVE_ = 1 and not exists( select J2.* from ACT_RU_JOB J2 where J2.PROCESS_INSTANCE_ID_ = RES.PROCESS_INSTANCE_ID_ -- from the same proc. inst. and (J2.EXCLUSIVE_ = 1) -- also exclusive and (J2.LOCK_OWNER_ is not null and J2.LOCK_EXP_TIME_ >= ?) -- in progress ) ) or RES.EXCLUSIVE_ = 0 ) ) a where ROWNUM < ?) where rnum >= ?\n### Cause: java.sql.SQLRecoverableException: No more data to read from socket\n\tat org.apache.ibatis.exceptions.ExceptionFactory.wrapException(ExceptionFactory.java:26)\n\tat org.apache.ibatis.session.defaults.DefaultSqlSession.selectList(DefaultSqlSession.java:111)\n\tat org.apache.ibatis.session.defaults.DefaultSqlSession.selectList(DefaultSqlSession.java:102)\n\tat org.camunda.bpm.engine.impl.db.sql.DbSqlSession.selectList(DbSqlSession.java:88)\n\tat org.camunda.bpm.engine.impl.db.entitymanager.DbEntityManager.selectListWithRawParameter(DbEntityManager.java:168)\n\tat org.camunda.bpm.engine.impl.db.entitymanager.DbEntityManager.selectList(DbEntityManager.java:160)\n\tat org.camunda.bpm.engine.impl.db.entitymanager.DbEntityManager.selectList(DbEntityManager.java:156)\n\tat org.camunda.bpm.engine.impl.db.entitymanager.DbEntityManager.selectList(DbEntityManager.java:145)\n\tat org.camunda.bpm.engine.impl.persistence.entity.JobManager.findNextJobsToExecute(JobManager.java:191)\n\tat org.camunda.bpm.engine.impl.cmd.AcquireJobsCmd.execute(AcquireJobsCmd.java:54)\n\tat org.camunda.bpm.engine.impl.cmd.AcquireJobsCmd.execute(AcquireJobsCmd.java:34)\n\tat org.camunda.bpm.engine.impl.interceptor.CommandExecutorImpl.execute(CommandExecutorImpl.java:24)\n\tat org.camunda.bpm.engine.impl.interceptor.CommandContextInterceptor.execute(CommandContextInterceptor.java:104)\n\tat org.camunda.bpm.engine.impl.interceptor.ProcessApplicationContextInterceptor.execute(ProcessApplicationContextInterceptor.java:66)\n\tat org.camunda.bpm.engine.impl.interceptor.LogInterceptor.execute(LogInterceptor.java:30)\n\tat org.camunda.bpm.engine.impl.jobexecutor.SequentialJobAcquisitionRunnable.acquireJobs(SequentialJobAcquisitionRunnable.java:142)\n\tat org.camunda.bpm.engine.impl.jobexecutor.SequentialJobAcquisitionRunnable.run(SequentialJobAcquisitionRunnable.java:60)\n\tat java.lang.Thread.run(Thread.java:748)\nCaused by: java.sql.SQLRecoverableException: No more data to read from socket\n\tat oracle.jdbc.driver.T4CMAREngine.unmarshalUB1(T4CMAREngine.java:1157)\n\tat oracle.jdbc.driver.T4CTTIfun.receive(T4CTTIfun.java:290)\n\tat oracle.jdbc.driver.T4CTTIfun.doRPC(T4CTTIfun.java:192)\n\tat oracle.jdbc.driver.T4C8Oall.doOALL(T4C8Oall.java:531)\n\tat oracle.jdbc.driver.T4CPreparedStatement.doOall8(T4CPreparedStatement.java:207)\n\tat oracle.jdbc.driver.T4CPreparedStatement.executeForDescribe(T4CPreparedStatement.java:884)\n\tat oracle.jdbc.driver.OracleStatement.executeMaybeDescribe(OracleStatement.java:1167)\n\tat oracle.jdbc.driver.OracleStatement.doExecuteWithTimeout(OracleStatement.java:1289)\n\tat oracle.jdbc.driver.OraclePreparedStatement.executeInternal(OraclePreparedStatement.java:3584)\n\tat oracle.jdbc.driver.OraclePreparedStatement.execute(OraclePreparedStatement.java:3685)\n\tat oracle.jdbc.driver.OraclePreparedStatementWrapper.execute(OraclePreparedStatementWrapper.java:1376)\n\tat sun.reflect.GeneratedMethodAccessor31.invoke(Unknown Source)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:498)\n\tat org.apache.tomcat.jdbc.pool.StatementFacade$StatementProxy.invoke(StatementFacade.java:138)\n\tat com.sun.proxy.$Proxy21.execute(Unknown Source)\n\tat org.apache.ibatis.executor.statement.PreparedStatementHandler.query(PreparedStatementHandler.java:59)\n\tat org.apache.ibatis.executor.statement.RoutingStatementHandler.query(RoutingStatementHandler.java:73)\n\tat org.apache.ibatis.executor.SimpleExecutor.doQuery(SimpleExecutor.java:60)\n\tat org.apache.ibatis.executor.BaseExecutor.queryFromDatabase(BaseExecutor.java:267)\n\tat org.apache.ibatis.executor.BaseExecutor.query(BaseExecutor.java:137)\n\tat org.apache.ibatis.executor.CachingExecutor.query(CachingExecutor.java:96)\n\tat org.apache.ibatis.executor.CachingExecutor.query(CachingExecutor.java:77)\n\tat org.apache.ibatis.session.defaults.DefaultSqlSession.selectList(DefaultSqlSession.java:108)\n\t... 16 more"}}
2019-10-24 16:04:59.955 5bb01d51 error org.camunda.bpm.engine.jobexecutor ENGINE-14019 Exception during job acquisition Process engine persistence exception process-engine BaseLogger.java:156 |-| {"context":{"isoTime":"2019-10-24T16:04:59.955+02:00","unixTime/s":1571925899.955,"thread":"5bb01d51","threadName":"Thread-4","level":"error","title":"org.camunda.bpm.engine.jobexecutor","text":"ENGINE-14019 Exception during job acquisition Process engine persistence exception","host":"lsaix02a","pid":"60315","component":"process-engine","componentType":"service","id":"process-engine","source":{"class":"org.camunda.commons.logging.BaseLogger","method":"logError","file":"BaseLogger.java","line":156},"tags":[],"map":{},"stacktrace":"org.camunda.bpm.engine.ProcessEngineException: Process engine persistence exception\n\tat org.camunda.bpm.engine.impl.interceptor.CommandInvocationContext.rethrow(CommandInvocationContext.java:148)\n\tat org.camunda.bpm.engine.impl.interceptor.CommandContext.close(CommandContext.java:173)\n\tat org.camunda.bpm.engine.impl.interceptor.CommandContextInterceptor.execute(CommandContextInterceptor.java:113)\n\tat org.camunda.bpm.engine.impl.interceptor.ProcessApplicationContextInterceptor.execute(ProcessApplicationContextInterceptor.java:66)\n\tat org.camunda.bpm.engine.impl.interceptor.LogInterceptor.execute(LogInterceptor.java:30)\n\tat org.camunda.bpm.engine.impl.jobexecutor.SequentialJobAcquisitionRunnable.acquireJobs(SequentialJobAcquisitionRunnable.java:142)\n\tat org.camunda.bpm.engine.impl.jobexecutor.SequentialJobAcquisitionRunnable.run(SequentialJobAcquisitionRunnable.java:60)\n\tat java.lang.Thread.run(Thread.java:748)\nCaused by: org.apache.ibatis.exceptions.PersistenceException: \n### Error querying database. Cause: java.sql.SQLRecoverableException: No more data to read from socket\n### The error may exist in org/camunda/bpm/engine/impl/mapping/entity/Job.xml\n### The error may involve org.camunda.bpm.engine.impl.persistence.entity.JobEntity.selectNextJobsToExecute-Inline\n### The error occurred while setting parameters\n### SQL: select * from ( select a.*, ROWNUM rnum from ( select RES.* from ACT_RU_JOB RES where (RES.RETRIES_ > 0) and (RES.DUEDATE_ is null or RES.DUEDATE_ <= ?) and (RES.LOCK_OWNER_ is null or RES.LOCK_EXP_TIME_ < ?) and RES.SUSPENSION_STATE_ = 1 and ( ( RES.EXCLUSIVE_ = 1 and not exists( select J2.* from ACT_RU_JOB J2 where J2.PROCESS_INSTANCE_ID_ = RES.PROCESS_INSTANCE_ID_ -- from the same proc. inst. and (J2.EXCLUSIVE_ = 1) -- also exclusive and (J2.LOCK_OWNER_ is not null and J2.LOCK_EXP_TIME_ >= ?) -- in progress ) ) or RES.EXCLUSIVE_ = 0 ) ) a where ROWNUM < ?) where rnum >= ?\n### Cause: java.sql.SQLRecoverableException: No more data to read from socket\n\tat org.apache.ibatis.exceptions.ExceptionFactory.wrapException(ExceptionFactory.java:26)\n\tat org.apache.ibatis.session.defaults.DefaultSqlSession.selectList(DefaultSqlSession.java:111)\n\tat org.apache.ibatis.session.defaults.DefaultSqlSession.selectList(DefaultSqlSession.java:102)\n\tat