Hi team,
We have configured High Availability in ITPAM using Azure load balancers,
However the HA architecture is not working as expected.
When both nodes are up, the runbooks are getting in queued state.
Also, when primary orchestrator service is stopped, and secondary orchestrator is up and running, then all the runbooks get stuck in running state and below error logs are getting generated on Secondary orchestrator node.
******************************************************************************************************************************************************************
ERROR [com.optinuity.c2o.transport.Resolver] [ Session Task-1] Cound not get node with uuid=**************************** from domain due to exceptionConnection timed out: connect
java.net.ConnectException: Connection timed out: connect
at java.net.DualStackPlainSocketImpl.connect0(Native Method)
at java.net.DualStackPlainSocketImpl.socketConnect(DualStackPlainSocketImpl.java:79)
at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
at java.net.PlainSocketImpl.connect(PlainSocketImpl.java:172)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
at java.net.Socket.connect(Socket.java:579)
at sun.security.ssl.SSLSocketImpl.connect(SSLSocketImpl.java:625)
at sun.security.ssl.BaseSSLSocketImpl.connect(BaseSSLSocketImpl.java:160)
at sun.net.NetworkClient.doConnect(NetworkClient.java:180)
at sun.net.www.http.HttpClient.openServer(HttpClient.java:432)
at sun.net.www.http.HttpClient.openServer(HttpClient.java:527)
at sun.net.www.protocol.https.HttpsClient.<init>(HttpsClient.java:275)
at sun.net.www.protocol.https.HttpsClient.New(HttpsClient.java:371)
at sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.getNewHttpClient(AbstractDelegateHttpsURLConnection.java:191)
at sun.net.www.protocol.http.HttpURLConnection.plainConnect(HttpURLConnection.java:933)
at sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.connect(AbstractDelegateHttpsURLConnection.java:177)
at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1301)
at sun.net.www.protocol.https.HttpsURLConnectionImpl.getInputStream(HttpsURLConnectionImpl.java:254)
at com.optinuity.c2o.transport.Resolver.getNodeFromDomain(Resolver.java:1088)
at com.optinuity.c2o.transport.Resolver.refreshNodeFromDomain(Resolver.java:1121)
at com.optinuity.c2o.transport.Resolver.getNodeURL(Resolver.java:485)
at com.optinuity.c2o.transport.Resolver.getNodeURL(Resolver.java:411)
at com.optinuity.c2o.c2oserver.ServerManager.sendMessageToNode(ServerManager.java:5104)
at com.optinuity.c2o.c2oserver.ServerManager.deliverRequest(ServerManager.java:3948)
at com.optinuity.c2o.server.mdb.RequestListener.onMessage(RequestListener.java:124)
at org.apache.activemq.ActiveMQMessageConsumer.dispatch(ActiveMQMessageConsumer.java:1401)
at org.apache.activemq.ActiveMQSessionExecutor.dispatch(ActiveMQSessionExecutor.java:131)
at org.apache.activemq.ActiveMQSessionExecutor.iterate(ActiveMQSessionExecutor.java:202)
at org.apache.activemq.thread.PooledTaskRunner.runTask(PooledTaskRunner.java:133)
at org.apache.activemq.thread.PooledTaskRunner$1.run(PooledTaskRunner.java:48)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
2019-12-05 17:21:20,741 ERROR [com.optinuity.c2o.config.https.TransportProtoFactory] [ Session Task-1] Error occurred while certificate installation
java.net.ConnectException: Connection timed out: connect
at java.net.DualStackPlainSocketImpl.connect0(Native Method)
at java.net.DualStackPlainSocketImpl.socketConnect(DualStackPlainSocketImpl.java:79)
at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
at java.net.PlainSocketImpl.connect(PlainSocketImpl.java:172)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
at java.net.Socket.connect(Socket.java:579)
at sun.security.ssl.SSLSocketImpl.connect(SSLSocketImpl.java:625)
at sun.security.ssl.SSLSocketImpl.<init>(SSLSocketImpl.java:413)
at sun.security.ssl.SSLSocketFactoryImpl.createSocket(SSLSocketFactoryImpl.java:88)
at com.optinuity.c2o.util.https.InstallCert.getCertificate(InstallCert.java:143)
at com.optinuity.c2o.config.https.TransportProtoFactory.handleSecurity(TransportProtoFactory.java:32)
at com.optinuity.c2o.config.https.TransportProtoFactory.handleSecurityIfSecure(TransportProtoFactory.java:137)
at com.optinuity.c2o.transport.Resolver.getNodeFromDomain(Resolver.java:1086)
at com.optinuity.c2o.transport.Resolver.refreshNodeFromDomain(Resolver.java:1121)
at com.optinuity.c2o.transport.Resolver.getNodeURL(Resolver.java:485)
at com.optinuity.c2o.transport.Resolver.getNodeURL(Resolver.java:411)
at com.optinuity.c2o.c2oserver.ServerManager.sendMessageToNode(ServerManager.java:5104)
at com.optinuity.c2o.c2oserver.ServerManager.deliverRequest(ServerManager.java:3948)
at com.optinuity.c2o.server.mdb.RequestListener.onMessage(RequestListener.java:124)
at org.apache.activemq.ActiveMQMessageConsumer.dispatch(ActiveMQMessageConsumer.java:1401)
at org.apache.activemq.ActiveMQSessionExecutor.dispatch(ActiveMQSessionExecutor.java:131)
at org.apache.activemq.ActiveMQSessionExecutor.iterate(ActiveMQSessionExecutor.java:202)
at org.apache.activemq.thread.PooledTaskRunner.runTask(PooledTaskRunner.java:133)
at org.apache.activemq.thread.PooledTaskRunner$1.run(PooledTaskRunner.java:48)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
2019-12-05 17:21:21,747 ERROR [com.optinuity.c2o.transport.Resolver] [ Session Task-1] Cound not get node with uuid=************************ from domain due to exceptionConnection timed out: connect
java.net.ConnectException: Connection timed out: connect
at java.net.DualStackPlainSocketImpl.connect0(Native Method)
at java.net.DualStackPlainSocketImpl.socketConnect(DualStackPlainSocketImpl.java:79)
at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:339)
at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:200)
at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:182)
at java.net.PlainSocketImpl.connect(PlainSocketImpl.java:172)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
at java.net.Socket.connect(Socket.java:579)
at sun.security.ssl.SSLSocketImpl.connect(SSLSocketImpl.java:625)
at sun.security.ssl.BaseSSLSocketImpl.connect(BaseSSLSocketImpl.java:160)
at sun.net.NetworkClient.doConnect(NetworkClient.java:180)
at sun.net.www.http.HttpClient.openServer(HttpClient.java:432)
at sun.net.www.http.HttpClient.openServer(HttpClient.java:527)
at sun.net.www.protocol.https.HttpsClient.<init>(HttpsClient.java:275)
at sun.net.www.protocol.https.HttpsClient.New(HttpsClient.java:371)
at sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.getNewHttpClient(AbstractDelegateHttpsURLConnection.java:191)
at sun.net.www.protocol.http.HttpURLConnection.plainConnect(HttpURLConnection.java:933)
at sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.connect(AbstractDelegateHttpsURLConnection.java:177)
at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1301)
at sun.net.www.protocol.https.HttpsURLConnectionImpl.getInputStream(HttpsURLConnectionImpl.java:254)
at com.optinuity.c2o.transport.Resolver.getNodeFromDomain(Resolver.java:1088)
at com.optinuity.c2o.transport.Resolver.refreshNodeFromDomain(Resolver.java:1121)
at com.optinuity.c2o.transport.Resolver.getNodeURL(Resolver.java:485)
at com.optinuity.c2o.transport.Resolver.getNodeURL(Resolver.java:411)
at com.optinuity.c2o.c2oserver.ServerManager.sendMessageToNode(ServerManager.java:5104)
at com.optinuity.c2o.c2oserver.ServerManager.deliverRequest(ServerManager.java:3948)
at com.optinuity.c2o.server.mdb.RequestListener.onMessage(RequestListener.java:124)
at org.apache.activemq.ActiveMQMessageConsumer.dispatch(ActiveMQMessageConsumer.java:1401)
at org.apache.activemq.ActiveMQSessionExecutor.dispatch(ActiveMQSessionExecutor.java:131)
at org.apache.activemq.ActiveMQSessionExecutor.iterate(ActiveMQSessionExecutor.java:202)
at org.apache.activemq.thread.PooledTaskRunner.runTask(PooledTaskRunner.java:133)
at org.apache.activemq.thread.PooledTaskRunner$1.run(PooledTaskRunner.java:48)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
******************************************************************************************************************************************************************