Hi everybody!
Keycloak fails to start in HA deployment when upgrading from 11.0.3 to 12.0.1.
Also discussed here
Logs on 12.0.1 version:
19:24:09,144 WARN [org.infinispan.topology.ClusterTopologyManagerImpl] (ServerService Thread Pool -- 61) ISPN000329: Unable to read rebalancing status from coordinator test-keycloak-1: java.util.concurrent.CompletionException: org.infinispan.remoting.RemoteException: ISPN000217: Received exception from test-keycloak-1, see cause for remote stack trace
at java.base/java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:331)
at java.base/java.util.concurrent.CompletableFuture.uniApplyNow(CompletableFuture.java:670)
at java.base/java.util.concurrent.CompletableFuture.uniApplyStage(CompletableFuture.java:658)
at java.base/java.util.concurrent.CompletableFuture.thenApply(CompletableFuture.java:2094)
at java.base/java.util.concurrent.CompletableFuture.thenApply(CompletableFuture.java:143)
at org.infinispan@11.0.4.Final//org.infinispan.topology.TopologyManagementHelper.executeOnCoordinator(TopologyManagementHelper.java:98)
at org.infinispan@11.0.4.Final//org.infinispan.topology.ClusterTopologyManagerImpl.fetchRebalancingStatusFromCoordinator(ClusterTopologyManagerImpl.java:163)
at org.infinispan@11.0.4.Final//org.infinispan.topology.ClusterTopologyManagerImpl.start(ClusterTopologyManagerImpl.java:154)
at org.infinispan@11.0.4.Final//org.infinispan.topology.CorePackageImpl$3.start(CorePackageImpl.java:74)
at org.infinispan@11.0.4.Final//org.infinispan.topology.CorePackageImpl$3.start(CorePackageImpl.java:58)
at org.infinispan@11.0.4.Final//org.infinispan.factories.impl.BasicComponentRegistryImpl.invokeStart(BasicComponentRegistryImpl.java:592)
at org.infinispan@11.0.4.Final//org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:583)
at org.infinispan@11.0.4.Final//org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:552)
at org.infinispan@11.0.4.Final//org.infinispan.factories.impl.BasicComponentRegistryImpl.access$700(BasicComponentRegistryImpl.java:30)
at org.infinispan@11.0.4.Final//org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:775)
at org.infinispan@11.0.4.Final//org.infinispan.factories.AbstractComponentRegistry.internalStart(AbstractComponentRegistry.java:341)
at org.infinispan@11.0.4.Final//org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:237)
at org.infinispan@11.0.4.Final//org.infinispan.manager.DefaultCacheManager.internalStart(DefaultCacheManager.java:746)
at org.infinispan@11.0.4.Final//org.infinispan.manager.DefaultCacheManager.start(DefaultCacheManager.java:717)
at org.jboss.as.clustering.infinispan@21.0.1.Final//org.jboss.as.clustering.infinispan.subsystem.CacheContainerServiceConfigurator.get(CacheContainerServiceConfigurator.java:126)
at org.jboss.as.clustering.infinispan@21.0.1.Final//org.jboss.as.clustering.infinispan.subsystem.CacheContainerServiceConfigurator.get(CacheContainerServiceConfigurator.java:77)
at org.wildfly.clustering.service@21.0.1.Final//org.wildfly.clustering.service.FunctionalService.start(FunctionalService.java:63)
at org.wildfly.clustering.service@21.0.1.Final//org.wildfly.clustering.service.AsyncServiceConfigurator$AsyncService.lambda$start$0(AsyncServiceConfigurator.java:117)
at org.jboss.threads@2.4.0.Final//org.jboss.threads.ContextClassLoaderSavingRunnable.run(ContextClassLoaderSavingRunnable.java:35)
at org.jboss.threads@2.4.0.Final//org.jboss.threads.EnhancedQueueExecutor.safeRun(EnhancedQueueExecutor.java:1990)
at org.jboss.threads@2.4.0.Final//org.jboss.threads.EnhancedQueueExecutor$ThreadBody.doRunTask(EnhancedQueueExecutor.java:1486)
at org.jboss.threads@2.4.0.Final//org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1348)
at java.base/java.lang.Thread.run(Thread.java:834)
at org.jboss.threads@2.4.0.Final//org.jboss.threads.JBossThread.run(JBossThread.java:513)
Caused by: org.infinispan.remoting.RemoteException: ISPN000217: Received exception from test-keycloak-1, see cause for remote stack trace
at org.infinispan@11.0.4.Final//org.infinispan.remoting.transport.ResponseCollectors.wrapRemoteException(ResponseCollectors.java:25)
at org.infinispan@11.0.4.Final//org.infinispan.remoting.transport.ValidSingleResponseCollector.withException(ValidSingleResponseCollector.java:37)
at org.infinispan@11.0.4.Final//org.infinispan.remoting.transport.ValidSingleResponseCollector.addResponse(ValidSingleResponseCollector.java:21)
at org.infinispan@11.0.4.Final//org.infinispan.remoting.transport.impl.SingleTargetRequest.addResponse(SingleTargetRequest.java:73)
at org.infinispan@11.0.4.Final//org.infinispan.remoting.transport.impl.SingleTargetRequest.onResponse(SingleTargetRequest.java:43)
at org.infinispan@11.0.4.Final//org.infinispan.remoting.transport.impl.RequestRepository.addResponse(RequestRepository.java:52)
at org.infinispan@11.0.4.Final//org.infinispan.remoting.transport.jgroups.JGroupsTransport.processResponse(JGroupsTransport.java:1402)
at org.infinispan@11.0.4.Final//org.infinispan.remoting.transport.jgroups.JGroupsTransport.processMessage(JGroupsTransport.java:1305)
at org.infinispan@11.0.4.Final//org.infinispan.remoting.transport.jgroups.JGroupsTransport.access$300(JGroupsTransport.java:131)
at org.infinispan@11.0.4.Final//org.infinispan.remoting.transport.jgroups.JGroupsTransport$ChannelCallbacks.up(JGroupsTransport.java:1445)
at org.jgroups@4.2.5.Final//org.jgroups.JChannel.up(JChannel.java:784)
at org.jgroups@4.2.5.Final//org.jgroups.fork.ForkProtocolStack.up(ForkProtocolStack.java:135)
at org.jgroups@4.2.5.Final//org.jgroups.stack.Protocol.up(Protocol.java:306)
at org.jgroups@4.2.5.Final//org.jgroups.protocols.FORK.up(FORK.java:142)
at org.jgroups@4.2.5.Final//org.jgroups.protocols.FRAG3.up(FRAG3.java:165)
at org.jgroups@4.2.5.Final//org.jgroups.protocols.FlowControl.up(FlowControl.java:343)
at org.jgroups@4.2.5.Final//org.jgroups.protocols.pbcast.GMS.up(GMS.java:868)
at org.jgroups@4.2.5.Final//org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:243)
at org.jgroups@4.2.5.Final//org.jgroups.protocols.UNICAST3.deliverMessage(UNICAST3.java:1049)
at org.jgroups@4.2.5.Final//org.jgroups.protocols.UNICAST3.addMessage(UNICAST3.java:772)
at org.jgroups@4.2.5.Final//org.jgroups.protocols.UNICAST3.handleDataReceived(UNICAST3.java:753)
at org.jgroups@4.2.5.Final//org.jgroups.protocols.UNICAST3.up(UNICAST3.java:405)
at org.jgroups@4.2.5.Final//org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:592)
at org.jgroups@4.2.5.Final//org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:132)
at org.jgroups@4.2.5.Final//org.jgroups.protocols.FailureDetection.up(FailureDetection.java:186)
at org.jgroups@4.2.5.Final//org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:254)
at org.jgroups@4.2.5.Final//org.jgroups.protocols.MERGE3.up(MERGE3.java:281)
at org.jgroups@4.2.5.Final//org.jgroups.protocols.Discovery.up(Discovery.java:300)
at org.jgroups@4.2.5.Final//org.jgroups.protocols.TP.passMessageUp(TP.java:1385)
at org.jgroups@4.2.5.Final//org.jgroups.util.SubmitToThreadPool$SingleMessageHandler.run(SubmitToThreadPool.java:87)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at org.jboss.as.clustering.common@21.0.1.Final//org.jboss.as.clustering.context.ContextReferenceExecutor.execute(ContextReferenceExecutor.java:49)
at org.jboss.as.clustering.common@21.0.1.Final//org.jboss.as.clustering.context.ContextualExecutor$1.run(ContextualExecutor.java:70)
at java.base/java.lang.Thread.run(Thread.java:834)
Caused by: org.infinispan.commons.CacheException: Unknown command id 90!
at org.infinispan@11.0.4.Final//org.infinispan.marshall.exts.ThrowableExternalizer.readObject(ThrowableExternalizer.java:181)
at org.infinispan@11.0.4.Final//org.infinispan.marshall.exts.ThrowableExternalizer.readObject(ThrowableExternalizer.java:42)
at org.infinispan@11.0.4.Final//org.infinispan.marshall.core.GlobalMarshaller.readWithExternalizer(GlobalMarshaller.java:728)
at org.infinispan@11.0.4.Final//org.infinispan.marshall.core.GlobalMarshaller.readNonNullableObject(GlobalMarshaller.java:709)
at org.infinispan@11.0.4.Final//org.infinispan.marshall.core.GlobalMarshaller.readNullableObject(GlobalMarshaller.java:358)
at org.infinispan@11.0.4.Final//org.infinispan.marshall.core.BytesObjectInput.readObject(BytesObjectInput.java:32)
at org.infinispan@11.0.4.Final//org.infinispan.remoting.responses.ExceptionResponse$Externalizer.readObject(ExceptionResponse.java:49)
at org.infinispan@11.0.4.Final//org.infinispan.remoting.responses.ExceptionResponse$Externalizer.readObject(ExceptionResponse.java:41)
at org.infinispan@11.0.4.Final//org.infinispan.marshall.core.GlobalMarshaller.readWithExternalizer(GlobalMarshaller.java:728)
at org.infinispan@11.0.4.Final//org.infinispan.marshall.core.GlobalMarshaller.readNonNullableObject(GlobalMarshaller.java:709)
at org.infinispan@11.0.4.Final//org.infinispan.marshall.core.GlobalMarshaller.readNullableObject(GlobalMarshaller.java:358)
at org.infinispan@11.0.4.Final//org.infinispan.marshall.core.GlobalMarshaller.objectFromObjectInput(GlobalMarshaller.java:192)
at org.infinispan@11.0.4.Final//org.infinispan.marshall.core.GlobalMarshaller.objectFromByteBuffer(GlobalMarshaller.java:221)
at org.infinispan@11.0.4.Final//org.infinispan.remoting.transport.jgroups.JGroupsTransport.processResponse(JGroupsTransport.java:1394)
... 28 more
19:24:29,294 ERROR [org.infinispan.CLUSTER] (thread-9,ejb,test-keycloak-2) ISPN000474: Error processing request 578275@test-keycloak-1: java.io.EOFException
at org.infinispan@11.0.4.Final//org.infinispan.marshall.core.AbstractBytesObjectInput.checkPosLength(AbstractBytesObjectInput.java:261)
at org.infinispan@11.0.4.Final//org.infinispan.marshall.core.AbstractBytesObjectInput.readInt(AbstractBytesObjectInput.java:142)
at org.infinispan@11.0.4.Final//org.infinispan.marshall.exts.ReplicableCommandExternalizer.readCommandParameters(ReplicableCommandExternalizer.java:116)
at org.infinispan@11.0.4.Final//org.infinispan.marshall.exts.ReplicableCommandExternalizer.readObject(ReplicableCommandExternalizer.java:103)
at org.infinispan@11.0.4.Final//org.infinispan.marshall.exts.ReplicableCommandExternalizer.readObject(ReplicableCommandExternalizer.java:65)
at org.infinispan@11.0.4.Final//org.infinispan.marshall.core.GlobalMarshaller.readWithExternalizer(GlobalMarshaller.java:728)
at org.infinispan@11.0.4.Final//org.infinispan.marshall.core.GlobalMarshaller.readNonNullableObject(GlobalMarshaller.java:709)
at org.infinispan@11.0.4.Final//org.infinispan.marshall.core.GlobalMarshaller.readNullableObject(GlobalMarshaller.java:358)
at org.infinispan@11.0.4.Final//org.infinispan.marshall.core.BytesObjectInput.readObject(BytesObjectInput.java:32)
at org.infinispan@11.0.4.Final//org.infinispan.commands.remote.SingleRpcCommand.readFrom(SingleRpcCommand.java:57)
at org.infinispan@11.0.4.Final//org.infinispan.marshall.exts.ReplicableCommandExternalizer.readCommandParameters(ReplicableCommandExternalizer.java:114)
at org.infinispan@11.0.4.Final//org.infinispan.marshall.exts.CacheRpcCommandExternalizer.readObject(CacheRpcCommandExternalizer.java:154)
at org.infinispan@11.0.4.Final//org.infinispan.marshall.exts.CacheRpcCommandExternalizer.readObject(CacheRpcCommandExternalizer.java:81)
at org.infinispan@11.0.4.Final//org.infinispan.marshall.core.GlobalMarshaller.readWithExternalizer(GlobalMarshaller.java:728)
at org.infinispan@11.0.4.Final//org.infinispan.marshall.core.GlobalMarshaller.readNonNullableObject(GlobalMarshaller.java:709)
at org.infinispan@11.0.4.Final//org.infinispan.marshall.core.GlobalMarshaller.readNullableObject(GlobalMarshaller.java:358)
at org.infinispan@11.0.4.Final//org.infinispan.marshall.core.GlobalMarshaller.objectFromObjectInput(GlobalMarshaller.java:192)
at org.infinispan@11.0.4.Final//org.infinispan.marshall.core.GlobalMarshaller.objectFromByteBuffer(GlobalMarshaller.java:221)
at org.infinispan@11.0.4.Final//org.infinispan.remoting.transport.jgroups.JGroupsTransport.processRequest(JGroupsTransport.java:1362)
at org.infinispan@11.0.4.Final//org.infinispan.remoting.transport.jgroups.JGroupsTransport.processMessage(JGroupsTransport.java:1302)
at org.infinispan@11.0.4.Final//org.infinispan.remoting.transport.jgroups.JGroupsTransport.access$300(JGroupsTransport.java:131)
at org.infinispan@11.0.4.Final//org.infinispan.remoting.transport.jgroups.JGroupsTransport$ChannelCallbacks.up(JGroupsTransport.java:1445)
at org.jgroups@4.2.5.Final//org.jgroups.JChannel.up(JChannel.java:784)
at org.jgroups@4.2.5.Final//org.jgroups.fork.ForkProtocolStack.up(ForkProtocolStack.java:135)
at org.jgroups@4.2.5.Final//org.jgroups.stack.Protocol.up(Protocol.java:306)
at org.jgroups@4.2.5.Final//org.jgroups.protocols.FORK.up(FORK.java:142)
at org.jgroups@4.2.5.Final//org.jgroups.protocols.FRAG3.up(FRAG3.java:165)
at org.jgroups@4.2.5.Final//org.jgroups.protocols.FlowControl.up(FlowControl.java:359)
at org.jgroups@4.2.5.Final//org.jgroups.protocols.pbcast.GMS.up(GMS.java:868)
at org.jgroups@4.2.5.Final//org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:243)
at org.jgroups@4.2.5.Final//org.jgroups.protocols.UNICAST3.up(UNICAST3.java:392)
at org.jgroups@4.2.5.Final//org.jgroups.protocols.pbcast.NAKACK2.deliver(NAKACK2.java:931)
at org.jgroups@4.2.5.Final//org.jgroups.protocols.pbcast.NAKACK2.handleMessage(NAKACK2.java:821)
at org.jgroups@4.2.5.Final//org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:602)
at org.jgroups@4.2.5.Final//org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:132)
at org.jgroups@4.2.5.Final//org.jgroups.protocols.FailureDetection.up(FailureDetection.java:186)
at org.jgroups@4.2.5.Final//org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:254)
at org.jgroups@4.2.5.Final//org.jgroups.protocols.MERGE3.up(MERGE3.java:281)
at org.jgroups@4.2.5.Final//org.jgroups.protocols.Discovery.up(Discovery.java:300)
at org.jgroups@4.2.5.Final//org.jgroups.protocols.TP.passMessageUp(TP.java:1385)
at org.jgroups@4.2.5.Final//org.jgroups.util.SubmitToThreadPool$SingleMessageHandler.run(SubmitToThreadPool.java:87)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at org.jboss.as.clustering.common@21.0.1.Final//org.jboss.as.clustering.context.ContextReferenceExecutor.execute(ContextReferenceExecutor.java:49)
at org.jboss.as.clustering.common@21.0.1.Final//org.jboss.as.clustering.context.ContextualExecutor$1.run(ContextualExecutor.java:70)
at java.base/java.lang.Thread.run(Thread.java:834)
Logs on 11.0.3 version:
20:10:37,487 ERROR [org.infinispan.CLUSTER] (thread-136,ejb,test-keycloak-1) ISPN000474: Error processing request 2373@test-keycloak-2: org.infinispan.commons.CacheException: Unknown command id 86!
at org.infinispan@10.1.8.Final//org.infinispan.commands.RemoteCommandsFactory.fromStream(RemoteCommandsFactory.java:214)
at org.infinispan@10.1.8.Final//org.infinispan.marshall.exts.ReplicableCommandExternalizer.readCommandHeader(ReplicableCommandExternalizer.java:99)
at org.infinispan@10.1.8.Final//org.infinispan.marshall.exts.ReplicableCommandExternalizer.readObject(ReplicableCommandExternalizer.java:91)
at org.infinispan@10.1.8.Final//org.infinispan.marshall.exts.ReplicableCommandExternalizer.readObject(ReplicableCommandExternalizer.java:54)
at org.infinispan@10.1.8.Final//org.infinispan.marshall.core.GlobalMarshaller.readWithExternalizer(GlobalMarshaller.java:728)
at org.infinispan@10.1.8.Final//org.infinispan.marshall.core.GlobalMarshaller.readNonNullableObject(GlobalMarshaller.java:709)
at org.infinispan@10.1.8.Final//org.infinispan.marshall.core.GlobalMarshaller.readNullableObject(GlobalMarshaller.java:358)
at org.infinispan@10.1.8.Final//org.infinispan.marshall.core.GlobalMarshaller.objectFromObjectInput(GlobalMarshaller.java:192)
at org.infinispan@10.1.8.Final//org.infinispan.marshall.core.GlobalMarshaller.objectFromByteBuffer(GlobalMarshaller.java:221)
at org.infinispan@10.1.8.Final//org.infinispan.remoting.transport.jgroups.JGroupsTransport.processRequest(JGroupsTransport.java:1395)
at org.infinispan@10.1.8.Final//org.infinispan.remoting.transport.jgroups.JGroupsTransport.processMessage(JGroupsTransport.java:1335)
at org.infinispan@10.1.8.Final//org.infinispan.remoting.transport.jgroups.JGroupsTransport.access$300(JGroupsTransport.java:132)
at org.infinispan@10.1.8.Final//org.infinispan.remoting.transport.jgroups.JGroupsTransport$ChannelCallbacks.up(JGroupsTransport.java:1487)
at org.jgroups@4.2.4.Final//org.jgroups.JChannel.up(JChannel.java:784)
at org.jgroups@4.2.4.Final//org.jgroups.fork.ForkProtocolStack.up(ForkProtocolStack.java:135)
at org.jgroups@4.2.4.Final//org.jgroups.stack.Protocol.up(Protocol.java:306)
at org.jgroups@4.2.4.Final//org.jgroups.protocols.FORK.up(FORK.java:142)
at org.jgroups@4.2.4.Final//org.jgroups.protocols.FRAG3.up(FRAG3.java:165)
at org.jgroups@4.2.4.Final//org.jgroups.protocols.FlowControl.up(FlowControl.java:351)
at org.jgroups@4.2.4.Final//org.jgroups.protocols.pbcast.GMS.up(GMS.java:868)
at org.jgroups@4.2.4.Final//org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:243)
at org.jgroups@4.2.4.Final//org.jgroups.protocols.UNICAST3.deliverMessage(UNICAST3.java:1049)
at org.jgroups@4.2.4.Final//org.jgroups.protocols.UNICAST3.addMessage(UNICAST3.java:772)
at org.jgroups@4.2.4.Final//org.jgroups.protocols.UNICAST3.handleDataReceived(UNICAST3.java:753)
at org.jgroups@4.2.4.Final//org.jgroups.protocols.UNICAST3.up(UNICAST3.java:405)
at org.jgroups@4.2.4.Final//org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:592)
at org.jgroups@4.2.4.Final//org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:132)
at org.jgroups@4.2.4.Final//org.jgroups.protocols.FailureDetection.up(FailureDetection.java:186)
at org.jgroups@4.2.4.Final//org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:254)
at org.jgroups@4.2.4.Final//org.jgroups.protocols.MERGE3.up(MERGE3.java:281)
at org.jgroups@4.2.4.Final//org.jgroups.protocols.Discovery.up(Discovery.java:300)
at org.jgroups@4.2.4.Final//org.jgroups.protocols.TP.passMessageUp(TP.java:1385)
at org.jgroups@4.2.4.Final//org.jgroups.util.SubmitToThreadPool$SingleMessageHandler.run(SubmitToThreadPool.java:87)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at org.jboss.as.clustering.common@20.0.1.Final//org.jboss.as.clustering.context.ContextReferenceExecutor.execute(ContextReferenceExecutor.java:49)
at org.jboss.as.clustering.common@20.0.1.Final//org.jboss.as.clustering.context.ContextualExecutor$1.run(ContextualExecutor.java:70)
at java.base/java.lang.Thread.run(Thread.java:834)