Infinispan Timeout

Dear All,

We are experiencing a problem with Keycloak communicating with external Infinispan.

The following log shows the error we are facing:

2024-08-29 13:13:53 2024-08-29 10:13:53,137 INFO  [org.infinispan.CONTAINER] (keycloak-cache-init) ISPN000556: Starting user marshaller 'org.infinispan.jboss.marshalling.core.JBossUserMarshaller'
2024-08-29 13:13:53 2024-08-29 10:13:53,339 INFO  [org.keycloak.broker.provider.AbstractIdentityProviderMapper] (main) Registering class org.keycloak.broker.provider.mappersync.ConfigSyncEventListener
2024-08-29 13:13:53 2024-08-29 10:13:53,463 INFO  [org.infinispan.CLUSTER] (keycloak-cache-init) ISPN000078: Starting JGroups channel `ISPN`
2024-08-29 13:13:53 2024-08-29 10:13:53,465 INFO  [org.jgroups.JChannel] (keycloak-cache-init) local_addr: cc49b35e-87f5-413d-8943-aebde564df27, name: authentication-application-3069
2024-08-29 13:13:53 2024-08-29 10:13:53,471 INFO  [org.jgroups.protocols.FD_SOCK2] (keycloak-cache-init) server listening on *.57800
2024-08-29 13:13:53 2024-08-29 10:13:53,537 INFO  [org.infinispan.CLUSTER] (keycloak-cache-init) ISPN000094: Received new cluster view for channel ISPN: [keycloak-infinispan-42737|1] (2) [keycloak-infinispan-42737, authentication-application-3069]
...
2024-08-29 13:13:53 2024-08-29 10:13:53,940 INFO  [org.infinispan.LIFECYCLE] (jgroups-7,authentication-application-3069) [Context=work] ISPN100002: Starting rebalance with members [keycloak-infinispan-42737, authentication-application-3069], phase READ_OLD_WRITE_ALL, topology id 2
2024-08-29 13:13:53 2024-08-29 10:13:53,942 INFO  [org.infinispan.LIFECYCLE] (non-blocking-thread--p2-t1) [Context=work] ISPN100010: Finished rebalance with members [keycloak-infinispan-42737, authentication-application-3069], topology id 2
2024-08-29 13:13:54 2024-08-29 10:13:54,003 INFO  [org.keycloak.connections.infinispan.DefaultInfinispanConnectionProviderFactory] (main) Node name: authentication-application-3069, Site name: null
2024-08-29 13:13:54 2024-08-29 10:13:54,547 INFO  [io.quarkus] (main) Keycloak 21.1.2 on JVM (powered by Quarkus 2.13.8.Final) started in 3.424s. Listening on: http://0.0.0.0:8080
2024-08-29 13:13:54 2024-08-29 10:13:54,548 INFO  [io.quarkus] (main) Profile prod activated. 
2024-08-29 13:13:54 2024-08-29 10:13:54,548 INFO  [io.quarkus] (main) Installed features: [agroal, cdi, hibernate-orm, jdbc-h2, jdbc-mariadb, jdbc-mssql, jdbc-mysql, jdbc-oracle, jdbc-postgresql, keycloak, logging-gelf, micrometer, narayana-jta, reactive-routes, resteasy, resteasy-jackson, smallrye-context-propagation, smallrye-health, vertx]
2024-08-29 13:13:54 2024-08-29 10:13:54,560 ERROR [org.keycloak.services] (main) KC-SERVICES0010: Failed to add user 'admin' to realm 'master': user with username exists
2024-08-29 13:15:12 2024-08-29 10:15:12,049 ERROR [org.infinispan.interceptors.impl.InvocationContextInterceptor] (timeout-thread--p4-t1) ISPN000136: Error executing command RemoveCommand on Cache 'authenticationSessions', writing keys [1f10a011-125e-4691-bec8-4cd2b4909b56]: org.infinispan.util.concurrent.TimeoutException: ISPN000427: Timeout after 15 seconds waiting for acks ([keycloak-infinispan-42737]). Id=50
2024-08-29 13:15:12     at org.infinispan.util.concurrent.CommandAckCollector.createTimeoutException(CommandAckCollector.java:228)
2024-08-29 13:15:12     at org.infinispan.util.concurrent.CommandAckCollector$BaseCollector.call(CommandAckCollector.java:281)
2024-08-29 13:15:12     at org.infinispan.util.concurrent.CommandAckCollector$BaseCollector.call(CommandAckCollector.java:261)
2024-08-29 13:15:12     at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
2024-08-29 13:15:12     at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
2024-08-29 13:15:12     at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
2024-08-29 13:15:12     at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
2024-08-29 13:15:12     at java.base/java.lang.Thread.run(Thread.java:833)
2024-08-29 13:15:12 
2024-08-29 13:15:27 2024-08-29 10:15:27,058 ERROR [org.infinispan.interceptors.impl.InvocationContextInterceptor] (timeout-thread--p4-t1) ISPN000136: Error executing command PutKeyValueCommand on Cache 'sessions', writing keys [1f10a011-125e-4691-bec8-4cd2b4909b56]: org.infinispan.util.concurrent.TimeoutException: ISPN000427: Timeout after 15 seconds waiting for acks ([keycloak-infinispan-42737]). Id=51
2024-08-29 13:15:27     at org.infinispan.util.concurrent.CommandAckCollector.createTimeoutException(CommandAckCollector.java:228)
2024-08-29 13:15:27     at org.infinispan.util.concurrent.CommandAckCollector$BaseCollector.call(CommandAckCollector.java:281)
2024-08-29 13:15:27     at org.infinispan.util.concurrent.CommandAckCollector$BaseCollector.call(CommandAckCollector.java:261)
2024-08-29 13:15:27     at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
2024-08-29 13:15:27     at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
2024-08-29 13:15:27     at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
2024-08-29 13:15:27     at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
2024-08-29 13:15:27     at java.base/java.lang.Thread.run(Thread.java:833)
2024-08-29 13:15:27 
2024-08-29 13:15:42 2024-08-29 10:15:42,063 ERROR [org.infinispan.interceptors.impl.InvocationContextInterceptor] (timeout-thread--p4-t1) ISPN000136: Error executing command PutKeyValueCommand on Cache 'clientSessions', writing keys [22a52165-ad0c-4f83-9d84-602c282233bd]: org.infinispan.util.concurrent.TimeoutException: ISPN000427: Timeout after 15 seconds waiting for acks ([authentication-application-3069]). Id=52
2024-08-29 13:15:42     at org.infinispan.util.concurrent.CommandAckCollector.createTimeoutException(CommandAckCollector.java:228)
2024-08-29 13:15:42     at org.infinispan.util.concurrent.CommandAckCollector$BaseCollector.call(CommandAckCollector.java:281)
2024-08-29 13:15:42     at org.infinispan.util.concurrent.CommandAckCollector$BaseCollector.call(CommandAckCollector.java:261)
2024-08-29 13:15:42     at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
2024-08-29 13:15:42     at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
2024-08-29 13:15:42     at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
2024-08-29 13:15:42     at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
2024-08-29 13:15:42     at java.base/java.lang.Thread.run(Thread.java:833)
2024-08-29 13:15:42 
2024-08-29 13:15:42 2024-08-29 10:15:42,073 ERROR [org.keycloak.services.error.KeycloakErrorHandler] (executor-thread-0) Uncaught server error: org.infinispan.util.concurrent.TimeoutException: ISPN000427: Timeout after 15 seconds waiting for acks ([keycloak-infinispan-42737]). Id=50
2024-08-29 13:15:42     at org.infinispan.interceptors.impl.AsyncInterceptorChainImpl.invoke(AsyncInterceptorChainImpl.java:258)
2024-08-29 13:15:42     at org.infinispan.cache.impl.InvocationHelper.doInvoke(InvocationHelper.java:323)
2024-08-29 13:15:42     at org.infinispan.cache.impl.InvocationHelper.invoke(InvocationHelper.java:111)
2024-08-29 13:15:42     at org.infinispan.cache.impl.InvocationHelper.invoke(InvocationHelper.java:93)
2024-08-29 13:15:42     at org.infinispan.cache.impl.CacheImpl.remove(CacheImpl.java:690)
2024-08-29 13:15:42     at org.infinispan.cache.impl.DecoratedCache.remove(DecoratedCache.java:557)
2024-08-29 13:15:42     at org.infinispan.cache.impl.EncoderCache.remove(EncoderCache.java:723)
2024-08-29 13:15:42     at org.keycloak.models.sessions.infinispan.InfinispanKeycloakTransaction$5.execute(InfinispanKeycloakTransaction.java:194)
2024-08-29 13:15:42     at java.base/java.util.LinkedHashMap$LinkedValues.forEach(LinkedHashMap.java:647)
2024-08-29 13:15:42     at org.keycloak.models.sessions.infinispan.InfinispanKeycloakTransaction.commit(InfinispanKeycloakTransaction.java:58)
2024-08-29 13:15:42     at org.keycloak.services.DefaultKeycloakTransactionManager.commit(DefaultKeycloakTransactionManager.java:146)
2024-08-29 13:15:42     at org.keycloak.services.DefaultKeycloakSession.closeTransactionManager(DefaultKeycloakSession.java:502)
2024-08-29 13:15:42     at org.keycloak.services.DefaultKeycloakSession.close(DefaultKeycloakSession.java:469)
2024-08-29 13:15:42     at org.keycloak.models.utils.KeycloakModelUtils.runJobInRetriableTransaction(KeycloakModelUtils.java:300)
2024-08-29 13:15:42     at org.keycloak.protocol.oidc.endpoints.TokenEndpoint.processGrantRequest(TokenEndpoint.java:178)
2024-08-29 13:15:42     at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
2024-08-29 13:15:42     at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
2024-08-29 13:15:42     at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
2024-08-29 13:15:42     at java.base/java.lang.reflect.Method.invoke(Method.java:568)
2024-08-29 13:15:42     at org.jboss.resteasy.core.MethodInjectorImpl.invoke(MethodInjectorImpl.java:170)
2024-08-29 13:15:42     at org.jboss.resteasy.core.MethodInjectorImpl.invoke(MethodInjectorImpl.java:130)
2024-08-29 13:15:42     at org.jboss.resteasy.core.ResourceMethodInvoker.internalInvokeOnTarget(ResourceMethodInvoker.java:660)
2024-08-29 13:15:42     at org.jboss.resteasy.core.ResourceMethodInvoker.invokeOnTargetAfterFilter(ResourceMethodInvoker.java:524)
2024-08-29 13:15:42     at org.jboss.resteasy.core.ResourceMethodInvoker.lambda$invokeOnTarget$2(ResourceMethodInvoker.java:474)
2024-08-29 13:15:42     at org.jboss.resteasy.core.interception.jaxrs.PreMatchContainerRequestContext.filter(PreMatchContainerRequestContext.java:364)
2024-08-29 13:15:42     at org.jboss.resteasy.core.ResourceMethodInvoker.invokeOnTarget(ResourceMethodInvoker.java:476)
2024-08-29 13:15:42     at org.jboss.resteasy.core.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:434)
2024-08-29 13:15:42     at org.jboss.resteasy.core.ResourceLocatorInvoker.invokeOnTargetObject(ResourceLocatorInvoker.java:192)
2024-08-29 13:15:42     at org.jboss.resteasy.core.ResourceLocatorInvoker.invoke(ResourceLocatorInvoker.java:152)
2024-08-29 13:15:42     at org.jboss.resteasy.core.ResourceLocatorInvoker.invokeOnTargetObject(ResourceLocatorInvoker.java:183)
2024-08-29 13:15:42     at org.jboss.resteasy.core.ResourceLocatorInvoker.invoke(ResourceLocatorInvoker.java:141)
2024-08-29 13:15:42     at org.jboss.resteasy.core.ResourceLocatorInvoker.invoke(ResourceLocatorInvoker.java:32)
2024-08-29 13:15:42     at org.jboss.resteasy.core.SynchronousDispatcher.invoke(SynchronousDispatcher.java:492)
2024-08-29 13:15:42     at org.jboss.resteasy.core.SynchronousDispatcher.lambda$invoke$4(SynchronousDispatcher.java:261)
2024-08-29 13:15:42     at org.jboss.resteasy.core.SynchronousDispatcher.lambda$preprocess$0(SynchronousDispatcher.java:161)
2024-08-29 13:15:42     at org.jboss.resteasy.core.interception.jaxrs.PreMatchContainerRequestContext.filter(PreMatchContainerRequestContext.java:364)
2024-08-29 13:15:42     at org.jboss.resteasy.core.SynchronousDispatcher.preprocess(SynchronousDispatcher.java:164)
2024-08-29 13:15:42     at org.jboss.resteasy.core.SynchronousDispatcher.invoke(SynchronousDispatcher.java:247)
2024-08-29 13:15:42     at io.quarkus.resteasy.runtime.standalone.RequestDispatcher.service(RequestDispatcher.java:73)
2024-08-29 13:15:42     at io.quarkus.resteasy.runtime.standalone.VertxRequestHandler.dispatch(VertxRequestHandler.java:151)
2024-08-29 13:15:42     at io.quarkus.resteasy.runtime.standalone.VertxRequestHandler.handle(VertxRequestHandler.java:82)
2024-08-29 13:15:42     at io.quarkus.resteasy.runtime.standalone.VertxRequestHandler.handle(VertxRequestHandler.java:42)
2024-08-29 13:15:42     at io.vertx.ext.web.impl.RouteState.handleContext(RouteState.java:1284)
2024-08-29 13:15:42     at io.vertx.ext.web.impl.RoutingContextImplBase.iterateNext(RoutingContextImplBase.java:173)
2024-08-29 13:15:42     at io.vertx.ext.web.impl.RoutingContextImpl.next(RoutingContextImpl.java:140)
2024-08-29 13:15:42     at io.quarkus.vertx.http.runtime.StaticResourcesRecorder$2.handle(StaticResourcesRecorder.java:84)
2024-08-29 13:15:42     at io.quarkus.vertx.http.runtime.StaticResourcesRecorder$2.handle(StaticResourcesRecorder.java:71)
2024-08-29 13:15:42     at io.vertx.ext.web.impl.RouteState.handleContext(RouteState.java:1284)
2024-08-29 13:15:42     at io.vertx.ext.web.impl.RoutingContextImplBase.iterateNext(RoutingContextImplBase.java:173)
2024-08-29 13:15:42     at io.vertx.ext.web.impl.RoutingContextImpl.next(RoutingContextImpl.java:140)
2024-08-29 13:15:42     at io.quarkus.vertx.http.runtime.VertxHttpRecorder$6.handle(VertxHttpRecorder.java:430)
2024-08-29 13:15:42     at io.quarkus.vertx.http.runtime.VertxHttpRecorder$6.handle(VertxHttpRecorder.java:408)
2024-08-29 13:15:42     at io.vertx.ext.web.impl.RouteState.handleContext(RouteState.java:1284)
2024-08-29 13:15:42     at io.vertx.ext.web.impl.RoutingContextImplBase.iterateNext(RoutingContextImplBase.java:173)
2024-08-29 13:15:42     at io.vertx.ext.web.impl.RoutingContextImpl.next(RoutingContextImpl.java:140)
2024-08-29 13:15:42     at org.keycloak.quarkus.runtime.integration.web.QuarkusRequestFilter.lambda$createBlockingHandler$0(QuarkusRequestFilter.java:82)
2024-08-29 13:15:42     at io.quarkus.vertx.core.runtime.VertxCoreRecorder$14.runWith(VertxCoreRecorder.java:576)
2024-08-29 13:15:42     at org.jboss.threads.EnhancedQueueExecutor$Task.run(EnhancedQueueExecutor.java:2449)
2024-08-29 13:15:42     at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1478)
2024-08-29 13:15:42     at org.jboss.threads.DelegatingRunnable.run(DelegatingRunnable.java:29)
2024-08-29 13:15:42     at org.jboss.threads.ThreadLocalResettingRunnable.run(ThreadLocalResettingRunnable.java:29)
2024-08-29 13:15:42     at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
2024-08-29 13:15:42     at java.base/java.lang.Thread.run(Thread.java:833)
2024-08-29 13:15:42 Caused by: org.infinispan.util.concurrent.TimeoutException: ISPN000427: Timeout after 15 seconds waiting for acks ([keycloak-infinispan-42737]). Id=50
2024-08-29 13:15:42     at org.infinispan.util.concurrent.CommandAckCollector.createTimeoutException(CommandAckCollector.java:228)
2024-08-29 13:15:42     at org.infinispan.util.concurrent.CommandAckCollector$BaseCollector.call(CommandAckCollector.java:281)
2024-08-29 13:15:42     at org.infinispan.util.concurrent.CommandAckCollector$BaseCollector.call(CommandAckCollector.java:261)
2024-08-29 13:15:42     at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
2024-08-29 13:15:42     at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
2024-08-29 13:15:42     at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
2024-08-29 13:15:42     at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
2024-08-29 13:15:42     ... 1 more
2024-08-29 13:15:42     Suppressed: org.infinispan.commons.util.logging.TraceException
2024-08-29 13:15:42             at org.infinispan.interceptors.impl.SimpleAsyncInvocationStage.get(SimpleAsyncInvocationStage.java:39)
2024-08-29 13:15:42             at org.infinispan.interceptors.impl.AsyncInterceptorChainImpl.invoke(AsyncInterceptorChainImpl.java:249)
2024-08-29 13:15:42             at org.infinispan.cache.impl.InvocationHelper.doInvoke(InvocationHelper.java:323)
2024-08-29 13:15:42             at org.infinispan.cache.impl.InvocationHelper.invoke(InvocationHelper.java:111)
2024-08-29 13:15:42             at org.infinispan.cache.impl.InvocationHelper.invoke(InvocationHelper.java:93)
2024-08-29 13:15:42             at org.infinispan.cache.impl.CacheImpl.remove(CacheImpl.java:690)
2024-08-29 13:15:42             at org.infinispan.cache.impl.DecoratedCache.remove(DecoratedCache.java:557)
2024-08-29 13:15:42             at org.infinispan.cache.impl.EncoderCache.remove(EncoderCache.java:723)
2024-08-29 13:15:42             at org.keycloak.models.sessions.infinispan.InfinispanKeycloakTransaction$5.execute(InfinispanKeycloakTransaction.java:194)
...

The topology we have is based on Docker containers for the development of the security service.

For Keycloak:

We have built an image, with 21.1.2 version, containing the following XML file (cache-ispn-remote.xml) to configure the internal Infinispan:

<?xml version="1.0" encoding="UTF-8"?>
<infinispan
  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="urn:infinispan:config:14.0 http://www.infinispan.org/schemas/infinispan-config-14.0.xsd"
  xmlns="urn:infinispan:config:14.0">

  <cache-container name="keycloak">
        <!--        <transport lock-timeout="60000" stack="jdbc-ping-tcp"/>-->
    <transport lock-timeout="60000"/>

        <local-cache name="realms">
            <encoding>
                <key media-type="application/x-java-object"/>
                <value media-type="application/x-java-object"/>
            </encoding>
            <memory max-count="10000"/>
        </local-cache>

        <local-cache name="users">
            <encoding>
                <key media-type="application/x-java-object"/>
                <value media-type="application/x-java-object"/>
            </encoding>
            <memory max-count="10000"/>
        </local-cache>

        <distributed-cache name="sessions" owners="2">
            <remote-store xmlns="urn:infinispan:config:store:remote:14.0"
                          cache="sessions"
                          purge="false"
                          preload="false"
                          shared="true"
                          segmented="false"
                          raw-values="true"
                          connect-timeout="5000">
                <remote-server host="${infinispan.bind.address:127.0.0.1}"
                               port="${infinispan.bind.port:11222}"/>

                <security>
                    <authentication server-name="infinispan">
                        <digest username="keycloak"
                                password="keycloak"
                                realm="default"/>
                    </authentication>
                </security>
            </remote-store>
            <expiration lifespan="-1"/>
        </distributed-cache>

        <distributed-cache name="authenticationSessions" owners="2">
            <remote-store xmlns="urn:infinispan:config:store:remote:14.0"
                          cache="authenticationSessions"
                          purge="false"
                          preload="false"
                          shared="true"
                          segmented="false"
                          raw-values="true"
                          connect-timeout="5000">
                <remote-server host="${infinispan.bind.address:127.0.0.1}"
                               port="${infinispan.bind.port:11222}"/>

                <security>
                    <authentication server-name="infinispan">
                        <digest username="keycloak"
                                password="keycloak"
                                realm="default"/>
                    </authentication>
                </security>
            </remote-store>
            <expiration lifespan="-1"/>
        </distributed-cache>

        <distributed-cache name="offlineSessions" owners="2">
            <remote-store xmlns="urn:infinispan:config:store:remote:14.0"
                          cache="offlineSessions"
                          purge="false"
                          preload="false"
                          shared="true"
                          segmented="false"
                          raw-values="true"
                          connect-timeout="5000">
                <remote-server host="${infinispan.bind.address:127.0.0.1}"
                               port="${infinispan.bind.port:11222}"/>

                <security>
                    <authentication server-name="infinispan">
                        <digest username="keycloak"
                                password="keycloak"
                                realm="default"/>
                    </authentication>
                </security>
            </remote-store>
            <expiration lifespan="-1"/>
        </distributed-cache>

        <distributed-cache name="clientSessions" owners="2">
            <remote-store xmlns="urn:infinispan:config:store:remote:14.0"
                          cache="clientSessions"
                          purge="false"
                          preload="false"
                          shared="true"
                          segmented="false"
                          raw-values="true"
                          connect-timeout="5000">
                <remote-server host="${infinispan.bind.address:127.0.0.1}"
                               port="${infinispan.bind.port:11222}"/>

                <security>
                    <authentication server-name="infinispan">
                        <digest username="keycloak"
                                password="keycloak"
                                realm="default"/>
                    </authentication>
                </security>
            </remote-store>
            <expiration lifespan="-1"/>
        </distributed-cache>

        <distributed-cache name="offlineClientSessions" owners="2">
            <remote-store xmlns="urn:infinispan:config:store:remote:14.0"
                          cache="offlineClientSessions"
                          purge="false"
                          preload="false"
                          shared="true"
                          segmented="false"
                          raw-values="true"
                          connect-timeout="5000">
                <remote-server host="${infinispan.bind.address:127.0.0.1}"
                               port="${infinispan.bind.port:11222}"/>

                <security>
                    <authentication server-name="infinispan">
                        <digest username="keycloak"
                                password="keycloak"
                                realm="default"/>
                    </authentication>
                </security>
            </remote-store>
            <expiration lifespan="-1"/>
        </distributed-cache>

        <distributed-cache name="loginFailures" owners="2">
            <remote-store xmlns="urn:infinispan:config:store:remote:14.0"
                          cache="loginFailures"
                          purge="false"
                          preload="false"
                          shared="true"
                          segmented="false"
                          raw-values="true"
                          connect-timeout="5000">
                <remote-server host="${infinispan.bind.address:127.0.0.1}"
                               port="${infinispan.bind.port:11222}"/>

                <security>
                    <authentication server-name="infinispan">
                        <digest username="keycloak"
                                password="keycloak"
                                realm="default"/>
                    </authentication>
                </security>
            </remote-store>
            <expiration lifespan="-1"/>
        </distributed-cache>

        <local-cache name="authorization">
            <encoding>
                <key media-type="application/x-java-object"/>
                <value media-type="application/x-java-object"/>
            </encoding>
            <memory max-count="10000"/>
        </local-cache>

        <replicated-cache name="work">
            <remote-store xmlns="urn:infinispan:config:store:remote:14.0"
                          cache="work"
                          purge="false"
                          preload="false"
                          shared="true"
                          segmented="false"
                          raw-values="true"
                          connect-timeout="5000">
                <remote-server host="${infinispan.bind.address:127.0.0.1}"
                               port="${infinispan.bind.port:11222}"/>

                <security>
                    <authentication server-name="infinispan">
                        <digest username="keycloak"
                                password="keycloak"
                                realm="default"/>
                    </authentication>
                </security>
            </remote-store>
            <expiration lifespan="-1"/>
        </replicated-cache>

        <local-cache name="keys">
            <encoding>
                <key media-type="application/x-java-object"/>
                <value media-type="application/x-java-object"/>
            </encoding>
            <expiration max-idle="3600000"/>
            <memory max-count="1000"/>
        </local-cache>

        <distributed-cache name="actionTokens" owners="2">
            <remote-store xmlns="urn:infinispan:config:store:remote:14.0"
                          cache="actionTokens"
                          purge="false"
                          preload="false"
                          shared="true"
                          segmented="false"
                          raw-values="true"
                          connect-timeout="5000">
                <remote-server host="${infinispan.bind.address:127.0.0.1}"
                               port="${infinispan.bind.port:11222}"/>

                <security>
                    <authentication server-name="infinispan">
                        <digest username="keycloak"
                                password="keycloak"
                                realm="default"/>
                    </authentication>
                </security>
            </remote-store>
            <encoding>
                <key media-type="application/x-java-object"/>
                <value media-type="application/x-java-object"/>
            </encoding>
            <expiration max-idle="-1" lifespan="-1" interval="300000"/>
            <memory max-count="-1"/>
        </distributed-cache>
    </cache-container>
</infinispan>

And we run it with the following properties:

		"Ports": {
			"8080/tcp": [
				{
					"HostIp": "0.0.0.0",
					"HostPort": "8090"
				}
			],
			"8443/tcp": null,
			"8787/tcp": [
				{
					"HostIp": "0.0.0.0",
					"HostPort": "8797"
				}
			]
		},
		"Env": [
			"KC_DB_PASSWORD=keycloak",
			"KEYCLOAK_ADMIN_PASSWORD=U6cDQm4Wy8vCd!BbSAEZ5#",
			"KC_CACHE=ispn",
			"KC_CACHE_REMOTE_USERNAME=keycloak",
			"KC_DB=mysql",
			"KC_HOSTNAME=host.docker.internal",
			"KEYCLOAK_ADMIN=admin",
			"KC_HEALTH_ENABLED=true",
			"KC_CACHE_CONFIG_FILE=cache-ispn-remote.xml",
			"KC_CACHE_REMOTE_TLS_ENABLED=false",
			"KC_LOG_LEVEL=INFO",
			"KC_DB_URL=jdbc:mysql://host.docker.internal:3317/keycloak",
			"KC_DB_USERNAME=keycloak",
			"KC_CACHE_STACK=tcp",
			"KC_CACHE_REMOTE_PORT=11222",
			"DEBUG_PORT=*:8787",
			"KC_CACHE_REMOTE_HOST=host.docker.internal",
			"KC_CACHE_REMOTE_PASSWORD=keycloak",
			"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
			"LANG=en_US.UTF-8",
			"KEYCLOAK_PRODUCTION=true",
			"KEYCLOAK_ENABLE_HTTPS=false",
			"KC_HOSTNAME_STRICT=false",
			"KC_HOSTNAME_STRICT_HTTPS=false",
			"KC_HTTP_ENABLED=true",
			"KC_METRICS_ENABLED=true",
			"KC_FEATURES=token-exchange"
		],
		"Cmd": [
			"start",
			"--optimized",
			"--debug"
		],

For Infinispan:

We have built an image, with 14.0.9.Final version, that contains the following XML file (infinispan.xml) for its configuration:

We have added the following dependencies:

  • keycloak-core-21.1.2.jar
  • keycloak-model-infinispan-21.1.2.jar
  • keycloak-server-spi-21.1.2.jar
  • keycloak-server-spi-private-21.1.2.jar

And we run it with the following properties:

		"Ports": {
			"11221/tcp": null,
			"11222/tcp": [
				{
					"HostIp": "0.0.0.0",
					"HostPort": "11222"
				}
			],
			"11223/tcp": null,
			"2157/tcp": null,
			"46655/tcp": null,
			"57600/tcp": null,
			"7800/tcp": null,
			"7900/tcp": null,
			"8080/tcp": null,
			"8443/tcp": null
		},
		"Env": [
			"USER=keycloak",
			"PASS=keycloak",
			"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
			"container=oci",
			"HOME=/home/jboss",
			"JAVA_HOME=/usr/lib/jvm/jre",
			"JAVA_VENDOR=openjdk",
			"JAVA_VERSION=17",
			"JBOSS_CONTAINER_OPENJDK_JRE_MODULE=/opt/jboss/container/openjdk/jre",
			"JBOSS_CONTAINER_JAVA_PROXY_MODULE=/opt/jboss/container/java/proxy",
			"JBOSS_CONTAINER_JAVA_JVM_MODULE=/opt/jboss/container/java/jvm",
			"JBOSS_CONTAINER_UTIL_LOGGING_MODULE=/opt/jboss/container/util/logging/",
			"JAVA_DATA_DIR=/deployments/data",
			"JBOSS_CONTAINER_JAVA_RUN_MODULE=/opt/jboss/container/java/run",
			"JBOSS_IMAGE_NAME=infinispan/server",
			"JBOSS_IMAGE_VERSION=14.0.9.Final-1",
			"LANG=C.utf8",
			"ISPN_HOME=/opt/infinispan",
			"JAVA_GC_MAX_METASPACE_SIZE=96m",
			"JAVA_GC_METASPACE_SIZE=32m",
			"JAVA_MAX_MEM_RATIO=50"
		],
		"Cmd": [
			"-c",
			"/keycloak-config/infinispan.xml"
		],

We have also tried using the basic settings for UDP Transport stacks, but it ends up with the same error.

Looking forward to your feedback

Thanks in advance,

Aris

And the infinispan.xml file:

<infinispan
  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="urn:infinispan:config:14.0 https://infinispan.org/schemas/infinispan-config-14.0.xsd
                        urn:infinispan:server:14.0 https://infinispan.org/schemas/infinispan-server-14.0.xsd"
  xmlns="urn:infinispan:config:14.0"
  xmlns:server="urn:infinispan:server:14.0">

    <cache-container name="default" statistics="true">
        <transport
          cluster="${infinispan.cluster.name:ISPN}"
          stack="${infinispan.cluster.stack:tcp}"
          node-name="${infinispan.node.name:}"
        />

        <security>
            <authorization/>
        </security>

        <serialization marshaller="org.infinispan.jboss.marshalling.commons.GenericJBossMarshaller">
            <allow-list>
                <class>org.*</class>
                <regex>.*</regex>
            </allow-list>
        </serialization>

        <distributed-cache name="sessions" owners="2">
            <encoding>
                <key media-type="application/x-jboss-marshalling"/>
                <value media-type="application/x-jboss-marshalling"/>
            </encoding>

            <expiration lifespan="900000000000000000"/>
        </distributed-cache>

        <distributed-cache name="authenticationSessions" owners="2">
            <encoding>
                <key media-type="application/x-jboss-marshalling"/>
                <value media-type="application/x-jboss-marshalling"/>
            </encoding>

            <expiration lifespan="900000000000000000"/>
        </distributed-cache>

        <distributed-cache name="offlineSessions" owners="2">
            <encoding>
                <key media-type="application/x-jboss-marshalling"/>
                <value media-type="application/x-jboss-marshalling"/>
            </encoding>

            <expiration lifespan="900000000000000000"/>
        </distributed-cache>

        <distributed-cache name="clientSessions" owners="2">
            <encoding>
                <key media-type="application/x-jboss-marshalling"/>
                <value media-type="application/x-jboss-marshalling"/>
            </encoding>

            <expiration lifespan="900000000000000000"/>
        </distributed-cache>

        <distributed-cache name="offlineClientSessions" owners="2">
            <encoding>
                <key media-type="application/x-jboss-marshalling"/>
                <value media-type="application/x-jboss-marshalling"/>
            </encoding>

            <expiration lifespan="900000000000000000"/>
        </distributed-cache>

        <distributed-cache name="loginFailures" owners="2">
            <encoding>
                <key media-type="application/x-jboss-marshalling"/>
                <value media-type="application/x-jboss-marshalling"/>
            </encoding>

            <expiration lifespan="900000000000000000"/>
        </distributed-cache>

        <replicated-cache name="work">
            <encoding>
                <key media-type="application/x-jboss-marshalling"/>
                <value media-type="application/x-jboss-marshalling"/>
            </encoding>

            <expiration lifespan="900000000000000000"/>
        </replicated-cache>

        <distributed-cache name="actionTokens" owners="2">
            <encoding>
                <key media-type="application/x-jboss-marshalling"/>
                <value media-type="application/x-jboss-marshalling"/>
            </encoding>

            <expiration interval="300000"/>
        </distributed-cache>
    </cache-container>

    <server xmlns="urn:infinispan:server:14.0">
        <interfaces>
            <interface name="public">
                <inet-address value="${infinispan.bind.address:127.0.0.1}"/>
            </interface>
        </interfaces>

        <socket-bindings default-interface="public"
                         port-offset="${infinispan.socket.binding.port-offset:0}">
            <socket-binding name="default" port="${infinispan.bind.port:11222}"/>
            <socket-binding name="memcached" port="11221"/>
        </socket-bindings>

        <security>
            <credential-stores>
                <credential-store name="credentials" path="credentials.pfx">
                    <clear-text-credential clear-text="secret"/>
                </credential-store>
            </credential-stores>
            <security-realms>
                <security-realm name="default">
                    <!-- Uncomment to enable TLS on the realm -->
                  <!-- server-identities>
           <ssl>
              <keystore path="application.keystore"
                        password="password" alias="server"
                        generate-self-signed-certificate-host="localhost"/>
           </ssl>
        </server-identities-->
                  <properties-realm groups-attribute="Roles">
                        <user-properties path="users.properties"/>
                        <group-properties path="groups.properties"/>
                    </properties-realm>
                </security-realm>
            </security-realms>
        </security>

        <endpoints socket-binding="default" security-realm="default">
            <endpoint>
                <hotrod-connector name="hotrod">
                    <authentication>
                        <sasl mechanisms="DIGEST-MD5"
                              server-name="infinispan"/>
                    </authentication>
                </hotrod-connector>
                <rest-connector name="rest"/>
            </endpoint>
        </endpoints>
    </server>
</infinispan>