Query related to Keycloak HA with separate Infinispan cache

Getting advice
1

Hi
I am trying to setup Keycloak with separate infinispan cache.
However I am facing couple of issues. Looking for guidance:
I have used bitnami helm chart for keycloak and openshift helm chart for Infinispan.

Infinispan version: 14.0.9.Final
Keycloak Version: 20.x and 21.x

Issues

  • Keycloak v 20.x:
    • Works well but when we increase number of replica from 1 to 2, Admin console fails to load. Reason: It creates resource folders under /opt/bitnami/keycloak/data/tmp/kc-gzip-cache/
    • It works well with single replica count.
    • Tried multiple configuration changes but did not work. They act as standalone
  • Keycloak v 21.x:
    • Transcoding error
      When we use same configuration even 13.0 xml urn, we get transcoding error as below

2023-05-04 04:03:29,630 ERROR [org.keycloak.quarkus.runtime.cli.ExecutionExceptionHandler] (main) ERROR: org.infinispan.commons.dataconversion.EncodingException: ISPN000492: Cannot find transcoder between 'application/x-protostream' to 'application/x-jboss-marshalling'

This states that Cache is using jboss-marshelling but getting data x-protostream.

  • AllowScriptManager realm not served error:
    So after this tried updating sessions cache encoding as x-protostream, however I got below error:

javax.security.sasl.SaslException: ELY05087: Client selected realm not offered by server (AllowScriptManager)

What should be the appropriate content for cache-ispn-remote.xml ?

Setup Details

Infinispan cache configuration

Configured Caches

sessions

  "sessions": {
    "distributed-cache": {
      "owners": "2",
      "mode": "SYNC",
      "statistics": true,
      "encoding": {
        "media-type": "application/x-jboss-marshalling"
      },
      "locking": {
        "isolation": "REPEATABLE_READ"
      }
    }
  }
}

actionTokens

  "actionTokens": {
    "distributed-cache": {
      "owners": "2",
      "mode": "SYNC",
      "statistics": true,
      "encoding": {
        "media-type": "application/x-jboss-marshalling"
      },
      "locking": {
        "isolation": "REPEATABLE_READ"
      }
    }
  }
}

authenticationSessions

  "authenticationSessions": {
    "distributed-cache": {
      "owners": "2",
      "mode": "SYNC",
      "statistics": true,
      "encoding": {
        "media-type": "application/x-jboss-marshalling"
      },
      "locking": {
        "isolation": "REPEATABLE_READ"
      }
    }
  }
}

clientSessions

  "clientSessions": {
    "distributed-cache": {
      "owners": "2",
      "mode": "SYNC",
      "statistics": true,
      "encoding": {
        "media-type": "application/x-jboss-marshalling"
      },
      "locking": {
        "isolation": "REPEATABLE_READ"
      }
    }
  }
}

loginFailures

  "loginFailures": {
    "distributed-cache": {
      "owners": "2",
      "mode": "SYNC",
      "statistics": true,
      "encoding": {
        "media-type": "application/x-jboss-marshalling"
      },
      "locking": {
        "isolation": "REPEATABLE_READ"
      }
    }
  }
}

offlineClientSessions

  "offlineClientSessions": {
    "distributed-cache": {
      "owners": "2",
      "mode": "SYNC",
      "statistics": true,
      "encoding": {
        "media-type": "application/x-jboss-marshalling"
      },
      "locking": {
        "isolation": "REPEATABLE_READ"
      }
    }
  }
}

offlineSessions

  "offlineSessions": {
    "distributed-cache": {
      "owners": "2",
      "mode": "SYNC",
      "statistics": true,
      "encoding": {
        "media-type": "application/x-jboss-marshalling"
      },
      "locking": {
        "isolation": "REPEATABLE_READ"
      }
    }
  }
}

work

  "work": {
    "replicated-cache": {
      "mode": "SYNC",
      "statistics": true,
      "encoding": {
        "media-type": "application/x-jboss-marshalling"
      },
      "locking": {
        "isolation": "REPEATABLE_READ"
      }
    }
  }
}

Keycloak Configuration

cache-ispn-remote.xml

<?xml version="1.0" encoding="UTF-8"?>
<infinispan
        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
        xsi:schemaLocation="urn:infinispan:config:13.0 http://www.infinispan.org/schemas/infinispan-config-13.0.xsd"
        xmlns="urn:infinispan:config:13.0">

    <cache-container name="keycloak">
        <transport lock-timeout="60000"/>

        <local-cache name="realms">
            <encoding>
                <key media-type="application/x-java-object"/>
                <value media-type="application/x-java-object"/>
            </encoding>
            <memory max-count="10000"/>
        </local-cache>
        <local-cache name="users">
            <encoding>
                <key media-type="application/x-java-object"/>
                <value media-type="application/x-java-object"/>
            </encoding>
            <memory max-count="10000"/>
        </local-cache>
        <local-cache name="keys">
            <encoding>
                <key media-type="application/x-java-object"/>
                <value media-type="application/x-java-object"/>
            </encoding>
            <expiration max-idle="3600000"/>
            <memory max-count="1000"/>
        </local-cache>
        <local-cache name="authorization">
            <encoding>
                <key media-type="application/x-java-object"/>
                <value media-type="application/x-java-object"/>
            </encoding>
            <memory max-count="10000"/>
        </local-cache>

        <distributed-cache name="sessions" owners="2">
            <expiration lifespan="-1"/>
            <remote-store cache="sessions" xmlns="urn:infinispan:config:store:remote:13.0"
                          fetch-state="false"
                          purge="false"
                          preload="false"
                          shared="true" segmented="false"
                          connect-timeout="${env.KEYCLOAK_REMOTE_ISPN_CONN_TIMEOUT:2000}">
                <remote-server host="${env.INFINISPAN_SERVER}" port="${infinispan.bind.port:11222}"/>

                <security>
                    <authentication>
                        <digest username="${env.KEYCLOAK_REMOTE_ISPN_USERNAME:keycloak}"
                                password="${env.KEYCLOAK_REMOTE_ISPN_PASSWORD:password}"
                                realm="default"/>
                    </authentication>
                </security>

                <property name="rawValues">true</property>
                <property name="marshaller">org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory</property>
            </remote-store>
        </distributed-cache>

        <distributed-cache name="authenticationSessions" owners="2">
            <expiration lifespan="-1"/>
            <remote-store cache="authenticationSessions" xmlns="urn:infinispan:config:store:remote:13.0"
                          fetch-state="false"
                          purge="false"
                          preload="false"
                          shared="true" segmented="false"
                          connect-timeout="${env.KEYCLOAK_REMOTE_ISPN_CONN_TIMEOUT:2000}">
                <remote-server host="${env.INFINISPAN_SERVER}" port="${infinispan.bind.port:11222}"/>

                <security>
                    <authentication>
                        <digest username="${env.KEYCLOAK_REMOTE_ISPN_USERNAME:keycloak}"
                                password="${env.KEYCLOAK_REMOTE_ISPN_PASSWORD:password}"
                                realm="default"/>
                    </authentication>
                </security>

                <property name="rawValues">true</property>
                <property name="marshaller">org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory</property>
            </remote-store>
        </distributed-cache>

        <distributed-cache name="offlineSessions" owners="2">
            <expiration lifespan="-1"/>
            <remote-store cache="offlineSessions" xmlns="urn:infinispan:config:store:remote:13.0"
                          fetch-state="false"
                          purge="false"
                          preload="false"
                          shared="true" segmented="false"
                          connect-timeout="${env.KEYCLOAK_REMOTE_ISPN_CONN_TIMEOUT:2000}">
                <remote-server host="${env.INFINISPAN_SERVER}" port="${infinispan.bind.port:11222}"/>

                <security>
                    <authentication>
                        <digest username="${env.KEYCLOAK_REMOTE_ISPN_USERNAME:keycloak}"
                                password="${env.KEYCLOAK_REMOTE_ISPN_PASSWORD:password}"
                                realm="default"/>
                    </authentication>
                </security>

                <property name="rawValues">true</property>
                <property name="marshaller">org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory</property>
            </remote-store>
        </distributed-cache>

        <distributed-cache name="clientSessions" owners="2">
            <expiration lifespan="-1"/>
            <remote-store cache="clientSessions" xmlns="urn:infinispan:config:store:remote:13.0"
                          fetch-state="false"
                          purge="false"
                          preload="false"
                          shared="true" segmented="false"
                          connect-timeout="${env.KEYCLOAK_REMOTE_ISPN_CONN_TIMEOUT:2000}">
                <remote-server host="${env.INFINISPAN_SERVER}" port="${infinispan.bind.port:11222}"/>

                <security>
                    <authentication>
                        <digest username="${env.KEYCLOAK_REMOTE_ISPN_USERNAME:keycloak}"
                                password="${env.KEYCLOAK_REMOTE_ISPN_PASSWORD:password}"
                                realm="default"/>
                    </authentication>
                </security>

                <property name="rawValues">true</property>
                <property name="marshaller">org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory</property>
            </remote-store>
        </distributed-cache>

        <distributed-cache name="offlineClientSessions" owners="2">
            <expiration lifespan="-1"/>
            <remote-store cache="offlineClientSessions" xmlns="urn:infinispan:config:store:remote:13.0"
                          fetch-state="false"
                          purge="false"
                          preload="false"
                          shared="true" segmented="false"
                          connect-timeout="${env.KEYCLOAK_REMOTE_ISPN_CONN_TIMEOUT:2000}">
                <remote-server host="${env.INFINISPAN_SERVER}" port="${infinispan.bind.port:11222}"/>

                <security>
                    <authentication>
                        <digest username="${env.KEYCLOAK_REMOTE_ISPN_USERNAME:keycloak}"
                                password="${env.KEYCLOAK_REMOTE_ISPN_PASSWORD:password}"
                                realm="default"/>
                    </authentication>
                </security>

                <property name="rawValues">true</property>
                <property name="marshaller">org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory</property>
            </remote-store>
        </distributed-cache>


        <distributed-cache name="loginFailures" owners="2">
            <expiration lifespan="-1"/>
            <remote-store cache="loginFailures" xmlns="urn:infinispan:config:store:remote:13.0"
                          fetch-state="false"
                          purge="false"
                          preload="false"
                          shared="true" segmented="false"
                          connect-timeout="${env.KEYCLOAK_REMOTE_ISPN_CONN_TIMEOUT:2000}">
                <remote-server host="${env.INFINISPAN_SERVER}" port="${infinispan.bind.port:11222}"/>

                <security>
                    <authentication>
                        <digest username="${env.KEYCLOAK_REMOTE_ISPN_USERNAME:keycloak}"
                                password="${env.KEYCLOAK_REMOTE_ISPN_PASSWORD:password}"
                                realm="default"/>
                    </authentication>
                </security>

                <property name="rawValues">true</property>
                <property name="marshaller">org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory</property>
            </remote-store>
        </distributed-cache>
        <distributed-cache name="actionTokens" owners="2">
            <encoding>
                <key media-type="application/x-java-object"/>
                <value media-type="application/x-java-object"/>
            </encoding>
            <expiration max-idle="-1" lifespan="-1" interval="300000"/>
            <memory max-count="-1"/>
            <remote-store cache="actionTokens" xmlns="urn:infinispan:config:store:remote:13.0"
                          fetch-state="false"
                          purge="false"
                          preload="false"
                          shared="true" segmented="false"
                          connect-timeout="${env.KEYCLOAK_REMOTE_ISPN_CONN_TIMEOUT:2000}">
                <remote-server host="${env.INFINISPAN_SERVER}" port="${infinispan.bind.port:11222}"/>

                <security>
                    <authentication>
                        <digest username="${env.KEYCLOAK_REMOTE_ISPN_USERNAME:keycloak}"
                                password="${env.KEYCLOAK_REMOTE_ISPN_PASSWORD:password}"
                                realm="default"/>
                    </authentication>
                </security>

                <property name="rawValues">true</property>
                <property name="marshaller">org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory</property>
            </remote-store>
        </distributed-cache>

        <replicated-cache name="work">
            <expiration lifespan="-1"/>
            <remote-store cache="work" xmlns="urn:infinispan:config:store:remote:13.0"
                          purge="false"
                          preload="false"
                          shared="true" segmented="false"
                          connect-timeout="${env.KEYCLOAK_REMOTE_ISPN_CONN_TIMEOUT:2000}">
                <remote-server host="${env.INFINISPAN_SERVER}" port="${infinispan.bind.port:11222}"/>

                <security>
                    <authentication>
                        <digest username="${env.KEYCLOAK_REMOTE_ISPN_USERNAME:keycloak}"
                                password="${env.KEYCLOAK_REMOTE_ISPN_PASSWORD:password}"
                                realm="default"/>
                    </authentication>
                </security>

                <property name="rawValues">true</property>
                <property name="marshaller">org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory</property>
            </remote-store>
        </replicated-cache>
        <serialization marshaller="org.infinispan.commons.marshall.JavaSerializationMarshaller">
            <allow-list>
                <regex>.*</regex>
            </allow-list>
        </serialization>
    </cache-container>
</infinispan>

keycloak-values.yaml

---
logging:
  level: DEBUG
replicaCount: 2
image:
  registry: docker.io
  repository: bitnami/keycloak
  tag: 21.0.2-debian-11-r0
command:
  - kc.sh
args:
  - "--verbose"
  - "start"
  - "--proxy=edge"
  - "--cache-config-file=cache-ispn.xml"
  - "--hostname=kcauth.example.com"
  - "--hostname-strict=false"
  - "-Djboss.site.name=site1"
  - "--http-enabled=true"
  - "-Dinfinispan.deserialization.allowlist.classes=org.keycloak.cluster.infinispan.WrapperClusterEvent"
  - "-Dinfinispan.deserialization.allowlist.regexps=.*"

production: true
resources:
  limits:
    cpu: 700m
    memory: 2048Mi
  requests:
    cpu: 700m
    memory: 2048Mi
tls:
  enabled: true
  autoGenerated: true
extraVolumeMounts: |
  - name: quarkus
    mountPath: /opt/bitnami/keycloak/conf/quarkus.properties
    subPath: quarkus.properties
  - name: keycloak
    mountPath: /opt/bitnami/keycloak/conf/cache-ispn.xml
    subPath: cache-ispn-remote.xml
extraVolumes: |
  - name: quarkus
    configMap:
      name: quarkus
      defaultMode: 0777
  - name: keycloak
    configMap:
      name: keycloak
      defaultMode: 0777
extraEnvVars:
   - name: KC_CACHE_STACK
     value: kubernetes
   - name: KC_PROXY
     value: edge
   - name: KEYCLOAK_PROXY
     value: edge
   - name: KEYCLOAK_PROXY_ADDRESS_FORWARDING
     value: "true"
   - name: QUARKUS_INFINISPAN_CLIENT_SERVER_LIST
     value: infinispan.keycloak.svc.cluster.local:11222
   - name: INFINISPAN_SERVER
     value: infinispan.keycloak.svc.cluster.local
   - name: KC_DB_URL_HOST
     value: <RDS Endpoint>
   - name: KC_DB_URL_DATABASE
     value: keycloak
   - name: KC_DB_USERNAME
     value: postgres
   - name: <DB PASS>
     value: cGFzc0AxMjMK
   - name: KEYCLOAK_REMOTE_ISPN_USERNAME
     value: developer
   - name: KEYCLOAK_REMOTE_ISPN_PASSWORD
     value: <password infini developer>
   - name: KC_CACHE
     value: ispn
   - name: KEYCLOAK_EXTRA_ARGS
     value: "--optimized"
   - name: JAVA_OPTS_APPEND
     value: -Djgroups.dns.query=keycloak-headless.keycloak.svc.cluster.local
   - name: KEYCLOAK_PRODUCTION
     value: "true"
cache: 
  enabled: true
  stackName: kubernetes
postgresql: 
  enabled: false
externalDatabase: 
  host: <RDS ENDPOINT>;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE
  port: 5432
  user: postgres
  password: <DB PASS>
  database: keycloak
auth: 
  adminUser: admin
  adminPassword: admin
service:
  type: ClusterIP

ingress:
  enabled: true
  ingressClassName: "alb"
  pathType: "Prefix"
  hostname: "kcauth.example.com"
  annotations:
    #kubernetes.io/ingress.class: alb
    alb.ingress.kubernetes.io/target-type: 'ip'
    alb.ingress.kubernetes.io/certificate-arn: "<ACM ARN>"
    alb.ingress.kubernetes.io/subnets: <subnet ids>
    alb.ingress.kubernetes.io/conditions.keycloak: |
      [{"Field":"host-header","HostHeaderConfig":{"Values":["*.example.com"]}}]
    alb.ingress.kubernetes.io/scheme: internet-facing
    alb.ingress.kubernetes.io/ssl-policy: ELBSecurityPolicy-TLS-1-1-2017-01
2

Found fix for this. Ref: Getting encode exception on Keycloak 21 with remote caches · Issue #20031 · keycloak/keycloak · GitHub

Now Keycloak version 21 is working with Infinispan cache.

Now my cache-ispn-remote.xml looks like:

<?xml version="1.0" encoding="UTF-8"?>
<infinispan
        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
        xsi:schemaLocation="urn:infinispan:config:14.0 http://www.infinispan.org/schemas/infinispan-config-14.0.xsd"
        xmlns="urn:infinispan:config:14.0">

    <cache-container name="keycloak">
        <transport lock-timeout="60000" site="site1"/>

        <local-cache name="realms">
            <encoding>
                <key media-type="application/x-java-object"/>
                <value media-type="application/x-java-object"/>
            </encoding>
            <memory max-count="10000"/>
        </local-cache>
        <local-cache name="users">
            <encoding>
                <key media-type="application/x-java-object"/>
                <value media-type="application/x-java-object"/>
            </encoding>
            <memory max-count="10000"/>
        </local-cache>
        <local-cache name="keys">
            <encoding>
                <key media-type="application/x-java-object"/>
                <value media-type="application/x-java-object"/>
            </encoding>
            <expiration max-idle="3600000"/>
            <memory max-count="1000"/>
        </local-cache>
        <local-cache name="authorization">
            <encoding>
                <key media-type="application/x-java-object"/>
                <value media-type="application/x-java-object"/>
            </encoding>
            <memory max-count="10000"/>
        </local-cache>
       <distributed-cache name="sessions" owners="2">
            <remote-store xmlns="urn:infinispan:config:store:remote:14.0"
                          cache="sessions"
                          purge="false"
                          preload="false"
                          segmented="false"
                          shared="true"
                          raw-values="true"
                          marshaller="org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory"
                          connect-timeout="${env.KEYCLOAK_REMOTE_ISPN_CONN_TIMEOUT:2000}">
                <remote-server host="${env.INFINISPAN_SERVER}" port="${infinispan.bind.port:11222}"/>
                <security>
                    <authentication>
                        <digest username="${env.KEYCLOAK_REMOTE_ISPN_USERNAME:keycloak}"
                                password="${env.KEYCLOAK_REMOTE_ISPN_PASSWORD:password}"
                                realm="default"/>
                    </authentication>
                </security>
            </remote-store>
       </distributed-cache>
       <distributed-cache name="authenticationSessions" owners="2">
            <remote-store xmlns="urn:infinispan:config:store:remote:14.0"
                          cache="authenticationSessions"
                          purge="false"
                          preload="false"
                          segmented="false"
                          shared="true"
                          raw-values="true"
                          marshaller="org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory"
                          connect-timeout="${env.KEYCLOAK_REMOTE_ISPN_CONN_TIMEOUT:2000}">
                <remote-server host="${env.INFINISPAN_SERVER}" port="${infinispan.bind.port:11222}"/>
                <security>
                    <authentication>
                        <digest username="${env.KEYCLOAK_REMOTE_ISPN_USERNAME:keycloak}"
                                password="${env.KEYCLOAK_REMOTE_ISPN_PASSWORD:password}"
                                realm="default"/>
                    </authentication>
                </security>
            </remote-store>
        </distributed-cache>
       <distributed-cache name="offlineSessions" owners="2">
            <remote-store xmlns="urn:infinispan:config:store:remote:14.0"
                          cache="offlineSessions"
                          purge="false"
                          preload="false"
                          segmented="false"
                          shared="true"
                          raw-values="true"
                          marshaller="org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory"
                          connect-timeout="${env.KEYCLOAK_REMOTE_ISPN_CONN_TIMEOUT:2000}">
                <remote-server host="${env.INFINISPAN_SERVER}" port="${infinispan.bind.port:11222}"/>
                <security>
                    <authentication>
                        <digest username="${env.KEYCLOAK_REMOTE_ISPN_USERNAME:keycloak}"
                                password="${env.KEYCLOAK_REMOTE_ISPN_PASSWORD:password}"
                                realm="default"/>
                    </authentication>
                </security>
            </remote-store>
        </distributed-cache>
       <distributed-cache name="clientSessions" owners="2">
            <remote-store xmlns="urn:infinispan:config:store:remote:14.0"
                          cache="clientSessions"
                          purge="false"
                          preload="false"
                          segmented="false"
                          shared="true"
                          raw-values="true"
                          marshaller="org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory"
                          connect-timeout="${env.KEYCLOAK_REMOTE_ISPN_CONN_TIMEOUT:2000}">
                <remote-server host="${env.INFINISPAN_SERVER}" port="${infinispan.bind.port:11222}"/>
                <security>
                    <authentication>
                        <digest username="${env.KEYCLOAK_REMOTE_ISPN_USERNAME:keycloak}"
                                password="${env.KEYCLOAK_REMOTE_ISPN_PASSWORD:password}"
                                realm="default"/>
                    </authentication>
                </security>
            </remote-store>
        </distributed-cache>
       <distributed-cache name="offlineClientSessions" owners="2">
            <remote-store xmlns="urn:infinispan:config:store:remote:14.0"
                          cache="offlineClientSessions"
                          purge="false"
                          preload="false"
                          segmented="false"
                          shared="true"
                          raw-values="true"
                          marshaller="org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory"
                          connect-timeout="${env.KEYCLOAK_REMOTE_ISPN_CONN_TIMEOUT:2000}">
                <remote-server host="${env.INFINISPAN_SERVER}" port="${infinispan.bind.port:11222}"/>
                <security>
                    <authentication>
                        <digest username="${env.KEYCLOAK_REMOTE_ISPN_USERNAME:keycloak}"
                                password="${env.KEYCLOAK_REMOTE_ISPN_PASSWORD:password}"
                                realm="default"/>
                    </authentication>
                </security>
            </remote-store>
        </distributed-cache>
        <distributed-cache name="loginFailures" owners="2">
            <remote-store xmlns="urn:infinispan:config:store:remote:14.0"
                          cache="loginFailures"
                          purge="false"
                          preload="false"
                          segmented="false"
                          shared="true"
                          raw-values="true"
                          marshaller="org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory"
                          connect-timeout="${env.KEYCLOAK_REMOTE_ISPN_CONN_TIMEOUT:2000}">
                <remote-server host="${env.INFINISPAN_SERVER}" port="${infinispan.bind.port:11222}"/>
                <security>
                    <authentication>
                        <digest username="${env.KEYCLOAK_REMOTE_ISPN_USERNAME:keycloak}"
                                password="${env.KEYCLOAK_REMOTE_ISPN_PASSWORD:password}"
                                realm="default"/>
                    </authentication>
                </security>
            </remote-store>
        </distributed-cache>
        <distributed-cache name="actionTokens" owners="2">
            <remote-store xmlns="urn:infinispan:config:store:remote:14.0"
                          cache="actionTokens"
                          purge="false"
                          preload="false"
                          segmented="false"
                          shared="true"
                          raw-values="true"
                          marshaller="org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory"
                          connect-timeout="${env.KEYCLOAK_REMOTE_ISPN_CONN_TIMEOUT:2000}">
                <remote-server host="${env.INFINISPAN_SERVER}" port="${infinispan.bind.port:11222}"/>
                <security>
                    <authentication>
                        <digest username="${env.KEYCLOAK_REMOTE_ISPN_USERNAME:keycloak}"
                                password="${env.KEYCLOAK_REMOTE_ISPN_PASSWORD:password}"
                                realm="default"/>
                    </authentication>
                </security>
            </remote-store>
        </distributed-cache>
        <replicated-cache name="work">
            <remote-store xmlns="urn:infinispan:config:store:remote:14.0"
                          cache="actionTokens"
                          purge="false"
                          preload="false"
                          segmented="false"
                          shared="true"
                          raw-values="true"
                          marshaller="org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory"
                          connect-timeout="${env.KEYCLOAK_REMOTE_ISPN_CONN_TIMEOUT:2000}">
                <remote-server host="${env.INFINISPAN_SERVER}" port="${infinispan.bind.port:11222}"/>
                <security>
                    <authentication>
                        <digest username="${env.KEYCLOAK_REMOTE_ISPN_USERNAME:keycloak}"
                                password="${env.KEYCLOAK_REMOTE_ISPN_PASSWORD:password}"
                                realm="default"/>
                    </authentication>
                </security>
            </remote-store>
        </replicated-cache>
    </cache-container>
</infinispan>