Keycloak 22.0.4 with external infinispan (docker compose)

1

I’m using keycloak 22.0.4 with custom spi and infinispan 14.0.21.Final docker images.
Im getting

ISPN000474: Error processing request 15@infinispan-server01-57681: java.lang.ClassNotFoundException: org.infinispan.server.hotrod.HotRodServer$ToEmptyBytesKeyValueFilterConverter$ToEmptyBytesKeyValueFilterConverterExternalizer

error when i up the keycloak container.But it’s working without any issues with keycloak 20.0.5 with below same configurations.
im using docker 24.0.7 version and docker compose v2.21.0

docker-compose yml

version: '3.8'
services:  
  
  keycloak:
    image: quay.io/keycloak/keycloak:22.0.4
    command: ["start-dev"]
    restart: always
    container_name: keycloak
    environment:
      KC_DB: mysql
      KC_DB_USERNAME: keycloak_user
      KC_DB_PASSWORD:  keycloak 
      KC_DB_URL: "jdbc:mysql://mysql:3306/keycloak"
      KC_METRICS_ENABLED: true
      KC_HOSTNAME_STRICT: "false"
      #KC_PROXY: edge
      KC_HTTP_ENABLED: true 
      KC_HOSTNAME_STRICT_HTTPS: true
      PROXY_ADDRESS_FORWARDING: true
#      KC_LOG_LEVEL: WARN 
      KEYCLOAK_ADMIN: admin
      KEYCLOAK_ADMIN_PASSWORD: keycloaknew
    volumes:
        - ./server_data/keycloak/cache-ispn.xml:/opt/keycloak/conf/cache-ispn.xml:ro
#        - ./server_data/keycloak/infinispan.xml:/opt/keycloak/conf/cache-ispn-jdbc-ping.xml:ro
        - ./server_data/keycloak/keycloak.conf:/opt/keycloak/conf/keycloak.conf:ro  
        
  mysql:
      image: mysql:8.0
      container_name: mysql
      restart: always
      environment:
        MYSQL_ROOT_PASSWORD: root_password
        MYSQL_DATABASE: keycloak
        MYSQL_USER: keycloak_user
        MYSQL_PASSWORD: keycloak 
      volumes:
        - ./server_data/mysql:/var/lib/mysql
      ports:
        - 3306:3306

  infinispan:
    image:  quay.io/infinispan/server:14.0.21.Final
    hostname: INFINISPAN-SERVER-01
    container_name: infinispan
    ports:
      - 11222:11222
    environment:
      - USER=admin
      - PASS=admin
    volumes:
     - ./server_data/infinispan/infinispan.xml:/opt/infinispan/server/conf/infinispan.xml
     - ./server_data/infinispan/org.keycloak.keycloak-model-infinispan-22.0.4.jar:/opt/infinispan/lib/org.keycloak.keycloak-model-infinispan-22.0.4.jar
     - ./server_data/infinispan/org.keycloak.keycloak-core-22.0.4.jar:/opt/infinispan/lib/org.keycloak.keycloak-core-22.0.4.jar
     - ./server_data/infinispan/org.keycloak.keycloak-server-spi-22.0.4.jar:/opt/infinispan/lib/org.keycloak.keycloak-server-spi-22.0.4.jar
     - ./server_data/infinispan/org.infinispan.infinispan-jboss-marshalling-14.0.17.Final.jar:/opt/infinispan/lib/org.infinispan.infinispan-jboss-marshalling-14.0.17.Final.jar
     - ./server_data/infinispan/org.jboss.marshalling.jboss-marshalling-2.0.11.Final.jar:/opt/infinispan/lib/org.jboss.marshalling.jboss-marshalling-2.0.11.Final.jar
     - ./server_data/infinispan/org.keycloak.keycloak-server-spi-private-22.0.4.jar:/opt/infinispan/lib/org.keycloak.keycloak-server-spi-private-22.0.4.jar
     - ./server_data/infinispan/org.infinispan.infinispan-client-hotrod-jakarta-14.0.17.Final.jar:/opt/infinispan/lib/org.infinispan.infinispan-client-hotrod-jakarta-14.0.17.Fi

infinispan.xml

<infinispan
      xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
      xsi:schemaLocation="urn:infinispan:config:14.0 https://infinispan.org/schemas/infinispan-config-14.0.xsd
                            urn:infinispan:server:14.0 https://infinispan.org/schemas/infinispan-server-14.0.xsd"
      xmlns="urn:infinispan:config:14.0"
      xmlns:server="urn:infinispan:server:14.0">

   <cache-container name="keycloak" statistics="true">
      <transport cluster="${infinispan.cluster.name:ISPN}" stack="${infinispan.cluster.stack:tcp}" node-name="${infinispan.node.name:}"/>
      <security>
         <authorization/>
      </security>

    <serialization marshaller="org.infinispan.jboss.marshalling.commons.GenericJBossMarshaller">
            <allow-list>
                <class>org.keycloak.cluster.infinispan.WrapperClusterEvent</class>
                <regex>.*</regex>
            </allow-list>
    </serialization> 

    <replicated-cache name="work">
        <encoding media-type="application/x-jboss-marshalling"/>
    </replicated-cache>
    
    <distributed-cache name="sessions">
        <encoding media-type="application/x-jboss-marshalling"/>
    </distributed-cache>
    
    <distributed-cache name="authenticationSessions">
        <encoding media-type="application/x-jboss-marshalling"/>
    </distributed-cache>
    
    <distributed-cache name="clientSessions">
        <encoding media-type="application/x-jboss-marshalling"/>
    </distributed-cache>
    
    <distributed-cache name="offlineSessions">
        <encoding media-type="application/x-jboss-marshalling"/>
    </distributed-cache>
    
    <distributed-cache name="offlineClientSessions">
        <encoding media-type="application/x-jboss-marshalling"/>
    </distributed-cache>
    
    <distributed-cache name="actionTokens">
        <encoding media-type="application/x-jboss-marshalling"/>
    </distributed-cache>
    
    <distributed-cache name="loginFailures">
        <encoding media-type="text/plain"/>
    </distributed-cache>-->       
          
   </cache-container>

   <server xmlns="urn:infinispan:server:14.0">
      <interfaces>
         <interface name="public">
            <inet-address value="${infinispan.bind.address:127.0.0.1}"/>
         </interface>
      </interfaces>

      <socket-bindings default-interface="public" port-offset="${infinispan.socket.binding.port-offset:0}">
         <socket-binding name="default" port="${infinispan.bind.port:11222}"/>
         <socket-binding name="memcached" port="11221"/>
      </socket-bindings>

      <security>
         <credential-stores>
            <credential-store name="credentials" path="credentials.pfx">
               <clear-text-credential clear-text="secret"/>
            </credential-store>
         </credential-stores>
         <security-realms>
            <security-realm name="default">
               <!-- Uncomment to enable TLS on the realm -->
               <!-- server-identities>
                  <ssl>
                     <keystore path="application.keystore"
                               password="password" alias="server"
                               generate-self-signed-certificate-host="localhost"/>
                  </ssl>
               </server-identities-->
               <properties-realm groups-attribute="Roles">
                  <user-properties path="users.properties"/>
                  <group-properties path="groups.properties"/>
               </properties-realm>
            </security-realm>
         </security-realms>
      </security>

      <endpoints socket-binding="default" security-realm="default" />
   </server>
</infinispan>

keycloak.conf file

# Basic settings for running in production. Change accordingly before deploying the server.

# Database

# The database vendor.
#db=postgres

# The username of the database user.
#db-username=keycloak

# The password of the database user.
#db-password=password

# The full database JDBC URL. If not provided, a default URL is set based on the selected database vendor.
#db-url=jdbc:postgresql://localhost/keycloak

# Observability

# If the server should expose healthcheck endpoints.
#health-enabled=true

# If the server should expose metrics endpoints.
#metrics-enabled=true

# HTTP

# The file path to a server certificate or certificate chain in PEM format.
#https-certificate-file=${kc.home.dir}conf/server.crt.pem

# The file path to a private key in PEM format.
#https-certificate-key-file=${kc.home.dir}conf/server.key.pem

# The proxy address forwarding mode if the server is behind a reverse proxy.
#proxy=reencrypt

# Do not attach route to cookies and rely on the session affinity capabilities from reverse proxy
#spi-sticky-session-encoder-infinispan-should-attach-route=false

# Hostname for the Keycloak server.
#hostname=myhostname
cache-config-file=cache-ispn.xml
2

Hi @madhuchamara383,
Review the example provided for Keycloak (this is quite new in the official docs):

2 Likes
3

Hi @embesozzi ,

Thanks for the reply.seems that is k8’s specific one.once i tried with without this part

<truststore filename="/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt"
                                type="pem"/>

but got an error.have a confusion with that part when it turns to docker compose

4

Hi @embesozzi,

since last week I am trying to follow the guide that you provided and I get the same error that was reported by @madhuchamara383.
Any hints about what could cause the error? Or someone else that could be mentioned in the post that could help as this is new and in preview? I am including my configuration details at the end of the post, omitting some parts due to the characters limit.

Before the common error, keycloak crashed because there was an additional node causing some conflicts (not sure where it came for, just mentioning if someone else wants to try my configuration).

WARN [org.jgroups.protocols.TCP] (TQ-Bundler-7,keycloak-node) JGRP000032: keycloak-node: no physical address for 5e2ddec1-8611-471f-9289-e93ed7919c6f, dropping message

I am not happy with this, but after some time “it healed itself” when the failing node was dropped and the pod restarted (No physical address for x, dropping message on node restart in jgroups cluster - Stack Overflow).

I am currently using this other guide for the infinispan deployment as it is simpler and I just want to do a POC, no need for more nodes: Infinispan Deployment: Single Cluster :: Keycloak Benchmark.
For the same reason, Keycloak is being started in dev mode and no TLS config is being used.

My environment details:

  • local kubernetes cluster v1.28.2 provided by docker desktop v4.25.2.
  • MySQL database, it is running in a docker container using the image: arm64v8/mysql:8.0.34.
  • Infinispan v14.0.19.Final, deployed following the guide from the benchmark site. It is on a separate namespace called infinispan.
  • Keycloak 23.0.0, is being deployed using vanilla kubernetes. It is in a namespace called keycloak.

single cluster deployment was successful:

➜  identity-provider git:(main) ✗ kubectl wait --for condition=WellFormed --timeout=300s infinispans.infinispan.org -n keycloak infinispan
infinispan.infinispan.org/infinispan condition met

Here is the my configuration:

identities.yml used to create the connect-secret

credentials:
  - username: admin
    password: admin
    roles:
      - admin

Infinispan CR:

apiVersion: infinispan.org/v1
kind: Infinispan
metadata:
  name: infinispan
  namespace: infinispan
  annotations:
    infinispan.org/monitoring: 'true'
spec:
  replicas: 1
  version: 14.0.19
  security:
    endpointSecretName: connect-secret
  service:
    type: DataGrid
  #logging:
  #  categories:
  #    org.infinispan: trace
  #    org.jgroups: trace

**Infinispan Cache CR** (omitted some cache definitions):

apiVersion: infinispan.org/v2alpha1
kind: Cache
metadata:
  name: sessions
  namespace: infinispan
spec:
  clusterName: infinispan
  name: sessions
  updates:
    strategy: recreate
  template: |-
    distributedCache:
      mode: "SYNC"
      owners: "2"
      statistics: "true"
      stateTransfer:
        chunkSize: 16
      
---
apiVersion: infinispan.org/v2alpha1
kind: Cache
metadata:
  name: work
  namespace: infinispan
spec:
  clusterName: infinispan
  name: work
  updates:
    strategy: recreate
  template: |-
    replicatedCache:
      mode: "SYNC"
      statistics: "true"
      stateTransfer:
        chunkSize: "16"

Keycloak’s infinispan cache configuration (omitted local-caches and some of the distributed):
I think that it may be necessary to include the jgroup configuration for the Keycloak cache configuration in the guide. Otherwise, the infinispan cluster wont be discovered.
Not sure if this is due to the fact that I am not using the Keycloak Operator.

<infinispan
        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
        xsi:schemaLocation="urn:infinispan:config:14.0 http://www.infinispan.org/schemas/infinispan-config-14.0.xsd"
        xmlns="urn:infinispan:config:14.0">

    <jgroups>
        <stack name="image-tcp" extends="tcp">
            <dns.DNS_PING dns_query="${ISPN_JGROUP_DNS_PING_DNS_QUERY}"
                          dns_record_type="A"
                          stack.combine="REPLACE" stack.position="MPING"/>

        </stack>
    </jgroups>

    <cache-container name="default">
        <transport cluster="infinispan" stack="image-tcp" node-name="keycloak-node"/>
        
        <distributed-cache name="sessions" owners="2">
            <expiration lifespan="-1"/>
            <persistence passivation="false">
                <remote-store xmlns="urn:infinispan:config:store:remote:14.0"
                              cache="sessions"
                              raw-values="true"
                              shared="true"
                              segmented="false">
                    <remote-server host="${env.KC_REMOTE_STORE_HOST}"
                                   port="${env.KC_REMOTE_STORE_PORT}"/>
                    <connection-pool max-active="16"
                                     exhausted-action="CREATE_NEW"/>
                    <security>
                        <authentication server-name="infinispan">
                            <digest username="${env.KC_REMOTE_STORE_USERNAME}"
                                    password="${env.KC_REMOTE_STORE_PASSWORD}"
                                    realm="default"/>
                        </authentication>
                    </security>
                </remote-store>
            </persistence>
        </distributed-cache>
        
        <replicated-cache name="work">
            <expiration lifespan="-1"/>
            <persistence passivation="false">
                <remote-store xmlns="urn:infinispan:config:store:remote:14.0"
                              cache="work"
                              raw-values="true"
                              shared="true"
                              segmented="false">
                    <remote-server host="${env.KC_REMOTE_STORE_HOST}"
                                   port="${env.KC_REMOTE_STORE_PORT}"/>
                    <connection-pool max-active="16"
                                     exhausted-action="CREATE_NEW"/>
                    <security>
                        <authentication server-name="infinispan">
                            <digest username="${env.KC_REMOTE_STORE_USERNAME}"
                                    password="${env.KC_REMOTE_STORE_PASSWORD}"
                                    realm="default"/>
                        </authentication>
                    </security>
                </remote-store>
            </persistence>
        </replicated-cache>
       
        <distributed-cache name="actionTokens" owners="2">
            <encoding>
                <key media-type="application/x-java-object"/>
                <value media-type="application/x-java-object"/>
            </encoding>
            <expiration max-idle="-1" lifespan="-1" interval="300000"/>
            <memory max-count="-1"/>
            <persistence passivation="false">
                <remote-store xmlns="urn:infinispan:config:store:remote:14.0"
                              cache="actionTokens"
                              raw-values="true"
                              shared="true"
                              segmented="false">
                    <remote-server host="${env.KC_REMOTE_STORE_HOST}"
                                   port="${env.KC_REMOTE_STORE_PORT}"/>
                    <connection-pool max-active="16"
                                     exhausted-action="CREATE_NEW"/>
                    <security>
                        <authentication server-name="infinispan">
                            <digest username="${env.KC_REMOTE_STORE_USERNAME}"
                                    password="${env.KC_REMOTE_STORE_PASSWORD}"
                                    realm="default"/>
                        </authentication>
                    </security>
                </remote-store>
            </persistence>
        </distributed-cache>
    </cache-container>
</infinispan>

Keycloak’s K8s deployment (omitted some env variables for: DB, admin user):

apiVersion: apps/v1
kind: Deployment
metadata:
  name: keycloak
  namespace: keycloak
  labels:
    app: keycloak
spec:
  replicas: 1
  selector:
    matchLabels:
      app: keycloak
  template:
    metadata:
      labels:
        app: keycloak
    spec:
      restartPolicy: Always
      containers:
        - name: keycloak
          image: quay.io/keycloak/keycloak:23.0.0
          args:
            - --verbose
            - start-dev
          imagePullPolicy: IfNotPresent
          env:
            - name: KC_CACHE
              value: ispn
            - name: KC_REMOTE_STORE_HOST
              value: infinispan.infinispan.svc
            - name: ISPN_JGROUP_DNS_PING_DNS_QUERY
              value: infinispan-ping.infinispan.svc.cluster.local
            - name: KC_REMOTE_STORE_PORT
              value: "11222"
            - name: KC_REMOTE_STORE_USERNAME
              value: admin
            - name: KC_REMOTE_STORE_PASSWORD
              value: admin
            - name: KC_CACHE_CONFIG_FILE
              value: custom-cache-ispn.xml
            - name: JAVA_OPTS_APPEND
              value: >
                -Djgroups.thread_dumps_threshold=1 -Djgroups.dns.query=keycloak-headless -Djboss.site.name=sso
          ports:
            - containerPort: 8080
          volumeMounts:
            - name: cache-config
              mountPath: /opt/keycloak/conf/custom-cache-ispn.xml
              subPath: custom-cache-ispn.xml
              readOnly: true
      volumes:
        - name: cache-config
          configMap:
            name: cache-config
            items:
              - key: custom-cache-ispn.xml
                path: custom-cache-ispn.xml

Logs:

Appending additional Java properties to JAVA_OPTS: -Djgroups.thread_dumps_threshold=1 -Djgroups.dns.query=keycloak-headless -Djboss.site.name=sso

Updating the configuration and installing your custom providers, if any. Please wait.
2023-11-29 17:49:20,871 INFO  [io.quarkus.deployment.QuarkusAugmentor] (main) Quarkus augmentation completed in 5234ms
2023-11-29 17:49:21,705 INFO  [org.keycloak.quarkus.runtime.hostname.DefaultHostnameProvider] (main) Hostname settings: Base URL: <unset>, Hostname: <request>, Strict HTTPS: false, Path: <request>, Strict BackChannel: false, Admin URL: <unset>, Admin: <request>, Port: -1, Proxied: false
2023-11-29 17:49:22,569 WARN  [io.quarkus.runtime.configuration.DeprecatedRuntimePropertiesRecorder] (main) The 'quarkus.log.console.color' config property is deprecated and should not be used anymore
2023-11-29 17:49:23,294 WARN  [org.infinispan.PERSISTENCE] (keycloak-cache-init) ISPN000554: jboss-marshalling is deprecated and planned for removal
2023-11-29 17:49:23,401 INFO  [org.infinispan.CONTAINER] (keycloak-cache-init) ISPN000556: Starting user marshaller 'org.infinispan.jboss.marshalling.core.JBossUserMarshaller'
2023-11-29 17:49:23,583 INFO  [org.infinispan.CLUSTER] (keycloak-cache-init) ISPN000078: Starting JGroups channel `infinispan` with stack `image-tcp`
2023-11-29 17:49:23,583 INFO  [org.jgroups.JChannel] (keycloak-cache-init) local_addr: 923a1b36-fefa-4e1b-af81-6e93670e8758, name: keycloak-node
2023-11-29 17:49:23,591 INFO  [org.jgroups.protocols.FD_SOCK2] (keycloak-cache-init) server listening on *.57800
2023-11-29 17:49:23,679 INFO  [org.infinispan.CLUSTER] (keycloak-cache-init) ISPN000094: Received new cluster view for channel infinispan: [infinispan-0-4262|42] (2) [infinispan-0-4262, keycloak-node]
2023-11-29 17:49:23,743 INFO  [org.infinispan.CLUSTER] (keycloak-cache-init) ISPN000079: Channel `infinispan` local address is `keycloak-node`, physical addresses are `[10.1.0.85:7800]`
2023-11-29 17:49:23,761 WARN  [org.infinispan.CONFIG] (keycloak-cache-init) ISPN000569: Unable to persist Infinispan internal caches as no global state enabled
2023-11-29 17:49:23,946 INFO  [org.infinispan.LIFECYCLE] (jgroups-5,keycloak-node) [Context=org.infinispan.PERMISSIONS] ISPN100002: Starting rebalance with members [infinispan-0-4262, keycloak-node], phase READ_OLD_WRITE_ALL, topology id 102
2023-11-29 17:49:23,965 INFO  [org.infinispan.LIFECYCLE] (non-blocking-thread-keycloak-node-p2-t5) [Context=org.infinispan.PERMISSIONS] ISPN100010: Finished rebalance with members [infinispan-0-4262, keycloak-node], topology id 102
2023-11-29 17:49:23,973 INFO  [org.infinispan.LIFECYCLE] (jgroups-6,keycloak-node) [Context=org.infinispan.ROLES] ISPN100002: Starting rebalance with members [infinispan-0-4262, keycloak-node], phase READ_OLD_WRITE_ALL, topology id 102
2023-11-29 17:49:23,976 INFO  [org.infinispan.LIFECYCLE] (non-blocking-thread-keycloak-node-p2-t8) [Context=org.infinispan.ROLES] ISPN100010: Finished rebalance with members [infinispan-0-4262, keycloak-node], topology id 102
2023-11-29 17:49:23,981 INFO  [org.infinispan.LIFECYCLE] (jgroups-5,keycloak-node) [Context=org.infinispan.CONFIG] ISPN100002: Starting rebalance with members [infinispan-0-4262, keycloak-node], phase READ_OLD_WRITE_ALL, topology id 102
2023-11-29 17:49:23,992 INFO  [org.infinispan.LIFECYCLE] (non-blocking-thread-keycloak-node-p2-t11) [Context=org.infinispan.CONFIG] ISPN100010: Finished rebalance with members [infinispan-0-4262, keycloak-node], topology id 102
2023-11-29 17:49:24,124 INFO  [org.infinispan.HOTROD] (blocking-thread-keycloak-node-p3-t2) ISPN004108: Native IOUring transport not available, using NIO instead: io.netty.incubator.channel.uring.IOUring
2023-11-29 17:49:24,260 INFO  [org.infinispan.HOTROD] (blocking-thread-keycloak-node-p3-t2) ISPN004021: Infinispan version: Infinispan 'Flying Saucer' 14.0.19.Final
2023-11-29 17:49:24,267 INFO  [org.infinispan.HOTROD] (Thread-0) ISPN004006: Server sent new topology view (id=228126203, age=0) containing 1 addresses: [10.1.0.69/<unresolved>:11222]
2023-11-29 17:49:24,269 INFO  [org.infinispan.HOTROD] (Thread-0) ISPN004014: New server added(10.1.0.69/<unresolved>:11222), adding to the pool.
2023-11-29 17:49:24,271 INFO  [org.infinispan.HOTROD] (Thread-0) ISPN004016: Server not in cluster anymore(infinispan.infinispan.svc/<unresolved>:11222), removing from the pool.
2023-11-29 17:49:24,289 INFO  [org.infinispan.LIFECYCLE] (jgroups-5,keycloak-node) [Context=actionTokens] ISPN100002: Starting rebalance with members [infinispan-0-4262, keycloak-node], phase READ_OLD_WRITE_ALL, topology id 96
2023-11-29 17:49:24,293 INFO  [org.infinispan.LIFECYCLE] (non-blocking-thread-keycloak-node-p2-t14) [Context=actionTokens] ISPN100010: Finished rebalance with members [infinispan-0-4262, keycloak-node], topology id 96
2023-11-29 17:49:24,359 INFO  [org.infinispan.HOTROD] (blocking-thread-keycloak-node-p3-t2) ISPN004021: Infinispan version: Infinispan 'Flying Saucer' 14.0.19.Final
2023-11-29 17:49:24,360 INFO  [org.infinispan.HOTROD] (Thread-0) ISPN004006: Server sent new topology view (id=228126203, age=0) containing 1 addresses: [10.1.0.69/<unresolved>:11222]
2023-11-29 17:49:24,361 INFO  [org.infinispan.HOTROD] (Thread-0) ISPN004014: New server added(10.1.0.69/<unresolved>:11222), adding to the pool.
2023-11-29 17:49:24,363 INFO  [org.infinispan.HOTROD] (Thread-0) ISPN004016: Server not in cluster anymore(infinispan.infinispan.svc/<unresolved>:11222), removing from the pool.
2023-11-29 17:49:24,370 INFO  [org.infinispan.LIFECYCLE] (jgroups-5,keycloak-node) [Context=authenticationSessions] ISPN100002: Starting rebalance with members [infinispan-0-4262, keycloak-node], phase READ_OLD_WRITE_ALL, topology id 96
2023-11-29 17:49:24,372 INFO  [org.infinispan.LIFECYCLE] (jgroups-5,keycloak-node) [Context=authenticationSessions] ISPN100010: Finished rebalance with members [infinispan-0-4262, keycloak-node], topology id 96
2023-11-29 17:49:24,410 INFO  [org.infinispan.HOTROD] (blocking-thread-keycloak-node-p3-t2) ISPN004021: Infinispan version: Infinispan 'Flying Saucer' 14.0.19.Final
2023-11-29 17:49:24,411 INFO  [org.infinispan.HOTROD] (Thread-0) ISPN004006: Server sent new topology view (id=228126203, age=0) containing 1 addresses: [10.1.0.69/<unresolved>:11222]
2023-11-29 17:49:24,411 INFO  [org.infinispan.HOTROD] (Thread-0) ISPN004014: New server added(10.1.0.69/<unresolved>:11222), adding to the pool.
2023-11-29 17:49:24,413 INFO  [org.infinispan.HOTROD] (Thread-0) ISPN004016: Server not in cluster anymore(infinispan.infinispan.svc/<unresolved>:11222), removing from the pool.
2023-11-29 17:49:24,437 INFO  [org.infinispan.LIFECYCLE] (jgroups-11,keycloak-node) [Context=clientSessions] ISPN100002: Starting rebalance with members [infinispan-0-4262, keycloak-node], phase READ_OLD_WRITE_ALL, topology id 58
2023-11-29 17:49:24,439 ERROR [org.infinispan.CLUSTER] (jgroups-5,keycloak-node) ISPN000475: Error processing response for request 18 from infinispan-0-4262: java.lang.ClassNotFoundException: org.infinispan.server.hotrod.HotRodServer$ToEmptyBytesKeyValueFilterConverter$ToEmptyBytesKeyValueFilterConverterExternalizer
        at java.base/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641)
        at java.base/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188)
        at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:525)
        at io.quarkus.bootstrap.runner.RunnerClassLoader.loadClass(RunnerClassLoader.java:115)
        at io.quarkus.bootstrap.runner.RunnerClassLoader.loadClass(RunnerClassLoader.java:65)
        at java.base/java.lang.Class.forName0(Native Method)
        at java.base/java.lang.Class.forName(Class.java:467)
        at org.infinispan.marshall.exts.ClassExternalizer.readObject(ClassExternalizer.java:37)
        at org.infinispan.marshall.exts.ClassExternalizer.readObject(ClassExternalizer.java:12)
        at org.infinispan.marshall.core.GlobalMarshaller.readWithExternalizer(GlobalMarshaller.java:727)
        at org.infinispan.marshall.core.GlobalMarshaller.readNonNullableObject(GlobalMarshaller.java:708)
        at org.infinispan.marshall.core.GlobalMarshaller.readNullableObject(GlobalMarshaller.java:357)
        at org.infinispan.marshall.core.BytesObjectInput.readObject(BytesObjectInput.java:32)
        at org.infinispan.marshall.core.GlobalMarshaller.readAnnotated(GlobalMarshaller.java:732)
        at org.infinispan.marshall.core.GlobalMarshaller.readNonNullableObject(GlobalMarshaller.java:712)
        at org.infinispan.marshall.core.GlobalMarshaller.readNullableObject(GlobalMarshaller.java:357)
        at org.infinispan.marshall.core.BytesObjectInput.readObject(BytesObjectInput.java:32)
        at org.infinispan.notifications.cachelistener.filter.KeyValueFilterConverterAsCacheEventFilterConverter$Externalizer.readObject(KeyValueFilterConverterAsCacheEventFilterConverter.java:75)
        at org.infinispan.notifications.cachelistener.filter.KeyValueFilterConverterAsCacheEventFilterConverter$Externalizer.readObject(KeyValueFilterConverterAsCacheEventFilterConverter.java:66)
        at org.infinispan.marshall.core.GlobalMarshaller.readWithExternalizer(GlobalMarshaller.java:727)
        at org.infinispan.marshall.core.GlobalMarshaller.readNonNullableObject(GlobalMarshaller.java:708)
        at org.infinispan.marshall.core.GlobalMarshaller.readNullableObject(GlobalMarshaller.java:357)
        at org.infinispan.marshall.core.BytesObjectInput.readObject(BytesObjectInput.java:32)
        at org.infinispan.notifications.cachelistener.cluster.ClusterListenerReplicateCallable$Externalizer.readObject(ClusterListenerReplicateCallable.java:179)
        at org.infinispan.notifications.cachelistener.cluster.ClusterListenerReplicateCallable$Externalizer.readObject(ClusterListenerReplicateCallable.java:143)
        at org.infinispan.marshall.core.GlobalMarshaller.readWithExternalizer(GlobalMarshaller.java:727)
        at org.infinispan.marshall.core.GlobalMarshaller.readNonNullableObject(GlobalMarshaller.java:708)
        at org.infinispan.marshall.core.GlobalMarshaller.readNullableObject(GlobalMarshaller.java:357)
        at org.infinispan.marshall.core.BytesObjectInput.readObject(BytesObjectInput.java:32)
        at org.infinispan.commons.marshall.MarshallUtil.lambda$unmarshallCollection$0(MarshallUtil.java:295)
        at org.infinispan.commons.marshall.MarshallUtil.unmarshallCollection(MarshallUtil.java:278)
        at org.infinispan.commons.marshall.MarshallUtil.unmarshallCollection(MarshallUtil.java:295)
        at org.infinispan.marshall.exts.CollectionExternalizer.readObject(CollectionExternalizer.java:111)
        at org.infinispan.marshall.exts.CollectionExternalizer.readObject(CollectionExternalizer.java:28)
        at org.infinispan.marshall.core.GlobalMarshaller.readWithExternalizer(GlobalMarshaller.java:727)
        at org.infinispan.marshall.core.GlobalMarshaller.readNonNullableObject(GlobalMarshaller.java:708)
        at org.infinispan.marshall.core.GlobalMarshaller.readNullableObject(GlobalMarshaller.java:357)
        at org.infinispan.marshall.core.BytesObjectInput.readObject(BytesObjectInput.java:32)
        at org.infinispan.remoting.responses.SuccessfulResponse$Externalizer.readObject(SuccessfulResponse.java:71)
        at org.infinispan.remoting.responses.SuccessfulResponse$Externalizer.readObject(SuccessfulResponse.java:63)
        at org.infinispan.marshall.core.GlobalMarshaller.readWithExternalizer(GlobalMarshaller.java:727)
        at org.infinispan.marshall.core.GlobalMarshaller.readNonNullableObject(GlobalMarshaller.java:708)
        at org.infinispan.marshall.core.GlobalMarshaller.readNullableObject(GlobalMarshaller.java:357)
        at org.infinispan.marshall.core.GlobalMarshaller.objectFromObjectInput(GlobalMarshaller.java:191)
        at org.infinispan.marshall.core.GlobalMarshaller.objectFromByteBuffer(GlobalMarshaller.java:220)
        at org.infinispan.remoting.transport.jgroups.JGroupsTransport.processResponse(JGroupsTransport.java:1571)
        at org.infinispan.remoting.transport.jgroups.JGroupsTransport.processMessage(JGroupsTransport.java:1479)
        at org.infinispan.remoting.transport.jgroups.JGroupsTransport$ChannelCallbacks.up(JGroupsTransport.java:1681)
        at org.jgroups.JChannel.up(JChannel.java:733)
        at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:936)
        at org.jgroups.protocols.FRAG2.up(FRAG2.java:139)
        at org.jgroups.protocols.FlowControl.up(FlowControl.java:245)
        at org.jgroups.protocols.FlowControl.up(FlowControl.java:245)
        at org.jgroups.protocols.pbcast.GMS.up(GMS.java:845)
        at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:226)
        at org.jgroups.protocols.UNICAST3.deliverMessage(UNICAST3.java:1083)
        at org.jgroups.protocols.UNICAST3.addMessage(UNICAST3.java:822)
        at org.jgroups.protocols.UNICAST3.handleDataReceived(UNICAST3.java:804)
        at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:453)
        at org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:680)
        at org.jgroups.protocols.VERIFY_SUSPECT2.up(VERIFY_SUSPECT2.java:105)
        at org.jgroups.protocols.FailureDetection.up(FailureDetection.java:180)
        at org.jgroups.protocols.FD_SOCK2.up(FD_SOCK2.java:188)
        at org.jgroups.protocols.MERGE3.up(MERGE3.java:274)
        at org.jgroups.protocols.Discovery.up(Discovery.java:294)
        at org.jgroups.stack.Protocol.up(Protocol.java:340)
        at org.jgroups.protocols.TP.passMessageUp(TP.java:1184)
        at org.jgroups.util.SubmitToThreadPool$SingleMessageHandler.run(SubmitToThreadPool.java:107)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base/java.lang.Thread.run(Thread.java:840)

2023-11-29 17:52:24,153 ERROR [org.infinispan.CONFIG] (keycloak-cache-init) ISPN000660: DefaultCacheManager start failed, stopping any running components: org.infinispan.commons.CacheException
        at org.infinispan.statetransfer.StateTransferManagerImpl.waitForInitialStateTransferToComplete(StateTransferManagerImpl.java:243)
        at org.infinispan.cache.impl.CacheImpl.start(CacheImpl.java:1013)
        at org.infinispan.cache.impl.AbstractDelegatingCache.start(AbstractDelegatingCache.java:504)
        at org.infinispan.manager.DefaultCacheManager.wireAndStartCache(DefaultCacheManager.java:727)
        at org.infinispan.manager.DefaultCacheManager.createCache(DefaultCacheManager.java:673)
        at org.infinispan.manager.DefaultCacheManager.internalGetCache(DefaultCacheManager.java:562)
        at org.infinispan.manager.DefaultCacheManager.getCache(DefaultCacheManager.java:525)
        at org.infinispan.security.actions.GetCacheAction.run(GetCacheAction.java:26)
        at org.infinispan.security.actions.GetCacheAction.run(GetCacheAction.java:14)
        at org.infinispan.security.Security.doPrivileged(Security.java:56)
        at org.infinispan.globalstate.impl.SecurityActions.doPrivileged(SecurityActions.java:30)
        at org.infinispan.globalstate.impl.SecurityActions.getCache(SecurityActions.java:39)
        at org.infinispan.globalstate.impl.GlobalConfigurationManagerImpl.start(GlobalConfigurationManagerImpl.java:114)
        at org.infinispan.globalstate.impl.CorePackageImpl$2.start(CorePackageImpl.java:61)
        at org.infinispan.globalstate.impl.CorePackageImpl$2.start(CorePackageImpl.java:48)
        at org.infinispan.factories.impl.BasicComponentRegistryImpl.invokeStart(BasicComponentRegistryImpl.java:616)
        at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:607)
        at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:576)
        at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:807)
        at org.infinispan.factories.AbstractComponentRegistry.internalStart(AbstractComponentRegistry.java:379)
        at org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:252)
        at org.infinispan.manager.DefaultCacheManager.internalStart(DefaultCacheManager.java:779)
        at org.infinispan.manager.DefaultCacheManager.start(DefaultCacheManager.java:747)
        at org.infinispan.manager.DefaultCacheManager.<init>(DefaultCacheManager.java:411)
        at org.keycloak.quarkus.runtime.storage.legacy.infinispan.CacheManagerFactory.startCacheManager(CacheManagerFactory.java:96)
        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base/java.lang.Thread.run(Thread.java:840)
Caused by: java.lang.InterruptedException
        at java.base/java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386)
        at java.base/java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2096)
        at org.infinispan.statetransfer.StateTransferManagerImpl.waitForInitialStateTransferToComplete(StateTransferManagerImpl.java:236)
        ... 28 more
2 Likes
5

From the Kubernetes side, the problem was solved after asking in the github discussion that was linked in the documentation: Active-passive multi-site deployments with Keycloak 23 · keycloak/keycloak · Discussion #25269 · GitHub

The configuration files can be found there too and it should be done with the Keycloak Operator.

1 Like
6

Hi @madhuchamara383, since we are facing the same issue that you initially posted in this thread, did you already find a solution for the local docker compose setup?

7

I have not been able to get this to work on either KC22 or KC23 using docker the exception is always the same. I am using the 14.0.21.Final version of Infinispan.

This is the error that I keep getting. what is the problem with the actionTokens ?

ISPN000660: Cache actionTokens start failed, stopping any running components: org.infinispan.commons.CacheConfigurationException: ISPN000659: Component ComponentWrapper{name=org.infinispan.persistence.manager.PersistenceManager, instance=org.infinispan.persistence.manager.PersistenceManagerImpl@4ab8a8b3, status=FAILED} failed

KC Side

<?xml version="1.0" encoding="UTF-8"?> 

<jgroups>
	<stack name="dns_ping_tcp">
		<!-- jgroups.tcp.address is deprecated and will be removed, see ISPN-11867 -->
		<TCP bind_addr="${jgroups.bind.address,jgroups.tcp.address:SITE_LOCAL}"
			 bind_port="${jgroups.bind.port,jgroups.tcp.port:7800}"
			 diag.enabled="${jgroups.diag.enabled:false}"
			 thread_naming_pattern="pl"
			 send_buf_size="640k"
			 sock_conn_timeout="500"
			 bundler_type="transfer-queue"
			 bundler.max_size="${jgroups.bundler.max_size:64000}"
			 logical_addr_cache_expiration="360000"
			 port_range="0"
			 thread_pool.min_threads="${jgroups.thread_pool.min_threads:0}"
			 thread_pool.max_threads="${jgroups.thread_pool.max_threads:200}"
			 thread_pool.keep_alive_time="60000"
			 thread_dumps_threshold="${jgroups.thread_dumps_threshold:10000}"
			 use_virtual_threads="${jgroups.thread.virtual,org.infinispan.threads.virtual:false}"
		/>
		<RED/>
		<dns.DNS_PING dns_query="keycloak-jgrp-svc-blue.XXXXX.svc.cluster.local"
					  num_discovery_runs="5"
					  stack.combine="REPLACE"
					  stack.position="MPING"
		/>
		<MERGE3 min_interval="10000" max_interval="30000"/>
		<FD_SOCK2/>
		<!-- Suspect node `timeout` to `timeout + timeout_check_interval` millis after the last heartbeat -->
		<FD_ALL3 timeout="10000" interval="2000"/>
		<VERIFY_SUSPECT2 timeout="1000"/>
		<pbcast.NAKACK2 use_mcast_xmit="false"
						xmit_interval="200" xmit_table_num_rows="50"
						xmit_table_msgs_per_row="1024"
						xmit_table_max_compaction_time="30000"
						resend_last_seqno="true"/>
		<UNICAST3 conn_close_timeout="5000"
				  xmit_interval="200"
				  xmit_table_num_rows="50"
				  xmit_table_msgs_per_row="1024"
				  xmit_table_max_compaction_time="30000"/>
		<pbcast.STABLE desired_avg_gossip="5000" max_bytes="1M"/>
		<pbcast.GMS print_local_addr="false"
					join_timeout="${jgroups.join_timeout:2000}"/>
		<UFC max_credits="4m" min_threshold="0.40"/>
		<MFC max_credits="4m" min_threshold="0.40"/>
		<FRAG4/>
	</stack>
</jgroups>

<cache-container name="keycloak" statistics="true">
	<transport cluster="infinispan" node-name="keycloak-node" site="site-keycloak" lock-timeout="60000" />
	<local-cache name="realms" simple-cache="true" statistics="true">
		<encoding>
			<key media-type="application/x-java-object"/>
			<value media-type="application/x-java-object"/>
		</encoding>
		<expiration max-idle="-1" lifespan="-1"/>
		<memory max-count="10000"/>
	</local-cache>

	<local-cache name="users" simple-cache="true" statistics="true">
		<encoding>
			<key media-type="application/x-java-object"/>
			<value media-type="application/x-java-object"/>
		</encoding>
		<expiration max-idle="900000"/>
		<memory max-count="10000"/>
	</local-cache>

	<local-cache name="keys" simple-cache="true" statistics="true">
		<encoding>
			<key media-type="application/x-java-object"/>
			<value media-type="application/x-java-object"/>
		</encoding>
		<expiration max-idle="3600000"/>
		<memory max-count="1000"/>
	</local-cache>

	<local-cache name="authorization" simple-cache="true" statistics="true">
		<encoding>
			<key media-type="application/x-java-object"/>
			<value media-type="application/x-java-object"/>
		</encoding>
		<expiration max-idle="900000"/>
		<memory max-count="10000"/>
	</local-cache>

	<distributed-cache name="sessions"  owners="4" mode="SYNC" remote-timeout="17500" statistics="true">
		<locking concurrency-level="1000" acquire-timeout="15000" striping="false"/>
		<expiration lifespan="900000000000000000"/>
		<persistence passivation="false">
			<remote-store xmlns="urn:infinispan:config:store:remote:14.0"
						  cache="sessions"
						  purge="false"
						  preload="false"
						  raw-values="true"
						  marshaller="org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory"
						  shared="true"
						  segmented="false"
						  >
				<remote-server host="${env.KEYCLOAK_REMOTE_ISPN_URL}" port="11222"/>
				<connection-pool max-active="16"
								 exhausted-action="CREATE_NEW"/>
				<security>
					<authentication>
						<digest
								username="${env.KEYCLOAK_REMOTE_ISPN_USERNAME:admin}"
								password="${env.KEYCLOAK_REMOTE_ISPN_PASSWORD:admin}"
								realm="default"/>
					</authentication>
					<!--<encryption protocol="TLSv1.3"
								sni-hostname="${env.KC_REMOTE_STORE_HOST}">
						<truststore filename="/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt"
									type="pem"/>
					</encryption>-->
				</security>
			</remote-store>
		</persistence>
		<state-transfer timeout="60000"/>
	</distributed-cache>

	<distributed-cache name="authenticationSessions" owners="4" statistics="true">
		<expiration lifespan="-1"/>
		<persistence passivation="false">
			<remote-store xmlns="urn:infinispan:config:store:remote:14.0"
						  cache="sessions"
						  purge="false"
						  preload="false"
						  raw-values="true"
						  marshaller="org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory"
						  shared="true"
						  segmented="false"
			>

				<remote-server host="${env.KEYCLOAK_REMOTE_ISPN_URL}" port="11222"/>
				<connection-pool max-active="16"
								 exhausted-action="CREATE_NEW"/>
				<security>
					<authentication >
						<digest
								username="${env.KEYCLOAK_REMOTE_ISPN_USERNAME:admin}"
								password="${env.KEYCLOAK_REMOTE_ISPN_PASSWORD:admin}"
								realm="default"/>
					</authentication>
					<!--<encryption protocol="TLSv1.3"
								sni-hostname="${env.KC_REMOTE_STORE_HOST}">
						<truststore filename="/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt"
									type="pem"/>
					</encryption>-->
				</security>
			</remote-store>
		</persistence>
	</distributed-cache>



	<distributed-cache name="offlineSessions" owners="2" statistics="true">
		<expiration lifespan="-1"/>
		<persistence passivation="false">
			<remote-store xmlns="urn:infinispan:config:store:remote:14.0"
						  cache="sessions"
						  purge="false"
						  preload="false"
						  raw-values="true"
						  marshaller="org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory"
						  shared="true"
						  segmented="false"
			>
				<remote-server host="${env.KEYCLOAK_REMOTE_ISPN_URL}" port="11222" />
				<connection-pool max-active="16"
								 exhausted-action="CREATE_NEW"/>
				<security>
					<authentication >
						<digest
								username="${env.KEYCLOAK_REMOTE_ISPN_USERNAME:admin}"
								password="${env.KEYCLOAK_REMOTE_ISPN_PASSWORD:admin}"
								realm="default"/>
					</authentication>
					<!--<encryption protocol="TLSv1.3"
								sni-hostname="${env.KC_REMOTE_STORE_HOST}">
						<truststore filename="/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt"
									type="pem"/>
					</encryption>-->
				</security>
			</remote-store>
		</persistence>
	</distributed-cache>

	<!--<distributed-cache name="clientSessions"  owners="4" mode="SYNC" remote-timeout="17500" statistics="true">
        <locking concurrency-level="1000" acquire-timeout="15000" striping="false"/>
        <expiration lifespan="900000000000000000"/>
        <state-transfer timeout="60000"/>
    </distributed-cache>-->

	<distributed-cache name="clientSessions" owners="2" statistics="true">
		<expiration lifespan="-1"/>
		<persistence passivation="false">
			<remote-store xmlns="urn:infinispan:config:store:remote:14.0"
						  cache="sessions"
						  purge="false"
						  preload="false"
						  raw-values="true"
						  marshaller="org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory"
						  shared="true"
						  segmented="false"
			>
				<remote-server host="${env.KEYCLOAK_REMOTE_ISPN_URL}" port="11222"/>
				<connection-pool max-active="16"
								 exhausted-action="CREATE_NEW"/>
				<security>
					<authentication >
						<digest
								username="${env.KEYCLOAK_REMOTE_ISPN_USERNAME:admin}"
								password="${env.KEYCLOAK_REMOTE_ISPN_PASSWORD:admin}"
								realm="default"/>
					</authentication>
					<!--<encryption protocol="TLSv1.3"
								sni-hostname="${env.KC_REMOTE_STORE_HOST}">
						<truststore filename="/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt"
									type="pem"/>
					</encryption>-->
				</security>
			</remote-store>
		</persistence>
	</distributed-cache>

	<distributed-cache name="offlineClientSessions" owners="2" statistics="true">
		<expiration lifespan="-1"/>
		<persistence passivation="false">
			<remote-store xmlns="urn:infinispan:config:store:remote:14.0"
						  cache="sessions"
						  purge="false"
						  preload="false"
						  raw-values="true"
						  marshaller="org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory"
						  shared="true"
						  segmented="false"
			>
				<remote-server host="${env.KEYCLOAK_REMOTE_ISPN_URL}" port="11222"/>
				<connection-pool max-active="16"
								 exhausted-action="CREATE_NEW"/>
				<security>
					<authentication >
						<digest
								username="${env.KEYCLOAK_REMOTE_ISPN_USERNAME:admin}"
								password="${env.KEYCLOAK_REMOTE_ISPN_PASSWORD:admin}"
								realm="default"/>
					</authentication>
					<!--<encryption protocol="TLSv1.3"
								sni-hostname="${env.KC_REMOTE_STORE_HOST}">
						<truststore filename="/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt"
									type="pem"/>
					</encryption>-->
				</security>
			</remote-store>
		</persistence>
	</distributed-cache>

	<distributed-cache name="loginFailures" owners="2" statistics="true">
		<expiration lifespan="-1"/>
		<persistence passivation="false">
			<remote-store xmlns="urn:infinispan:config:store:remote:14.0"
						  cache="sessions"
						  purge="false"
						  preload="false"
						  raw-values="true"
						  marshaller="org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory"
						  shared="true"
						  segmented="false"
			>
				<remote-server host="${env.KEYCLOAK_REMOTE_ISPN_URL}" port="11222"/>
				<connection-pool max-active="16"
								 exhausted-action="CREATE_NEW"/>
				<security>
					<authentication >
						<digest
								username="${env.KEYCLOAK_REMOTE_ISPN_USERNAME:admin}"
								password="${env.KEYCLOAK_REMOTE_ISPN_PASSWORD:admin}"
								realm="default"/>
					</authentication>
					<!--<encryption protocol="TLSv1.3"
								sni-hostname="${env.KC_REMOTE_STORE_HOST}">
						<truststore filename="/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt"
									type="pem"/>
					</encryption>-->
				</security>
			</remote-store>
		</persistence>
	</distributed-cache>
	<!--<distributed-cache name="loginFailures"  owners="4" mode="SYNC" remote-timeout="17500" statistics="true">
        <locking concurrency-level="1000" acquire-timeout="15000" striping="false"/>
        <expiration lifespan="900000000000000000"/>
        <state-transfer timeout="60000"/>
    </distributed-cache>-->



	<!--<replicated-cache name="work" mode="SYNC">
		<expiration lifespan="-1"/>
	</replicated-cache>-->

	<replicated-cache name="work" mode="SYNC" statistics="true">
        <expiration lifespan="-1"/>
        <persistence passivation="false">
            <remote-store xmlns="urn:infinispan:config:store:remote:14.0"
                          cache="work"
						  purge="false"
						  preload="false"
                          segmented="false"
						  shared="true"
                          raw-values="true"
                          marshaller="org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory">
                <remote-server host="${env.KEYCLOAK_REMOTE_ISPN_URL}" port="11222"/>
                <connection-pool max-active="16"
                                 exhausted-action="CREATE_NEW"/>
                <security>
                    <authentication >
                        <digest username="${env.KC_REMOTE_STORE_USERNAME:admin}"
                                password="${env.KC_REMOTE_STORE_PASSWORD:admin}"
                                realm="default"/>
                    </authentication>
                </security>
            </remote-store>
        </persistence>
    </replicated-cache>


	<!--<distributed-cache name="actionTokens"  owners="4" mode="SYNC" remote-timeout="17500" statistics="true">
        <locking concurrency-level="1000" acquire-timeout="15000" striping="false"/>
        <expiration lifespan="900000000000000000" max-idle="-1" interval="300000"/>
        <memory/>
        <state-transfer timeout="60000"/>
    </distributed-cache>-->


	<distributed-cache name="actionTokens" owners="2" statistics="true">
		<encoding>
			<key media-type="application/x-java-object"/>
			<value media-type="application/x-java-object"/>
		</encoding>
		<expiration max-idle="-1" lifespan="-1" interval="300000"/>
		<memory max-count="-1"/>
		<persistence passivation="false">
			<remote-store xmlns="urn:infinispan:config:store:remote:14.0"
						  cache="sessions"
						  purge="false"
						  preload="false"
						  raw-values="true"
						  marshaller="org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory"
						  shared="true"
						  segmented="false"
			>
				<remote-server host="${env.KEYCLOAK_REMOTE_ISPN_URL}" port="11222"/>
				<connection-pool max-active="16"
								 exhausted-action="CREATE_NEW"/>
				<security>
					<authentication >
						<digest
								username="${env.KEYCLOAK_REMOTE_ISPN_USERNAME:admin}"
								password="${env.KEYCLOAK_REMOTE_ISPN_PASSWORD:admin}"
								realm="default"/>
					</authentication>
					<!--<encryption protocol="TLSv1.3"
								sni-hostname="${env.KC_REMOTE_STORE_HOST}">
						<truststore filename="/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt"
									type="pem"/>
					</encryption>-->
				</security>

			</remote-store>
		</persistence>
		<!--            <state-transfer enabled="false"/>-->
	</distributed-cache>
</cache-container>

Infinispan Side

<infinispan
xmlns:xsi=“http://www.w3.org/2001/XMLSchema-instance”
xsi:schemaLocation=“urn:infinispan:config:14.0 https://infinispan.org/schemas/infinispan-config-14.0.xsd
urn:infinispan:server:14.0 https://infinispan.org/schemas/infinispan-server-14.0.xsd”
xmlns=“urn:infinispan:config:14.0”
xmlns:server=“urn:infinispan:server:14.0”

<!-- TODO configure JGROUPS tcp Stack with encryption -->

<!-- see https://docs.jboss.org/infinispan/13.0/configdocs/infinispan-config-13.0.html -->
<jgroups>
    <!-- Allows custom jgroups configuration -->
    <stack-file path="/opt/infinispan/server/conf/default-jgroups-kubernetes.xml" name="custom"/>
</jgroups>

<cache-container name="default" statistics="true">

    <transport stack="custom" />

    <replicated-cache-configuration name="replicated-cache-cfg">
        <encoding>
            <key media-type="application/x-jboss-marshalling"/>
            <value media-type="application/x-jboss-marshalling"/>
        </encoding>

        <expiration lifespan="900000000000000000"/>
    </replicated-cache-configuration>

    <distributed-cache-configuration name="distributed-cache-cfg">
        <encoding>
            <key media-type="application/x-jboss-marshalling"/>
            <value media-type="application/x-jboss-marshalling"/>
        </encoding>

        <expiration lifespan="900000000000000000"/>
    </distributed-cache-configuration>

    <distributed-cache name="sessions" owners="4" configuration="distributed-cache-cfg">
        <persistence passivation="true">
            <!-- purge="false" fetch-state="false" see:  https://infinispan.org/docs/stable/titles/configuring/configuring.html#configuring_cache_stores-persistence-->
            <file-store preload="true" purge="false" path="../mydata/sessions">
            </file-store>
        </persistence>
    </distributed-cache>
    <!--Number of owners needs to be n-1 for n number of replicas-->
    <distributed-cache name="authenticationSessions" owners="4" configuration="distributed-cache-cfg">
    </distributed-cache>

    <distributed-cache name="offlineSessions" owners="4" configuration="distributed-cache-cfg">
    </distributed-cache>

    <distributed-cache name="clientSessions" owners="4" configuration="distributed-cache-cfg">
    </distributed-cache>

    <distributed-cache name="offlineClientSessions" owners="4" configuration="distributed-cache-cfg">
    </distributed-cache>

    <distributed-cache name="loginFailures" owners="4" configuration="distributed-cache-cfg">
    </distributed-cache>

    <distributed-cache name="actionTokens" owners="4" configuration="distributed-cache-cfg">
        <memory max-count="-1">
        </memory>
        <expiration interval="300000" max-idle="-1"/>
    </distributed-cache>
</cache-container>

<!-- https://docs.jboss.org/infinispan/13.0/configdocs/infinispan-server-13.0.html# -->
<server xmlns="urn:infinispan:server:14.0">

    <interfaces>
        <interface name="public">
            <!-- we bind to the eth0 interface instead of a specific ip address to ease access -->
            <!--                <inet-address value="${infinispan.bind.address:127.0.0.1}"/>-->
            <match-interface value="eth0"/>
        </interface>
    </interfaces>

    <socket-bindings default-interface="public" port-offset="${infinispan.socket.binding.port-offset:0}">
        <socket-binding name="default" port="${infinispan.bind.port:11222}"/>
        <socket-binding name="memcached" port="11221"/>
    </socket-bindings>

    <security>
        <security-realms>
            <security-realm name="default">
                <!--  Uncomment to enable TLS on the realm
               <server-identities>
                    <ssl>
                        <keystore path="ispn-server.jks" relative-to="infinispan.server.config.path" password="password" alias="server" key-password="password" generate-self-signed-certificate-host="localhost"/>
                    </ssl>
                </server-identities> -->
                <properties-realm groups-attribute="Roles">
                    <user-properties path="users.properties" relative-to="infinispan.server.config.path" plain-text="true"/>
                    <group-properties path="groups.properties" relative-to="infinispan.server.config.path"/>
                </properties-realm>
            </security-realm>
        </security-realms>
    </security>

    <!-- see https://docs.jboss.org/infinispan/13.0/configdocs/infinispan-server-13.0.html#endpoints -->
    <endpoints>
        <endpoint socket-binding="default" security-realm="default">
            <hotrod-connector security-realm="default">

            </hotrod-connector>
            <rest-connector></rest-connector>
        </endpoint>
    </endpoints>
</server>
8

@aadharsharma Did you define the distributed cache for actionToken on both sides (Keycloak and infinispan)?

9

@dbubenheim Yes I did updated my earlier comment with what I have on both sides

10

with some configuration changes i was able to connect keycoak 22 with infinispan 14

11

@madhuchamara383 Sounds great! Do you mind sharing your configuration files or do you remember, what changes you implemented?

12

@dbubenheim / @madhuchamara383 can you check what is wrong with the conifgs I have shared
or share your changes please

13

hi @dbubenheim and @aadharsharma try my below configs

infinispan.xml

<infinispan
      xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
      xsi:schemaLocation="urn:infinispan:config:14.0 https://infinispan.org/schemas/infinispan-config-14.0.xsd
                            urn:infinispan:server:14.0 https://infinispan.org/schemas/infinispan-server-14.0.xsd"
      xmlns="urn:infinispan:config:14.0"
      xmlns:server="urn:infinispan:server:14.0">
       <jgroups>
        <stack name="jdbc-ping-tcp" extends="tcp">
         <TCP bind_addr="${jgroups.bind.address,jgroups.tcp.address:SITE_LOCAL}"
              bind_port="7801" /> 
         <MPING 
                  mcast_addr="239.6.1.10"
                  mcast_port="46777"
         />
         <FD_SOCK2/>
<!--        </stack>
       </jgroups> -->
<!--      <jgroups>  -->
 
   </jgroups>  
   <cache-container name="remoteKeycloak" statistics="true">
      <transport cluster="${infinispan.cluster.name:ISPN}" stack="jdbc-ping-tcp"  site="site1"/>
      
      <serialization marshaller="org.infinispan.jboss.marshalling.commons.GenericJBossMarshaller">
         <allow-list>
         <!--    <class>org.*</class> -->
             <regex>org.keycloak.*</regex>
             <regex>java.util.*</regex>
         </allow-list>
     </serialization>

      <security>
         <authorization/>
      </security>
     <serialization marshaller="org.infinispan.jboss.marshalling.commons.GenericJBossMarshaller">
            <allow-list>
                <class>org.keycloak.cluster.infinispan.WrapperClusterEvent</class>
                <regex>.*</regex>
            </allow-list>
        </serialization>      

    <replicated-cache name="work">
      <transaction mode="NON_XA" locking="PESSIMISTIC"/>
    <!--  <transaction mode="NONE"/> -->
      <encoding media-type="application/x-jboss-marshalling"/>
   </replicated-cache>
    

    <distributed-cache name="sessions" owners="1">
      <transaction mode="NON_XA" locking="PESSIMISTIC"/>
      <encoding media-type="application/x-jboss-marshalling"/>
    </distributed-cache>
    
    <distributed-cache name="authenticationSessions" owners="1">
      <transaction mode="NON_XA" locking="PESSIMISTIC"/>
      <encoding media-type="application/x-jboss-marshalling"/>
    </distributed-cache>
    
    <distributed-cache name="clientSessions" owners="1">
      <transaction mode="NON_XA" locking="PESSIMISTIC"/>
      <encoding media-type="application/x-jboss-marshalling"/>
    </distributed-cache>
    
    <distributed-cache name="offlineSessions" owners="1">
      <transaction mode="NON_XA" locking="PESSIMISTIC"/>
      <encoding media-type="application/x-jboss-marshalling"/>
   </distributed-cache>
    
    <distributed-cache name="offlineClientSessions" owners="1">
      <transaction mode="NON_XA" locking="PESSIMISTIC"/>
      <encoding media-type="application/x-jboss-marshalling"/>
   </distributed-cache>
    
    <distributed-cache name="actionTokens" owners="1">
      <transaction mode="NON_XA" locking="PESSIMISTIC"/>
      <encoding media-type="application/x-jboss-marshalling"/>
   </distributed-cache>
    
    <distributed-cache name="loginFailures" owners="1">
      <transaction mode="NON_XA" locking="PESSIMISTIC"/>
      <encoding media-type="application/x-jboss-marshalling"/>
      </distributed-cache>
   </cache-container>

   <server xmlns="urn:infinispan:server:14.0">
      <interfaces>
         <interface name="public">
            <inet-address value="${infinispan.bind.address:127.0.0.1}"/>
         </interface>
      </interfaces>

      <socket-bindings default-interface="public" port-offset="${infinispan.socket.binding.port-offset:0}">
         <socket-binding name="default" port="${infinispan.bind.port:11222}"/>
         <socket-binding name="memcached" port="11221"/>
      </socket-bindings>

      <security>
         <credential-stores>
            <credential-store name="credentials" path="credentials.pfx">
               <clear-text-credential clear-text="secret"/>
            </credential-store>
         </credential-stores>
         <security-realms>
            <security-realm name="default">
               <!-- Uncomment to enable TLS on the realm -->
               <!-- server-identities>
                  <ssl>
                     <keystore path="application.keystore"
                               password="password" alias="server"
                               generate-self-signed-certificate-host="localhost"/>
                  </ssl>
               </server-identities-->
               <properties-realm groups-attribute="Roles">
                  <user-properties path="users.properties"/>
                  <group-properties path="groups.properties"/>
               </properties-realm>
            </security-realm>
         </security-realms>
      </security>

      <endpoints socket-binding="default" security-realm="default">
    <endpoint>
      <hotrod-connector name="hotrod">
        <authentication>
          <sasl mechanisms="DIGEST-MD5"
                server-name="infinispan" />
        </authentication>
      </hotrod-connector>
      <rest-connector name="rest"/>
    </endpoint>
  </endpoints>
   </server>
</infinispan>

cache-ispn.xml

<?xml version="1.0" encoding="UTF-8"?>

<infinispan
        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
        xsi:schemaLocation="urn:infinispan:config:14.0 http://www.infinispan.org/schemas/infinispan-config-14.0.xsd"
        xmlns="urn:infinispan:config:14.0">

        <jgroups>
         <stack name="jdbc-ping-tcp" extends="tcp">
   <!--     <TCP external_addr="${env.JGROUPS_DISCOVERY_EXTERNAL_IP}" bind_port="7600"/> -->
         <JDBC_PING connection_driver="com.mysql.cj.jdbc.Driver"
                     connection_username="${env.DB_USERNAME}" connection_password="${env.DB_PASSWORD}"
                     connection_url="jdbc:mysql://${env.DB_HOST}:${env.DB_PORT}/${env.DB_DATABASE}"
                     initialize_sql="CREATE TABLE IF NOT EXISTS JGROUPSPING (own_addr varchar(200) NOT NULL, cluster_name varchar(200) NOT NULL, ping_data BINARY(124), constraint PK_JGROUPSPING PRIMARY KEY (own_addr, cluster_name));"
                     info_writer_sleep_time="500"
                     remove_all_data_on_view_change="true"
                     stack.combine="REPLACE"
                     stack.position="MPING" />
         <MERGE3 min_interval="10000"
                  max_interval="30000" />
         <FD_ALL3 timeout="40000" interval="5000" />
         <VERIFY_SUSPECT2 timeout="1500" />
         <pbcast.NAKACK2 use_mcast_xmit="fals" />
         <pbcast.STABLE desired_avg_gossip="50000" 
                        max_bytes="4M" />
         <pbcast.GMS print_local_addr="true" join_timeout="2000"/>
         <UFC max_credits="2M"
              min_threshold="0.4" />
         <MFC max_credits="2M" 
              min_threshold="0.4" />
        <FRAG2 frag_size="60K"/>
         </stack> 
        </jgroups> 

    <cache-container name="keycloak">
        <transport cluster="${infinispan.cluster.name:ISPN}"  stack="jdbc-ping-tcp" lock-timeout="60000" site="site1"/>
        <local-cache name="realms">
            <encoding>
                <key media-type="application/x-java-object"/>
                <value media-type="application/x-java-object"/>
            </encoding>
            <memory max-count="10000"/>
        </local-cache>
        <local-cache name="users">
            <encoding>
                <key media-type="application/x-java-object"/>
                <value media-type="application/x-java-object"/>
            </encoding>
            <memory max-count="10000"/>
        </local-cache>
        <local-cache name="authorization">
            <encoding>
                <key media-type="application/x-java-object"/>
                <value media-type="application/x-java-object"/>
            </encoding>
            <memory max-count="10000"/>
        </local-cache>
        <local-cache name="keys">
            <encoding>
                <key media-type="application/x-java-object"/>
                <value media-type="application/x-java-object"/>
            </encoding>
            <expiration max-idle="3600000"/>
            <memory max-count="1000"/>
        </local-cache>

        <distributed-cache name="sessions" owners="1">
            <remote-store xmlns="urn:infinispan:config:store:remote:14.0"
                          cache="sessions"
                          purge="false"
                          preload="false"
                          segmented="false"
                          shared="true"
                          raw-values="true"
                          marshaller="org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory">
                <remote-server host="infinispan-01" port="11222" />
                <security>
                    <authentication server-name="infinispan">
                        <digest username="admin" password="admin" realm="default"/>
                    </authentication>
                </security>
            </remote-store>
        </distributed-cache>
        <distributed-cache name="authenticationSessions" owners="1">
            <remote-store xmlns="urn:infinispan:config:store:remote:14.0"
                          cache="authenticationSessions"
                          purge="false"
                          preload="false"
                          segmented="false"
                          shared="true"
                          raw-values="true"
                          marshaller="org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory">
                <remote-server host="infinispan-01" port="11222" />
                <security>
                    <authentication server-name="infinispan">
                        <digest username="admin" password="admin" realm="default"/>
                    </authentication>
                </security>
            </remote-store>
        </distributed-cache>
        <distributed-cache name="offlineSessions" owners="1">
            <remote-store xmlns="urn:infinispan:config:store:remote:14.0"
                          cache="offlineSessions"
                          purge="false"
                          preload="false"
                          segmented="false"
                          shared="true"
                          raw-values="true"
                          marshaller="org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory">
                <remote-server host="infinispan-01" port="11222" />
                <security>
                    <authentication server-name="infinispan">
                        <digest username="admin" password="admin" realm="default"/>
                    </authentication>
                </security>
            </remote-store>
        </distributed-cache> 
        <distributed-cache name="clientSessions" owners="1">
            <remote-store xmlns="urn:infinispan:config:store:remote:14.0"
                          cache="clientSessions"
                          purge="false"
                          preload="false"
                          segmented="false"
                          shared="true"
                          raw-values="true"
                          marshaller="org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory">
                <remote-server host="infinispan-01" port="11222" />
                <security>
                    <authentication server-name="infinispan">
                        <digest username="admin" password="admin" realm="default"/>
                    </authentication>
                </security>
            </remote-store>
        </distributed-cache>
        <distributed-cache name="offlineClientSessions" owners="1">
            <remote-store xmlns="urn:infinispan:config:store:remote:14.0"
                          cache="offlineClientSessions"
                          purge="false"
                          preload="false"
                          segmented="false"
                          shared="true"
                          raw-values="true"
                          marshaller="org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory">
                <remote-server host="infinispan-01" port="11222" />
                <security>
                    <authentication server-name="infinispan">
                        <digest username="admin" password="admin" realm="default"/>
                    </authentication>
                </security>
            </remote-store>
        </distributed-cache>
        <distributed-cache name="loginFailures" owners="1">
            <remote-store xmlns="urn:infinispan:config:store:remote:14.0"
                          cache="loginFailures"
                          purge="false"
                          preload="false"
                          segmented="false"
                          shared="true"
                          raw-values="false"
                          marshaller="org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory">
                <remote-server host="infinispan-01" port="11222" />
                <security>
                    <authentication server-name="infinispan">
                        <digest username="admin" password="admin" realm="default"/>
                    </authentication>
                </security>
            </remote-store>
        </distributed-cache> 
        <distributed-cache name="actionTokens" owners="1">
                <expiration max-idle="-1" interval="300000"/>
            <remote-store xmlns="urn:infinispan:config:store:remote:14.0"
                          cache="actionTokens"
                          purge="false"
                          preload="false"
                          segmented="false"
                          shared="true"
                          raw-values="true"
                          marshaller="org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory">
                <remote-server host="infinispan-01" port="11222" />
                <security>
                    <authentication server-name="infinispan">
                        <digest username="admin" password="admin" realm="default"/>
                    </authentication>
                </security>
            </remote-store>
        </distributed-cache> 
        <replicated-cache name="work">
            <remote-store xmlns="urn:infinispan:config:store:remote:14.0"
                          cache="work"
                          purge="false"
                          preload="false"
                          segmented="false"
                          shared="true"
                          raw-values="true"
                          marshaller="org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory">
                <remote-server host="infinispan-01" port="11222" />
                <security>
                    <authentication server-name="infinispan">
                        <digest username="admin" password="admin" realm="default"/>
                    </authentication>
                </security>
            </remote-store>
        </replicated-cache>
    </cache-container>
</infinispan>
1 Like