Hello,
I’m configuring a keycloak cluster with 2 servers running in 2 different machines as containers. They have each 2 interfaces, one to talk to the reverse proxy that will loadbalancer (10.0.x.x) between them and another to distribute the cache (1.1.1.x)
I have managed to start the containers (wasn’t an easy task…), but when I try to configure infinispan it tells me it cannot bind the address
My interface config
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
link/etherxxxxx
altname enp11s0
altname ens192
inet 10.0.43.11/24 brd 10.0.43.255 scope global noprefixroute eth0
valid_lft forever preferred_lft forever
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
link/ether xxxx
altname enp19s0
altname ens224
inet 1.1.1.1/24 brd 1.1.1.255 scope global noprefixroute eth1
valid_lft forever preferred_lft forever
Here’s my chache-ispn.xml file:
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Copyright 2019 Red Hat, Inc. and/or its affiliates
~ and other contributors as indicated by the @author tags.
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<infinispan
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="urn:infinispan:config:11.0 http://www.infinispan.org/schemas/infinispan-config-11.0.xsd"
xmlns="urn:infinispan:config:11.0">
<!-- Contains one or more JGroups stack definitions. -->
<jgroups>
<stack name="prod" extends="tcp">
<TCP bind_addr="${jgroups.bind.address,jgroups.tcp.address:{{ server_ispn_ip }}}"
bind_port="${jgroups.bind.port,jgroups.tcp.port:7600}"
enable_diagnostics="false"
thread_naming_pattern="pl"
send_buf_size="640k"
sock_conn_timeout="300"
bundler_type="transfer-queue"
thread_pool.min_threads="${jgroups.thread_pool.min_threads:0}"
thread_pool.max_threads="${jgroups.thread_pool.max_threads:200}"
thread_pool.keep_alive_time="60000"
thread_dumps_threshold="${jgroups.thread_dumps_threshold:10000}"
/>
<TCP bind_port="7800" />
<TCPPING initial_hosts="${jgroups.tcpping.initial_hosts:kc-app01-ispn[7600],kc-app02-ispn[7600]}" port_range="0" num_initial_members="3"/>
</stack>
</jgroups>
<cache-container name="keycloak">
<transport lock-timeout="60000"/>
<!-- Adds cluster transport that uses the default JGroups TCP stack. -->
<transport cluster="${infinispan.cluster.name:keycloak-cluster}"
stack="${infinispan.cluster.stack:prod}"
node-name="${infinispan.node.name:{{ server_hostname }}}"/>
<local-cache name="realms">
<encoding>
<key media-type="application/x-java-object"/>
<value media-type="application/x-java-object"/>
</encoding>
<memory max-count="10000"/>
</local-cache>
<local-cache name="users">
<encoding>
<key media-type="application/x-java-object"/>
<value media-type="application/x-java-object"/>
</encoding>
<memory max-count="10000"/>
</local-cache>
<distributed-cache name="sessions" owners="2">
<expiration lifespan="-1"/>
</distributed-cache>
<distributed-cache name="authenticationSessions" owners="2">
<expiration lifespan="-1"/>
</distributed-cache>
<distributed-cache name="offlineSessions" owners="2">
<expiration lifespan="-1"/>
</distributed-cache>
<distributed-cache name="clientSessions" owners="2">
<expiration lifespan="-1"/>
</distributed-cache>
<distributed-cache name="offlineClientSessions" owners="2">
<expiration lifespan="-1"/>
</distributed-cache>
<distributed-cache name="loginFailures" owners="2">
<expiration lifespan="-1"/>
</distributed-cache>
<local-cache name="authorization">
<encoding>
<key media-type="application/x-java-object"/>
<value media-type="application/x-java-object"/>
</encoding>
<memory max-count="10000"/>
</local-cache>
<replicated-cache name="work">
<expiration lifespan="-1"/>
</replicated-cache>
<local-cache name="keys">
<encoding>
<key media-type="application/x-java-object"/>
<value media-type="application/x-java-object"/>
</encoding>
<expiration max-idle="3600000"/>
<memory max-count="1000"/>
</local-cache>
<distributed-cache name="actionTokens" owners="2">
<encoding>
<key media-type="application/x-java-object"/>
<value media-type="application/x-java-object"/>
</encoding>
<expiration max-idle="-1" lifespan="-1" interval="300000"/>
<memory max-count="-1"/>
</distributed-cache>
</cache-container>
</infinispan>
My ansible podman keycloak config (the other server has an “almost” identical ansible task)
- name: KEYCLOAK DEPLOY | Start Keycloak container 2
when: "inventory_hostname == 'kc-app01'"
become: true
become_user: "{{ podman_user }}"
containers.podman.podman_container:
name: "{{ container_filename }}"
image: "localhost/keycloak"
state: started
recreate: no
ports:
- "8443:8443"
- "7600:7600"
env:
KC_HTTP_ENABLED: "true"
KC_FEATURES: "token-exchange"
KC_HOSTNAME_STRICT: "false"
KC_HOSTNAME_STRICT_HTTPS: "true"
KC_PROXY: "edge"
# Do not use KC_HOSTHAME_URL as it crashes the container
KC_HOSTNAME: "kc-app01.my.domain"
KC_HOSTNAME_PORT: 8443
KC_HTTPS_CERTIFICATE_FILE: /opt/keycloak/certs/cert.pem
KC_HTTPS_CERTIFICATE_KEY_FILE: /opt/keycloak/certs/privkey.pem
KC_CACHE_CONFIG_FILE: /opt/keycloak/conf/cache-ispn.xml
entrypoint: "/opt/keycloak/bin/kc.sh"
command: start --optimized # --log-level=DEBUG
restart_policy: unless-stopped
volumes:
- "/var/vhosts/cert-store/:/opt/keycloak/certs:Z"
- "{{ containers_path }}/images/keycloak/cache-ispn.xml:/opt/keycloak/conf/cache-ispn.xml"
user: keycloak
My Dockerimage for building keycloak
FROM quay.io/keycloak/keycloak:21.0.1 as builder
# Enable health and metrics support
ENV KC_HEALTH_ENABLED=true
ENV KC_METRICS_ENABLED=true
# Configure a database vendor
ENV KC_DB=postgres
COPY cache-ispn.xml /opt/keycloak/conf/
WORKDIR /opt/keycloak
RUN /opt/keycloak/bin/kc.sh build --cache=ispn --cache-config-file=cache-ispn.xml --cache-stack=tcp
FROM quay.io/keycloak/keycloak:21.0.1
COPY --from=builder /opt/keycloak/ /opt/keycloak/
RUN mkdir /opt/keycloak/certs
ENV KC_DB=postgres
ENV KC_DB_URL=jdbc:postgresql://myDB/keycloak
ENV KC_DB_USERNAME=keycloak
ENV KC_DB_PASSWORD=xxxxxxxx
ENV KC_HOSTNAME=kc-app01
ENV KC_HTTPS_KEY_STORE_PASSWORD=xxxxx
ENV KC_HTTPS_KEY_STORE_FILE=/opt/keycloak/conf/server.keystore
ENV KEYCLOAK_ADMIN=admin
ENV KEYCLOAK_ADMIN_PASSWORD=xxxxxxx
ENTRYPOINT ["/opt/keycloak/bin/kc.sh"]
My error logs:
2023-04-18 13:33:38,853 INFO [org.keycloak.quarkus.runtime.hostname.DefaultHostnameProvider] (main) Hostname settings: Base URL: <unset>, Hostname: kc-app01.my.domain, Strict HTTPS: true, Path: <request>, Strict BackChannel: false, Admin URL: <unset>, Admin: <request>, Port: 8443, Proxied: true
2023-04-18 13:33:40,281 WARN [io.quarkus.agroal.runtime.DataSources] (main) Datasource <default> enables XA but transaction recovery is not enabled. Please enable transaction recovery by setting quarkus.transaction-manager.enable-recovery=true, otherwise data may be lost if the application is terminated abruptly
2023-04-18 13:33:41,487 INFO [org.infinispan.SERVER] (keycloak-cache-init) ISPN005054: Native IOUring transport not available, using NIO instead: io.netty.incubator.channel.uring.IOUring
2023-04-18 13:33:41,636 WARN [io.quarkus.vertx.http.runtime.VertxHttpRecorder] (main) The X-Forwarded-* and Forwarded headers will be considered when determining the proxy address. This configuration can cause a security issue as clients can forge requests and send a forwarded header that is not overwritten by the proxy. Please consider use one of these headers just to forward the proxy address in requests.
2023-04-18 13:33:41,672 WARN [org.infinispan.CONFIG] (keycloak-cache-init) ISPN000569: Unable to persist Infinispan internal caches as no global state enabled
2023-04-18 13:33:41,717 WARN [org.infinispan.PERSISTENCE] (keycloak-cache-init) ISPN000554: jboss-marshalling is deprecated and planned for removal
2023-04-18 13:33:41,760 INFO [org.infinispan.CONTAINER] (keycloak-cache-init) ISPN000556: Starting user marshaller 'org.infinispan.jboss.marshalling.core.JBossUserMarshaller'
2023-04-18 13:33:42,030 INFO [org.keycloak.broker.provider.AbstractIdentityProviderMapper] (main) Registering class org.keycloak.broker.provider.mappersync.ConfigSyncEventListener
2023-04-18 13:33:42,264 INFO [org.infinispan.CLUSTER] (keycloak-cache-init) ISPN000078: Starting JGroups channel `keycloak-cluster` with stack `prod`
2023-04-18 13:33:42,265 INFO [org.jgroups.JChannel] (keycloak-cache-init) local_addr: aefd5072-e9d5-47b3-a1ad-dd7318f498f1, name: kc-app01
2023-04-18 13:33:42,267 ERROR [org.infinispan.CONFIG] (keycloak-cache-init) ISPN000660: DefaultCacheManager start failed, stopping any running components: org.infinispan.commons.CacheException: Unable to start JGroups Channel
at org.infinispan.remoting.transport.jgroups.JGroupsTransport.startJGroupsChannelIfNeeded(JGroupsTransport.java:623)
at org.infinispan.remoting.transport.jgroups.JGroupsTransport.start(JGroupsTransport.java:490)
at org.infinispan.remoting.transport.jgroups.CorePackageImpl$1.start(CorePackageImpl.java:42)
at org.infinispan.remoting.transport.jgroups.CorePackageImpl$1.start(CorePackageImpl.java:27)
at org.infinispan.factories.impl.BasicComponentRegistryImpl.invokeStart(BasicComponentRegistryImpl.java:617)
at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:608)
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:577)
at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:808)
at org.infinispan.metrics.impl.MetricsCollector.start(MetricsCollector.java:78)
at org.infinispan.metrics.impl.CorePackageImpl$1.start(CorePackageImpl.java:41)
at org.infinispan.metrics.impl.CorePackageImpl$1.start(CorePackageImpl.java:34)
at org.infinispan.factories.impl.BasicComponentRegistryImpl.invokeStart(BasicComponentRegistryImpl.java:617)
at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:608)
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:577)
at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:808)
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:635)
at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:599)
at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:577)
at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:808)
at org.infinispan.factories.AbstractComponentRegistry.internalStart(AbstractComponentRegistry.java:357)
at org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:250)
at org.infinispan.manager.DefaultCacheManager.internalStart(DefaultCacheManager.java:774)
at org.infinispan.manager.DefaultCacheManager.start(DefaultCacheManager.java:742)
at org.infinispan.manager.DefaultCacheManager.<init>(DefaultCacheManager.java:406)
at org.keycloak.quarkus.runtime.storage.legacy.infinispan.CacheManagerFactory.startCacheManager(CacheManagerFactory.java:96)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:833)
Caused by: java.net.BindException: bind_addr kc-app01-ispn/1.1.1.1 is not a valid interface: java.net.BindException: Cannot assign requested address
at org.jgroups.util.Util.bind(Util.java:3977)
at org.jgroups.util.Util.createServerSocket(Util.java:3949)
at org.jgroups.blocks.cs.TcpServer.<init>(TcpServer.java:74)
at org.jgroups.blocks.cs.TcpServer.<init>(TcpServer.java:49)
at org.jgroups.protocols.TCP.start(TCP.java:101)
at org.jgroups.stack.ProtocolStack.startStack(ProtocolStack.java:890)
at org.jgroups.JChannel.startStack(JChannel.java:919)
at org.jgroups.JChannel._preConnect(JChannel.java:797)
at org.jgroups.JChannel.connect(JChannel.java:322)
at org.jgroups.JChannel.connect(JChannel.java:316)
at org.infinispan.remoting.transport.jgroups.JGroupsTransport.startJGroupsChannelIfNeeded(JGroupsTransport.java:621)
... 28 more
2023-04-18 13:33:42,269 ERROR [org.jgroups.JChannel] (keycloak-cache-init) JGRP000020: failed destroying the protocol stack: java.lang.NullPointerException: Cannot invoke "org.jgroups.blocks.cs.TcpServer.stop()" because "this.srv" is null
at org.jgroups.protocols.TCP.stop(TCP.java:141)
at java.base/java.util.ArrayList.forEach(ArrayList.java:1511)
at org.jgroups.stack.ProtocolStack.stopStack(ProtocolStack.java:907)
at org.jgroups.JChannel.stopStack(JChannel.java:1016)
at org.jgroups.JChannel._close(JChannel.java:1003)
at org.jgroups.JChannel.close(JChannel.java:427)
at org.infinispan.remoting.transport.jgroups.JGroupsTransport.stop(JGroupsTransport.java:900)
at org.infinispan.remoting.transport.jgroups.CorePackageImpl$1.stop(CorePackageImpl.java:46)
at org.infinispan.remoting.transport.jgroups.CorePackageImpl$1.stop(CorePackageImpl.java:27)
at org.infinispan.factories.impl.BasicComponentRegistryImpl.invokeStop(BasicComponentRegistryImpl.java:678)
at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStopWrapper(BasicComponentRegistryImpl.java:674)
at org.infinispan.factories.impl.BasicComponentRegistryImpl.stopWrapper(BasicComponentRegistryImpl.java:662)
at org.infinispan.factories.impl.BasicComponentRegistryImpl.stop(BasicComponentRegistryImpl.java:529)
at org.infinispan.factories.AbstractComponentRegistry.internalStop(AbstractComponentRegistry.java:377)
at org.infinispan.factories.AbstractComponentRegistry.stop(AbstractComponentRegistry.java:311)
at org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:266)
at org.infinispan.manager.DefaultCacheManager.internalStart(DefaultCacheManager.java:774)
at org.infinispan.manager.DefaultCacheManager.start(DefaultCacheManager.java:742)
at org.infinispan.manager.DefaultCacheManager.<init>(DefaultCacheManager.java:406)
at org.keycloak.quarkus.runtime.storage.legacy.infinispan.CacheManagerFactory.startCacheManager(CacheManagerFactory.java:96)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:833)
2023-04-18 13:33:42,444 ERROR [org.keycloak.quarkus.runtime.cli.ExecutionExceptionHandler] (main) ERROR: Failed to start server in (production) mode
2023-04-18 13:33:42,444 ERROR [org.keycloak.quarkus.runtime.cli.ExecutionExceptionHandler] (main) ERROR: Failed to start caches
2023-04-18 13:33:42,444 ERROR [org.keycloak.quarkus.runtime.cli.ExecutionExceptionHandler] (main) ERROR: org.infinispan.manager.EmbeddedCacheManagerStartupException: org.infinispan.commons.CacheException: Unable to start JGroups Channel
2023-04-18 13:33:42,444 ERROR [org.keycloak.quarkus.runtime.cli.ExecutionExceptionHandler] (main) ERROR: org.infinispan.commons.CacheException: Unable to start JGroups Channel
2023-04-18 13:33:42,444 ERROR [org.keycloak.quarkus.runtime.cli.ExecutionExceptionHandler] (main) ERROR: Unable to start JGroups Channel
2023-04-18 13:33:42,444 ERROR [org.keycloak.quarkus.runtime.cli.ExecutionExceptionHandler] (main) ERROR: bind_addr kc-app01-ispn/1.1.1.1 is not a valid interface: java.net.BindException: Cannot assign requested address
2023-04-18 13:33:42,444 ERROR [org.keycloak.quarkus.runtime.cli.ExecutionExceptionHandler] (main) For more details run the same command passing the '--verbose' option. Also you can use '--help' to see the details about the usage of the particular command.
The addresses 1.1.1.1 and 1.1.1.2 are attainable from both nodes (as well as kc-app0X-isp fqdns)
I am very despreate and I find nothing in the documentation to help…
Could someone please give me a hand?
Thanks in advance