Struggling with Keycloak, X509 Authentication, and using an Apache2 Reverse Proxy. I can’t seem to get the SSL certificate from the header as a a result of the RequestHeader set SSL_CLIENT_CERT. We are running Apache 2.4.41, Ubuntu 20.0.4 and Keycloak 11.0.2.
I have tried the with using the expressions for Base64, unbase64, and plain. The keystore and Truststore are created using LetsEncrypt SSL certs, Intermediate certs and Root Certs, which
inserted into PKC12/JKS stores
Error Log from KeyCloak
20:10:40,103 ERROR [org.keycloak.services.x509.AbstractClientCertificateFromHttpHeadersLookup] (default task-1) org.keycloak.common.util.PemException: java.io.IOException: Bad Base64 input character decimal 40 in array position 0: org.keycloak.common.util.PemException: org.keycloak.common.util.PemException: java.io.IOException: Bad Base64 input character decimal 40 in array position 0
.
.
.
Caused by: java.io.IOException: Bad Base64 input character decimal 40 in array position 0
at org.keycloak.common.util.Base64.decode(Base64.java:1201)
at org.keycloak.common.util.Base64.decode(Base64.java:1256)
at org.keycloak.common.util.Base64.decode(Base64.java:1224)
at org.keycloak.common.util.PemUtils.pemToDer(PemUtils.java:148)
20:10:40,572 INFO [io.undertow.request.dump] (default task-1)
==================
Standalone.XML file for KeyCloak Config
<subsystem xmlns="urn:jboss:domain:bean-validation:1.0"/>
<subsystem xmlns="urn:jboss:domain:core-management:1.0"/>
<subsystem xmlns="urn:jboss:domain:datasources:5.0">
<datasources>
<datasource jndi-name="java:jboss/datasources/ExampleDS" pool-name="ExampleDS" enabled="true" use-java-context="true" statistics-enabled="${wildfly.datasources.statistics-enabled:${wildfly.statistics-enabled:false}}">
<connection-url>jdbc:h2:mem:test;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE</connection-url>
<driver>h2</driver>
<security>
<user-name>sa</user-name>
<password>sa</password>
</security>
</datasource>
<datasource jndi-name="java:jboss/datasources/KeycloakDS" pool-name="KeycloakDS" enabled="true" use-java-context="true" statistics-enabled="${wildfly.datasources.statistics-enabled:${wildfly.statistics-enabled:false}}">
<connection-url>jdbc:h2:${jboss.server.data.dir}/keycloak;AUTO_SERVER=TRUE</connection-url>
<driver>h2</driver>
<security>
<user-name>sa</user-name>
<password>sa</password>
</security>
<pool>
<max-pool-size>100</max-pool-size>
</pool>
</datasource>
<drivers>
<driver name="h2" module="com.h2database.h2">
<xa-datasource-class>org.h2.jdbcx.JdbcDataSource</xa-datasource-class>
</driver>
</drivers>
</datasources>
</subsystem>
<subsystem xmlns="urn:jboss:domain:io:3.0">
<worker name="default"/>
<buffer-pool name="default"/>
</subsystem>
<subsystem xmlns="urn:jboss:domain:infinispan:10.0">
<cache-container name="keycloak" module="org.keycloak.keycloak-model-infinispan">
<local-cache name="realms">
<object-memory size="10000"/>
</local-cache>
<local-cache name="users">
<object-memory size="10000"/>
</local-cache>
<local-cache name="sessions"/>
<local-cache name="authenticationSessions"/>
<local-cache name="offlineSessions"/>
<local-cache name="clientSessions"/>
<local-cache name="offlineClientSessions"/>
<local-cache name="loginFailures"/>
<local-cache name="work"/>
<local-cache name="authorization">
<object-memory size="10000"/>
</local-cache>
<local-cache name="keys">
<object-memory size="1000"/>
<expiration max-idle="3600000"/>
</local-cache>
<local-cache name="actionTokens">
<object-memory size="-1"/>
<expiration max-idle="-1" interval="300000"/>
</local-cache>
</cache-container>
<cache-container name="server" default-cache="default" module="org.wildfly.clustering.server">
<local-cache name="default">
<transaction mode="BATCH"/>
</local-cache>
</cache-container>
<cache-container name="web" default-cache="passivation" module="org.wildfly.clustering.web.infinispan">
<local-cache name="passivation">
<locking isolation="REPEATABLE_READ"/>
<transaction mode="BATCH"/>
<file-store passivation="true" purge="false"/>
</local-cache>
<local-cache name="sso">
<locking isolation="REPEATABLE_READ"/>
<transaction mode="BATCH"/>
</local-cache>
<local-cache name="routing"/>
</cache-container>
<cache-container name="ejb" aliases="sfsb" default-cache="passivation" module="org.wildfly.clustering.ejb.infinispan">
<local-cache name="passivation">
<locking isolation="REPEATABLE_READ"/>
<transaction mode="BATCH"/>
<file-store passivation="true" purge="false"/>
</local-cache>
</cache-container>
<cache-container name="hibernate" module="org.infinispan.hibernate-cache">
<local-cache name="entity">
<object-memory size="10000"/>
<expiration max-idle="100000"/>
</local-cache>
<local-cache name="local-query">
<object-memory size="10000"/>
<expiration max-idle="100000"/>
</local-cache>
<local-cache name="timestamps"/>
</cache-container>
</subsystem>
<subsystem xmlns="urn:jboss:domain:jmx:1.3">
<expose-resolved-model/>
<expose-expression-model/>
<remoting-connector/>
</subsystem>
<subsystem xmlns="urn:jboss:domain:jpa:1.1">
<jpa default-datasource="" default-extended-persistence-inheritance="DEEP"/>
</subsystem>
<subsystem xmlns="urn:jboss:domain:mail:4.0">
<mail-session name="default" jndi-name="java:jboss/mail/Default">
<smtp-server outbound-socket-binding-ref="mail-smtp"/>
</mail-session>
</subsystem>
<subsystem xmlns="urn:jboss:domain:naming:2.0">
<remote-naming/>
</subsystem>
<subsystem xmlns="urn:jboss:domain:remoting:4.0">
<http-connector name="http-remoting-connector" connector-ref="default" security-realm="ApplicationRealm"/>
</subsystem>
<subsystem xmlns="urn:jboss:domain:request-controller:1.0"/>
<subsystem xmlns="urn:jboss:domain:security-manager:1.0">
<deployment-permissions>
<maximum-set>
<permission class="java.security.AllPermission"/>
</maximum-set>
</deployment-permissions>
</subsystem>
<subsystem xmlns="urn:wildfly:elytron:10.0" final-providers="combined-providers" disallowed-providers="OracleUcrypto">
<providers>
<aggregate-providers name="combined-providers">
<providers name="elytron"/>
<providers name="openssl"/>
</aggregate-providers>
<provider-loader name="elytron" module="org.wildfly.security.elytron"/>
<provider-loader name="openssl" module="org.wildfly.openssl"/>
</providers>
<audit-logging>
<file-audit-log name="local-audit" path="audit.log" relative-to="jboss.server.log.dir" format="JSON"/>
</audit-logging>
<security-domains>
<security-domain name="ApplicationDomain" default-realm="ApplicationRealm" permission-mapper="default-permission-mapper">
<realm name="ApplicationRealm" role-decoder="groups-to-roles"/>
<realm name="local"/>
</security-domain>
<security-domain name="ManagementDomain" default-realm="ManagementRealm" permission-mapper="default-permission-mapper">
<realm name="ManagementRealm" role-decoder="groups-to-roles"/>
<realm name="local" role-mapper="super-user-mapper"/>
</security-domain>
</security-domains>
<security-realms>
<identity-realm name="local" identity="$local"/>
<properties-realm name="ApplicationRealm">
<users-properties path="application-users.properties" relative-to="jboss.server.config.dir" digest-realm-name="ApplicationRealm"/>
<groups-properties path="application-roles.properties" relative-to="jboss.server.config.dir"/>
</properties-realm>
<properties-realm name="ManagementRealm">
<users-properties path="mgmt-users.properties" relative-to="jboss.server.config.dir" digest-realm-name="ManagementRealm"/>
<groups-properties path="mgmt-groups.properties" relative-to="jboss.server.config.dir"/>
</properties-realm>
</security-realms>
<tls>
<key-stores>
<key-store name="KCKeyStore">
<credential-reference clear-text="123456"/>
<implementation type="PKCS12"/>
<file path="/etc/certs/jboss.keystore"/>
</key-store>
<key-store name="KCTrustStore">
<credential-reference clear-text="123456"/>
<implementation type="JKS"/>
<file path="/etc/certs/jboss.truststore"/>
</key-store>
</key-stores>
<key-managers>
<key-manager name="KCKeyManager" key-store="KCKeyStore">
<credential-reference clear-text="123456"/>
</key-manager>
</key-managers>
<trust-managers>
<trust-manager name="KCTrustManager" key-store="KCTrustStore"/>
</trust-managers>
<server-ssl-contexts>
<server-ssl-context name="MySSLContext" protocols="TLSv1.2" need-client-auth="true" key-manager="KCKeyManager" trust-manager="KCTrustManager"/>
</server-ssl-contexts>
</tls>
<mappers>
<simple-permission-mapper name="default-permission-mapper" mapping-mode="first">
<permission-mapping>
<principal name="anonymous"/>
<permission-set name="default-permissions"/>
</permission-mapping>
<permission-mapping match-all="true">
<permission-set name="login-permission"/>
<permission-set name="default-permissions"/>
</permission-mapping>
</simple-permission-mapper>
<constant-realm-mapper name="local" realm-name="local"/>
<simple-role-decoder name="groups-to-roles" attribute="groups"/>
<constant-role-mapper name="super-user-mapper">
<role name="SuperUser"/>
</constant-role-mapper>
</mappers>
<permission-sets>
<permission-set name="login-permission">
<permission class-name="org.wildfly.security.auth.permission.LoginPermission"/>
</permission-set>
<permission-set name="default-permissions">
<permission class-name="org.wildfly.extension.batch.jberet.deployment.BatchPermission" module="org.wildfly.extension.batch.jberet" target-name="*"/>
<permission class-name="org.wildfly.transaction.client.RemoteTransactionPermission" module="org.wildfly.transaction.client"/>
<permission class-name="org.jboss.ejb.client.RemoteEJBPermission" module="org.jboss.ejb-client"/>
</permission-set>
</permission-sets>
<http>
<http-authentication-factory name="management-http-authentication" security-domain="ManagementDomain" http-server-mechanism-factory="global">
<mechanism-configuration>
<mechanism mechanism-name="DIGEST">
<mechanism-realm realm-name="ManagementRealm"/>
</mechanism>
</mechanism-configuration>
</http-authentication-factory>
<provider-http-server-mechanism-factory name="global"/>
</http>
<sasl>
<sasl-authentication-factory name="application-sasl-authentication" sasl-server-factory="configured" security-domain="ApplicationDomain">
<mechanism-configuration>
<mechanism mechanism-name="JBOSS-LOCAL-USER" realm-mapper="local"/>
<mechanism mechanism-name="DIGEST-MD5">
<mechanism-realm realm-name="ApplicationRealm"/>
</mechanism>
</mechanism-configuration>
</sasl-authentication-factory>
<sasl-authentication-factory name="management-sasl-authentication" sasl-server-factory="configured" security-domain="ManagementDomain">
<mechanism-configuration>
<mechanism mechanism-name="JBOSS-LOCAL-USER" realm-mapper="local"/>
<mechanism mechanism-name="DIGEST-MD5">
<mechanism-realm realm-name="ManagementRealm"/>
</mechanism>
</mechanism-configuration>
</sasl-authentication-factory>
<configurable-sasl-server-factory name="configured" sasl-server-factory="elytron">
<properties>
<property name="wildfly.sasl.local-user.default-user" value="$local"/>
</properties>
</configurable-sasl-server-factory>
<mechanism-provider-filtering-sasl-server-factory name="elytron" sasl-server-factory="global">
<filters>
<filter provider-name="WildFlyElytron"/>
</filters>
</mechanism-provider-filtering-sasl-server-factory>
<provider-sasl-server-factory name="global"/>
</sasl>
</subsystem>
<subsystem xmlns="urn:jboss:domain:security:2.0">
<security-domains>
<security-domain name="other" cache-type="default">
<authentication>
<login-module code="Remoting" flag="optional">
<module-option name="password-stacking" value="useFirstPass"/>
</login-module>
<login-module code="RealmDirect" flag="required">
<module-option name="password-stacking" value="useFirstPass"/>
</login-module>
</authentication>
</security-domain>
<security-domain name="jboss-web-policy" cache-type="default">
<authorization>
<policy-module code="Delegating" flag="required"/>
</authorization>
</security-domain>
<security-domain name="jaspitest" cache-type="default">
<authentication-jaspi>
<login-module-stack name="dummy">
<login-module code="Dummy" flag="optional"/>
</login-module-stack>
<auth-module code="Dummy"/>
</authentication-jaspi>
</security-domain>
<security-domain name="jboss-ejb-policy" cache-type="default">
<authorization>
<policy-module code="Delegating" flag="required"/>
</authorization>
</security-domain>
<security-domain name="keycloak">
<authentication>
<login-module code="org.keycloak.adapters.jboss.KeycloakLoginModule" flag="required"/>
</authentication>
</security-domain>
</security-domains>
</subsystem>
<subsystem xmlns="urn:jboss:domain:transactions:5.0">
<core-environment node-identifier="${jboss.tx.node.id:1}">
<process-id>
<uuid/>
</process-id>
</core-environment>
<recovery-environment socket-binding="txn-recovery-environment" status-socket-binding="txn-status-manager"/>
<coordinator-environment statistics-enabled="${wildfly.transactions.statistics-enabled:${wildfly.statistics-enabled:false}}"/>
<object-store path="tx-object-store" relative-to="jboss.server.data.dir"/>
</subsystem>
<subsystem xmlns="urn:jboss:domain:weld:4.0"/>
<subsystem xmlns="urn:wildfly:microprofile-config-smallrye:1.0"/>
<subsystem xmlns="urn:wildfly:microprofile-health-smallrye:2.0" security-enabled="false" empty-liveness-checks-status="${env.MP_HEALTH_EMPTY_LIVENESS_CHECKS_STATUS:UP}" empty-readiness-checks-status="${env.MP_HEALTH_EMPTY_READINESS_CHECKS_STATUS:UP}"/>
<subsystem xmlns="urn:wildfly:microprofile-metrics-smallrye:2.0" security-enabled="false" exposed-subsystems="*" prefix="${wildfly.metrics.prefix:wildfly}"/>
<subsystem xmlns="urn:jboss:domain:undertow:11.0" default-server="default-server" default-virtual-host="default-host" default-servlet-container="default" default-security-domain="other" statistics-enabled="${wildfly.undertow.statistics-enabled:${wildfly.statistics-enabled:false}}">
<buffer-cache name="default"/>
<server name="default-server">
<http-listener name="default" socket-binding="http" proxy-address-forwarding="true" redirect-socket="https" enable-http2="true"/>
<https-listener name="https" socket-binding="https" security-realm="ssl-realm" verify-client="REQUESTED"/>
<!--<https-listener name="https" socket-binding="https" ssl-context="MySSLContext" enable-http2="true"/> -->
<host name="default-host" alias="localhost">
<filter-ref name="request-dumper"/>
<location name="/" handler="welcome-content"/>
<http-invoker security-realm="ssl-realm"/>
</host>
</server>
<filters>
<filter name="request-dumper" module="io.undertow.core"
class-name="io.undertow.server.handlers.RequestDumpingHandler"/>
</filters>
<servlet-container name="default">
<jsp-config/>
<websockets/>
</servlet-container>
<handlers>
<file name="welcome-content" path="${jboss.home.dir}/welcome-content"/>
</handlers>
</subsystem>
<subsystem xmlns="urn:jboss:domain:keycloak-server:1.1">
<web-context>auth</web-context>
<providers>
<provider>classpath:${jboss.home.dir}/providers/*</provider>
</providers>
<master-realm-name>master</master-realm-name>
<scheduled-task-interval>900</scheduled-task-interval>
<theme>
<staticMaxAge>2592000</staticMaxAge>
<cacheThemes>true</cacheThemes>
<cacheTemplates>true</cacheTemplates>
<dir>${jboss.home.dir}/themes</dir>
</theme>
<spi name="eventsStore">
<provider name="jpa" enabled="true">
<properties>
<property name="exclude-events" value="["REFRESH_TOKEN"]"/>
</properties>
</provider>
</spi>
<spi name="userCache">
<provider name="default" enabled="true"/>
</spi>
<spi name="userSessionPersister">
<default-provider>jpa</default-provider>
</spi>
<spi name="timer">
<default-provider>basic</default-provider>
</spi>
<spi name="connectionsHttpClient">
<provider name="default" enabled="true">
<properties>
<property name="connection-pool-size" value="256"/>
</properties>
</provider>
</spi>
<spi name="realmCache">
<provider name="default" enabled="true"/>
</spi>
<spi name="connectionsInfinispan">
<default-provider>default</default-provider>
<provider name="default" enabled="true">
<properties>
<property name="cacheContainer" value="java:jboss/infinispan/container/keycloak"/>
</properties>
</provider>
</spi>
<spi name="jta-lookup">
<default-provider>${keycloak.jta.lookup.provider:jboss}</default-provider>
<provider name="jboss" enabled="true"/>
</spi>
<spi name="publicKeyStorage">
<provider name="infinispan" enabled="true">
<properties>
<property name="minTimeBetweenRequests" value="10"/>
</properties>
</provider>
</spi>
<spi name="truststore">
<provider name="file" enabled="true">
<properties>
<property name="file" value="/home/ubuntu/keycloak-11.0.2/standalone/configuration/enclaveserver.trust"/>
<property name="password" value="123456"/>
<property name="hostname-verification-policy" value="WILDCARD"/>
<property name="disabled" value="false"/>
</properties>
</provider>
</spi>
<spi name="x509cert-lookup">
<default-provider>apache</default-provider>
<provider name="apache" enabled="true">
<properties>
<property name="sslClientCert" value="SSL_CLIENT_CERT"/>
<property name="sslCertChainPrefix" value="SSL_CLIENT_CERT_CHAIN"/>
<property name="certificateChainLength" value="3"/>
</properties>
</provider>
</spi>
</subsystem>
</profile>
<interfaces>
<interface name="management">
<inet-address value="${jboss.bind.address.management:127.0.0.1}"/>
</interface>
<interface name="public">
<inet-address value="${jboss.bind.address:127.0.0.1}"/>
</interface>
</interfaces>
<socket-binding-group name="standard-sockets" default-interface="public" port-offset="${jboss.socket.binding.port-offset:0}">
<socket-binding name="ajp" port="${jboss.ajp.port:8009}"/>
<socket-binding name="http" port="${jboss.http.port:8080}"/>
<socket-binding name="https" port="${jboss.https.port:8443}"/>
<socket-binding name="management-http" interface="management" port="${jboss.management.http.port:9990}"/>
<socket-binding name="management-https" interface="management" port="${jboss.management.https.port:9993}"/>
<socket-binding name="https-proxy" port="443"/>
<socket-binding name="txn-recovery-environment" port="4712"/>
<socket-binding name="txn-status-manager" port="4713"/>
<outbound-socket-binding name="mail-smtp">
<remote-destination host="localhost" port="25"/>
</outbound-socket-binding>
</socket-binding-group>
SSLProxyVerify none
SSLEngine on
SSLProxyEngine on
ProxyRequests off
ProxyPreserveHost on
ProxyTimeout 600
SSLProxyCheckPeerExpire off
SSLProxyCheckPeerCN on
SSLProtocol all -SSLv3 -TLSv1.3
ErrorLog ${APACHE_LOG_DIR}/error.log
CustomLog ${APACHE_LOG_DIR}/access.log combined
CustomLog ${APACHE_LOG_DIR}/ssl_request_log \
"%t %h %{SSL_PROTOCOL}x %{SSL_CIPHER}x \"%r\" %b"
LogLevel debug
SSLCertificateFile "/etc/letsencrypt/live/enclave.ridgegroup.com/cert.pem"
SSLCertificateKeyFile "/etc/letsencrypt/live/enclave.ridgegroup.com/privkey.pem"
SSLCACertificateFile "/etc/letsencrypt/live/enclave.ridgegroup.com/chain.pem"
SSLProxyCACertificateFile /etc/certs/jboss_cert.pem
SSLProxyMachineCertificateFile /etc/certs/apache_proxy.pem
<Proxy */>
AddDefaultCharset Off
Order allow,deny
Allow from all
SSLVerifyClient optional_no_ca
SSLVerifyDepth 3
SSLOptions +ExportCertData
RequestHeader set X-Forwarded-Port "443"
RequestHeader add "X-forwarded-proto" "https"
ProxyPass "/auth" "https://127.0.0.1:8443/auth"
ProxyPassReverse "/auth" "https://127.0.0.1:8443/auth"
<Location />
#RequestHeader set SSL_CLIENT_CERT "expr=%{unbase64:%{SSL_CLIENT_CERT}s}"
#RequestHeader set SSL_CLIENT_CERT "expr=%{base64:%{SSL_CLIENT_CERT}s}"
RequestHeader set SSL_CLIENT_CERT "%{SSL_CLIENT_CERT}s"
RequestHeader set SSL_CLIENT_S_DN "%{SSL_CLIENT_S_DN}s"
RequestHeader set SSL_CLIENT_I_DN "%{SSL_CLIENT_I_DN}s"
RequestHeader set SSL_CLIENT_S_DN_CN "%{SSL_CLIENT_S_DN_CN}s"
#RequestHeader set SSL_SERVER_S_DN_OU "%{SSL_SERVER_S_DN_OU}s"
#RequestHeader set SSL_CLIENT_VERIFY "%{SSL_CLIENT_VERIFY}s"
#Requestheader set SSL_CLIENT_CERT_CHAIN_0 "%{SSL_CLIENT_CERT_CHAIN_0}s"
#RequestHeader set SSL_CLIENT_CERT_CHAIN_1 "%{SSL_CLIENT_CERT_CHAIN_1}s"
# RequestHeader set SSL_CLIENT_S_DN_x509 "%{SSL_CLIENT_S_DN_x509}s"
</Location>
Any insights would be greatly appreciated.
Thanks,
JF