From 2a9b3551f8518e657e3bff9911f8e196a78170fc Mon Sep 17 00:00:00 2001 From: James Andariese Date: Fri, 22 Dec 2023 16:36:41 -0600 Subject: [PATCH] fix multiple tls ingress on mosquitto * adds a server cert from the mtls CA for mtls clients * adds gost as a tls offload for non-mtls clients to work around https://github.com/eclipse/mosquitto/issues/1839 * adds wildcard-tls through external secrets and removes template operator version * removes non-working websockets listener * fixes tests to work with mtls and non-mtls --- mosquitto/cert-issuer.yaml | 20 +++++++++- mosquitto/cm.yaml | 15 ++------ mosquitto/ns.yaml | 3 -- mosquitto/sts.yaml | 34 ++++++++++++++--- mosquitto/svc.yaml | 9 ++++- mosquitto/test.sh | 14 +++++-- mosquitto/wildcard-tls.yaml | 74 +++++++++++++++++++++++++++++++++++++ 7 files changed, 142 insertions(+), 27 deletions(-) create mode 100644 mosquitto/wildcard-tls.yaml diff --git a/mosquitto/cert-issuer.yaml b/mosquitto/cert-issuer.yaml index 1c6d7d4..97555b2 100644 --- a/mosquitto/cert-issuer.yaml +++ b/mosquitto/cert-issuer.yaml @@ -13,7 +13,7 @@ metadata: namespace: mosquitto spec: isCA: true - commonName: mosquitto + commonName: mosquitto mTLS CA secretName: mosquitto-mtls-root-ca privateKey: algorithm: ECDSA @@ -34,6 +34,24 @@ spec: --- apiVersion: cert-manager.io/v1 kind: Certificate +metadata: + name: mosquitto-mtls-cert + namespace: mosquitto +spec: + commonName: mosquitto + secretName: mosquitto-mtls-server-cert + dnsNames: + - 172.16.17.83 + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: mosquitto-mtls-issuer + kind: Issuer + group: cert-manager.io +--- +apiVersion: cert-manager.io/v1 +kind: Certificate metadata: name: mosquitto-mtls-write-user namespace: mosquitto diff --git a/mosquitto/cm.yaml b/mosquitto/cm.yaml index d510a2f..a91e167 100644 --- a/mosquitto/cm.yaml +++ b/mosquitto/cm.yaml @@ -16,20 +16,11 @@ data: allow_anonymous false password_file /mosquitto/passwords/mosquitto.passwd protocol mqtt - listener 9001 - protocol websockets - allow_anonymous false - http_dir /http - certfile /mosquitto/tls/tls.crt - keyfile /mosquitto/tls/tls.key - cafile /mosquitto/tls/ca.crt - require_certificate true - use_identity_as_username true listener 8883 allow_anonymous false - certfile /mosquitto/tls/tls.crt - keyfile /mosquitto/tls/tls.key - cafile /mosquitto/tls/ca.crt + certfile /mosquitto/mtls/tls.crt + keyfile /mosquitto/mtls/tls.key + cafile /mosquitto/ca/ca.crt require_certificate true use_identity_as_username true --- diff --git a/mosquitto/ns.yaml b/mosquitto/ns.yaml index 8be6099..fe2f8f7 100644 --- a/mosquitto/ns.yaml +++ b/mosquitto/ns.yaml @@ -1,7 +1,4 @@ apiVersion: v1 kind: Namespace metadata: - labels: - kubernetes.io/metadata.name: gitea - wildcard-tls.kn8v.com/copy: "true" name: mosquitto diff --git a/mosquitto/sts.yaml b/mosquitto/sts.yaml index b30320a..b4ce453 100644 --- a/mosquitto/sts.yaml +++ b/mosquitto/sts.yaml @@ -45,6 +45,20 @@ spec: - mountPath: /users name: users containers: + - image: ginuerzh/gost:latest + imagePullPolicy: IfNotPresent + name: gost + command: + - gost + - -L + - tls://:1884/:1883?cert=/tls/tls.crt&key=/tls/tls.key + ports: + - containerPort: 1884 + protocol: TCP + name: mqtts + volumeMounts: + - mountPath: /tls + name: tls - name: mosquitto image: eclipse-mosquitto command: @@ -61,10 +75,10 @@ spec: ports: - containerPort: 1883 protocol: TCP - name: ssh - - containerPort: 9001 + name: mqtt + - containerPort: 8883 protocol: TCP - name: http + name: mqtts-mtls volumeMounts: - mountPath: /mosquitto/data name: mosquitto-data @@ -72,8 +86,10 @@ spec: name: mosquitto-config - mountPath: /mosquitto/passwords name: passwords - - mountPath: /mosquitto/tls - name: tls + - mountPath: /mosquitto/mtls + name: mtls-server-cert + - mountPath: /mosquitto/ca + name: ca - mountPath: /http name: mosquitto-http-dir dnsPolicy: ClusterFirst @@ -92,7 +108,15 @@ spec: secret: secretName: mosquitto-users optional: true + - name: mtls-server-cert + secret: + secretName: mosquitto-mtls-server-cert + optional: false - name: tls + secret: + secretName: wildcard-tls + optional: false + - name: ca secret: secretName: mosquitto-mtls-root-ca optional: false diff --git a/mosquitto/svc.yaml b/mosquitto/svc.yaml index 757f056..c213464 100644 --- a/mosquitto/svc.yaml +++ b/mosquitto/svc.yaml @@ -13,17 +13,22 @@ spec: ipFamilies: - IPv4 ipFamilyPolicy: SingleStack + clusterRoutingPolicy: Local ports: - port: 1883 name: mqtt protocol: TCP targetPort: 1883 - - port: 8883 + - port: 1884 name: mqtts protocol: TCP + targetPort: 1884 + - port: 8883 + name: mqtts-mtls + protocol: TCP targetPort: 8883 - port: 9001 - name: http + name: mqttwss protocol: TCP targetPort: 9001 selector: diff --git a/mosquitto/test.sh b/mosquitto/test.sh index 18bf01e..e7ca3d7 100644 --- a/mosquitto/test.sh +++ b/mosquitto/test.sh @@ -1,5 +1,11 @@ +#!/bin/bash + +set -e + TEST=test-$(date +%s)-$RANDOM +trap "rm $TEST-*" EXIT + ./extract-mtls-ca.sh > $TEST-ca.pem ./extract-mtls-cert.sh > $TEST-user.crt ./extract-mtls-key.sh > $TEST-user.key @@ -9,8 +15,8 @@ TEST=test-$(date +%s)-$RANDOM eval "$(kubectl get secret -o json mosquitto-users | jq -r '.data|to_entries[0] | @sh "USERNAME=\(.key)\nPASSWORD=\(@base64d "\(.value)")\n"')" -mosquitto_pub -h 172.16.17.83 -p 1883 -u "$USERNAME" -P "$PASSWORD" -t tests/1 -m success -mosquitto_pub --insecure -L mqtts://172.16.17.83:8883/tests/2 -m success --cert $TEST-user.crt --key $TEST-user.key --keyform pem --cafile $TEST-ca.pem -mosquitto_pub --insecure -L mqtts://172.16.17.83:8883/tests/3 -m success -u "$USERNAME" -P "$PASSWORD" +mosquitto_pub -h 172.16.17.83 -p 1883 -u "$USERNAME" -P "$PASSWORD" -t tests/1 -m success --debug +mosquitto_pub -L mqtts://172.16.17.83:8883/tests/2 -m success --cert $TEST-user.crt --key $TEST-user.key --keyform pem --cafile $TEST-ca.pem --debug +mosquitto_pub -L mqtts://mqtt.strudelline.net:1884/tests/3 -u "$USERNAME" -P "$PASSWORD" -m success --debug -rm $TEST-* +echo test passed diff --git a/mosquitto/wildcard-tls.yaml b/mosquitto/wildcard-tls.yaml new file mode 100644 index 0000000..1c15567 --- /dev/null +++ b/mosquitto/wildcard-tls.yaml @@ -0,0 +1,74 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: wildcard-tls + name: wildcard-tls-reader +rules: +- apiGroups: [""] + resources: + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - authorization.k8s.io + resources: + - selfsubjectrulesreviews + verbs: + - create +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: wildcard-tls-sa + namespace: mosquitto +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: wildcard-tls-reader-from-mosquitto + namespace: wildcard-tls +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: wildcard-tls-reader +subjects: +- kind: ServiceAccount + name: wildcard-tls-sa + namespace: mosquitto +--- +apiVersion: external-secrets.io/v1beta1 +kind: SecretStore +metadata: + name: wildcard-tls + namespace: mosquitto +spec: + provider: + kubernetes: + # with this, the store is able to pull only from `default` namespace + remoteNamespace: wildcard-tls + server: + caProvider: + type: ConfigMap + name: kube-root-ca.crt + key: ca.crt + auth: + serviceAccount: + name: "wildcard-tls-sa" +--- +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: wildcard-tls + namespace: mosquitto +spec: + refreshInterval: 1h + secretStoreRef: + kind: SecretStore + name: wildcard-tls + target: + name: wildcard-tls + dataFrom: + - extract: + key: wildcard-tls