fix multiple tls ingress on mosquitto

* adds a server cert from the mtls CA for mtls clients
* adds gost as a tls offload for non-mtls clients to work
  around https://github.com/eclipse/mosquitto/issues/1839
* adds wildcard-tls through external secrets and removes
  template operator version
* removes non-working websockets listener
* fixes tests to work with mtls and non-mtls

mosquitto/cert-issuer.yaml

@@ -13,7 +13,7 @@ metadata:
   namespace: mosquitto
 spec:
   isCA: true
-  commonName: mosquitto
+  commonName: mosquitto mTLS CA
   secretName: mosquitto-mtls-root-ca
   privateKey:
     algorithm: ECDSA
@@ -34,6 +34,24 @@ spec:
 ---
 apiVersion: cert-manager.io/v1
 kind: Certificate
+metadata:
+  name: mosquitto-mtls-cert
+  namespace: mosquitto
+spec:
+  commonName: mosquitto
+  secretName: mosquitto-mtls-server-cert
+  dnsNames:
+  - 172.16.17.83
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+  issuerRef:
+    name: mosquitto-mtls-issuer
+    kind: Issuer
+    group: cert-manager.io
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
 metadata:
   name: mosquitto-mtls-write-user
   namespace: mosquitto

mosquitto/cm.yaml

@@ -16,20 +16,11 @@ data:
     allow_anonymous false
     password_file /mosquitto/passwords/mosquitto.passwd
     protocol mqtt
-    listener 9001
-    protocol websockets
-    allow_anonymous false
-    http_dir /http
-    certfile /mosquitto/tls/tls.crt
-    keyfile /mosquitto/tls/tls.key
-    cafile  /mosquitto/tls/ca.crt
-    require_certificate true
-    use_identity_as_username true
     listener 8883
     allow_anonymous false
-    certfile /mosquitto/tls/tls.crt
+    certfile /mosquitto/mtls/tls.crt
-    keyfile /mosquitto/tls/tls.key
+    keyfile  /mosquitto/mtls/tls.key
-    cafile  /mosquitto/tls/ca.crt
+    cafile   /mosquitto/ca/ca.crt
     require_certificate true
     use_identity_as_username true
 ---

mosquitto/ns.yaml

@@ -1,7 +1,4 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  labels:
-    kubernetes.io/metadata.name: gitea
-    wildcard-tls.kn8v.com/copy: "true"
   name: mosquitto

mosquitto/sts.yaml

@@ -45,6 +45,20 @@ spec:
         - mountPath: /users
           name: users
       containers:
+      - image: ginuerzh/gost:latest
+        imagePullPolicy: IfNotPresent
+        name: gost
+        command:
+        - gost
+        - -L
+        - tls://:1884/:1883?cert=/tls/tls.crt&key=/tls/tls.key
+        ports:
+        - containerPort: 1884
+          protocol: TCP
+          name: mqtts
+        volumeMounts:
+        - mountPath: /tls
+          name: tls
       - name: mosquitto
         image: eclipse-mosquitto
         command:
@@ -61,10 +75,10 @@ spec:
         ports:
         - containerPort: 1883
           protocol: TCP
-          name: ssh
+          name: mqtt
-        - containerPort: 9001
+        - containerPort: 8883
           protocol: TCP
-          name: http
+          name: mqtts-mtls
         volumeMounts:
         - mountPath: /mosquitto/data
           name: mosquitto-data
@@ -72,8 +86,10 @@ spec:
           name: mosquitto-config
         - mountPath: /mosquitto/passwords
           name: passwords
-        - mountPath: /mosquitto/tls
+        - mountPath: /mosquitto/mtls
-          name: tls
+          name: mtls-server-cert
+        - mountPath: /mosquitto/ca
+          name: ca
         - mountPath: /http
           name: mosquitto-http-dir
       dnsPolicy: ClusterFirst
@@ -92,7 +108,15 @@ spec:
           secret:
             secretName: mosquitto-users
             optional: true
+        - name: mtls-server-cert
+          secret:
+            secretName: mosquitto-mtls-server-cert
+            optional: false
         - name: tls
+          secret:
+            secretName: wildcard-tls
+            optional: false
+        - name: ca
           secret:
             secretName: mosquitto-mtls-root-ca
             optional: false

mosquitto/svc.yaml

@@ -13,17 +13,22 @@ spec:
   ipFamilies:
   - IPv4
   ipFamilyPolicy: SingleStack
+  clusterRoutingPolicy: Local
   ports:
   - port: 1883
     name: mqtt
     protocol: TCP
     targetPort: 1883
-  - port: 8883
+  - port: 1884
     name: mqtts
     protocol: TCP
+    targetPort: 1884
+  - port: 8883
+    name: mqtts-mtls
+    protocol: TCP
     targetPort: 8883
   - port: 9001
-    name: http
+    name: mqttwss
     protocol: TCP
     targetPort: 9001
   selector:

mosquitto/test.sh

@@ -1,5 +1,11 @@
+#!/bin/bash
+
+set -e
+
 TEST=test-$(date +%s)-$RANDOM
 
+trap "rm $TEST-*" EXIT
+
 ./extract-mtls-ca.sh > $TEST-ca.pem
 ./extract-mtls-cert.sh > $TEST-user.crt
 ./extract-mtls-key.sh > $TEST-user.key
@@ -9,8 +15,8 @@ TEST=test-$(date +%s)-$RANDOM
 
 eval "$(kubectl get secret -o json mosquitto-users | jq -r '.data|to_entries[0] | @sh "USERNAME=\(.key)\nPASSWORD=\(@base64d "\(.value)")\n"')"
 
-mosquitto_pub -h 172.16.17.83 -p 1883 -u "$USERNAME" -P "$PASSWORD" -t tests/1 -m success
+mosquitto_pub -h 172.16.17.83 -p 1883 -u "$USERNAME" -P "$PASSWORD" -t tests/1 -m success --debug
-mosquitto_pub --insecure -L mqtts://172.16.17.83:8883/tests/2 -m success --cert $TEST-user.crt --key $TEST-user.key --keyform pem --cafile $TEST-ca.pem
+mosquitto_pub -L mqtts://172.16.17.83:8883/tests/2 -m success --cert $TEST-user.crt --key $TEST-user.key --keyform pem --cafile $TEST-ca.pem --debug
-mosquitto_pub --insecure -L mqtts://172.16.17.83:8883/tests/3 -m success -u "$USERNAME" -P "$PASSWORD"
+mosquitto_pub -L mqtts://mqtt.strudelline.net:1884/tests/3 -u "$USERNAME" -P "$PASSWORD" -m success --debug
 
-rm $TEST-*
+echo test passed

mosquitto/wildcard-tls.yaml

@@ -0,0 +1,74 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  namespace: wildcard-tls
+  name: wildcard-tls-reader
+rules:
+- apiGroups: [""]
+  resources:
+  - secrets
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - authorization.k8s.io
+  resources:
+  - selfsubjectrulesreviews
+  verbs:
+  - create
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: wildcard-tls-sa
+  namespace: mosquitto
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: wildcard-tls-reader-from-mosquitto
+  namespace: wildcard-tls
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: wildcard-tls-reader
+subjects:
+- kind: ServiceAccount
+  name: wildcard-tls-sa
+  namespace: mosquitto
+---
+apiVersion: external-secrets.io/v1beta1
+kind: SecretStore
+metadata:
+  name: wildcard-tls
+  namespace: mosquitto
+spec:
+  provider:
+    kubernetes:
+      # with this, the store is able to pull only from `default` namespace
+      remoteNamespace: wildcard-tls
+      server:
+        caProvider:
+          type: ConfigMap
+          name: kube-root-ca.crt
+          key: ca.crt
+      auth:
+        serviceAccount:
+          name: "wildcard-tls-sa"
+---
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: wildcard-tls
+  namespace: mosquitto
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: SecretStore
+    name: wildcard-tls
+  target:
+    name: wildcard-tls
+  dataFrom:
+  - extract:
+      key: wildcard-tls