initial import

0ns.yaml

@@ -0,0 +1,4 @@
+kind: Namespace
+apiVersion: v1
+metadata:
+  name: garage

cm.yaml

@@ -0,0 +1,25 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: garage-config
+  namespace: garage
+data:
+  metadata_dir: "/var/lib/garage/meta"
+  data_dir: "/var/lib/garage/data"
+  rpc_bind_addr: "[::]:3901"
+
+  replication_factor: "2"
+  consistency_mode: "consistent"
+
+  bootstrap_peers: |
+    []
+
+  s3_api.s3_region: "us-east-1"
+  s3_api.api_bind_addr: "[::]:3900"
+  s3_api.root_domain: ".strudelline.net"
+
+  s3_web.bind_addr: "[::]:3902"
+  s3_web.root_domain: ".web.strudelline.net"
+  s3_web.add_host_to_metrics: "true"
+
+  admin.api_bind_addr: "[::]:3903"

ds.yaml

@@ -0,0 +1,172 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  namespace: garage
+  name: garage
+  labels:
+    app: garage
+  annotations:
+    reloader.stakater.com/auto: "true"
+spec:
+  selector:
+    matchLabels:
+      app: garage
+  template:
+    metadata:
+      labels:
+        app: garage
+    spec:
+      terminationGracePeriodSeconds: 0
+      hostNetwork: true
+      initContainers:
+      - name: config
+        restartPolicy: Always
+        image: git.strudelline.net/cascade/tools:latest
+        command: [bash, -c]
+        args:
+        - |
+          set -e
+
+          cd /config-in
+
+          # enumerate the global configs
+          # with the while loop, restartPolicy, and startupProbe: operate as a sidecar.
+          # this has a huge delay and probably isn't safe so it's disabled for now.
+          #while true;do
+            (
+              find . -maxdepth 1 -mindepth 1 -name '[a-z]*' -not -name '*.*' -print | (
+                while read -r f;do
+                  echo -n "${f#./} = "
+                  # if it's valid json, keep it that way (true/false, numbers, etc).
+                  # IF IT IS REQUIRED TO PASS "true" AS A STRING:
+                  #   put it in quoted quotes in the configmap like stupid_hostname: '"true"'
+                  jq . "$f" 2> /dev/null || jq -R . "$f"
+                done)
+              # enumerate the sectioned configs
+              find . -maxdepth 1 -mindepth 1 -name '[a-z]*.*' -not -name garage.toml -print | (
+                cut -c 3- | cut -d. -f1 |
+                sort | uniq | while read -r SECTION;do
+                  echo
+                  echo "[$SECTION]"
+                  for f in "$SECTION".*;do
+                    echo -n "${f#*.} = "
+                    # if it's valid json, keep it that way (true/false, numbers, etc).
+                    # IF IT IS REQUIRED TO PASS "true" AS A STRING:
+                    #   put it in quoted quotes in the configmap like stupid_hostname: '"true"'
+                    jq . "$f" 2> /dev/null || jq -R . "$f"
+                  done
+                done)
+            ) > /config/garage.toml
+            #) > /config/garage.toml.new
+            #if diff -Nq "/config/garage.toml.new" "/config/garage.toml";then
+            #  mv /config/garage.toml.new /config/garage.toml
+            #  echo "config settled."
+            #  inotifywait -r -t 30 /config-in
+            #else
+            #  mv /config/garage.toml.new /config/garage.toml
+            #  echo "rerunning shortly to settle config"
+            #  sleep 1
+            #fi
+          #done
+          while true;do
+            (
+              DD="$(cat data_dir)"
+              DDFREE="$(stat -fc "%a * %s" "$DD" | bc)"
+              echo "strudelline.net/garage-data-free-bytes"="$DDFREE"
+
+              MD="$(cat metadata_dir)"
+              MDFREE="$(stat -fc "%a * %s" "$MD" | bc)"
+              echo "strudelline.net/garage-meta-free-bytes"="$MDFREE"
+            ) | sponge /nfd-features-d/garage
+            cat /nfd-features-d/garage
+            echo "sleeping."
+            sleep 60
+          done
+        startupProbe:
+          exec:
+            command: [ grep, -q, "metadata_dir", /config/garage.toml ]
+          initialDelaySeconds: 1
+          periodSeconds: 1
+          failureThreshold: 20
+        volumeMounts:
+        - name: meta
+          mountPath: /var/lib/garage/meta
+        - name: data
+          mountPath: /var/lib/garage/data
+        - name: config-in
+          mountPath: /config-in
+        - name: config-xfr
+          mountPath: /config
+        - name: nfd-features-d
+          mountPath: /nfd-features-d
+      containers:
+      - image: dxflrs/garage:v1.1.0
+        name: garage
+        env:
+        - name: PATH
+          value: "/"
+        - name: GARAGE_ADMIN_TOKEN_FILE
+          value: /secrets/admin-token
+        - name: GARAGE_RPC_SECRET_FILE
+          value: /secrets/rpc-secret
+        - name: GARAGE_METRICS_TOKEN_FILE
+          value: /secrets/metrics-token
+        - name: GARAGE_CONFIG_FILE
+          value: /config/garage.toml
+        - name: GARAGE_ALLOW_WORLD_READABLE_SECRETS
+          value: "true"
+        ports:
+        - containerPort: 3900
+          name: s3-api
+        - containerPort: 3901
+          name: rpc
+        - containerPort: 3902
+          name: s3-web
+        - containerPort: 3903
+          name: admin-api
+        volumeMounts:
+        - name: meta
+          mountPath: /var/lib/garage/meta
+        - name: data
+          mountPath: /var/lib/garage/data
+        - name: config-xfr
+          mountPath: /config
+        - name: secrets
+          mountPath: /secrets
+        startupProbe:
+          httpGet:
+            path: /health
+            port: admin-api
+          initialDelaySeconds: 1
+          periodSeconds: 1
+          failureThreshold: 120
+        livenessProbe:
+          httpGet:
+            path: /health
+            port: admin-api
+          initialDelaySeconds: 1
+          periodSeconds: 5
+          failureThreshold: 10
+      volumes:
+      - name: config-xfr
+        emptyDir: {}
+      - name: config-in
+        configMap:
+          name: garage-config
+      - name: meta
+        hostPath:
+          path: /var/lib/garage/meta
+          type: DirectoryOrCreate
+      - name: data
+        hostPath:
+          path: /var/lib/garage/data
+          type: DirectoryOrCreate
+      - name: secrets
+        secret:
+          secretName: garage-secrets
+      - name: nfd-features-d
+        hostPath:
+          path: /etc/kubernetes/node-feature-discovery/features.d
+          type: DirectoryOrCreate
+      restartPolicy: Always

ingress.yaml

@@ -0,0 +1,29 @@
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: garage-web
+  namespace: garage
+spec:
+  ingressClassName: haproxy
+  rules:
+  - host: 'strudelline.net'
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: garage
+            port:
+              name: s3-web
+  - host: '*.strudelline.net'
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: garage
+            port:
+              name: s3-api

svc.yaml

@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: garage
+  name: garage
+  namespace: garage
+spec:
+  selector:
+    app: garage
+  ports:
+  - port: 3900
+    name: s3-api
+    protocol: TCP
+    targetPort: s3-api
+  - port: 3901
+    name: rpc
+    protocol: TCP
+    targetPort: rpc
+  - port: 3902
+    name: s3-web
+    protocol: TCP
+    targetPort: s3-web
+  - port: 3903
+    name: admin
+    protocol: TCP
+    targetPort: admin
+
+  type: ClusterIP

tools

@@ -0,0 +1,160 @@
+#!/bin/bash
+
+set -e -o pipefail
+
+shq() {(
+  while [ $# -gt 0 ];do
+    echo -n "'$(echo -n "$1" | sed -e "s/'/'\"'\"'/g")'"
+    shift
+  done
+)}
+
+json() {( # json encode the argument
+  jq -n --arg v "$@" '$v'
+)}
+
+GARAGE_NAMESPACE="${GARAGE_NAMESPACE-garage}"
+
+ARGS="$(shq "$@")"
+ZERO="$0"
+ONE="$1"
+ONE_UNDER="$(echo -n "$ONE" | tr - _)"
+NL="$(echo)"
+
+enumerate_pods() { # enumerate kubernetes-based garage pods
+  kubectl get pod -n "$GARAGE_NAMESPACE" -l app=garage -o json | yq -ot '.items[] | [
+      .metadata.name,
+      .status.podIP,
+      .spec.containers[].ports[]|select(.name == "rpc")|.hostPort,
+      .spec.nodeName
+  ]' | while read -r pod ip port node;do
+      gnid="$(grun "$pod" node id -q)"
+      printf "%${#pod}s %25s %s\n" $pod $node "$gnid@$ip:$port"
+  done
+
+}
+
+get_ids() { # get the public id of all discoverable servers
+  enumerate_pods | while read -r pod node gnid;do
+    echo "$gnid"
+  done
+}
+
+iterstats() { # get stats from running garage nodes via iteration over discovered ids
+  get_ids | while read -r id;do
+    garage -h "$id" stats
+  done
+}
+
+grun() {
+  TARGET="$1"
+  shift
+  kubectl exec -n "$GARAGE_NAMESPACE" -c garage "$TARGET" -- garage "$@"
+}
+
+garage() { # run an arbitrary garage command via a random garage pod
+  grun ds/garage "$@"
+}
+
+generate_secrets() { # generates secrets required to run garage
+  kubectl create secret -n "$GARAGE_NAMESPACE" generic garage-secrets \
+    --from-literal=admin-token="$( openssl rand -base64 32 )" \
+    --from-literal=metrics-token="$( openssl rand -base64 32 )" \
+    --from-literal=rpc-secret="$( openssl rand -hex 32 )"
+}
+
+generate_layout() {( # generates a sample layout, (args are included verbatim, e.g. -t k8s)
+  enumerate_pods | while read -r pod node gnid;do
+  kubectl get node/$node -o json | jq -r --arg zero "$ZERO" '
+    [
+      $zero,
+      "garage", "layout", "assign",
+      "-t", .metadata.name,
+      "-c", "\(.metadata.labels["strudelline.net/garage-meta-free-bytes"]|tonumber/1024/1024/1024|floor)G"
+    ]+$ARGS.positional
+    | @sh' --args -- "$@" "${gnid%%@*}" | while read -r cmd;do
+      eval "set $cmd"
+      printf " %q" "$@" | cut -c 2-
+    done
+  done
+)}
+
+make_bucket() { # make a bucket along with metadata in a namespace
+  TARGETBUCKET="$1"
+  TARGETKEYNAME="${2-$TARGETBUCKET}-app-key"
+  TARGETNS="${3-$TARGETBUCKET}"
+  TARGETSECRET="${3-$TARGETKEYNAME}"
+
+  kubectl get namespace "$TARGETNS" > /dev/null
+
+  garage bucket create "$TARGETBUCKET"
+  eval "$(
+  (garage key info "$TARGETKEYNAME" 2> /dev/null || garage key create "$TARGETKEYNAME") | awk '
+    /^key id: /i {printf("KID=%s\n", $3);}
+    /^secret key: /i {printf("SK=%s\n", $3);}
+  ')"
+  garage bucket allow --read --write --owner "$TARGETBUCKET" --key "$TARGETKEYNAME"
+
+  kubectl create secret generic \
+    -n "$TARGETNS" "$TARGETSECRET" \
+    --type kubernetes.io/basic-auth \
+    --from-literal=username="$KID" \
+    --from-literal=password="$SK" \
+    --from-literal=AWS_ACCESS_KEY_ID="$KID" \
+    --from-literal=AWS_SECRET_ACCESS_KEY="$SK" \
+    --from-literal=bucket="$TARGETBUCKET"
+
+  kubectl label -n "$TARGETNS" secret/$TARGETSECRET bucket="$TARGETBUCKET" key="$TARGETKEYNAME"
+}
+
+connect() { # attempt to connect all nodes
+  allids="$(get_ids)"
+  primary="$(echo "$allids" | head -1)"
+  echo "$allids" | sed 1d | while read -r id;do
+    garage -h $primary node connect "$id"
+  done
+}
+
+env() { # connect your local garage cli to a random node
+  jq -n -r \
+    --argjson secret "$(kubectl get secret -n "$GARAGE_NAMESPACE" garage-secrets -o json)" \
+    --arg gnid "$(get_ids | head -1)" \
+  '  
+    rpc_secret=\($secret.data."rpc-secret")",
+    RPC_HOST=\($gnid)"
+  '
+}
+
+help() {( # this help
+  exec 1>&2
+
+  echo "usage: $0 <subcmd> [options]"
+  echo
+  echo GARAGE SUBCOMMANDS:
+  ( 2>&1 garage --help || true ) | awk 'p==1&&/^        / {print lc,$0;next} p==1 {lc=$1;print;} /^SUBCOMMANDS:$/ {p=1}' | while read -r cmd help;do
+    printf "  %-25s %s\n" "garage $cmd" "$help"
+  done
+  echo
+  echo "GARAGE CLUSTER TOOLS:"
+  grep -E "^[a-z_]+[(][)] {[(]? # .*" "$ZERO" | sed -Ee 's@[(][)] [{][(]? # @: @' |
+    while read cmd help;do
+      printf "  %-25s %s\n" "$(echo -n $cmd | tr _ -)" "$help"
+    done
+  echo
+)}
+
+if [ "x$1" = x ];then
+    help
+    exit 1
+fi
+
+# we translate - to _ to allow enumerate-pods or enumerate_pods.
+# it also changes -h to _h.
+
+if [ "x$ONE" = "x-h" ];then
+    help
+    exit 0
+fi
+
+shift 1
+"$ONE_UNDER" "$@"