initial import
This commit is contained in:
commit
04d764c492
4
0ns.yaml
Normal file
4
0ns.yaml
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
kind: Namespace
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: garage
|
||||
25
cm.yaml
Normal file
25
cm.yaml
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: garage-config
|
||||
namespace: garage
|
||||
data:
|
||||
metadata_dir: "/var/lib/garage/meta"
|
||||
data_dir: "/var/lib/garage/data"
|
||||
rpc_bind_addr: "[::]:3901"
|
||||
|
||||
replication_factor: "2"
|
||||
consistency_mode: "consistent"
|
||||
|
||||
bootstrap_peers: |
|
||||
[]
|
||||
|
||||
s3_api.s3_region: "us-east-1"
|
||||
s3_api.api_bind_addr: "[::]:3900"
|
||||
s3_api.root_domain: ".strudelline.net"
|
||||
|
||||
s3_web.bind_addr: "[::]:3902"
|
||||
s3_web.root_domain: ".web.strudelline.net"
|
||||
s3_web.add_host_to_metrics: "true"
|
||||
|
||||
admin.api_bind_addr: "[::]:3903"
|
||||
172
ds.yaml
Normal file
172
ds.yaml
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
---
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
namespace: garage
|
||||
name: garage
|
||||
labels:
|
||||
app: garage
|
||||
annotations:
|
||||
reloader.stakater.com/auto: "true"
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: garage
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: garage
|
||||
spec:
|
||||
terminationGracePeriodSeconds: 0
|
||||
hostNetwork: true
|
||||
initContainers:
|
||||
- name: config
|
||||
restartPolicy: Always
|
||||
image: git.strudelline.net/cascade/tools:latest
|
||||
command: [bash, -c]
|
||||
args:
|
||||
- |
|
||||
set -e
|
||||
|
||||
cd /config-in
|
||||
|
||||
# enumerate the global configs
|
||||
# with the while loop, restartPolicy, and startupProbe: operate as a sidecar.
|
||||
# this has a huge delay and probably isn't safe so it's disabled for now.
|
||||
#while true;do
|
||||
(
|
||||
find . -maxdepth 1 -mindepth 1 -name '[a-z]*' -not -name '*.*' -print | (
|
||||
while read -r f;do
|
||||
echo -n "${f#./} = "
|
||||
# if it's valid json, keep it that way (true/false, numbers, etc).
|
||||
# IF IT IS REQUIRED TO PASS "true" AS A STRING:
|
||||
# put it in quoted quotes in the configmap like stupid_hostname: '"true"'
|
||||
jq . "$f" 2> /dev/null || jq -R . "$f"
|
||||
done)
|
||||
# enumerate the sectioned configs
|
||||
find . -maxdepth 1 -mindepth 1 -name '[a-z]*.*' -not -name garage.toml -print | (
|
||||
cut -c 3- | cut -d. -f1 |
|
||||
sort | uniq | while read -r SECTION;do
|
||||
echo
|
||||
echo "[$SECTION]"
|
||||
for f in "$SECTION".*;do
|
||||
echo -n "${f#*.} = "
|
||||
# if it's valid json, keep it that way (true/false, numbers, etc).
|
||||
# IF IT IS REQUIRED TO PASS "true" AS A STRING:
|
||||
# put it in quoted quotes in the configmap like stupid_hostname: '"true"'
|
||||
jq . "$f" 2> /dev/null || jq -R . "$f"
|
||||
done
|
||||
done)
|
||||
) > /config/garage.toml
|
||||
#) > /config/garage.toml.new
|
||||
#if diff -Nq "/config/garage.toml.new" "/config/garage.toml";then
|
||||
# mv /config/garage.toml.new /config/garage.toml
|
||||
# echo "config settled."
|
||||
# inotifywait -r -t 30 /config-in
|
||||
#else
|
||||
# mv /config/garage.toml.new /config/garage.toml
|
||||
# echo "rerunning shortly to settle config"
|
||||
# sleep 1
|
||||
#fi
|
||||
#done
|
||||
while true;do
|
||||
(
|
||||
DD="$(cat data_dir)"
|
||||
DDFREE="$(stat -fc "%a * %s" "$DD" | bc)"
|
||||
echo "strudelline.net/garage-data-free-bytes"="$DDFREE"
|
||||
|
||||
MD="$(cat metadata_dir)"
|
||||
MDFREE="$(stat -fc "%a * %s" "$MD" | bc)"
|
||||
echo "strudelline.net/garage-meta-free-bytes"="$MDFREE"
|
||||
) | sponge /nfd-features-d/garage
|
||||
cat /nfd-features-d/garage
|
||||
echo "sleeping."
|
||||
sleep 60
|
||||
done
|
||||
startupProbe:
|
||||
exec:
|
||||
command: [ grep, -q, "metadata_dir", /config/garage.toml ]
|
||||
initialDelaySeconds: 1
|
||||
periodSeconds: 1
|
||||
failureThreshold: 20
|
||||
volumeMounts:
|
||||
- name: meta
|
||||
mountPath: /var/lib/garage/meta
|
||||
- name: data
|
||||
mountPath: /var/lib/garage/data
|
||||
- name: config-in
|
||||
mountPath: /config-in
|
||||
- name: config-xfr
|
||||
mountPath: /config
|
||||
- name: nfd-features-d
|
||||
mountPath: /nfd-features-d
|
||||
containers:
|
||||
- image: dxflrs/garage:v1.1.0
|
||||
name: garage
|
||||
env:
|
||||
- name: PATH
|
||||
value: "/"
|
||||
- name: GARAGE_ADMIN_TOKEN_FILE
|
||||
value: /secrets/admin-token
|
||||
- name: GARAGE_RPC_SECRET_FILE
|
||||
value: /secrets/rpc-secret
|
||||
- name: GARAGE_METRICS_TOKEN_FILE
|
||||
value: /secrets/metrics-token
|
||||
- name: GARAGE_CONFIG_FILE
|
||||
value: /config/garage.toml
|
||||
- name: GARAGE_ALLOW_WORLD_READABLE_SECRETS
|
||||
value: "true"
|
||||
ports:
|
||||
- containerPort: 3900
|
||||
name: s3-api
|
||||
- containerPort: 3901
|
||||
name: rpc
|
||||
- containerPort: 3902
|
||||
name: s3-web
|
||||
- containerPort: 3903
|
||||
name: admin-api
|
||||
volumeMounts:
|
||||
- name: meta
|
||||
mountPath: /var/lib/garage/meta
|
||||
- name: data
|
||||
mountPath: /var/lib/garage/data
|
||||
- name: config-xfr
|
||||
mountPath: /config
|
||||
- name: secrets
|
||||
mountPath: /secrets
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: admin-api
|
||||
initialDelaySeconds: 1
|
||||
periodSeconds: 1
|
||||
failureThreshold: 120
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: admin-api
|
||||
initialDelaySeconds: 1
|
||||
periodSeconds: 5
|
||||
failureThreshold: 10
|
||||
volumes:
|
||||
- name: config-xfr
|
||||
emptyDir: {}
|
||||
- name: config-in
|
||||
configMap:
|
||||
name: garage-config
|
||||
- name: meta
|
||||
hostPath:
|
||||
path: /var/lib/garage/meta
|
||||
type: DirectoryOrCreate
|
||||
- name: data
|
||||
hostPath:
|
||||
path: /var/lib/garage/data
|
||||
type: DirectoryOrCreate
|
||||
- name: secrets
|
||||
secret:
|
||||
secretName: garage-secrets
|
||||
- name: nfd-features-d
|
||||
hostPath:
|
||||
path: /etc/kubernetes/node-feature-discovery/features.d
|
||||
type: DirectoryOrCreate
|
||||
restartPolicy: Always
|
||||
29
ingress.yaml
Normal file
29
ingress.yaml
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: garage-web
|
||||
namespace: garage
|
||||
spec:
|
||||
ingressClassName: haproxy
|
||||
rules:
|
||||
- host: 'strudelline.net'
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: garage
|
||||
port:
|
||||
name: s3-web
|
||||
- host: '*.strudelline.net'
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: garage
|
||||
port:
|
||||
name: s3-api
|
||||
29
svc.yaml
Normal file
29
svc.yaml
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
labels:
|
||||
app: garage
|
||||
name: garage
|
||||
namespace: garage
|
||||
spec:
|
||||
selector:
|
||||
app: garage
|
||||
ports:
|
||||
- port: 3900
|
||||
name: s3-api
|
||||
protocol: TCP
|
||||
targetPort: s3-api
|
||||
- port: 3901
|
||||
name: rpc
|
||||
protocol: TCP
|
||||
targetPort: rpc
|
||||
- port: 3902
|
||||
name: s3-web
|
||||
protocol: TCP
|
||||
targetPort: s3-web
|
||||
- port: 3903
|
||||
name: admin
|
||||
protocol: TCP
|
||||
targetPort: admin
|
||||
|
||||
type: ClusterIP
|
||||
160
tools
Executable file
160
tools
Executable file
|
|
@ -0,0 +1,160 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e -o pipefail
|
||||
|
||||
shq() {(
|
||||
while [ $# -gt 0 ];do
|
||||
echo -n "'$(echo -n "$1" | sed -e "s/'/'\"'\"'/g")'"
|
||||
shift
|
||||
done
|
||||
)}
|
||||
|
||||
json() {( # json encode the argument
|
||||
jq -n --arg v "$@" '$v'
|
||||
)}
|
||||
|
||||
GARAGE_NAMESPACE="${GARAGE_NAMESPACE-garage}"
|
||||
|
||||
ARGS="$(shq "$@")"
|
||||
ZERO="$0"
|
||||
ONE="$1"
|
||||
ONE_UNDER="$(echo -n "$ONE" | tr - _)"
|
||||
NL="$(echo)"
|
||||
|
||||
enumerate_pods() { # enumerate kubernetes-based garage pods
|
||||
kubectl get pod -n "$GARAGE_NAMESPACE" -l app=garage -o json | yq -ot '.items[] | [
|
||||
.metadata.name,
|
||||
.status.podIP,
|
||||
.spec.containers[].ports[]|select(.name == "rpc")|.hostPort,
|
||||
.spec.nodeName
|
||||
]' | while read -r pod ip port node;do
|
||||
gnid="$(grun "$pod" node id -q)"
|
||||
printf "%${#pod}s %25s %s\n" $pod $node "$gnid@$ip:$port"
|
||||
done
|
||||
|
||||
}
|
||||
|
||||
get_ids() { # get the public id of all discoverable servers
|
||||
enumerate_pods | while read -r pod node gnid;do
|
||||
echo "$gnid"
|
||||
done
|
||||
}
|
||||
|
||||
iterstats() { # get stats from running garage nodes via iteration over discovered ids
|
||||
get_ids | while read -r id;do
|
||||
garage -h "$id" stats
|
||||
done
|
||||
}
|
||||
|
||||
grun() {
|
||||
TARGET="$1"
|
||||
shift
|
||||
kubectl exec -n "$GARAGE_NAMESPACE" -c garage "$TARGET" -- garage "$@"
|
||||
}
|
||||
|
||||
garage() { # run an arbitrary garage command via a random garage pod
|
||||
grun ds/garage "$@"
|
||||
}
|
||||
|
||||
generate_secrets() { # generates secrets required to run garage
|
||||
kubectl create secret -n "$GARAGE_NAMESPACE" generic garage-secrets \
|
||||
--from-literal=admin-token="$( openssl rand -base64 32 )" \
|
||||
--from-literal=metrics-token="$( openssl rand -base64 32 )" \
|
||||
--from-literal=rpc-secret="$( openssl rand -hex 32 )"
|
||||
}
|
||||
|
||||
generate_layout() {( # generates a sample layout, (args are included verbatim, e.g. -t k8s)
|
||||
enumerate_pods | while read -r pod node gnid;do
|
||||
kubectl get node/$node -o json | jq -r --arg zero "$ZERO" '
|
||||
[
|
||||
$zero,
|
||||
"garage", "layout", "assign",
|
||||
"-t", .metadata.name,
|
||||
"-c", "\(.metadata.labels["strudelline.net/garage-meta-free-bytes"]|tonumber/1024/1024/1024|floor)G"
|
||||
]+$ARGS.positional
|
||||
| @sh' --args -- "$@" "${gnid%%@*}" | while read -r cmd;do
|
||||
eval "set $cmd"
|
||||
printf " %q" "$@" | cut -c 2-
|
||||
done
|
||||
done
|
||||
)}
|
||||
|
||||
make_bucket() { # make a bucket along with metadata in a namespace
|
||||
TARGETBUCKET="$1"
|
||||
TARGETKEYNAME="${2-$TARGETBUCKET}-app-key"
|
||||
TARGETNS="${3-$TARGETBUCKET}"
|
||||
TARGETSECRET="${3-$TARGETKEYNAME}"
|
||||
|
||||
kubectl get namespace "$TARGETNS" > /dev/null
|
||||
|
||||
garage bucket create "$TARGETBUCKET"
|
||||
eval "$(
|
||||
(garage key info "$TARGETKEYNAME" 2> /dev/null || garage key create "$TARGETKEYNAME") | awk '
|
||||
/^key id: /i {printf("KID=%s\n", $3);}
|
||||
/^secret key: /i {printf("SK=%s\n", $3);}
|
||||
')"
|
||||
garage bucket allow --read --write --owner "$TARGETBUCKET" --key "$TARGETKEYNAME"
|
||||
|
||||
kubectl create secret generic \
|
||||
-n "$TARGETNS" "$TARGETSECRET" \
|
||||
--type kubernetes.io/basic-auth \
|
||||
--from-literal=username="$KID" \
|
||||
--from-literal=password="$SK" \
|
||||
--from-literal=AWS_ACCESS_KEY_ID="$KID" \
|
||||
--from-literal=AWS_SECRET_ACCESS_KEY="$SK" \
|
||||
--from-literal=bucket="$TARGETBUCKET"
|
||||
|
||||
kubectl label -n "$TARGETNS" secret/$TARGETSECRET bucket="$TARGETBUCKET" key="$TARGETKEYNAME"
|
||||
}
|
||||
|
||||
connect() { # attempt to connect all nodes
|
||||
allids="$(get_ids)"
|
||||
primary="$(echo "$allids" | head -1)"
|
||||
echo "$allids" | sed 1d | while read -r id;do
|
||||
garage -h $primary node connect "$id"
|
||||
done
|
||||
}
|
||||
|
||||
env() { # connect your local garage cli to a random node
|
||||
jq -n -r \
|
||||
--argjson secret "$(kubectl get secret -n "$GARAGE_NAMESPACE" garage-secrets -o json)" \
|
||||
--arg gnid "$(get_ids | head -1)" \
|
||||
'
|
||||
rpc_secret=\($secret.data."rpc-secret")",
|
||||
RPC_HOST=\($gnid)"
|
||||
'
|
||||
}
|
||||
|
||||
help() {( # this help
|
||||
exec 1>&2
|
||||
|
||||
echo "usage: $0 <subcmd> [options]"
|
||||
echo
|
||||
echo GARAGE SUBCOMMANDS:
|
||||
( 2>&1 garage --help || true ) | awk 'p==1&&/^ / {print lc,$0;next} p==1 {lc=$1;print;} /^SUBCOMMANDS:$/ {p=1}' | while read -r cmd help;do
|
||||
printf " %-25s %s\n" "garage $cmd" "$help"
|
||||
done
|
||||
echo
|
||||
echo "GARAGE CLUSTER TOOLS:"
|
||||
grep -E "^[a-z_]+[(][)] {[(]? # .*" "$ZERO" | sed -Ee 's@[(][)] [{][(]? # @: @' |
|
||||
while read cmd help;do
|
||||
printf " %-25s %s\n" "$(echo -n $cmd | tr _ -)" "$help"
|
||||
done
|
||||
echo
|
||||
)}
|
||||
|
||||
if [ "x$1" = x ];then
|
||||
help
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# we translate - to _ to allow enumerate-pods or enumerate_pods.
|
||||
# it also changes -h to _h.
|
||||
|
||||
if [ "x$ONE" = "x-h" ];then
|
||||
help
|
||||
exit 0
|
||||
fi
|
||||
|
||||
shift 1
|
||||
"$ONE_UNDER" "$@"
|
||||
Loading…
Reference in New Issue
Block a user