pvc creation, job control

This commit is contained in:
James Andariese 2023-10-31 03:25:16 -05:00
parent c06e4f754f
commit 07d0474ba1
3 changed files with 433 additions and 32 deletions

258
Cargo.lock generated
View File

@ -106,6 +106,12 @@ version = "1.0.75"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
[[package]]
name = "arrayvec"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
[[package]] [[package]]
name = "async-trait" name = "async-trait"
version = "0.1.74" version = "0.1.74"
@ -114,7 +120,7 @@ checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 2.0.38",
] ]
[[package]] [[package]]
@ -168,6 +174,18 @@ version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07"
[[package]]
name = "bitvec"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c"
dependencies = [
"funty",
"radium",
"tap",
"wyz",
]
[[package]] [[package]]
name = "block-buffer" name = "block-buffer"
version = "0.10.4" version = "0.10.4"
@ -177,12 +195,79 @@ dependencies = [
"generic-array", "generic-array",
] ]
[[package]]
name = "borsh"
version = "0.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4114279215a005bc675e386011e594e1d9b800918cea18fcadadcce864a2046b"
dependencies = [
"borsh-derive",
"hashbrown 0.12.3",
]
[[package]]
name = "borsh-derive"
version = "0.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0754613691538d51f329cce9af41d7b7ca150bc973056f1156611489475f54f7"
dependencies = [
"borsh-derive-internal",
"borsh-schema-derive-internal",
"proc-macro-crate",
"proc-macro2",
"syn 1.0.109",
]
[[package]]
name = "borsh-derive-internal"
version = "0.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "afb438156919598d2c7bad7e1c0adf3d26ed3840dbc010db1a882a65583ca2fb"
dependencies = [
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]]
name = "borsh-schema-derive-internal"
version = "0.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "634205cc43f74a1b9046ef87c4540ebda95696ec0f315024860cad7c5b0f5ccd"
dependencies = [
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]] [[package]]
name = "bumpalo" name = "bumpalo"
version = "3.14.0" version = "3.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec"
[[package]]
name = "bytecheck"
version = "0.6.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b6372023ac861f6e6dc89c8344a8f398fb42aaba2b5dbc649ca0c0e9dbcb627"
dependencies = [
"bytecheck_derive",
"ptr_meta",
"simdutf8",
]
[[package]]
name = "bytecheck_derive"
version = "0.6.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7ec4c6f261935ad534c0c22dbef2201b45918860eb1c574b972bd213a76af61"
dependencies = [
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]] [[package]]
name = "byteorder" name = "byteorder"
version = "1.5.0" version = "1.5.0"
@ -254,7 +339,7 @@ dependencies = [
"heck", "heck",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 2.0.38",
] ]
[[package]] [[package]]
@ -372,6 +457,12 @@ dependencies = [
"percent-encoding", "percent-encoding",
] ]
[[package]]
name = "funty"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c"
[[package]] [[package]]
name = "futures" name = "futures"
version = "0.3.29" version = "0.3.29"
@ -428,7 +519,7 @@ checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 2.0.38",
] ]
[[package]] [[package]]
@ -498,6 +589,7 @@ dependencies = [
"futures", "futures",
"k8s-openapi", "k8s-openapi",
"kube", "kube",
"kube_quantity",
"once_cell", "once_cell",
"regex", "regex",
"scopeguard", "scopeguard",
@ -797,6 +889,18 @@ dependencies = [
"thiserror", "thiserror",
] ]
[[package]]
name = "kube_quantity"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ce035f193161d3e736b24018c4a95d42ee6e12fd14728056884e8b404942f6a"
dependencies = [
"k8s-openapi",
"nom",
"rust_decimal",
"thiserror",
]
[[package]] [[package]]
name = "lazy_static" name = "lazy_static"
version = "1.4.0" version = "1.4.0"
@ -997,7 +1101,7 @@ dependencies = [
"pest_meta", "pest_meta",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 2.0.38",
] ]
[[package]] [[package]]
@ -1028,7 +1132,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 2.0.38",
] ]
[[package]] [[package]]
@ -1049,6 +1153,15 @@ version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
[[package]]
name = "proc-macro-crate"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785"
dependencies = [
"toml",
]
[[package]] [[package]]
name = "proc-macro2" name = "proc-macro2"
version = "1.0.69" version = "1.0.69"
@ -1058,6 +1171,26 @@ dependencies = [
"unicode-ident", "unicode-ident",
] ]
[[package]]
name = "ptr_meta"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1"
dependencies = [
"ptr_meta_derive",
]
[[package]]
name = "ptr_meta_derive"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac"
dependencies = [
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]] [[package]]
name = "quote" name = "quote"
version = "1.0.33" version = "1.0.33"
@ -1067,6 +1200,12 @@ dependencies = [
"proc-macro2", "proc-macro2",
] ]
[[package]]
name = "radium"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09"
[[package]] [[package]]
name = "rand" name = "rand"
version = "0.8.5" version = "0.8.5"
@ -1126,6 +1265,15 @@ version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
[[package]]
name = "rend"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2571463863a6bd50c32f94402933f03457a3fbaf697a707c5be741e459f08fd"
dependencies = [
"bytecheck",
]
[[package]] [[package]]
name = "ring" name = "ring"
version = "0.17.5" version = "0.17.5"
@ -1140,6 +1288,34 @@ dependencies = [
"windows-sys", "windows-sys",
] ]
[[package]]
name = "rkyv"
version = "0.7.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0200c8230b013893c0b2d6213d6ec64ed2b9be2e0e016682b7224ff82cff5c58"
dependencies = [
"bitvec",
"bytecheck",
"hashbrown 0.12.3",
"ptr_meta",
"rend",
"rkyv_derive",
"seahash",
"tinyvec",
"uuid",
]
[[package]]
name = "rkyv_derive"
version = "0.7.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2e06b915b5c230a17d7a736d1e2e63ee753c256a8614ef3f5147b13a4f5541d"
dependencies = [
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]] [[package]]
name = "ron" name = "ron"
version = "0.7.1" version = "0.7.1"
@ -1161,6 +1337,22 @@ dependencies = [
"ordered-multimap", "ordered-multimap",
] ]
[[package]]
name = "rust_decimal"
version = "1.32.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4c4216490d5a413bc6d10fa4742bd7d4955941d062c0ef873141d6b0e7b30fd"
dependencies = [
"arrayvec",
"borsh",
"bytes",
"num-traits",
"rand",
"rkyv",
"serde",
"serde_json",
]
[[package]] [[package]]
name = "rustc-demangle" name = "rustc-demangle"
version = "0.1.23" version = "0.1.23"
@ -1241,6 +1433,12 @@ dependencies = [
"untrusted", "untrusted",
] ]
[[package]]
name = "seahash"
version = "4.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b"
[[package]] [[package]]
name = "secrecy" name = "secrecy"
version = "0.8.0" version = "0.8.0"
@ -1301,7 +1499,7 @@ checksum = "67c5609f394e5c2bd7fc51efda478004ea80ef42fee983d5c67a65e34f32c0e3"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 2.0.38",
] ]
[[package]] [[package]]
@ -1369,6 +1567,12 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "simdutf8"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a"
[[package]] [[package]]
name = "slab" name = "slab"
version = "0.4.9" version = "0.4.9"
@ -1416,6 +1620,17 @@ version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
[[package]]
name = "syn"
version = "1.0.109"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]] [[package]]
name = "syn" name = "syn"
version = "2.0.38" version = "2.0.38"
@ -1427,6 +1642,12 @@ dependencies = [
"unicode-ident", "unicode-ident",
] ]
[[package]]
name = "tap"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
[[package]] [[package]]
name = "thiserror" name = "thiserror"
version = "1.0.50" version = "1.0.50"
@ -1444,7 +1665,7 @@ checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 2.0.38",
] ]
[[package]] [[package]]
@ -1508,7 +1729,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 2.0.38",
] ]
[[package]] [[package]]
@ -1626,7 +1847,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 2.0.38",
] ]
[[package]] [[package]]
@ -1757,6 +1978,12 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
[[package]]
name = "uuid"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "88ad59a7560b41a70d191093a945f0b87bc1deeda46fb237479708a1d6b6cdfc"
[[package]] [[package]]
name = "valuable" name = "valuable"
version = "0.1.0" version = "0.1.0"
@ -1805,7 +2032,7 @@ dependencies = [
"once_cell", "once_cell",
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 2.0.38",
"wasm-bindgen-shared", "wasm-bindgen-shared",
] ]
@ -1827,7 +2054,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"syn", "syn 2.0.38",
"wasm-bindgen-backend", "wasm-bindgen-backend",
"wasm-bindgen-shared", "wasm-bindgen-shared",
] ]
@ -1935,6 +2162,15 @@ version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
[[package]]
name = "wyz"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed"
dependencies = [
"tap",
]
[[package]] [[package]]
name = "yaml-rust" name = "yaml-rust"
version = "0.4.5" version = "0.4.5"

View File

@ -10,6 +10,7 @@ config = "0.13.3"
futures = "0.3.29" futures = "0.3.29"
k8s-openapi = { version = "0.20.0", features = ["v1_27"] } k8s-openapi = { version = "0.20.0", features = ["v1_27"] }
kube = { version = "0.86.0", features = ["ws"] } kube = { version = "0.86.0", features = ["ws"] }
kube_quantity = "0.7.0"
once_cell = "1.18.0" once_cell = "1.18.0"
regex = "1.10.2" regex = "1.10.2"
scopeguard = "1.2.0" scopeguard = "1.2.0"

View File

@ -1,7 +1,23 @@
// TODO:
//
// * add the ability to create its pvc
// * add tooling mode when it's not GIT_DIR=
// * delete pvc
// * check disk usage
// * resize disk maybe?
// * create pvc with a specific storageclass or similar
// * transfer pvc contents
// * config mode (maybe)
// * set the pvc storageclass instead
use std::f32::consts::E;
use std::process::ExitCode; use std::process::ExitCode;
use std::time::Duration;
use std::{future::Future, collections::BTreeMap}; use std::{future::Future, collections::BTreeMap};
use std::sync::Arc; use std::sync::Arc;
use futures::{StreamExt,TryStreamExt}; use futures::{StreamExt,TryStreamExt};
use k8s::apimachinery::pkg::api::resource::Quantity;
use kube_quantity::{ParsedQuantity, ParseQuantityError};
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
use anyhow::{bail, anyhow, Result}; use anyhow::{bail, anyhow, Result};
@ -58,6 +74,18 @@ struct Config {
)] )]
/// verbosity, may be specified more than once /// verbosity, may be specified more than once
verbose: u8, verbose: u8,
#[arg(long("storage-class-name"),env="GIT_REMOTE_K8S_PVC_STORAGECLASSNAME")]
/// storageClassName to use for the backing PVC _if it is created_.
///
/// this will be used to create a new PVC but if a PVC already
/// exists by the requested name, it will be used. If there is
/// no repo present in the PVC, one will be created on the first
/// run.
storageclass: Option<String>,
#[arg(short('s'),long("initial-volume-size"), default_value("1Gi"), env="GIT_REMOTE_K8S_INITIAL_VOLUME_SIZE")]
initial_size: String,
} }
#[derive(ThisError,Debug)] #[derive(ThisError,Debug)]
@ -77,6 +105,10 @@ pub enum ApplicationError {
PodCouldNotOpenStdout, PodCouldNotOpenStdout,
#[error("couldn't open pod's stderr")] #[error("couldn't open pod's stderr")]
PodCouldNotOpenStderr, PodCouldNotOpenStderr,
#[error("pod failed to start")]
PodCouldNotStart,
#[error("worker pod did not continue running")]
PodDidNotWait,
} }
#[derive(ThisError,Debug)] #[derive(ThisError,Debug)]
@ -238,6 +270,14 @@ async fn wait_for_pod_running_watch(pods: &Api<Pod>, pod: Pod) -> Result<()> {
info!("Ready to attach to {}", o.name_any()); info!("Ready to attach to {}", o.name_any());
break; break;
} }
if s.phase.clone().unwrap_or_default() == "Failed" {
error!("Pod which we are awaiting has entered failed state instead of running");
return Err(ApplicationError::PodCouldNotStart.into());
}
if s.phase.clone().unwrap_or_default() == "Succeeded" {
error!("Pod which we are awaiting has completed instead");
return Err(ApplicationError::PodDidNotWait.into());
}
} }
_ => {} _ => {}
} }
@ -257,7 +297,22 @@ async fn is_pod_running(pods: &Api<Pod>, pod: Pod) -> Result<bool> {
} }
} }
async fn wait_for_pod_running(pods: &Api<Pod>, pod: Pod) -> Result<()> { async fn is_pod_finished(pods: &Api<Pod>, pod: &Pod) -> Result<bool> {
let got_pod = pods.get(&pod.metadata.name.as_ref().ok_or(anyhow!("pod metadata must have a name"))?).await?;
let phase = got_pod
.status.ok_or(anyhow!("pod has no status"))?
.phase.ok_or(anyhow!("pod has no status.phase"))?;
if phase == "Failed" {
Ok(true)
} else if phase == "Succeeded" {
Ok(true)
} else {
Ok(false)
}
}
async fn wait_for_pod_running(pods: &Api<Pod>, pod: &Pod) -> Result<()> {
let (tx, mut rx) = tokio::sync::mpsc::channel(1); let (tx, mut rx) = tokio::sync::mpsc::channel(1);
let xtx = tx.clone(); let xtx = tx.clone();
let xpods = pods.clone(); let xpods = pods.clone();
@ -318,11 +373,6 @@ async fn main_wrapped(rc: &mut ExitCode) -> crate::Result<()> {
_ => Level::TRACE, _ => Level::TRACE,
}).await; }).await;
if let Err(_) = std::env::var("GIT_DIR") {
error!("Please see https://github.com/jamesandariese/git-remote-k8s for details on use.");
bail!("not running in git");
}
debug!("{:?}", &cfg); debug!("{:?}", &cfg);
@ -343,9 +393,38 @@ async fn main_wrapped(rc: &mut ExitCode) -> crate::Result<()> {
let kube_container_name = kube_job_label; let kube_container_name = kube_job_label;
let kube_shell_executable = "/bin/sh"; let kube_shell_executable = "/bin/sh";
let kube_shell_parameters = "-c"; let kube_shell_parameters = "-c";
let kube_shell_sleeper_command = "set -e; while true;do sleep 5;done"; let kube_shell_sleeper_command = r#"
set -e
SD="$(date +%s)"
CD=$SD
LD=$SD
echo $SD > /repo.timeout
while [ $(( LD + 900 )) -ge $CD ];do
sleep 5
CD="$(date +%s)"
NLD=$(cat /repo.timeout)
if [ $NLD -ne $LD ];then
echo "updated last run time to $NLD (from $LD)"
fi
LD=$NLD
done
echo "worker ran for $(( CD - SD )) seconds"
"#;
let kube_repo_mount_path = "/repo"; let kube_repo_mount_path = "/repo";
let parsed_size: Result<ParsedQuantity, ParseQuantityError> = cfg.initial_size.clone().try_into();
let quantity_size: Quantity = match parsed_size {
Err(e) => {
error!("could not parse initial PVC size: {e}");
return Err(e.into());
},
Ok(s) => {
s.into()
}
};
debug!("parsed size is {quantity_size:#?}");
let kube_pod_labels = vec![ let kube_pod_labels = vec![
("com.kn8v.git-remote-k8s/repo-name", kube_pvc), ("com.kn8v.git-remote-k8s/repo-name", kube_pvc),
("com.kn8v.git-remote-k8s/job", kube_job_label), ("com.kn8v.git-remote-k8s/job", kube_job_label),
@ -355,12 +434,45 @@ async fn main_wrapped(rc: &mut ExitCode) -> crate::Result<()> {
info!("Remote Namespace: {}", kube_ns); info!("Remote Namespace: {}", kube_ns);
info!("Remote PVC Name: {}", kube_pvc); info!("Remote PVC Name: {}", kube_pvc);
debug!("option parsing complete. continuing to job selection.");
if let Err(_) = std::env::var("GIT_DIR") {
}
let client = ctx.ktx(Some(kube_context.to_owned())).await?; let client = ctx.ktx(Some(kube_context.to_owned())).await?;
let pvcs_api = kube::Api::<PersistentVolumeClaim>::namespaced(client.clone(), &kube_ns); let pvcs_api = kube::Api::<PersistentVolumeClaim>::namespaced(client.clone(), &kube_ns);
let pods_api = kube::Api::<Pod>::namespaced(client.clone(), &kube_ns); let pods_api = kube::Api::<Pod>::namespaced(client.clone(), &kube_ns);
// TODO: create the pvc
let existing_pvc = pvcs_api.get_opt(kube_pvc).await?;
if let None = existing_pvc {
info!("pvc doesn't exist yet. creating now.");
let mut repo_pvc = PersistentVolumeClaim {
metadata: ObjectMeta::default(),
spec: Some(PersistentVolumeClaimSpec{
access_modes:Some(vec!["ReadWriteOnce".to_owned()]),
resources: Some(ResourceRequirements{
claims: None,
limits: None,
requests: Some(BTreeMap::from([("storage".to_owned(),quantity_size)]))
}),
storage_class_name: cfg.storageclass.clone(),
volume_mode: Some("Filesystem".to_owned()),
volume_name: None, data_source: None, data_source_ref: None, selector: None,
}),
status: None,
};
let mut meta = ObjectMeta::default();
meta.name = Some(kube_pvc.to_owned());
meta.namespace = Some(kube_ns.to_owned());
repo_pvc.metadata = meta;
let pp = PostParams::default();
let created_pvc = pvcs_api.create(&pp, &repo_pvc).await?;
debug!("created pvc: {created_pvc:#?}");
}
debug!("{:#?}",existing_pvc);
// create the worker pod // create the worker pod
let mut worker_pod = Pod::default(); let mut worker_pod = Pod::default();
@ -394,6 +506,7 @@ async fn main_wrapped(rc: &mut ExitCode) -> crate::Result<()> {
volume_mount.name = "repo".to_owned(); volume_mount.name = "repo".to_owned();
container.volume_mounts = Some(vec![volume_mount]); container.volume_mounts = Some(vec![volume_mount]);
} }
container.image_pull_policy = Some("IfNotPresent".to_owned());
spec.containers = vec![container]; spec.containers = vec![container];
} }
{ {
@ -412,37 +525,88 @@ async fn main_wrapped(rc: &mut ExitCode) -> crate::Result<()> {
// debug!("Pod: {:?}", worker_pod); // debug!("Pod: {:?}", worker_pod);
let mut lp = let mut lp =
ListParams::default(); ListParams::default();
let mut ls: String = String::with_capacity(kube_pod_labels.len()*100);
for (k,v) in kube_pod_labels { for (k,v) in kube_pod_labels {
lp = lp.labels(&format!("{}={}", k.to_owned(), v)); if ls.len() > 0 {
ls.push(',');
} }
ls.push_str(format!("{}={}", k.to_owned(), v).as_ref());
};
lp = lp.labels(&ls);
debug!("list params: {lp:#?}"); debug!("list params: {lp:#?}");
let worker_pods = pods_api.list(&lp).await?; let worker_pods = pods_api.list(&lp).await?;
// debug!("worker_pods: {worker_pods:#?}"); // debug!("worker_pods: {worker_pods:#?}");
if worker_pods.items.len() > 1 {
// 1: if there is >1 pod, bail
// 2: if there is 1 pod and that pod is running or pending, use it
// 3: if there is 1 pod and the pod has ended, delete it and create a new one
// 4: if there are 0 pods, start one
let worker_pods = worker_pods.items;
let mut opod: Option<Pod> = None;
// 1
if worker_pods.len() > 1 {
error!("GIT-REMOTE CLUSTER IS IN AN INCONSISTENT STATE"); error!("GIT-REMOTE CLUSTER IS IN AN INCONSISTENT STATE");
error!("Your cluster has multiple pods running which are uniquely used for this repo."); error!("Your cluster has multiple pods running which are uniquely used for this repo.");
let mut i = 0; let mut i = 0;
for pod in worker_pods.items.iter() { for pod in worker_pods.iter() {
i += 1; i += 1;
let pn = pod.metadata.name.as_ref(); let pn = pod.metadata.name.as_ref();
error!("pod {i}: {:?}", pn); error!("pod {i}: {:?}", pn);
} }
error!("Cannot continue while these pods are all running."); error!("Cannot continue while these pods are all running.");
bail!(ApplicationError::RemoteClusterInconsistent); bail!(ApplicationError::RemoteClusterInconsistent);
} else if worker_pods.len() == 1 {
// 3
let p = worker_pods.iter().next()
.expect("failed to take an item from an iter which is known to have enough items")
.to_owned();
if is_pod_finished(&pods_api, &p).await? {
info!("existing worker is finished. deleting.");
let pn = p.metadata.name.expect("trying to delete an ended pod which has no name");
pods_api.delete(&pn, &DeleteParams::default()).await.expect("failed to delete existing ended pod");
let mut sleeptime = 0.2;
loop {
tokio::time::sleep(Duration::from_secs_f64(sleeptime)).await;
if pods_api.get_opt(&pn).await?.is_none() {
break;
} }
let pod; if sleeptime >= 5.0 {
if worker_pods.items.len() == 0 { info!("still waiting for pod to be deleted. Ctrl-C to cancel.")
}
sleeptime *= 1.7;
if sleeptime >= 10.0 {
sleeptime = 10.0;
}
}
info!("deletion complete. recreating worker.");
let created_pod = pods_api.create(&PostParams::default(), &worker_pod).await?; let created_pod = pods_api.create(&PostParams::default(), &worker_pod).await?;
pod = created_pod; opod = Some(created_pod);
} else { } else {
pod = worker_pods.items.into_iter().next() // 2
.expect("failed to take an item from an iter which is known to have enough items"); opod = Some(p);
}
} else if worker_pods.len() == 0 {
// 4
let created_pod = pods_api.create(&PostParams::default(), &worker_pod).await?;
opod = Some(created_pod);
} }
wait_for_pod_running(&pods_api, pod).await?; let pod: Pod;
if let Some(p) = opod {
pod = p;
} else {
error!("could not find or start a worker pod (pod name: {})", kube_worker_name);
bail!("could not find or start a worker pod");
}
let mut gitcommand = "1>&2 echo welcome from the git-remote-k8s worker pod ; [ -f HEAD ] || git init --bare 1>&2".to_owned(); info!("waiting for worker pod to come online");
wait_for_pod_running(&pods_api, &pod).await?;
info!("work pod is online. continuing.");
let mut gitcommand = "date +%s > /repo.timeout; 1>&2 echo welcome from the git-remote-k8s worker pod ; [ -f HEAD ] || git init --bare 1>&2".to_owned();
let mut ttyout = tokio::io::stdout(); let mut ttyout = tokio::io::stdout();
let mut ttyin = tokio::io::stdin(); let mut ttyin = tokio::io::stdin();