mirror of
https://code.tvl.fyi/depot.git:/tools/nixery.git
synced 2025-03-15 14:11:51 +00:00
feat(server): Cache built manifests to the GCS bucket
Caches manifests under `manifests/$cacheKey` in the GCS bucket and introduces two-tiered retrieval of manifests from the caches (local first, bucket second). There is some cleanup to be done in this code, but the initial version works.
This commit is contained in:
parent
7e886e6728
commit
c9237845ec
|
@ -109,8 +109,8 @@ func convenienceNames(packages []string) []string {
|
||||||
|
|
||||||
// Call out to Nix and request that an image be built. Nix will, upon success,
|
// Call out to Nix and request that an image be built. Nix will, upon success,
|
||||||
// return a manifest for the container image.
|
// return a manifest for the container image.
|
||||||
func BuildImage(ctx *context.Context, cfg *config.Config, cache *BuildCache, image *Image, bucket *storage.BucketHandle) (*BuildResult, error) {
|
func BuildImage(ctx *context.Context, cfg *config.Config, cache *LocalCache, image *Image, bucket *storage.BucketHandle) (*BuildResult, error) {
|
||||||
resultFile, cached := cache.manifestFromCache(cfg.Pkgs, image)
|
resultFile, cached := manifestFromCache(ctx, bucket, cfg.Pkgs, cache, image)
|
||||||
|
|
||||||
if !cached {
|
if !cached {
|
||||||
packages, err := json.Marshal(image.Packages)
|
packages, err := json.Marshal(image.Packages)
|
||||||
|
@ -158,7 +158,7 @@ func BuildImage(ctx *context.Context, cfg *config.Config, cache *BuildCache, ima
|
||||||
log.Println("Finished Nix image build")
|
log.Println("Finished Nix image build")
|
||||||
|
|
||||||
resultFile = strings.TrimSpace(string(stdout))
|
resultFile = strings.TrimSpace(string(stdout))
|
||||||
cache.cacheManifest(cfg.Pkgs, image, resultFile)
|
cacheManifest(ctx, bucket, cfg.Pkgs, cache, image, resultFile)
|
||||||
}
|
}
|
||||||
|
|
||||||
buildOutput, err := ioutil.ReadFile(resultFile)
|
buildOutput, err := ioutil.ReadFile(resultFile)
|
||||||
|
|
|
@ -14,13 +14,21 @@
|
||||||
package builder
|
package builder
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/google/nixery/config"
|
"context"
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
|
"cloud.google.com/go/storage"
|
||||||
|
"github.com/google/nixery/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
type void struct{}
|
type void struct{}
|
||||||
|
|
||||||
type BuildCache struct {
|
// LocalCache implements the structure used for local caching of
|
||||||
|
// manifests and layer uploads.
|
||||||
|
type LocalCache struct {
|
||||||
mmtx sync.RWMutex
|
mmtx sync.RWMutex
|
||||||
mcache map[string]string
|
mcache map[string]string
|
||||||
|
|
||||||
|
@ -28,8 +36,8 @@ type BuildCache struct {
|
||||||
lcache map[string]void
|
lcache map[string]void
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewCache() BuildCache {
|
func NewCache() LocalCache {
|
||||||
return BuildCache{
|
return LocalCache{
|
||||||
mcache: make(map[string]string),
|
mcache: make(map[string]string),
|
||||||
lcache: make(map[string]void),
|
lcache: make(map[string]void),
|
||||||
}
|
}
|
||||||
|
@ -38,7 +46,7 @@ func NewCache() BuildCache {
|
||||||
// Has this layer hash already been seen by this Nixery instance? If
|
// Has this layer hash already been seen by this Nixery instance? If
|
||||||
// yes, we can skip upload checking and such because it has already
|
// yes, we can skip upload checking and such because it has already
|
||||||
// been done.
|
// been done.
|
||||||
func (c *BuildCache) hasSeenLayer(hash string) bool {
|
func (c *LocalCache) hasSeenLayer(hash string) bool {
|
||||||
c.lmtx.RLock()
|
c.lmtx.RLock()
|
||||||
defer c.lmtx.RUnlock()
|
defer c.lmtx.RUnlock()
|
||||||
_, seen := c.lcache[hash]
|
_, seen := c.lcache[hash]
|
||||||
|
@ -46,19 +54,14 @@ func (c *BuildCache) hasSeenLayer(hash string) bool {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Layer has now been seen and should be stored.
|
// Layer has now been seen and should be stored.
|
||||||
func (c *BuildCache) sawLayer(hash string) {
|
func (c *LocalCache) sawLayer(hash string) {
|
||||||
c.lmtx.Lock()
|
c.lmtx.Lock()
|
||||||
defer c.lmtx.Unlock()
|
defer c.lmtx.Unlock()
|
||||||
c.lcache[hash] = void{}
|
c.lcache[hash] = void{}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Retrieve a cached manifest if the build is cacheable and it exists.
|
// Retrieve a cached manifest if the build is cacheable and it exists.
|
||||||
func (c *BuildCache) manifestFromCache(src config.PkgSource, image *Image) (string, bool) {
|
func (c *LocalCache) manifestFromLocalCache(key string) (string, bool) {
|
||||||
key := src.CacheKey(image.Packages, image.Tag)
|
|
||||||
if key == "" {
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
|
|
||||||
c.mmtx.RLock()
|
c.mmtx.RLock()
|
||||||
path, ok := c.mcache[key]
|
path, ok := c.mcache[key]
|
||||||
c.mmtx.RUnlock()
|
c.mmtx.RUnlock()
|
||||||
|
@ -70,15 +73,85 @@ func (c *BuildCache) manifestFromCache(src config.PkgSource, image *Image) (stri
|
||||||
return path, true
|
return path, true
|
||||||
}
|
}
|
||||||
|
|
||||||
// Adds the result of a manifest build to the cache, if the manifest
|
// Adds the result of a manifest build to the local cache, if the
|
||||||
// is considered cacheable.
|
// manifest is considered cacheable.
|
||||||
func (c *BuildCache) cacheManifest(src config.PkgSource, image *Image, path string) {
|
func (c *LocalCache) localCacheManifest(key, path string) {
|
||||||
key := src.CacheKey(image.Packages, image.Tag)
|
|
||||||
if key == "" {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
c.mmtx.Lock()
|
c.mmtx.Lock()
|
||||||
c.mcache[key] = path
|
c.mcache[key] = path
|
||||||
c.mmtx.Unlock()
|
c.mmtx.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Retrieve a manifest from the cache(s). First the local cache is
|
||||||
|
// checked, then the GCS-bucket cache.
|
||||||
|
func manifestFromCache(ctx *context.Context, bucket *storage.BucketHandle, pkgs config.PkgSource, cache *LocalCache, image *Image) (string, bool) {
|
||||||
|
key := pkgs.CacheKey(image.Packages, image.Tag)
|
||||||
|
if key == "" {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
|
||||||
|
path, cached := cache.manifestFromLocalCache(key)
|
||||||
|
if cached {
|
||||||
|
return path, true
|
||||||
|
}
|
||||||
|
|
||||||
|
obj := bucket.Object("manifests/" + key)
|
||||||
|
|
||||||
|
// Probe whether the file exists before trying to fetch it.
|
||||||
|
_, err := obj.Attrs(*ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
|
||||||
|
r, err := obj.NewReader(*ctx)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("Failed to retrieve manifest '%s' from cache: %s\n", key, err)
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
defer r.Close()
|
||||||
|
|
||||||
|
path = os.TempDir() + "/" + key
|
||||||
|
f, _ := os.Create(path)
|
||||||
|
defer f.Close()
|
||||||
|
|
||||||
|
_, err = io.Copy(f, r)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("Failed to read cached manifest for '%s': %s\n", key, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("Retrieved manifest for '%s' (%s) from GCS\n", image.Name, key)
|
||||||
|
cache.localCacheManifest(key, path)
|
||||||
|
|
||||||
|
return path, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func cacheManifest(ctx *context.Context, bucket *storage.BucketHandle, pkgs config.PkgSource, cache *LocalCache, image *Image, path string) {
|
||||||
|
key := pkgs.CacheKey(image.Packages, image.Tag)
|
||||||
|
if key == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
cache.localCacheManifest(key, path)
|
||||||
|
|
||||||
|
obj := bucket.Object("manifests/" + key)
|
||||||
|
w := obj.NewWriter(*ctx)
|
||||||
|
|
||||||
|
f, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("failed to open '%s' manifest for cache upload: %s\n", image.Name, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
|
||||||
|
size, err := io.Copy(w, f)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("failed to cache manifest sha1:%s: %s\n", key, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = w.Close(); err != nil {
|
||||||
|
log.Printf("failed to cache manifest sha1:%s: %s\n", key, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("Cached manifest sha1:%s (%v bytes written)\n", key, size)
|
||||||
|
}
|
||||||
|
|
|
@ -125,7 +125,7 @@ type registryHandler struct {
|
||||||
cfg *config.Config
|
cfg *config.Config
|
||||||
ctx *context.Context
|
ctx *context.Context
|
||||||
bucket *storage.BucketHandle
|
bucket *storage.BucketHandle
|
||||||
cache *builder.BuildCache
|
cache *builder.LocalCache
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *registryHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
func (h *registryHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
Loading…
Reference in New Issue
Block a user