feat(server): Cache built manifests to the GCS bucket

Caches manifests under `manifests/$cacheKey` in the GCS bucket and
introduces two-tiered retrieval of manifests from the caches (local
first, bucket second).

There is some cleanup to be done in this code, but the initial version
works.
This commit is contained in:
Vincent Ambo 2019-09-09 16:41:52 +01:00 committed by Vincent Ambo
parent 7e886e6728
commit c9237845ec
3 changed files with 97 additions and 24 deletions

View File

@ -109,8 +109,8 @@ func convenienceNames(packages []string) []string {
// Call out to Nix and request that an image be built. Nix will, upon success, // Call out to Nix and request that an image be built. Nix will, upon success,
// return a manifest for the container image. // return a manifest for the container image.
func BuildImage(ctx *context.Context, cfg *config.Config, cache *BuildCache, image *Image, bucket *storage.BucketHandle) (*BuildResult, error) { func BuildImage(ctx *context.Context, cfg *config.Config, cache *LocalCache, image *Image, bucket *storage.BucketHandle) (*BuildResult, error) {
resultFile, cached := cache.manifestFromCache(cfg.Pkgs, image) resultFile, cached := manifestFromCache(ctx, bucket, cfg.Pkgs, cache, image)
if !cached { if !cached {
packages, err := json.Marshal(image.Packages) packages, err := json.Marshal(image.Packages)
@ -158,7 +158,7 @@ func BuildImage(ctx *context.Context, cfg *config.Config, cache *BuildCache, ima
log.Println("Finished Nix image build") log.Println("Finished Nix image build")
resultFile = strings.TrimSpace(string(stdout)) resultFile = strings.TrimSpace(string(stdout))
cache.cacheManifest(cfg.Pkgs, image, resultFile) cacheManifest(ctx, bucket, cfg.Pkgs, cache, image, resultFile)
} }
buildOutput, err := ioutil.ReadFile(resultFile) buildOutput, err := ioutil.ReadFile(resultFile)

View File

@ -14,13 +14,21 @@
package builder package builder
import ( import (
"github.com/google/nixery/config" "context"
"io"
"log"
"os"
"sync" "sync"
"cloud.google.com/go/storage"
"github.com/google/nixery/config"
) )
type void struct{} type void struct{}
type BuildCache struct { // LocalCache implements the structure used for local caching of
// manifests and layer uploads.
type LocalCache struct {
mmtx sync.RWMutex mmtx sync.RWMutex
mcache map[string]string mcache map[string]string
@ -28,8 +36,8 @@ type BuildCache struct {
lcache map[string]void lcache map[string]void
} }
func NewCache() BuildCache { func NewCache() LocalCache {
return BuildCache{ return LocalCache{
mcache: make(map[string]string), mcache: make(map[string]string),
lcache: make(map[string]void), lcache: make(map[string]void),
} }
@ -38,7 +46,7 @@ func NewCache() BuildCache {
// Has this layer hash already been seen by this Nixery instance? If // Has this layer hash already been seen by this Nixery instance? If
// yes, we can skip upload checking and such because it has already // yes, we can skip upload checking and such because it has already
// been done. // been done.
func (c *BuildCache) hasSeenLayer(hash string) bool { func (c *LocalCache) hasSeenLayer(hash string) bool {
c.lmtx.RLock() c.lmtx.RLock()
defer c.lmtx.RUnlock() defer c.lmtx.RUnlock()
_, seen := c.lcache[hash] _, seen := c.lcache[hash]
@ -46,19 +54,14 @@ func (c *BuildCache) hasSeenLayer(hash string) bool {
} }
// Layer has now been seen and should be stored. // Layer has now been seen and should be stored.
func (c *BuildCache) sawLayer(hash string) { func (c *LocalCache) sawLayer(hash string) {
c.lmtx.Lock() c.lmtx.Lock()
defer c.lmtx.Unlock() defer c.lmtx.Unlock()
c.lcache[hash] = void{} c.lcache[hash] = void{}
} }
// Retrieve a cached manifest if the build is cacheable and it exists. // Retrieve a cached manifest if the build is cacheable and it exists.
func (c *BuildCache) manifestFromCache(src config.PkgSource, image *Image) (string, bool) { func (c *LocalCache) manifestFromLocalCache(key string) (string, bool) {
key := src.CacheKey(image.Packages, image.Tag)
if key == "" {
return "", false
}
c.mmtx.RLock() c.mmtx.RLock()
path, ok := c.mcache[key] path, ok := c.mcache[key]
c.mmtx.RUnlock() c.mmtx.RUnlock()
@ -70,15 +73,85 @@ func (c *BuildCache) manifestFromCache(src config.PkgSource, image *Image) (stri
return path, true return path, true
} }
// Adds the result of a manifest build to the cache, if the manifest // Adds the result of a manifest build to the local cache, if the
// is considered cacheable. // manifest is considered cacheable.
func (c *BuildCache) cacheManifest(src config.PkgSource, image *Image, path string) { func (c *LocalCache) localCacheManifest(key, path string) {
key := src.CacheKey(image.Packages, image.Tag)
if key == "" {
return
}
c.mmtx.Lock() c.mmtx.Lock()
c.mcache[key] = path c.mcache[key] = path
c.mmtx.Unlock() c.mmtx.Unlock()
} }
// Retrieve a manifest from the cache(s). First the local cache is
// checked, then the GCS-bucket cache.
func manifestFromCache(ctx *context.Context, bucket *storage.BucketHandle, pkgs config.PkgSource, cache *LocalCache, image *Image) (string, bool) {
key := pkgs.CacheKey(image.Packages, image.Tag)
if key == "" {
return "", false
}
path, cached := cache.manifestFromLocalCache(key)
if cached {
return path, true
}
obj := bucket.Object("manifests/" + key)
// Probe whether the file exists before trying to fetch it.
_, err := obj.Attrs(*ctx)
if err != nil {
return "", false
}
r, err := obj.NewReader(*ctx)
if err != nil {
log.Printf("Failed to retrieve manifest '%s' from cache: %s\n", key, err)
return "", false
}
defer r.Close()
path = os.TempDir() + "/" + key
f, _ := os.Create(path)
defer f.Close()
_, err = io.Copy(f, r)
if err != nil {
log.Printf("Failed to read cached manifest for '%s': %s\n", key, err)
}
log.Printf("Retrieved manifest for '%s' (%s) from GCS\n", image.Name, key)
cache.localCacheManifest(key, path)
return path, true
}
func cacheManifest(ctx *context.Context, bucket *storage.BucketHandle, pkgs config.PkgSource, cache *LocalCache, image *Image, path string) {
key := pkgs.CacheKey(image.Packages, image.Tag)
if key == "" {
return
}
cache.localCacheManifest(key, path)
obj := bucket.Object("manifests/" + key)
w := obj.NewWriter(*ctx)
f, err := os.Open(path)
if err != nil {
log.Printf("failed to open '%s' manifest for cache upload: %s\n", image.Name, err)
return
}
defer f.Close()
size, err := io.Copy(w, f)
if err != nil {
log.Printf("failed to cache manifest sha1:%s: %s\n", key, err)
return
}
if err = w.Close(); err != nil {
log.Printf("failed to cache manifest sha1:%s: %s\n", key, err)
return
}
log.Printf("Cached manifest sha1:%s (%v bytes written)\n", key, size)
}

View File

@ -125,7 +125,7 @@ type registryHandler struct {
cfg *config.Config cfg *config.Config
ctx *context.Context ctx *context.Context
bucket *storage.BucketHandle bucket *storage.BucketHandle
cache *builder.BuildCache cache *builder.LocalCache
} }
func (h *registryHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { func (h *registryHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {