james/nixery
1
0
Fork 0
mirror of https://code.tvl.fyi/depot.git:/tools/nixery.git synced 2025-03-15 14:11:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

refactor(server): Use package source specific cache keys

Browse Source 
Use the PackageSource.CacheKey function introduced in the previous
commit to determine the key at which a manifest should be cached in
the local cache.

Due to this change, manifests for moving target sources are no longer
cached and the recency threshold logic has been removed.
This commit is contained in:
Vincent Ambo 2019-09-08 22:21:14 +01:00 committed by Vincent Ambo
parent 72fac25c97
commit 7e886e6728
2 changed files with 21 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
server/builder/builder.go
View File

@ -110,7 +110,7 @@ func convenienceNames(packages []string) []string {
// Call out to Nix and request that an image be built. Nix will, upon success, // Call out to Nix and request that an image be built. Nix will, upon success,
// return a manifest for the container image. // return a manifest for the container image.
func BuildImage(ctx *context.Context, cfg *config.Config, cache *BuildCache, image *Image, bucket *storage.BucketHandle) (*BuildResult, error) { func BuildImage(ctx *context.Context, cfg *config.Config, cache *BuildCache, image *Image, bucket *storage.BucketHandle) (*BuildResult, error) {
resultFile, cached := cache.manifestFromCache(image) resultFile, cached := cache.manifestFromCache(cfg.Pkgs, image)
if !cached { if !cached {
packages, err := json.Marshal(image.Packages) packages, err := json.Marshal(image.Packages)
@ -158,7 +158,7 @@ func BuildImage(ctx *context.Context, cfg *config.Config, cache *BuildCache, ima
log.Println("Finished Nix image build") log.Println("Finished Nix image build")
resultFile = strings.TrimSpace(string(stdout)) resultFile = strings.TrimSpace(string(stdout))
cache.cacheManifest(image, resultFile) cache.cacheManifest(cfg.Pkgs, image, resultFile)
} }
buildOutput, err := ioutil.ReadFile(resultFile) buildOutput, err := ioutil.ReadFile(resultFile)

49
server/builder/cache.go
View File

@ -14,26 +14,15 @@
package builder package builder
import ( import (
"github.com/google/nixery/config"
"sync" "sync"
"time"
) )
// recencyThreshold is the amount of time that a manifest build will be cached
// for. When using the channel mechanism for retrieving nixpkgs, Nix will
// occasionally re-fetch the channel so things can in fact change while the
// instance is running.
const recencyThreshold = time.Duration(6) * time.Hour
type manifestEntry struct {
built time.Time
path string
}
type void struct{} type void struct{}
type BuildCache struct { type BuildCache struct {
mmtx sync.RWMutex mmtx sync.RWMutex
mcache map[string]manifestEntry mcache map[string]string
lmtx sync.RWMutex lmtx sync.RWMutex
lcache map[string]void lcache map[string]void
@ -41,7 +30,7 @@ type BuildCache struct {
func NewCache() BuildCache { func NewCache() BuildCache {
return BuildCache{ return BuildCache{
mcache: make(map[string]manifestEntry), mcache: make(map[string]string),
lcache: make(map[string]void), lcache: make(map[string]void),
} }
} }
@ -63,33 +52,33 @@ func (c *BuildCache) sawLayer(hash string) {
c.lcache[hash] = void{} c.lcache[hash] = void{}
} }
// Has this manifest been built already? If yes, we can reuse the // Retrieve a cached manifest if the build is cacheable and it exists.
// result given that the build happened recently enough. func (c *BuildCache) manifestFromCache(src config.PkgSource, image *Image) (string, bool) {
func (c *BuildCache) manifestFromCache(image *Image) (string, bool) { key := src.CacheKey(image.Packages, image.Tag)
c.mmtx.RLock() if key == "" {
return "", false
}
entry, ok := c.mcache[image.Name+image.Tag] c.mmtx.RLock()
path, ok := c.mcache[key]
c.mmtx.RUnlock() c.mmtx.RUnlock()
if !ok { if !ok {
return "", false return "", false
} }
if time.Since(entry.built) > recencyThreshold { return path, true
return "", false
}
return entry.path, true
} }
// Adds the result of a manifest build to the cache. // Adds the result of a manifest build to the cache, if the manifest
func (c *BuildCache) cacheManifest(image *Image, path string) { // is considered cacheable.
entry := manifestEntry{ func (c *BuildCache) cacheManifest(src config.PkgSource, image *Image, path string) {
built: time.Now(), key := src.CacheKey(image.Packages, image.Tag)
path: path, if key == "" {
return
} }
c.mmtx.Lock() c.mmtx.Lock()
c.mcache[image.Name+image.Tag] = entry c.mcache[key] = path
c.mmtx.Unlock() c.mmtx.Unlock()
} }