james/nixery
1
0
Fork 0
mirror of https://code.tvl.fyi/depot.git:/tools/nixery.git synced 2024-10-22 20:33:02 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat(storage): Store blob content-type in extended attributes

Browse Source 
After the discussion in #116, this stores the blob content types
in extended attributes when using the filesystem backend.

If the underlying filesystem doesn't support extended attributes,
storing blobs won't work; also, if extended attributes get removed,
blobs won't be served anymore. We can relax this behavior if
needed (i.e. log errors but still accept to store or serve blobs).
However, since the Docker Engine (and possibly other container
engines) won't accept to pull images from a registry that doesn't
use correct content types for manifest files, it could be argued
that it's better to give a hard fail. (Otherwise, the container
engine gives cryptic error messages like "missing signature key".)

I can change that behavior (and log errors but still store/serve
blobs to the filesystem) if you think it's better.
This commit is contained in:
Jérôme Petazzoni 2021-04-22 16:52:12 +02:00 committed by Vincent Ambo
parent 1aa8f31709
commit 0face4273d
4 changed files with 20 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
default.nix
View File

@ -38,7 +38,7 @@ let
doCheck = true;
# Needs to be updated after every modification of go.mod/go.sum
vendorSha256 = "1ff0kfww6fy6pnvyva7x8cc6l1d12aafps48wrkwawk2qjy9a8b9";
vendorSha256 = "1adjav0dxb97ws0w2k50rhk6r46wvfry6aj4sik3ninl525kd15s";
buildFlagsArray = [
"-ldflags=-s -w -X main.version=${nixery-commit-hash}"

1
go.mod
View File

@ -5,6 +5,7 @@ go 1.15
require (
cloud.google.com/go/storage v1.15.0
github.com/google/go-cmp v0.5.5
github.com/pkg/xattr v0.4.3
github.com/sirupsen/logrus v1.8.1
golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c
gonum.org/v1/gonum v0.9.1

3
go.sum
View File

@ -158,6 +158,8 @@ github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2
github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/xattr v0.4.3 h1:5Jx4GCg5ABtqWZH8WLzeI4fOtM1HyX4RBawuCoua1es=
github.com/pkg/xattr v0.4.3/go.mod h1:sBD3RAqlr8Q+RC3FutZcikpT8nyDrIEEBw2J744gVWs=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@ -322,6 +324,7 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

17
storage/filesystem.go
View File

@ -23,6 +23,7 @@ import (
"os"
"path"
"github.com/pkg/xattr"
log "github.com/sirupsen/logrus"
)
@ -49,8 +50,7 @@ func (b *FSBackend) Name() string {
return fmt.Sprintf("Filesystem (%s)", b.path)
}
// TODO(tazjin): Implement support for persisting content-types for the filesystem backend.
func (b *FSBackend) Persist(ctx context.Context, key, _type string, f Persister) (string, int64, error) {
func (b *FSBackend) Persist(ctx context.Context, key, contentType string, f Persister) (string, int64, error) {
full := path.Join(b.path, key)
dir := path.Dir(full)
err := os.MkdirAll(dir, 0755)
@ -66,6 +66,12 @@ func (b *FSBackend) Persist(ctx context.Context, key, _type string, f Persister)
}
defer file.Close()
err = xattr.Set(full, "user.mime_type", []byte(contentType))
if err != nil {
log.WithError(err).WithField("file", full).Error("failed to store file type in xattrs")
return "", 0, err
}
return f(file)
}
@ -92,6 +98,13 @@ func (b *FSBackend) Serve(digest string, r *http.Request, w http.ResponseWriter)
"path": p,
}).Info("serving blob from filesystem")
contentType, err := xattr.Get(p, "user.mime_type")
if err != nil {
log.WithError(err).WithField("file", p).Error("failed to read file type from xattrs")
return err
}
w.Header().Add("Content-Type", string(contentType))
http.ServeFile(w, r, p)
return nil
}