diff --git a/adder/adder.go b/adder/adder.go index f57feaeb..81a212cf 100644 --- a/adder/adder.go +++ b/adder/adder.go @@ -2,8 +2,10 @@ package adder import ( "context" + "fmt" "io" "mime/multipart" + "strings" "github.com/ipfs/ipfs-cluster/adder/ipfsadd" "github.com/ipfs/ipfs-cluster/api" @@ -12,6 +14,8 @@ import ( files "github.com/ipfs/go-ipfs-cmdkit/files" ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log" + merkledag "github.com/ipfs/go-merkledag" + multihash "github.com/multiformats/go-multihash" ) var logger = logging.Logger("adder") @@ -113,6 +117,20 @@ func (a *Adder) FromFiles(ctx context.Context, f files.File) (*cid.Cid, error) { ipfsAdder.Out = a.output ipfsAdder.Progress = a.params.Progress + // Set up prefix + prefix, err := merkledag.PrefixForCidVersion(a.params.CidVersion) + if err != nil { + return nil, fmt.Errorf("bad CID Version: %s", err) + } + + hashFunCode, ok := multihash.Names[strings.ToLower(a.params.HashFun)] + if !ok { + return nil, fmt.Errorf("unrecognized hash function: %s", a.params.HashFun) + } + prefix.MhType = hashFunCode + prefix.MhLength = -1 + ipfsAdder.CidBuilder = &prefix + for { select { case <-a.ctx.Done(): diff --git a/adder/ipfsadd/add.go b/adder/ipfsadd/add.go index c974f1a8..052653ce 100644 --- a/adder/ipfsadd/add.go +++ b/adder/ipfsadd/add.go @@ -14,10 +14,10 @@ import ( chunker "github.com/ipfs/go-ipfs-chunker" files "github.com/ipfs/go-ipfs-cmdkit/files" posinfo "github.com/ipfs/go-ipfs-posinfo" - mfs "github.com/ipfs/go-ipfs/mfs" ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log" dag "github.com/ipfs/go-merkledag" + mfs "github.com/ipfs/go-mfs" unixfs "github.com/ipfs/go-unixfs" balanced "github.com/ipfs/go-unixfs/importer/balanced" ihelper "github.com/ipfs/go-unixfs/importer/helpers" @@ -61,6 +61,7 @@ type Adder struct { mroot *mfs.Root tempRoot *cid.Cid Prefix *cid.Prefix + CidBuilder cid.Builder liveNodes uint64 lastFile mfs.FSNode } @@ -70,7 +71,7 @@ func (adder *Adder) mfsRoot() (*mfs.Root, error) { return adder.mroot, nil } rnode := unixfs.EmptyDirNode() - rnode.SetPrefix(adder.Prefix) + rnode.SetCidBuilder(adder.CidBuilder) mr, err := mfs.NewRoot(adder.ctx, adder.dagService, rnode, nil) if err != nil { return nil, err @@ -92,11 +93,11 @@ func (adder *Adder) add(reader io.Reader) (ipld.Node, error) { } params := ihelper.DagBuilderParams{ - Dagserv: adder.dagService, - RawLeaves: adder.RawLeaves, - Maxlinks: ihelper.DefaultLinksPerBlock, - NoCopy: adder.NoCopy, - Prefix: adder.Prefix, + Dagserv: adder.dagService, + RawLeaves: adder.RawLeaves, + Maxlinks: ihelper.DefaultLinksPerBlock, + NoCopy: adder.NoCopy, + CidBuilder: adder.CidBuilder, } if adder.Trickle { @@ -247,9 +248,9 @@ func (adder *Adder) addNode(node ipld.Node, path string) error { dir := gopath.Dir(path) if dir != "." { opts := mfs.MkdirOpts{ - Mkparents: true, - Flush: false, - Prefix: adder.Prefix, + Mkparents: true, + Flush: false, + CidBuilder: adder.CidBuilder, } if err := mfs.Mkdir(mr, dir, opts); err != nil { return err @@ -307,7 +308,7 @@ func (adder *Adder) addFile(file files.File) error { } dagnode := dag.NodeWithData(sdata) - dagnode.SetPrefix(adder.Prefix) + dagnode.SetCidBuilder(adder.CidBuilder) err = adder.dagService.Add(adder.ctx, dagnode) if err != nil { return err @@ -346,9 +347,9 @@ func (adder *Adder) addDir(dir files.File) error { return err } err = mfs.Mkdir(mr, dir.FileName(), mfs.MkdirOpts{ - Mkparents: true, - Flush: false, - Prefix: adder.Prefix, + Mkparents: true, + Flush: false, + CidBuilder: adder.CidBuilder, }) if err != nil { return err diff --git a/api/add.go b/api/add.go index 5bb6a4e4..3257b187 100644 --- a/api/add.go +++ b/api/add.go @@ -25,27 +25,31 @@ type AddedOutput struct { type AddParams struct { PinOptions - Recursive bool - Layout string - Chunker string - RawLeaves bool - Hidden bool - Wrap bool - Shard bool - Progress bool + Recursive bool + Layout string + Chunker string + RawLeaves bool + Hidden bool + Wrap bool + Shard bool + Progress bool + CidVersion int + HashFun string } // DefaultAddParams returns a AddParams object with standard defaults func DefaultAddParams() *AddParams { return &AddParams{ - Recursive: false, - Layout: "", // corresponds to balanced layout - Chunker: "size-262144", - RawLeaves: false, - Hidden: false, - Wrap: false, - Shard: false, - Progress: false, + Recursive: false, + Layout: "", // corresponds to balanced layout + Chunker: "size-262144", + RawLeaves: false, + Hidden: false, + Wrap: false, + Shard: false, + Progress: false, + CidVersion: 0, + HashFun: "sha2-256", PinOptions: PinOptions{ ReplicationFactorMin: 0, ReplicationFactorMax: 0, @@ -96,6 +100,11 @@ func AddParamsFromQuery(query url.Values) (*AddParams, error) { name := query.Get("name") params.Name = name + hashF := query.Get("hash") + if hashF != "" { + params.HashFun = hashF + } + err := parseBoolParam(query, "recursive", ¶ms.Recursive) if err != nil { return nil, err @@ -132,6 +141,11 @@ func AddParamsFromQuery(query url.Values) (*AddParams, error) { return nil, err } + err = parseIntParam(query, "cid-version", ¶ms.CidVersion) + if err != nil { + return nil, err + } + if v := query.Get("shard-size"); v != "" { shardSize, err := strconv.ParseUint(v, 10, 64) if err != nil { @@ -148,7 +162,8 @@ func (p *AddParams) ToQueryString() string { fmtStr := "replication-min=%d&replication-max=%d&name=%s&" fmtStr += "shard=%t&shard-size=%d&recursive=%t&" fmtStr += "layout=%s&chunker=%s&raw-leaves=%t&hidden=%t&" - fmtStr += "wrap-with-directory=%t&progress=%t" + fmtStr += "wrap-with-directory=%t&progress=%t&" + fmtStr += "cid-version=%d&hash=%s" query := fmt.Sprintf( fmtStr, p.ReplicationFactorMin, @@ -163,6 +178,8 @@ func (p *AddParams) ToQueryString() string { p.Hidden, p.Wrap, p.Progress, + p.CidVersion, + p.HashFun, ) return query } @@ -179,5 +196,7 @@ func (p *AddParams) Equals(p2 *AddParams) bool { p.Chunker == p2.Chunker && p.RawLeaves == p2.RawLeaves && p.Hidden == p2.Hidden && - p.Wrap == p2.Wrap + p.Wrap == p2.Wrap && + p.CidVersion == p2.CidVersion && + p.HashFun == p2.HashFun } diff --git a/ipfs-cluster-ctl/main.go b/ipfs-cluster-ctl/main.go index c40e6280..cd6d7532 100644 --- a/ipfs-cluster-ctl/main.go +++ b/ipfs-cluster-ctl/main.go @@ -298,6 +298,16 @@ If you prefer faster adding, add directly to the local IPFS and trigger a Name: "raw-leaves", Usage: "Use raw blocks for leaves (experimental)", }, + cli.IntFlag{ + Name: "cid-version", + Usage: "CID version. Non default implies raw-leaves", + Value: defaultAddParams.CidVersion, + }, + cli.StringFlag{ + Name: "hash", + Usage: "Hash function to use. Implies cid-version=1.", + Value: defaultAddParams.HashFun, + }, cli.StringFlag{ Name: "name, n", Value: defaultAddParams.Name, @@ -366,6 +376,14 @@ If you prefer faster adding, add directly to the local IPFS and trigger a p.RawLeaves = c.Bool("raw-leaves") p.Hidden = c.Bool("hidden") p.Wrap = c.Bool("wrap-with-directory") || len(paths) > 1 + p.CidVersion = c.Int("cid-version") + p.HashFun = c.String("hash") + if p.HashFun != defaultAddParams.HashFun { + p.CidVersion = 1 + } + if p.CidVersion > 0 { + p.RawLeaves = true + } out := make(chan *api.AddedOutput, 1) var wg sync.WaitGroup diff --git a/package.json b/package.json index 3d7f9850..f38703af 100644 --- a/package.json +++ b/package.json @@ -75,9 +75,9 @@ }, { "author": "ipfs", - "hash": "QmPBf1azRsvCAMqEwZn3GMW5So5VHxW7hFMb5jz8sE7yFY", + "hash": "QmbjmbWbDCTKGba7zvtVx7qyyegziT3KWRtVN9mPaysmcF", "name": "go-ipfs-api", - "version": "1.3.2" + "version": "1.3.5" }, { "author": "whyrusleeping", @@ -87,9 +87,9 @@ }, { "author": "whyrusleeping", - "hash": "QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP", + "hash": "QmYjnkEL7i731PirfVH1sis89evN7jt4otSHw5D2xXXwUV", "name": "go-cid", - "version": "0.7.22" + "version": "0.7.25" }, { "author": "satori", @@ -99,20 +99,15 @@ }, { "author": "hsanjuan", - "hash": "Qmc3UwSvJkntxu2gKDPCqJEzmhqVeJtTbrVxJm6tsdmMF1", + "hash": "QmWbCAB5f3LDumj4ncz1UCHSiyXrXxkMxZB6Wv35xi4P8z", "name": "go-ipfs-chunker", - "version": "0.0.8" + "version": "0.0.10" }, { "author": "hector", - "hash": "QmSHjPDw8yNgLZ7cBfX7w3Smn7PHwYhNEpd4LHQQxUg35L", + "hash": "QmdBpJ5VTfL79VwKDU93z7fyZJ3mm4UaBHrE73CWRw2Bjd", "name": "go-ipfs-posinfo", - "version": "0.0.3" - }, - { - "hash": "QmPmdpQJRbpWM65XTrKMvpDDjqfSBVwnpQwij1Wq8xwHzk", - "name": "go-ipfs", - "version": "0.4.18-dev" + "version": "0.0.5" }, { "author": "dustin", @@ -122,20 +117,20 @@ }, { "author": "why", - "hash": "QmWJRM6rLjXGEXb5JkKu17Y68eJtCFcKPyRhb8JH2ELZ2Q", + "hash": "QmVKnUHAik1RuY38k775fuS9Um4jTNFYrNXxpPAmbUq6JW", "name": "go-unixfs", - "version": "1.0.6" + "version": "1.0.11" }, { "author": "why", - "hash": "QmXkZeJmx4c3ddjw81DQMUpM1e5LjAack5idzZYWUb2qAJ", + "hash": "QmQzSpSjkdGHW6WFBhUG6P3t9K8yv7iucucT1cQaqJ6tgd", "name": "go-merkledag", - "version": "1.0.6" + "version": "1.0.9" }, { - "hash": "QmZAsayEQakfFbHyakgHRKHwBTWrwuSBTfaMyxJZUG97VC", + "hash": "QmdP3wKxB6x6vJ57tDrewAJF2qv4ULejCZ6dspJRnk3993", "name": "go-libp2p-kad-dht", - "version": "4.3.1" + "version": "4.3.4" }, { "hash": "Qmbq7kGxgcpALGLPaWDyTa6KUq5kBUKdEvkvPZcBkJoLex", @@ -146,6 +141,12 @@ "hash": "QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky", "name": "go-ipfs-cmdkit", "version": "1.1.3" + }, + { + "author": "hsanjuan", + "hash": "QmZJMX4qjdLMKKPFcZN8PgkASZK9JHG9P8vZAfbiKyhzxv", + "name": "go-mfs", + "version": "0.0.2" } ], "gxVersion": "0.11.0", diff --git a/sharness/t0031-ctl-add.sh b/sharness/t0031-ctl-add.sh index 3f6ce59f..e40d2332 100755 --- a/sharness/t0031-ctl-add.sh +++ b/sharness/t0031-ctl-add.sh @@ -33,6 +33,9 @@ test_expect_success IPFS,CLUSTER "add files locally and compare with ipfs" ' ipfs-cluster-ctl add --quiet -r testFolder >> cidscluster.txt ipfs-cluster-ctl add --quiet -r -w testFolder >> cidscluster.txt + ipfs-cluster-ctl add --quiet --cid-version 1 -r testFolder >> cidscluster.txt + ipfs-cluster-ctl add --quiet --hash sha3-512 -r testFolder >> cidscluster.txt + ipfsCmd add --quiet /tmp/smallfile.bin > cidsipfs.txt ipfsCmd add --quiet -w /tmp/smallfile.bin >> cidsipfs.txt ipfsCmd add --quiet --raw-leaves -w /tmp/smallfile.bin >> cidsipfs.txt @@ -47,6 +50,9 @@ test_expect_success IPFS,CLUSTER "add files locally and compare with ipfs" ' ipfsCmd add --quiet -r /tmp/testFolder >> cidsipfs.txt ipfsCmd add --quiet -r -w /tmp/testFolder >> cidsipfs.txt + ipfsCmd add --quiet --cid-version 1 -r /tmp/testFolder >> cidsipfs.txt + ipfsCmd add --quiet --hash sha3-512 -r /tmp/testFolder >> cidsipfs.txt + test_cmp cidscluster.txt cidsipfs.txt '