Merge pull request #513 from ipfs/remove-go-ipfs-dep
AddParams: support "cid-version" and "hash" function.
This commit is contained in:
Hector Sanjuan
GitHub
commit
d6e516b84a
|
|
@ -2,8 +2,10 @@ package adder
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"strings"
|
||||
|
||||
"github.com/ipfs/ipfs-cluster/adder/ipfsadd"
|
||||
"github.com/ipfs/ipfs-cluster/api"
|
||||
|
|
@ -12,6 +14,8 @@ import (
|
|||
files "github.com/ipfs/go-ipfs-cmdkit/files"
|
||||
ipld "github.com/ipfs/go-ipld-format"
|
||||
logging "github.com/ipfs/go-log"
|
||||
merkledag "github.com/ipfs/go-merkledag"
|
||||
multihash "github.com/multiformats/go-multihash"
|
||||
)
|
||||
|
||||
var logger = logging.Logger("adder")
|
||||
|
|
@ -113,6 +117,20 @@ func (a *Adder) FromFiles(ctx context.Context, f files.File) (*cid.Cid, error) {
|
|||
ipfsAdder.Out = a.output
|
||||
ipfsAdder.Progress = a.params.Progress
|
||||
|
||||
// Set up prefix
|
||||
prefix, err := merkledag.PrefixForCidVersion(a.params.CidVersion)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("bad CID Version: %s", err)
|
||||
}
|
||||
|
||||
hashFunCode, ok := multihash.Names[strings.ToLower(a.params.HashFun)]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unrecognized hash function: %s", a.params.HashFun)
|
||||
}
|
||||
prefix.MhType = hashFunCode
|
||||
prefix.MhLength = -1
|
||||
ipfsAdder.CidBuilder = &prefix
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-a.ctx.Done():
|
||||
|
|
|
|||
|
|
@ -14,10 +14,10 @@ import (
|
|||
chunker "github.com/ipfs/go-ipfs-chunker"
|
||||
files "github.com/ipfs/go-ipfs-cmdkit/files"
|
||||
posinfo "github.com/ipfs/go-ipfs-posinfo"
|
||||
mfs "github.com/ipfs/go-ipfs/mfs"
|
||||
ipld "github.com/ipfs/go-ipld-format"
|
||||
logging "github.com/ipfs/go-log"
|
||||
dag "github.com/ipfs/go-merkledag"
|
||||
mfs "github.com/ipfs/go-mfs"
|
||||
unixfs "github.com/ipfs/go-unixfs"
|
||||
balanced "github.com/ipfs/go-unixfs/importer/balanced"
|
||||
ihelper "github.com/ipfs/go-unixfs/importer/helpers"
|
||||
|
|
@ -61,6 +61,7 @@ type Adder struct {
|
|||
mroot *mfs.Root
|
||||
tempRoot *cid.Cid
|
||||
Prefix *cid.Prefix
|
||||
CidBuilder cid.Builder
|
||||
liveNodes uint64
|
||||
lastFile mfs.FSNode
|
||||
}
|
||||
|
|
@ -70,7 +71,7 @@ func (adder *Adder) mfsRoot() (*mfs.Root, error) {
|
|||
return adder.mroot, nil
|
||||
}
|
||||
rnode := unixfs.EmptyDirNode()
|
||||
rnode.SetPrefix(adder.Prefix)
|
||||
rnode.SetCidBuilder(adder.CidBuilder)
|
||||
mr, err := mfs.NewRoot(adder.ctx, adder.dagService, rnode, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -92,11 +93,11 @@ func (adder *Adder) add(reader io.Reader) (ipld.Node, error) {
|
|||
}
|
||||
|
||||
params := ihelper.DagBuilderParams{
|
||||
Dagserv: adder.dagService,
|
||||
RawLeaves: adder.RawLeaves,
|
||||
Maxlinks: ihelper.DefaultLinksPerBlock,
|
||||
NoCopy: adder.NoCopy,
|
||||
Prefix: adder.Prefix,
|
||||
Dagserv: adder.dagService,
|
||||
RawLeaves: adder.RawLeaves,
|
||||
Maxlinks: ihelper.DefaultLinksPerBlock,
|
||||
NoCopy: adder.NoCopy,
|
||||
CidBuilder: adder.CidBuilder,
|
||||
}
|
||||
|
||||
if adder.Trickle {
|
||||
|
|
@ -247,9 +248,9 @@ func (adder *Adder) addNode(node ipld.Node, path string) error {
|
|||
dir := gopath.Dir(path)
|
||||
if dir != "." {
|
||||
opts := mfs.MkdirOpts{
|
||||
Mkparents: true,
|
||||
Flush: false,
|
||||
Prefix: adder.Prefix,
|
||||
Mkparents: true,
|
||||
Flush: false,
|
||||
CidBuilder: adder.CidBuilder,
|
||||
}
|
||||
if err := mfs.Mkdir(mr, dir, opts); err != nil {
|
||||
return err
|
||||
|
|
@ -307,7 +308,7 @@ func (adder *Adder) addFile(file files.File) error {
|
|||
}
|
||||
|
||||
dagnode := dag.NodeWithData(sdata)
|
||||
dagnode.SetPrefix(adder.Prefix)
|
||||
dagnode.SetCidBuilder(adder.CidBuilder)
|
||||
err = adder.dagService.Add(adder.ctx, dagnode)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -346,9 +347,9 @@ func (adder *Adder) addDir(dir files.File) error {
|
|||
return err
|
||||
}
|
||||
err = mfs.Mkdir(mr, dir.FileName(), mfs.MkdirOpts{
|
||||
Mkparents: true,
|
||||
Flush: false,
|
||||
Prefix: adder.Prefix,
|
||||
Mkparents: true,
|
||||
Flush: false,
|
||||
CidBuilder: adder.CidBuilder,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
|
|||
55
api/add.go
55
api/add.go
|
|
@ -25,27 +25,31 @@ type AddedOutput struct {
|
|||
type AddParams struct {
|
||||
PinOptions
|
||||
|
||||
Recursive bool
|
||||
Layout string
|
||||
Chunker string
|
||||
RawLeaves bool
|
||||
Hidden bool
|
||||
Wrap bool
|
||||
Shard bool
|
||||
Progress bool
|
||||
Recursive bool
|
||||
Layout string
|
||||
Chunker string
|
||||
RawLeaves bool
|
||||
Hidden bool
|
||||
Wrap bool
|
||||
Shard bool
|
||||
Progress bool
|
||||
CidVersion int
|
||||
HashFun string
|
||||
}
|
||||
|
||||
// DefaultAddParams returns a AddParams object with standard defaults
|
||||
func DefaultAddParams() *AddParams {
|
||||
return &AddParams{
|
||||
Recursive: false,
|
||||
Layout: "", // corresponds to balanced layout
|
||||
Chunker: "size-262144",
|
||||
RawLeaves: false,
|
||||
Hidden: false,
|
||||
Wrap: false,
|
||||
Shard: false,
|
||||
Progress: false,
|
||||
Recursive: false,
|
||||
Layout: "", // corresponds to balanced layout
|
||||
Chunker: "size-262144",
|
||||
RawLeaves: false,
|
||||
Hidden: false,
|
||||
Wrap: false,
|
||||
Shard: false,
|
||||
Progress: false,
|
||||
CidVersion: 0,
|
||||
HashFun: "sha2-256",
|
||||
PinOptions: PinOptions{
|
||||
ReplicationFactorMin: 0,
|
||||
ReplicationFactorMax: 0,
|
||||
|
|
@ -96,6 +100,11 @@ func AddParamsFromQuery(query url.Values) (*AddParams, error) {
|
|||
name := query.Get("name")
|
||||
params.Name = name
|
||||
|
||||
hashF := query.Get("hash")
|
||||
if hashF != "" {
|
||||
params.HashFun = hashF
|
||||
}
|
||||
|
||||
err := parseBoolParam(query, "recursive", ¶ms.Recursive)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -132,6 +141,11 @@ func AddParamsFromQuery(query url.Values) (*AddParams, error) {
|
|||
return nil, err
|
||||
}
|
||||
|
||||
err = parseIntParam(query, "cid-version", ¶ms.CidVersion)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if v := query.Get("shard-size"); v != "" {
|
||||
shardSize, err := strconv.ParseUint(v, 10, 64)
|
||||
if err != nil {
|
||||
|
|
@ -148,7 +162,8 @@ func (p *AddParams) ToQueryString() string {
|
|||
fmtStr := "replication-min=%d&replication-max=%d&name=%s&"
|
||||
fmtStr += "shard=%t&shard-size=%d&recursive=%t&"
|
||||
fmtStr += "layout=%s&chunker=%s&raw-leaves=%t&hidden=%t&"
|
||||
fmtStr += "wrap-with-directory=%t&progress=%t"
|
||||
fmtStr += "wrap-with-directory=%t&progress=%t&"
|
||||
fmtStr += "cid-version=%d&hash=%s"
|
||||
query := fmt.Sprintf(
|
||||
fmtStr,
|
||||
p.ReplicationFactorMin,
|
||||
|
|
@ -163,6 +178,8 @@ func (p *AddParams) ToQueryString() string {
|
|||
p.Hidden,
|
||||
p.Wrap,
|
||||
p.Progress,
|
||||
p.CidVersion,
|
||||
p.HashFun,
|
||||
)
|
||||
return query
|
||||
}
|
||||
|
|
@ -179,5 +196,7 @@ func (p *AddParams) Equals(p2 *AddParams) bool {
|
|||
p.Chunker == p2.Chunker &&
|
||||
p.RawLeaves == p2.RawLeaves &&
|
||||
p.Hidden == p2.Hidden &&
|
||||
p.Wrap == p2.Wrap
|
||||
p.Wrap == p2.Wrap &&
|
||||
p.CidVersion == p2.CidVersion &&
|
||||
p.HashFun == p2.HashFun
|
||||
}
|
||||
|
|
|
|||
|
|
@ -298,6 +298,16 @@ If you prefer faster adding, add directly to the local IPFS and trigger a
|
|||
Name: "raw-leaves",
|
||||
Usage: "Use raw blocks for leaves (experimental)",
|
||||
},
|
||||
cli.IntFlag{
|
||||
Name: "cid-version",
|
||||
Usage: "CID version. Non default implies raw-leaves",
|
||||
Value: defaultAddParams.CidVersion,
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "hash",
|
||||
Usage: "Hash function to use. Implies cid-version=1.",
|
||||
Value: defaultAddParams.HashFun,
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "name, n",
|
||||
Value: defaultAddParams.Name,
|
||||
|
|
@ -366,6 +376,14 @@ If you prefer faster adding, add directly to the local IPFS and trigger a
|
|||
p.RawLeaves = c.Bool("raw-leaves")
|
||||
p.Hidden = c.Bool("hidden")
|
||||
p.Wrap = c.Bool("wrap-with-directory") || len(paths) > 1
|
||||
p.CidVersion = c.Int("cid-version")
|
||||
p.HashFun = c.String("hash")
|
||||
if p.HashFun != defaultAddParams.HashFun {
|
||||
p.CidVersion = 1
|
||||
}
|
||||
if p.CidVersion > 0 {
|
||||
p.RawLeaves = true
|
||||
}
|
||||
|
||||
out := make(chan *api.AddedOutput, 1)
|
||||
var wg sync.WaitGroup
|
||||
|
|
|
|||
39
package.json
39
package.json
|
|
@ -75,9 +75,9 @@
|
|||
},
|
||||
{
|
||||
"author": "ipfs",
|
||||
"hash": "QmPBf1azRsvCAMqEwZn3GMW5So5VHxW7hFMb5jz8sE7yFY",
|
||||
"hash": "QmbjmbWbDCTKGba7zvtVx7qyyegziT3KWRtVN9mPaysmcF",
|
||||
"name": "go-ipfs-api",
|
||||
"version": "1.3.2"
|
||||
"version": "1.3.5"
|
||||
},
|
||||
{
|
||||
"author": "whyrusleeping",
|
||||
|
|
@ -87,9 +87,9 @@
|
|||
},
|
||||
{
|
||||
"author": "whyrusleeping",
|
||||
"hash": "QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP",
|
||||
"hash": "QmYjnkEL7i731PirfVH1sis89evN7jt4otSHw5D2xXXwUV",
|
||||
"name": "go-cid",
|
||||
"version": "0.7.22"
|
||||
"version": "0.7.25"
|
||||
},
|
||||
{
|
||||
"author": "satori",
|
||||
|
|
@ -99,20 +99,15 @@
|
|||
},
|
||||
{
|
||||
"author": "hsanjuan",
|
||||
"hash": "Qmc3UwSvJkntxu2gKDPCqJEzmhqVeJtTbrVxJm6tsdmMF1",
|
||||
"hash": "QmWbCAB5f3LDumj4ncz1UCHSiyXrXxkMxZB6Wv35xi4P8z",
|
||||
"name": "go-ipfs-chunker",
|
||||
"version": "0.0.8"
|
||||
"version": "0.0.10"
|
||||
},
|
||||
{
|
||||
"author": "hector",
|
||||
"hash": "QmSHjPDw8yNgLZ7cBfX7w3Smn7PHwYhNEpd4LHQQxUg35L",
|
||||
"hash": "QmdBpJ5VTfL79VwKDU93z7fyZJ3mm4UaBHrE73CWRw2Bjd",
|
||||
"name": "go-ipfs-posinfo",
|
||||
"version": "0.0.3"
|
||||
},
|
||||
{
|
||||
"hash": "QmPmdpQJRbpWM65XTrKMvpDDjqfSBVwnpQwij1Wq8xwHzk",
|
||||
"name": "go-ipfs",
|
||||
"version": "0.4.18-dev"
|
||||
"version": "0.0.5"
|
||||
},
|
||||
{
|
||||
"author": "dustin",
|
||||
|
|
@ -122,20 +117,20 @@
|
|||
},
|
||||
{
|
||||
"author": "why",
|
||||
"hash": "QmWJRM6rLjXGEXb5JkKu17Y68eJtCFcKPyRhb8JH2ELZ2Q",
|
||||
"hash": "QmVKnUHAik1RuY38k775fuS9Um4jTNFYrNXxpPAmbUq6JW",
|
||||
"name": "go-unixfs",
|
||||
"version": "1.0.6"
|
||||
"version": "1.0.11"
|
||||
},
|
||||
{
|
||||
"author": "why",
|
||||
"hash": "QmXkZeJmx4c3ddjw81DQMUpM1e5LjAack5idzZYWUb2qAJ",
|
||||
"hash": "QmQzSpSjkdGHW6WFBhUG6P3t9K8yv7iucucT1cQaqJ6tgd",
|
||||
"name": "go-merkledag",
|
||||
"version": "1.0.6"
|
||||
"version": "1.0.9"
|
||||
},
|
||||
{
|
||||
"hash": "QmZAsayEQakfFbHyakgHRKHwBTWrwuSBTfaMyxJZUG97VC",
|
||||
"hash": "QmdP3wKxB6x6vJ57tDrewAJF2qv4ULejCZ6dspJRnk3993",
|
||||
"name": "go-libp2p-kad-dht",
|
||||
"version": "4.3.1"
|
||||
"version": "4.3.4"
|
||||
},
|
||||
{
|
||||
"hash": "Qmbq7kGxgcpALGLPaWDyTa6KUq5kBUKdEvkvPZcBkJoLex",
|
||||
|
|
@ -146,6 +141,12 @@
|
|||
"hash": "QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky",
|
||||
"name": "go-ipfs-cmdkit",
|
||||
"version": "1.1.3"
|
||||
},
|
||||
{
|
||||
"author": "hsanjuan",
|
||||
"hash": "QmZJMX4qjdLMKKPFcZN8PgkASZK9JHG9P8vZAfbiKyhzxv",
|
||||
"name": "go-mfs",
|
||||
"version": "0.0.2"
|
||||
}
|
||||
],
|
||||
"gxVersion": "0.11.0",
|
||||
|
|
|
|||
|
|
@ -33,6 +33,9 @@ test_expect_success IPFS,CLUSTER "add files locally and compare with ipfs" '
|
|||
ipfs-cluster-ctl add --quiet -r testFolder >> cidscluster.txt
|
||||
ipfs-cluster-ctl add --quiet -r -w testFolder >> cidscluster.txt
|
||||
|
||||
ipfs-cluster-ctl add --quiet --cid-version 1 -r testFolder >> cidscluster.txt
|
||||
ipfs-cluster-ctl add --quiet --hash sha3-512 -r testFolder >> cidscluster.txt
|
||||
|
||||
ipfsCmd add --quiet /tmp/smallfile.bin > cidsipfs.txt
|
||||
ipfsCmd add --quiet -w /tmp/smallfile.bin >> cidsipfs.txt
|
||||
ipfsCmd add --quiet --raw-leaves -w /tmp/smallfile.bin >> cidsipfs.txt
|
||||
|
|
@ -47,6 +50,9 @@ test_expect_success IPFS,CLUSTER "add files locally and compare with ipfs" '
|
|||
ipfsCmd add --quiet -r /tmp/testFolder >> cidsipfs.txt
|
||||
ipfsCmd add --quiet -r -w /tmp/testFolder >> cidsipfs.txt
|
||||
|
||||
ipfsCmd add --quiet --cid-version 1 -r /tmp/testFolder >> cidsipfs.txt
|
||||
ipfsCmd add --quiet --hash sha3-512 -r /tmp/testFolder >> cidsipfs.txt
|
||||
|
||||
test_cmp cidscluster.txt cidsipfs.txt
|
||||
'
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user