2018-08-23 12:12:55 +00:00
|
|
|
// Package adder implements functionality to add content to IPFS daemons
|
|
|
|
// managed by the Cluster.
|
2018-07-04 16:30:24 +00:00
|
|
|
package adder
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
2018-08-20 14:25:26 +00:00
|
|
|
"fmt"
|
2018-07-04 16:30:24 +00:00
|
|
|
"mime/multipart"
|
2018-08-20 14:25:26 +00:00
|
|
|
"strings"
|
2018-07-04 16:30:24 +00:00
|
|
|
|
2018-08-07 18:01:02 +00:00
|
|
|
"github.com/ipfs/ipfs-cluster/adder/ipfsadd"
|
2018-08-06 10:44:44 +00:00
|
|
|
"github.com/ipfs/ipfs-cluster/api"
|
|
|
|
|
2018-12-06 18:59:05 +00:00
|
|
|
cid "github.com/ipfs/go-cid"
|
|
|
|
files "github.com/ipfs/go-ipfs-files"
|
|
|
|
ipld "github.com/ipfs/go-ipld-format"
|
2020-03-13 20:40:02 +00:00
|
|
|
logging "github.com/ipfs/go-log/v2"
|
2018-12-06 18:59:05 +00:00
|
|
|
merkledag "github.com/ipfs/go-merkledag"
|
|
|
|
multihash "github.com/multiformats/go-multihash"
|
2018-07-04 16:30:24 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
var logger = logging.Logger("adder")
|
|
|
|
|
2018-08-07 18:01:02 +00:00
|
|
|
// ClusterDAGService is an implementation of ipld.DAGService plus a Finalize
|
2018-08-08 19:10:42 +00:00
|
|
|
// method. ClusterDAGServices can be used to provide Adders with a different
|
|
|
|
// add implementation.
|
2018-08-07 18:01:02 +00:00
|
|
|
type ClusterDAGService interface {
|
|
|
|
ipld.DAGService
|
2018-08-08 23:16:30 +00:00
|
|
|
// Finalize receives the IPFS content root CID as
|
|
|
|
// returned by the ipfs adder.
|
2018-09-22 01:00:10 +00:00
|
|
|
Finalize(ctx context.Context, ipfsRoot cid.Cid) (cid.Cid, error)
|
2018-08-07 18:01:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Adder is used to add content to IPFS Cluster using an implementation of
|
|
|
|
// ClusterDAGService.
|
|
|
|
type Adder struct {
|
|
|
|
ctx context.Context
|
|
|
|
cancel context.CancelFunc
|
|
|
|
|
2018-08-09 11:24:10 +00:00
|
|
|
dgs ClusterDAGService
|
2018-08-07 18:01:02 +00:00
|
|
|
|
|
|
|
params *api.AddParams
|
2018-08-08 19:10:42 +00:00
|
|
|
|
|
|
|
// AddedOutput updates are placed on this channel
|
|
|
|
// whenever a block is processed. They contain information
|
|
|
|
// about the block, the CID, the Name etc. and are mostly
|
|
|
|
// meant to be streamed back to the user.
|
2018-08-07 18:01:02 +00:00
|
|
|
output chan *api.AddedOutput
|
|
|
|
}
|
|
|
|
|
2018-08-08 19:10:42 +00:00
|
|
|
// New returns a new Adder with the given ClusterDAGService, add options and a
|
2018-08-07 18:01:02 +00:00
|
|
|
// channel to send updates during the adding process.
|
|
|
|
//
|
|
|
|
// An Adder may only be used once.
|
2018-08-08 19:10:42 +00:00
|
|
|
func New(ds ClusterDAGService, p *api.AddParams, out chan *api.AddedOutput) *Adder {
|
|
|
|
// Discard all progress update output as the caller has not provided
|
|
|
|
// a channel for them to listen on.
|
2018-08-07 18:01:02 +00:00
|
|
|
if out == nil {
|
|
|
|
out = make(chan *api.AddedOutput, 100)
|
|
|
|
go func() {
|
|
|
|
for range out {
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
}
|
|
|
|
|
|
|
|
return &Adder{
|
2018-08-10 12:39:34 +00:00
|
|
|
dgs: ds,
|
2018-08-07 18:01:02 +00:00
|
|
|
params: p,
|
|
|
|
output: out,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-08-08 19:10:42 +00:00
|
|
|
func (a *Adder) setContext(ctx context.Context) {
|
2018-08-10 09:51:27 +00:00
|
|
|
if a.ctx == nil { // only allows first context
|
|
|
|
ctxc, cancel := context.WithCancel(ctx)
|
|
|
|
a.ctx = ctxc
|
|
|
|
a.cancel = cancel
|
|
|
|
}
|
2018-08-08 19:10:42 +00:00
|
|
|
}
|
|
|
|
|
2018-08-07 18:01:02 +00:00
|
|
|
// FromMultipart adds content from a multipart.Reader. The adder will
|
|
|
|
// no longer be usable after calling this method.
|
2018-09-22 01:00:10 +00:00
|
|
|
func (a *Adder) FromMultipart(ctx context.Context, r *multipart.Reader) (cid.Cid, error) {
|
2018-08-07 18:01:02 +00:00
|
|
|
logger.Debugf("adding from multipart with params: %+v", a.params)
|
|
|
|
|
2018-12-01 04:58:55 +00:00
|
|
|
f, err := files.NewFileFromPartReader(r, "multipart/form-data")
|
|
|
|
if err != nil {
|
|
|
|
return cid.Undef, err
|
2018-08-07 18:01:02 +00:00
|
|
|
}
|
|
|
|
defer f.Close()
|
2018-08-08 19:10:42 +00:00
|
|
|
return a.FromFiles(ctx, f)
|
2018-08-07 18:01:02 +00:00
|
|
|
}
|
|
|
|
|
2018-12-01 04:58:55 +00:00
|
|
|
// FromFiles adds content from a files.Directory. The adder will no longer
|
2018-08-07 18:01:02 +00:00
|
|
|
// be usable after calling this method.
|
2018-12-01 04:58:55 +00:00
|
|
|
func (a *Adder) FromFiles(ctx context.Context, f files.Directory) (cid.Cid, error) {
|
2019-02-27 18:43:29 +00:00
|
|
|
logger.Debug("adding from files")
|
2018-08-08 19:10:42 +00:00
|
|
|
a.setContext(ctx)
|
|
|
|
|
2018-08-07 18:01:02 +00:00
|
|
|
if a.ctx.Err() != nil { // don't allow running twice
|
2018-09-22 01:00:10 +00:00
|
|
|
return cid.Undef, a.ctx.Err()
|
2018-08-07 18:01:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
defer a.cancel()
|
|
|
|
defer close(a.output)
|
|
|
|
|
2018-08-09 11:24:10 +00:00
|
|
|
ipfsAdder, err := ipfsadd.NewAdder(a.ctx, a.dgs)
|
2018-08-07 18:01:02 +00:00
|
|
|
if err != nil {
|
2018-08-08 19:43:20 +00:00
|
|
|
logger.Error(err)
|
2018-09-22 01:00:10 +00:00
|
|
|
return cid.Undef, err
|
2018-08-07 18:01:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
ipfsAdder.Trickle = a.params.Layout == "trickle"
|
|
|
|
ipfsAdder.RawLeaves = a.params.RawLeaves
|
|
|
|
ipfsAdder.Chunker = a.params.Chunker
|
|
|
|
ipfsAdder.Out = a.output
|
2018-08-16 15:40:32 +00:00
|
|
|
ipfsAdder.Progress = a.params.Progress
|
2019-03-23 13:47:45 +00:00
|
|
|
ipfsAdder.NoCopy = a.params.NoCopy
|
2018-08-07 18:01:02 +00:00
|
|
|
|
2018-08-20 14:25:26 +00:00
|
|
|
// Set up prefix
|
|
|
|
prefix, err := merkledag.PrefixForCidVersion(a.params.CidVersion)
|
|
|
|
if err != nil {
|
2018-09-22 01:00:10 +00:00
|
|
|
return cid.Undef, fmt.Errorf("bad CID Version: %s", err)
|
2018-08-20 14:25:26 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
hashFunCode, ok := multihash.Names[strings.ToLower(a.params.HashFun)]
|
|
|
|
if !ok {
|
2018-09-22 01:00:10 +00:00
|
|
|
return cid.Undef, fmt.Errorf("unrecognized hash function: %s", a.params.HashFun)
|
2018-08-20 14:25:26 +00:00
|
|
|
}
|
|
|
|
prefix.MhType = hashFunCode
|
|
|
|
prefix.MhLength = -1
|
|
|
|
ipfsAdder.CidBuilder = &prefix
|
|
|
|
|
2019-10-01 09:32:58 +00:00
|
|
|
// setup wrapping
|
|
|
|
if a.params.Wrap {
|
|
|
|
f = files.NewSliceDirectory(
|
|
|
|
[]files.DirEntry{files.FileEntry("", f)},
|
|
|
|
)
|
|
|
|
}
|
|
|
|
|
2018-12-01 04:58:55 +00:00
|
|
|
it := f.Entries()
|
2019-09-27 16:54:49 +00:00
|
|
|
var adderRoot ipld.Node
|
2018-12-01 04:58:55 +00:00
|
|
|
for it.Next() {
|
2019-12-17 23:24:16 +00:00
|
|
|
// In order to set the AddedOutput names right, we use
|
|
|
|
// OutputPrefix:
|
|
|
|
//
|
|
|
|
// When adding a folder, this is the root folder name which is
|
|
|
|
// prepended to the addedpaths. When adding a single file,
|
|
|
|
// this is the name of the file which overrides the empty
|
|
|
|
// AddedOutput name.
|
|
|
|
//
|
2019-12-18 10:32:28 +00:00
|
|
|
// After coreunix/add.go was refactored in go-ipfs and we
|
|
|
|
// followed suit, it no longer receives the name of the
|
|
|
|
// file/folder being added and does not emit AddedOutput
|
|
|
|
// events with the right names. We addressed this by adding
|
2020-02-03 09:30:04 +00:00
|
|
|
// OutputPrefix to our version. go-ipfs modifies emitted
|
2019-12-18 10:32:28 +00:00
|
|
|
// events before sending to user).
|
2019-12-17 23:24:16 +00:00
|
|
|
ipfsAdder.OutputPrefix = it.Name()
|
2019-12-17 22:01:25 +00:00
|
|
|
|
2018-08-07 18:01:02 +00:00
|
|
|
select {
|
|
|
|
case <-a.ctx.Done():
|
2018-09-22 01:00:10 +00:00
|
|
|
return cid.Undef, a.ctx.Err()
|
2018-08-07 18:01:02 +00:00
|
|
|
default:
|
2018-12-01 04:58:55 +00:00
|
|
|
logger.Debugf("ipfsAdder AddFile(%s)", it.Name())
|
|
|
|
|
2019-09-27 16:54:49 +00:00
|
|
|
adderRoot, err = ipfsAdder.AddAllAndPin(it.Node())
|
|
|
|
if err != nil {
|
2018-08-08 19:29:21 +00:00
|
|
|
logger.Error("error adding to cluster: ", err)
|
2018-09-22 01:00:10 +00:00
|
|
|
return cid.Undef, err
|
2018-08-07 18:01:02 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2018-12-01 04:58:55 +00:00
|
|
|
if it.Err() != nil {
|
|
|
|
return cid.Undef, it.Err()
|
|
|
|
}
|
2018-08-07 18:24:33 +00:00
|
|
|
|
2018-08-09 11:24:10 +00:00
|
|
|
clusterRoot, err := a.dgs.Finalize(a.ctx, adderRoot.Cid())
|
2018-08-08 19:29:21 +00:00
|
|
|
if err != nil {
|
|
|
|
logger.Error("error finalizing adder:", err)
|
2018-09-22 01:00:10 +00:00
|
|
|
return cid.Undef, err
|
2018-08-08 19:29:21 +00:00
|
|
|
}
|
2018-08-08 23:16:30 +00:00
|
|
|
logger.Infof("%s successfully added to cluster", clusterRoot)
|
|
|
|
return clusterRoot, nil
|
2018-08-07 18:01:02 +00:00
|
|
|
}
|