2018-08-23 12:12:55 +00:00
|
|
|
// Package adder implements functionality to add content to IPFS daemons
|
|
|
|
// managed by the Cluster.
|
2018-07-04 16:30:24 +00:00
|
|
|
package adder
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
2018-08-20 14:25:26 +00:00
|
|
|
"fmt"
|
2018-08-07 18:01:02 +00:00
|
|
|
"io"
|
2018-07-04 16:30:24 +00:00
|
|
|
"mime/multipart"
|
2018-08-20 14:25:26 +00:00
|
|
|
"strings"
|
2018-07-04 16:30:24 +00:00
|
|
|
|
2018-08-07 18:01:02 +00:00
|
|
|
"github.com/ipfs/ipfs-cluster/adder/ipfsadd"
|
2018-08-06 10:44:44 +00:00
|
|
|
"github.com/ipfs/ipfs-cluster/api"
|
|
|
|
|
2018-07-19 13:17:27 +00:00
|
|
|
cid "github.com/ipfs/go-cid"
|
2018-10-23 23:28:05 +00:00
|
|
|
files "github.com/ipfs/go-ipfs-files"
|
2018-08-07 18:01:02 +00:00
|
|
|
ipld "github.com/ipfs/go-ipld-format"
|
2018-07-04 16:30:24 +00:00
|
|
|
logging "github.com/ipfs/go-log"
|
2018-08-20 14:25:26 +00:00
|
|
|
merkledag "github.com/ipfs/go-merkledag"
|
|
|
|
multihash "github.com/multiformats/go-multihash"
|
2018-07-04 16:30:24 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
var logger = logging.Logger("adder")
|
|
|
|
|
2018-08-07 18:01:02 +00:00
|
|
|
// ClusterDAGService is an implementation of ipld.DAGService plus a Finalize
|
2018-08-08 19:10:42 +00:00
|
|
|
// method. ClusterDAGServices can be used to provide Adders with a different
|
|
|
|
// add implementation.
|
2018-08-07 18:01:02 +00:00
|
|
|
type ClusterDAGService interface {
|
|
|
|
ipld.DAGService
|
2018-08-08 23:16:30 +00:00
|
|
|
// Finalize receives the IPFS content root CID as
|
|
|
|
// returned by the ipfs adder.
|
2018-09-22 01:00:10 +00:00
|
|
|
Finalize(ctx context.Context, ipfsRoot cid.Cid) (cid.Cid, error)
|
2018-08-07 18:01:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Adder is used to add content to IPFS Cluster using an implementation of
|
|
|
|
// ClusterDAGService.
|
|
|
|
type Adder struct {
|
|
|
|
ctx context.Context
|
|
|
|
cancel context.CancelFunc
|
|
|
|
|
2018-08-09 11:24:10 +00:00
|
|
|
dgs ClusterDAGService
|
2018-08-07 18:01:02 +00:00
|
|
|
|
|
|
|
params *api.AddParams
|
2018-08-08 19:10:42 +00:00
|
|
|
|
|
|
|
// AddedOutput updates are placed on this channel
|
|
|
|
// whenever a block is processed. They contain information
|
|
|
|
// about the block, the CID, the Name etc. and are mostly
|
|
|
|
// meant to be streamed back to the user.
|
2018-08-07 18:01:02 +00:00
|
|
|
output chan *api.AddedOutput
|
|
|
|
}
|
|
|
|
|
2018-08-08 19:10:42 +00:00
|
|
|
// New returns a new Adder with the given ClusterDAGService, add options and a
|
2018-08-07 18:01:02 +00:00
|
|
|
// channel to send updates during the adding process.
|
|
|
|
//
|
|
|
|
// An Adder may only be used once.
|
2018-08-08 19:10:42 +00:00
|
|
|
func New(ds ClusterDAGService, p *api.AddParams, out chan *api.AddedOutput) *Adder {
|
|
|
|
// Discard all progress update output as the caller has not provided
|
|
|
|
// a channel for them to listen on.
|
2018-08-07 18:01:02 +00:00
|
|
|
if out == nil {
|
|
|
|
out = make(chan *api.AddedOutput, 100)
|
|
|
|
go func() {
|
|
|
|
for range out {
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
}
|
|
|
|
|
|
|
|
return &Adder{
|
2018-08-10 12:39:34 +00:00
|
|
|
dgs: ds,
|
2018-08-07 18:01:02 +00:00
|
|
|
params: p,
|
|
|
|
output: out,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-08-08 19:10:42 +00:00
|
|
|
func (a *Adder) setContext(ctx context.Context) {
|
2018-08-10 09:51:27 +00:00
|
|
|
if a.ctx == nil { // only allows first context
|
|
|
|
ctxc, cancel := context.WithCancel(ctx)
|
|
|
|
a.ctx = ctxc
|
|
|
|
a.cancel = cancel
|
|
|
|
}
|
2018-08-08 19:10:42 +00:00
|
|
|
}
|
|
|
|
|
2018-08-07 18:01:02 +00:00
|
|
|
// FromMultipart adds content from a multipart.Reader. The adder will
|
|
|
|
// no longer be usable after calling this method.
|
2018-09-22 01:00:10 +00:00
|
|
|
func (a *Adder) FromMultipart(ctx context.Context, r *multipart.Reader) (cid.Cid, error) {
|
2018-08-07 18:01:02 +00:00
|
|
|
logger.Debugf("adding from multipart with params: %+v", a.params)
|
|
|
|
|
|
|
|
f := &files.MultipartFile{
|
|
|
|
Mediatype: "multipart/form-data",
|
|
|
|
Reader: r,
|
|
|
|
}
|
|
|
|
defer f.Close()
|
2018-08-08 19:10:42 +00:00
|
|
|
return a.FromFiles(ctx, f)
|
2018-08-07 18:01:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// FromFiles adds content from a files.File. The adder will no longer
|
|
|
|
// be usable after calling this method.
|
2018-09-22 01:00:10 +00:00
|
|
|
func (a *Adder) FromFiles(ctx context.Context, f files.File) (cid.Cid, error) {
|
2018-08-07 18:01:02 +00:00
|
|
|
logger.Debugf("adding from files")
|
2018-08-08 19:10:42 +00:00
|
|
|
a.setContext(ctx)
|
|
|
|
|
2018-08-07 18:01:02 +00:00
|
|
|
if a.ctx.Err() != nil { // don't allow running twice
|
2018-09-22 01:00:10 +00:00
|
|
|
return cid.Undef, a.ctx.Err()
|
2018-08-07 18:01:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
defer a.cancel()
|
|
|
|
defer close(a.output)
|
|
|
|
|
2018-08-09 11:24:10 +00:00
|
|
|
ipfsAdder, err := ipfsadd.NewAdder(a.ctx, a.dgs)
|
2018-08-07 18:01:02 +00:00
|
|
|
if err != nil {
|
2018-08-08 19:43:20 +00:00
|
|
|
logger.Error(err)
|
2018-09-22 01:00:10 +00:00
|
|
|
return cid.Undef, err
|
2018-08-07 18:01:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
ipfsAdder.Hidden = a.params.Hidden
|
|
|
|
ipfsAdder.Trickle = a.params.Layout == "trickle"
|
|
|
|
ipfsAdder.RawLeaves = a.params.RawLeaves
|
2018-08-08 23:16:30 +00:00
|
|
|
ipfsAdder.Wrap = a.params.Wrap
|
2018-08-07 18:01:02 +00:00
|
|
|
ipfsAdder.Chunker = a.params.Chunker
|
|
|
|
ipfsAdder.Out = a.output
|
2018-08-16 15:40:32 +00:00
|
|
|
ipfsAdder.Progress = a.params.Progress
|
2018-08-07 18:01:02 +00:00
|
|
|
|
2018-08-20 14:25:26 +00:00
|
|
|
// Set up prefix
|
|
|
|
prefix, err := merkledag.PrefixForCidVersion(a.params.CidVersion)
|
|
|
|
if err != nil {
|
2018-09-22 01:00:10 +00:00
|
|
|
return cid.Undef, fmt.Errorf("bad CID Version: %s", err)
|
2018-08-20 14:25:26 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
hashFunCode, ok := multihash.Names[strings.ToLower(a.params.HashFun)]
|
|
|
|
if !ok {
|
2018-09-22 01:00:10 +00:00
|
|
|
return cid.Undef, fmt.Errorf("unrecognized hash function: %s", a.params.HashFun)
|
2018-08-20 14:25:26 +00:00
|
|
|
}
|
|
|
|
prefix.MhType = hashFunCode
|
|
|
|
prefix.MhLength = -1
|
|
|
|
ipfsAdder.CidBuilder = &prefix
|
|
|
|
|
2018-08-07 18:01:02 +00:00
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-a.ctx.Done():
|
2018-09-22 01:00:10 +00:00
|
|
|
return cid.Undef, a.ctx.Err()
|
2018-08-07 18:01:02 +00:00
|
|
|
default:
|
|
|
|
err := addFile(f, ipfsAdder)
|
|
|
|
if err == io.EOF {
|
|
|
|
goto FINALIZE
|
|
|
|
}
|
|
|
|
if err != nil {
|
2018-08-08 19:29:21 +00:00
|
|
|
logger.Error("error adding to cluster: ", err)
|
2018-09-22 01:00:10 +00:00
|
|
|
return cid.Undef, err
|
2018-08-07 18:01:02 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2018-08-07 18:24:33 +00:00
|
|
|
|
2018-08-07 18:01:02 +00:00
|
|
|
FINALIZE:
|
2018-08-08 23:16:30 +00:00
|
|
|
adderRoot, err := ipfsAdder.Finalize()
|
2018-08-07 18:01:02 +00:00
|
|
|
if err != nil {
|
2018-09-22 01:00:10 +00:00
|
|
|
return cid.Undef, err
|
2018-08-07 18:01:02 +00:00
|
|
|
}
|
2018-08-09 11:24:10 +00:00
|
|
|
clusterRoot, err := a.dgs.Finalize(a.ctx, adderRoot.Cid())
|
2018-08-08 19:29:21 +00:00
|
|
|
if err != nil {
|
|
|
|
logger.Error("error finalizing adder:", err)
|
2018-09-22 01:00:10 +00:00
|
|
|
return cid.Undef, err
|
2018-08-08 19:29:21 +00:00
|
|
|
}
|
2018-08-08 23:16:30 +00:00
|
|
|
logger.Infof("%s successfully added to cluster", clusterRoot)
|
|
|
|
return clusterRoot, nil
|
2018-08-07 18:01:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func addFile(fs files.File, ipfsAdder *ipfsadd.Adder) error {
|
|
|
|
f, err := fs.NextFile()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2018-08-09 08:21:22 +00:00
|
|
|
defer f.Close()
|
2018-08-06 20:49:28 +00:00
|
|
|
|
2018-08-07 18:01:02 +00:00
|
|
|
logger.Debugf("ipfsAdder AddFile(%s)", f.FullPath())
|
|
|
|
return ipfsAdder.AddFile(f)
|
2018-07-04 16:30:24 +00:00
|
|
|
}
|