2018-07-04 16:30:24 +00:00
|
|
|
package adder
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"errors"
|
|
|
|
"io"
|
|
|
|
"sync"
|
|
|
|
|
|
|
|
"github.com/ipfs/ipfs-cluster/adder/ipfsadd"
|
|
|
|
"github.com/ipfs/ipfs-cluster/api"
|
2018-08-06 10:44:44 +00:00
|
|
|
|
|
|
|
"github.com/ipfs/go-ipfs-cmdkit/files"
|
2018-07-04 16:30:24 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
// BlockHandler is a function used to process a block as is received by the
|
|
|
|
// Importer. Used in Importer.Run().
|
|
|
|
type BlockHandler func(ctx context.Context, n *api.NodeWithMeta) (string, error)
|
|
|
|
|
|
|
|
// Importer facilitates converting a file into a stream
|
|
|
|
// of chunked blocks.
|
|
|
|
type Importer struct {
|
|
|
|
startedMux sync.Mutex
|
|
|
|
started bool
|
|
|
|
|
|
|
|
files files.File
|
2018-08-06 10:44:44 +00:00
|
|
|
params *api.AddParams
|
2018-07-04 16:30:24 +00:00
|
|
|
|
|
|
|
output chan *api.AddedOutput
|
|
|
|
blocks chan *api.NodeWithMeta
|
|
|
|
errors chan error
|
|
|
|
}
|
|
|
|
|
|
|
|
// NewImporter sets up an Importer ready to Go().
|
2018-08-06 20:49:28 +00:00
|
|
|
func NewImporter(f files.File, p *api.AddParams, out chan *api.AddedOutput) (*Importer, error) {
|
2018-07-04 16:30:24 +00:00
|
|
|
blocks := make(chan *api.NodeWithMeta, 1)
|
|
|
|
errors := make(chan error, 1)
|
|
|
|
|
|
|
|
return &Importer{
|
|
|
|
started: false,
|
|
|
|
files: f,
|
|
|
|
params: p,
|
2018-08-06 20:49:28 +00:00
|
|
|
output: out,
|
2018-07-04 16:30:24 +00:00
|
|
|
blocks: blocks,
|
|
|
|
errors: errors,
|
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Blocks returns a channel where each imported block is sent.
|
|
|
|
func (imp *Importer) Blocks() <-chan *api.NodeWithMeta {
|
|
|
|
return imp.blocks
|
|
|
|
}
|
|
|
|
|
|
|
|
// Errors returns a channel to which any errors during the import
|
|
|
|
// process are sent.
|
|
|
|
func (imp *Importer) Errors() <-chan error {
|
|
|
|
return imp.errors
|
|
|
|
}
|
|
|
|
|
|
|
|
func (imp *Importer) start() bool {
|
|
|
|
imp.startedMux.Lock()
|
|
|
|
defer imp.startedMux.Unlock()
|
|
|
|
retVal := imp.started
|
|
|
|
imp.started = true
|
|
|
|
return !retVal
|
|
|
|
}
|
|
|
|
|
2018-07-24 13:22:29 +00:00
|
|
|
func (imp *Importer) addFile(ipfsAdder *ipfsadd.Adder) error {
|
|
|
|
f, err := imp.files.NextFile()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
logger.Debugf("ipfsAdder AddFile(%s)", f.FullPath())
|
|
|
|
return ipfsAdder.AddFile(f)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (imp *Importer) addFiles(ctx context.Context, ipfsAdder *ipfsadd.Adder) {
|
|
|
|
defer close(imp.blocks)
|
|
|
|
defer close(imp.errors)
|
|
|
|
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
imp.errors <- ctx.Err()
|
|
|
|
return
|
|
|
|
default:
|
|
|
|
err := imp.addFile(ipfsAdder)
|
2018-07-24 13:27:46 +00:00
|
|
|
if err == io.EOF {
|
|
|
|
goto FINALIZE
|
|
|
|
}
|
|
|
|
|
2018-07-24 13:22:29 +00:00
|
|
|
if err != nil {
|
|
|
|
imp.errors <- err
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
FINALIZE:
|
|
|
|
_, err := ipfsAdder.Finalize()
|
|
|
|
if err != nil {
|
|
|
|
imp.errors <- err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-07-24 12:21:29 +00:00
|
|
|
// Go starts a goroutine which reads the blocks as outputted by the
|
|
|
|
// ipfsadd module called with the parameters of this importer. The blocks,
|
|
|
|
// errors and output are placed in the respective importer channels for
|
|
|
|
// further processing. When there are no more blocks, or an error happen,
|
|
|
|
// the channels will be closed.
|
2018-07-04 16:30:24 +00:00
|
|
|
func (imp *Importer) Go(ctx context.Context) error {
|
|
|
|
if !imp.start() {
|
2018-07-24 12:21:29 +00:00
|
|
|
return errors.New("importing process already started or finished")
|
2018-07-04 16:30:24 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
dagsvc := newAdderDAGService(imp.blocks)
|
|
|
|
|
|
|
|
ipfsAdder, err := ipfsadd.NewAdder(ctx, dagsvc)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
ipfsAdder.Hidden = imp.params.Hidden
|
|
|
|
ipfsAdder.Trickle = imp.params.Layout == "trickle"
|
|
|
|
ipfsAdder.RawLeaves = imp.params.RawLeaves
|
|
|
|
ipfsAdder.Wrap = true
|
|
|
|
ipfsAdder.Chunker = imp.params.Chunker
|
|
|
|
ipfsAdder.Out = imp.output
|
|
|
|
|
2018-07-24 13:22:29 +00:00
|
|
|
go imp.addFiles(ctx, ipfsAdder)
|
2018-07-04 16:30:24 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2018-07-24 12:21:29 +00:00
|
|
|
// Run triggers the importing process (calling Go) and calls the given BlockHandler
|
|
|
|
// on every node read from the importer.
|
2018-07-04 16:30:24 +00:00
|
|
|
// It returns the value returned by the last-called BlockHandler.
|
|
|
|
func (imp *Importer) Run(ctx context.Context, blockF BlockHandler) (string, error) {
|
|
|
|
var retVal string
|
|
|
|
|
|
|
|
errors := imp.Errors()
|
|
|
|
blocks := imp.Blocks()
|
|
|
|
|
|
|
|
err := imp.Go(ctx)
|
|
|
|
if err != nil {
|
|
|
|
return retVal, err
|
|
|
|
}
|
|
|
|
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
return retVal, ctx.Err()
|
|
|
|
case err, ok := <-errors:
|
|
|
|
if ok {
|
2018-07-19 13:17:27 +00:00
|
|
|
logger.Error(err)
|
2018-07-04 16:30:24 +00:00
|
|
|
return retVal, err
|
|
|
|
}
|
|
|
|
case node, ok := <-blocks:
|
|
|
|
if !ok {
|
2018-07-17 11:35:23 +00:00
|
|
|
goto BREAK // finished importing
|
2018-07-04 16:30:24 +00:00
|
|
|
}
|
2018-07-17 11:35:23 +00:00
|
|
|
retVal, err = blockF(ctx, node)
|
2018-07-04 16:30:24 +00:00
|
|
|
if err != nil {
|
|
|
|
return retVal, err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2018-07-17 11:35:23 +00:00
|
|
|
BREAK:
|
2018-07-04 16:30:24 +00:00
|
|
|
// grab any last errors from errors if necessary
|
|
|
|
// (this waits for errors to be closed)
|
|
|
|
err = <-errors
|
|
|
|
return retVal, err
|
|
|
|
}
|