2018-02-09 22:20:44 +00:00
|
|
|
package importer
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"io"
|
|
|
|
"strings"
|
|
|
|
|
|
|
|
"github.com/ipfs/go-ipfs-cmdkit/files"
|
|
|
|
"github.com/ipfs/go-ipfs/core/coreunix"
|
|
|
|
ipld "github.com/ipfs/go-ipld-format"
|
|
|
|
)
|
|
|
|
|
|
|
|
// ToPrint is a first step towards a streaming importer. Verify that we
|
|
|
|
// can hijack the blockstore abstraction to redirect blocks as they arrive
|
|
|
|
// Closely follows go-ipfs/core/commands/add.go: Run func
|
|
|
|
func ToPrint(f files.File) error {
|
|
|
|
dserv := &pDAGService{
|
|
|
|
membership: make(map[string]bool),
|
|
|
|
}
|
|
|
|
|
|
|
|
ctx := context.Background() // using background for now, should upgrade later
|
|
|
|
fileAdder, err := coreunix.NewAdder(ctx, nil, nil, dserv)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
fileAdder.Pin = false // This way we can be honest that blockstore doesn't exist
|
|
|
|
|
|
|
|
// add all files under the root, as in ipfs
|
|
|
|
for {
|
|
|
|
file, err := f.NextFile()
|
|
|
|
if err == io.EOF {
|
|
|
|
break
|
|
|
|
} else if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if err := fileAdder.AddFile(file); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Without this call all of the directory nodes (stored in MFS) do not get
|
|
|
|
// written through to the dagservice and its blockstore
|
|
|
|
_, err = fileAdder.Finalize()
|
|
|
|
// ignore errors caused by printing-blockstore Get not finding blocks
|
|
|
|
if err != nil && !strings.Contains(err.Error(), "dagservice: block not found") {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// ToChannel imports file to ipfs ipld nodes, outputting nodes on the
|
|
|
|
// provided channel
|
2018-02-15 00:02:55 +00:00
|
|
|
func ToChannel(f files.File, ctx context.Context) (<-chan *ipld.Node, error) {
|
2018-02-12 20:57:04 +00:00
|
|
|
outChan := make(chan *ipld.Node)
|
2018-02-09 22:20:44 +00:00
|
|
|
dserv := &outDAGService{
|
2018-02-15 00:02:55 +00:00
|
|
|
membership: make(map[string]struct{}),
|
2018-02-09 22:20:44 +00:00
|
|
|
outChan: outChan,
|
|
|
|
}
|
|
|
|
|
|
|
|
fileAdder, err := coreunix.NewAdder(ctx, nil, nil, dserv)
|
|
|
|
if err != nil {
|
2018-02-15 00:02:55 +00:00
|
|
|
return outChan, err
|
2018-02-09 22:20:44 +00:00
|
|
|
}
|
|
|
|
fileAdder.Pin = false
|
|
|
|
|
2018-02-12 20:57:04 +00:00
|
|
|
go func() {
|
|
|
|
defer close(outChan)
|
|
|
|
// add all files under the root, as in ipfs
|
|
|
|
for {
|
|
|
|
file, err := f.NextFile()
|
|
|
|
if err == io.EOF {
|
|
|
|
break
|
|
|
|
} else if err != nil {
|
2018-02-15 00:02:55 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
return
|
|
|
|
default:
|
2018-02-12 20:57:04 +00:00
|
|
|
}
|
|
|
|
if err := fileAdder.AddFile(file); err != nil {
|
2018-02-15 00:02:55 +00:00
|
|
|
return
|
2018-02-12 20:57:04 +00:00
|
|
|
}
|
2018-02-09 22:20:44 +00:00
|
|
|
}
|
|
|
|
|
2018-02-12 20:57:04 +00:00
|
|
|
_, err = fileAdder.Finalize()
|
|
|
|
if !strings.Contains(err.Error(), "dagservice: block not found") {
|
2018-02-15 00:02:55 +00:00
|
|
|
return
|
2018-02-09 22:20:44 +00:00
|
|
|
}
|
2018-02-12 20:57:04 +00:00
|
|
|
}()
|
2018-02-15 00:02:55 +00:00
|
|
|
return outChan, nil
|
2018-02-09 22:20:44 +00:00
|
|
|
}
|