70 lines
1.4 KiB
Go
70 lines
1.4 KiB
Go
// package sharding implements a sharding adder that chunks and
|
|
// shards content while it's added, creating Cluster DAGs and
|
|
// pinning them.
|
|
package sharding
|
|
|
|
import (
|
|
"context"
|
|
"mime/multipart"
|
|
|
|
"github.com/ipfs/ipfs-cluster/adder"
|
|
"github.com/ipfs/ipfs-cluster/api"
|
|
|
|
rpc "github.com/hsanjuan/go-libp2p-gorpc"
|
|
"github.com/ipfs/go-ipfs-cmdkit/files"
|
|
logging "github.com/ipfs/go-log"
|
|
)
|
|
|
|
var logger = logging.Logger("addshard")
|
|
|
|
type Adder struct {
|
|
rpcClient *rpc.Client
|
|
}
|
|
|
|
func New(rpc *rpc.Client) *Adder {
|
|
return &Adder{
|
|
rpcClient: rpc,
|
|
}
|
|
}
|
|
|
|
func (a *Adder) FromMultipart(ctx context.Context, r *multipart.Reader, p *adder.Params) error {
|
|
f := &files.MultipartFile{
|
|
Mediatype: "multipart/form-data",
|
|
Reader: r,
|
|
}
|
|
|
|
pinOpts := api.PinOptions{
|
|
ReplicationFactorMin: p.ReplicationFactorMin,
|
|
ReplicationFactorMax: p.ReplicationFactorMax,
|
|
Name: p.Name,
|
|
ShardSize: p.ShardSize,
|
|
}
|
|
|
|
dagBuilder := newClusterDAGBuilder(a.rpcClient, pinOpts)
|
|
// Always stop the builder
|
|
defer dagBuilder.Cancel()
|
|
|
|
blockSink := dagBuilder.Blocks()
|
|
|
|
blockHandle := func(ctx context.Context, n *api.NodeWithMeta) (string, error) {
|
|
blockSink <- n
|
|
return "", nil
|
|
}
|
|
|
|
importer, err := adder.NewImporter(f, p)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
_, err = importer.Run(ctx, blockHandle)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Trigger shard finalize
|
|
close(blockSink)
|
|
|
|
<-dagBuilder.Done() // wait for the builder to finish
|
|
return nil
|
|
}
|