2018-08-17 14:17:28 +00:00
|
|
|
// Package adderutils provides some utilities for adding content to cluster.
|
|
|
|
package adderutils
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"encoding/json"
|
|
|
|
"mime/multipart"
|
|
|
|
"net/http"
|
|
|
|
"sync"
|
|
|
|
|
|
|
|
"github.com/ipfs/ipfs-cluster/adder"
|
|
|
|
"github.com/ipfs/ipfs-cluster/adder/local"
|
|
|
|
"github.com/ipfs/ipfs-cluster/adder/sharding"
|
|
|
|
"github.com/ipfs/ipfs-cluster/api"
|
|
|
|
|
|
|
|
cid "github.com/ipfs/go-cid"
|
|
|
|
logging "github.com/ipfs/go-log"
|
2018-10-17 13:28:03 +00:00
|
|
|
rpc "github.com/libp2p/go-libp2p-gorpc"
|
2018-08-17 14:17:28 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
var logger = logging.Logger("adder")
|
|
|
|
|
|
|
|
// AddMultipartHTTPHandler is a helper function to add content
|
2018-10-03 21:03:30 +00:00
|
|
|
// uploaded using a multipart request. The outputTransform parameter
|
|
|
|
// allows to customize the http response output format to something
|
|
|
|
// else than api.AddedOutput objects.
|
2018-08-17 14:17:28 +00:00
|
|
|
func AddMultipartHTTPHandler(
|
|
|
|
ctx context.Context,
|
|
|
|
rpc *rpc.Client,
|
|
|
|
params *api.AddParams,
|
|
|
|
reader *multipart.Reader,
|
|
|
|
w http.ResponseWriter,
|
2018-10-03 21:03:30 +00:00
|
|
|
outputTransform func(*api.AddedOutput) interface{},
|
2018-09-22 01:00:10 +00:00
|
|
|
) (cid.Cid, error) {
|
2018-08-17 14:17:28 +00:00
|
|
|
var dags adder.ClusterDAGService
|
|
|
|
output := make(chan *api.AddedOutput, 200)
|
2019-01-04 18:30:41 +00:00
|
|
|
|
2018-08-17 14:17:28 +00:00
|
|
|
if params.Shard {
|
|
|
|
dags = sharding.New(rpc, params.PinOptions, output)
|
|
|
|
} else {
|
|
|
|
dags = local.New(rpc, params.PinOptions)
|
|
|
|
}
|
|
|
|
|
2019-01-04 18:30:41 +00:00
|
|
|
if outputTransform == nil {
|
|
|
|
outputTransform = func(in *api.AddedOutput) interface{} { return in }
|
|
|
|
}
|
|
|
|
|
2018-08-18 00:45:47 +00:00
|
|
|
// This must be application/json otherwise go-ipfs client
|
|
|
|
// will break.
|
2018-10-03 21:03:30 +00:00
|
|
|
w.Header().Set("Content-Type", "application/json")
|
2019-01-04 18:30:41 +00:00
|
|
|
// Browsers should not cache these responses.
|
2018-10-03 21:03:30 +00:00
|
|
|
w.Header().Set("Cache-Control", "no-cache")
|
2018-09-29 00:07:57 +00:00
|
|
|
// We need to ask the clients to close the connection
|
|
|
|
// (no keep-alive) of things break badly when adding.
|
|
|
|
// https://github.com/ipfs/go-ipfs-cmds/pull/116
|
|
|
|
w.Header().Set("Connection", "close")
|
2018-08-17 14:17:28 +00:00
|
|
|
|
2019-01-04 18:30:41 +00:00
|
|
|
var wg sync.WaitGroup
|
|
|
|
if params.StreamChannels {
|
|
|
|
// handle stream-adding
|
|
|
|
// this should be the default.
|
|
|
|
|
|
|
|
// https://github.com/ipfs-shipyard/ipfs-companion/issues/600
|
|
|
|
w.Header().Set("X-Chunked-Output", "1")
|
|
|
|
// Used by go-ipfs to signal errors half-way through the stream.
|
|
|
|
w.Header().Set("Trailer", "X-Stream-Error")
|
|
|
|
w.WriteHeader(http.StatusOK)
|
|
|
|
wg.Add(1)
|
|
|
|
go func() {
|
|
|
|
defer wg.Done()
|
|
|
|
streamOutput(w, output, outputTransform)
|
|
|
|
}()
|
|
|
|
add := adder.New(dags, params, output)
|
|
|
|
root, err := add.FromMultipart(ctx, reader)
|
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
// Set trailer with error
|
|
|
|
w.Header().Set("X-Stream-Error", err.Error())
|
|
|
|
}
|
|
|
|
wg.Wait()
|
|
|
|
return root, err
|
2018-10-03 21:03:30 +00:00
|
|
|
}
|
|
|
|
|
2019-01-04 18:30:41 +00:00
|
|
|
// Add buffering the AddedOutput (StreamChannels=false)
|
2018-08-17 14:17:28 +00:00
|
|
|
wg.Add(1)
|
2019-01-04 18:30:41 +00:00
|
|
|
var bufOutput []interface{} // a slice of transformed AddedOutput
|
2018-08-17 14:17:28 +00:00
|
|
|
go func() {
|
|
|
|
defer wg.Done()
|
2019-01-04 18:30:41 +00:00
|
|
|
bufOutput = buildOutput(w, output, outputTransform)
|
2018-08-17 14:17:28 +00:00
|
|
|
}()
|
|
|
|
|
2019-01-04 18:30:41 +00:00
|
|
|
enc := json.NewEncoder(w)
|
2018-08-17 14:17:28 +00:00
|
|
|
add := adder.New(dags, params, output)
|
|
|
|
root, err := add.FromMultipart(ctx, reader)
|
2019-01-04 18:30:41 +00:00
|
|
|
if err != nil { // Send an error
|
|
|
|
logger.Error(err)
|
|
|
|
w.WriteHeader(http.StatusInternalServerError)
|
|
|
|
errorResp := api.Error{
|
|
|
|
Code: http.StatusInternalServerError,
|
|
|
|
Message: err.Error(),
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := enc.Encode(errorResp); err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
}
|
|
|
|
wg.Wait()
|
|
|
|
return root, err
|
2018-10-03 21:03:30 +00:00
|
|
|
}
|
2018-08-17 14:17:28 +00:00
|
|
|
wg.Wait()
|
2019-01-04 18:30:41 +00:00
|
|
|
w.WriteHeader(http.StatusOK)
|
|
|
|
enc.Encode(bufOutput)
|
2018-08-17 14:17:28 +00:00
|
|
|
return root, err
|
|
|
|
}
|
2019-01-04 18:30:41 +00:00
|
|
|
|
|
|
|
func streamOutput(w http.ResponseWriter, output chan *api.AddedOutput, transform func(*api.AddedOutput) interface{}) {
|
|
|
|
flusher, flush := w.(http.Flusher)
|
|
|
|
enc := json.NewEncoder(w)
|
|
|
|
for v := range output {
|
|
|
|
err := enc.Encode(transform(v))
|
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
break
|
|
|
|
}
|
|
|
|
if flush {
|
|
|
|
flusher.Flush()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func buildOutput(w http.ResponseWriter, output chan *api.AddedOutput, transform func(*api.AddedOutput) interface{}) []interface{} {
|
|
|
|
var finalOutput []interface{}
|
|
|
|
for v := range output {
|
|
|
|
finalOutput = append(finalOutput, transform(v))
|
|
|
|
}
|
|
|
|
return finalOutput
|
|
|
|
}
|