ipfs-cluster/api/ipfsproxy/ipfsproxy_test.go
Hector Sanjuan acbd7fda60 Consensus: add new "crdt" consensus component
This adds a new "crdt" consensus component using go-ds-crdt.

This implies several refactors to fully make cluster consensus-component
independent:

* Delete mapstate and fully adopt dsstate (after people have migrated).
* Return errors from state methods rather than ignoring them.
* Add a new "datastore" modules so that we can configure datastores in the
   main configuration like other components.
* Let the consensus components fully define the "state.State". Thus, they do
not receive the state, they receive the storage where we put the state (a
go-datastore).
* Allow to customize how the monitor component obtains Peers() (the current
  peerset), including avoiding using the current peerset. At the moment the
  crdt consensus uses the monitoring component to define the current peerset.
  Therefore the monitor component cannot rely on the consensus component to
  produce a peerset.
* Re-factor/re-implementation of "ipfs-cluster-service state"
  operations. Includes the dissapearance of the "migrate" one.

The CRDT consensus component defines creates a crdt-datastore (with ipfs-lite)
and uses it to intitialize a dssate. Thus the crdt-store is elegantly
wrapped. Any modifications to the state get automatically replicated to other
peers. We store all the CRDT DAG blocks in the local datastore.

The consensus components only expose a ReadOnly state, as any modifications to
the shared state should happen through them.

DHT and PubSub facilities must now be created outside of Cluster and passed in
so they can be re-used by different components.
2019-04-17 19:14:26 +02:00

663 lines
15 KiB
Go

package ipfsproxy
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
"testing"
"time"
cid "github.com/ipfs/go-cid"
"github.com/ipfs/ipfs-cluster/api"
"github.com/ipfs/ipfs-cluster/test"
logging "github.com/ipfs/go-log"
ma "github.com/multiformats/go-multiaddr"
)
func init() {
_ = logging.Logger
}
func testIPFSProxyWithConfig(t *testing.T, cfg *Config) (*Server, *test.IpfsMock) {
mock := test.NewIpfsMock(t)
nodeMAddr, _ := ma.NewMultiaddr(fmt.Sprintf("/ip4/%s/tcp/%d",
mock.Addr, mock.Port))
proxyMAddr, _ := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/0")
cfg.NodeAddr = nodeMAddr
cfg.ListenAddr = proxyMAddr
cfg.ExtractHeadersExtra = []string{
test.IpfsCustomHeaderName,
test.IpfsTimeHeaderName,
}
proxy, err := New(cfg)
if err != nil {
t.Fatal("creating an IPFSProxy should work: ", err)
}
proxy.server.SetKeepAlivesEnabled(false)
proxy.SetClient(test.NewMockRPCClient(t))
return proxy, mock
}
func testIPFSProxy(t *testing.T) (*Server, *test.IpfsMock) {
cfg := &Config{}
cfg.Default()
return testIPFSProxyWithConfig(t, cfg)
}
func TestIPFSProxyVersion(t *testing.T) {
ctx := context.Background()
proxy, mock := testIPFSProxy(t)
defer mock.Close()
defer proxy.Shutdown(ctx)
res, err := http.Post(fmt.Sprintf("%s/version", proxyURL(proxy)), "", nil)
if err != nil {
t.Fatal("should forward requests to ipfs host: ", err)
}
defer res.Body.Close()
resBytes, _ := ioutil.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
t.Error("the request should have succeeded")
t.Fatal(string(resBytes))
}
var resp struct {
Version string
}
err = json.Unmarshal(resBytes, &resp)
if err != nil {
t.Fatal(err)
}
if resp.Version != "m.o.c.k" {
t.Error("wrong version")
}
}
func TestIPFSProxyPin(t *testing.T) {
ctx := context.Background()
proxy, mock := testIPFSProxy(t)
defer mock.Close()
defer proxy.Shutdown(ctx)
type args struct {
urlPath string
testCid string
statusCode int
}
tests := []struct {
name string
args args
want cid.Cid
wantErr bool
}{
{
"pin good cid query arg",
args{
"/pin/add?arg=",
test.Cid1.String(),
http.StatusOK,
},
test.Cid1,
false,
},
{
"pin good path query arg",
args{
"/pin/add?arg=",
test.PathIPFS2,
http.StatusOK,
},
test.CidResolved,
false,
},
{
"pin good cid url arg",
args{
"/pin/add/",
test.Cid1.String(),
http.StatusOK,
},
test.Cid1,
false,
},
{
"pin bad cid query arg",
args{
"/pin/add?arg=",
test.ErrorCid.String(),
http.StatusInternalServerError,
},
cid.Undef,
true,
},
{
"pin bad cid url arg",
args{
"/pin/add/",
test.ErrorCid.String(),
http.StatusInternalServerError,
},
cid.Undef,
true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
u := fmt.Sprintf(
"%s%s%s",
proxyURL(proxy),
tt.args.urlPath,
tt.args.testCid,
)
res, err := http.Post(u, "", nil)
if err != nil {
t.Fatal("should have succeeded: ", err)
}
defer res.Body.Close()
if res.StatusCode != tt.args.statusCode {
t.Errorf("statusCode: got = %v, want %v", res.StatusCode, tt.args.statusCode)
}
resBytes, _ := ioutil.ReadAll(res.Body)
switch tt.wantErr {
case false:
var resp ipfsPinOpResp
err = json.Unmarshal(resBytes, &resp)
if err != nil {
t.Fatal(err)
}
if len(resp.Pins) != 1 {
t.Fatalf("wrong number of pins: got = %d, want %d", len(resp.Pins), 1)
}
if resp.Pins[0] != tt.want.String() {
t.Errorf("wrong pin cid: got = %s, want = %s", resp.Pins[0], tt.want)
}
case true:
var respErr ipfsError
err = json.Unmarshal(resBytes, &respErr)
if err != nil {
t.Fatal(err)
}
if respErr.Message != test.ErrBadCid.Error() {
t.Errorf("wrong response: got = %s, want = %s", respErr.Message, test.ErrBadCid.Error())
}
}
})
}
}
func TestIPFSProxyUnpin(t *testing.T) {
ctx := context.Background()
proxy, mock := testIPFSProxy(t)
defer mock.Close()
defer proxy.Shutdown(ctx)
type args struct {
urlPath string
testCid string
statusCode int
}
tests := []struct {
name string
args args
want cid.Cid
wantErr bool
}{
{
"unpin good cid query arg",
args{
"/pin/rm?arg=",
test.Cid1.String(),
http.StatusOK,
},
test.Cid1,
false,
},
{
"unpin good path query arg",
args{
"/pin/rm?arg=",
test.PathIPFS2,
http.StatusOK,
},
test.CidResolved,
false,
},
{
"unpin good cid url arg",
args{
"/pin/rm/",
test.Cid1.String(),
http.StatusOK,
},
test.Cid1,
false,
},
{
"unpin bad cid query arg",
args{
"/pin/rm?arg=",
test.ErrorCid.String(),
http.StatusInternalServerError,
},
cid.Undef,
true,
},
{
"unpin bad cid url arg",
args{
"/pin/rm/",
test.ErrorCid.String(),
http.StatusInternalServerError,
},
cid.Undef,
true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
u := fmt.Sprintf("%s%s%s", proxyURL(proxy), tt.args.urlPath, tt.args.testCid)
res, err := http.Post(u, "", nil)
if err != nil {
t.Fatal("should have succeeded: ", err)
}
defer res.Body.Close()
if res.StatusCode != tt.args.statusCode {
t.Errorf("statusCode: got = %v, want %v", res.StatusCode, tt.args.statusCode)
}
resBytes, _ := ioutil.ReadAll(res.Body)
switch tt.wantErr {
case false:
var resp ipfsPinOpResp
err = json.Unmarshal(resBytes, &resp)
if err != nil {
t.Fatal(err)
}
if len(resp.Pins) != 1 {
t.Fatalf("wrong number of pins: got = %d, want %d", len(resp.Pins), 1)
}
if resp.Pins[0] != tt.want.String() {
t.Errorf("wrong pin cid: got = %s, want = %s", resp.Pins[0], tt.want)
}
case true:
var respErr ipfsError
err = json.Unmarshal(resBytes, &respErr)
if err != nil {
t.Fatal(err)
}
if respErr.Message != test.ErrBadCid.Error() {
t.Errorf("wrong response: got = %s, want = %s", respErr.Message, test.ErrBadCid.Error())
}
}
})
}
}
func TestIPFSProxyPinLs(t *testing.T) {
ctx := context.Background()
proxy, mock := testIPFSProxy(t)
defer mock.Close()
defer proxy.Shutdown(ctx)
t.Run("pin/ls query arg", func(t *testing.T) {
res, err := http.Post(fmt.Sprintf("%s/pin/ls?arg=%s", proxyURL(proxy), test.Cid1), "", nil)
if err != nil {
t.Fatal("should have succeeded: ", err)
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
t.Error("the request should have succeeded")
}
resBytes, _ := ioutil.ReadAll(res.Body)
var resp ipfsPinLsResp
err = json.Unmarshal(resBytes, &resp)
if err != nil {
t.Fatal(err)
}
_, ok := resp.Keys[test.Cid1.String()]
if len(resp.Keys) != 1 || !ok {
t.Error("wrong response")
}
})
t.Run("pin/ls url arg", func(t *testing.T) {
res, err := http.Post(fmt.Sprintf("%s/pin/ls/%s", proxyURL(proxy), test.Cid1), "", nil)
if err != nil {
t.Fatal("should have succeeded: ", err)
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
t.Error("the request should have succeeded")
}
resBytes, _ := ioutil.ReadAll(res.Body)
var resp ipfsPinLsResp
err = json.Unmarshal(resBytes, &resp)
if err != nil {
t.Fatal(err)
}
_, ok := resp.Keys[test.Cid1.String()]
if len(resp.Keys) != 1 || !ok {
t.Error("wrong response")
}
})
t.Run("pin/ls all no arg", func(t *testing.T) {
res2, err := http.Post(fmt.Sprintf("%s/pin/ls", proxyURL(proxy)), "", nil)
if err != nil {
t.Fatal("should have succeeded: ", err)
}
defer res2.Body.Close()
if res2.StatusCode != http.StatusOK {
t.Error("the request should have succeeded")
}
resBytes, _ := ioutil.ReadAll(res2.Body)
var resp ipfsPinLsResp
err = json.Unmarshal(resBytes, &resp)
if err != nil {
t.Fatal(err)
}
if len(resp.Keys) != 3 {
t.Error("wrong response")
}
})
t.Run("pin/ls bad cid query arg", func(t *testing.T) {
res3, err := http.Post(fmt.Sprintf("%s/pin/ls?arg=%s", proxyURL(proxy), test.ErrorCid), "", nil)
if err != nil {
t.Fatal("should have succeeded: ", err)
}
defer res3.Body.Close()
if res3.StatusCode != http.StatusInternalServerError {
t.Error("the request should have failed")
}
})
}
func TestProxyRepoStat(t *testing.T) {
ctx := context.Background()
proxy, mock := testIPFSProxy(t)
defer mock.Close()
defer proxy.Shutdown(ctx)
res, err := http.Post(fmt.Sprintf("%s/repo/stat", proxyURL(proxy)), "", nil)
if err != nil {
t.Fatal(err)
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
t.Error("request should have succeeded")
}
resBytes, _ := ioutil.ReadAll(res.Body)
var stat api.IPFSRepoStat
err = json.Unmarshal(resBytes, &stat)
if err != nil {
t.Fatal(err)
}
// The mockRPC returns 3 peers. Since no host is set,
// all calls are local.
if stat.RepoSize != 6000 || stat.StorageMax != 300000 {
t.Errorf("expected different stats: %+v", stat)
}
}
func TestProxyAdd(t *testing.T) {
ctx := context.Background()
proxy, mock := testIPFSProxy(t)
defer mock.Close()
defer proxy.Shutdown(ctx)
type testcase struct {
query string
expectedCid string
}
testcases := []testcase{
testcase{
query: "",
expectedCid: test.ShardingDirBalancedRootCID,
},
testcase{
query: "progress=true",
expectedCid: test.ShardingDirBalancedRootCID,
},
testcase{
query: "wrap-with-directory=true",
expectedCid: test.ShardingDirBalancedRootCIDWrapped,
},
testcase{
query: "trickle=true",
expectedCid: test.ShardingDirTrickleRootCID,
},
}
reqs := make([]*http.Request, len(testcases), len(testcases))
sth := test.NewShardingTestHelper()
defer sth.Clean(t)
for i, tc := range testcases {
mr, closer := sth.GetTreeMultiReader(t)
defer closer.Close()
cType := "multipart/form-data; boundary=" + mr.Boundary()
url := fmt.Sprintf("%s/add?"+tc.query, proxyURL(proxy))
req, _ := http.NewRequest("POST", url, mr)
req.Header.Set("Content-Type", cType)
reqs[i] = req
}
for i, tc := range testcases {
t.Run(tc.query, func(t *testing.T) {
res, err := http.DefaultClient.Do(reqs[i])
if err != nil {
t.Fatal("should have succeeded: ", err)
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
t.Fatalf("Bad response status: got = %d, want = %d", res.StatusCode, http.StatusOK)
}
var resp ipfsAddResp
dec := json.NewDecoder(res.Body)
for dec.More() {
err := dec.Decode(&resp)
if err != nil {
t.Fatal(err)
}
}
if resp.Hash != tc.expectedCid {
t.Logf("%+v", resp.Hash)
t.Error("expected CID does not match")
}
})
}
}
func TestProxyAddError(t *testing.T) {
ctx := context.Background()
proxy, mock := testIPFSProxy(t)
defer mock.Close()
defer proxy.Shutdown(ctx)
res, err := http.Post(fmt.Sprintf("%s/add?recursive=true", proxyURL(proxy)), "", nil)
if err != nil {
t.Fatal(err)
}
res.Body.Close()
if res.StatusCode != http.StatusInternalServerError {
t.Errorf("wrong status code: got = %d, want = %d", res.StatusCode, http.StatusInternalServerError)
}
}
func TestProxyError(t *testing.T) {
ctx := context.Background()
proxy, mock := testIPFSProxy(t)
defer mock.Close()
defer proxy.Shutdown(ctx)
res, err := http.Post(fmt.Sprintf("%s/bad/command", proxyURL(proxy)), "", nil)
if err != nil {
t.Fatal("should have succeeded: ", err)
}
defer res.Body.Close()
if res.StatusCode != 404 {
t.Error("should have respected the status code")
}
}
func proxyURL(c *Server) string {
addr := c.listener.Addr()
return fmt.Sprintf("http://%s/api/v0", addr.String())
}
func TestIPFSProxy(t *testing.T) {
ctx := context.Background()
proxy, mock := testIPFSProxy(t)
defer mock.Close()
if err := proxy.Shutdown(ctx); err != nil {
t.Error("expected a clean shutdown")
}
if err := proxy.Shutdown(ctx); err != nil {
t.Error("expected a second clean shutdown")
}
}
func mustParseURL(rawurl string) *url.URL {
u, err := url.Parse(rawurl)
if err != nil {
panic(err)
}
return u
}
func TestHeaderExtraction(t *testing.T) {
ctx := context.Background()
proxy, mock := testIPFSProxy(t)
proxy.config.ExtractHeadersTTL = time.Second
defer mock.Close()
defer proxy.Shutdown(ctx)
req, err := http.NewRequest("POST", fmt.Sprintf("%s/pin/ls", proxyURL(proxy)), nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Origin", test.IpfsACAOrigin)
res, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal("should forward requests to ipfs host: ", err)
}
res.Body.Close()
for k, v := range res.Header {
t.Logf("%s: %s", k, v)
}
if h := res.Header.Get("Access-Control-Allow-Origin"); h != test.IpfsACAOrigin {
t.Error("We did not find out the AC-Allow-Origin header: ", h)
}
for _, h := range corsHeaders {
if v := res.Header.Get(h); v == "" {
t.Error("We did not set CORS header: ", h)
}
}
if res.Header.Get(test.IpfsCustomHeaderName) != test.IpfsCustomHeaderValue {
t.Error("the proxy should have extracted custom headers from ipfs")
}
if !strings.HasPrefix(res.Header.Get("Server"), "ipfs-cluster") {
t.Error("wrong value for Server header")
}
// Test ExtractHeaderTTL
t1 := res.Header.Get(test.IpfsTimeHeaderName)
res, err = http.DefaultClient.Do(req)
if err != nil {
t.Fatal("should forward requests to ipfs host: ", err)
}
t2 := res.Header.Get(test.IpfsTimeHeaderName)
if t1 != t2 {
t.Error("should have cached the headers during TTL")
}
time.Sleep(1200 * time.Millisecond)
res, err = http.DefaultClient.Do(req)
if err != nil {
t.Fatal("should forward requests to ipfs host: ", err)
}
res.Body.Close()
t3 := res.Header.Get(test.IpfsTimeHeaderName)
if t3 == t2 {
t.Error("should have refreshed the headers after TTL")
}
}
func TestAttackHeaderSize(t *testing.T) {
const testHeaderSize = minMaxHeaderBytes * 4
ctx := context.Background()
cfg := &Config{}
cfg.Default()
cfg.MaxHeaderBytes = testHeaderSize
proxy, mock := testIPFSProxyWithConfig(t, cfg)
defer mock.Close()
defer proxy.Shutdown(ctx)
type testcase struct {
headerSize int
expectedStatus int
}
testcases := []testcase{
testcase{testHeaderSize / 2, http.StatusNotFound},
testcase{testHeaderSize * 2, http.StatusRequestHeaderFieldsTooLarge},
}
req, err := http.NewRequest("POST", fmt.Sprintf("%s/foo", proxyURL(proxy)), nil)
if err != nil {
t.Fatal(err)
}
for _, tc := range testcases {
for size := 0; size < tc.headerSize; size += 8 {
req.Header.Add("Foo", "bar")
}
res, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal("should forward requests to ipfs host: ", err)
}
res.Body.Close()
if res.StatusCode != tc.expectedStatus {
t.Errorf("proxy returned unexpected status %d, expected status code was %d",
res.StatusCode, tc.expectedStatus)
}
}
}