cascade/ipfs-cluster
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into feat/add-authless-healthcheck

Browse Source
This commit is contained in:
Arthur Gavazza 2023-04-20 08:54:13 -03:00 committed by GitHub
commit 093d234da9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 108 additions and 593 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
add_test.go
View File

@ -9,10 +9,10 @@ import (
"testing"
"time"
files "github.com/ipfs/go-libipfs/files"
"github.com/ipfs-cluster/ipfs-cluster/adder"
"github.com/ipfs-cluster/ipfs-cluster/api"
"github.com/ipfs-cluster/ipfs-cluster/test"
files "github.com/ipfs/boxo/files"
peer "github.com/libp2p/go-libp2p/core/peer"
)

8
adder/adder.go
View File

@ -12,16 +12,16 @@ import (
"github.com/ipfs-cluster/ipfs-cluster/adder/ipfsadd"
"github.com/ipfs-cluster/ipfs-cluster/api"
"github.com/ipfs/go-unixfs"
"github.com/ipld/go-car"
"github.com/ipfs/boxo/ipld/car"
"github.com/ipfs/boxo/ipld/unixfs"
peer "github.com/libp2p/go-libp2p/core/peer"
files "github.com/ipfs/boxo/files"
merkledag "github.com/ipfs/boxo/ipld/merkledag"
cid "github.com/ipfs/go-cid"
files "github.com/ipfs/go-libipfs/files"
cbor "github.com/ipfs/go-ipld-cbor"
ipld "github.com/ipfs/go-ipld-format"
logging "github.com/ipfs/go-log/v2"
merkledag "github.com/ipfs/go-merkledag"
multihash "github.com/multiformats/go-multihash"
)

4
adder/adder_test.go
View File

@ -11,11 +11,11 @@ import (
"github.com/ipfs-cluster/ipfs-cluster/api"
"github.com/ipfs-cluster/ipfs-cluster/test"
"github.com/ipld/go-car"
"github.com/ipfs/boxo/ipld/car"
peer "github.com/libp2p/go-libp2p/core/peer"
files "github.com/ipfs/boxo/files"
cid "github.com/ipfs/go-cid"
files "github.com/ipfs/go-libipfs/files"
)
type mockCDAGServ struct {

18
adder/ipfsadd/add.go
View File

@ -11,18 +11,18 @@ import (
"github.com/ipfs-cluster/ipfs-cluster/api"
chunker "github.com/ipfs/boxo/chunker"
files "github.com/ipfs/boxo/files"
posinfo "github.com/ipfs/boxo/filestore/posinfo"
dag "github.com/ipfs/boxo/ipld/merkledag"
unixfs "github.com/ipfs/boxo/ipld/unixfs"
balanced "github.com/ipfs/boxo/ipld/unixfs/importer/balanced"
ihelper "github.com/ipfs/boxo/ipld/unixfs/importer/helpers"
trickle "github.com/ipfs/boxo/ipld/unixfs/importer/trickle"
mfs "github.com/ipfs/boxo/mfs"
cid "github.com/ipfs/go-cid"
chunker "github.com/ipfs/go-ipfs-chunker"
files "github.com/ipfs/go-libipfs/files"
posinfo "github.com/ipfs/go-ipfs-posinfo"
ipld "github.com/ipfs/go-ipld-format"
logging "github.com/ipfs/go-log/v2"
dag "github.com/ipfs/go-merkledag"
mfs "github.com/ipfs/go-mfs"
unixfs "github.com/ipfs/go-unixfs"
balanced "github.com/ipfs/go-unixfs/importer/balanced"
ihelper "github.com/ipfs/go-unixfs/importer/helpers"
trickle "github.com/ipfs/go-unixfs/importer/trickle"
peer "github.com/libp2p/go-libp2p/core/peer"
)

4
adder/sharding/dag.go
View File

@ -20,11 +20,11 @@ import (
"context"
"fmt"
dag "github.com/ipfs/boxo/ipld/merkledag"
blocks "github.com/ipfs/go-block-format"
cid "github.com/ipfs/go-cid"
cbor "github.com/ipfs/go-ipld-cbor"
ipld "github.com/ipfs/go-ipld-format"
blocks "github.com/ipfs/go-libipfs/blocks"
dag "github.com/ipfs/go-merkledag"
mh "github.com/multiformats/go-multihash"
)

2
api/common/api.go
View File

@ -30,8 +30,8 @@ import (
jwt "github.com/golang-jwt/jwt/v4"
types "github.com/ipfs-cluster/ipfs-cluster/api"
state "github.com/ipfs-cluster/ipfs-cluster/state"
gopath "github.com/ipfs/boxo/path"
logging "github.com/ipfs/go-log/v2"
gopath "github.com/ipfs/go-path"
libp2p "github.com/libp2p/go-libp2p"
rpc "github.com/libp2p/go-libp2p-gorpc"
gostream "github.com/libp2p/go-libp2p-gostream"

2
api/ipfsproxy/ipfsproxy.go
View File

@ -27,10 +27,10 @@ import (
handlers "github.com/gorilla/handlers"
mux "github.com/gorilla/mux"
path "github.com/ipfs/boxo/path"
cid "github.com/ipfs/go-cid"
cmd "github.com/ipfs/go-ipfs-cmds"
logging "github.com/ipfs/go-log/v2"
path "github.com/ipfs/go-path"
rpc "github.com/libp2p/go-libp2p-gorpc"
peer "github.com/libp2p/go-libp2p/core/peer"
madns "github.com/multiformats/go-multiaddr-dns"

2
api/rest/client/client.go
View File

@ -12,8 +12,8 @@ import (
"github.com/ipfs-cluster/ipfs-cluster/api"
files "github.com/ipfs/boxo/files"
shell "github.com/ipfs/go-ipfs-api"
files "github.com/ipfs/go-libipfs/files"
logging "github.com/ipfs/go-log/v2"
host "github.com/libp2p/go-libp2p/core/host"
peer "github.com/libp2p/go-libp2p/core/peer"

4
api/rest/client/lbclient.go
View File

@ -5,8 +5,10 @@ import (
"sync/atomic"
"github.com/ipfs-cluster/ipfs-cluster/api"
files "github.com/ipfs/boxo/files"
shell "github.com/ipfs/go-ipfs-api"
files "github.com/ipfs/go-libipfs/files"
peer "github.com/libp2p/go-libp2p/core/peer"
)

4
api/rest/client/methods.go
View File

@ -15,8 +15,8 @@ import (
"github.com/ipfs-cluster/ipfs-cluster/api"
files "github.com/ipfs/go-libipfs/files"
gopath "github.com/ipfs/go-path"
files "github.com/ipfs/boxo/files"
gopath "github.com/ipfs/boxo/path"
peer "github.com/libp2p/go-libp2p/core/peer"
"go.opencensus.io/trace"

2
cluster_test.go
View File

@ -21,7 +21,7 @@ import (
"github.com/ipfs-cluster/ipfs-cluster/test"
"github.com/ipfs-cluster/ipfs-cluster/version"
gopath "github.com/ipfs/go-path"
gopath "github.com/ipfs/boxo/path"
rpc "github.com/libp2p/go-libp2p-gorpc"
peer "github.com/libp2p/go-libp2p/core/peer"
)

2
clusterhost.go
View File

@ -5,9 +5,9 @@ import (
"encoding/hex"
config "github.com/ipfs-cluster/ipfs-cluster/config"
ipns "github.com/ipfs/boxo/ipns"
ds "github.com/ipfs/go-datastore"
namespace "github.com/ipfs/go-datastore/namespace"
ipns "github.com/ipfs/go-ipns"
libp2p "github.com/libp2p/go-libp2p"
dht "github.com/libp2p/go-libp2p-kad-dht"
dual "github.com/libp2p/go-libp2p-kad-dht/dual"

2
cmd/ipfs-cluster-ctl/main.go
View File

@ -27,7 +27,7 @@ const programName = `ipfs-cluster-ctl`
// Version is the cluster-ctl tool version. It should match
// the IPFS cluster's version
const Version = "1.0.5"
const Version = "1.0.6"
var (
defaultHost = "/ip4/127.0.0.1/tcp/9094"

11
cmd/ipfs-cluster-service/daemon.go
View File

@ -31,6 +31,7 @@ import (
ma "github.com/multiformats/go-multiaddr"
sddaemon "github.com/coreos/go-systemd/v22/daemon"
errors "github.com/pkg/errors"
cli "github.com/urfave/cli"
)
@ -105,6 +106,16 @@ func daemon(c *cli.Context) error {
// will realize).
go bootstrap(ctx, cluster, bootstraps)
// send readiness notification to systemd
go func() {
select {
case <-ctx.Done():
return
case <-cluster.Ready():
sddaemon.SdNotify(false, sddaemon.SdNotifyReady)
}
}()
return cmdutils.HandleSignals(ctx, cancel, cluster, host, dht, store)
}

2
consensus/crdt/consensus.go
View File

@ -15,11 +15,11 @@ import (
"github.com/ipfs-cluster/ipfs-cluster/state"
"github.com/ipfs-cluster/ipfs-cluster/state/dsstate"
dshelp "github.com/ipfs/boxo/datastore/dshelp"
ds "github.com/ipfs/go-datastore"
namespace "github.com/ipfs/go-datastore/namespace"
query "github.com/ipfs/go-datastore/query"
crdt "github.com/ipfs/go-ds-crdt"
dshelp "github.com/ipfs/go-ipfs-ds-help"
logging "github.com/ipfs/go-log/v2"
rpc "github.com/libp2p/go-libp2p-gorpc"
pubsub "github.com/libp2p/go-libp2p-pubsub"

6
consensus/crdt/consensus_test.go
View File

@ -10,14 +10,14 @@ import (
"github.com/ipfs-cluster/ipfs-cluster/datastore/inmem"
"github.com/ipfs-cluster/ipfs-cluster/test"
ipns "github.com/ipfs/go-ipns"
ipns "github.com/ipfs/boxo/ipns"
libp2p "github.com/libp2p/go-libp2p"
host "github.com/libp2p/go-libp2p/core/host"
peerstore "github.com/libp2p/go-libp2p/core/peerstore"
dht "github.com/libp2p/go-libp2p-kad-dht"
dual "github.com/libp2p/go-libp2p-kad-dht/dual"
pubsub "github.com/libp2p/go-libp2p-pubsub"
record "github.com/libp2p/go-libp2p-record"
host "github.com/libp2p/go-libp2p/core/host"
peerstore "github.com/libp2p/go-libp2p/core/peerstore"
routedhost "github.com/libp2p/go-libp2p/p2p/host/routed"
)

3
datastore/pebble/config.go
View File

@ -107,6 +107,7 @@ type pebbleOptions struct {
ReadOnly bool `json:"read_only"`
WALBytesPerSync int `json:"wal_bytes_per_sync"`
Levels []levelOptions `json:"levels"`
MaxConcurrentCompactions int `json:"max_concurrent_compactions"`
}
func (po *pebbleOptions) Unmarshal() *pebble.Options {
@ -133,6 +134,7 @@ func (po *pebbleOptions) Unmarshal() *pebble.Options {
pebbleOpts.MemTableStopWritesThreshold = po.MemTableStopWritesThreshold
pebbleOpts.ReadOnly = po.ReadOnly
pebbleOpts.WALBytesPerSync = po.WALBytesPerSync
pebbleOpts.MaxConcurrentCompactions = func() int { return po.MaxConcurrentCompactions }
return pebbleOpts
}
@ -158,6 +160,7 @@ func (po *pebbleOptions) Marshal(pebbleOpts *pebble.Options) {
po.MemTableStopWritesThreshold = pebbleOpts.MemTableStopWritesThreshold
po.ReadOnly = pebbleOpts.ReadOnly
po.WALBytesPerSync = pebbleOpts.WALBytesPerSync
po.MaxConcurrentCompactions = pebbleOpts.MaxConcurrentCompactions()
}
// levelOptions carries options for pebble's per-level parameters.

7
datastore/pebble/config_test.go
View File

@ -35,7 +35,8 @@ var cfgJSON = []byte(`
"mem_table_size": 4194304,
"mem_table_stop_writes_threshold": 2,
"read_only": false,
"wal_bytes_per_sync": 0
"wal_bytes_per_sync": 0,
"max_concurrent_compactions": 2
}
}
`)
@ -60,6 +61,10 @@ func TestToJSON(t *testing.T) {
t.Fatal("Disable WAL should be true")
}
if cfg.PebbleOptions.MaxConcurrentCompactions() != 2 {
t.Fatalf("Wrong max concuncurrent compactions value, got: %d, want: %d", cfg.PebbleOptions.MaxConcurrentCompactions(), 2)
}
newjson, err := cfg.ToJSON()
if err != nil {
t.Fatal(err)

49
go.mod
View File

@ -5,6 +5,7 @@ require (
contrib.go.opencensus.io/exporter/prometheus v0.4.2
github.com/blang/semver v3.5.1+incompatible
github.com/cockroachdb/pebble v0.0.0-20230227185959-8285e8dd5c08
github.com/coreos/go-systemd/v22 v22.5.0
github.com/dgraph-io/badger v1.6.2
github.com/dgraph-io/badger/v3 v3.2103.5
github.com/dustin/go-humanize v1.0.1
@ -14,41 +15,32 @@ require (
github.com/gorilla/mux v1.8.0
github.com/hashicorp/raft v1.3.11
github.com/hashicorp/raft-boltdb v0.0.0-20190605210249-ef2e128ed477
github.com/hsanjuan/ipfs-lite v1.6.0
github.com/hsanjuan/ipfs-lite v1.7.0
github.com/imdario/mergo v0.3.13
github.com/ipfs/go-cid v0.3.2
github.com/ipfs/boxo v0.8.0
github.com/ipfs/go-block-format v0.1.2
github.com/ipfs/go-cid v0.4.0
github.com/ipfs/go-datastore v0.6.0
github.com/ipfs/go-ds-badger v0.3.0
github.com/ipfs/go-ds-badger3 v0.0.2
github.com/ipfs/go-ds-crdt v0.3.10
github.com/ipfs/go-ds-crdt v0.4.0
github.com/ipfs/go-ds-leveldb v0.5.0
github.com/ipfs/go-ds-pebble v0.2.3
github.com/ipfs/go-fs-lock v0.0.7
github.com/ipfs/go-ipfs-api v0.3.0
github.com/ipfs/go-ipfs-chunker v0.0.5
github.com/ipfs/go-ipfs-cmds v0.6.0
github.com/ipfs/go-ipfs-ds-help v1.1.0
github.com/ipfs/go-ipfs-pinner v0.3.0
github.com/ipfs/go-ipfs-posinfo v0.0.1
github.com/ipfs/go-ipfs-api v0.6.0
github.com/ipfs/go-ipfs-cmds v0.9.0
github.com/ipfs/go-ipld-cbor v0.0.6
github.com/ipfs/go-ipld-format v0.4.0
github.com/ipfs/go-ipns v0.3.0
github.com/ipfs/go-libipfs v0.6.0
github.com/ipfs/go-log/v2 v2.5.1
github.com/ipfs/go-merkledag v0.10.0
github.com/ipfs/go-mfs v0.2.1
github.com/ipfs/go-path v0.3.1
github.com/ipfs/go-unixfs v0.4.4
github.com/ipld/go-car v0.6.0
github.com/kelseyhightower/envconfig v1.4.0
github.com/kishansagathiya/go-dot v0.1.0
github.com/lanzafame/go-libp2p-ocgorpc v0.1.1
github.com/libp2p/go-libp2p v0.26.0
github.com/libp2p/go-libp2p v0.26.3
github.com/libp2p/go-libp2p-consensus v0.0.1
github.com/libp2p/go-libp2p-gorpc v0.5.0
github.com/libp2p/go-libp2p-gostream v0.6.0
github.com/libp2p/go-libp2p-http v0.5.0
github.com/libp2p/go-libp2p-kad-dht v0.21.1
github.com/libp2p/go-libp2p-kad-dht v0.22.0
github.com/libp2p/go-libp2p-pubsub v0.9.2
github.com/libp2p/go-libp2p-raft v0.4.0
github.com/libp2p/go-libp2p-record v0.2.0
@ -59,7 +51,7 @@ require (
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.14.0
github.com/rs/cors v1.8.3
github.com/syndtr/goleveldb v1.0.0
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926
github.com/ugorji/go/codec v1.2.10
github.com/urfave/cli v1.22.12
@ -86,7 +78,6 @@ require (
github.com/cockroachdb/redact v1.0.8 // indirect
github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect
github.com/containerd/cgroups v1.0.4 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
github.com/cskr/pubsub v1.0.2 // indirect
@ -124,23 +115,14 @@ require (
github.com/huin/goupnp v1.0.3 // indirect
github.com/ipfs/bbloom v0.0.4 // indirect
github.com/ipfs/go-bitfield v1.1.0 // indirect
github.com/ipfs/go-block-format v0.1.1 // indirect
github.com/ipfs/go-blockservice v0.5.0 // indirect
github.com/ipfs/go-cidutil v0.1.0 // indirect
github.com/ipfs/go-fetcher v1.6.1 // indirect
github.com/ipfs/go-ipfs-blockstore v1.2.0 // indirect
github.com/ipfs/go-ipfs-delay v0.0.1 // indirect
github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect
github.com/ipfs/go-ipfs-exchange-offline v0.3.0 // indirect
github.com/ipfs/go-ipfs-files v0.3.0 // indirect
github.com/ipfs/go-ipfs-pq v0.0.3 // indirect
github.com/ipfs/go-ipfs-provider v0.8.1 // indirect
github.com/ipfs/go-ipfs-util v0.0.2 // indirect
github.com/ipfs/go-ipld-legacy v0.1.1 // indirect
github.com/ipfs/go-log v1.0.5 // indirect
github.com/ipfs/go-metrics-interface v0.0.1 // indirect
github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
github.com/ipfs/go-verifcid v0.0.2 // indirect
github.com/ipld/go-codec-dagpb v1.6.0 // indirect
github.com/ipld/go-ipld-prime v0.20.0 // indirect
github.com/jackpal/go-nat-pmp v1.0.2 // indirect
@ -155,7 +137,6 @@ require (
github.com/libp2p/go-cidranger v1.1.0 // indirect
github.com/libp2p/go-flow-metrics v0.1.0 // indirect
github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect
github.com/libp2p/go-libp2p-core v0.20.0 // indirect
github.com/libp2p/go-libp2p-kbucket v0.5.0 // indirect
github.com/libp2p/go-libp2p-routing-helpers v0.4.0 // indirect
github.com/libp2p/go-msgio v0.3.0 // indirect
@ -193,7 +174,7 @@ require (
github.com/quic-go/qtls-go1-19 v0.2.1 // indirect
github.com/quic-go/qtls-go1-20 v0.1.1 // indirect
github.com/quic-go/quic-go v0.33.0 // indirect
github.com/quic-go/webtransport-go v0.5.1 // indirect
github.com/quic-go/webtransport-go v0.5.2 // indirect
github.com/raulk/go-watchdog v1.3.0 // indirect
github.com/rogpeppe/go-internal v1.9.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
@ -204,14 +185,14 @@ require (
github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
go.opentelemetry.io/otel v1.12.0 // indirect
go.opentelemetry.io/otel/trace v1.12.0 // indirect
go.opentelemetry.io/otel v1.14.0 // indirect
go.opentelemetry.io/otel/trace v1.14.0 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/dig v1.15.0 // indirect
go.uber.org/fx v1.18.2 // indirect
go.uber.org/zap v1.24.0 // indirect
go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
golang.org/x/exp v0.0.0-20230129154200-a960b3787bd2 // indirect
golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb // indirect
golang.org/x/mod v0.8.0 // indirect
golang.org/x/net v0.8.0 // indirect
golang.org/x/sync v0.1.0 // indirect

551
go.sum
View File

File diff suppressed because it is too large Load Diff

6
ipfsconn/ipfshttp/ipfshttp.go
View File

@ -19,11 +19,11 @@ import (
"github.com/ipfs-cluster/ipfs-cluster/api"
"github.com/ipfs-cluster/ipfs-cluster/observations"
files "github.com/ipfs/boxo/files"
gopath "github.com/ipfs/boxo/path"
ipfspinner "github.com/ipfs/boxo/pinning/pinner"
cid "github.com/ipfs/go-cid"
files "github.com/ipfs/go-libipfs/files"
ipfspinner "github.com/ipfs/go-ipfs-pinner"
logging "github.com/ipfs/go-log/v2"
gopath "github.com/ipfs/go-path"
rpc "github.com/libp2p/go-libp2p-gorpc"
peer "github.com/libp2p/go-libp2p/core/peer"
madns "github.com/multiformats/go-multiaddr-dns"

2
ipfsconn/ipfshttp/ipfshttp_test.go
View File

@ -10,9 +10,9 @@ import (
logging "github.com/ipfs/go-log/v2"
ma "github.com/multiformats/go-multiaddr"
merkledag "github.com/ipfs/go-merkledag"
"github.com/ipfs-cluster/ipfs-cluster/api"
"github.com/ipfs-cluster/ipfs-cluster/test"
merkledag "github.com/ipfs/boxo/ipld/merkledag"
)
func init() {

2
state/dsstate/datastore.go
View File

@ -12,9 +12,9 @@ import (
"github.com/ipfs-cluster/ipfs-cluster/observations"
"github.com/ipfs-cluster/ipfs-cluster/state"
dshelp "github.com/ipfs/boxo/datastore/dshelp"
ds "github.com/ipfs/go-datastore"
query "github.com/ipfs/go-datastore/query"
dshelp "github.com/ipfs/go-ipfs-ds-help"
logging "github.com/ipfs/go-log/v2"
codec "github.com/ugorji/go/codec"

2
test/rpc_api_mock.go
View File

@ -10,7 +10,7 @@ import (
"github.com/ipfs-cluster/ipfs-cluster/api"
"github.com/ipfs-cluster/ipfs-cluster/state"
gopath "github.com/ipfs/go-path"
gopath "github.com/ipfs/boxo/path"
rpc "github.com/libp2p/go-libp2p-gorpc"
host "github.com/libp2p/go-libp2p/core/host"
peer "github.com/libp2p/go-libp2p/core/peer"

2
test/sharding.go
View File

@ -12,7 +12,7 @@ import (
"testing"
"github.com/ipfs-cluster/ipfs-cluster/api"
files "github.com/ipfs/go-libipfs/files"
files "github.com/ipfs/boxo/files"
format "github.com/ipfs/go-ipld-format"
cid "github.com/ipfs/go-cid"

2
version/version.go
View File

@ -7,7 +7,7 @@ import (
)
// Version is the current cluster version.
var Version = semver.MustParse("1.0.5")
var Version = semver.MustParse("1.0.6")
// RPCProtocol is protocol handler used to send libp2p-rpc messages between
// cluster peers. All peers in the cluster need to speak the same protocol