From 154d4dab7467a5c3034574c88bd37a05bbf0f5e1 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 24 Jan 2017 21:40:43 +0100 Subject: [PATCH 1/6] Issue #20: Download a local installation of gx This way we do not mess with the users stuff License: MIT Signed-off-by: Hector Sanjuan --- .gitignore | 1 + Makefile | 42 +++++++++++++++++++++++++++++++++--------- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 37f034c6..a3e8e98c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ coverage.out ipfs-cluster-service/ipfs-cluster-service ipfs-cluster-ctl/ipfs-cluster-ctl +deptools # Compiled Object files, Static and Dynamic libs (Shared Objects) *.o diff --git a/Makefile b/Makefile index 224b91d3..0fe70eac 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,16 @@ -all: server client +gx_version=v0.10.0 +gx-go_version=v1.4.0 +gx=gx_$(gx_version) +gx-go=gx-go_$(gx-go_version) +gx_bin=deptools/$(gx) +gx-go_bin=deptools/$(gx-go) +bin_env=$(shell go env GOHOSTOS)-$(shell go env GOHOSTARCH) + +all: service ctl clean: rwundo $(MAKE) -C ipfs-cluster-service clean $(MAKE) -C ipfs-cluster-ctl clean + rm -rf deptools install: deps $(MAKE) -C ipfs-cluster-service install $(MAKE) -C ipfs-cluster-ctl install @@ -16,22 +25,37 @@ service: deps ctl: deps $(MAKE) -C ipfs-cluster-ctl ipfs-cluster-ctl -gx: - go get github.com/whyrusleeping/gx - go get github.com/whyrusleeping/gx-go +$(gx_bin): + @echo "Downloading gx" + @mkdir -p ./deptools + @wget -nc -q -O $(gx_bin).tgz https://dist.ipfs.io/gx/$(gx_version)/$(gx)_$(bin_env).tar.gz + @tar -zxf $(gx_bin).tgz -C deptools --strip-components=1 gx/gx + @mv deptools/gx $(gx_bin) + @rm $(gx_bin).tgz + +$(gx-go_bin): + echo "Downloading gx-go" + mkdir -p ./deptools + wget -nc -q -O $(gx-go_bin).tgz https://dist.ipfs.io/gx-go/$(gx-go_version)/$(gx-go)_$(bin_env).tar.gz + tar -zxf $(gx-go_bin).tgz -C deptools --strip-components=1 gx-go/gx-go + mv deptools/gx-go $(gx-go_bin) + rm $(gx-go_bin).tgz + +gx: $(gx_bin) $(gx-go_bin) + deps: gx go get github.com/gorilla/mux go get github.com/hashicorp/raft go get github.com/hashicorp/raft-boltdb go get github.com/ugorji/go/codec - gx --verbose install --global - gx-go rewrite + $(gx_bin) --verbose install --global + $(gx-go_bin) rewrite test: deps go test -tags silent -v -covermode count -coverprofile=coverage.out . rw: - gx-go rewrite + $(gx-go_bin) rewrite rwundo: - gx-go rewrite --undo + $(gx-go_bin) rewrite --undo publish: rwundo - gx publish + $(gx_bin) publish .PHONY: all gx deps test rw rwundo publish service ctl install clean From 8719d88e130f555c82eeed29bb76d801d9c38aa4 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 24 Jan 2017 23:09:36 +0100 Subject: [PATCH 2/6] gx all dependencies License: MIT Signed-off-by: Hector Sanjuan --- Makefile | 4 ---- package.json | 16 ++++++++++++++-- raft.go | 10 ++++------ 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 0fe70eac..99f9e58e 100644 --- a/Makefile +++ b/Makefile @@ -44,10 +44,6 @@ $(gx-go_bin): gx: $(gx_bin) $(gx-go_bin) deps: gx - go get github.com/gorilla/mux - go get github.com/hashicorp/raft - go get github.com/hashicorp/raft-boltdb - go get github.com/ugorji/go/codec $(gx_bin) --verbose install --global $(gx-go_bin) rewrite test: deps diff --git a/package.json b/package.json index 7334b3f0..fc163736 100644 --- a/package.json +++ b/package.json @@ -21,9 +21,9 @@ }, { "author": "hsanjuan", - "hash": "QmbzVbfqoZ3ovvjCuZ7D7MAnmzNyTsU7cmwyfnZRYBPn69", + "hash": "QmbdNJqhQ1jmNmLmig2f99a49XwKJHp6XJUTht4VdrYT2M", "name": "go-libp2p-raft", - "version": "1.0.0" + "version": "1.0.1" }, { "author": "whyrusleeping", @@ -42,6 +42,18 @@ "hash": "Qmc1AtgBdoUHP8oYSqU81NRYdzohmF45t5XNwVMvhCxsBA", "name": "cli", "version": "1.19.1" + }, + { + "author": "hashicorp", + "hash": "QmUDCcPkPMPJ149YBpfFLWJtRFeqace5GNdBPD2cW4Z8E6", + "name": "raft-boltdb", + "version": "2017.1.24" + }, + { + "author": "gorilla", + "hash": "QmVrBjGjskzhW7HRfC56f4mT3nmNBHj2WWnEXr39G7Y6w8", + "name": "mux", + "version": "1.3.0" } ], "gxVersion": "0.10.0", diff --git a/raft.go b/raft.go index dfcf96a5..0accef59 100644 --- a/raft.go +++ b/raft.go @@ -4,13 +4,11 @@ import ( "io/ioutil" "path/filepath" - host "github.com/libp2p/go-libp2p-host" - libp2praft "github.com/libp2p/go-libp2p-raft" - - peer "github.com/libp2p/go-libp2p-peer" - hashiraft "github.com/hashicorp/raft" - raftboltdb "github.com/hashicorp/raft-boltdb" + host "github.com/libp2p/go-libp2p-host" + peer "github.com/libp2p/go-libp2p-peer" + libp2praft "github.com/libp2p/go-libp2p-raft" + raftboltdb "gx/ipfs/QmWrdreJ88vAtixq3k7PA9fX6hCLVzWwQu9oYr48R81NEa/raft-boltdb" ) // libp2pRaftWrap wraps the stuff that we need to run From 85092ede9a60848b2588240a36a8d248f9f57494 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 25 Jan 2017 00:20:51 +0100 Subject: [PATCH 3/6] Update libs. Make sure libp2p versions match License: MIT Signed-off-by: Hector Sanjuan --- package.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index fc163736..49cfbc86 100644 --- a/package.json +++ b/package.json @@ -15,15 +15,15 @@ }, { "author": "whyrusleeping", - "hash": "QmbzCT1CwxVZ2ednptC9RavuJe7Bv8DDi2Ne89qUrA37XM", + "hash": "QmdzDdLZ7nj133QvNHypyS9Y39g35bMFk5DJ2pmX7YqtKU", "name": "go-libp2p", - "version": "4.3.0" + "version": "4.3.5" }, { "author": "hsanjuan", - "hash": "QmbdNJqhQ1jmNmLmig2f99a49XwKJHp6XJUTht4VdrYT2M", + "hash": "QmURcg2K5geS5fgV5EUKYdyqhsRTwCSzYsFjJpGq91vLCn", "name": "go-libp2p-raft", - "version": "1.0.1" + "version": "1.0.2" }, { "author": "whyrusleeping", From cd307b08ad9719f8a64af60ab4ed187ae680f5a6 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 25 Jan 2017 01:41:52 +0100 Subject: [PATCH 4/6] Try to fix gx-go not rewriting correctly License: MIT Signed-off-by: Hector Sanjuan --- Makefile | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 99f9e58e..977beebb 100644 --- a/Makefile +++ b/Makefile @@ -1,16 +1,21 @@ gx_version=v0.10.0 gx-go_version=v1.4.0 + +deptools=deptools + gx=gx_$(gx_version) gx-go=gx-go_$(gx-go_version) -gx_bin=deptools/$(gx) -gx-go_bin=deptools/$(gx-go) +gx_bin=$(deptools)/$(gx) +gx-go_bin=$(deptools)/$(gx-go) bin_env=$(shell go env GOHOSTOS)-$(shell go env GOHOSTARCH) +export PATH := $(deptools):$(PATH) + all: service ctl -clean: rwundo +clean: $(MAKE) -C ipfs-cluster-service clean $(MAKE) -C ipfs-cluster-ctl clean - rm -rf deptools + rm -rf $(deptools) install: deps $(MAKE) -C ipfs-cluster-service install $(MAKE) -C ipfs-cluster-ctl install @@ -27,19 +32,23 @@ ctl: deps $(gx_bin): @echo "Downloading gx" - @mkdir -p ./deptools + @mkdir -p ./$(deptools) + @rm -f $(deptools)/gx @wget -nc -q -O $(gx_bin).tgz https://dist.ipfs.io/gx/$(gx_version)/$(gx)_$(bin_env).tar.gz - @tar -zxf $(gx_bin).tgz -C deptools --strip-components=1 gx/gx - @mv deptools/gx $(gx_bin) + @tar -zxf $(gx_bin).tgz -C $(deptools) --strip-components=1 gx/gx + @mv $(deptools)/gx $(gx_bin) + @ln -s $(gx) $(deptools)/gx @rm $(gx_bin).tgz $(gx-go_bin): - echo "Downloading gx-go" - mkdir -p ./deptools - wget -nc -q -O $(gx-go_bin).tgz https://dist.ipfs.io/gx-go/$(gx-go_version)/$(gx-go)_$(bin_env).tar.gz - tar -zxf $(gx-go_bin).tgz -C deptools --strip-components=1 gx-go/gx-go - mv deptools/gx-go $(gx-go_bin) - rm $(gx-go_bin).tgz + @echo "Downloading gx-go" + @mkdir -p ./$(deptools) + @rm -f $(deptools)gx-go + @wget -nc -q -O $(gx-go_bin).tgz https://dist.ipfs.io/gx-go/$(gx-go_version)/$(gx-go)_$(bin_env).tar.gz + @tar -zxf $(gx-go_bin).tgz -C $(deptools) --strip-components=1 gx-go/gx-go + @mv $(deptools)/gx-go $(gx-go_bin) + @ln -s $(gx-go) $(deptools)/gx-go + @rm $(gx-go_bin).tgz gx: $(gx_bin) $(gx-go_bin) @@ -48,9 +57,9 @@ deps: gx $(gx-go_bin) rewrite test: deps go test -tags silent -v -covermode count -coverprofile=coverage.out . -rw: +rw: gx $(gx-go_bin) rewrite -rwundo: +rwundo: gx $(gx-go_bin) rewrite --undo publish: rwundo $(gx_bin) publish From 9a47e6dd1f5e0ad37c9a1a69a7a0f8c245947047 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 25 Jan 2017 12:14:39 +0100 Subject: [PATCH 5/6] Update go-libp2p-gorpc Uses experimental version of multicodecs but should finally pin all deps License: MIT Signed-off-by: Hector Sanjuan --- architecture.md | 2 +- cluster.go | 2 +- cluster_test.go | 2 +- consensus.go | 2 +- ipfs_http_connector.go | 2 +- ipfscluster.go | 2 +- map_pin_tracker.go | 2 +- package.json | 12 ++++++------ rest_api.go | 2 +- rpc_api.go | 2 +- rpc_api_test.go | 2 +- 11 files changed, 16 insertions(+), 16 deletions(-) diff --git a/architecture.md b/architecture.md index 2955f78e..d681b3b3 100644 --- a/architecture.md +++ b/architecture.md @@ -44,7 +44,7 @@ Components perform a number of functions and need to be able to communicate with Communication between components happens through the RPC API: a set of functions which stablishes which functions are available to components (`rpc_api.go`). -The RPC API uses `go-libp2p-rpc`. The main Cluster component runs an RPC server. RPC Clients are provided to all components for their use. The main feature of this setup is that **Components can use `go-libp2p-rpc` to perform operations in the local cluster and in any remote cluster node using the same API**. +The RPC API uses `go-libp2p-gorpc`. The main Cluster component runs an RPC server. RPC Clients are provided to all components for their use. The main feature of this setup is that **Components can use `go-libp2p-gorpc` to perform operations in the local cluster and in any remote cluster node using the same API**. This makes broadcasting operations, contacting the Cluster leader really easy. It also allows to think of a future where components may be completely arbitrary and run from different applications. Local RPC calls, on their side, do not suffer any penalty as the execution is short cut directly to the server component of the Cluster, without network intervention. diff --git a/cluster.go b/cluster.go index db1fabe5..80f7f79e 100644 --- a/cluster.go +++ b/cluster.go @@ -6,7 +6,7 @@ import ( "sync" "time" - rpc "github.com/hsanjuan/go-libp2p-rpc" + rpc "github.com/hsanjuan/go-libp2p-gorpc" cid "github.com/ipfs/go-cid" crypto "github.com/libp2p/go-libp2p-crypto" host "github.com/libp2p/go-libp2p-host" diff --git a/cluster_test.go b/cluster_test.go index 27972b09..1bc72fd8 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -4,7 +4,7 @@ import ( "errors" "testing" - rpc "github.com/hsanjuan/go-libp2p-rpc" + rpc "github.com/hsanjuan/go-libp2p-gorpc" cid "github.com/ipfs/go-cid" ) diff --git a/consensus.go b/consensus.go index e0e2c32a..bbaea9e1 100644 --- a/consensus.go +++ b/consensus.go @@ -7,7 +7,7 @@ import ( "sync" "time" - rpc "github.com/hsanjuan/go-libp2p-rpc" + rpc "github.com/hsanjuan/go-libp2p-gorpc" cid "github.com/ipfs/go-cid" consensus "github.com/libp2p/go-libp2p-consensus" host "github.com/libp2p/go-libp2p-host" diff --git a/ipfs_http_connector.go b/ipfs_http_connector.go index 1b126ac0..91deb02a 100644 --- a/ipfs_http_connector.go +++ b/ipfs_http_connector.go @@ -14,7 +14,7 @@ import ( "sync" "time" - rpc "github.com/hsanjuan/go-libp2p-rpc" + rpc "github.com/hsanjuan/go-libp2p-gorpc" cid "github.com/ipfs/go-cid" ma "github.com/multiformats/go-multiaddr" ) diff --git a/ipfscluster.go b/ipfscluster.go index 96513c0c..35eb10d1 100644 --- a/ipfscluster.go +++ b/ipfscluster.go @@ -11,7 +11,7 @@ package ipfscluster import ( "time" - rpc "github.com/hsanjuan/go-libp2p-rpc" + rpc "github.com/hsanjuan/go-libp2p-gorpc" cid "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log" peer "github.com/libp2p/go-libp2p-peer" diff --git a/map_pin_tracker.go b/map_pin_tracker.go index 73177151..9b95981f 100644 --- a/map_pin_tracker.go +++ b/map_pin_tracker.go @@ -5,7 +5,7 @@ import ( "sync" "time" - rpc "github.com/hsanjuan/go-libp2p-rpc" + rpc "github.com/hsanjuan/go-libp2p-gorpc" cid "github.com/ipfs/go-cid" peer "github.com/libp2p/go-libp2p-peer" ) diff --git a/package.json b/package.json index 49cfbc86..4e8a21aa 100644 --- a/package.json +++ b/package.json @@ -31,12 +31,6 @@ "name": "go-cid", "version": "0.7.7" }, - { - "author": "hector", - "hash": "QmZvXudUXTc2G3o2SYPU7uQX1gzbeQkAZXJ6bnN8C4rcXE", - "name": "go-libp2p-rpc", - "version": "0.0.10" - }, { "author": "urfave", "hash": "Qmc1AtgBdoUHP8oYSqU81NRYdzohmF45t5XNwVMvhCxsBA", @@ -54,6 +48,12 @@ "hash": "QmVrBjGjskzhW7HRfC56f4mT3nmNBHj2WWnEXr39G7Y6w8", "name": "mux", "version": "1.3.0" + }, + { + "author": "hector", + "hash": "QmcmpQc4j164VuyLxxvYM5VZYf8RZRnouttAbvK6ZknUu3", + "name": "go-libp2p-gorpc", + "version": "0.0.14" } ], "gxVersion": "0.10.0", diff --git a/rest_api.go b/rest_api.go index c43ebb4a..18c1b911 100644 --- a/rest_api.go +++ b/rest_api.go @@ -12,7 +12,7 @@ import ( "sync" "time" - rpc "github.com/hsanjuan/go-libp2p-rpc" + rpc "github.com/hsanjuan/go-libp2p-gorpc" cid "github.com/ipfs/go-cid" peer "github.com/libp2p/go-libp2p-peer" ma "github.com/multiformats/go-multiaddr" diff --git a/rpc_api.go b/rpc_api.go index f7932ea6..4d36cf1f 100644 --- a/rpc_api.go +++ b/rpc_api.go @@ -5,7 +5,7 @@ import ( peer "github.com/libp2p/go-libp2p-peer" ) -// RPCAPI is a go-libp2p-rpc service which provides the internal ipfs-cluster +// RPCAPI is a go-libp2p-gorpc service which provides the internal ipfs-cluster // API, which enables components and members of the cluster to communicate and // request actions from each other. // diff --git a/rpc_api_test.go b/rpc_api_test.go index 21e5b8d3..999b9eea 100644 --- a/rpc_api_test.go +++ b/rpc_api_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - rpc "github.com/hsanjuan/go-libp2p-rpc" + rpc "github.com/hsanjuan/go-libp2p-gorpc" cid "github.com/ipfs/go-cid" crypto "github.com/libp2p/go-libp2p-crypto" peer "github.com/libp2p/go-libp2p-peer" From 78aa03e2d81a08cad3d955458221ef66938644c4 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 25 Jan 2017 12:20:49 +0100 Subject: [PATCH 6/6] A raft-boltdb import was permanently gx'ed License: MIT Signed-off-by: Hector Sanjuan --- raft.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/raft.go b/raft.go index 0accef59..197acdd4 100644 --- a/raft.go +++ b/raft.go @@ -5,10 +5,10 @@ import ( "path/filepath" hashiraft "github.com/hashicorp/raft" + raftboltdb "github.com/hashicorp/raft-boltdb" host "github.com/libp2p/go-libp2p-host" peer "github.com/libp2p/go-libp2p-peer" libp2praft "github.com/libp2p/go-libp2p-raft" - raftboltdb "gx/ipfs/QmWrdreJ88vAtixq3k7PA9fX6hCLVzWwQu9oYr48R81NEa/raft-boltdb" ) // libp2pRaftWrap wraps the stuff that we need to run