From e99b7b4f798b55be2aa0dadc1053139fc23ceccd Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Fri, 10 Mar 2017 15:29:11 +0100 Subject: [PATCH] [WIP]: Move REST API and IPFS HTTP Connector to its own submodules Part of Issue #18. License: MIT Signed-off-by: Hector Sanjuan --- rest_api.go => api/restapi/restapi.go | 17 +++++---- .../restapi/restapi_test.go | 8 ++-- debug.go | 2 + ipfs-cluster-service/main.go | 9 ++++- .../ipfshttp/ipfshttp.go | 19 ++++++---- .../ipfshttp/ipfshttp_test.go | 37 +++++++------------ ipfscluster_test.go | 8 +++- nodebug.go | 2 + silent.go | 2 + 9 files changed, 58 insertions(+), 46 deletions(-) rename rest_api.go => api/restapi/restapi.go (96%) rename rest_api_test.go => api/restapi/restapi_test.go (97%) rename ipfs_http_connector.go => ipfs-connector/ipfshttp/ipfshttp.go (95%) rename ipfs_http_connector_test.go => ipfs-connector/ipfshttp/ipfshttp_test.go (88%) diff --git a/rest_api.go b/api/restapi/restapi.go similarity index 96% rename from rest_api.go rename to api/restapi/restapi.go index 180aaffb..b1319e83 100644 --- a/rest_api.go +++ b/api/restapi/restapi.go @@ -1,4 +1,4 @@ -package ipfscluster +package restapi import ( "context" @@ -16,10 +16,13 @@ import ( mux "github.com/gorilla/mux" rpc "github.com/hsanjuan/go-libp2p-gorpc" cid "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log" peer "github.com/libp2p/go-libp2p-peer" ma "github.com/multiformats/go-multiaddr" ) +var logger = logging.Logger("restapi") + // Server settings var ( // maximum duration before timing out read of the request @@ -72,14 +75,14 @@ func (e errorResp) Error() string { return e.Message } -// NewRESTAPI creates a new object which is ready to be -// started. -func NewRESTAPI(cfg *Config) (*RESTAPI, error) { - listenAddr, err := cfg.APIAddr.ValueForProtocol(ma.P_IP4) +// NewRESTAPI creates a new REST API component. It receives +// the multiaddress on which the API listens. +func NewRESTAPI(apiMAddr ma.Multiaddr) (*RESTAPI, error) { + listenAddr, err := apiMAddr.ValueForProtocol(ma.P_IP4) if err != nil { return nil, err } - listenPortStr, err := cfg.APIAddr.ValueForProtocol(ma.P_TCP) + listenPortStr, err := apiMAddr.ValueForProtocol(ma.P_TCP) if err != nil { return nil, err } @@ -108,7 +111,7 @@ func NewRESTAPI(cfg *Config) (*RESTAPI, error) { api := &RESTAPI{ ctx: ctx, cancel: cancel, - apiAddr: cfg.APIAddr, + apiAddr: apiMAddr, listenAddr: listenAddr, listenPort: listenPort, listener: l, diff --git a/rest_api_test.go b/api/restapi/restapi_test.go similarity index 97% rename from rest_api_test.go rename to api/restapi/restapi_test.go index 39857c94..cac22dfb 100644 --- a/rest_api_test.go +++ b/api/restapi/restapi_test.go @@ -1,4 +1,4 @@ -package ipfscluster +package restapi import ( "bytes" @@ -10,6 +10,8 @@ import ( "github.com/ipfs/ipfs-cluster/api" "github.com/ipfs/ipfs-cluster/test" + + ma "github.com/multiformats/go-multiaddr" ) var ( @@ -18,8 +20,8 @@ var ( func testRESTAPI(t *testing.T) *RESTAPI { //logging.SetDebugLogging() - cfg := testingConfig() - rest, err := NewRESTAPI(cfg) + apiMAddr, _ := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/10002") + rest, err := NewRESTAPI(apiMAddr) if err != nil { t.Fatal("should be able to create a new Api: ", err) } diff --git a/debug.go b/debug.go index eff42743..b3ab400e 100644 --- a/debug.go +++ b/debug.go @@ -5,6 +5,8 @@ package ipfscluster func init() { l := "DEBUG" SetFacilityLogLevel("cluster", l) + SetFacilityLogLevel("restapi", l) + SetFacilityLogLevel("ipfshttp", l) //SetFacilityLogLevel("raft", l) //SetFacilityLogLevel("p2p-gorpc", l) //SetFacilityLogLevel("swarm2", l) diff --git a/ipfs-cluster-service/main.go b/ipfs-cluster-service/main.go index faf5d0d1..d14e691b 100644 --- a/ipfs-cluster-service/main.go +++ b/ipfs-cluster-service/main.go @@ -16,7 +16,9 @@ import ( ipfscluster "github.com/ipfs/ipfs-cluster" "github.com/ipfs/ipfs-cluster/allocator/numpinalloc" + "github.com/ipfs/ipfs-cluster/api/restapi" "github.com/ipfs/ipfs-cluster/informer/numpin" + "github.com/ipfs/ipfs-cluster/ipfs-connector/ipfshttp" "github.com/ipfs/ipfs-cluster/state/mapstate" ) @@ -237,10 +239,11 @@ func run(c *cli.Context) error { cfg.LeaveOnShutdown = true } - api, err := ipfscluster.NewRESTAPI(cfg) + api, err := restapi.NewRESTAPI(cfg.APIAddr) checkErr("creating REST API component", err) - proxy, err := ipfscluster.NewIPFSHTTPConnector(cfg) + proxy, err := ipfshttp.NewIPFSHTTPConnector( + cfg.IPFSNodeAddr, cfg.IPFSProxyAddr) checkErr("creating IPFS Connector component", err) state := mapstate.NewMapState() @@ -284,6 +287,8 @@ func setupLogging(lvl string) { func setupDebug() { l := "DEBUG" ipfscluster.SetFacilityLogLevel("cluster", l) + ipfscluster.SetFacilityLogLevel("ipfshttp", l) + ipfscluster.SetFacilityLogLevel("restapi", l) ipfscluster.SetFacilityLogLevel("raft", l) ipfscluster.SetFacilityLogLevel("p2p-gorpc", l) //SetFacilityLogLevel("swarm2", l) diff --git a/ipfs_http_connector.go b/ipfs-connector/ipfshttp/ipfshttp.go similarity index 95% rename from ipfs_http_connector.go rename to ipfs-connector/ipfshttp/ipfshttp.go index f7655548..8bc7aff3 100644 --- a/ipfs_http_connector.go +++ b/ipfs-connector/ipfshttp/ipfshttp.go @@ -1,4 +1,4 @@ -package ipfscluster +package ipfshttp import ( "context" @@ -18,10 +18,13 @@ import ( rpc "github.com/hsanjuan/go-libp2p-gorpc" cid "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log" peer "github.com/libp2p/go-libp2p-peer" ma "github.com/multiformats/go-multiaddr" ) +var logger = logging.Logger("ipfshttp") + // IPFS Proxy settings var ( // maximum duration before timing out read of the request @@ -88,12 +91,12 @@ type ipfsIDResp struct { } // NewIPFSHTTPConnector creates the component and leaves it ready to be started -func NewIPFSHTTPConnector(cfg *Config) (*IPFSHTTPConnector, error) { - destHost, err := cfg.IPFSNodeAddr.ValueForProtocol(ma.P_IP4) +func NewIPFSHTTPConnector(ipfsNodeMAddr ma.Multiaddr, ipfsProxyMAddr ma.Multiaddr) (*IPFSHTTPConnector, error) { + destHost, err := ipfsNodeMAddr.ValueForProtocol(ma.P_IP4) if err != nil { return nil, err } - destPortStr, err := cfg.IPFSNodeAddr.ValueForProtocol(ma.P_TCP) + destPortStr, err := ipfsNodeMAddr.ValueForProtocol(ma.P_TCP) if err != nil { return nil, err } @@ -102,11 +105,11 @@ func NewIPFSHTTPConnector(cfg *Config) (*IPFSHTTPConnector, error) { return nil, err } - listenAddr, err := cfg.IPFSProxyAddr.ValueForProtocol(ma.P_IP4) + listenAddr, err := ipfsProxyMAddr.ValueForProtocol(ma.P_IP4) if err != nil { return nil, err } - listenPortStr, err := cfg.IPFSProxyAddr.ValueForProtocol(ma.P_TCP) + listenPortStr, err := ipfsProxyMAddr.ValueForProtocol(ma.P_TCP) if err != nil { return nil, err } @@ -135,8 +138,8 @@ func NewIPFSHTTPConnector(cfg *Config) (*IPFSHTTPConnector, error) { ipfs := &IPFSHTTPConnector{ ctx: ctx, cancel: cancel, - nodeAddr: cfg.IPFSNodeAddr, - proxyAddr: cfg.IPFSProxyAddr, + nodeAddr: ipfsNodeMAddr, + proxyAddr: ipfsProxyMAddr, destHost: destHost, destPort: destPort, diff --git a/ipfs_http_connector_test.go b/ipfs-connector/ipfshttp/ipfshttp_test.go similarity index 88% rename from ipfs_http_connector_test.go rename to ipfs-connector/ipfshttp/ipfshttp_test.go index a40a8fd3..6e23cbe6 100644 --- a/ipfs_http_connector_test.go +++ b/ipfs-connector/ipfshttp/ipfshttp_test.go @@ -1,4 +1,4 @@ -package ipfscluster +package ipfshttp import ( "encoding/json" @@ -14,18 +14,13 @@ import ( ma "github.com/multiformats/go-multiaddr" ) -func testIPFSConnectorConfig(mock *test.IpfsMock) *Config { - cfg := testingConfig() - addr, _ := ma.NewMultiaddr(fmt.Sprintf("/ip4/%s/tcp/%d", mock.Addr, mock.Port)) - cfg.IPFSNodeAddr = addr - return cfg -} - func testIPFSConnector(t *testing.T) (*IPFSHTTPConnector, *test.IpfsMock) { mock := test.NewIpfsMock() - cfg := testIPFSConnectorConfig(mock) + nodeMAddr, _ := ma.NewMultiaddr(fmt.Sprintf("/ip4/%s/tcp/%d", + mock.Addr, mock.Port)) + proxyMAddr, _ := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/10001") - ipfs, err := NewIPFSHTTPConnector(cfg) + ipfs, err := NewIPFSHTTPConnector(nodeMAddr, proxyMAddr) if err != nil { t.Fatal("creating an IPFSConnector should work: ", err) } @@ -148,15 +143,12 @@ func TestIPFSPinLs(t *testing.T) { } func TestIPFSProxyVersion(t *testing.T) { - // This makes sure default handler is used - ipfs, mock := testIPFSConnector(t) defer mock.Close() defer ipfs.Shutdown() - cfg := testingConfig() - host, _ := cfg.IPFSProxyAddr.ValueForProtocol(ma.P_IP4) - port, _ := cfg.IPFSProxyAddr.ValueForProtocol(ma.P_TCP) + host, _ := ipfs.proxyAddr.ValueForProtocol(ma.P_IP4) + port, _ := ipfs.proxyAddr.ValueForProtocol(ma.P_TCP) res, err := http.Get(fmt.Sprintf("http://%s:%s/api/v0/version", host, port)) @@ -188,9 +180,8 @@ func TestIPFSProxyPin(t *testing.T) { defer mock.Close() defer ipfs.Shutdown() - cfg := testingConfig() - host, _ := cfg.IPFSProxyAddr.ValueForProtocol(ma.P_IP4) - port, _ := cfg.IPFSProxyAddr.ValueForProtocol(ma.P_TCP) + host, _ := ipfs.proxyAddr.ValueForProtocol(ma.P_IP4) + port, _ := ipfs.proxyAddr.ValueForProtocol(ma.P_TCP) res, err := http.Get(fmt.Sprintf("http://%s:%s/api/v0/pin/add?arg=%s", host, port, @@ -244,9 +235,8 @@ func TestIPFSProxyUnpin(t *testing.T) { defer mock.Close() defer ipfs.Shutdown() - cfg := testingConfig() - host, _ := cfg.IPFSProxyAddr.ValueForProtocol(ma.P_IP4) - port, _ := cfg.IPFSProxyAddr.ValueForProtocol(ma.P_TCP) + host, _ := ipfs.proxyAddr.ValueForProtocol(ma.P_IP4) + port, _ := ipfs.proxyAddr.ValueForProtocol(ma.P_TCP) res, err := http.Get(fmt.Sprintf("http://%s:%s/api/v0/pin/rm?arg=%s", host, port, @@ -301,9 +291,8 @@ func TestIPFSProxyPinLs(t *testing.T) { defer mock.Close() defer ipfs.Shutdown() - cfg := testingConfig() - host, _ := cfg.IPFSProxyAddr.ValueForProtocol(ma.P_IP4) - port, _ := cfg.IPFSProxyAddr.ValueForProtocol(ma.P_TCP) + host, _ := ipfs.proxyAddr.ValueForProtocol(ma.P_IP4) + port, _ := ipfs.proxyAddr.ValueForProtocol(ma.P_TCP) res, err := http.Get(fmt.Sprintf("http://%s:%s/api/v0/pin/ls?arg=%s", host, port, diff --git a/ipfscluster_test.go b/ipfscluster_test.go index bb00d9f1..271c0042 100644 --- a/ipfscluster_test.go +++ b/ipfscluster_test.go @@ -12,7 +12,9 @@ import ( "github.com/ipfs/ipfs-cluster/allocator/numpinalloc" "github.com/ipfs/ipfs-cluster/api" + "github.com/ipfs/ipfs-cluster/api/restapi" "github.com/ipfs/ipfs-cluster/informer/numpin" + "github.com/ipfs/ipfs-cluster/ipfs-connector/ipfshttp" "github.com/ipfs/ipfs-cluster/state/mapstate" "github.com/ipfs/ipfs-cluster/test" @@ -79,9 +81,11 @@ func createComponents(t *testing.T, i int) (*Config, API, IPFSConnector, State, cfg.ReplicationFactor = -1 cfg.MonitoringIntervalSeconds = 2 - api, err := NewRESTAPI(cfg) + api, err := restapi.NewRESTAPI(cfg.APIAddr) checkErr(t, err) - ipfs, err := NewIPFSHTTPConnector(cfg) + ipfs, err := ipfshttp.NewIPFSHTTPConnector( + cfg.IPFSNodeAddr, + cfg.IPFSProxyAddr) checkErr(t, err) state := mapstate.NewMapState() tracker := NewMapPinTracker(cfg) diff --git a/nodebug.go b/nodebug.go index 9918c7dd..74cdd0d4 100644 --- a/nodebug.go +++ b/nodebug.go @@ -5,6 +5,8 @@ package ipfscluster // This is our default logs levels func init() { SetFacilityLogLevel("cluster", "INFO") + SetFacilityLogLevel("restapi", "INFO") + SetFacilityLogLevel("ipfshttp", "INFO") SetFacilityLogLevel("raft", "ERROR") SetFacilityLogLevel("p2p-gorpc", "ERROR") //SetFacilityLogLevel("swarm2", l) diff --git a/silent.go b/silent.go index d1fabf36..91bc4de7 100644 --- a/silent.go +++ b/silent.go @@ -5,6 +5,8 @@ package ipfscluster func init() { l := "CRITICAL" SetFacilityLogLevel("cluster", l) + SetFacilityLogLevel("restapi", l) + SetFacilityLogLevel("ipfshttp", l) SetFacilityLogLevel("raft", l) SetFacilityLogLevel("p2p-gorpc", l) SetFacilityLogLevel("swarm2", l)