2017-03-14 15:37:29 +00:00
|
|
|
// Package ipfshttp implements an IPFS Cluster IPFSConnector component. It
|
|
|
|
// uses the IPFS HTTP API to communicate to IPFS.
|
2017-03-10 14:29:11 +00:00
|
|
|
package ipfshttp
|
2016-12-02 18:33:39 +00:00
|
|
|
|
|
|
|
import (
|
2017-03-29 20:52:13 +00:00
|
|
|
"bytes"
|
2016-12-02 18:33:39 +00:00
|
|
|
"context"
|
|
|
|
"encoding/json"
|
|
|
|
"errors"
|
|
|
|
"fmt"
|
|
|
|
"io"
|
|
|
|
"io/ioutil"
|
2016-12-05 15:24:41 +00:00
|
|
|
"net"
|
2016-12-02 18:33:39 +00:00
|
|
|
"net/http"
|
2018-10-04 05:03:27 +00:00
|
|
|
"net/http/httputil"
|
2017-03-29 20:52:13 +00:00
|
|
|
"net/url"
|
2018-10-03 21:03:30 +00:00
|
|
|
"strconv"
|
2016-12-02 18:33:39 +00:00
|
|
|
"strings"
|
2016-12-15 13:07:19 +00:00
|
|
|
"sync"
|
2016-12-22 10:31:09 +00:00
|
|
|
"time"
|
2016-12-02 18:33:39 +00:00
|
|
|
|
2018-08-17 14:17:28 +00:00
|
|
|
"github.com/ipfs/ipfs-cluster/adder/adderutils"
|
2017-02-08 17:04:08 +00:00
|
|
|
"github.com/ipfs/ipfs-cluster/api"
|
2018-08-20 18:43:27 +00:00
|
|
|
"github.com/ipfs/ipfs-cluster/rpcutil"
|
2017-02-08 17:04:08 +00:00
|
|
|
|
2017-10-13 16:44:43 +00:00
|
|
|
rpc "github.com/hsanjuan/go-libp2p-gorpc"
|
|
|
|
cid "github.com/ipfs/go-cid"
|
2018-02-09 05:23:24 +00:00
|
|
|
"github.com/ipfs/go-ipfs-cmdkit/files"
|
2017-10-13 16:44:43 +00:00
|
|
|
logging "github.com/ipfs/go-log"
|
|
|
|
peer "github.com/libp2p/go-libp2p-peer"
|
|
|
|
ma "github.com/multiformats/go-multiaddr"
|
2018-06-12 08:09:03 +00:00
|
|
|
madns "github.com/multiformats/go-multiaddr-dns"
|
2017-10-13 16:44:43 +00:00
|
|
|
manet "github.com/multiformats/go-multiaddr-net"
|
2016-12-02 18:33:39 +00:00
|
|
|
)
|
|
|
|
|
2018-06-12 08:09:03 +00:00
|
|
|
// DNSTimeout is used when resolving DNS multiaddresses in this module
|
|
|
|
var DNSTimeout = 5 * time.Second
|
|
|
|
|
2017-03-10 14:29:11 +00:00
|
|
|
var logger = logging.Logger("ipfshttp")
|
|
|
|
|
2018-08-07 09:49:31 +00:00
|
|
|
// updateMetricsMod only makes updates to informer metrics
|
|
|
|
// on the nth occasion. So, for example, for every BlockPut,
|
|
|
|
// only the 10th will trigger a SendInformerMetrics call.
|
|
|
|
var updateMetricMod = 10
|
|
|
|
|
2017-03-14 15:37:29 +00:00
|
|
|
// Connector implements the IPFSConnector interface
|
2016-12-02 18:33:39 +00:00
|
|
|
// and provides a component which does two tasks:
|
|
|
|
//
|
|
|
|
// On one side, it proxies HTTP requests to the configured IPFS
|
|
|
|
// daemon. It is able to intercept these requests though, and
|
|
|
|
// perform extra operations on them.
|
|
|
|
//
|
|
|
|
// On the other side, it is used to perform on-demand requests
|
|
|
|
// against the configured IPFS daemom (such as a pin request).
|
2017-03-14 15:37:29 +00:00
|
|
|
type Connector struct {
|
2017-03-02 12:57:37 +00:00
|
|
|
ctx context.Context
|
|
|
|
cancel func()
|
|
|
|
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
config *Config
|
|
|
|
nodeAddr string
|
2016-12-23 18:35:37 +00:00
|
|
|
|
|
|
|
handlers map[string]func(http.ResponseWriter, *http.Request)
|
|
|
|
|
|
|
|
rpcClient *rpc.Client
|
|
|
|
rpcReady chan struct{}
|
2016-12-05 15:24:41 +00:00
|
|
|
|
2018-03-08 10:29:23 +00:00
|
|
|
listener net.Listener // proxy listener
|
|
|
|
server *http.Server // proxy server
|
|
|
|
client *http.Client // client to ipfs daemon
|
2016-12-09 19:54:46 +00:00
|
|
|
|
2018-08-07 09:49:31 +00:00
|
|
|
updateMetricMutex sync.Mutex
|
|
|
|
updateMetricCount int
|
|
|
|
|
2016-12-15 13:07:19 +00:00
|
|
|
shutdownLock sync.Mutex
|
|
|
|
shutdown bool
|
|
|
|
wg sync.WaitGroup
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
type ipfsError struct {
|
|
|
|
Message string
|
|
|
|
}
|
|
|
|
|
2017-01-26 21:49:53 +00:00
|
|
|
type ipfsPinType struct {
|
|
|
|
Type string
|
|
|
|
}
|
|
|
|
|
|
|
|
type ipfsPinLsResp struct {
|
|
|
|
Keys map[string]ipfsPinType
|
|
|
|
}
|
|
|
|
|
|
|
|
type ipfsPinOpResp struct {
|
|
|
|
Pins []string
|
2017-01-25 17:07:19 +00:00
|
|
|
}
|
|
|
|
|
2017-01-26 18:59:31 +00:00
|
|
|
type ipfsIDResp struct {
|
|
|
|
ID string
|
|
|
|
Addresses []string
|
|
|
|
}
|
|
|
|
|
2018-10-03 21:03:30 +00:00
|
|
|
// From https://github.com/ipfs/go-ipfs/blob/master/core/coreunix/add.go#L49
|
2017-03-29 20:52:13 +00:00
|
|
|
type ipfsAddResp struct {
|
2017-11-13 12:52:33 +00:00
|
|
|
Name string
|
2018-10-03 21:03:30 +00:00
|
|
|
Hash string `json:",omitempty"`
|
|
|
|
Bytes int64 `json:",omitempty"`
|
|
|
|
Size string `json:",omitempty"`
|
2017-03-29 20:52:13 +00:00
|
|
|
}
|
|
|
|
|
2017-10-13 21:12:46 +00:00
|
|
|
type ipfsSwarmPeersResp struct {
|
|
|
|
Peers []ipfsPeer
|
|
|
|
}
|
|
|
|
|
|
|
|
type ipfsPeer struct {
|
2018-01-18 02:49:35 +00:00
|
|
|
Peer string
|
2017-10-13 21:12:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
type ipfsStream struct {
|
|
|
|
Protocol string
|
|
|
|
}
|
|
|
|
|
2017-03-14 15:37:29 +00:00
|
|
|
// NewConnector creates the component and leaves it ready to be started
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
func NewConnector(cfg *Config) (*Connector, error) {
|
|
|
|
err := cfg.Validate()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2018-06-12 08:09:03 +00:00
|
|
|
nodeMAddr := cfg.NodeAddr
|
|
|
|
// dns multiaddresses need to be resolved first
|
|
|
|
if madns.Matches(nodeMAddr) {
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), DNSTimeout)
|
|
|
|
defer cancel()
|
|
|
|
resolvedAddrs, err := madns.Resolve(ctx, cfg.NodeAddr)
|
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
nodeMAddr = resolvedAddrs[0]
|
|
|
|
}
|
|
|
|
|
|
|
|
_, nodeAddr, err := manet.DialArgs(nodeMAddr)
|
2017-01-23 17:38:59 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
proxyNet, proxyAddr, err := manet.DialArgs(cfg.ProxyAddr)
|
2017-01-23 17:38:59 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2017-03-16 14:51:24 +00:00
|
|
|
l, err := net.Listen(proxyNet, proxyAddr)
|
2016-12-05 15:24:41 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2016-12-09 19:54:46 +00:00
|
|
|
|
2018-10-04 05:03:27 +00:00
|
|
|
nodeHTTPAddr := "http://" + nodeAddr
|
|
|
|
proxyURL, err := url.Parse(nodeHTTPAddr)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
proxyHandler := httputil.NewSingleHostReverseProxy(proxyURL)
|
|
|
|
|
2016-12-09 19:54:46 +00:00
|
|
|
smux := http.NewServeMux()
|
|
|
|
s := &http.Server{
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
ReadTimeout: cfg.ProxyReadTimeout,
|
|
|
|
WriteTimeout: cfg.ProxyWriteTimeout,
|
|
|
|
ReadHeaderTimeout: cfg.ProxyReadHeaderTimeout,
|
|
|
|
IdleTimeout: cfg.ProxyIdleTimeout,
|
|
|
|
Handler: smux,
|
2016-12-09 19:54:46 +00:00
|
|
|
}
|
2018-08-18 00:45:47 +00:00
|
|
|
|
|
|
|
// See: https://github.com/ipfs/go-ipfs/issues/5168
|
2018-09-29 00:07:57 +00:00
|
|
|
// See: https://github.com/ipfs/ipfs-cluster/issues/548
|
|
|
|
// on why this is re-enabled.
|
2018-08-18 00:45:47 +00:00
|
|
|
s.SetKeepAlivesEnabled(false) // A reminder that this can be changed
|
2016-12-09 19:54:46 +00:00
|
|
|
|
2018-05-24 07:12:05 +00:00
|
|
|
c := &http.Client{} // timeouts are handled by context timeouts
|
2018-03-08 10:29:23 +00:00
|
|
|
|
2017-03-02 12:57:37 +00:00
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
|
|
|
2017-03-14 15:37:29 +00:00
|
|
|
ipfs := &Connector{
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
ctx: ctx,
|
|
|
|
config: cfg,
|
|
|
|
cancel: cancel,
|
|
|
|
nodeAddr: nodeAddr,
|
|
|
|
handlers: make(map[string]func(http.ResponseWriter, *http.Request)),
|
|
|
|
rpcReady: make(chan struct{}, 1),
|
|
|
|
listener: l,
|
|
|
|
server: s,
|
2018-03-08 10:29:23 +00:00
|
|
|
client: c,
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
|
|
|
|
2018-10-04 05:03:27 +00:00
|
|
|
smux.Handle("/", proxyHandler)
|
2018-04-20 04:09:07 +00:00
|
|
|
smux.HandleFunc("/api/v0/pin/add/", ipfs.pinHandler)
|
|
|
|
smux.HandleFunc("/api/v0/pin/rm/", ipfs.unpinHandler)
|
2018-04-30 03:03:42 +00:00
|
|
|
smux.HandleFunc("/api/v0/pin/ls", ipfs.pinLsHandler) // required to handle /pin/ls for all pins
|
2018-04-20 04:09:07 +00:00
|
|
|
smux.HandleFunc("/api/v0/pin/ls/", ipfs.pinLsHandler)
|
|
|
|
smux.HandleFunc("/api/v0/add", ipfs.addHandler)
|
|
|
|
smux.HandleFunc("/api/v0/add/", ipfs.addHandler)
|
2018-08-20 18:43:27 +00:00
|
|
|
smux.HandleFunc("/api/v0/repo/stat", ipfs.repoStatHandler)
|
|
|
|
smux.HandleFunc("/api/v0/repo/stat/", ipfs.repoStatHandler)
|
2016-12-09 19:54:46 +00:00
|
|
|
|
2017-03-23 18:34:33 +00:00
|
|
|
go ipfs.run()
|
2016-12-02 18:33:39 +00:00
|
|
|
return ipfs, nil
|
|
|
|
}
|
|
|
|
|
2017-03-23 18:34:33 +00:00
|
|
|
// launches proxy and connects all ipfs daemons when
|
|
|
|
// we receive the rpcReady signal.
|
2017-03-14 15:37:29 +00:00
|
|
|
func (ipfs *Connector) run() {
|
2017-03-23 18:34:33 +00:00
|
|
|
<-ipfs.rpcReady
|
|
|
|
|
2018-03-08 10:29:23 +00:00
|
|
|
// Do not shutdown while launching threads
|
|
|
|
// -- prevents race conditions with ipfs.wg.
|
|
|
|
ipfs.shutdownLock.Lock()
|
|
|
|
defer ipfs.shutdownLock.Unlock()
|
|
|
|
|
2017-01-26 21:49:53 +00:00
|
|
|
// This launches the proxy
|
|
|
|
ipfs.wg.Add(1)
|
|
|
|
go func() {
|
|
|
|
defer ipfs.wg.Done()
|
2018-04-20 04:09:07 +00:00
|
|
|
logger.Infof(
|
|
|
|
"IPFS Proxy: %s -> %s",
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
ipfs.config.ProxyAddr,
|
2018-04-20 04:09:07 +00:00
|
|
|
ipfs.config.NodeAddr,
|
|
|
|
)
|
2017-03-23 18:34:33 +00:00
|
|
|
err := ipfs.server.Serve(ipfs.listener) // hangs here
|
2017-01-26 21:49:53 +00:00
|
|
|
if err != nil && !strings.Contains(err.Error(), "closed network connection") {
|
|
|
|
logger.Error(err)
|
|
|
|
}
|
|
|
|
}()
|
2017-03-23 18:34:33 +00:00
|
|
|
|
|
|
|
// This runs ipfs swarm connect to the daemons of other cluster members
|
|
|
|
ipfs.wg.Add(1)
|
|
|
|
go func() {
|
|
|
|
defer ipfs.wg.Done()
|
|
|
|
|
|
|
|
// It does not hurt to wait a little bit. i.e. think cluster
|
|
|
|
// peers which are started at the same time as the ipfs
|
|
|
|
// daemon...
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
tmr := time.NewTimer(ipfs.config.ConnectSwarmsDelay)
|
2017-03-23 18:34:33 +00:00
|
|
|
defer tmr.Stop()
|
|
|
|
select {
|
|
|
|
case <-tmr.C:
|
2018-01-16 19:57:54 +00:00
|
|
|
// do not hang this goroutine if this call hangs
|
|
|
|
// otherwise we hang during shutdown
|
|
|
|
go ipfs.ConnectSwarms()
|
2017-03-23 18:34:33 +00:00
|
|
|
case <-ipfs.ctx.Done():
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}()
|
2017-01-26 21:49:53 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func ipfsErrorResponder(w http.ResponseWriter, errMsg string) {
|
2017-03-29 20:52:13 +00:00
|
|
|
res := ipfsError{errMsg}
|
|
|
|
resBytes, _ := json.Marshal(res)
|
2017-04-06 02:27:02 +00:00
|
|
|
w.Header().Add("Content-Type", "application/json")
|
2017-01-26 21:49:53 +00:00
|
|
|
w.WriteHeader(http.StatusInternalServerError)
|
2017-03-29 20:52:13 +00:00
|
|
|
w.Write(resBytes)
|
2017-01-26 21:49:53 +00:00
|
|
|
return
|
|
|
|
}
|
2016-12-23 18:35:37 +00:00
|
|
|
|
2017-03-14 15:37:29 +00:00
|
|
|
func (ipfs *Connector) pinOpHandler(op string, w http.ResponseWriter, r *http.Request) {
|
2018-04-20 04:09:07 +00:00
|
|
|
arg, ok := extractArgument(r.URL)
|
|
|
|
if !ok {
|
2017-01-26 21:49:53 +00:00
|
|
|
ipfsErrorResponder(w, "Error: bad argument")
|
|
|
|
return
|
|
|
|
}
|
2018-07-04 16:30:24 +00:00
|
|
|
c, err := cid.Decode(arg)
|
2017-01-26 21:49:53 +00:00
|
|
|
if err != nil {
|
|
|
|
ipfsErrorResponder(w, "Error parsing CID: "+err.Error())
|
|
|
|
return
|
|
|
|
}
|
2016-12-23 18:35:37 +00:00
|
|
|
|
2018-04-20 04:09:07 +00:00
|
|
|
err = ipfs.rpcClient.Call(
|
|
|
|
"",
|
2017-01-26 21:49:53 +00:00
|
|
|
"Cluster",
|
|
|
|
op,
|
2018-07-04 16:30:24 +00:00
|
|
|
api.PinCid(c).ToSerial(),
|
2018-04-20 04:09:07 +00:00
|
|
|
&struct{}{},
|
|
|
|
)
|
2017-01-26 21:49:53 +00:00
|
|
|
if err != nil {
|
|
|
|
ipfsErrorResponder(w, err.Error())
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2017-03-29 20:52:13 +00:00
|
|
|
res := ipfsPinOpResp{
|
2017-01-26 21:49:53 +00:00
|
|
|
Pins: []string{arg},
|
|
|
|
}
|
2017-03-29 20:52:13 +00:00
|
|
|
resBytes, _ := json.Marshal(res)
|
2017-04-06 02:27:02 +00:00
|
|
|
w.Header().Add("Content-Type", "application/json")
|
2017-01-26 21:49:53 +00:00
|
|
|
w.WriteHeader(http.StatusOK)
|
2017-03-29 20:52:13 +00:00
|
|
|
w.Write(resBytes)
|
2017-01-26 21:49:53 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2017-03-14 15:37:29 +00:00
|
|
|
func (ipfs *Connector) pinHandler(w http.ResponseWriter, r *http.Request) {
|
2017-01-26 21:49:53 +00:00
|
|
|
ipfs.pinOpHandler("Pin", w, r)
|
|
|
|
}
|
|
|
|
|
2017-03-14 15:37:29 +00:00
|
|
|
func (ipfs *Connector) unpinHandler(w http.ResponseWriter, r *http.Request) {
|
2017-01-26 21:49:53 +00:00
|
|
|
ipfs.pinOpHandler("Unpin", w, r)
|
|
|
|
}
|
|
|
|
|
2017-03-14 15:37:29 +00:00
|
|
|
func (ipfs *Connector) pinLsHandler(w http.ResponseWriter, r *http.Request) {
|
2017-01-26 21:49:53 +00:00
|
|
|
pinLs := ipfsPinLsResp{}
|
|
|
|
pinLs.Keys = make(map[string]ipfsPinType)
|
|
|
|
|
2018-04-20 04:09:07 +00:00
|
|
|
arg, ok := extractArgument(r.URL)
|
|
|
|
if ok {
|
2017-04-06 02:27:02 +00:00
|
|
|
c, err := cid.Decode(arg)
|
|
|
|
if err != nil {
|
|
|
|
ipfsErrorResponder(w, err.Error())
|
2017-01-26 21:49:53 +00:00
|
|
|
return
|
|
|
|
}
|
2017-04-06 02:27:02 +00:00
|
|
|
var pin api.PinSerial
|
2018-04-20 04:09:07 +00:00
|
|
|
err = ipfs.rpcClient.Call(
|
|
|
|
"",
|
2017-04-06 02:27:02 +00:00
|
|
|
"Cluster",
|
|
|
|
"PinGet",
|
|
|
|
api.PinCid(c).ToSerial(),
|
2018-04-20 04:09:07 +00:00
|
|
|
&pin,
|
|
|
|
)
|
2017-04-06 02:27:02 +00:00
|
|
|
if err != nil {
|
2018-04-20 04:09:07 +00:00
|
|
|
ipfsErrorResponder(w, fmt.Sprintf("Error: path '%s' is not pinned", arg))
|
2017-01-26 21:49:53 +00:00
|
|
|
return
|
|
|
|
}
|
2017-04-06 02:27:02 +00:00
|
|
|
pinLs.Keys[pin.Cid] = ipfsPinType{
|
|
|
|
Type: "recursive",
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
var pins []api.PinSerial
|
2018-04-20 04:09:07 +00:00
|
|
|
err := ipfs.rpcClient.Call(
|
|
|
|
"",
|
2017-04-06 02:27:02 +00:00
|
|
|
"Cluster",
|
|
|
|
"Pins",
|
|
|
|
struct{}{},
|
2018-04-20 04:09:07 +00:00
|
|
|
&pins,
|
|
|
|
)
|
2017-04-06 02:27:02 +00:00
|
|
|
if err != nil {
|
|
|
|
ipfsErrorResponder(w, err.Error())
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, pin := range pins {
|
|
|
|
pinLs.Keys[pin.Cid] = ipfsPinType{
|
|
|
|
Type: "recursive",
|
|
|
|
}
|
|
|
|
}
|
2017-01-26 21:49:53 +00:00
|
|
|
}
|
|
|
|
|
2017-03-29 20:52:13 +00:00
|
|
|
resBytes, _ := json.Marshal(pinLs)
|
2017-04-06 02:27:02 +00:00
|
|
|
w.Header().Add("Content-Type", "application/json")
|
2017-01-26 21:49:53 +00:00
|
|
|
w.WriteHeader(http.StatusOK)
|
2017-03-29 20:52:13 +00:00
|
|
|
w.Write(resBytes)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (ipfs *Connector) addHandler(w http.ResponseWriter, r *http.Request) {
|
2018-08-16 15:40:32 +00:00
|
|
|
reader, err := r.MultipartReader()
|
2017-03-29 20:52:13 +00:00
|
|
|
if err != nil {
|
2018-08-16 15:40:32 +00:00
|
|
|
ipfsErrorResponder(w, "error reading request: "+err.Error())
|
2017-03-29 20:52:13 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2018-08-16 15:40:32 +00:00
|
|
|
q := r.URL.Query()
|
|
|
|
if q.Get("only-hash") == "true" {
|
|
|
|
ipfsErrorResponder(w, "only-hash is not supported when adding to cluster")
|
2017-03-29 20:52:13 +00:00
|
|
|
}
|
|
|
|
|
2018-08-16 15:40:32 +00:00
|
|
|
unpin := q.Get("pin") == "false"
|
|
|
|
|
|
|
|
// Luckily, most IPFS add query params are compatible with cluster's
|
|
|
|
// /add params. We can parse most of them directly from the query.
|
|
|
|
params, err := api.AddParamsFromQuery(q)
|
|
|
|
if err != nil {
|
|
|
|
ipfsErrorResponder(w, "error parsing options:"+err.Error())
|
2017-03-29 20:52:13 +00:00
|
|
|
return
|
|
|
|
}
|
2018-08-16 15:40:32 +00:00
|
|
|
trickle := q.Get("trickle")
|
|
|
|
if trickle == "true" {
|
|
|
|
params.Layout = "trickle"
|
2017-03-29 20:52:13 +00:00
|
|
|
}
|
|
|
|
|
2018-08-16 15:40:32 +00:00
|
|
|
logger.Warningf("Proxy/add does not support all IPFS params. Current options: %+v", params)
|
2017-03-29 20:52:13 +00:00
|
|
|
|
2018-10-03 21:03:30 +00:00
|
|
|
outputTransform := func(in *api.AddedOutput) interface{} {
|
|
|
|
r := &ipfsAddResp{
|
|
|
|
Name: in.Name,
|
|
|
|
Hash: in.Cid,
|
|
|
|
Bytes: int64(in.Bytes),
|
2017-03-29 20:52:13 +00:00
|
|
|
}
|
2018-10-03 21:03:30 +00:00
|
|
|
if in.Size != 0 {
|
|
|
|
r.Size = strconv.FormatUint(in.Size, 10)
|
2017-03-29 20:52:13 +00:00
|
|
|
}
|
2018-10-03 21:03:30 +00:00
|
|
|
return r
|
2017-03-29 20:52:13 +00:00
|
|
|
}
|
|
|
|
|
2018-08-17 14:17:28 +00:00
|
|
|
root, err := adderutils.AddMultipartHTTPHandler(
|
|
|
|
ipfs.ctx,
|
|
|
|
ipfs.rpcClient,
|
|
|
|
params,
|
|
|
|
reader,
|
|
|
|
w,
|
2018-10-03 21:03:30 +00:00
|
|
|
outputTransform,
|
2018-08-17 14:17:28 +00:00
|
|
|
)
|
|
|
|
|
2018-10-03 21:03:30 +00:00
|
|
|
// any errors have been sent as Trailer
|
2018-08-16 15:40:32 +00:00
|
|
|
if err != nil {
|
|
|
|
return
|
2017-03-29 20:52:13 +00:00
|
|
|
}
|
|
|
|
|
2018-08-16 15:40:32 +00:00
|
|
|
if !unpin {
|
|
|
|
return
|
2017-03-29 20:52:13 +00:00
|
|
|
}
|
2018-08-16 15:40:32 +00:00
|
|
|
|
|
|
|
// Unpin because the user doesn't want to pin
|
|
|
|
time.Sleep(100 * time.Millisecond)
|
|
|
|
err = ipfs.rpcClient.CallContext(
|
|
|
|
ipfs.ctx,
|
|
|
|
"",
|
|
|
|
"Cluster",
|
|
|
|
"Unpin",
|
2018-08-17 14:17:28 +00:00
|
|
|
api.PinCid(root).ToSerial(),
|
2018-08-16 15:40:32 +00:00
|
|
|
&struct{}{},
|
|
|
|
)
|
|
|
|
if err != nil {
|
2018-10-03 21:03:30 +00:00
|
|
|
w.Header().Set("X-Stream-Error", err.Error())
|
2018-08-16 15:40:32 +00:00
|
|
|
return
|
2017-03-29 20:52:13 +00:00
|
|
|
}
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
|
|
|
|
2018-08-20 18:43:27 +00:00
|
|
|
func (ipfs *Connector) repoStatHandler(w http.ResponseWriter, r *http.Request) {
|
|
|
|
var peers []peer.ID
|
|
|
|
err := ipfs.rpcClient.Call(
|
|
|
|
"",
|
|
|
|
"Cluster",
|
|
|
|
"ConsensusPeers",
|
|
|
|
struct{}{},
|
|
|
|
&peers,
|
|
|
|
)
|
|
|
|
if err != nil {
|
|
|
|
ipfsErrorResponder(w, err.Error())
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
ctxs, cancels := rpcutil.CtxsWithTimeout(ipfs.ctx, len(peers), ipfs.config.IPFSRequestTimeout)
|
|
|
|
defer rpcutil.MultiCancel(cancels)
|
|
|
|
|
|
|
|
repoStats := make([]api.IPFSRepoStat, len(peers), len(peers))
|
|
|
|
repoStatsIfaces := make([]interface{}, len(repoStats), len(repoStats))
|
|
|
|
for i := range repoStats {
|
|
|
|
repoStatsIfaces[i] = &repoStats[i]
|
|
|
|
}
|
|
|
|
|
|
|
|
errs := ipfs.rpcClient.MultiCall(
|
|
|
|
ctxs,
|
|
|
|
peers,
|
|
|
|
"Cluster",
|
|
|
|
"IPFSRepoStat",
|
|
|
|
struct{}{},
|
|
|
|
repoStatsIfaces,
|
|
|
|
)
|
|
|
|
|
|
|
|
totalStats := api.IPFSRepoStat{}
|
|
|
|
|
|
|
|
for i, err := range errs {
|
|
|
|
if err != nil {
|
|
|
|
logger.Errorf("%s repo/stat errored: %s", peers[i], err)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
totalStats.RepoSize += repoStats[i].RepoSize
|
|
|
|
totalStats.StorageMax += repoStats[i].StorageMax
|
|
|
|
}
|
|
|
|
|
|
|
|
resBytes, _ := json.Marshal(totalStats)
|
|
|
|
w.Header().Add("Content-Type", "application/json")
|
|
|
|
w.WriteHeader(http.StatusOK)
|
|
|
|
w.Write(resBytes)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2016-12-23 18:35:37 +00:00
|
|
|
// SetClient makes the component ready to perform RPC
|
|
|
|
// requests.
|
2017-03-14 15:37:29 +00:00
|
|
|
func (ipfs *Connector) SetClient(c *rpc.Client) {
|
2016-12-23 18:35:37 +00:00
|
|
|
ipfs.rpcClient = c
|
|
|
|
ipfs.rpcReady <- struct{}{}
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Shutdown stops any listeners and stops the component from taking
|
|
|
|
// any requests.
|
2017-03-14 15:37:29 +00:00
|
|
|
func (ipfs *Connector) Shutdown() error {
|
2016-12-15 13:07:19 +00:00
|
|
|
ipfs.shutdownLock.Lock()
|
|
|
|
defer ipfs.shutdownLock.Unlock()
|
|
|
|
|
|
|
|
if ipfs.shutdown {
|
|
|
|
logger.Debug("already shutdown")
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2016-12-15 13:19:41 +00:00
|
|
|
logger.Info("stopping IPFS Proxy")
|
2016-12-15 13:07:19 +00:00
|
|
|
|
2017-03-02 12:57:37 +00:00
|
|
|
ipfs.cancel()
|
2016-12-23 18:35:37 +00:00
|
|
|
close(ipfs.rpcReady)
|
2016-12-09 19:54:46 +00:00
|
|
|
ipfs.server.SetKeepAlivesEnabled(false)
|
2016-12-05 15:24:41 +00:00
|
|
|
ipfs.listener.Close()
|
2016-12-15 13:07:19 +00:00
|
|
|
|
|
|
|
ipfs.wg.Wait()
|
|
|
|
ipfs.shutdown = true
|
2016-12-02 18:33:39 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2017-01-26 18:59:31 +00:00
|
|
|
// ID performs an ID request against the configured
|
|
|
|
// IPFS daemon. It returns the fetched information.
|
|
|
|
// If the request fails, or the parsing fails, it
|
|
|
|
// returns an error and an empty IPFSID which also
|
|
|
|
// contains the error message.
|
2017-03-14 15:37:29 +00:00
|
|
|
func (ipfs *Connector) ID() (api.IPFSID, error) {
|
2018-05-24 07:12:05 +00:00
|
|
|
ctx, cancel := context.WithTimeout(ipfs.ctx, ipfs.config.IPFSRequestTimeout)
|
|
|
|
defer cancel()
|
2017-02-08 17:04:08 +00:00
|
|
|
id := api.IPFSID{}
|
2018-06-28 15:01:39 +00:00
|
|
|
body, err := ipfs.postCtx(ctx, "id", "", nil)
|
2017-01-26 18:59:31 +00:00
|
|
|
if err != nil {
|
|
|
|
id.Error = err.Error()
|
|
|
|
return id, err
|
|
|
|
}
|
|
|
|
|
2017-03-29 20:52:13 +00:00
|
|
|
var res ipfsIDResp
|
|
|
|
err = json.Unmarshal(body, &res)
|
2017-01-26 18:59:31 +00:00
|
|
|
if err != nil {
|
|
|
|
id.Error = err.Error()
|
|
|
|
return id, err
|
|
|
|
}
|
|
|
|
|
2017-03-29 20:52:13 +00:00
|
|
|
pID, err := peer.IDB58Decode(res.ID)
|
2017-01-26 18:59:31 +00:00
|
|
|
if err != nil {
|
|
|
|
id.Error = err.Error()
|
|
|
|
return id, err
|
|
|
|
}
|
|
|
|
id.ID = pID
|
|
|
|
|
2017-03-29 20:52:13 +00:00
|
|
|
mAddrs := make([]ma.Multiaddr, len(res.Addresses), len(res.Addresses))
|
|
|
|
for i, strAddr := range res.Addresses {
|
2017-01-26 18:59:31 +00:00
|
|
|
mAddr, err := ma.NewMultiaddr(strAddr)
|
|
|
|
if err != nil {
|
|
|
|
id.Error = err.Error()
|
|
|
|
return id, err
|
|
|
|
}
|
|
|
|
mAddrs[i] = mAddr
|
|
|
|
}
|
|
|
|
id.Addresses = mAddrs
|
|
|
|
return id, nil
|
|
|
|
}
|
|
|
|
|
2016-12-02 18:33:39 +00:00
|
|
|
// Pin performs a pin request against the configured IPFS
|
|
|
|
// daemon.
|
2018-09-22 01:00:10 +00:00
|
|
|
func (ipfs *Connector) Pin(ctx context.Context, hash cid.Cid, maxDepth int) error {
|
2018-04-18 04:53:41 +00:00
|
|
|
ctx, cancel := context.WithTimeout(ctx, ipfs.config.PinTimeout)
|
|
|
|
defer cancel()
|
2018-04-16 09:01:20 +00:00
|
|
|
pinStatus, err := ipfs.PinLsCid(ctx, hash)
|
2016-12-05 14:30:11 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2018-03-07 13:51:37 +00:00
|
|
|
|
2018-07-04 16:30:24 +00:00
|
|
|
if pinStatus.IsPinned(maxDepth) {
|
|
|
|
logger.Debug("IPFS object is already pinned: ", hash)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2018-08-07 09:49:31 +00:00
|
|
|
defer ipfs.updateInformerMetric()
|
|
|
|
|
2018-07-04 16:30:24 +00:00
|
|
|
var pinArgs string
|
|
|
|
switch {
|
|
|
|
case maxDepth < 0:
|
|
|
|
pinArgs = "recursive=true"
|
|
|
|
case maxDepth == 0:
|
|
|
|
pinArgs = "recursive=false"
|
|
|
|
default:
|
|
|
|
pinArgs = fmt.Sprintf("recursive=true&max-depth=%d", maxDepth)
|
|
|
|
}
|
|
|
|
|
|
|
|
switch ipfs.config.PinMethod {
|
|
|
|
case "refs": // do refs -r first
|
|
|
|
path := fmt.Sprintf("refs?arg=%s&%s", hash, pinArgs)
|
|
|
|
err := ipfs.postDiscardBodyCtx(ctx, path)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
2016-12-19 17:35:24 +00:00
|
|
|
}
|
2018-07-04 16:30:24 +00:00
|
|
|
logger.Debugf("Refs for %s sucessfully fetched", hash)
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
2018-07-04 16:30:24 +00:00
|
|
|
|
|
|
|
path := fmt.Sprintf("pin/add?arg=%s&%s", hash, pinArgs)
|
|
|
|
_, err = ipfs.postCtx(ctx, path, "", nil)
|
|
|
|
if err == nil {
|
|
|
|
logger.Info("IPFS Pin request succeeded: ", hash)
|
|
|
|
}
|
|
|
|
return err
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
|
|
|
|
2016-12-28 15:25:24 +00:00
|
|
|
// Unpin performs an unpin request against the configured IPFS
|
2016-12-02 18:33:39 +00:00
|
|
|
// daemon.
|
2018-09-22 01:00:10 +00:00
|
|
|
func (ipfs *Connector) Unpin(ctx context.Context, hash cid.Cid) error {
|
2018-04-18 04:53:41 +00:00
|
|
|
ctx, cancel := context.WithTimeout(ctx, ipfs.config.UnpinTimeout)
|
|
|
|
defer cancel()
|
2018-08-07 09:49:31 +00:00
|
|
|
|
2018-04-16 09:01:20 +00:00
|
|
|
pinStatus, err := ipfs.PinLsCid(ctx, hash)
|
2016-12-05 14:30:11 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2018-07-04 16:30:24 +00:00
|
|
|
if pinStatus.IsPinned(-1) {
|
2018-08-07 09:49:31 +00:00
|
|
|
defer ipfs.updateInformerMetric()
|
2016-12-02 18:33:39 +00:00
|
|
|
path := fmt.Sprintf("pin/rm?arg=%s", hash)
|
2018-06-28 15:01:39 +00:00
|
|
|
_, err := ipfs.postCtx(ctx, path, "", nil)
|
2016-12-19 17:35:24 +00:00
|
|
|
if err == nil {
|
|
|
|
logger.Info("IPFS Unpin request succeeded:", hash)
|
|
|
|
}
|
2016-12-02 18:33:39 +00:00
|
|
|
return err
|
|
|
|
}
|
2016-12-07 16:21:29 +00:00
|
|
|
|
2017-01-23 22:58:04 +00:00
|
|
|
logger.Debug("IPFS object is already unpinned: ", hash)
|
2016-12-02 18:33:39 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2017-02-09 15:29:17 +00:00
|
|
|
// PinLs performs a "pin ls --type typeFilter" request against the configured
|
|
|
|
// IPFS daemon and returns a map of cid strings and their status.
|
2018-04-16 09:01:20 +00:00
|
|
|
func (ipfs *Connector) PinLs(ctx context.Context, typeFilter string) (map[string]api.IPFSPinStatus, error) {
|
2018-05-24 07:12:05 +00:00
|
|
|
ctx, cancel := context.WithTimeout(ctx, ipfs.config.IPFSRequestTimeout)
|
|
|
|
defer cancel()
|
2018-06-28 15:01:39 +00:00
|
|
|
body, err := ipfs.postCtx(ctx, "pin/ls?type="+typeFilter, "", nil)
|
2017-01-25 17:07:19 +00:00
|
|
|
|
|
|
|
// Some error talking to the daemon
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2017-03-29 20:52:13 +00:00
|
|
|
var res ipfsPinLsResp
|
|
|
|
err = json.Unmarshal(body, &res)
|
2016-12-07 16:21:29 +00:00
|
|
|
if err != nil {
|
2017-01-25 17:07:19 +00:00
|
|
|
logger.Error("parsing pin/ls response")
|
|
|
|
logger.Error(string(body))
|
|
|
|
return nil, err
|
2016-12-07 16:21:29 +00:00
|
|
|
}
|
|
|
|
|
2017-02-08 17:04:08 +00:00
|
|
|
statusMap := make(map[string]api.IPFSPinStatus)
|
2017-03-29 20:52:13 +00:00
|
|
|
for k, v := range res.Keys {
|
2017-02-08 17:04:08 +00:00
|
|
|
statusMap[k] = api.IPFSPinStatusFromString(v.Type)
|
2016-12-07 16:21:29 +00:00
|
|
|
}
|
2017-01-25 17:07:19 +00:00
|
|
|
return statusMap, nil
|
2016-12-07 16:21:29 +00:00
|
|
|
}
|
|
|
|
|
2018-08-15 11:16:52 +00:00
|
|
|
// PinLsCid performs a "pin ls <hash>" request. It first tries with
|
|
|
|
// "type=recursive" and then, if not found, with "type=direct". It returns an
|
|
|
|
// api.IPFSPinStatus for that hash.
|
2018-09-22 01:00:10 +00:00
|
|
|
func (ipfs *Connector) PinLsCid(ctx context.Context, hash cid.Cid) (api.IPFSPinStatus, error) {
|
2018-08-15 11:16:52 +00:00
|
|
|
pinLsType := func(pinType string) ([]byte, error) {
|
|
|
|
ctx, cancel := context.WithTimeout(ctx, ipfs.config.IPFSRequestTimeout)
|
|
|
|
defer cancel()
|
|
|
|
lsPath := fmt.Sprintf("pin/ls?arg=%s&type=%s", hash, pinType)
|
|
|
|
return ipfs.postCtx(ctx, lsPath, "", nil)
|
|
|
|
}
|
|
|
|
|
|
|
|
var body []byte
|
|
|
|
var err error
|
|
|
|
// FIXME: Sharding may need to check more pin types here.
|
|
|
|
for _, pinType := range []string{"recursive", "direct"} {
|
|
|
|
body, err = pinLsType(pinType)
|
|
|
|
// Network error, daemon down
|
|
|
|
if body == nil && err != nil {
|
|
|
|
return api.IPFSPinStatusError, err
|
|
|
|
}
|
2016-12-05 14:30:11 +00:00
|
|
|
|
2018-08-20 09:39:34 +00:00
|
|
|
// Pin found. Do not keep looking.
|
|
|
|
if err == nil {
|
|
|
|
break
|
2018-08-15 11:16:52 +00:00
|
|
|
}
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
|
|
|
|
2018-08-15 11:16:52 +00:00
|
|
|
if err != nil { // we could not find the pin
|
2017-02-08 17:04:08 +00:00
|
|
|
return api.IPFSPinStatusUnpinned, nil
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
|
|
|
|
2017-03-29 20:52:13 +00:00
|
|
|
var res ipfsPinLsResp
|
|
|
|
err = json.Unmarshal(body, &res)
|
2016-12-02 18:33:39 +00:00
|
|
|
if err != nil {
|
2017-01-25 17:07:19 +00:00
|
|
|
logger.Error("parsing pin/ls?arg=cid response:")
|
2016-12-08 16:24:38 +00:00
|
|
|
logger.Error(string(body))
|
2017-02-08 17:04:08 +00:00
|
|
|
return api.IPFSPinStatusError, err
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
2017-03-29 20:52:13 +00:00
|
|
|
pinObj, ok := res.Keys[hash.String()]
|
2016-12-02 18:33:39 +00:00
|
|
|
if !ok {
|
2017-02-08 17:04:08 +00:00
|
|
|
return api.IPFSPinStatusError, errors.New("expected to find the pin in the response")
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
2017-01-25 17:07:19 +00:00
|
|
|
|
2017-02-08 17:04:08 +00:00
|
|
|
return api.IPFSPinStatusFromString(pinObj.Type), nil
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
|
|
|
|
2018-02-09 05:23:24 +00:00
|
|
|
func (ipfs *Connector) doPostCtx(ctx context.Context, client *http.Client, apiURL, path string, contentType string, postBody io.Reader) (*http.Response, error) {
|
2017-10-13 16:38:18 +00:00
|
|
|
logger.Debugf("posting %s", path)
|
2018-04-09 02:49:02 +00:00
|
|
|
urlstr := fmt.Sprintf("%s/%s", apiURL, path)
|
2018-03-08 10:29:23 +00:00
|
|
|
|
2018-02-09 05:23:24 +00:00
|
|
|
req, err := http.NewRequest("POST", urlstr, postBody)
|
2018-04-09 02:49:02 +00:00
|
|
|
if err != nil {
|
|
|
|
logger.Error("error creating POST request:", err)
|
|
|
|
}
|
|
|
|
|
2018-02-09 05:23:24 +00:00
|
|
|
req.Header.Set("Content-Type", contentType)
|
2018-04-09 02:49:02 +00:00
|
|
|
req = req.WithContext(ctx)
|
|
|
|
res, err := ipfs.client.Do(req)
|
2018-03-08 10:29:23 +00:00
|
|
|
if err != nil {
|
|
|
|
logger.Error("error posting to IPFS:", err)
|
|
|
|
}
|
2018-04-09 02:49:02 +00:00
|
|
|
|
2018-03-08 10:29:23 +00:00
|
|
|
return res, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// checkResponse tries to parse an error message on non StatusOK responses
|
|
|
|
// from ipfs.
|
|
|
|
func checkResponse(path string, code int, body []byte) error {
|
|
|
|
if code == http.StatusOK {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
var ipfsErr ipfsError
|
2016-12-02 18:33:39 +00:00
|
|
|
|
2018-03-08 10:29:23 +00:00
|
|
|
if body != nil && json.Unmarshal(body, &ipfsErr) == nil {
|
|
|
|
return fmt.Errorf("IPFS unsuccessful: %d: %s", code, ipfsErr.Message)
|
|
|
|
}
|
|
|
|
// No error response with useful message from ipfs
|
|
|
|
return fmt.Errorf("IPFS-post '%s' unsuccessful: %d: %s", path, code, body)
|
|
|
|
}
|
|
|
|
|
2018-05-24 07:12:05 +00:00
|
|
|
// postCtx makes a POST request against
|
2018-03-08 10:29:23 +00:00
|
|
|
// the ipfs daemon, reads the full body of the response and
|
|
|
|
// returns it after checking for errors.
|
2018-02-09 05:23:24 +00:00
|
|
|
func (ipfs *Connector) postCtx(ctx context.Context, path string, contentType string, postBody io.Reader) ([]byte, error) {
|
|
|
|
res, err := ipfs.doPostCtx(ctx, ipfs.client, ipfs.apiURL(), path, contentType, postBody)
|
2016-12-02 18:33:39 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2017-03-29 20:52:13 +00:00
|
|
|
defer res.Body.Close()
|
|
|
|
body, err := ioutil.ReadAll(res.Body)
|
2016-12-02 18:33:39 +00:00
|
|
|
if err != nil {
|
2016-12-15 13:19:41 +00:00
|
|
|
logger.Errorf("error reading response body: %s", err)
|
2016-12-02 18:33:39 +00:00
|
|
|
return nil, err
|
|
|
|
}
|
2018-03-08 10:29:23 +00:00
|
|
|
return body, checkResponse(path, res.StatusCode, body)
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
|
|
|
|
2018-04-18 04:53:41 +00:00
|
|
|
// postDiscardBodyCtx makes a POST requests but discards the body
|
2018-03-08 10:29:23 +00:00
|
|
|
// of the response directly after reading it.
|
2018-04-18 04:53:41 +00:00
|
|
|
func (ipfs *Connector) postDiscardBodyCtx(ctx context.Context, path string) error {
|
2018-03-15 17:31:06 +00:00
|
|
|
res, err := ipfs.doPostCtx(ctx, ipfs.client, ipfs.apiURL(), path, "", nil)
|
2018-03-07 13:51:37 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer res.Body.Close()
|
|
|
|
_, err = io.Copy(ioutil.Discard, res.Body)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2018-03-08 10:29:23 +00:00
|
|
|
return checkResponse(path, res.StatusCode, nil)
|
2018-03-07 13:51:37 +00:00
|
|
|
}
|
|
|
|
|
2016-12-02 18:33:39 +00:00
|
|
|
// apiURL is a short-hand for building the url of the IPFS
|
|
|
|
// daemon API.
|
2017-03-14 15:37:29 +00:00
|
|
|
func (ipfs *Connector) apiURL() string {
|
2017-03-16 14:51:24 +00:00
|
|
|
return fmt.Sprintf("http://%s/api/v0", ipfs.nodeAddr)
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
2017-03-23 18:34:33 +00:00
|
|
|
|
|
|
|
// ConnectSwarms requests the ipfs addresses of other peers and
|
|
|
|
// triggers ipfs swarm connect requests
|
2017-03-27 13:07:12 +00:00
|
|
|
func (ipfs *Connector) ConnectSwarms() error {
|
2018-05-24 07:12:05 +00:00
|
|
|
ctx, cancel := context.WithTimeout(ipfs.ctx, ipfs.config.IPFSRequestTimeout)
|
|
|
|
defer cancel()
|
2017-03-23 18:34:33 +00:00
|
|
|
var idsSerial []api.IDSerial
|
2018-04-20 04:09:07 +00:00
|
|
|
err := ipfs.rpcClient.Call(
|
|
|
|
"",
|
2017-03-23 18:34:33 +00:00
|
|
|
"Cluster",
|
|
|
|
"Peers",
|
|
|
|
struct{}{},
|
2018-04-20 04:09:07 +00:00
|
|
|
&idsSerial,
|
|
|
|
)
|
2017-03-23 18:34:33 +00:00
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
2017-03-27 13:07:12 +00:00
|
|
|
return err
|
2017-03-23 18:34:33 +00:00
|
|
|
}
|
|
|
|
logger.Debugf("%+v", idsSerial)
|
|
|
|
|
|
|
|
for _, idSerial := range idsSerial {
|
|
|
|
ipfsID := idSerial.IPFS
|
|
|
|
for _, addr := range ipfsID.Addresses {
|
|
|
|
// This is a best effort attempt
|
|
|
|
// We ignore errors which happens
|
|
|
|
// when passing in a bunch of addresses
|
2018-02-09 05:23:24 +00:00
|
|
|
_, err := ipfs.postCtx(
|
|
|
|
ctx,
|
|
|
|
fmt.Sprintf("swarm/connect?arg=%s", addr),
|
2018-06-28 15:01:39 +00:00
|
|
|
"",
|
2018-02-09 05:23:24 +00:00
|
|
|
nil,
|
|
|
|
)
|
2017-03-23 18:34:33 +00:00
|
|
|
if err != nil {
|
|
|
|
logger.Debug(err)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
logger.Debugf("ipfs successfully connected to %s", addr)
|
|
|
|
}
|
|
|
|
}
|
2017-03-27 13:07:12 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// ConfigKey fetches the IPFS daemon configuration and retrieves the value for
|
|
|
|
// a given configuration key. For example, "Datastore/StorageMax" will return
|
|
|
|
// the value for StorageMax in the Datastore configuration object.
|
|
|
|
func (ipfs *Connector) ConfigKey(keypath string) (interface{}, error) {
|
2018-05-24 07:12:05 +00:00
|
|
|
ctx, cancel := context.WithTimeout(ipfs.ctx, ipfs.config.IPFSRequestTimeout)
|
|
|
|
defer cancel()
|
2018-06-28 15:01:39 +00:00
|
|
|
res, err := ipfs.postCtx(ctx, "config/show", "", nil)
|
2017-03-27 13:07:12 +00:00
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
var cfg map[string]interface{}
|
2017-03-29 20:52:13 +00:00
|
|
|
err = json.Unmarshal(res, &cfg)
|
2017-03-27 13:07:12 +00:00
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
path := strings.SplitN(keypath, "/", 2)
|
|
|
|
if len(path) == 0 {
|
|
|
|
return nil, errors.New("cannot lookup without a path")
|
|
|
|
}
|
|
|
|
|
|
|
|
return getConfigValue(path, cfg)
|
|
|
|
}
|
|
|
|
|
|
|
|
func getConfigValue(path []string, cfg map[string]interface{}) (interface{}, error) {
|
|
|
|
value, ok := cfg[path[0]]
|
|
|
|
if !ok {
|
|
|
|
return nil, errors.New("key not found in configuration")
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(path) == 1 {
|
|
|
|
return value, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
switch value.(type) {
|
|
|
|
case map[string]interface{}:
|
|
|
|
v := value.(map[string]interface{})
|
|
|
|
return getConfigValue(path[1:], v)
|
|
|
|
default:
|
|
|
|
return nil, errors.New("invalid path")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-08-20 18:43:27 +00:00
|
|
|
// RepoStat returns the DiskUsage and StorageMax repo/stat values from the
|
|
|
|
// ipfs daemon, in bytes, wrapped as an IPFSRepoStat object.
|
|
|
|
func (ipfs *Connector) RepoStat() (api.IPFSRepoStat, error) {
|
2018-05-24 07:12:05 +00:00
|
|
|
ctx, cancel := context.WithTimeout(ipfs.ctx, ipfs.config.IPFSRequestTimeout)
|
|
|
|
defer cancel()
|
2018-08-17 14:53:40 +00:00
|
|
|
res, err := ipfs.postCtx(ctx, "repo/stat?size-only=true", "", nil)
|
2017-03-27 13:07:12 +00:00
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
2018-08-20 18:43:27 +00:00
|
|
|
return api.IPFSRepoStat{}, err
|
2017-03-27 13:07:12 +00:00
|
|
|
}
|
|
|
|
|
2018-08-20 18:43:27 +00:00
|
|
|
var stats api.IPFSRepoStat
|
2017-03-29 20:52:13 +00:00
|
|
|
err = json.Unmarshal(res, &stats)
|
2017-03-27 13:07:12 +00:00
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
2018-08-20 18:43:27 +00:00
|
|
|
return stats, err
|
2017-03-27 13:07:12 +00:00
|
|
|
}
|
2018-08-20 18:43:27 +00:00
|
|
|
return stats, nil
|
2017-03-23 18:34:33 +00:00
|
|
|
}
|
2017-10-13 21:12:46 +00:00
|
|
|
|
2018-02-09 05:23:24 +00:00
|
|
|
// SwarmPeers returns the peers currently connected to this ipfs daemon.
|
2017-10-13 21:12:46 +00:00
|
|
|
func (ipfs *Connector) SwarmPeers() (api.SwarmPeers, error) {
|
2018-05-24 07:12:05 +00:00
|
|
|
ctx, cancel := context.WithTimeout(ipfs.ctx, ipfs.config.IPFSRequestTimeout)
|
|
|
|
defer cancel()
|
2017-10-13 21:12:46 +00:00
|
|
|
swarm := api.SwarmPeers{}
|
2018-06-28 15:01:39 +00:00
|
|
|
res, err := ipfs.postCtx(ctx, "swarm/peers", "", nil)
|
2017-10-13 21:12:46 +00:00
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
return swarm, err
|
|
|
|
}
|
|
|
|
var peersRaw ipfsSwarmPeersResp
|
|
|
|
err = json.Unmarshal(res, &peersRaw)
|
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
return swarm, err
|
|
|
|
}
|
|
|
|
|
2018-01-18 02:49:35 +00:00
|
|
|
swarm = make([]peer.ID, len(peersRaw.Peers))
|
2017-10-13 21:12:46 +00:00
|
|
|
for i, p := range peersRaw.Peers {
|
|
|
|
pID, err := peer.IDB58Decode(p.Peer)
|
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
return swarm, err
|
|
|
|
}
|
2018-01-18 02:49:35 +00:00
|
|
|
swarm[i] = pID
|
2017-10-13 21:12:46 +00:00
|
|
|
}
|
|
|
|
return swarm, nil
|
|
|
|
}
|
2018-04-20 04:09:07 +00:00
|
|
|
|
2018-06-28 15:01:39 +00:00
|
|
|
// BlockPut triggers an ipfs block put on the given data, inserting the block
|
|
|
|
// into the ipfs daemon's repo.
|
|
|
|
func (ipfs *Connector) BlockPut(b api.NodeWithMeta) error {
|
2018-08-08 19:11:26 +00:00
|
|
|
logger.Debugf("putting block to IPFS: %s", b.Cid)
|
2018-06-28 15:01:39 +00:00
|
|
|
ctx, cancel := context.WithTimeout(ipfs.ctx, ipfs.config.IPFSRequestTimeout)
|
|
|
|
defer cancel()
|
2018-08-07 09:49:31 +00:00
|
|
|
defer ipfs.updateInformerMetric()
|
|
|
|
|
2018-06-28 15:01:39 +00:00
|
|
|
r := ioutil.NopCloser(bytes.NewReader(b.Data))
|
|
|
|
rFile := files.NewReaderFile("", "", r, nil)
|
|
|
|
sliceFile := files.NewSliceFile("", "", []files.File{rFile}) // IPFS reqs require a wrapping directory
|
|
|
|
multiFileR := files.NewMultiFileReader(sliceFile, true)
|
|
|
|
if b.Format == "" {
|
|
|
|
b.Format = "v0"
|
|
|
|
}
|
|
|
|
url := "block/put?f=" + b.Format
|
|
|
|
contentType := "multipart/form-data; boundary=" + multiFileR.Boundary()
|
2018-08-07 09:49:31 +00:00
|
|
|
|
2018-06-28 15:01:39 +00:00
|
|
|
_, err := ipfs.postCtx(ctx, url, contentType, multiFileR)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// BlockGet retrieves an ipfs block with the given cid
|
2018-09-22 01:00:10 +00:00
|
|
|
func (ipfs *Connector) BlockGet(c cid.Cid) ([]byte, error) {
|
2018-06-28 15:01:39 +00:00
|
|
|
ctx, cancel := context.WithTimeout(ipfs.ctx, ipfs.config.IPFSRequestTimeout)
|
|
|
|
defer cancel()
|
|
|
|
url := "block/get?arg=" + c.String()
|
|
|
|
return ipfs.postCtx(ctx, url, "", nil)
|
|
|
|
}
|
|
|
|
|
2018-04-20 04:09:07 +00:00
|
|
|
// extractArgument extracts the cid argument from a url.URL, either via
|
|
|
|
// the query string parameters or from the url path itself.
|
|
|
|
func extractArgument(u *url.URL) (string, bool) {
|
|
|
|
arg := u.Query().Get("arg")
|
|
|
|
if arg != "" {
|
|
|
|
return arg, true
|
|
|
|
}
|
|
|
|
|
|
|
|
p := strings.TrimPrefix(u.Path, "/api/v0/")
|
|
|
|
segs := strings.Split(p, "/")
|
|
|
|
|
|
|
|
if len(segs) > 2 {
|
|
|
|
warnMsg := "You are using an undocumented form of the IPFS API."
|
|
|
|
warnMsg += "Consider passing your command arguments"
|
|
|
|
warnMsg += "with the '?arg=' query parameter"
|
|
|
|
logger.Warning(warnMsg)
|
|
|
|
return segs[len(segs)-1], true
|
|
|
|
}
|
|
|
|
return "", false
|
|
|
|
}
|
2018-08-07 09:49:31 +00:00
|
|
|
|
|
|
|
// Returns true every updateMetricsMod-th time that we
|
|
|
|
// call this function.
|
|
|
|
func (ipfs *Connector) shouldUpdateMetric() bool {
|
|
|
|
ipfs.updateMetricMutex.Lock()
|
|
|
|
defer ipfs.updateMetricMutex.Unlock()
|
|
|
|
ipfs.updateMetricCount++
|
|
|
|
if ipfs.updateMetricCount%updateMetricMod == 0 {
|
|
|
|
ipfs.updateMetricCount = 0
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
// Trigger a broadcast of the local informer metrics.
|
|
|
|
func (ipfs *Connector) updateInformerMetric() error {
|
|
|
|
if !ipfs.shouldUpdateMetric() {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
var metric api.Metric
|
|
|
|
|
|
|
|
err := ipfs.rpcClient.GoContext(
|
|
|
|
ipfs.ctx,
|
|
|
|
"",
|
|
|
|
"Cluster",
|
|
|
|
"SendInformerMetric",
|
|
|
|
struct{}{},
|
|
|
|
&metric,
|
|
|
|
nil,
|
|
|
|
)
|
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
}
|
|
|
|
return err
|
|
|
|
}
|