2017-03-14 15:37:29 +00:00
|
|
|
// Package ipfshttp implements an IPFS Cluster IPFSConnector component. It
|
|
|
|
// uses the IPFS HTTP API to communicate to IPFS.
|
2017-03-10 14:29:11 +00:00
|
|
|
package ipfshttp
|
2016-12-02 18:33:39 +00:00
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"encoding/json"
|
|
|
|
"errors"
|
|
|
|
"fmt"
|
|
|
|
"io"
|
|
|
|
"io/ioutil"
|
|
|
|
"net/http"
|
Add PinPath/UnpinPath support.
Squashed commit of the following:
commit 38cf569c6aed77c46ee4e0f8baa4d1a9daf8f03e
Merge: d125f69 aaada42
Author: Hector Sanjuan <hsanjuan@users.noreply.github.com>
Date: Wed Feb 20 11:02:00 2019 +0000
Merge pull request #634 from ipfs/issue_450
Support PinPath, UnpinPath (resolve before pinning)
commit aaada42054e1f1c7b2abb1270859d0de41a0e5d8
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 19 22:16:25 2019 +0530
formatResponse accepts api.Pin and not api.PinSerial
commit b5da4bea045865814cc422da71827b44ddd44b90
Merge: ba59036 cc8dd7e
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 19 21:36:46 2019 +0530
Merge branch 'master' into issue_450
commit ba5903649c1df1dba20f4d6f7e3573d6fe24921f
Merge: f002914 d59880c
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Feb 18 08:41:11 2019 +0530
Merge branch 'issue_450' of github.com:ipfs/ipfs-cluster into issue_450
commit f00291494c0c02621c2296cbb7ac71e4c23aa9ec
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Feb 18 08:31:39 2019 +0530
PinPath: more improvements
Added tracing for new methods
commit d59880c338eaa8214fe06b4f930a540793d78407
Merge: 0ca4c7c b4f0eb3
Author: Hector Sanjuan <hsanjuan@users.noreply.github.com>
Date: Wed Feb 13 15:22:49 2019 +0000
Merge branch 'master' into issue_450
commit 0ca4c7c3b0670ed9c8279f8274d36e3485c10030
Merge: d35017a ecef9ea
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 12 13:10:13 2019 +0530
Merge branch 'master' into issue_450
commit d35017a8de91ca9fc9a9a047c48c75134cee9f98
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 12 13:07:25 2019 +0530
PinPath: more improvements
- Worth having `PinOptions` as a separate field in the struct and
constructing the query in the test with ToQuery()
- sharness: "intialization" line can be placed outside the tests at
the top
commit 68e3b90417ffbad89d41a70ac81d85f9037f8848
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Sun Feb 10 21:43:50 2019 +0530
Using if-continue pattern instead of if-else
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 3c29799f3b85be328b27508332ab92049d8b82f3
Merge: 956790b 4324889
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Thu Feb 7 10:25:52 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 956790b381db9858e4194f983e898b07dc51ba66
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 21:11:20 2019 +0530
Removing resolved path
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 7191cc46cedfbec116a9746937e28881b50ca044
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 16:45:07 2019 +0530
Fix go vet
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit f8b3d5b63b1b7569e2a3e0d82894fd4491c246c4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 16:07:03 2019 +0530
Fixed linting error
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 23c57eb467755a1f21387a1615a7f34e97348053
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 09:20:41 2019 +0530
Fixed tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 0caedd94aefeb3b6649dedc214cb4b849ace2ea4
Merge: 17e555e 5a7ee1d
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 00:07:10 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 17e555e4a7c574413df90aac70c5cc29cab98f54
Author: Hector Sanjuan <code@hector.link>
Date: Tue Feb 5 16:58:50 2019 +0000
PinPath: address some feedback + improvements
* Changed client's Pin() API and PinPath to be consistent
* Added helper methods to turn PinPath to query and back
* Make code and tests build
* Use TestCidResolved everywhere
* Fix cluster.PinPath arguments
* Fix formatting of responses with --no-status
* Make tests readable and call Fatal when needed
* Use a pathTestCases variable
commit f0e7369c47c5ddadc8ed45df5fd2d4d9b2d42b38
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 5 18:34:26 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Addressed review comments as in
https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-198751932
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit a8b4f181d2d7afed32ee41331dfaab19fd66a173
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 29 22:41:27 2019 +0530
Fixing tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit e39b95ca19e4d75506f4f492678245ef13936a44
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 29 14:52:53 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- PinPath and UnpinPath should return api.Pin
- PinPath should accept pin options
- Removing duplicate logic for Resolve from cluster
- And many other review comments https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-195509504
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit d146075126320896665ba58d337a13789f68ea86
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 23 17:08:41 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
PinPath(in both rest and rpc) should return a serializable struct in the
form `{"\":"Q...cid..string..."}` (as used in "github.com/ipfs/go-cid"
to marshal and unmarshal)
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 1f4869568a8adb450275257154ea3a26d03a30f3
Merge: 7acfd28 a244af9
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 23 07:18:56 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 7acfd282732ddf2282a67d4f9d0170a494eb3ed4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 22 18:14:32 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- RPC must always use serializable structs
- In command, just use pin with path as cid is also a valid path
- Addressing many other small review comments as in
https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-192122534
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 36905041e1e3f0b204942030aab3ab7b5b9e4d62
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 16 09:36:42 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Extra logic for path checking should go into resolve so that it can be
properly reused
Added sharness tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 9116bda3534e77bb391d873051bb520a1b01a326
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 16 08:08:07 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
error strings should not be capitalized
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit ca7e61861374f456300a85ddc0374e594f74f963
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 15 23:40:25 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Tests
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 522fbcd899f01c01680375561a32a87464157c0a
Merge: f1a56ab f7bc468
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 15 10:40:54 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit f1a56ab925fb74c0c44273a4524afa4843cf757f
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Jan 14 20:58:17 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- IPFS Connector should act as a pure IPFS client, any extra logic
should go to cluster.go
- Use cid.Undef, instead of cid.Cid{}
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit c83b91054f6774f1f9d4930cfc3f1fa28236f57c
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Thu Jan 10 08:57:17 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- Separate handlers, methods and rpc apis for PinPath and UnpinPath from
Pin and Unpin
- Support ipld paths as well
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 719dff88129366ce3ccb5e04cb6f8082a0915c5c
Merge: 91ceb47 21170c4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 9 19:38:35 2019 +0530
Merge branch 'issue_450_old' into HEAD
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 91ceb4796259ca7ef2974ec43e6a278a12796b13
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 9 19:36:41 2019 +0530
Revert "WIP: Figure out why test does not impleme"
This reverts commit 28a3a3f25dce6f296c8cbef86221644c099a7e75.
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 28a3a3f25dce6f296c8cbef86221644c099a7e75
Author: cd10012 <ced361@nyu.edu>
Date: Tue Jul 24 23:23:10 2018 -0400
WIP: Figure out why test does not implement IPFSConnector interface...
License: MIT
Signed-off-by: cd10012 <ced361@nyu.edu>
commit 21170c48e77e69583db64544b08120a9baf40d8d
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 8 10:37:59 2019 +0530
Support PinPath, UnpinPath (resolve before pinning)
This commit adds API support for pinning using path
`POST /pins/<ipfs or ipns path>` and `DELETE /pins/<ipfs or ipns path>`
will resolve the path into a cid and perform perform pinning or
unpinning
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Co-authored-by: Hector Sanjuan <hector@protocol.ai>
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2019-02-20 11:07:50 +00:00
|
|
|
"net/url"
|
2019-05-17 14:26:27 +00:00
|
|
|
"strconv"
|
2016-12-02 18:33:39 +00:00
|
|
|
"strings"
|
2016-12-15 13:07:19 +00:00
|
|
|
"sync"
|
2022-01-31 16:44:35 +00:00
|
|
|
"sync/atomic"
|
2016-12-22 10:31:09 +00:00
|
|
|
"time"
|
2016-12-02 18:33:39 +00:00
|
|
|
|
2017-02-08 17:04:08 +00:00
|
|
|
"github.com/ipfs/ipfs-cluster/api"
|
Adders: stream blocks to destinations
This commit fixes #810 and adds block streaming to the final destinations when
adding. This should add major performance gains when adding data to clusters.
Before, everytime cluster issued a block, it was broadcasted individually to
all destinations (new libp2p stream), where it was block/put to IPFS (a single
block/put http roundtrip per block).
Now, blocks are streamed all the way from the adder module to the ipfs daemon,
by making every block as it arrives a single part in a multipart block/put
request.
Before, block-broadcast needed to wait for all destinations to finish in order
to process the next block. Now, buffers allow some destinations to be faster
than others while sending and receiving blocks.
Before, if a block put request failed to be broadcasted everywhere, an error
would happen at that moment.
Now, we keep streaming until the end and only then report any errors. The
operation succeeds as long as at least one stream finished successfully.
Errors block/putting to IPFS will not abort streams. Instead, subsequent
blocks are retried with a new request, although the method will return an
error when the stream finishes if there were errors at any point.
2022-03-24 01:17:10 +00:00
|
|
|
"go.uber.org/multierr"
|
2018-06-27 04:03:15 +00:00
|
|
|
|
2018-12-06 18:59:05 +00:00
|
|
|
cid "github.com/ipfs/go-cid"
|
|
|
|
files "github.com/ipfs/go-ipfs-files"
|
2021-08-16 15:52:31 +00:00
|
|
|
ipfspinner "github.com/ipfs/go-ipfs-pinner"
|
2020-03-13 20:40:02 +00:00
|
|
|
logging "github.com/ipfs/go-log/v2"
|
2019-06-14 10:41:11 +00:00
|
|
|
gopath "github.com/ipfs/go-path"
|
|
|
|
peer "github.com/libp2p/go-libp2p-core/peer"
|
2018-12-06 18:59:05 +00:00
|
|
|
rpc "github.com/libp2p/go-libp2p-gorpc"
|
|
|
|
madns "github.com/multiformats/go-multiaddr-dns"
|
2020-08-27 12:10:58 +00:00
|
|
|
manet "github.com/multiformats/go-multiaddr/net"
|
2021-02-19 09:13:45 +00:00
|
|
|
multihash "github.com/multiformats/go-multihash"
|
2019-06-14 10:41:11 +00:00
|
|
|
|
|
|
|
"go.opencensus.io/plugin/ochttp"
|
|
|
|
"go.opencensus.io/plugin/ochttp/propagation/tracecontext"
|
|
|
|
"go.opencensus.io/trace"
|
2016-12-02 18:33:39 +00:00
|
|
|
)
|
|
|
|
|
2018-06-12 08:09:03 +00:00
|
|
|
// DNSTimeout is used when resolving DNS multiaddresses in this module
|
|
|
|
var DNSTimeout = 5 * time.Second
|
|
|
|
|
2017-03-10 14:29:11 +00:00
|
|
|
var logger = logging.Logger("ipfshttp")
|
|
|
|
|
2017-03-14 15:37:29 +00:00
|
|
|
// Connector implements the IPFSConnector interface
|
2018-10-13 14:27:03 +00:00
|
|
|
// and provides a component which is used to perform
|
|
|
|
// on-demand requests against the configured IPFS daemom
|
|
|
|
// (such as a pin request).
|
2017-03-14 15:37:29 +00:00
|
|
|
type Connector struct {
|
2017-03-02 12:57:37 +00:00
|
|
|
ctx context.Context
|
|
|
|
cancel func()
|
|
|
|
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
config *Config
|
|
|
|
nodeAddr string
|
2016-12-23 18:35:37 +00:00
|
|
|
|
|
|
|
rpcClient *rpc.Client
|
|
|
|
rpcReady chan struct{}
|
2016-12-05 15:24:41 +00:00
|
|
|
|
2018-10-13 14:27:03 +00:00
|
|
|
client *http.Client // client to ipfs daemon
|
2016-12-09 19:54:46 +00:00
|
|
|
|
2022-01-31 16:44:35 +00:00
|
|
|
updateMetricCount uint64
|
2018-08-07 09:49:31 +00:00
|
|
|
|
2016-12-15 13:07:19 +00:00
|
|
|
shutdownLock sync.Mutex
|
|
|
|
shutdown bool
|
|
|
|
wg sync.WaitGroup
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
type ipfsError struct {
|
2020-04-20 14:21:27 +00:00
|
|
|
path string
|
|
|
|
code int
|
2016-12-02 18:33:39 +00:00
|
|
|
Message string
|
|
|
|
}
|
|
|
|
|
2020-04-20 14:21:27 +00:00
|
|
|
func (ie ipfsError) Error() string {
|
|
|
|
return fmt.Sprintf(
|
2022-03-22 09:56:16 +00:00
|
|
|
"IPFS error (%s). Code: %d. Message: %s",
|
2020-04-20 14:21:27 +00:00
|
|
|
ie.path,
|
|
|
|
ie.code,
|
|
|
|
ie.Message,
|
|
|
|
)
|
|
|
|
}
|
|
|
|
|
2022-03-22 09:56:16 +00:00
|
|
|
type ipfsUnpinnedError ipfsError
|
|
|
|
|
|
|
|
func (unpinned ipfsUnpinnedError) Is(target error) bool {
|
|
|
|
ierr, ok := target.(ipfsError)
|
|
|
|
if !ok {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
return strings.HasSuffix(ierr.Message, "not pinned")
|
2017-01-26 21:49:53 +00:00
|
|
|
}
|
|
|
|
|
2022-03-22 09:56:16 +00:00
|
|
|
func (unpinned ipfsUnpinnedError) Error() string {
|
|
|
|
return ipfsError(unpinned).Error()
|
2017-01-26 21:49:53 +00:00
|
|
|
}
|
|
|
|
|
2017-01-26 18:59:31 +00:00
|
|
|
type ipfsIDResp struct {
|
|
|
|
ID string
|
|
|
|
Addresses []string
|
|
|
|
}
|
|
|
|
|
Add PinPath/UnpinPath support.
Squashed commit of the following:
commit 38cf569c6aed77c46ee4e0f8baa4d1a9daf8f03e
Merge: d125f69 aaada42
Author: Hector Sanjuan <hsanjuan@users.noreply.github.com>
Date: Wed Feb 20 11:02:00 2019 +0000
Merge pull request #634 from ipfs/issue_450
Support PinPath, UnpinPath (resolve before pinning)
commit aaada42054e1f1c7b2abb1270859d0de41a0e5d8
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 19 22:16:25 2019 +0530
formatResponse accepts api.Pin and not api.PinSerial
commit b5da4bea045865814cc422da71827b44ddd44b90
Merge: ba59036 cc8dd7e
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 19 21:36:46 2019 +0530
Merge branch 'master' into issue_450
commit ba5903649c1df1dba20f4d6f7e3573d6fe24921f
Merge: f002914 d59880c
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Feb 18 08:41:11 2019 +0530
Merge branch 'issue_450' of github.com:ipfs/ipfs-cluster into issue_450
commit f00291494c0c02621c2296cbb7ac71e4c23aa9ec
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Feb 18 08:31:39 2019 +0530
PinPath: more improvements
Added tracing for new methods
commit d59880c338eaa8214fe06b4f930a540793d78407
Merge: 0ca4c7c b4f0eb3
Author: Hector Sanjuan <hsanjuan@users.noreply.github.com>
Date: Wed Feb 13 15:22:49 2019 +0000
Merge branch 'master' into issue_450
commit 0ca4c7c3b0670ed9c8279f8274d36e3485c10030
Merge: d35017a ecef9ea
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 12 13:10:13 2019 +0530
Merge branch 'master' into issue_450
commit d35017a8de91ca9fc9a9a047c48c75134cee9f98
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 12 13:07:25 2019 +0530
PinPath: more improvements
- Worth having `PinOptions` as a separate field in the struct and
constructing the query in the test with ToQuery()
- sharness: "intialization" line can be placed outside the tests at
the top
commit 68e3b90417ffbad89d41a70ac81d85f9037f8848
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Sun Feb 10 21:43:50 2019 +0530
Using if-continue pattern instead of if-else
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 3c29799f3b85be328b27508332ab92049d8b82f3
Merge: 956790b 4324889
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Thu Feb 7 10:25:52 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 956790b381db9858e4194f983e898b07dc51ba66
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 21:11:20 2019 +0530
Removing resolved path
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 7191cc46cedfbec116a9746937e28881b50ca044
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 16:45:07 2019 +0530
Fix go vet
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit f8b3d5b63b1b7569e2a3e0d82894fd4491c246c4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 16:07:03 2019 +0530
Fixed linting error
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 23c57eb467755a1f21387a1615a7f34e97348053
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 09:20:41 2019 +0530
Fixed tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 0caedd94aefeb3b6649dedc214cb4b849ace2ea4
Merge: 17e555e 5a7ee1d
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 00:07:10 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 17e555e4a7c574413df90aac70c5cc29cab98f54
Author: Hector Sanjuan <code@hector.link>
Date: Tue Feb 5 16:58:50 2019 +0000
PinPath: address some feedback + improvements
* Changed client's Pin() API and PinPath to be consistent
* Added helper methods to turn PinPath to query and back
* Make code and tests build
* Use TestCidResolved everywhere
* Fix cluster.PinPath arguments
* Fix formatting of responses with --no-status
* Make tests readable and call Fatal when needed
* Use a pathTestCases variable
commit f0e7369c47c5ddadc8ed45df5fd2d4d9b2d42b38
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 5 18:34:26 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Addressed review comments as in
https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-198751932
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit a8b4f181d2d7afed32ee41331dfaab19fd66a173
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 29 22:41:27 2019 +0530
Fixing tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit e39b95ca19e4d75506f4f492678245ef13936a44
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 29 14:52:53 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- PinPath and UnpinPath should return api.Pin
- PinPath should accept pin options
- Removing duplicate logic for Resolve from cluster
- And many other review comments https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-195509504
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit d146075126320896665ba58d337a13789f68ea86
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 23 17:08:41 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
PinPath(in both rest and rpc) should return a serializable struct in the
form `{"\":"Q...cid..string..."}` (as used in "github.com/ipfs/go-cid"
to marshal and unmarshal)
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 1f4869568a8adb450275257154ea3a26d03a30f3
Merge: 7acfd28 a244af9
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 23 07:18:56 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 7acfd282732ddf2282a67d4f9d0170a494eb3ed4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 22 18:14:32 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- RPC must always use serializable structs
- In command, just use pin with path as cid is also a valid path
- Addressing many other small review comments as in
https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-192122534
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 36905041e1e3f0b204942030aab3ab7b5b9e4d62
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 16 09:36:42 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Extra logic for path checking should go into resolve so that it can be
properly reused
Added sharness tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 9116bda3534e77bb391d873051bb520a1b01a326
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 16 08:08:07 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
error strings should not be capitalized
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit ca7e61861374f456300a85ddc0374e594f74f963
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 15 23:40:25 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Tests
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 522fbcd899f01c01680375561a32a87464157c0a
Merge: f1a56ab f7bc468
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 15 10:40:54 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit f1a56ab925fb74c0c44273a4524afa4843cf757f
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Jan 14 20:58:17 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- IPFS Connector should act as a pure IPFS client, any extra logic
should go to cluster.go
- Use cid.Undef, instead of cid.Cid{}
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit c83b91054f6774f1f9d4930cfc3f1fa28236f57c
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Thu Jan 10 08:57:17 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- Separate handlers, methods and rpc apis for PinPath and UnpinPath from
Pin and Unpin
- Support ipld paths as well
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 719dff88129366ce3ccb5e04cb6f8082a0915c5c
Merge: 91ceb47 21170c4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 9 19:38:35 2019 +0530
Merge branch 'issue_450_old' into HEAD
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 91ceb4796259ca7ef2974ec43e6a278a12796b13
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 9 19:36:41 2019 +0530
Revert "WIP: Figure out why test does not impleme"
This reverts commit 28a3a3f25dce6f296c8cbef86221644c099a7e75.
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 28a3a3f25dce6f296c8cbef86221644c099a7e75
Author: cd10012 <ced361@nyu.edu>
Date: Tue Jul 24 23:23:10 2018 -0400
WIP: Figure out why test does not implement IPFSConnector interface...
License: MIT
Signed-off-by: cd10012 <ced361@nyu.edu>
commit 21170c48e77e69583db64544b08120a9baf40d8d
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 8 10:37:59 2019 +0530
Support PinPath, UnpinPath (resolve before pinning)
This commit adds API support for pinning using path
`POST /pins/<ipfs or ipns path>` and `DELETE /pins/<ipfs or ipns path>`
will resolve the path into a cid and perform perform pinning or
unpinning
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Co-authored-by: Hector Sanjuan <hector@protocol.ai>
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2019-02-20 11:07:50 +00:00
|
|
|
type ipfsResolveResp struct {
|
|
|
|
Path string
|
|
|
|
}
|
|
|
|
|
2019-10-22 05:40:32 +00:00
|
|
|
type ipfsRepoGCResp struct {
|
|
|
|
Key cid.Cid
|
|
|
|
Error string
|
|
|
|
}
|
|
|
|
|
2019-04-17 06:24:02 +00:00
|
|
|
type ipfsPinsResp struct {
|
2019-04-19 03:47:02 +00:00
|
|
|
Pins []string
|
|
|
|
Progress int
|
2019-04-17 06:24:02 +00:00
|
|
|
}
|
|
|
|
|
2017-10-13 21:12:46 +00:00
|
|
|
type ipfsSwarmPeersResp struct {
|
|
|
|
Peers []ipfsPeer
|
|
|
|
}
|
|
|
|
|
2020-03-06 15:35:15 +00:00
|
|
|
type ipfsBlockPutResp struct {
|
Adders: stream blocks to destinations
This commit fixes #810 and adds block streaming to the final destinations when
adding. This should add major performance gains when adding data to clusters.
Before, everytime cluster issued a block, it was broadcasted individually to
all destinations (new libp2p stream), where it was block/put to IPFS (a single
block/put http roundtrip per block).
Now, blocks are streamed all the way from the adder module to the ipfs daemon,
by making every block as it arrives a single part in a multipart block/put
request.
Before, block-broadcast needed to wait for all destinations to finish in order
to process the next block. Now, buffers allow some destinations to be faster
than others while sending and receiving blocks.
Before, if a block put request failed to be broadcasted everywhere, an error
would happen at that moment.
Now, we keep streaming until the end and only then report any errors. The
operation succeeds as long as at least one stream finished successfully.
Errors block/putting to IPFS will not abort streams. Instead, subsequent
blocks are retried with a new request, although the method will return an
error when the stream finishes if there were errors at any point.
2022-03-24 01:17:10 +00:00
|
|
|
Key api.Cid
|
2020-03-06 15:35:15 +00:00
|
|
|
Size int
|
|
|
|
}
|
|
|
|
|
2017-10-13 21:12:46 +00:00
|
|
|
type ipfsPeer struct {
|
2018-01-18 02:49:35 +00:00
|
|
|
Peer string
|
2017-10-13 21:12:46 +00:00
|
|
|
}
|
|
|
|
|
2017-03-14 15:37:29 +00:00
|
|
|
// NewConnector creates the component and leaves it ready to be started
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
func NewConnector(cfg *Config) (*Connector, error) {
|
|
|
|
err := cfg.Validate()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2018-06-12 08:09:03 +00:00
|
|
|
nodeMAddr := cfg.NodeAddr
|
|
|
|
// dns multiaddresses need to be resolved first
|
|
|
|
if madns.Matches(nodeMAddr) {
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), DNSTimeout)
|
|
|
|
defer cancel()
|
|
|
|
resolvedAddrs, err := madns.Resolve(ctx, cfg.NodeAddr)
|
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
nodeMAddr = resolvedAddrs[0]
|
|
|
|
}
|
|
|
|
|
|
|
|
_, nodeAddr, err := manet.DialArgs(nodeMAddr)
|
2017-01-23 17:38:59 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2018-05-24 07:12:05 +00:00
|
|
|
c := &http.Client{} // timeouts are handled by context timeouts
|
2018-06-27 04:03:15 +00:00
|
|
|
if cfg.Tracing {
|
|
|
|
c.Transport = &ochttp.Transport{
|
|
|
|
Base: http.DefaultTransport,
|
|
|
|
Propagation: &tracecontext.HTTPFormat{},
|
|
|
|
StartOptions: trace.StartOptions{SpanKind: trace.SpanKindClient},
|
|
|
|
FormatSpanName: func(req *http.Request) string { return req.Host + ":" + req.URL.Path + ":" + req.Method },
|
|
|
|
NewClientTrace: ochttp.NewSpanAnnotatingClientTrace,
|
|
|
|
}
|
|
|
|
}
|
2018-03-08 10:29:23 +00:00
|
|
|
|
2017-03-02 12:57:37 +00:00
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
|
|
|
2017-03-14 15:37:29 +00:00
|
|
|
ipfs := &Connector{
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
ctx: ctx,
|
|
|
|
config: cfg,
|
|
|
|
cancel: cancel,
|
|
|
|
nodeAddr: nodeAddr,
|
|
|
|
rpcReady: make(chan struct{}, 1),
|
2018-03-08 10:29:23 +00:00
|
|
|
client: c,
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
|
|
|
|
2017-03-23 18:34:33 +00:00
|
|
|
go ipfs.run()
|
2016-12-02 18:33:39 +00:00
|
|
|
return ipfs, nil
|
|
|
|
}
|
|
|
|
|
2018-11-11 15:39:13 +00:00
|
|
|
// connects all ipfs daemons when
|
2017-03-23 18:34:33 +00:00
|
|
|
// we receive the rpcReady signal.
|
2017-03-14 15:37:29 +00:00
|
|
|
func (ipfs *Connector) run() {
|
2017-03-23 18:34:33 +00:00
|
|
|
<-ipfs.rpcReady
|
|
|
|
|
2018-03-08 10:29:23 +00:00
|
|
|
// Do not shutdown while launching threads
|
|
|
|
// -- prevents race conditions with ipfs.wg.
|
|
|
|
ipfs.shutdownLock.Lock()
|
|
|
|
defer ipfs.shutdownLock.Unlock()
|
|
|
|
|
2019-06-07 17:02:27 +00:00
|
|
|
if ipfs.config.ConnectSwarmsDelay == 0 {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2017-03-23 18:34:33 +00:00
|
|
|
// This runs ipfs swarm connect to the daemons of other cluster members
|
|
|
|
ipfs.wg.Add(1)
|
|
|
|
go func() {
|
|
|
|
defer ipfs.wg.Done()
|
|
|
|
|
|
|
|
// It does not hurt to wait a little bit. i.e. think cluster
|
|
|
|
// peers which are started at the same time as the ipfs
|
|
|
|
// daemon...
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
tmr := time.NewTimer(ipfs.config.ConnectSwarmsDelay)
|
2017-03-23 18:34:33 +00:00
|
|
|
defer tmr.Stop()
|
|
|
|
select {
|
|
|
|
case <-tmr.C:
|
2018-01-16 19:57:54 +00:00
|
|
|
// do not hang this goroutine if this call hangs
|
|
|
|
// otherwise we hang during shutdown
|
2018-06-27 04:03:15 +00:00
|
|
|
go ipfs.ConnectSwarms(ipfs.ctx)
|
2017-03-23 18:34:33 +00:00
|
|
|
case <-ipfs.ctx.Done():
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}()
|
2017-01-26 21:49:53 +00:00
|
|
|
}
|
|
|
|
|
2018-10-14 17:12:50 +00:00
|
|
|
// SetClient makes the component ready to perform RPC
|
|
|
|
// requests.
|
|
|
|
func (ipfs *Connector) SetClient(c *rpc.Client) {
|
|
|
|
ipfs.rpcClient = c
|
|
|
|
ipfs.rpcReady <- struct{}{}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Shutdown stops any listeners and stops the component from taking
|
|
|
|
// any requests.
|
2018-06-27 04:03:15 +00:00
|
|
|
func (ipfs *Connector) Shutdown(ctx context.Context) error {
|
2020-04-14 17:58:00 +00:00
|
|
|
_, span := trace.StartSpan(ctx, "ipfsconn/ipfshttp/Shutdown")
|
2018-06-27 04:03:15 +00:00
|
|
|
defer span.End()
|
|
|
|
|
2018-10-14 17:12:50 +00:00
|
|
|
ipfs.shutdownLock.Lock()
|
|
|
|
defer ipfs.shutdownLock.Unlock()
|
|
|
|
|
|
|
|
if ipfs.shutdown {
|
|
|
|
logger.Debug("already shutdown")
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2018-11-12 00:03:07 +00:00
|
|
|
logger.Info("stopping IPFS Connector")
|
|
|
|
|
2018-10-14 17:12:50 +00:00
|
|
|
ipfs.cancel()
|
|
|
|
close(ipfs.rpcReady)
|
|
|
|
|
|
|
|
ipfs.wg.Wait()
|
|
|
|
ipfs.shutdown = true
|
2018-11-01 10:24:05 +00:00
|
|
|
|
2018-10-14 17:12:50 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2017-01-26 18:59:31 +00:00
|
|
|
// ID performs an ID request against the configured
|
|
|
|
// IPFS daemon. It returns the fetched information.
|
|
|
|
// If the request fails, or the parsing fails, it
|
2019-02-27 17:04:35 +00:00
|
|
|
// returns an error.
|
2022-03-19 01:52:46 +00:00
|
|
|
func (ipfs *Connector) ID(ctx context.Context) (api.IPFSID, error) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx, span := trace.StartSpan(ctx, "ipfsconn/ipfshttp/ID")
|
|
|
|
defer span.End()
|
|
|
|
|
|
|
|
ctx, cancel := context.WithTimeout(ctx, ipfs.config.IPFSRequestTimeout)
|
2018-05-24 07:12:05 +00:00
|
|
|
defer cancel()
|
2019-02-27 17:04:35 +00:00
|
|
|
|
2018-06-28 15:01:39 +00:00
|
|
|
body, err := ipfs.postCtx(ctx, "id", "", nil)
|
2017-01-26 18:59:31 +00:00
|
|
|
if err != nil {
|
2022-03-19 01:52:46 +00:00
|
|
|
return api.IPFSID{}, err
|
2017-01-26 18:59:31 +00:00
|
|
|
}
|
|
|
|
|
2017-03-29 20:52:13 +00:00
|
|
|
var res ipfsIDResp
|
|
|
|
err = json.Unmarshal(body, &res)
|
2017-01-26 18:59:31 +00:00
|
|
|
if err != nil {
|
2022-03-19 01:52:46 +00:00
|
|
|
return api.IPFSID{}, err
|
2017-01-26 18:59:31 +00:00
|
|
|
}
|
|
|
|
|
2020-04-14 17:58:00 +00:00
|
|
|
pID, err := peer.Decode(res.ID)
|
2017-01-26 18:59:31 +00:00
|
|
|
if err != nil {
|
2022-03-19 01:52:46 +00:00
|
|
|
return api.IPFSID{}, err
|
2019-02-27 17:04:35 +00:00
|
|
|
}
|
|
|
|
|
2022-03-19 01:52:46 +00:00
|
|
|
id := api.IPFSID{
|
2019-02-27 17:04:35 +00:00
|
|
|
ID: pID,
|
2017-01-26 18:59:31 +00:00
|
|
|
}
|
|
|
|
|
2020-04-14 17:58:00 +00:00
|
|
|
mAddrs := make([]api.Multiaddr, len(res.Addresses))
|
2017-03-29 20:52:13 +00:00
|
|
|
for i, strAddr := range res.Addresses {
|
2019-02-27 17:04:35 +00:00
|
|
|
mAddr, err := api.NewMultiaddr(strAddr)
|
2017-01-26 18:59:31 +00:00
|
|
|
if err != nil {
|
|
|
|
id.Error = err.Error()
|
|
|
|
return id, err
|
|
|
|
}
|
|
|
|
mAddrs[i] = mAddr
|
|
|
|
}
|
|
|
|
id.Addresses = mAddrs
|
|
|
|
return id, nil
|
|
|
|
}
|
|
|
|
|
2020-04-20 13:15:08 +00:00
|
|
|
func pinArgs(maxDepth api.PinDepth) string {
|
2019-05-17 14:26:27 +00:00
|
|
|
q := url.Values{}
|
|
|
|
switch {
|
|
|
|
case maxDepth < 0:
|
|
|
|
q.Set("recursive", "true")
|
|
|
|
case maxDepth == 0:
|
|
|
|
q.Set("recursive", "false")
|
|
|
|
default:
|
|
|
|
q.Set("recursive", "true")
|
2020-04-20 13:15:08 +00:00
|
|
|
q.Set("max-depth", strconv.Itoa(int(maxDepth)))
|
2019-05-17 14:26:27 +00:00
|
|
|
}
|
|
|
|
return q.Encode()
|
|
|
|
}
|
|
|
|
|
2016-12-02 18:33:39 +00:00
|
|
|
// Pin performs a pin request against the configured IPFS
|
|
|
|
// daemon.
|
2022-03-19 01:52:46 +00:00
|
|
|
func (ipfs *Connector) Pin(ctx context.Context, pin api.Pin) error {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx, span := trace.StartSpan(ctx, "ipfsconn/ipfshttp/Pin")
|
|
|
|
defer span.End()
|
|
|
|
|
2019-07-12 14:40:29 +00:00
|
|
|
hash := pin.Cid
|
|
|
|
maxDepth := pin.MaxDepth
|
|
|
|
|
2020-04-20 14:21:27 +00:00
|
|
|
pinStatus, err := ipfs.PinLsCid(ctx, pin)
|
2016-12-05 14:30:11 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2018-03-07 13:51:37 +00:00
|
|
|
|
2018-07-04 16:30:24 +00:00
|
|
|
if pinStatus.IsPinned(maxDepth) {
|
|
|
|
logger.Debug("IPFS object is already pinned: ", hash)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
defer ipfs.updateInformerMetric(ctx)
|
2018-08-07 09:49:31 +00:00
|
|
|
|
2019-05-17 14:26:27 +00:00
|
|
|
ctx, cancelRequest := context.WithCancel(ctx)
|
|
|
|
defer cancelRequest()
|
2018-07-04 16:30:24 +00:00
|
|
|
|
2021-07-02 00:20:39 +00:00
|
|
|
// If the pin has origins, tell ipfs to connect to a maximum of 10.
|
|
|
|
bound := len(pin.Origins)
|
|
|
|
if bound > 10 {
|
|
|
|
bound = 10
|
|
|
|
}
|
|
|
|
for _, orig := range pin.Origins[0:bound] {
|
|
|
|
// do it in the background, ignoring errors.
|
2021-07-02 00:09:27 +00:00
|
|
|
go func(o string) {
|
2021-07-02 00:20:39 +00:00
|
|
|
logger.Debugf("swarm-connect to origin before pinning: %s", o)
|
2021-07-02 00:09:27 +00:00
|
|
|
_, err := ipfs.postCtx(
|
|
|
|
ctx,
|
|
|
|
fmt.Sprintf("swarm/connect?arg=%s", o),
|
|
|
|
"",
|
|
|
|
nil,
|
|
|
|
)
|
|
|
|
if err != nil {
|
|
|
|
logger.Debug(err)
|
|
|
|
return
|
|
|
|
}
|
2021-07-02 00:20:39 +00:00
|
|
|
logger.Debugf("swarm-connect success to origin: %s", o)
|
2021-07-02 00:09:27 +00:00
|
|
|
}(url.QueryEscape(orig.String()))
|
|
|
|
}
|
|
|
|
|
2019-07-12 14:40:29 +00:00
|
|
|
// If we have a pin-update, and the old object
|
|
|
|
// is pinned recursively, then do pin/update.
|
|
|
|
// Otherwise do a normal pin.
|
2022-04-07 11:53:30 +00:00
|
|
|
if from := pin.PinUpdate; from.Defined() {
|
2020-04-20 14:21:27 +00:00
|
|
|
fromPin := api.PinWithOpts(from, pin.PinOptions)
|
|
|
|
pinStatus, _ := ipfs.PinLsCid(ctx, fromPin)
|
2019-08-09 14:45:54 +00:00
|
|
|
if pinStatus.IsPinned(-1) { // pinned recursively.
|
|
|
|
// As a side note, if PinUpdate == pin.Cid, we are
|
|
|
|
// somehow pinning an already pinned thing and we'd
|
|
|
|
// better use update for that
|
|
|
|
return ipfs.pinUpdate(ctx, from, pin.Cid)
|
2016-12-19 17:35:24 +00:00
|
|
|
}
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
2018-07-04 16:30:24 +00:00
|
|
|
|
2019-05-17 14:26:27 +00:00
|
|
|
// Pin request and timeout if there is no progress
|
|
|
|
outPins := make(chan int)
|
|
|
|
go func() {
|
|
|
|
var lastProgress int
|
|
|
|
lastProgressTime := time.Now()
|
|
|
|
|
|
|
|
ticker := time.NewTicker(ipfs.config.PinTimeout)
|
|
|
|
defer ticker.Stop()
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-ticker.C:
|
|
|
|
if time.Since(lastProgressTime) > ipfs.config.PinTimeout {
|
|
|
|
// timeout request
|
|
|
|
cancelRequest()
|
|
|
|
return
|
|
|
|
}
|
|
|
|
case p := <-outPins:
|
|
|
|
// ipfs will send status messages every second
|
|
|
|
// or so but we need make sure there was
|
|
|
|
// progress by looking at number of nodes
|
|
|
|
// fetched.
|
|
|
|
if p > lastProgress {
|
|
|
|
lastProgress = p
|
|
|
|
lastProgressTime = time.Now()
|
|
|
|
}
|
|
|
|
case <-ctx.Done():
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
|
|
|
err = ipfs.pinProgress(ctx, hash, maxDepth, outPins)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2019-04-22 13:54:34 +00:00
|
|
|
|
2019-05-17 14:26:27 +00:00
|
|
|
logger.Info("IPFS Pin request succeeded: ", hash)
|
|
|
|
return nil
|
|
|
|
}
|
2019-04-22 13:54:34 +00:00
|
|
|
|
2019-05-17 14:26:27 +00:00
|
|
|
// pinProgress pins an item and sends fetched node's progress on a
|
|
|
|
// channel. Blocks until done or error. pinProgress will always close the out
|
|
|
|
// channel. pinProgress will not block on sending to the channel if it is full.
|
2022-04-07 11:53:30 +00:00
|
|
|
func (ipfs *Connector) pinProgress(ctx context.Context, hash api.Cid, maxDepth api.PinDepth, out chan<- int) error {
|
2019-05-17 14:26:27 +00:00
|
|
|
defer close(out)
|
2018-07-04 16:30:24 +00:00
|
|
|
|
2019-05-17 14:26:27 +00:00
|
|
|
ctx, span := trace.StartSpan(ctx, "ipfsconn/ipfshttp/pinsProgress")
|
|
|
|
defer span.End()
|
|
|
|
|
|
|
|
pinArgs := pinArgs(maxDepth)
|
2019-04-04 06:42:01 +00:00
|
|
|
path := fmt.Sprintf("pin/add?arg=%s&%s&progress=true", hash, pinArgs)
|
2019-04-22 13:54:34 +00:00
|
|
|
res, err := ipfs.doPostCtx(ctx, ipfs.client, ipfs.apiURL(), path, "", nil)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer res.Body.Close()
|
|
|
|
|
2019-05-17 14:26:27 +00:00
|
|
|
_, err = checkResponse(path, res)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
2018-07-04 16:30:24 +00:00
|
|
|
}
|
2019-04-22 13:54:34 +00:00
|
|
|
|
2019-05-17 14:26:27 +00:00
|
|
|
dec := json.NewDecoder(res.Body)
|
|
|
|
for {
|
|
|
|
var pins ipfsPinsResp
|
|
|
|
if err := dec.Decode(&pins); err != nil {
|
|
|
|
// If we cancelled the request we should tell the user
|
|
|
|
// (in case dec.Decode() exited cleanly with an EOF).
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
return ctx.Err()
|
|
|
|
default:
|
|
|
|
if err == io.EOF {
|
|
|
|
return nil // clean exit. Pinned!
|
|
|
|
}
|
|
|
|
return err // error decoding
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
select {
|
|
|
|
case out <- pins.Progress:
|
|
|
|
default:
|
|
|
|
}
|
2018-07-04 16:30:24 +00:00
|
|
|
}
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
|
|
|
|
2022-04-07 11:53:30 +00:00
|
|
|
func (ipfs *Connector) pinUpdate(ctx context.Context, from, to api.Cid) error {
|
2019-07-12 14:40:29 +00:00
|
|
|
ctx, span := trace.StartSpan(ctx, "ipfsconn/ipfshttp/pinUpdate")
|
|
|
|
defer span.End()
|
|
|
|
|
|
|
|
path := fmt.Sprintf("pin/update?arg=%s&arg=%s&unpin=false", from, to)
|
|
|
|
_, err := ipfs.postCtx(ctx, path, "", nil)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
logger.Infof("IPFS Pin Update request succeeded. %s -> %s (unpin=false)", from, to)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2016-12-28 15:25:24 +00:00
|
|
|
// Unpin performs an unpin request against the configured IPFS
|
2016-12-02 18:33:39 +00:00
|
|
|
// daemon.
|
2022-04-07 11:53:30 +00:00
|
|
|
func (ipfs *Connector) Unpin(ctx context.Context, hash api.Cid) error {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx, span := trace.StartSpan(ctx, "ipfsconn/ipfshttp/Unpin")
|
|
|
|
defer span.End()
|
|
|
|
|
2020-04-20 14:21:27 +00:00
|
|
|
if ipfs.config.UnpinDisable {
|
|
|
|
return errors.New("ipfs unpinning is disallowed by configuration on this peer")
|
2016-12-05 14:30:11 +00:00
|
|
|
}
|
2019-04-22 13:54:34 +00:00
|
|
|
|
2020-04-20 14:21:27 +00:00
|
|
|
defer ipfs.updateInformerMetric(ctx)
|
2019-08-26 12:56:19 +00:00
|
|
|
|
2020-04-20 14:21:27 +00:00
|
|
|
path := fmt.Sprintf("pin/rm?arg=%s", hash)
|
2019-04-22 13:54:34 +00:00
|
|
|
|
2020-04-20 14:21:27 +00:00
|
|
|
ctx, cancel := context.WithTimeout(ctx, ipfs.config.UnpinTimeout)
|
|
|
|
defer cancel()
|
2019-04-22 13:54:34 +00:00
|
|
|
|
2020-04-20 14:21:27 +00:00
|
|
|
// We will call unpin in any case, if the CID is not pinned,
|
|
|
|
// then we ignore the error (although this is a bit flaky).
|
|
|
|
_, err := ipfs.postCtx(ctx, path, "", nil)
|
|
|
|
if err != nil {
|
|
|
|
ipfsErr, ok := err.(ipfsError)
|
2021-08-16 15:52:31 +00:00
|
|
|
if !ok || ipfsErr.Message != ipfspinner.ErrNotPinned.Error() {
|
2018-06-27 04:03:15 +00:00
|
|
|
return err
|
2016-12-19 17:35:24 +00:00
|
|
|
}
|
2020-04-20 14:21:27 +00:00
|
|
|
logger.Debug("IPFS object is already unpinned: ", hash)
|
2020-05-18 20:41:05 +00:00
|
|
|
return nil
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
2016-12-07 16:21:29 +00:00
|
|
|
|
2020-04-20 14:21:27 +00:00
|
|
|
logger.Info("IPFS Unpin request succeeded:", hash)
|
2016-12-02 18:33:39 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2017-02-09 15:29:17 +00:00
|
|
|
// PinLs performs a "pin ls --type typeFilter" request against the configured
|
2022-03-22 09:56:16 +00:00
|
|
|
// IPFS daemon and sends the results on the given channel. Returns when done.
|
|
|
|
func (ipfs *Connector) PinLs(ctx context.Context, typeFilters []string, out chan<- api.IPFSPinInfo) error {
|
|
|
|
defer close(out)
|
|
|
|
bodies := make([]io.ReadCloser, len(typeFilters))
|
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx, span := trace.StartSpan(ctx, "ipfsconn/ipfshttp/PinLs")
|
|
|
|
defer span.End()
|
|
|
|
|
2018-05-24 07:12:05 +00:00
|
|
|
ctx, cancel := context.WithTimeout(ctx, ipfs.config.IPFSRequestTimeout)
|
|
|
|
defer cancel()
|
2017-01-25 17:07:19 +00:00
|
|
|
|
2022-03-22 09:56:16 +00:00
|
|
|
var err error
|
2017-01-25 17:07:19 +00:00
|
|
|
|
2022-03-22 09:56:16 +00:00
|
|
|
nextFilter:
|
|
|
|
for i, typeFilter := range typeFilters {
|
|
|
|
// Post and read streaming response
|
|
|
|
path := "pin/ls?stream=true&type=" + typeFilter
|
|
|
|
bodies[i], err = ipfs.postCtxStreamResponse(ctx, path, "", nil)
|
|
|
|
if err != nil {
|
|
|
|
logger.Error("error querying pinset: %s", err)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer bodies[i].Close()
|
2016-12-07 16:21:29 +00:00
|
|
|
|
2022-03-22 09:56:16 +00:00
|
|
|
dec := json.NewDecoder(bodies[i])
|
|
|
|
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
err = fmt.Errorf("aborting pin/ls operation: %w", ctx.Err())
|
|
|
|
logger.Error(err)
|
|
|
|
return err
|
|
|
|
default:
|
|
|
|
}
|
|
|
|
|
|
|
|
var ipfsPin api.IPFSPinInfo
|
|
|
|
err = dec.Decode(&ipfsPin)
|
|
|
|
if err == io.EOF {
|
|
|
|
break nextFilter
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
err = fmt.Errorf("error decoding ipfs pin: %w", err)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
err = fmt.Errorf("aborting pin/ls operation: %w", ctx.Err())
|
|
|
|
logger.Error(err)
|
|
|
|
return err
|
|
|
|
case out <- ipfsPin:
|
|
|
|
}
|
|
|
|
}
|
2016-12-07 16:21:29 +00:00
|
|
|
}
|
2022-03-22 09:56:16 +00:00
|
|
|
|
|
|
|
return nil
|
2016-12-07 16:21:29 +00:00
|
|
|
}
|
|
|
|
|
2020-04-20 14:21:27 +00:00
|
|
|
// PinLsCid performs a "pin ls <hash>" request. It will use "type=recursive" or
|
|
|
|
// "type=direct" (or other) depending on the given pin's MaxDepth setting.
|
|
|
|
// It returns an api.IPFSPinStatus for that hash.
|
2022-03-19 01:52:46 +00:00
|
|
|
func (ipfs *Connector) PinLsCid(ctx context.Context, pin api.Pin) (api.IPFSPinStatus, error) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx, span := trace.StartSpan(ctx, "ipfsconn/ipfshttp/PinLsCid")
|
|
|
|
defer span.End()
|
|
|
|
|
2020-04-20 14:21:27 +00:00
|
|
|
ctx, cancel := context.WithTimeout(ctx, ipfs.config.IPFSRequestTimeout)
|
|
|
|
defer cancel()
|
2016-12-05 14:30:11 +00:00
|
|
|
|
2022-03-22 09:56:16 +00:00
|
|
|
if !pin.Defined() {
|
|
|
|
return api.IPFSPinStatusBug, errors.New("calling PinLsCid without a defined CID")
|
|
|
|
}
|
|
|
|
|
2020-04-20 14:21:27 +00:00
|
|
|
pinType := pin.MaxDepth.ToPinMode().String()
|
2022-03-22 09:56:16 +00:00
|
|
|
lsPath := fmt.Sprintf("pin/ls?stream=true&arg=%s&type=%s", pin.Cid, pinType)
|
|
|
|
body, err := ipfs.postCtxStreamResponse(ctx, lsPath, "", nil)
|
|
|
|
if err != nil {
|
|
|
|
if errors.Is(ipfsUnpinnedError{}, err) {
|
|
|
|
return api.IPFSPinStatusUnpinned, nil
|
|
|
|
}
|
2020-04-20 14:21:27 +00:00
|
|
|
return api.IPFSPinStatusError, err
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
2022-03-22 09:56:16 +00:00
|
|
|
defer body.Close()
|
2016-12-02 18:33:39 +00:00
|
|
|
|
2022-03-22 09:56:16 +00:00
|
|
|
var res api.IPFSPinInfo
|
|
|
|
dec := json.NewDecoder(body)
|
2016-12-02 18:33:39 +00:00
|
|
|
|
2022-03-22 09:56:16 +00:00
|
|
|
err = dec.Decode(&res)
|
2016-12-02 18:33:39 +00:00
|
|
|
if err != nil {
|
2022-03-22 09:56:16 +00:00
|
|
|
logger.Error("error parsing pin/ls?arg=cid response")
|
2017-02-08 17:04:08 +00:00
|
|
|
return api.IPFSPinStatusError, err
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
2017-01-25 17:07:19 +00:00
|
|
|
|
2022-03-22 09:56:16 +00:00
|
|
|
return res.Type, nil
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
|
|
|
|
2018-02-09 05:23:24 +00:00
|
|
|
func (ipfs *Connector) doPostCtx(ctx context.Context, client *http.Client, apiURL, path string, contentType string, postBody io.Reader) (*http.Response, error) {
|
2022-03-28 18:05:01 +00:00
|
|
|
logger.Debugf("posting /%s", path)
|
2018-04-09 02:49:02 +00:00
|
|
|
urlstr := fmt.Sprintf("%s/%s", apiURL, path)
|
2018-03-08 10:29:23 +00:00
|
|
|
|
2018-02-09 05:23:24 +00:00
|
|
|
req, err := http.NewRequest("POST", urlstr, postBody)
|
2018-04-09 02:49:02 +00:00
|
|
|
if err != nil {
|
|
|
|
logger.Error("error creating POST request:", err)
|
|
|
|
}
|
|
|
|
|
2018-02-09 05:23:24 +00:00
|
|
|
req.Header.Set("Content-Type", contentType)
|
2018-04-09 02:49:02 +00:00
|
|
|
req = req.WithContext(ctx)
|
|
|
|
res, err := ipfs.client.Do(req)
|
2018-03-08 10:29:23 +00:00
|
|
|
if err != nil {
|
|
|
|
logger.Error("error posting to IPFS:", err)
|
|
|
|
}
|
2018-04-09 02:49:02 +00:00
|
|
|
|
2018-03-08 10:29:23 +00:00
|
|
|
return res, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// checkResponse tries to parse an error message on non StatusOK responses
|
|
|
|
// from ipfs.
|
2019-05-17 14:26:27 +00:00
|
|
|
func checkResponse(path string, res *http.Response) ([]byte, error) {
|
|
|
|
if res.StatusCode == http.StatusOK {
|
|
|
|
return nil, nil
|
2018-03-08 10:29:23 +00:00
|
|
|
}
|
|
|
|
|
2019-05-17 14:26:27 +00:00
|
|
|
body, err := ioutil.ReadAll(res.Body)
|
|
|
|
if err == nil {
|
|
|
|
var ipfsErr ipfsError
|
|
|
|
if err := json.Unmarshal(body, &ipfsErr); err == nil {
|
2020-04-20 14:21:27 +00:00
|
|
|
ipfsErr.code = res.StatusCode
|
|
|
|
ipfsErr.path = path
|
|
|
|
return body, ipfsErr
|
2019-04-22 13:54:34 +00:00
|
|
|
}
|
2018-03-08 10:29:23 +00:00
|
|
|
}
|
2019-04-22 13:54:34 +00:00
|
|
|
|
2018-03-08 10:29:23 +00:00
|
|
|
// No error response with useful message from ipfs
|
2019-05-17 14:26:27 +00:00
|
|
|
return nil, fmt.Errorf(
|
2022-03-22 09:56:16 +00:00
|
|
|
"IPFS request failed (is it running?) (%s). Code %d: %s",
|
2019-05-17 14:26:27 +00:00
|
|
|
path,
|
|
|
|
res.StatusCode,
|
|
|
|
string(body))
|
2018-03-08 10:29:23 +00:00
|
|
|
}
|
|
|
|
|
2018-05-24 07:12:05 +00:00
|
|
|
// postCtx makes a POST request against
|
2018-03-08 10:29:23 +00:00
|
|
|
// the ipfs daemon, reads the full body of the response and
|
|
|
|
// returns it after checking for errors.
|
2018-02-09 05:23:24 +00:00
|
|
|
func (ipfs *Connector) postCtx(ctx context.Context, path string, contentType string, postBody io.Reader) ([]byte, error) {
|
2022-03-22 09:56:16 +00:00
|
|
|
rdr, err := ipfs.postCtxStreamResponse(ctx, path, contentType, postBody)
|
2016-12-02 18:33:39 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2022-03-22 09:56:16 +00:00
|
|
|
defer rdr.Close()
|
2019-05-17 14:26:27 +00:00
|
|
|
|
2022-03-22 09:56:16 +00:00
|
|
|
body, err := ioutil.ReadAll(rdr)
|
2019-05-17 14:26:27 +00:00
|
|
|
if err != nil {
|
2022-03-22 09:56:16 +00:00
|
|
|
logger.Errorf("error reading response body: %s", err)
|
|
|
|
return nil, err
|
2019-05-17 14:26:27 +00:00
|
|
|
}
|
2022-03-22 09:56:16 +00:00
|
|
|
return body, nil
|
|
|
|
}
|
2019-05-17 14:26:27 +00:00
|
|
|
|
2022-03-22 09:56:16 +00:00
|
|
|
// postCtxStreamResponse makes a POST request against the ipfs daemon, and
|
|
|
|
// returns the body reader after checking the request for errros.
|
|
|
|
func (ipfs *Connector) postCtxStreamResponse(ctx context.Context, path string, contentType string, postBody io.Reader) (io.ReadCloser, error) {
|
|
|
|
res, err := ipfs.doPostCtx(ctx, ipfs.client, ipfs.apiURL(), path, contentType, postBody)
|
2016-12-02 18:33:39 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2022-03-22 09:56:16 +00:00
|
|
|
|
|
|
|
_, err = checkResponse(path, res)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return res.Body, nil
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// apiURL is a short-hand for building the url of the IPFS
|
|
|
|
// daemon API.
|
2017-03-14 15:37:29 +00:00
|
|
|
func (ipfs *Connector) apiURL() string {
|
2017-03-16 14:51:24 +00:00
|
|
|
return fmt.Sprintf("http://%s/api/v0", ipfs.nodeAddr)
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
2017-03-23 18:34:33 +00:00
|
|
|
|
|
|
|
// ConnectSwarms requests the ipfs addresses of other peers and
|
|
|
|
// triggers ipfs swarm connect requests
|
2018-06-27 04:03:15 +00:00
|
|
|
func (ipfs *Connector) ConnectSwarms(ctx context.Context) error {
|
|
|
|
ctx, span := trace.StartSpan(ctx, "ipfsconn/ipfshttp/ConnectSwarms")
|
|
|
|
defer span.End()
|
|
|
|
|
|
|
|
ctx, cancel := context.WithTimeout(ctx, ipfs.config.IPFSRequestTimeout)
|
2018-05-24 07:12:05 +00:00
|
|
|
defer cancel()
|
2017-03-23 18:34:33 +00:00
|
|
|
|
2022-03-23 00:27:57 +00:00
|
|
|
in := make(chan struct{})
|
|
|
|
close(in)
|
|
|
|
out := make(chan api.ID)
|
|
|
|
go func() {
|
|
|
|
err := ipfs.rpcClient.Stream(
|
|
|
|
ctx,
|
|
|
|
"",
|
|
|
|
"Cluster",
|
|
|
|
"Peers",
|
|
|
|
in,
|
|
|
|
out,
|
|
|
|
)
|
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
|
|
|
for id := range out {
|
2019-02-27 17:04:35 +00:00
|
|
|
ipfsID := id.IPFS
|
2022-01-28 17:21:11 +00:00
|
|
|
if id.Error != "" || ipfsID.Error != "" {
|
2019-03-18 12:52:34 +00:00
|
|
|
continue
|
|
|
|
}
|
2017-03-23 18:34:33 +00:00
|
|
|
for _, addr := range ipfsID.Addresses {
|
|
|
|
// This is a best effort attempt
|
|
|
|
// We ignore errors which happens
|
|
|
|
// when passing in a bunch of addresses
|
2018-02-09 05:23:24 +00:00
|
|
|
_, err := ipfs.postCtx(
|
|
|
|
ctx,
|
2021-07-02 00:09:27 +00:00
|
|
|
fmt.Sprintf("swarm/connect?arg=%s", url.QueryEscape(addr.String())),
|
2018-06-28 15:01:39 +00:00
|
|
|
"",
|
2018-02-09 05:23:24 +00:00
|
|
|
nil,
|
|
|
|
)
|
2017-03-23 18:34:33 +00:00
|
|
|
if err != nil {
|
|
|
|
logger.Debug(err)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
logger.Debugf("ipfs successfully connected to %s", addr)
|
|
|
|
}
|
|
|
|
}
|
2017-03-27 13:07:12 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// ConfigKey fetches the IPFS daemon configuration and retrieves the value for
|
|
|
|
// a given configuration key. For example, "Datastore/StorageMax" will return
|
|
|
|
// the value for StorageMax in the Datastore configuration object.
|
|
|
|
func (ipfs *Connector) ConfigKey(keypath string) (interface{}, error) {
|
2018-05-24 07:12:05 +00:00
|
|
|
ctx, cancel := context.WithTimeout(ipfs.ctx, ipfs.config.IPFSRequestTimeout)
|
|
|
|
defer cancel()
|
2018-06-28 15:01:39 +00:00
|
|
|
res, err := ipfs.postCtx(ctx, "config/show", "", nil)
|
2017-03-27 13:07:12 +00:00
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
var cfg map[string]interface{}
|
2017-03-29 20:52:13 +00:00
|
|
|
err = json.Unmarshal(res, &cfg)
|
2017-03-27 13:07:12 +00:00
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
path := strings.SplitN(keypath, "/", 2)
|
|
|
|
if len(path) == 0 {
|
|
|
|
return nil, errors.New("cannot lookup without a path")
|
|
|
|
}
|
|
|
|
|
|
|
|
return getConfigValue(path, cfg)
|
|
|
|
}
|
|
|
|
|
|
|
|
func getConfigValue(path []string, cfg map[string]interface{}) (interface{}, error) {
|
|
|
|
value, ok := cfg[path[0]]
|
|
|
|
if !ok {
|
|
|
|
return nil, errors.New("key not found in configuration")
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(path) == 1 {
|
|
|
|
return value, nil
|
|
|
|
}
|
|
|
|
|
2020-04-14 17:58:00 +00:00
|
|
|
switch v := value.(type) {
|
2017-03-27 13:07:12 +00:00
|
|
|
case map[string]interface{}:
|
|
|
|
return getConfigValue(path[1:], v)
|
|
|
|
default:
|
|
|
|
return nil, errors.New("invalid path")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-08-20 18:43:27 +00:00
|
|
|
// RepoStat returns the DiskUsage and StorageMax repo/stat values from the
|
|
|
|
// ipfs daemon, in bytes, wrapped as an IPFSRepoStat object.
|
2022-03-19 01:52:46 +00:00
|
|
|
func (ipfs *Connector) RepoStat(ctx context.Context) (api.IPFSRepoStat, error) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx, span := trace.StartSpan(ctx, "ipfsconn/ipfshttp/RepoStat")
|
|
|
|
defer span.End()
|
|
|
|
|
|
|
|
ctx, cancel := context.WithTimeout(ctx, ipfs.config.IPFSRequestTimeout)
|
2018-05-24 07:12:05 +00:00
|
|
|
defer cancel()
|
2018-08-17 14:53:40 +00:00
|
|
|
res, err := ipfs.postCtx(ctx, "repo/stat?size-only=true", "", nil)
|
2017-03-27 13:07:12 +00:00
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
2022-03-19 01:52:46 +00:00
|
|
|
return api.IPFSRepoStat{}, err
|
2017-03-27 13:07:12 +00:00
|
|
|
}
|
|
|
|
|
2018-08-20 18:43:27 +00:00
|
|
|
var stats api.IPFSRepoStat
|
2017-03-29 20:52:13 +00:00
|
|
|
err = json.Unmarshal(res, &stats)
|
2017-03-27 13:07:12 +00:00
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
2022-03-19 01:52:46 +00:00
|
|
|
return api.IPFSRepoStat{}, err
|
2017-03-27 13:07:12 +00:00
|
|
|
}
|
2022-03-19 01:52:46 +00:00
|
|
|
return stats, nil
|
2017-03-23 18:34:33 +00:00
|
|
|
}
|
2017-10-13 21:12:46 +00:00
|
|
|
|
2019-10-22 05:40:32 +00:00
|
|
|
// RepoGC performs a garbage collection sweep on the cluster peer's IPFS repo.
|
2022-03-19 01:52:46 +00:00
|
|
|
func (ipfs *Connector) RepoGC(ctx context.Context) (api.RepoGC, error) {
|
2019-10-22 05:40:32 +00:00
|
|
|
ctx, span := trace.StartSpan(ctx, "ipfsconn/ipfshttp/RepoGC")
|
|
|
|
defer span.End()
|
|
|
|
|
|
|
|
ctx, cancel := context.WithTimeout(ctx, ipfs.config.RepoGCTimeout)
|
|
|
|
defer cancel()
|
|
|
|
|
|
|
|
res, err := ipfs.doPostCtx(ctx, ipfs.client, ipfs.apiURL(), "repo/gc?stream-errors=true", "", nil)
|
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
2022-03-19 01:52:46 +00:00
|
|
|
return api.RepoGC{}, err
|
2019-10-22 05:40:32 +00:00
|
|
|
}
|
|
|
|
defer res.Body.Close()
|
|
|
|
|
|
|
|
dec := json.NewDecoder(res.Body)
|
2022-03-19 01:52:46 +00:00
|
|
|
repoGC := api.RepoGC{
|
2019-10-22 05:40:32 +00:00
|
|
|
Keys: []api.IPFSRepoGC{},
|
|
|
|
}
|
|
|
|
for {
|
|
|
|
resp := ipfsRepoGCResp{}
|
|
|
|
|
|
|
|
if err := dec.Decode(&resp); err != nil {
|
|
|
|
// If we cancelled the request we should tell the user
|
|
|
|
// (in case dec.Decode() exited cleanly with an EOF).
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
return repoGC, ctx.Err()
|
|
|
|
default:
|
|
|
|
if err == io.EOF {
|
|
|
|
return repoGC, nil // clean exit
|
|
|
|
}
|
|
|
|
logger.Error(err)
|
|
|
|
return repoGC, err // error decoding
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-04-07 11:53:30 +00:00
|
|
|
repoGC.Keys = append(repoGC.Keys, api.IPFSRepoGC{Key: api.NewCid(resp.Key), Error: resp.Error})
|
2019-10-22 05:40:32 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
Add PinPath/UnpinPath support.
Squashed commit of the following:
commit 38cf569c6aed77c46ee4e0f8baa4d1a9daf8f03e
Merge: d125f69 aaada42
Author: Hector Sanjuan <hsanjuan@users.noreply.github.com>
Date: Wed Feb 20 11:02:00 2019 +0000
Merge pull request #634 from ipfs/issue_450
Support PinPath, UnpinPath (resolve before pinning)
commit aaada42054e1f1c7b2abb1270859d0de41a0e5d8
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 19 22:16:25 2019 +0530
formatResponse accepts api.Pin and not api.PinSerial
commit b5da4bea045865814cc422da71827b44ddd44b90
Merge: ba59036 cc8dd7e
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 19 21:36:46 2019 +0530
Merge branch 'master' into issue_450
commit ba5903649c1df1dba20f4d6f7e3573d6fe24921f
Merge: f002914 d59880c
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Feb 18 08:41:11 2019 +0530
Merge branch 'issue_450' of github.com:ipfs/ipfs-cluster into issue_450
commit f00291494c0c02621c2296cbb7ac71e4c23aa9ec
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Feb 18 08:31:39 2019 +0530
PinPath: more improvements
Added tracing for new methods
commit d59880c338eaa8214fe06b4f930a540793d78407
Merge: 0ca4c7c b4f0eb3
Author: Hector Sanjuan <hsanjuan@users.noreply.github.com>
Date: Wed Feb 13 15:22:49 2019 +0000
Merge branch 'master' into issue_450
commit 0ca4c7c3b0670ed9c8279f8274d36e3485c10030
Merge: d35017a ecef9ea
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 12 13:10:13 2019 +0530
Merge branch 'master' into issue_450
commit d35017a8de91ca9fc9a9a047c48c75134cee9f98
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 12 13:07:25 2019 +0530
PinPath: more improvements
- Worth having `PinOptions` as a separate field in the struct and
constructing the query in the test with ToQuery()
- sharness: "intialization" line can be placed outside the tests at
the top
commit 68e3b90417ffbad89d41a70ac81d85f9037f8848
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Sun Feb 10 21:43:50 2019 +0530
Using if-continue pattern instead of if-else
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 3c29799f3b85be328b27508332ab92049d8b82f3
Merge: 956790b 4324889
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Thu Feb 7 10:25:52 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 956790b381db9858e4194f983e898b07dc51ba66
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 21:11:20 2019 +0530
Removing resolved path
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 7191cc46cedfbec116a9746937e28881b50ca044
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 16:45:07 2019 +0530
Fix go vet
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit f8b3d5b63b1b7569e2a3e0d82894fd4491c246c4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 16:07:03 2019 +0530
Fixed linting error
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 23c57eb467755a1f21387a1615a7f34e97348053
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 09:20:41 2019 +0530
Fixed tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 0caedd94aefeb3b6649dedc214cb4b849ace2ea4
Merge: 17e555e 5a7ee1d
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 00:07:10 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 17e555e4a7c574413df90aac70c5cc29cab98f54
Author: Hector Sanjuan <code@hector.link>
Date: Tue Feb 5 16:58:50 2019 +0000
PinPath: address some feedback + improvements
* Changed client's Pin() API and PinPath to be consistent
* Added helper methods to turn PinPath to query and back
* Make code and tests build
* Use TestCidResolved everywhere
* Fix cluster.PinPath arguments
* Fix formatting of responses with --no-status
* Make tests readable and call Fatal when needed
* Use a pathTestCases variable
commit f0e7369c47c5ddadc8ed45df5fd2d4d9b2d42b38
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 5 18:34:26 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Addressed review comments as in
https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-198751932
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit a8b4f181d2d7afed32ee41331dfaab19fd66a173
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 29 22:41:27 2019 +0530
Fixing tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit e39b95ca19e4d75506f4f492678245ef13936a44
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 29 14:52:53 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- PinPath and UnpinPath should return api.Pin
- PinPath should accept pin options
- Removing duplicate logic for Resolve from cluster
- And many other review comments https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-195509504
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit d146075126320896665ba58d337a13789f68ea86
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 23 17:08:41 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
PinPath(in both rest and rpc) should return a serializable struct in the
form `{"\":"Q...cid..string..."}` (as used in "github.com/ipfs/go-cid"
to marshal and unmarshal)
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 1f4869568a8adb450275257154ea3a26d03a30f3
Merge: 7acfd28 a244af9
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 23 07:18:56 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 7acfd282732ddf2282a67d4f9d0170a494eb3ed4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 22 18:14:32 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- RPC must always use serializable structs
- In command, just use pin with path as cid is also a valid path
- Addressing many other small review comments as in
https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-192122534
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 36905041e1e3f0b204942030aab3ab7b5b9e4d62
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 16 09:36:42 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Extra logic for path checking should go into resolve so that it can be
properly reused
Added sharness tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 9116bda3534e77bb391d873051bb520a1b01a326
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 16 08:08:07 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
error strings should not be capitalized
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit ca7e61861374f456300a85ddc0374e594f74f963
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 15 23:40:25 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Tests
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 522fbcd899f01c01680375561a32a87464157c0a
Merge: f1a56ab f7bc468
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 15 10:40:54 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit f1a56ab925fb74c0c44273a4524afa4843cf757f
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Jan 14 20:58:17 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- IPFS Connector should act as a pure IPFS client, any extra logic
should go to cluster.go
- Use cid.Undef, instead of cid.Cid{}
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit c83b91054f6774f1f9d4930cfc3f1fa28236f57c
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Thu Jan 10 08:57:17 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- Separate handlers, methods and rpc apis for PinPath and UnpinPath from
Pin and Unpin
- Support ipld paths as well
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 719dff88129366ce3ccb5e04cb6f8082a0915c5c
Merge: 91ceb47 21170c4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 9 19:38:35 2019 +0530
Merge branch 'issue_450_old' into HEAD
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 91ceb4796259ca7ef2974ec43e6a278a12796b13
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 9 19:36:41 2019 +0530
Revert "WIP: Figure out why test does not impleme"
This reverts commit 28a3a3f25dce6f296c8cbef86221644c099a7e75.
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 28a3a3f25dce6f296c8cbef86221644c099a7e75
Author: cd10012 <ced361@nyu.edu>
Date: Tue Jul 24 23:23:10 2018 -0400
WIP: Figure out why test does not implement IPFSConnector interface...
License: MIT
Signed-off-by: cd10012 <ced361@nyu.edu>
commit 21170c48e77e69583db64544b08120a9baf40d8d
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 8 10:37:59 2019 +0530
Support PinPath, UnpinPath (resolve before pinning)
This commit adds API support for pinning using path
`POST /pins/<ipfs or ipns path>` and `DELETE /pins/<ipfs or ipns path>`
will resolve the path into a cid and perform perform pinning or
unpinning
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Co-authored-by: Hector Sanjuan <hector@protocol.ai>
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2019-02-20 11:07:50 +00:00
|
|
|
// Resolve accepts ipfs or ipns path and resolves it into a cid
|
2022-04-07 11:53:30 +00:00
|
|
|
func (ipfs *Connector) Resolve(ctx context.Context, path string) (api.Cid, error) {
|
Add PinPath/UnpinPath support.
Squashed commit of the following:
commit 38cf569c6aed77c46ee4e0f8baa4d1a9daf8f03e
Merge: d125f69 aaada42
Author: Hector Sanjuan <hsanjuan@users.noreply.github.com>
Date: Wed Feb 20 11:02:00 2019 +0000
Merge pull request #634 from ipfs/issue_450
Support PinPath, UnpinPath (resolve before pinning)
commit aaada42054e1f1c7b2abb1270859d0de41a0e5d8
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 19 22:16:25 2019 +0530
formatResponse accepts api.Pin and not api.PinSerial
commit b5da4bea045865814cc422da71827b44ddd44b90
Merge: ba59036 cc8dd7e
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 19 21:36:46 2019 +0530
Merge branch 'master' into issue_450
commit ba5903649c1df1dba20f4d6f7e3573d6fe24921f
Merge: f002914 d59880c
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Feb 18 08:41:11 2019 +0530
Merge branch 'issue_450' of github.com:ipfs/ipfs-cluster into issue_450
commit f00291494c0c02621c2296cbb7ac71e4c23aa9ec
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Feb 18 08:31:39 2019 +0530
PinPath: more improvements
Added tracing for new methods
commit d59880c338eaa8214fe06b4f930a540793d78407
Merge: 0ca4c7c b4f0eb3
Author: Hector Sanjuan <hsanjuan@users.noreply.github.com>
Date: Wed Feb 13 15:22:49 2019 +0000
Merge branch 'master' into issue_450
commit 0ca4c7c3b0670ed9c8279f8274d36e3485c10030
Merge: d35017a ecef9ea
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 12 13:10:13 2019 +0530
Merge branch 'master' into issue_450
commit d35017a8de91ca9fc9a9a047c48c75134cee9f98
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 12 13:07:25 2019 +0530
PinPath: more improvements
- Worth having `PinOptions` as a separate field in the struct and
constructing the query in the test with ToQuery()
- sharness: "intialization" line can be placed outside the tests at
the top
commit 68e3b90417ffbad89d41a70ac81d85f9037f8848
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Sun Feb 10 21:43:50 2019 +0530
Using if-continue pattern instead of if-else
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 3c29799f3b85be328b27508332ab92049d8b82f3
Merge: 956790b 4324889
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Thu Feb 7 10:25:52 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 956790b381db9858e4194f983e898b07dc51ba66
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 21:11:20 2019 +0530
Removing resolved path
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 7191cc46cedfbec116a9746937e28881b50ca044
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 16:45:07 2019 +0530
Fix go vet
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit f8b3d5b63b1b7569e2a3e0d82894fd4491c246c4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 16:07:03 2019 +0530
Fixed linting error
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 23c57eb467755a1f21387a1615a7f34e97348053
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 09:20:41 2019 +0530
Fixed tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 0caedd94aefeb3b6649dedc214cb4b849ace2ea4
Merge: 17e555e 5a7ee1d
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 00:07:10 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 17e555e4a7c574413df90aac70c5cc29cab98f54
Author: Hector Sanjuan <code@hector.link>
Date: Tue Feb 5 16:58:50 2019 +0000
PinPath: address some feedback + improvements
* Changed client's Pin() API and PinPath to be consistent
* Added helper methods to turn PinPath to query and back
* Make code and tests build
* Use TestCidResolved everywhere
* Fix cluster.PinPath arguments
* Fix formatting of responses with --no-status
* Make tests readable and call Fatal when needed
* Use a pathTestCases variable
commit f0e7369c47c5ddadc8ed45df5fd2d4d9b2d42b38
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 5 18:34:26 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Addressed review comments as in
https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-198751932
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit a8b4f181d2d7afed32ee41331dfaab19fd66a173
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 29 22:41:27 2019 +0530
Fixing tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit e39b95ca19e4d75506f4f492678245ef13936a44
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 29 14:52:53 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- PinPath and UnpinPath should return api.Pin
- PinPath should accept pin options
- Removing duplicate logic for Resolve from cluster
- And many other review comments https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-195509504
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit d146075126320896665ba58d337a13789f68ea86
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 23 17:08:41 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
PinPath(in both rest and rpc) should return a serializable struct in the
form `{"\":"Q...cid..string..."}` (as used in "github.com/ipfs/go-cid"
to marshal and unmarshal)
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 1f4869568a8adb450275257154ea3a26d03a30f3
Merge: 7acfd28 a244af9
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 23 07:18:56 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 7acfd282732ddf2282a67d4f9d0170a494eb3ed4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 22 18:14:32 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- RPC must always use serializable structs
- In command, just use pin with path as cid is also a valid path
- Addressing many other small review comments as in
https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-192122534
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 36905041e1e3f0b204942030aab3ab7b5b9e4d62
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 16 09:36:42 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Extra logic for path checking should go into resolve so that it can be
properly reused
Added sharness tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 9116bda3534e77bb391d873051bb520a1b01a326
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 16 08:08:07 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
error strings should not be capitalized
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit ca7e61861374f456300a85ddc0374e594f74f963
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 15 23:40:25 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Tests
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 522fbcd899f01c01680375561a32a87464157c0a
Merge: f1a56ab f7bc468
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 15 10:40:54 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit f1a56ab925fb74c0c44273a4524afa4843cf757f
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Jan 14 20:58:17 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- IPFS Connector should act as a pure IPFS client, any extra logic
should go to cluster.go
- Use cid.Undef, instead of cid.Cid{}
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit c83b91054f6774f1f9d4930cfc3f1fa28236f57c
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Thu Jan 10 08:57:17 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- Separate handlers, methods and rpc apis for PinPath and UnpinPath from
Pin and Unpin
- Support ipld paths as well
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 719dff88129366ce3ccb5e04cb6f8082a0915c5c
Merge: 91ceb47 21170c4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 9 19:38:35 2019 +0530
Merge branch 'issue_450_old' into HEAD
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 91ceb4796259ca7ef2974ec43e6a278a12796b13
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 9 19:36:41 2019 +0530
Revert "WIP: Figure out why test does not impleme"
This reverts commit 28a3a3f25dce6f296c8cbef86221644c099a7e75.
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 28a3a3f25dce6f296c8cbef86221644c099a7e75
Author: cd10012 <ced361@nyu.edu>
Date: Tue Jul 24 23:23:10 2018 -0400
WIP: Figure out why test does not implement IPFSConnector interface...
License: MIT
Signed-off-by: cd10012 <ced361@nyu.edu>
commit 21170c48e77e69583db64544b08120a9baf40d8d
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 8 10:37:59 2019 +0530
Support PinPath, UnpinPath (resolve before pinning)
This commit adds API support for pinning using path
`POST /pins/<ipfs or ipns path>` and `DELETE /pins/<ipfs or ipns path>`
will resolve the path into a cid and perform perform pinning or
unpinning
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Co-authored-by: Hector Sanjuan <hector@protocol.ai>
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2019-02-20 11:07:50 +00:00
|
|
|
ctx, span := trace.StartSpan(ctx, "ipfsconn/ipfshttp/Resolve")
|
|
|
|
defer span.End()
|
|
|
|
|
|
|
|
validPath, err := gopath.ParsePath(path)
|
|
|
|
if err != nil {
|
|
|
|
logger.Error("could not parse path: " + err.Error())
|
2022-04-07 11:53:30 +00:00
|
|
|
return api.CidUndef, err
|
Add PinPath/UnpinPath support.
Squashed commit of the following:
commit 38cf569c6aed77c46ee4e0f8baa4d1a9daf8f03e
Merge: d125f69 aaada42
Author: Hector Sanjuan <hsanjuan@users.noreply.github.com>
Date: Wed Feb 20 11:02:00 2019 +0000
Merge pull request #634 from ipfs/issue_450
Support PinPath, UnpinPath (resolve before pinning)
commit aaada42054e1f1c7b2abb1270859d0de41a0e5d8
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 19 22:16:25 2019 +0530
formatResponse accepts api.Pin and not api.PinSerial
commit b5da4bea045865814cc422da71827b44ddd44b90
Merge: ba59036 cc8dd7e
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 19 21:36:46 2019 +0530
Merge branch 'master' into issue_450
commit ba5903649c1df1dba20f4d6f7e3573d6fe24921f
Merge: f002914 d59880c
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Feb 18 08:41:11 2019 +0530
Merge branch 'issue_450' of github.com:ipfs/ipfs-cluster into issue_450
commit f00291494c0c02621c2296cbb7ac71e4c23aa9ec
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Feb 18 08:31:39 2019 +0530
PinPath: more improvements
Added tracing for new methods
commit d59880c338eaa8214fe06b4f930a540793d78407
Merge: 0ca4c7c b4f0eb3
Author: Hector Sanjuan <hsanjuan@users.noreply.github.com>
Date: Wed Feb 13 15:22:49 2019 +0000
Merge branch 'master' into issue_450
commit 0ca4c7c3b0670ed9c8279f8274d36e3485c10030
Merge: d35017a ecef9ea
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 12 13:10:13 2019 +0530
Merge branch 'master' into issue_450
commit d35017a8de91ca9fc9a9a047c48c75134cee9f98
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 12 13:07:25 2019 +0530
PinPath: more improvements
- Worth having `PinOptions` as a separate field in the struct and
constructing the query in the test with ToQuery()
- sharness: "intialization" line can be placed outside the tests at
the top
commit 68e3b90417ffbad89d41a70ac81d85f9037f8848
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Sun Feb 10 21:43:50 2019 +0530
Using if-continue pattern instead of if-else
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 3c29799f3b85be328b27508332ab92049d8b82f3
Merge: 956790b 4324889
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Thu Feb 7 10:25:52 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 956790b381db9858e4194f983e898b07dc51ba66
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 21:11:20 2019 +0530
Removing resolved path
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 7191cc46cedfbec116a9746937e28881b50ca044
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 16:45:07 2019 +0530
Fix go vet
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit f8b3d5b63b1b7569e2a3e0d82894fd4491c246c4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 16:07:03 2019 +0530
Fixed linting error
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 23c57eb467755a1f21387a1615a7f34e97348053
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 09:20:41 2019 +0530
Fixed tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 0caedd94aefeb3b6649dedc214cb4b849ace2ea4
Merge: 17e555e 5a7ee1d
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 00:07:10 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 17e555e4a7c574413df90aac70c5cc29cab98f54
Author: Hector Sanjuan <code@hector.link>
Date: Tue Feb 5 16:58:50 2019 +0000
PinPath: address some feedback + improvements
* Changed client's Pin() API and PinPath to be consistent
* Added helper methods to turn PinPath to query and back
* Make code and tests build
* Use TestCidResolved everywhere
* Fix cluster.PinPath arguments
* Fix formatting of responses with --no-status
* Make tests readable and call Fatal when needed
* Use a pathTestCases variable
commit f0e7369c47c5ddadc8ed45df5fd2d4d9b2d42b38
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 5 18:34:26 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Addressed review comments as in
https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-198751932
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit a8b4f181d2d7afed32ee41331dfaab19fd66a173
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 29 22:41:27 2019 +0530
Fixing tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit e39b95ca19e4d75506f4f492678245ef13936a44
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 29 14:52:53 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- PinPath and UnpinPath should return api.Pin
- PinPath should accept pin options
- Removing duplicate logic for Resolve from cluster
- And many other review comments https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-195509504
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit d146075126320896665ba58d337a13789f68ea86
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 23 17:08:41 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
PinPath(in both rest and rpc) should return a serializable struct in the
form `{"\":"Q...cid..string..."}` (as used in "github.com/ipfs/go-cid"
to marshal and unmarshal)
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 1f4869568a8adb450275257154ea3a26d03a30f3
Merge: 7acfd28 a244af9
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 23 07:18:56 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 7acfd282732ddf2282a67d4f9d0170a494eb3ed4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 22 18:14:32 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- RPC must always use serializable structs
- In command, just use pin with path as cid is also a valid path
- Addressing many other small review comments as in
https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-192122534
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 36905041e1e3f0b204942030aab3ab7b5b9e4d62
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 16 09:36:42 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Extra logic for path checking should go into resolve so that it can be
properly reused
Added sharness tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 9116bda3534e77bb391d873051bb520a1b01a326
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 16 08:08:07 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
error strings should not be capitalized
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit ca7e61861374f456300a85ddc0374e594f74f963
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 15 23:40:25 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Tests
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 522fbcd899f01c01680375561a32a87464157c0a
Merge: f1a56ab f7bc468
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 15 10:40:54 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit f1a56ab925fb74c0c44273a4524afa4843cf757f
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Jan 14 20:58:17 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- IPFS Connector should act as a pure IPFS client, any extra logic
should go to cluster.go
- Use cid.Undef, instead of cid.Cid{}
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit c83b91054f6774f1f9d4930cfc3f1fa28236f57c
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Thu Jan 10 08:57:17 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- Separate handlers, methods and rpc apis for PinPath and UnpinPath from
Pin and Unpin
- Support ipld paths as well
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 719dff88129366ce3ccb5e04cb6f8082a0915c5c
Merge: 91ceb47 21170c4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 9 19:38:35 2019 +0530
Merge branch 'issue_450_old' into HEAD
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 91ceb4796259ca7ef2974ec43e6a278a12796b13
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 9 19:36:41 2019 +0530
Revert "WIP: Figure out why test does not impleme"
This reverts commit 28a3a3f25dce6f296c8cbef86221644c099a7e75.
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 28a3a3f25dce6f296c8cbef86221644c099a7e75
Author: cd10012 <ced361@nyu.edu>
Date: Tue Jul 24 23:23:10 2018 -0400
WIP: Figure out why test does not implement IPFSConnector interface...
License: MIT
Signed-off-by: cd10012 <ced361@nyu.edu>
commit 21170c48e77e69583db64544b08120a9baf40d8d
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 8 10:37:59 2019 +0530
Support PinPath, UnpinPath (resolve before pinning)
This commit adds API support for pinning using path
`POST /pins/<ipfs or ipns path>` and `DELETE /pins/<ipfs or ipns path>`
will resolve the path into a cid and perform perform pinning or
unpinning
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Co-authored-by: Hector Sanjuan <hector@protocol.ai>
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2019-02-20 11:07:50 +00:00
|
|
|
}
|
|
|
|
if !strings.HasPrefix(path, "/ipns") && validPath.IsJustAKey() {
|
|
|
|
ci, _, err := gopath.SplitAbsPath(validPath)
|
2022-04-07 11:53:30 +00:00
|
|
|
return api.NewCid(ci), err
|
Add PinPath/UnpinPath support.
Squashed commit of the following:
commit 38cf569c6aed77c46ee4e0f8baa4d1a9daf8f03e
Merge: d125f69 aaada42
Author: Hector Sanjuan <hsanjuan@users.noreply.github.com>
Date: Wed Feb 20 11:02:00 2019 +0000
Merge pull request #634 from ipfs/issue_450
Support PinPath, UnpinPath (resolve before pinning)
commit aaada42054e1f1c7b2abb1270859d0de41a0e5d8
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 19 22:16:25 2019 +0530
formatResponse accepts api.Pin and not api.PinSerial
commit b5da4bea045865814cc422da71827b44ddd44b90
Merge: ba59036 cc8dd7e
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 19 21:36:46 2019 +0530
Merge branch 'master' into issue_450
commit ba5903649c1df1dba20f4d6f7e3573d6fe24921f
Merge: f002914 d59880c
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Feb 18 08:41:11 2019 +0530
Merge branch 'issue_450' of github.com:ipfs/ipfs-cluster into issue_450
commit f00291494c0c02621c2296cbb7ac71e4c23aa9ec
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Feb 18 08:31:39 2019 +0530
PinPath: more improvements
Added tracing for new methods
commit d59880c338eaa8214fe06b4f930a540793d78407
Merge: 0ca4c7c b4f0eb3
Author: Hector Sanjuan <hsanjuan@users.noreply.github.com>
Date: Wed Feb 13 15:22:49 2019 +0000
Merge branch 'master' into issue_450
commit 0ca4c7c3b0670ed9c8279f8274d36e3485c10030
Merge: d35017a ecef9ea
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 12 13:10:13 2019 +0530
Merge branch 'master' into issue_450
commit d35017a8de91ca9fc9a9a047c48c75134cee9f98
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 12 13:07:25 2019 +0530
PinPath: more improvements
- Worth having `PinOptions` as a separate field in the struct and
constructing the query in the test with ToQuery()
- sharness: "intialization" line can be placed outside the tests at
the top
commit 68e3b90417ffbad89d41a70ac81d85f9037f8848
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Sun Feb 10 21:43:50 2019 +0530
Using if-continue pattern instead of if-else
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 3c29799f3b85be328b27508332ab92049d8b82f3
Merge: 956790b 4324889
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Thu Feb 7 10:25:52 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 956790b381db9858e4194f983e898b07dc51ba66
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 21:11:20 2019 +0530
Removing resolved path
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 7191cc46cedfbec116a9746937e28881b50ca044
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 16:45:07 2019 +0530
Fix go vet
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit f8b3d5b63b1b7569e2a3e0d82894fd4491c246c4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 16:07:03 2019 +0530
Fixed linting error
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 23c57eb467755a1f21387a1615a7f34e97348053
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 09:20:41 2019 +0530
Fixed tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 0caedd94aefeb3b6649dedc214cb4b849ace2ea4
Merge: 17e555e 5a7ee1d
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 00:07:10 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 17e555e4a7c574413df90aac70c5cc29cab98f54
Author: Hector Sanjuan <code@hector.link>
Date: Tue Feb 5 16:58:50 2019 +0000
PinPath: address some feedback + improvements
* Changed client's Pin() API and PinPath to be consistent
* Added helper methods to turn PinPath to query and back
* Make code and tests build
* Use TestCidResolved everywhere
* Fix cluster.PinPath arguments
* Fix formatting of responses with --no-status
* Make tests readable and call Fatal when needed
* Use a pathTestCases variable
commit f0e7369c47c5ddadc8ed45df5fd2d4d9b2d42b38
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 5 18:34:26 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Addressed review comments as in
https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-198751932
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit a8b4f181d2d7afed32ee41331dfaab19fd66a173
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 29 22:41:27 2019 +0530
Fixing tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit e39b95ca19e4d75506f4f492678245ef13936a44
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 29 14:52:53 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- PinPath and UnpinPath should return api.Pin
- PinPath should accept pin options
- Removing duplicate logic for Resolve from cluster
- And many other review comments https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-195509504
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit d146075126320896665ba58d337a13789f68ea86
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 23 17:08:41 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
PinPath(in both rest and rpc) should return a serializable struct in the
form `{"\":"Q...cid..string..."}` (as used in "github.com/ipfs/go-cid"
to marshal and unmarshal)
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 1f4869568a8adb450275257154ea3a26d03a30f3
Merge: 7acfd28 a244af9
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 23 07:18:56 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 7acfd282732ddf2282a67d4f9d0170a494eb3ed4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 22 18:14:32 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- RPC must always use serializable structs
- In command, just use pin with path as cid is also a valid path
- Addressing many other small review comments as in
https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-192122534
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 36905041e1e3f0b204942030aab3ab7b5b9e4d62
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 16 09:36:42 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Extra logic for path checking should go into resolve so that it can be
properly reused
Added sharness tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 9116bda3534e77bb391d873051bb520a1b01a326
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 16 08:08:07 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
error strings should not be capitalized
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit ca7e61861374f456300a85ddc0374e594f74f963
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 15 23:40:25 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Tests
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 522fbcd899f01c01680375561a32a87464157c0a
Merge: f1a56ab f7bc468
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 15 10:40:54 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit f1a56ab925fb74c0c44273a4524afa4843cf757f
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Jan 14 20:58:17 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- IPFS Connector should act as a pure IPFS client, any extra logic
should go to cluster.go
- Use cid.Undef, instead of cid.Cid{}
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit c83b91054f6774f1f9d4930cfc3f1fa28236f57c
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Thu Jan 10 08:57:17 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- Separate handlers, methods and rpc apis for PinPath and UnpinPath from
Pin and Unpin
- Support ipld paths as well
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 719dff88129366ce3ccb5e04cb6f8082a0915c5c
Merge: 91ceb47 21170c4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 9 19:38:35 2019 +0530
Merge branch 'issue_450_old' into HEAD
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 91ceb4796259ca7ef2974ec43e6a278a12796b13
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 9 19:36:41 2019 +0530
Revert "WIP: Figure out why test does not impleme"
This reverts commit 28a3a3f25dce6f296c8cbef86221644c099a7e75.
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 28a3a3f25dce6f296c8cbef86221644c099a7e75
Author: cd10012 <ced361@nyu.edu>
Date: Tue Jul 24 23:23:10 2018 -0400
WIP: Figure out why test does not implement IPFSConnector interface...
License: MIT
Signed-off-by: cd10012 <ced361@nyu.edu>
commit 21170c48e77e69583db64544b08120a9baf40d8d
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 8 10:37:59 2019 +0530
Support PinPath, UnpinPath (resolve before pinning)
This commit adds API support for pinning using path
`POST /pins/<ipfs or ipns path>` and `DELETE /pins/<ipfs or ipns path>`
will resolve the path into a cid and perform perform pinning or
unpinning
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Co-authored-by: Hector Sanjuan <hector@protocol.ai>
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2019-02-20 11:07:50 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
ctx, cancel := context.WithTimeout(ctx, ipfs.config.IPFSRequestTimeout)
|
|
|
|
defer cancel()
|
|
|
|
res, err := ipfs.postCtx(ctx, "resolve?arg="+url.QueryEscape(path), "", nil)
|
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
2022-04-07 11:53:30 +00:00
|
|
|
return api.CidUndef, err
|
Add PinPath/UnpinPath support.
Squashed commit of the following:
commit 38cf569c6aed77c46ee4e0f8baa4d1a9daf8f03e
Merge: d125f69 aaada42
Author: Hector Sanjuan <hsanjuan@users.noreply.github.com>
Date: Wed Feb 20 11:02:00 2019 +0000
Merge pull request #634 from ipfs/issue_450
Support PinPath, UnpinPath (resolve before pinning)
commit aaada42054e1f1c7b2abb1270859d0de41a0e5d8
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 19 22:16:25 2019 +0530
formatResponse accepts api.Pin and not api.PinSerial
commit b5da4bea045865814cc422da71827b44ddd44b90
Merge: ba59036 cc8dd7e
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 19 21:36:46 2019 +0530
Merge branch 'master' into issue_450
commit ba5903649c1df1dba20f4d6f7e3573d6fe24921f
Merge: f002914 d59880c
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Feb 18 08:41:11 2019 +0530
Merge branch 'issue_450' of github.com:ipfs/ipfs-cluster into issue_450
commit f00291494c0c02621c2296cbb7ac71e4c23aa9ec
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Feb 18 08:31:39 2019 +0530
PinPath: more improvements
Added tracing for new methods
commit d59880c338eaa8214fe06b4f930a540793d78407
Merge: 0ca4c7c b4f0eb3
Author: Hector Sanjuan <hsanjuan@users.noreply.github.com>
Date: Wed Feb 13 15:22:49 2019 +0000
Merge branch 'master' into issue_450
commit 0ca4c7c3b0670ed9c8279f8274d36e3485c10030
Merge: d35017a ecef9ea
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 12 13:10:13 2019 +0530
Merge branch 'master' into issue_450
commit d35017a8de91ca9fc9a9a047c48c75134cee9f98
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 12 13:07:25 2019 +0530
PinPath: more improvements
- Worth having `PinOptions` as a separate field in the struct and
constructing the query in the test with ToQuery()
- sharness: "intialization" line can be placed outside the tests at
the top
commit 68e3b90417ffbad89d41a70ac81d85f9037f8848
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Sun Feb 10 21:43:50 2019 +0530
Using if-continue pattern instead of if-else
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 3c29799f3b85be328b27508332ab92049d8b82f3
Merge: 956790b 4324889
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Thu Feb 7 10:25:52 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 956790b381db9858e4194f983e898b07dc51ba66
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 21:11:20 2019 +0530
Removing resolved path
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 7191cc46cedfbec116a9746937e28881b50ca044
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 16:45:07 2019 +0530
Fix go vet
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit f8b3d5b63b1b7569e2a3e0d82894fd4491c246c4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 16:07:03 2019 +0530
Fixed linting error
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 23c57eb467755a1f21387a1615a7f34e97348053
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 09:20:41 2019 +0530
Fixed tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 0caedd94aefeb3b6649dedc214cb4b849ace2ea4
Merge: 17e555e 5a7ee1d
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 00:07:10 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 17e555e4a7c574413df90aac70c5cc29cab98f54
Author: Hector Sanjuan <code@hector.link>
Date: Tue Feb 5 16:58:50 2019 +0000
PinPath: address some feedback + improvements
* Changed client's Pin() API and PinPath to be consistent
* Added helper methods to turn PinPath to query and back
* Make code and tests build
* Use TestCidResolved everywhere
* Fix cluster.PinPath arguments
* Fix formatting of responses with --no-status
* Make tests readable and call Fatal when needed
* Use a pathTestCases variable
commit f0e7369c47c5ddadc8ed45df5fd2d4d9b2d42b38
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 5 18:34:26 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Addressed review comments as in
https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-198751932
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit a8b4f181d2d7afed32ee41331dfaab19fd66a173
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 29 22:41:27 2019 +0530
Fixing tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit e39b95ca19e4d75506f4f492678245ef13936a44
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 29 14:52:53 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- PinPath and UnpinPath should return api.Pin
- PinPath should accept pin options
- Removing duplicate logic for Resolve from cluster
- And many other review comments https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-195509504
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit d146075126320896665ba58d337a13789f68ea86
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 23 17:08:41 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
PinPath(in both rest and rpc) should return a serializable struct in the
form `{"\":"Q...cid..string..."}` (as used in "github.com/ipfs/go-cid"
to marshal and unmarshal)
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 1f4869568a8adb450275257154ea3a26d03a30f3
Merge: 7acfd28 a244af9
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 23 07:18:56 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 7acfd282732ddf2282a67d4f9d0170a494eb3ed4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 22 18:14:32 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- RPC must always use serializable structs
- In command, just use pin with path as cid is also a valid path
- Addressing many other small review comments as in
https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-192122534
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 36905041e1e3f0b204942030aab3ab7b5b9e4d62
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 16 09:36:42 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Extra logic for path checking should go into resolve so that it can be
properly reused
Added sharness tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 9116bda3534e77bb391d873051bb520a1b01a326
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 16 08:08:07 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
error strings should not be capitalized
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit ca7e61861374f456300a85ddc0374e594f74f963
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 15 23:40:25 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Tests
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 522fbcd899f01c01680375561a32a87464157c0a
Merge: f1a56ab f7bc468
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 15 10:40:54 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit f1a56ab925fb74c0c44273a4524afa4843cf757f
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Jan 14 20:58:17 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- IPFS Connector should act as a pure IPFS client, any extra logic
should go to cluster.go
- Use cid.Undef, instead of cid.Cid{}
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit c83b91054f6774f1f9d4930cfc3f1fa28236f57c
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Thu Jan 10 08:57:17 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- Separate handlers, methods and rpc apis for PinPath and UnpinPath from
Pin and Unpin
- Support ipld paths as well
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 719dff88129366ce3ccb5e04cb6f8082a0915c5c
Merge: 91ceb47 21170c4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 9 19:38:35 2019 +0530
Merge branch 'issue_450_old' into HEAD
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 91ceb4796259ca7ef2974ec43e6a278a12796b13
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 9 19:36:41 2019 +0530
Revert "WIP: Figure out why test does not impleme"
This reverts commit 28a3a3f25dce6f296c8cbef86221644c099a7e75.
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 28a3a3f25dce6f296c8cbef86221644c099a7e75
Author: cd10012 <ced361@nyu.edu>
Date: Tue Jul 24 23:23:10 2018 -0400
WIP: Figure out why test does not implement IPFSConnector interface...
License: MIT
Signed-off-by: cd10012 <ced361@nyu.edu>
commit 21170c48e77e69583db64544b08120a9baf40d8d
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 8 10:37:59 2019 +0530
Support PinPath, UnpinPath (resolve before pinning)
This commit adds API support for pinning using path
`POST /pins/<ipfs or ipns path>` and `DELETE /pins/<ipfs or ipns path>`
will resolve the path into a cid and perform perform pinning or
unpinning
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Co-authored-by: Hector Sanjuan <hector@protocol.ai>
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2019-02-20 11:07:50 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
var resp ipfsResolveResp
|
|
|
|
err = json.Unmarshal(res, &resp)
|
|
|
|
if err != nil {
|
|
|
|
logger.Error("could not unmarshal response: " + err.Error())
|
2022-04-07 11:53:30 +00:00
|
|
|
return api.CidUndef, err
|
Add PinPath/UnpinPath support.
Squashed commit of the following:
commit 38cf569c6aed77c46ee4e0f8baa4d1a9daf8f03e
Merge: d125f69 aaada42
Author: Hector Sanjuan <hsanjuan@users.noreply.github.com>
Date: Wed Feb 20 11:02:00 2019 +0000
Merge pull request #634 from ipfs/issue_450
Support PinPath, UnpinPath (resolve before pinning)
commit aaada42054e1f1c7b2abb1270859d0de41a0e5d8
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 19 22:16:25 2019 +0530
formatResponse accepts api.Pin and not api.PinSerial
commit b5da4bea045865814cc422da71827b44ddd44b90
Merge: ba59036 cc8dd7e
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 19 21:36:46 2019 +0530
Merge branch 'master' into issue_450
commit ba5903649c1df1dba20f4d6f7e3573d6fe24921f
Merge: f002914 d59880c
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Feb 18 08:41:11 2019 +0530
Merge branch 'issue_450' of github.com:ipfs/ipfs-cluster into issue_450
commit f00291494c0c02621c2296cbb7ac71e4c23aa9ec
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Feb 18 08:31:39 2019 +0530
PinPath: more improvements
Added tracing for new methods
commit d59880c338eaa8214fe06b4f930a540793d78407
Merge: 0ca4c7c b4f0eb3
Author: Hector Sanjuan <hsanjuan@users.noreply.github.com>
Date: Wed Feb 13 15:22:49 2019 +0000
Merge branch 'master' into issue_450
commit 0ca4c7c3b0670ed9c8279f8274d36e3485c10030
Merge: d35017a ecef9ea
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 12 13:10:13 2019 +0530
Merge branch 'master' into issue_450
commit d35017a8de91ca9fc9a9a047c48c75134cee9f98
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 12 13:07:25 2019 +0530
PinPath: more improvements
- Worth having `PinOptions` as a separate field in the struct and
constructing the query in the test with ToQuery()
- sharness: "intialization" line can be placed outside the tests at
the top
commit 68e3b90417ffbad89d41a70ac81d85f9037f8848
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Sun Feb 10 21:43:50 2019 +0530
Using if-continue pattern instead of if-else
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 3c29799f3b85be328b27508332ab92049d8b82f3
Merge: 956790b 4324889
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Thu Feb 7 10:25:52 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 956790b381db9858e4194f983e898b07dc51ba66
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 21:11:20 2019 +0530
Removing resolved path
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 7191cc46cedfbec116a9746937e28881b50ca044
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 16:45:07 2019 +0530
Fix go vet
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit f8b3d5b63b1b7569e2a3e0d82894fd4491c246c4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 16:07:03 2019 +0530
Fixed linting error
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 23c57eb467755a1f21387a1615a7f34e97348053
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 09:20:41 2019 +0530
Fixed tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 0caedd94aefeb3b6649dedc214cb4b849ace2ea4
Merge: 17e555e 5a7ee1d
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 00:07:10 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 17e555e4a7c574413df90aac70c5cc29cab98f54
Author: Hector Sanjuan <code@hector.link>
Date: Tue Feb 5 16:58:50 2019 +0000
PinPath: address some feedback + improvements
* Changed client's Pin() API and PinPath to be consistent
* Added helper methods to turn PinPath to query and back
* Make code and tests build
* Use TestCidResolved everywhere
* Fix cluster.PinPath arguments
* Fix formatting of responses with --no-status
* Make tests readable and call Fatal when needed
* Use a pathTestCases variable
commit f0e7369c47c5ddadc8ed45df5fd2d4d9b2d42b38
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 5 18:34:26 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Addressed review comments as in
https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-198751932
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit a8b4f181d2d7afed32ee41331dfaab19fd66a173
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 29 22:41:27 2019 +0530
Fixing tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit e39b95ca19e4d75506f4f492678245ef13936a44
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 29 14:52:53 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- PinPath and UnpinPath should return api.Pin
- PinPath should accept pin options
- Removing duplicate logic for Resolve from cluster
- And many other review comments https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-195509504
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit d146075126320896665ba58d337a13789f68ea86
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 23 17:08:41 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
PinPath(in both rest and rpc) should return a serializable struct in the
form `{"\":"Q...cid..string..."}` (as used in "github.com/ipfs/go-cid"
to marshal and unmarshal)
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 1f4869568a8adb450275257154ea3a26d03a30f3
Merge: 7acfd28 a244af9
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 23 07:18:56 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 7acfd282732ddf2282a67d4f9d0170a494eb3ed4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 22 18:14:32 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- RPC must always use serializable structs
- In command, just use pin with path as cid is also a valid path
- Addressing many other small review comments as in
https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-192122534
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 36905041e1e3f0b204942030aab3ab7b5b9e4d62
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 16 09:36:42 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Extra logic for path checking should go into resolve so that it can be
properly reused
Added sharness tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 9116bda3534e77bb391d873051bb520a1b01a326
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 16 08:08:07 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
error strings should not be capitalized
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit ca7e61861374f456300a85ddc0374e594f74f963
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 15 23:40:25 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Tests
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 522fbcd899f01c01680375561a32a87464157c0a
Merge: f1a56ab f7bc468
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 15 10:40:54 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit f1a56ab925fb74c0c44273a4524afa4843cf757f
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Jan 14 20:58:17 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- IPFS Connector should act as a pure IPFS client, any extra logic
should go to cluster.go
- Use cid.Undef, instead of cid.Cid{}
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit c83b91054f6774f1f9d4930cfc3f1fa28236f57c
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Thu Jan 10 08:57:17 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- Separate handlers, methods and rpc apis for PinPath and UnpinPath from
Pin and Unpin
- Support ipld paths as well
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 719dff88129366ce3ccb5e04cb6f8082a0915c5c
Merge: 91ceb47 21170c4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 9 19:38:35 2019 +0530
Merge branch 'issue_450_old' into HEAD
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 91ceb4796259ca7ef2974ec43e6a278a12796b13
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 9 19:36:41 2019 +0530
Revert "WIP: Figure out why test does not impleme"
This reverts commit 28a3a3f25dce6f296c8cbef86221644c099a7e75.
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 28a3a3f25dce6f296c8cbef86221644c099a7e75
Author: cd10012 <ced361@nyu.edu>
Date: Tue Jul 24 23:23:10 2018 -0400
WIP: Figure out why test does not implement IPFSConnector interface...
License: MIT
Signed-off-by: cd10012 <ced361@nyu.edu>
commit 21170c48e77e69583db64544b08120a9baf40d8d
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 8 10:37:59 2019 +0530
Support PinPath, UnpinPath (resolve before pinning)
This commit adds API support for pinning using path
`POST /pins/<ipfs or ipns path>` and `DELETE /pins/<ipfs or ipns path>`
will resolve the path into a cid and perform perform pinning or
unpinning
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Co-authored-by: Hector Sanjuan <hector@protocol.ai>
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2019-02-20 11:07:50 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
ci, _, err := gopath.SplitAbsPath(gopath.FromString(resp.Path))
|
2022-04-07 11:53:30 +00:00
|
|
|
return api.NewCid(ci), err
|
Add PinPath/UnpinPath support.
Squashed commit of the following:
commit 38cf569c6aed77c46ee4e0f8baa4d1a9daf8f03e
Merge: d125f69 aaada42
Author: Hector Sanjuan <hsanjuan@users.noreply.github.com>
Date: Wed Feb 20 11:02:00 2019 +0000
Merge pull request #634 from ipfs/issue_450
Support PinPath, UnpinPath (resolve before pinning)
commit aaada42054e1f1c7b2abb1270859d0de41a0e5d8
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 19 22:16:25 2019 +0530
formatResponse accepts api.Pin and not api.PinSerial
commit b5da4bea045865814cc422da71827b44ddd44b90
Merge: ba59036 cc8dd7e
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 19 21:36:46 2019 +0530
Merge branch 'master' into issue_450
commit ba5903649c1df1dba20f4d6f7e3573d6fe24921f
Merge: f002914 d59880c
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Feb 18 08:41:11 2019 +0530
Merge branch 'issue_450' of github.com:ipfs/ipfs-cluster into issue_450
commit f00291494c0c02621c2296cbb7ac71e4c23aa9ec
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Feb 18 08:31:39 2019 +0530
PinPath: more improvements
Added tracing for new methods
commit d59880c338eaa8214fe06b4f930a540793d78407
Merge: 0ca4c7c b4f0eb3
Author: Hector Sanjuan <hsanjuan@users.noreply.github.com>
Date: Wed Feb 13 15:22:49 2019 +0000
Merge branch 'master' into issue_450
commit 0ca4c7c3b0670ed9c8279f8274d36e3485c10030
Merge: d35017a ecef9ea
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 12 13:10:13 2019 +0530
Merge branch 'master' into issue_450
commit d35017a8de91ca9fc9a9a047c48c75134cee9f98
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 12 13:07:25 2019 +0530
PinPath: more improvements
- Worth having `PinOptions` as a separate field in the struct and
constructing the query in the test with ToQuery()
- sharness: "intialization" line can be placed outside the tests at
the top
commit 68e3b90417ffbad89d41a70ac81d85f9037f8848
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Sun Feb 10 21:43:50 2019 +0530
Using if-continue pattern instead of if-else
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 3c29799f3b85be328b27508332ab92049d8b82f3
Merge: 956790b 4324889
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Thu Feb 7 10:25:52 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 956790b381db9858e4194f983e898b07dc51ba66
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 21:11:20 2019 +0530
Removing resolved path
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 7191cc46cedfbec116a9746937e28881b50ca044
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 16:45:07 2019 +0530
Fix go vet
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit f8b3d5b63b1b7569e2a3e0d82894fd4491c246c4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 16:07:03 2019 +0530
Fixed linting error
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 23c57eb467755a1f21387a1615a7f34e97348053
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 09:20:41 2019 +0530
Fixed tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 0caedd94aefeb3b6649dedc214cb4b849ace2ea4
Merge: 17e555e 5a7ee1d
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 00:07:10 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 17e555e4a7c574413df90aac70c5cc29cab98f54
Author: Hector Sanjuan <code@hector.link>
Date: Tue Feb 5 16:58:50 2019 +0000
PinPath: address some feedback + improvements
* Changed client's Pin() API and PinPath to be consistent
* Added helper methods to turn PinPath to query and back
* Make code and tests build
* Use TestCidResolved everywhere
* Fix cluster.PinPath arguments
* Fix formatting of responses with --no-status
* Make tests readable and call Fatal when needed
* Use a pathTestCases variable
commit f0e7369c47c5ddadc8ed45df5fd2d4d9b2d42b38
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 5 18:34:26 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Addressed review comments as in
https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-198751932
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit a8b4f181d2d7afed32ee41331dfaab19fd66a173
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 29 22:41:27 2019 +0530
Fixing tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit e39b95ca19e4d75506f4f492678245ef13936a44
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 29 14:52:53 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- PinPath and UnpinPath should return api.Pin
- PinPath should accept pin options
- Removing duplicate logic for Resolve from cluster
- And many other review comments https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-195509504
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit d146075126320896665ba58d337a13789f68ea86
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 23 17:08:41 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
PinPath(in both rest and rpc) should return a serializable struct in the
form `{"\":"Q...cid..string..."}` (as used in "github.com/ipfs/go-cid"
to marshal and unmarshal)
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 1f4869568a8adb450275257154ea3a26d03a30f3
Merge: 7acfd28 a244af9
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 23 07:18:56 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 7acfd282732ddf2282a67d4f9d0170a494eb3ed4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 22 18:14:32 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- RPC must always use serializable structs
- In command, just use pin with path as cid is also a valid path
- Addressing many other small review comments as in
https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-192122534
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 36905041e1e3f0b204942030aab3ab7b5b9e4d62
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 16 09:36:42 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Extra logic for path checking should go into resolve so that it can be
properly reused
Added sharness tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 9116bda3534e77bb391d873051bb520a1b01a326
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 16 08:08:07 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
error strings should not be capitalized
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit ca7e61861374f456300a85ddc0374e594f74f963
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 15 23:40:25 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Tests
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 522fbcd899f01c01680375561a32a87464157c0a
Merge: f1a56ab f7bc468
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 15 10:40:54 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit f1a56ab925fb74c0c44273a4524afa4843cf757f
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Jan 14 20:58:17 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- IPFS Connector should act as a pure IPFS client, any extra logic
should go to cluster.go
- Use cid.Undef, instead of cid.Cid{}
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit c83b91054f6774f1f9d4930cfc3f1fa28236f57c
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Thu Jan 10 08:57:17 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- Separate handlers, methods and rpc apis for PinPath and UnpinPath from
Pin and Unpin
- Support ipld paths as well
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 719dff88129366ce3ccb5e04cb6f8082a0915c5c
Merge: 91ceb47 21170c4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 9 19:38:35 2019 +0530
Merge branch 'issue_450_old' into HEAD
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 91ceb4796259ca7ef2974ec43e6a278a12796b13
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 9 19:36:41 2019 +0530
Revert "WIP: Figure out why test does not impleme"
This reverts commit 28a3a3f25dce6f296c8cbef86221644c099a7e75.
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 28a3a3f25dce6f296c8cbef86221644c099a7e75
Author: cd10012 <ced361@nyu.edu>
Date: Tue Jul 24 23:23:10 2018 -0400
WIP: Figure out why test does not implement IPFSConnector interface...
License: MIT
Signed-off-by: cd10012 <ced361@nyu.edu>
commit 21170c48e77e69583db64544b08120a9baf40d8d
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 8 10:37:59 2019 +0530
Support PinPath, UnpinPath (resolve before pinning)
This commit adds API support for pinning using path
`POST /pins/<ipfs or ipns path>` and `DELETE /pins/<ipfs or ipns path>`
will resolve the path into a cid and perform perform pinning or
unpinning
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Co-authored-by: Hector Sanjuan <hector@protocol.ai>
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2019-02-20 11:07:50 +00:00
|
|
|
}
|
|
|
|
|
2018-02-09 05:23:24 +00:00
|
|
|
// SwarmPeers returns the peers currently connected to this ipfs daemon.
|
2019-02-27 17:04:35 +00:00
|
|
|
func (ipfs *Connector) SwarmPeers(ctx context.Context) ([]peer.ID, error) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx, span := trace.StartSpan(ctx, "ipfsconn/ipfshttp/SwarmPeers")
|
|
|
|
defer span.End()
|
|
|
|
|
|
|
|
ctx, cancel := context.WithTimeout(ctx, ipfs.config.IPFSRequestTimeout)
|
2018-05-24 07:12:05 +00:00
|
|
|
defer cancel()
|
2019-02-27 17:04:35 +00:00
|
|
|
|
2018-06-28 15:01:39 +00:00
|
|
|
res, err := ipfs.postCtx(ctx, "swarm/peers", "", nil)
|
2017-10-13 21:12:46 +00:00
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
2019-02-27 17:04:35 +00:00
|
|
|
return nil, err
|
2017-10-13 21:12:46 +00:00
|
|
|
}
|
|
|
|
var peersRaw ipfsSwarmPeersResp
|
|
|
|
err = json.Unmarshal(res, &peersRaw)
|
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
2019-02-27 17:04:35 +00:00
|
|
|
return nil, err
|
2017-10-13 21:12:46 +00:00
|
|
|
}
|
|
|
|
|
2019-02-27 17:04:35 +00:00
|
|
|
swarm := make([]peer.ID, len(peersRaw.Peers))
|
2017-10-13 21:12:46 +00:00
|
|
|
for i, p := range peersRaw.Peers {
|
2020-04-14 17:58:00 +00:00
|
|
|
pID, err := peer.Decode(p.Peer)
|
2017-10-13 21:12:46 +00:00
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
return swarm, err
|
|
|
|
}
|
2018-01-18 02:49:35 +00:00
|
|
|
swarm[i] = pID
|
2017-10-13 21:12:46 +00:00
|
|
|
}
|
|
|
|
return swarm, nil
|
|
|
|
}
|
2018-04-20 04:09:07 +00:00
|
|
|
|
Adders: stream blocks to destinations
This commit fixes #810 and adds block streaming to the final destinations when
adding. This should add major performance gains when adding data to clusters.
Before, everytime cluster issued a block, it was broadcasted individually to
all destinations (new libp2p stream), where it was block/put to IPFS (a single
block/put http roundtrip per block).
Now, blocks are streamed all the way from the adder module to the ipfs daemon,
by making every block as it arrives a single part in a multipart block/put
request.
Before, block-broadcast needed to wait for all destinations to finish in order
to process the next block. Now, buffers allow some destinations to be faster
than others while sending and receiving blocks.
Before, if a block put request failed to be broadcasted everywhere, an error
would happen at that moment.
Now, we keep streaming until the end and only then report any errors. The
operation succeeds as long as at least one stream finished successfully.
Errors block/putting to IPFS will not abort streams. Instead, subsequent
blocks are retried with a new request, although the method will return an
error when the stream finishes if there were errors at any point.
2022-03-24 01:17:10 +00:00
|
|
|
// chanDirectory implementes the files.Directory interace
|
|
|
|
type chanDirectory struct {
|
|
|
|
iterator files.DirIterator
|
|
|
|
}
|
2018-06-27 04:03:15 +00:00
|
|
|
|
Adders: stream blocks to destinations
This commit fixes #810 and adds block streaming to the final destinations when
adding. This should add major performance gains when adding data to clusters.
Before, everytime cluster issued a block, it was broadcasted individually to
all destinations (new libp2p stream), where it was block/put to IPFS (a single
block/put http roundtrip per block).
Now, blocks are streamed all the way from the adder module to the ipfs daemon,
by making every block as it arrives a single part in a multipart block/put
request.
Before, block-broadcast needed to wait for all destinations to finish in order
to process the next block. Now, buffers allow some destinations to be faster
than others while sending and receiving blocks.
Before, if a block put request failed to be broadcasted everywhere, an error
would happen at that moment.
Now, we keep streaming until the end and only then report any errors. The
operation succeeds as long as at least one stream finished successfully.
Errors block/putting to IPFS will not abort streams. Instead, subsequent
blocks are retried with a new request, although the method will return an
error when the stream finishes if there were errors at any point.
2022-03-24 01:17:10 +00:00
|
|
|
// Close is a no-op and it is not used.
|
|
|
|
func (cd *chanDirectory) Close() error {
|
|
|
|
return nil
|
|
|
|
}
|
2018-08-07 09:49:31 +00:00
|
|
|
|
Adders: stream blocks to destinations
This commit fixes #810 and adds block streaming to the final destinations when
adding. This should add major performance gains when adding data to clusters.
Before, everytime cluster issued a block, it was broadcasted individually to
all destinations (new libp2p stream), where it was block/put to IPFS (a single
block/put http roundtrip per block).
Now, blocks are streamed all the way from the adder module to the ipfs daemon,
by making every block as it arrives a single part in a multipart block/put
request.
Before, block-broadcast needed to wait for all destinations to finish in order
to process the next block. Now, buffers allow some destinations to be faster
than others while sending and receiving blocks.
Before, if a block put request failed to be broadcasted everywhere, an error
would happen at that moment.
Now, we keep streaming until the end and only then report any errors. The
operation succeeds as long as at least one stream finished successfully.
Errors block/putting to IPFS will not abort streams. Instead, subsequent
blocks are retried with a new request, although the method will return an
error when the stream finishes if there were errors at any point.
2022-03-24 01:17:10 +00:00
|
|
|
// not implemented, I think not needed for multipart.
|
|
|
|
func (cd *chanDirectory) Size() (int64, error) {
|
|
|
|
return 0, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (cd *chanDirectory) Entries() files.DirIterator {
|
|
|
|
return cd.iterator
|
|
|
|
}
|
|
|
|
|
|
|
|
// chanIterator implements the files.DirIterator interface.
|
|
|
|
type chanIterator struct {
|
|
|
|
ctx context.Context
|
|
|
|
blocks <-chan api.NodeWithMeta
|
|
|
|
|
|
|
|
current api.NodeWithMeta
|
|
|
|
peeked api.NodeWithMeta
|
|
|
|
done bool
|
|
|
|
err error
|
|
|
|
|
|
|
|
seenMu sync.Mutex
|
Fix warnings on block-cid mismatch when adding
Because we are adding blocks on a single call, and we choose the format
parameter based on the prefix of the first block, IPFS will return block CIDs
based on that option.
This caused warnings when adding content that has multiple CID prefixes: for
example, any cid-version=1 file will include both dag-pb CIDs and raw
CIDs. Since the first block is usually a leave, IPFS will only return
raw-cids, and cause a warning because of the CID-mistmatch.
This fixes things by comparing multihashes only.
But! We might be writing blocks with the wrong CID and then the good CID won't
work!
Correct, we might, in some corner cases.
In go-ipfs >= 0.12.0, all blocks are addressed by multihash so CID prefixes
are irrelevant. This problem does not exist in that case.
In go-ipfs < 0.12.0, if a read for a CIDv1 DAG-PB fails, it is retried as it
it was raw. This means that if we wrote something with cidv1/format=raw, that
should have been a cidv1/format=dag-pb, the read will still work. That covers
some common cases (i.e. adding with cid-version=1) because the first block
should be a raw-leaf. Default-params (cidv0) is not affected since everything
is raw multihashes. However, there are still possible CAR layouts etc. where
cluster will write blocks wrongly to older IPFS versions.
2022-04-11 18:26:06 +00:00
|
|
|
seen *multihash.Set
|
Adders: stream blocks to destinations
This commit fixes #810 and adds block streaming to the final destinations when
adding. This should add major performance gains when adding data to clusters.
Before, everytime cluster issued a block, it was broadcasted individually to
all destinations (new libp2p stream), where it was block/put to IPFS (a single
block/put http roundtrip per block).
Now, blocks are streamed all the way from the adder module to the ipfs daemon,
by making every block as it arrives a single part in a multipart block/put
request.
Before, block-broadcast needed to wait for all destinations to finish in order
to process the next block. Now, buffers allow some destinations to be faster
than others while sending and receiving blocks.
Before, if a block put request failed to be broadcasted everywhere, an error
would happen at that moment.
Now, we keep streaming until the end and only then report any errors. The
operation succeeds as long as at least one stream finished successfully.
Errors block/putting to IPFS will not abort streams. Instead, subsequent
blocks are retried with a new request, although the method will return an
error when the stream finishes if there were errors at any point.
2022-03-24 01:17:10 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (ci *chanIterator) Name() string {
|
|
|
|
if !ci.current.Cid.Defined() {
|
|
|
|
return ""
|
|
|
|
}
|
|
|
|
return ci.current.Cid.String()
|
|
|
|
}
|
|
|
|
|
|
|
|
// return NewBytesFile.
|
|
|
|
func (ci *chanIterator) Node() files.Node {
|
|
|
|
if !ci.current.Cid.Defined() {
|
|
|
|
return nil
|
|
|
|
}
|
Fix warnings on block-cid mismatch when adding
Because we are adding blocks on a single call, and we choose the format
parameter based on the prefix of the first block, IPFS will return block CIDs
based on that option.
This caused warnings when adding content that has multiple CID prefixes: for
example, any cid-version=1 file will include both dag-pb CIDs and raw
CIDs. Since the first block is usually a leave, IPFS will only return
raw-cids, and cause a warning because of the CID-mistmatch.
This fixes things by comparing multihashes only.
But! We might be writing blocks with the wrong CID and then the good CID won't
work!
Correct, we might, in some corner cases.
In go-ipfs >= 0.12.0, all blocks are addressed by multihash so CID prefixes
are irrelevant. This problem does not exist in that case.
In go-ipfs < 0.12.0, if a read for a CIDv1 DAG-PB fails, it is retried as it
it was raw. This means that if we wrote something with cidv1/format=raw, that
should have been a cidv1/format=dag-pb, the read will still work. That covers
some common cases (i.e. adding with cid-version=1) because the first block
should be a raw-leaf. Default-params (cidv0) is not affected since everything
is raw multihashes. However, there are still possible CAR layouts etc. where
cluster will write blocks wrongly to older IPFS versions.
2022-04-11 18:26:06 +00:00
|
|
|
logger.Debugf("it.node(): %s", ci.current.Cid)
|
Adders: stream blocks to destinations
This commit fixes #810 and adds block streaming to the final destinations when
adding. This should add major performance gains when adding data to clusters.
Before, everytime cluster issued a block, it was broadcasted individually to
all destinations (new libp2p stream), where it was block/put to IPFS (a single
block/put http roundtrip per block).
Now, blocks are streamed all the way from the adder module to the ipfs daemon,
by making every block as it arrives a single part in a multipart block/put
request.
Before, block-broadcast needed to wait for all destinations to finish in order
to process the next block. Now, buffers allow some destinations to be faster
than others while sending and receiving blocks.
Before, if a block put request failed to be broadcasted everywhere, an error
would happen at that moment.
Now, we keep streaming until the end and only then report any errors. The
operation succeeds as long as at least one stream finished successfully.
Errors block/putting to IPFS will not abort streams. Instead, subsequent
blocks are retried with a new request, although the method will return an
error when the stream finishes if there were errors at any point.
2022-03-24 01:17:10 +00:00
|
|
|
ci.seenMu.Lock()
|
Fix warnings on block-cid mismatch when adding
Because we are adding blocks on a single call, and we choose the format
parameter based on the prefix of the first block, IPFS will return block CIDs
based on that option.
This caused warnings when adding content that has multiple CID prefixes: for
example, any cid-version=1 file will include both dag-pb CIDs and raw
CIDs. Since the first block is usually a leave, IPFS will only return
raw-cids, and cause a warning because of the CID-mistmatch.
This fixes things by comparing multihashes only.
But! We might be writing blocks with the wrong CID and then the good CID won't
work!
Correct, we might, in some corner cases.
In go-ipfs >= 0.12.0, all blocks are addressed by multihash so CID prefixes
are irrelevant. This problem does not exist in that case.
In go-ipfs < 0.12.0, if a read for a CIDv1 DAG-PB fails, it is retried as it
it was raw. This means that if we wrote something with cidv1/format=raw, that
should have been a cidv1/format=dag-pb, the read will still work. That covers
some common cases (i.e. adding with cid-version=1) because the first block
should be a raw-leaf. Default-params (cidv0) is not affected since everything
is raw multihashes. However, there are still possible CAR layouts etc. where
cluster will write blocks wrongly to older IPFS versions.
2022-04-11 18:26:06 +00:00
|
|
|
ci.seen.Add(ci.current.Cid.Hash())
|
Adders: stream blocks to destinations
This commit fixes #810 and adds block streaming to the final destinations when
adding. This should add major performance gains when adding data to clusters.
Before, everytime cluster issued a block, it was broadcasted individually to
all destinations (new libp2p stream), where it was block/put to IPFS (a single
block/put http roundtrip per block).
Now, blocks are streamed all the way from the adder module to the ipfs daemon,
by making every block as it arrives a single part in a multipart block/put
request.
Before, block-broadcast needed to wait for all destinations to finish in order
to process the next block. Now, buffers allow some destinations to be faster
than others while sending and receiving blocks.
Before, if a block put request failed to be broadcasted everywhere, an error
would happen at that moment.
Now, we keep streaming until the end and only then report any errors. The
operation succeeds as long as at least one stream finished successfully.
Errors block/putting to IPFS will not abort streams. Instead, subsequent
blocks are retried with a new request, although the method will return an
error when the stream finishes if there were errors at any point.
2022-03-24 01:17:10 +00:00
|
|
|
ci.seenMu.Unlock()
|
|
|
|
return files.NewBytesFile(ci.current.Data)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (ci *chanIterator) Seen(c api.Cid) bool {
|
|
|
|
ci.seenMu.Lock()
|
Fix warnings on block-cid mismatch when adding
Because we are adding blocks on a single call, and we choose the format
parameter based on the prefix of the first block, IPFS will return block CIDs
based on that option.
This caused warnings when adding content that has multiple CID prefixes: for
example, any cid-version=1 file will include both dag-pb CIDs and raw
CIDs. Since the first block is usually a leave, IPFS will only return
raw-cids, and cause a warning because of the CID-mistmatch.
This fixes things by comparing multihashes only.
But! We might be writing blocks with the wrong CID and then the good CID won't
work!
Correct, we might, in some corner cases.
In go-ipfs >= 0.12.0, all blocks are addressed by multihash so CID prefixes
are irrelevant. This problem does not exist in that case.
In go-ipfs < 0.12.0, if a read for a CIDv1 DAG-PB fails, it is retried as it
it was raw. This means that if we wrote something with cidv1/format=raw, that
should have been a cidv1/format=dag-pb, the read will still work. That covers
some common cases (i.e. adding with cid-version=1) because the first block
should be a raw-leaf. Default-params (cidv0) is not affected since everything
is raw multihashes. However, there are still possible CAR layouts etc. where
cluster will write blocks wrongly to older IPFS versions.
2022-04-11 18:26:06 +00:00
|
|
|
has := ci.seen.Has(c.Cid.Hash())
|
|
|
|
ci.seen.Remove(c.Cid.Hash())
|
Adders: stream blocks to destinations
This commit fixes #810 and adds block streaming to the final destinations when
adding. This should add major performance gains when adding data to clusters.
Before, everytime cluster issued a block, it was broadcasted individually to
all destinations (new libp2p stream), where it was block/put to IPFS (a single
block/put http roundtrip per block).
Now, blocks are streamed all the way from the adder module to the ipfs daemon,
by making every block as it arrives a single part in a multipart block/put
request.
Before, block-broadcast needed to wait for all destinations to finish in order
to process the next block. Now, buffers allow some destinations to be faster
than others while sending and receiving blocks.
Before, if a block put request failed to be broadcasted everywhere, an error
would happen at that moment.
Now, we keep streaming until the end and only then report any errors. The
operation succeeds as long as at least one stream finished successfully.
Errors block/putting to IPFS will not abort streams. Instead, subsequent
blocks are retried with a new request, although the method will return an
error when the stream finishes if there were errors at any point.
2022-03-24 01:17:10 +00:00
|
|
|
ci.seenMu.Unlock()
|
|
|
|
return has
|
|
|
|
}
|
|
|
|
|
|
|
|
func (ci *chanIterator) Done() bool {
|
|
|
|
return ci.done
|
|
|
|
}
|
|
|
|
|
|
|
|
// Peek reads one block from the channel but saves it so that Next also
|
|
|
|
// returns it.
|
|
|
|
func (ci *chanIterator) Peek() (api.NodeWithMeta, bool) {
|
|
|
|
if ci.done {
|
|
|
|
return api.NodeWithMeta{}, false
|
|
|
|
}
|
|
|
|
|
|
|
|
select {
|
|
|
|
case <-ci.ctx.Done():
|
|
|
|
return api.NodeWithMeta{}, false
|
|
|
|
case next, ok := <-ci.blocks:
|
|
|
|
if !ok {
|
|
|
|
return api.NodeWithMeta{}, false
|
|
|
|
}
|
|
|
|
ci.peeked = next
|
|
|
|
return next, true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (ci *chanIterator) Next() bool {
|
|
|
|
if ci.done {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
if ci.peeked.Cid.Defined() {
|
|
|
|
ci.current = ci.peeked
|
|
|
|
ci.peeked = api.NodeWithMeta{}
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
select {
|
|
|
|
case <-ci.ctx.Done():
|
|
|
|
ci.done = true
|
|
|
|
ci.err = ci.ctx.Err()
|
|
|
|
return false
|
|
|
|
case next, ok := <-ci.blocks:
|
|
|
|
if !ok {
|
|
|
|
ci.done = true
|
|
|
|
return false
|
|
|
|
}
|
Fix warnings on block-cid mismatch when adding
Because we are adding blocks on a single call, and we choose the format
parameter based on the prefix of the first block, IPFS will return block CIDs
based on that option.
This caused warnings when adding content that has multiple CID prefixes: for
example, any cid-version=1 file will include both dag-pb CIDs and raw
CIDs. Since the first block is usually a leave, IPFS will only return
raw-cids, and cause a warning because of the CID-mistmatch.
This fixes things by comparing multihashes only.
But! We might be writing blocks with the wrong CID and then the good CID won't
work!
Correct, we might, in some corner cases.
In go-ipfs >= 0.12.0, all blocks are addressed by multihash so CID prefixes
are irrelevant. This problem does not exist in that case.
In go-ipfs < 0.12.0, if a read for a CIDv1 DAG-PB fails, it is retried as it
it was raw. This means that if we wrote something with cidv1/format=raw, that
should have been a cidv1/format=dag-pb, the read will still work. That covers
some common cases (i.e. adding with cid-version=1) because the first block
should be a raw-leaf. Default-params (cidv0) is not affected since everything
is raw multihashes. However, there are still possible CAR layouts etc. where
cluster will write blocks wrongly to older IPFS versions.
2022-04-11 18:26:06 +00:00
|
|
|
logger.Debugf("it.Next() %s", next.Cid)
|
Adders: stream blocks to destinations
This commit fixes #810 and adds block streaming to the final destinations when
adding. This should add major performance gains when adding data to clusters.
Before, everytime cluster issued a block, it was broadcasted individually to
all destinations (new libp2p stream), where it was block/put to IPFS (a single
block/put http roundtrip per block).
Now, blocks are streamed all the way from the adder module to the ipfs daemon,
by making every block as it arrives a single part in a multipart block/put
request.
Before, block-broadcast needed to wait for all destinations to finish in order
to process the next block. Now, buffers allow some destinations to be faster
than others while sending and receiving blocks.
Before, if a block put request failed to be broadcasted everywhere, an error
would happen at that moment.
Now, we keep streaming until the end and only then report any errors. The
operation succeeds as long as at least one stream finished successfully.
Errors block/putting to IPFS will not abort streams. Instead, subsequent
blocks are retried with a new request, although the method will return an
error when the stream finishes if there were errors at any point.
2022-03-24 01:17:10 +00:00
|
|
|
ci.current = next
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
2018-12-17 12:44:06 +00:00
|
|
|
|
Adders: stream blocks to destinations
This commit fixes #810 and adds block streaming to the final destinations when
adding. This should add major performance gains when adding data to clusters.
Before, everytime cluster issued a block, it was broadcasted individually to
all destinations (new libp2p stream), where it was block/put to IPFS (a single
block/put http roundtrip per block).
Now, blocks are streamed all the way from the adder module to the ipfs daemon,
by making every block as it arrives a single part in a multipart block/put
request.
Before, block-broadcast needed to wait for all destinations to finish in order
to process the next block. Now, buffers allow some destinations to be faster
than others while sending and receiving blocks.
Before, if a block put request failed to be broadcasted everywhere, an error
would happen at that moment.
Now, we keep streaming until the end and only then report any errors. The
operation succeeds as long as at least one stream finished successfully.
Errors block/putting to IPFS will not abort streams. Instead, subsequent
blocks are retried with a new request, although the method will return an
error when the stream finishes if there were errors at any point.
2022-03-24 01:17:10 +00:00
|
|
|
func (ci *chanIterator) Err() error {
|
|
|
|
return ci.err
|
|
|
|
}
|
2020-03-06 15:35:15 +00:00
|
|
|
|
Adders: stream blocks to destinations
This commit fixes #810 and adds block streaming to the final destinations when
adding. This should add major performance gains when adding data to clusters.
Before, everytime cluster issued a block, it was broadcasted individually to
all destinations (new libp2p stream), where it was block/put to IPFS (a single
block/put http roundtrip per block).
Now, blocks are streamed all the way from the adder module to the ipfs daemon,
by making every block as it arrives a single part in a multipart block/put
request.
Before, block-broadcast needed to wait for all destinations to finish in order
to process the next block. Now, buffers allow some destinations to be faster
than others while sending and receiving blocks.
Before, if a block put request failed to be broadcasted everywhere, an error
would happen at that moment.
Now, we keep streaming until the end and only then report any errors. The
operation succeeds as long as at least one stream finished successfully.
Errors block/putting to IPFS will not abort streams. Instead, subsequent
blocks are retried with a new request, although the method will return an
error when the stream finishes if there were errors at any point.
2022-03-24 01:17:10 +00:00
|
|
|
func blockPutQuery(prefix cid.Prefix) (url.Values, error) {
|
2020-03-06 15:35:15 +00:00
|
|
|
q := make(url.Values, 3)
|
|
|
|
format, ok := cid.CodecToStr[prefix.Codec]
|
|
|
|
if !ok {
|
Adders: stream blocks to destinations
This commit fixes #810 and adds block streaming to the final destinations when
adding. This should add major performance gains when adding data to clusters.
Before, everytime cluster issued a block, it was broadcasted individually to
all destinations (new libp2p stream), where it was block/put to IPFS (a single
block/put http roundtrip per block).
Now, blocks are streamed all the way from the adder module to the ipfs daemon,
by making every block as it arrives a single part in a multipart block/put
request.
Before, block-broadcast needed to wait for all destinations to finish in order
to process the next block. Now, buffers allow some destinations to be faster
than others while sending and receiving blocks.
Before, if a block put request failed to be broadcasted everywhere, an error
would happen at that moment.
Now, we keep streaming until the end and only then report any errors. The
operation succeeds as long as at least one stream finished successfully.
Errors block/putting to IPFS will not abort streams. Instead, subsequent
blocks are retried with a new request, although the method will return an
error when the stream finishes if there were errors at any point.
2022-03-24 01:17:10 +00:00
|
|
|
return q, fmt.Errorf("cannot find name for the blocks' CID codec: %x", prefix.Codec)
|
2020-03-06 15:35:15 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
mhType, ok := multihash.Codes[prefix.MhType]
|
|
|
|
if !ok {
|
Adders: stream blocks to destinations
This commit fixes #810 and adds block streaming to the final destinations when
adding. This should add major performance gains when adding data to clusters.
Before, everytime cluster issued a block, it was broadcasted individually to
all destinations (new libp2p stream), where it was block/put to IPFS (a single
block/put http roundtrip per block).
Now, blocks are streamed all the way from the adder module to the ipfs daemon,
by making every block as it arrives a single part in a multipart block/put
request.
Before, block-broadcast needed to wait for all destinations to finish in order
to process the next block. Now, buffers allow some destinations to be faster
than others while sending and receiving blocks.
Before, if a block put request failed to be broadcasted everywhere, an error
would happen at that moment.
Now, we keep streaming until the end and only then report any errors. The
operation succeeds as long as at least one stream finished successfully.
Errors block/putting to IPFS will not abort streams. Instead, subsequent
blocks are retried with a new request, although the method will return an
error when the stream finishes if there were errors at any point.
2022-03-24 01:17:10 +00:00
|
|
|
return q, fmt.Errorf("cannot find name for the blocks' Multihash type: %x", prefix.MhType)
|
2018-06-28 15:01:39 +00:00
|
|
|
}
|
2020-05-17 00:39:02 +00:00
|
|
|
|
|
|
|
// IPFS behaves differently when using v0 or protobuf which are
|
|
|
|
// actually the same.
|
|
|
|
if prefix.Version == 0 {
|
|
|
|
q.Set("format", "v0")
|
|
|
|
} else {
|
|
|
|
q.Set("format", format)
|
|
|
|
}
|
|
|
|
|
2020-03-06 15:35:15 +00:00
|
|
|
q.Set("mhtype", mhType)
|
|
|
|
q.Set("mhlen", strconv.Itoa(prefix.MhLength))
|
Fix warnings on block-cid mismatch when adding
Because we are adding blocks on a single call, and we choose the format
parameter based on the prefix of the first block, IPFS will return block CIDs
based on that option.
This caused warnings when adding content that has multiple CID prefixes: for
example, any cid-version=1 file will include both dag-pb CIDs and raw
CIDs. Since the first block is usually a leave, IPFS will only return
raw-cids, and cause a warning because of the CID-mistmatch.
This fixes things by comparing multihashes only.
But! We might be writing blocks with the wrong CID and then the good CID won't
work!
Correct, we might, in some corner cases.
In go-ipfs >= 0.12.0, all blocks are addressed by multihash so CID prefixes
are irrelevant. This problem does not exist in that case.
In go-ipfs < 0.12.0, if a read for a CIDv1 DAG-PB fails, it is retried as it
it was raw. This means that if we wrote something with cidv1/format=raw, that
should have been a cidv1/format=dag-pb, the read will still work. That covers
some common cases (i.e. adding with cid-version=1) because the first block
should be a raw-leaf. Default-params (cidv0) is not affected since everything
is raw multihashes. However, there are still possible CAR layouts etc. where
cluster will write blocks wrongly to older IPFS versions.
2022-04-11 18:26:06 +00:00
|
|
|
q.Set("pin", "false")
|
Adders: stream blocks to destinations
This commit fixes #810 and adds block streaming to the final destinations when
adding. This should add major performance gains when adding data to clusters.
Before, everytime cluster issued a block, it was broadcasted individually to
all destinations (new libp2p stream), where it was block/put to IPFS (a single
block/put http roundtrip per block).
Now, blocks are streamed all the way from the adder module to the ipfs daemon,
by making every block as it arrives a single part in a multipart block/put
request.
Before, block-broadcast needed to wait for all destinations to finish in order
to process the next block. Now, buffers allow some destinations to be faster
than others while sending and receiving blocks.
Before, if a block put request failed to be broadcasted everywhere, an error
would happen at that moment.
Now, we keep streaming until the end and only then report any errors. The
operation succeeds as long as at least one stream finished successfully.
Errors block/putting to IPFS will not abort streams. Instead, subsequent
blocks are retried with a new request, although the method will return an
error when the stream finishes if there were errors at any point.
2022-03-24 01:17:10 +00:00
|
|
|
return q, nil
|
|
|
|
}
|
2020-03-06 15:35:15 +00:00
|
|
|
|
Adders: stream blocks to destinations
This commit fixes #810 and adds block streaming to the final destinations when
adding. This should add major performance gains when adding data to clusters.
Before, everytime cluster issued a block, it was broadcasted individually to
all destinations (new libp2p stream), where it was block/put to IPFS (a single
block/put http roundtrip per block).
Now, blocks are streamed all the way from the adder module to the ipfs daemon,
by making every block as it arrives a single part in a multipart block/put
request.
Before, block-broadcast needed to wait for all destinations to finish in order
to process the next block. Now, buffers allow some destinations to be faster
than others while sending and receiving blocks.
Before, if a block put request failed to be broadcasted everywhere, an error
would happen at that moment.
Now, we keep streaming until the end and only then report any errors. The
operation succeeds as long as at least one stream finished successfully.
Errors block/putting to IPFS will not abort streams. Instead, subsequent
blocks are retried with a new request, although the method will return an
error when the stream finishes if there were errors at any point.
2022-03-24 01:17:10 +00:00
|
|
|
// BlockStream performs a multipart request to block/put with the blocks
|
|
|
|
// received on the channel.
|
|
|
|
func (ipfs *Connector) BlockStream(ctx context.Context, blocks <-chan api.NodeWithMeta) error {
|
|
|
|
ctx, span := trace.StartSpan(ctx, "ipfsconn/ipfshttp/BlockStream")
|
|
|
|
defer span.End()
|
2018-08-07 09:49:31 +00:00
|
|
|
|
Adders: stream blocks to destinations
This commit fixes #810 and adds block streaming to the final destinations when
adding. This should add major performance gains when adding data to clusters.
Before, everytime cluster issued a block, it was broadcasted individually to
all destinations (new libp2p stream), where it was block/put to IPFS (a single
block/put http roundtrip per block).
Now, blocks are streamed all the way from the adder module to the ipfs daemon,
by making every block as it arrives a single part in a multipart block/put
request.
Before, block-broadcast needed to wait for all destinations to finish in order
to process the next block. Now, buffers allow some destinations to be faster
than others while sending and receiving blocks.
Before, if a block put request failed to be broadcasted everywhere, an error
would happen at that moment.
Now, we keep streaming until the end and only then report any errors. The
operation succeeds as long as at least one stream finished successfully.
Errors block/putting to IPFS will not abort streams. Instead, subsequent
blocks are retried with a new request, although the method will return an
error when the stream finishes if there were errors at any point.
2022-03-24 01:17:10 +00:00
|
|
|
logger.Debug("streaming blocks to IPFS")
|
|
|
|
defer ipfs.updateInformerMetric(ctx)
|
|
|
|
|
|
|
|
var errs error
|
|
|
|
|
|
|
|
it := &chanIterator{
|
|
|
|
ctx: ctx,
|
|
|
|
blocks: blocks,
|
Fix warnings on block-cid mismatch when adding
Because we are adding blocks on a single call, and we choose the format
parameter based on the prefix of the first block, IPFS will return block CIDs
based on that option.
This caused warnings when adding content that has multiple CID prefixes: for
example, any cid-version=1 file will include both dag-pb CIDs and raw
CIDs. Since the first block is usually a leave, IPFS will only return
raw-cids, and cause a warning because of the CID-mistmatch.
This fixes things by comparing multihashes only.
But! We might be writing blocks with the wrong CID and then the good CID won't
work!
Correct, we might, in some corner cases.
In go-ipfs >= 0.12.0, all blocks are addressed by multihash so CID prefixes
are irrelevant. This problem does not exist in that case.
In go-ipfs < 0.12.0, if a read for a CIDv1 DAG-PB fails, it is retried as it
it was raw. This means that if we wrote something with cidv1/format=raw, that
should have been a cidv1/format=dag-pb, the read will still work. That covers
some common cases (i.e. adding with cid-version=1) because the first block
should be a raw-leaf. Default-params (cidv0) is not affected since everything
is raw multihashes. However, there are still possible CAR layouts etc. where
cluster will write blocks wrongly to older IPFS versions.
2022-04-11 18:26:06 +00:00
|
|
|
seen: multihash.NewSet(),
|
Adders: stream blocks to destinations
This commit fixes #810 and adds block streaming to the final destinations when
adding. This should add major performance gains when adding data to clusters.
Before, everytime cluster issued a block, it was broadcasted individually to
all destinations (new libp2p stream), where it was block/put to IPFS (a single
block/put http roundtrip per block).
Now, blocks are streamed all the way from the adder module to the ipfs daemon,
by making every block as it arrives a single part in a multipart block/put
request.
Before, block-broadcast needed to wait for all destinations to finish in order
to process the next block. Now, buffers allow some destinations to be faster
than others while sending and receiving blocks.
Before, if a block put request failed to be broadcasted everywhere, an error
would happen at that moment.
Now, we keep streaming until the end and only then report any errors. The
operation succeeds as long as at least one stream finished successfully.
Errors block/putting to IPFS will not abort streams. Instead, subsequent
blocks are retried with a new request, although the method will return an
error when the stream finishes if there were errors at any point.
2022-03-24 01:17:10 +00:00
|
|
|
}
|
|
|
|
dir := &chanDirectory{
|
|
|
|
iterator: it,
|
2020-03-06 15:35:15 +00:00
|
|
|
}
|
|
|
|
|
Adders: stream blocks to destinations
This commit fixes #810 and adds block streaming to the final destinations when
adding. This should add major performance gains when adding data to clusters.
Before, everytime cluster issued a block, it was broadcasted individually to
all destinations (new libp2p stream), where it was block/put to IPFS (a single
block/put http roundtrip per block).
Now, blocks are streamed all the way from the adder module to the ipfs daemon,
by making every block as it arrives a single part in a multipart block/put
request.
Before, block-broadcast needed to wait for all destinations to finish in order
to process the next block. Now, buffers allow some destinations to be faster
than others while sending and receiving blocks.
Before, if a block put request failed to be broadcasted everywhere, an error
would happen at that moment.
Now, we keep streaming until the end and only then report any errors. The
operation succeeds as long as at least one stream finished successfully.
Errors block/putting to IPFS will not abort streams. Instead, subsequent
blocks are retried with a new request, although the method will return an
error when the stream finishes if there were errors at any point.
2022-03-24 01:17:10 +00:00
|
|
|
// We need to pick into the first block to know which Cid prefix we
|
|
|
|
// are writing blocks with, so that ipfs calculates the expected
|
|
|
|
// multihash (we select the function used). This means that all blocks
|
|
|
|
// in a stream should use the same.
|
|
|
|
peek, ok := it.Peek()
|
|
|
|
if !ok {
|
|
|
|
return errors.New("BlockStream: no blocks to peek in blocks channel")
|
2020-03-06 15:35:15 +00:00
|
|
|
}
|
|
|
|
|
Adders: stream blocks to destinations
This commit fixes #810 and adds block streaming to the final destinations when
adding. This should add major performance gains when adding data to clusters.
Before, everytime cluster issued a block, it was broadcasted individually to
all destinations (new libp2p stream), where it was block/put to IPFS (a single
block/put http roundtrip per block).
Now, blocks are streamed all the way from the adder module to the ipfs daemon,
by making every block as it arrives a single part in a multipart block/put
request.
Before, block-broadcast needed to wait for all destinations to finish in order
to process the next block. Now, buffers allow some destinations to be faster
than others while sending and receiving blocks.
Before, if a block put request failed to be broadcasted everywhere, an error
would happen at that moment.
Now, we keep streaming until the end and only then report any errors. The
operation succeeds as long as at least one stream finished successfully.
Errors block/putting to IPFS will not abort streams. Instead, subsequent
blocks are retried with a new request, although the method will return an
error when the stream finishes if there were errors at any point.
2022-03-24 01:17:10 +00:00
|
|
|
q, err := blockPutQuery(peek.Cid.Prefix())
|
2020-03-06 15:35:15 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
Adders: stream blocks to destinations
This commit fixes #810 and adds block streaming to the final destinations when
adding. This should add major performance gains when adding data to clusters.
Before, everytime cluster issued a block, it was broadcasted individually to
all destinations (new libp2p stream), where it was block/put to IPFS (a single
block/put http roundtrip per block).
Now, blocks are streamed all the way from the adder module to the ipfs daemon,
by making every block as it arrives a single part in a multipart block/put
request.
Before, block-broadcast needed to wait for all destinations to finish in order
to process the next block. Now, buffers allow some destinations to be faster
than others while sending and receiving blocks.
Before, if a block put request failed to be broadcasted everywhere, an error
would happen at that moment.
Now, we keep streaming until the end and only then report any errors. The
operation succeeds as long as at least one stream finished successfully.
Errors block/putting to IPFS will not abort streams. Instead, subsequent
blocks are retried with a new request, although the method will return an
error when the stream finishes if there were errors at any point.
2022-03-24 01:17:10 +00:00
|
|
|
url := "block/put?" + q.Encode()
|
2020-03-06 15:35:15 +00:00
|
|
|
|
Adders: stream blocks to destinations
This commit fixes #810 and adds block streaming to the final destinations when
adding. This should add major performance gains when adding data to clusters.
Before, everytime cluster issued a block, it was broadcasted individually to
all destinations (new libp2p stream), where it was block/put to IPFS (a single
block/put http roundtrip per block).
Now, blocks are streamed all the way from the adder module to the ipfs daemon,
by making every block as it arrives a single part in a multipart block/put
request.
Before, block-broadcast needed to wait for all destinations to finish in order
to process the next block. Now, buffers allow some destinations to be faster
than others while sending and receiving blocks.
Before, if a block put request failed to be broadcasted everywhere, an error
would happen at that moment.
Now, we keep streaming until the end and only then report any errors. The
operation succeeds as long as at least one stream finished successfully.
Errors block/putting to IPFS will not abort streams. Instead, subsequent
blocks are retried with a new request, although the method will return an
error when the stream finishes if there were errors at any point.
2022-03-24 01:17:10 +00:00
|
|
|
// We essentially keep going on any request errors and keep putting
|
|
|
|
// blocks until we are done. We will, however, return a final error if
|
|
|
|
// there were errors along the way, but we do not abort the blocks
|
|
|
|
// stream because we could not block/put.
|
|
|
|
for !it.Done() {
|
2022-03-28 15:54:55 +00:00
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
logger.Error("BlockStream aborted: %s", ctx.Err())
|
|
|
|
return ctx.Err()
|
|
|
|
default:
|
|
|
|
}
|
|
|
|
|
Adders: stream blocks to destinations
This commit fixes #810 and adds block streaming to the final destinations when
adding. This should add major performance gains when adding data to clusters.
Before, everytime cluster issued a block, it was broadcasted individually to
all destinations (new libp2p stream), where it was block/put to IPFS (a single
block/put http roundtrip per block).
Now, blocks are streamed all the way from the adder module to the ipfs daemon,
by making every block as it arrives a single part in a multipart block/put
request.
Before, block-broadcast needed to wait for all destinations to finish in order
to process the next block. Now, buffers allow some destinations to be faster
than others while sending and receiving blocks.
Before, if a block put request failed to be broadcasted everywhere, an error
would happen at that moment.
Now, we keep streaming until the end and only then report any errors. The
operation succeeds as long as at least one stream finished successfully.
Errors block/putting to IPFS will not abort streams. Instead, subsequent
blocks are retried with a new request, although the method will return an
error when the stream finishes if there were errors at any point.
2022-03-24 01:17:10 +00:00
|
|
|
multiFileR := files.NewMultiFileReader(dir, true)
|
|
|
|
contentType := "multipart/form-data; boundary=" + multiFileR.Boundary()
|
|
|
|
body, err := ipfs.postCtxStreamResponse(ctx, url, contentType, multiFileR)
|
|
|
|
if err != nil {
|
|
|
|
errs = multierr.Append(errs, err)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
dec := json.NewDecoder(body)
|
|
|
|
for {
|
|
|
|
var res ipfsBlockPutResp
|
|
|
|
err := dec.Decode(&res)
|
|
|
|
if err == io.EOF {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
errs = multierr.Append(errs, err)
|
|
|
|
break
|
|
|
|
}
|
Fix warnings on block-cid mismatch when adding
Because we are adding blocks on a single call, and we choose the format
parameter based on the prefix of the first block, IPFS will return block CIDs
based on that option.
This caused warnings when adding content that has multiple CID prefixes: for
example, any cid-version=1 file will include both dag-pb CIDs and raw
CIDs. Since the first block is usually a leave, IPFS will only return
raw-cids, and cause a warning because of the CID-mistmatch.
This fixes things by comparing multihashes only.
But! We might be writing blocks with the wrong CID and then the good CID won't
work!
Correct, we might, in some corner cases.
In go-ipfs >= 0.12.0, all blocks are addressed by multihash so CID prefixes
are irrelevant. This problem does not exist in that case.
In go-ipfs < 0.12.0, if a read for a CIDv1 DAG-PB fails, it is retried as it
it was raw. This means that if we wrote something with cidv1/format=raw, that
should have been a cidv1/format=dag-pb, the read will still work. That covers
some common cases (i.e. adding with cid-version=1) because the first block
should be a raw-leaf. Default-params (cidv0) is not affected since everything
is raw multihashes. However, there are still possible CAR layouts etc. where
cluster will write blocks wrongly to older IPFS versions.
2022-04-11 18:26:06 +00:00
|
|
|
logger.Debugf("response block: %s", res.Key)
|
Adders: stream blocks to destinations
This commit fixes #810 and adds block streaming to the final destinations when
adding. This should add major performance gains when adding data to clusters.
Before, everytime cluster issued a block, it was broadcasted individually to
all destinations (new libp2p stream), where it was block/put to IPFS (a single
block/put http roundtrip per block).
Now, blocks are streamed all the way from the adder module to the ipfs daemon,
by making every block as it arrives a single part in a multipart block/put
request.
Before, block-broadcast needed to wait for all destinations to finish in order
to process the next block. Now, buffers allow some destinations to be faster
than others while sending and receiving blocks.
Before, if a block put request failed to be broadcasted everywhere, an error
would happen at that moment.
Now, we keep streaming until the end and only then report any errors. The
operation succeeds as long as at least one stream finished successfully.
Errors block/putting to IPFS will not abort streams. Instead, subsequent
blocks are retried with a new request, although the method will return an
error when the stream finishes if there were errors at any point.
2022-03-24 01:17:10 +00:00
|
|
|
if !it.Seen(res.Key) {
|
Fix warnings on block-cid mismatch when adding
Because we are adding blocks on a single call, and we choose the format
parameter based on the prefix of the first block, IPFS will return block CIDs
based on that option.
This caused warnings when adding content that has multiple CID prefixes: for
example, any cid-version=1 file will include both dag-pb CIDs and raw
CIDs. Since the first block is usually a leave, IPFS will only return
raw-cids, and cause a warning because of the CID-mistmatch.
This fixes things by comparing multihashes only.
But! We might be writing blocks with the wrong CID and then the good CID won't
work!
Correct, we might, in some corner cases.
In go-ipfs >= 0.12.0, all blocks are addressed by multihash so CID prefixes
are irrelevant. This problem does not exist in that case.
In go-ipfs < 0.12.0, if a read for a CIDv1 DAG-PB fails, it is retried as it
it was raw. This means that if we wrote something with cidv1/format=raw, that
should have been a cidv1/format=dag-pb, the read will still work. That covers
some common cases (i.e. adding with cid-version=1) because the first block
should be a raw-leaf. Default-params (cidv0) is not affected since everything
is raw multihashes. However, there are still possible CAR layouts etc. where
cluster will write blocks wrongly to older IPFS versions.
2022-04-11 18:26:06 +00:00
|
|
|
logger.Warningf("blockPut response CID (%s) does not match the multihash of any blocks sent", res.Key)
|
Adders: stream blocks to destinations
This commit fixes #810 and adds block streaming to the final destinations when
adding. This should add major performance gains when adding data to clusters.
Before, everytime cluster issued a block, it was broadcasted individually to
all destinations (new libp2p stream), where it was block/put to IPFS (a single
block/put http roundtrip per block).
Now, blocks are streamed all the way from the adder module to the ipfs daemon,
by making every block as it arrives a single part in a multipart block/put
request.
Before, block-broadcast needed to wait for all destinations to finish in order
to process the next block. Now, buffers allow some destinations to be faster
than others while sending and receiving blocks.
Before, if a block put request failed to be broadcasted everywhere, an error
would happen at that moment.
Now, we keep streaming until the end and only then report any errors. The
operation succeeds as long as at least one stream finished successfully.
Errors block/putting to IPFS will not abort streams. Instead, subsequent
blocks are retried with a new request, although the method will return an
error when the stream finishes if there were errors at any point.
2022-03-24 01:17:10 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
// continue until it.Done()
|
2020-03-06 15:35:15 +00:00
|
|
|
}
|
Adders: stream blocks to destinations
This commit fixes #810 and adds block streaming to the final destinations when
adding. This should add major performance gains when adding data to clusters.
Before, everytime cluster issued a block, it was broadcasted individually to
all destinations (new libp2p stream), where it was block/put to IPFS (a single
block/put http roundtrip per block).
Now, blocks are streamed all the way from the adder module to the ipfs daemon,
by making every block as it arrives a single part in a multipart block/put
request.
Before, block-broadcast needed to wait for all destinations to finish in order
to process the next block. Now, buffers allow some destinations to be faster
than others while sending and receiving blocks.
Before, if a block put request failed to be broadcasted everywhere, an error
would happen at that moment.
Now, we keep streaming until the end and only then report any errors. The
operation succeeds as long as at least one stream finished successfully.
Errors block/putting to IPFS will not abort streams. Instead, subsequent
blocks are retried with a new request, although the method will return an
error when the stream finishes if there were errors at any point.
2022-03-24 01:17:10 +00:00
|
|
|
|
|
|
|
return errs
|
2018-06-28 15:01:39 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// BlockGet retrieves an ipfs block with the given cid
|
2022-04-07 11:53:30 +00:00
|
|
|
func (ipfs *Connector) BlockGet(ctx context.Context, c api.Cid) ([]byte, error) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx, span := trace.StartSpan(ctx, "ipfsconn/ipfshttp/BlockGet")
|
|
|
|
defer span.End()
|
|
|
|
|
|
|
|
ctx, cancel := context.WithTimeout(ctx, ipfs.config.IPFSRequestTimeout)
|
2018-06-28 15:01:39 +00:00
|
|
|
defer cancel()
|
|
|
|
url := "block/get?arg=" + c.String()
|
|
|
|
return ipfs.postCtx(ctx, url, "", nil)
|
|
|
|
}
|
|
|
|
|
2019-02-20 14:24:25 +00:00
|
|
|
// // FetchRefs asks IPFS to download blocks recursively to the given depth.
|
|
|
|
// // It discards the response, but waits until it completes.
|
2022-04-07 11:53:30 +00:00
|
|
|
// func (ipfs *Connector) FetchRefs(ctx context.Context, c api.Cid, maxDepth int) error {
|
2019-02-20 14:24:25 +00:00
|
|
|
// ctx, cancel := context.WithTimeout(ipfs.ctx, ipfs.config.PinTimeout)
|
|
|
|
// defer cancel()
|
|
|
|
|
|
|
|
// q := url.Values{}
|
|
|
|
// q.Set("recursive", "true")
|
|
|
|
// q.Set("unique", "false") // same memory on IPFS side
|
|
|
|
// q.Set("max-depth", fmt.Sprintf("%d", maxDepth))
|
|
|
|
// q.Set("arg", c.String())
|
|
|
|
|
|
|
|
// url := fmt.Sprintf("refs?%s", q.Encode())
|
|
|
|
// err := ipfs.postDiscardBodyCtx(ctx, url)
|
|
|
|
// if err != nil {
|
|
|
|
// return err
|
|
|
|
// }
|
2020-02-03 09:30:04 +00:00
|
|
|
// logger.Debugf("refs for %s successfully fetched", c)
|
2019-02-20 14:24:25 +00:00
|
|
|
// return nil
|
|
|
|
// }
|
|
|
|
|
2018-08-07 09:49:31 +00:00
|
|
|
// Returns true every updateMetricsMod-th time that we
|
|
|
|
// call this function.
|
|
|
|
func (ipfs *Connector) shouldUpdateMetric() bool {
|
2022-01-31 16:44:35 +00:00
|
|
|
if ipfs.config.InformerTriggerInterval <= 0 {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
curCount := atomic.AddUint64(&ipfs.updateMetricCount, 1)
|
|
|
|
if curCount%uint64(ipfs.config.InformerTriggerInterval) == 0 {
|
|
|
|
atomic.StoreUint64(&ipfs.updateMetricCount, 0)
|
2018-08-07 09:49:31 +00:00
|
|
|
return true
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
// Trigger a broadcast of the local informer metrics.
|
2018-06-27 04:03:15 +00:00
|
|
|
func (ipfs *Connector) updateInformerMetric(ctx context.Context) error {
|
2019-02-16 00:11:07 +00:00
|
|
|
_, span := trace.StartSpan(ctx, "ipfsconn/ipfshttp/updateInformerMetric")
|
2018-06-27 04:03:15 +00:00
|
|
|
defer span.End()
|
2019-02-16 00:11:07 +00:00
|
|
|
ctx = trace.NewContext(ipfs.ctx, span)
|
2018-06-27 04:03:15 +00:00
|
|
|
|
2018-08-07 09:49:31 +00:00
|
|
|
if !ipfs.shouldUpdateMetric() {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
err := ipfs.rpcClient.GoContext(
|
2019-02-16 00:11:07 +00:00
|
|
|
ctx,
|
2018-08-07 09:49:31 +00:00
|
|
|
"",
|
|
|
|
"Cluster",
|
2019-12-05 14:08:43 +00:00
|
|
|
"SendInformersMetrics",
|
2018-08-07 09:49:31 +00:00
|
|
|
struct{}{},
|
2021-09-13 12:12:36 +00:00
|
|
|
&struct{}{},
|
2018-08-07 09:49:31 +00:00
|
|
|
nil,
|
|
|
|
)
|
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
}
|
|
|
|
return err
|
|
|
|
}
|