Merge pull request #915 from ipfs/feat/proxy-logging
API logs for IPFS Proxy
This commit is contained in:
commit
582968f44e
|
@ -4,6 +4,7 @@ import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"path/filepath"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/kelseyhightower/envconfig"
|
"github.com/kelseyhightower/envconfig"
|
||||||
|
@ -46,6 +47,12 @@ type Config struct {
|
||||||
// Should we talk to the IPFS API over HTTPS? (experimental, untested)
|
// Should we talk to the IPFS API over HTTPS? (experimental, untested)
|
||||||
NodeHTTPS bool
|
NodeHTTPS bool
|
||||||
|
|
||||||
|
// LogFile is path of the file that would save Proxy API logs. If this
|
||||||
|
// path is empty, logs would be sent to standard output. This path
|
||||||
|
// should either be absolute or relative to cluster base directory. Its
|
||||||
|
// default value is empty.
|
||||||
|
LogFile string
|
||||||
|
|
||||||
// Maximum duration before timing out reading a full request
|
// Maximum duration before timing out reading a full request
|
||||||
ReadTimeout time.Duration
|
ReadTimeout time.Duration
|
||||||
|
|
||||||
|
@ -90,6 +97,8 @@ type jsonConfig struct {
|
||||||
NodeMultiaddress string `json:"node_multiaddress"`
|
NodeMultiaddress string `json:"node_multiaddress"`
|
||||||
NodeHTTPS bool `json:"node_https,omitempty"`
|
NodeHTTPS bool `json:"node_https,omitempty"`
|
||||||
|
|
||||||
|
LogFile string `json:"log_file"`
|
||||||
|
|
||||||
ReadTimeout string `json:"read_timeout"`
|
ReadTimeout string `json:"read_timeout"`
|
||||||
ReadHeaderTimeout string `json:"read_header_timeout"`
|
ReadHeaderTimeout string `json:"read_header_timeout"`
|
||||||
WriteTimeout string `json:"write_timeout"`
|
WriteTimeout string `json:"write_timeout"`
|
||||||
|
@ -101,6 +110,20 @@ type jsonConfig struct {
|
||||||
ExtractHeadersTTL string `json:"extract_headers_ttl,omitempty"`
|
ExtractHeadersTTL string `json:"extract_headers_ttl,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// getLogPath gets full path of the file where proxy logs should be
|
||||||
|
// saved.
|
||||||
|
func (cfg *Config) getLogPath() string {
|
||||||
|
if filepath.IsAbs(cfg.LogFile) {
|
||||||
|
return cfg.LogFile
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg.BaseDir == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
return filepath.Join(cfg.BaseDir, cfg.LogFile)
|
||||||
|
}
|
||||||
|
|
||||||
// ConfigKey provides a human-friendly identifier for this type of Config.
|
// ConfigKey provides a human-friendly identifier for this type of Config.
|
||||||
func (cfg *Config) ConfigKey() string {
|
func (cfg *Config) ConfigKey() string {
|
||||||
return configKey
|
return configKey
|
||||||
|
@ -118,6 +141,7 @@ func (cfg *Config) Default() error {
|
||||||
}
|
}
|
||||||
cfg.ListenAddr = proxy
|
cfg.ListenAddr = proxy
|
||||||
cfg.NodeAddr = node
|
cfg.NodeAddr = node
|
||||||
|
cfg.LogFile = ""
|
||||||
cfg.ReadTimeout = DefaultReadTimeout
|
cfg.ReadTimeout = DefaultReadTimeout
|
||||||
cfg.ReadHeaderTimeout = DefaultReadHeaderTimeout
|
cfg.ReadHeaderTimeout = DefaultReadHeaderTimeout
|
||||||
cfg.WriteTimeout = DefaultWriteTimeout
|
cfg.WriteTimeout = DefaultWriteTimeout
|
||||||
|
@ -222,6 +246,8 @@ func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error {
|
||||||
}
|
}
|
||||||
config.SetIfNotDefault(jcfg.NodeHTTPS, &cfg.NodeHTTPS)
|
config.SetIfNotDefault(jcfg.NodeHTTPS, &cfg.NodeHTTPS)
|
||||||
|
|
||||||
|
config.SetIfNotDefault(jcfg.LogFile, &cfg.LogFile)
|
||||||
|
|
||||||
err := config.ParseDurations(
|
err := config.ParseDurations(
|
||||||
"ipfsproxy",
|
"ipfsproxy",
|
||||||
&config.DurationOpt{Duration: jcfg.ReadTimeout, Dst: &cfg.ReadTimeout, Name: "read_timeout"},
|
&config.DurationOpt{Duration: jcfg.ReadTimeout, Dst: &cfg.ReadTimeout, Name: "read_timeout"},
|
||||||
|
@ -278,6 +304,7 @@ func (cfg *Config) toJSONConfig() (jcfg *jsonConfig, err error) {
|
||||||
jcfg.IdleTimeout = cfg.IdleTimeout.String()
|
jcfg.IdleTimeout = cfg.IdleTimeout.String()
|
||||||
jcfg.MaxHeaderBytes = cfg.MaxHeaderBytes
|
jcfg.MaxHeaderBytes = cfg.MaxHeaderBytes
|
||||||
jcfg.NodeHTTPS = cfg.NodeHTTPS
|
jcfg.NodeHTTPS = cfg.NodeHTTPS
|
||||||
|
jcfg.LogFile = cfg.LogFile
|
||||||
|
|
||||||
jcfg.ExtractHeadersExtra = cfg.ExtractHeadersExtra
|
jcfg.ExtractHeadersExtra = cfg.ExtractHeadersExtra
|
||||||
if cfg.ExtractHeadersPath != DefaultExtractHeadersPath {
|
if cfg.ExtractHeadersPath != DefaultExtractHeadersPath {
|
||||||
|
|
|
@ -9,16 +9,17 @@ import (
|
||||||
|
|
||||||
var cfgJSON = []byte(`
|
var cfgJSON = []byte(`
|
||||||
{
|
{
|
||||||
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
|
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
|
||||||
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
|
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
|
||||||
"read_timeout": "10m0s",
|
"log_file": "",
|
||||||
"read_header_timeout": "5s",
|
"read_timeout": "10m0s",
|
||||||
"write_timeout": "10m0s",
|
"read_header_timeout": "5s",
|
||||||
"idle_timeout": "1m0s",
|
"write_timeout": "10m0s",
|
||||||
"max_header_bytes": 16384,
|
"idle_timeout": "1m0s",
|
||||||
"extract_headers_extra": [],
|
"max_header_bytes": 16384,
|
||||||
"extract_headers_path": "/api/v0/version",
|
"extract_headers_extra": [],
|
||||||
"extract_headers_ttl": "5m"
|
"extract_headers_path": "/api/v0/version",
|
||||||
|
"extract_headers_ttl": "5m"
|
||||||
}
|
}
|
||||||
`)
|
`)
|
||||||
|
|
||||||
|
|
|
@ -4,10 +4,12 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httputil"
|
"net/http/httputil"
|
||||||
"net/url"
|
"net/url"
|
||||||
|
"os"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
@ -17,6 +19,7 @@ import (
|
||||||
"github.com/ipfs/ipfs-cluster/api"
|
"github.com/ipfs/ipfs-cluster/api"
|
||||||
"github.com/ipfs/ipfs-cluster/rpcutil"
|
"github.com/ipfs/ipfs-cluster/rpcutil"
|
||||||
|
|
||||||
|
handlers "github.com/gorilla/handlers"
|
||||||
mux "github.com/gorilla/mux"
|
mux "github.com/gorilla/mux"
|
||||||
cid "github.com/ipfs/go-cid"
|
cid "github.com/ipfs/go-cid"
|
||||||
logging "github.com/ipfs/go-log"
|
logging "github.com/ipfs/go-log"
|
||||||
|
@ -34,7 +37,10 @@ import (
|
||||||
// DNSTimeout is used when resolving DNS multiaddresses in this module
|
// DNSTimeout is used when resolving DNS multiaddresses in this module
|
||||||
var DNSTimeout = 5 * time.Second
|
var DNSTimeout = 5 * time.Second
|
||||||
|
|
||||||
var logger = logging.Logger("ipfsproxy")
|
var (
|
||||||
|
logger = logging.Logger("ipfsproxy")
|
||||||
|
proxyLogger = logging.Logger("ipfsproxylog")
|
||||||
|
)
|
||||||
|
|
||||||
// Server offers an IPFS API, hijacking some interesting requests
|
// Server offers an IPFS API, hijacking some interesting requests
|
||||||
// and forwarding the rest to the ipfs daemon
|
// and forwarding the rest to the ipfs daemon
|
||||||
|
@ -87,6 +93,14 @@ type ipfsAddResp struct {
|
||||||
Size string `json:",omitempty"`
|
Size string `json:",omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type logWriter struct {
|
||||||
|
}
|
||||||
|
|
||||||
|
func (lw logWriter) Write(b []byte) (int, error) {
|
||||||
|
proxyLogger.Infof(string(b))
|
||||||
|
return len(b), nil
|
||||||
|
}
|
||||||
|
|
||||||
// New returns and ipfs Proxy component
|
// New returns and ipfs Proxy component
|
||||||
func New(cfg *Config) (*Server, error) {
|
func New(cfg *Config) (*Server, error) {
|
||||||
err := cfg.Validate()
|
err := cfg.Validate()
|
||||||
|
@ -148,12 +162,23 @@ func New(cfg *Config) (*Server, error) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var writer io.Writer
|
||||||
|
if cfg.LogFile != "" {
|
||||||
|
f, err := os.OpenFile(cfg.getLogPath(), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
writer = f
|
||||||
|
} else {
|
||||||
|
writer = logWriter{}
|
||||||
|
}
|
||||||
|
|
||||||
s := &http.Server{
|
s := &http.Server{
|
||||||
ReadTimeout: cfg.ReadTimeout,
|
ReadTimeout: cfg.ReadTimeout,
|
||||||
WriteTimeout: cfg.WriteTimeout,
|
WriteTimeout: cfg.WriteTimeout,
|
||||||
ReadHeaderTimeout: cfg.ReadHeaderTimeout,
|
ReadHeaderTimeout: cfg.ReadHeaderTimeout,
|
||||||
IdleTimeout: cfg.IdleTimeout,
|
IdleTimeout: cfg.IdleTimeout,
|
||||||
Handler: handler,
|
Handler: handlers.LoggingHandler(writer, handler),
|
||||||
MaxHeaderBytes: cfg.MaxHeaderBytes,
|
MaxHeaderBytes: cfg.MaxHeaderBytes,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -7,6 +7,8 @@ import (
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
@ -762,3 +764,69 @@ func TestAttackHeaderSize(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestProxyLogging(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
cfg := &Config{}
|
||||||
|
cfg.Default()
|
||||||
|
|
||||||
|
logFile, err := filepath.Abs("proxy.log")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cfg.LogFile = logFile
|
||||||
|
|
||||||
|
proxy, mock := testIPFSProxyWithConfig(t, cfg)
|
||||||
|
defer os.Remove(cfg.LogFile)
|
||||||
|
|
||||||
|
info, err := os.Stat(cfg.LogFile)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if info.Size() > 0 {
|
||||||
|
t.Errorf("expected empty log file")
|
||||||
|
}
|
||||||
|
|
||||||
|
res, err := http.Post(fmt.Sprintf("%s/version", proxyURL(proxy)), "", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("should forward requests to ipfs host: ", err)
|
||||||
|
}
|
||||||
|
res.Body.Close()
|
||||||
|
|
||||||
|
info, err = os.Stat(cfg.LogFile)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
size1 := info.Size()
|
||||||
|
if size1 == 0 {
|
||||||
|
t.Error("did not expect an empty log file")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Restart proxy and make sure that logs are being appended
|
||||||
|
mock.Close()
|
||||||
|
proxy.Shutdown(ctx)
|
||||||
|
|
||||||
|
proxy, mock = testIPFSProxyWithConfig(t, cfg)
|
||||||
|
defer mock.Close()
|
||||||
|
defer proxy.Shutdown(ctx)
|
||||||
|
|
||||||
|
res1, err := http.Post(fmt.Sprintf("%s/version", proxyURL(proxy)), "", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("should forward requests to ipfs host: ", err)
|
||||||
|
}
|
||||||
|
res1.Body.Close()
|
||||||
|
|
||||||
|
info, err = os.Stat(cfg.LogFile)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
size2 := info.Size()
|
||||||
|
if size2 == 0 {
|
||||||
|
t.Error("did not expect an empty log file")
|
||||||
|
}
|
||||||
|
|
||||||
|
if !(size2 > size1) {
|
||||||
|
t.Error("logs were not appended")
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
|
@ -26,6 +26,7 @@ var LoggingFacilities = map[string]string{
|
||||||
"restapi": "INFO",
|
"restapi": "INFO",
|
||||||
"restapilog": "INFO",
|
"restapilog": "INFO",
|
||||||
"ipfsproxy": "INFO",
|
"ipfsproxy": "INFO",
|
||||||
|
"ipfsproxylog": "INFO",
|
||||||
"ipfshttp": "INFO",
|
"ipfshttp": "INFO",
|
||||||
"monitor": "INFO",
|
"monitor": "INFO",
|
||||||
"dsstate": "INFO",
|
"dsstate": "INFO",
|
||||||
|
|
Loading…
Reference in New Issue
Block a user