Fix #382: Add TTL for cached headers

License: MIT
Signed-off-by: Hector Sanjuan <code@hector.link>

api/ipfsproxy/config_test.go

@@ -55,7 +55,7 @@ func TestLoadJSON(t *testing.T) {
 
 	j = &jsonConfig{}
 	json.Unmarshal(cfgJSON, j)
-	j.ExtractHeadersTTL = -10
+	j.ExtractHeadersTTL = "-10"
 	tst, _ = json.Marshal(j)
 	err = cfg.LoadJSON(tst)
 	if err == nil {

api/ipfsproxy/headers.go

@@ -3,6 +3,7 @@ package ipfsproxy
 import (
 	"fmt"
 	"net/http"
+	"time"
 
 	"github.com/ipfs/ipfs-cluster/version"
 )
@@ -45,6 +46,8 @@ var extractHeadersDefault = []string{
 	"Access-Control-Expose-Headers",
 }
 
+const ipfsHeadersTimestampKey = "proxyHeadersTS"
+
 // ipfsHeaders returns all the headers we want to extract-once from IPFS: a
 // concatenation of extractHeadersDefault and config.ExtractHeadersExtra.
 func (proxy *Server) ipfsHeaders() []string {
@@ -57,14 +60,47 @@ func (proxy *Server) rememberIPFSHeaders(hdrs http.Header) {
 	for _, h := range proxy.ipfsHeaders() {
 		proxy.ipfsHeadersStore.Store(h, hdrs[h])
 	}
+	// use the sync map to store the ts
+	proxy.ipfsHeadersStore.Store(ipfsHeadersTimestampKey, time.Now())
+}
+
+// returns whether we can consider that whatever headers we are
+// storing have a valid TTL still.
+func (proxy *Server) headersWithinTTL() bool {
+	ttl := proxy.config.ExtractHeadersTTL
+	if ttl == 0 {
+		return true
+	}
+
+	tsRaw, ok := proxy.ipfsHeadersStore.Load(ipfsHeadersTimestampKey)
+	if !ok {
+		return false
+	}
+
+	ts, ok := tsRaw.(time.Time)
+	if !ok {
+		return false
+	}
+
+	lifespan := time.Since(ts)
+	return lifespan < ttl
 }
 
 // rememberIPFSHeaders adds the known IPFS Headers to the destination
-// and returns true if we could set all the headers in the list.
+// and returns true if we could set all the headers in the list and
+// the TTL has not expired.
 // False is used to determine if we need to make a request to try
 // to extract these headers.
 func (proxy *Server) setIPFSHeaders(dest http.Header) bool {
 	r := true
+
+	if !proxy.headersWithinTTL() {
+		r = false
+		// still set those headers we can set in the destination.
+		// We do our best there, since maybe the ipfs daemon
+		// is down and what we have now is all we can use.
+	}
+
 	for _, h := range proxy.ipfsHeaders() {
 		v, ok := proxy.ipfsHeadersStore.Load(h)
 		if !ok {

api/ipfsproxy/ipfsproxy_test.go

@@ -8,6 +8,7 @@ import (
 	"net/url"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/ipfs/ipfs-cluster/api"
 	"github.com/ipfs/ipfs-cluster/test"
@@ -30,7 +31,10 @@ func testIPFSProxy(t *testing.T) (*Server, *test.IpfsMock) {
 	cfg.Default()
 	cfg.NodeAddr = nodeMAddr
 	cfg.ListenAddr = proxyMAddr
-	cfg.ExtractHeadersExtra = []string{test.IpfsCustomHeaderName}
+	cfg.ExtractHeadersExtra = []string{
+		test.IpfsCustomHeaderName,
+		test.IpfsTimeHeaderName,
+	}
 
 	proxy, err := New(cfg)
 	if err != nil {
@@ -518,6 +522,7 @@ func mustParseURL(rawurl string) *url.URL {
 
 func TestHeaderExtraction(t *testing.T) {
 	proxy, mock := testIPFSProxy(t)
+	proxy.config.ExtractHeadersTTL = time.Second
 	defer mock.Close()
 	defer proxy.Shutdown()
 
@@ -554,4 +559,25 @@ func TestHeaderExtraction(t *testing.T) {
 	if !strings.HasPrefix(res.Header.Get("Server"), "ipfs-cluster") {
 		t.Error("wrong value for Server header")
 	}
+
+	// Test ExtractHeaderTTL
+	t1 := res.Header.Get(test.IpfsTimeHeaderName)
+	res, err = http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatal("should forward requests to ipfs host: ", err)
+	}
+	t2 := res.Header.Get(test.IpfsTimeHeaderName)
+	if t1 != t2 {
+		t.Error("should have cached the headers during TTL")
+	}
+	time.Sleep(1200 * time.Millisecond)
+	res, err = http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatal("should forward requests to ipfs host: ", err)
+	}
+	res.Body.Close()
+	t3 := res.Header.Get(test.IpfsTimeHeaderName)
+	if t3 == t2 {
+		t.Error("should have refreshed the headers after TTL")
+	}
 }

test/ipfs_mock.go

@@ -9,10 +9,12 @@ import (
 	"net/url"
 	"strconv"
 	"strings"
+	"time"
+
+	"github.com/rs/cors"
 
 	"github.com/ipfs/ipfs-cluster/api"
 	"github.com/ipfs/ipfs-cluster/state/mapstate"
-	"github.com/rs/cors"
 
 	cid "github.com/ipfs/go-cid"
 	u "github.com/ipfs/go-ipfs-util"
@@ -20,6 +22,7 @@ import (
 
 var (
 	IpfsCustomHeaderName  = "X-Custom-Header"
+	IpfsTimeHeaderName    = "X-Time-Now"
 	IpfsCustomHeaderValue = "42"
 	IpfsACAOrigin         = "myorigin"
 )
@@ -128,6 +131,7 @@ func (m *IpfsMock) handler(w http.ResponseWriter, r *http.Request) {
 	p := r.URL.Path
 	w.Header().Set(IpfsCustomHeaderName, IpfsCustomHeaderValue)
 	w.Header().Set("Server", "ipfs-mock")
+	w.Header().Set(IpfsTimeHeaderName, fmt.Sprintf("%d", time.Now().Unix()))
 	endp := strings.TrimPrefix(p, "/api/v0/")
 	switch endp {
 	case "id":