add cascade-release.sh to make releases

Changelog for v1.0.8

.github/workflows/build.yaml

@@ -0,0 +1,20 @@
+name: Github Actions Demo
+run-name: ${{ github.actor }} is testing out GitHub Actions (on Gitea) 🚀
+on: [push]
+
+jobs:
+  Explore-Gitea-Actions:
+    runs-on: metal-docker
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+      - run: |
+          ls -la
+          docker build --progress plain --iidfile iid.txt .
+          stringprefix() { [ ${#1} -le $2 ] && echo $1 && return 0 || stringprefix "${1%?}" $2 ; }
+          SHORTSHA="$(stringprefix "$GITHUB_SHA" 8)"
+          for TAG in "$SHORTSHA" "$GITHUB_REF_NAME";do
+            docker tag "`cat iid.txt`" "$DOCKER_REGISTRY/$GITHUB_REPOSITORY:$TAG"
+            docker push "$DOCKER_REGISTRY/$GITHUB_REPOSITORY:$TAG"
+          done
+

.github/workflows/codeql-analysis.yml

@@ -1,80 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: "CodeQL"
-
-on:
-  push:
-    branches: [ "master" ]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [ "master" ]
-  schedule:
-    - cron: '20 2 * * 5'
-
-env:
-  GO: 1.21.4
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [ 'go' ]
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
-        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
-
-    - name: Set up Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: ${{ env.GO }}
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-        
-        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
-        # queries: security-extended,security-and-quality
-
-        
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
-
-    #   If the Autobuild fails above, remove it and uncomment the following three lines. 
-    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
-
-    # - run: |
-    #   echo "Run, Build Application using script"
-    #   ./location_of_script_within_repo/buildscript.sh
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2

.github/workflows/docker-image.yml

@@ -1,70 +0,0 @@
-name: Publish Docker image
-
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - 'master'
-    tags:
-      - 'v*'
-
-env:
-  REGISTRY: ""
-  IMAGE_NAME: ipfs/ipfs-cluster
-
-jobs:
-  build-and-push-image:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-      
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Cache Docker layers
-        uses: actions/cache@v3
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-
-
-      - name: Log in to the Container registry
-        uses: docker/login-action@v3
-        with:
-          # registry: ${{ env.REGISTRY }}
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - name: Get tags
-        id: tags
-        run: |
-          echo "value<<EOF" >> $GITHUB_OUTPUT
-          ./docker/get-docker-tags.sh "$(date -u +%F)" >> $GITHUB_OUTPUT
-          echo "EOF" >> $GITHUB_OUTPUT
-        shell: bash
-
-      - name: Build Docker image and publish to Docker Hub
-        uses: docker/build-push-action@v4
-        with:
-          platforms: linux/amd64,linux/arm/v7,linux/arm64/v8
-          context: .
-          push: true
-          file: ./Dockerfile
-          tags: "${{ steps.tags.outputs.value }}"
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache-new
-
-      # https://github.com/docker/build-push-action/issues/252
-      # https://github.com/moby/buildkit/issues/1896
-      - name: Move cache to limit growth
-        run: |
-          rm -rf /tmp/.buildx-cache
-          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

.github/workflows/stale.yml

@@ -1,26 +0,0 @@
-name: Close and mark stale issue
-
-on:
-  schedule:
-  - cron: '0 0 * * *'
-
-jobs:
-  stale:
-
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-      pull-requests: write
-
-    steps:
-    - uses: actions/stale@v3
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        stale-issue-message: 'Oops, seems like we needed more information for this issue, please comment with more details or this issue will be closed in 7 days.'
-        close-issue-message: 'This issue was closed because it is missing author input.'
-        stale-issue-label: 'kind/stale'
-        any-of-labels: 'need/author-input'
-        exempt-issue-labels: 'need/triage,need/community-input,need/maintainer-input,need/maintainers-input,need/analysis'
-        days-before-issue-stale: 6
-        days-before-issue-close: 7
-        enable-statistics: true

.github/workflows/tests.yml

@@ -1,170 +0,0 @@
-name: Tests
-
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    branches: [ master, dependency-upgrades ]
-
-env:
-  GO: 1.21.4
-
-jobs:
-  tests-all:
-    name: "All modules + Coverage"
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-      with:
-        fetch-depth: 2
-
-    - name: Set up Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: ${{ env.GO }}
-
-    - name: "Tests"
-      run: go test -v -timeout 15m -coverprofile=coverage.txt -covermode=atomic ./...
-
-    - name: "Coverage"
-      uses: codecov/codecov-action@v2
-
-  tests-raft:
-    name: "Using Raft"
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-      with:
-        fetch-depth: 2
-
-    - name: Set up Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: ${{ env.GO }}
-
-    - name: "Tests"
-      run: go test -v -timeout 15m -failfast -consensus raft .
-
-  tests-leveldb:
-    name: "Using LevelDB"
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-      with:
-        fetch-depth: 2
-
-    - name: Set up Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: ${{ env.GO }}
-
-    - name: "Tests"
-      run: go test -v -timeout 15m -failfast -datastore leveldb .
-
-  tests-badger3:
-    name: "Using Badger3"
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-      with:
-        fetch-depth: 2
-
-    - name: Set up Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: ${{ env.GO }}
-
-    - name: "Tests"
-      run: go test -v -timeout 15m -failfast -datastore badger3 .
-
-  tests-badger:
-    name: "Using Badger"
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-      with:
-        fetch-depth: 2
-
-    - name: Set up Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: ${{ env.GO }}
-
-    - name: "Tests"
-      run: go test -v -timeout 15m -failfast -datastore badger .
-
-      
-  tests-check:
-    name: "Build, syntax and spelling checks"
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-      with:
-        fetch-depth: 2
-
-    - name: Set up Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: ${{ env.GO }}
-
-    - name: Install staticcheck
-      run: go install honnef.co/go/tools/cmd/staticcheck@latest
-
-    - name: Install misspell
-      run: go install github.com/client9/misspell/cmd/misspell@latest
-
-    - name: Check
-      run: make check
-
-    - name: Build binaries
-      run: make build
-
-  docker:
-    name: "Docker build"
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-      with:
-        fetch-depth: 2
-
-    - name: Docker builds
-      run: make docker
-
-  docker-compose:
-    name: "Docker Compose"
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-      with:
-        fetch-depth: 2
-
-    - name: Docker compose
-      run: make docker-compose
-
-  sharness:
-    name: Sharness
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-      with:
-        fetch-depth: 2
-
-    - name: Set up Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: ${{ env.GO }}
-
-    - name: Apt-get update
-      run: sudo apt-get update
-
-    - name: Install jq and curl
-      run: sudo apt-get install -y jq curl
-
-    - name: Make install
-      run: make install
-
-    - name: Pull kubo
-      run: docker pull ipfs/kubo
-
-    - name: Sharness
-      run: make test_sharness

CHANGELOG.md

@@ -1,5 +1,74 @@
 # IPFS Cluster Changelog
 
+### v1.0.8 - 2024-01-30
+
+IPFS Cluster v1.0.8 is a maintenance release.
+
+This release updates dependencies (latest boxo and libp2p) and should bring a couple of Pebble-related improvements:
+  * We have upgraded Pebble's version. Some users have reported deadlocks in writes to Pebble ([ipfs/ipfs-cluster#2009](https://github.com/ipfs/ipfs-cluster/issues/2009)) and this seems to have helped.
+  * Pebble now supports 32-bit so it can be the default for all archs.
+  * We added a warning when Pebble's newest `MajorFormatVersion` is higher than
+  what is used in the configuration. **Users should increase their `major_format_version`
+  to maintain forward-compatibility with future versions of Pebble.**
+
+Additionally, some bugs have been fixed and a couple of useful features added, as mentioned below.
+
+#### List of changes
+
+##### Breaking changes
+
+There are no breaking changes on this release.
+
+##### Features
+
+* ipfshttp: support talking to Kubo over unix sockets | [ipfs/ipfs-cluster#1999](https://github.com/ipfs/ipfs-cluster/issues/1999)
+* ipfsproxy: support talking to Kubo over unix sockets | [ipfs/ipfs-cluster#2027](https://github.com/ipfs/ipfs-cluster/issues/2027)
+* pebble: enable in all archs as default datastore | [ipfs/ipfs-cluster#2005](https://github.com/ipfs/ipfs-cluster/issues/2005) | [ipfs/ipfs-cluster#2007](https://github.com/ipfs/ipfs-cluster/issues/2007)
+* pebble: set default MajorVersionFormat to newest | [ipfs/ipfs-cluster#2019](https://github.com/ipfs/ipfs-cluster/issues/2019)
+* cluster: Announce and NoAnnounce options | [ipfs/ipfs-cluster#952](https://github.com/ipfs/ipfs-cluster/issues/952) | [ipfs/ipfs-cluster#2010](https://github.com/ipfs/ipfs-cluster/issues/2010)
+
+
+##### Bug fixes
+
+* ipfs-cluster-follow: issue numpin and pinqueue metrics to other peers | [ipfs/ipfs-cluster#2011](https://github.com/ipfs/ipfs-cluster/issues/2011) | [ipfs/ipfs-cluster#2016](https://github.com/ipfs/ipfs-cluster/issues/2016)
+* ipfshttp: do no pre-resolve node_multiaddresses | [ipfs/ipfs-cluster#2004](https://github.com/ipfs/ipfs-cluster/issues/2004) | [ipfs/ipfs-cluster#2017](https://github.com/ipfs/ipfs-cluster/issues/2017)
+* ipfsproxy: do no pre-resolve node_multiaddresses | [ipfs/ipfs-cluster#2027](https://github.com/ipfs/ipfs-cluster/issues/2027)
+* pebble: deadlock | [ipfs/ipfs-cluster#2009](https://github.com/ipfs/ipfs-cluster/issues/2009)
+
+##### Other changes
+
+* The `Dockerfile-bundle` file has been removed (unmaintained) | [ipfs/ipfs-cluster#1986](https://github.com/ipfs/ipfs-cluster/issues/1986)
+* Dependency upgrades | [ipfs/ipfs-cluster#2007](https://github.com/ipfs/ipfs-cluster/issues/2007) | [ipfs/ipfs-cluster#2018](https://github.com/ipfs/ipfs-cluster/issues/2018) | [ipfs/ipfs-cluster#2026](https://github.com/ipfs/ipfs-cluster/issues/2026)
+
+#### Upgrading notices
+
+##### Configuration changes
+
+Two new options have been added to forcefully control the cluster peer libp2p host address announcements: `cluster.announce_multiaddress` and `cluster.no_announce_multiaddress`. Both take a slice of multiaddresses.
+
+##### REST API
+
+No changes.
+
+##### Pinning Service API
+
+No changes.
+
+##### IPFS Proxy API
+
+No changes.
+
+##### Go APIs
+
+No relevant changes.
+
+##### Other
+
+Nothing.
+
+---
+
+
 ### v1.0.7 - 2023-10-12
 
 IPFS Cluster v1.0.7 is a maintenance release.

Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.21-bullseye AS builder
+FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.21-alpine3.19 AS builder
 MAINTAINER Hector Sanjuan <code@hector.link>
 
 # This dockerfile builds and runs ipfs-cluster-service.
@@ -8,6 +8,8 @@ ENV GOPATH      /go
 ENV SRC_PATH    $GOPATH/src/github.com/ipfs-cluster/ipfs-cluster
 ENV GOPROXY     https://proxy.golang.org
 
+RUN apk add --no-cache git alpine-sdk
+
 COPY --chown=1000:users go.* $SRC_PATH/
 WORKDIR $SRC_PATH
 RUN go mod download -x
@@ -20,7 +22,7 @@ RUN make install
 
 
 #------------------------------------------------------
-FROM alpine:3.18
+FROM alpine:3.19
 MAINTAINER Hector Sanjuan <hector@protocol.ai>
 
 LABEL org.opencontainers.image.source=https://github.com/ipfs-cluster/ipfs-cluster
@@ -28,7 +30,7 @@ LABEL org.opencontainers.image.description="Pinset orchestration for IPFS"
 LABEL org.opencontainers.image.licenses=MIT+APACHE_2.0
 
 # Install binaries for $TARGETARCH
-RUN apk add --no-cache tini su-exec ca-certificates
+RUN apk add --no-cache tini su-exec ca-certificates curl yq
 
 ENV GOPATH                 /go
 ENV SRC_PATH               /go/src/github.com/ipfs-cluster/ipfs-cluster

api/ipfsproxy/ipfsproxy.go

@@ -24,6 +24,7 @@ import (
 	"github.com/ipfs-cluster/ipfs-cluster/adder/adderutils"
 	"github.com/ipfs-cluster/ipfs-cluster/api"
 	"github.com/ipfs-cluster/ipfs-cluster/rpcutil"
+	"github.com/tv42/httpunix"
 
 	handlers "github.com/gorilla/handlers"
 	mux "github.com/gorilla/mux"
@@ -33,7 +34,6 @@ import (
 	logging "github.com/ipfs/go-log/v2"
 	rpc "github.com/libp2p/go-libp2p-gorpc"
 	peer "github.com/libp2p/go-libp2p/core/peer"
-	madns "github.com/multiformats/go-multiaddr-dns"
 	manet "github.com/multiformats/go-multiaddr/net"
 
 	"go.opencensus.io/plugin/ochttp"
@@ -58,13 +58,16 @@ type Server struct {
 	ctx    context.Context
 	cancel func()
 
-	config     *Config
+	config      *Config
-	nodeScheme string
+	nodeScheme  string
-	nodeAddr   string
+	nodeAddr    string
+	nodeNetwork string
 
 	rpcClient *rpc.Client
 	rpcReady  chan struct{}
 
+	transport http.RoundTripper // to the proxied kubo RPC API
+
 	listeners    []net.Listener         // proxy listener
 	server       *http.Server           // proxy server
 	reverseProxy *httputil.ReverseProxy // allows to talk to IPFS
@@ -111,24 +114,23 @@ func New(cfg *Config) (*Server, error) {
 		return nil, err
 	}
 
-	nodeMAddr := cfg.NodeAddr
+	nodeNetwork, nodeAddr, err := manet.DialArgs(cfg.NodeAddr)
-	// dns multiaddresses need to be resolved first
-	if madns.Matches(nodeMAddr) {
-		ctx, cancel := context.WithTimeout(context.Background(), DNSTimeout)
-		defer cancel()
-		resolvedAddrs, err := madns.Resolve(ctx, cfg.NodeAddr)
-		if err != nil {
-			logger.Error(err)
-			return nil, err
-		}
-		nodeMAddr = resolvedAddrs[0]
-	}
-
-	_, nodeAddr, err := manet.DialArgs(nodeMAddr)
 	if err != nil {
 		return nil, err
 	}
 
+	transport := http.DefaultTransport
+
+	if nodeNetwork == "unix" {
+		unixTransport := &httpunix.Transport{
+			DialTimeout: time.Second,
+		}
+		unixTransport.RegisterLocation("ipfs", nodeAddr)
+		t := &http.Transport{}
+		t.RegisterProtocol(httpunix.Scheme, unixTransport)
+		transport = t
+	}
+
 	var listeners []net.Listener
 	for _, addr := range cfg.ListenAddr {
 		proxyNet, proxyAddr, err := manet.DialArgs(addr)
@@ -195,14 +197,16 @@ func New(cfg *Config) (*Server, error) {
 	s.SetKeepAlivesEnabled(true) // A reminder that this can be changed
 
 	reverseProxy := httputil.NewSingleHostReverseProxy(proxyURL)
-	reverseProxy.Transport = http.DefaultTransport
+	reverseProxy.Transport = transport
 	ctx, cancel := context.WithCancel(context.Background())
 	proxy := &Server{
 		ctx:          ctx,
 		config:       cfg,
 		cancel:       cancel,
 		nodeAddr:     nodeHTTPAddr,
+		nodeNetwork:  nodeNetwork,
 		nodeScheme:   nodeScheme,
+		transport:    transport,
 		rpcReady:     make(chan struct{}, 1),
 		listeners:    listeners,
 		server:       s,

cascade-release.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e
+
+if [ x"$(git branch --show)" != x"cascade" ];then
+  echo "this cannot be run on any branches other than cascade"
+  exit 1
+fi
+
+git fetch upstream
+git checkout -b cascade-"$1"
+git rebase "$1" || (git checkout cascade && git branch -D cascade-"$1";exit 1)
+git push -u origin cascade-"$1"
+git checkout cascade

cmd/ipfs-cluster-ctl/main.go

@@ -27,7 +27,7 @@ const programName = `ipfs-cluster-ctl`
 
 // Version is the cluster-ctl tool version. It should match
 // the IPFS cluster's version
-const Version = "1.0.7"
+const Version = "1.0.8"
 
 var (
 	defaultHost          = "/ip4/127.0.0.1/tcp/9094"

datastore/pebble/config.go

@@ -66,7 +66,8 @@ var (
 	// DefaultFilterPolicy defines the number of bits used per key for
 	// bloom filters. 10 yields a 1% false positive rate.
 	DefaultFilterPolicy bloom.FilterPolicy = 10 // Pebble's default: 10
-
+	// DefaultFormatMajorVersion sets the format of Pebble on-disk files.
+	DefaultFormatMajorVersion = pebble.FormatNewest
 )
 
 func init() {
@@ -219,6 +220,7 @@ func (cfg *Config) Default() error {
 	cfg.PebbleOptions = DefaultPebbleOptions
 	cache := pebble.NewCache(DefaultCacheSize)
 	cfg.PebbleOptions.Cache = cache
+	cfg.PebbleOptions.FormatMajorVersion = DefaultFormatMajorVersion
 	cfg.PebbleOptions.MemTableSize = DefaultMemTableSize
 	cfg.PebbleOptions.MemTableStopWritesThreshold = DefaultMemTableStopWritesThreshold
 	cfg.PebbleOptions.BytesPerSync = DefaultBytesPerSync

datastore/pebble/pebble.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"time"
 
+	"github.com/cockroachdb/pebble"
 	ds "github.com/ipfs/go-datastore"
 	pebbleds "github.com/ipfs/go-ds-pebble"
 	logging "github.com/ipfs/go-log/v2"
@@ -24,6 +25,19 @@ func New(cfg *Config) (ds.Datastore, error) {
 		return nil, errors.Wrap(err, "creating pebble folder")
 	}
 
+	// Deal with Pebble updates... user should try to be up to date with
+	// latest Pebble table formats.
+	fmv := cfg.PebbleOptions.FormatMajorVersion
+	newest := pebble.FormatNewest
+	if fmv < newest {
+		logger.Warnf(`Pebble's format_major_version is set to %d, but newest version is %d.
+
+It is recommended to increase format_major_version and restart. If an error
+occurrs when increasing the number several versions at once, it may help to
+increase them one by one, restarting the daemon every time.
+`, fmv, newest)
+	}
+
 	db, err := pebbleds.NewDatastore(folder, &cfg.PebbleOptions)
 	if err != nil {
 		return nil, err
@@ -32,11 +46,13 @@ func New(cfg *Config) (ds.Datastore, error) {
 	// Calling regularly DB's DiskUsage is a way to printout debug
 	// database statistics.
 	go func() {
+		ctx := context.Background()
+		db.DiskUsage(ctx)
 		ticker := time.NewTicker(time.Minute)
 		defer ticker.Stop()
 		for {
 			<-ticker.C
-			db.DiskUsage(context.Background())
+			db.DiskUsage(ctx)
 		}
 	}()
 	return db, nil

version/version.go

@@ -7,7 +7,7 @@ import (
 )
 
 // Version is the current cluster version.
-var Version = semver.MustParse("1.0.7")
+var Version = semver.MustParse("1.0.8")
 
 // RPCProtocol is protocol handler used to send libp2p-rpc messages between
 // cluster peers.  All peers in the cluster need to speak the same protocol