Merge pull request #262 from ipfs/fix/lock-service

ipfs-cluster-service lock

.travis.yml

@@ -17,7 +17,8 @@ script:
 - make test_sharness && make clean_sharness
 env:
   global:
-    secure: M3K3y9+D933tCda7+blW3qqVV8fA6PBDRdJoQvmQc1f0XYbWinJ+bAziFp6diKkF8sMQ+cPwLMONYJuaNT2h7/PkG+sIwF0PuUo5VVCbhGmSDrn2qOjmSnfawNs8wW31f44FQA8ICka1EFZcihohoIMf0e5xZ0tXA9jqw+ngPJiRnv4zyzC3r6t4JMAZcbS9w4KTYpIev5Yj72eCvk6lGjadSVCDVXo2sVs27tNt+BSgtMXiH6Sv8GLOnN2kFspGITgivHgB/jtU6QVtFXB+cbBJJAs3lUYnzmQZ5INecbjweYll07ilwFiCVNCX67+L15gpymKGJbQggloIGyTWrAOa2TMaB/bvblzwwQZ8wE5P3Rss5L0TFkUAcdU+3BUHM+TwV4e8F9x10v1PjgWNBRJQzd1sjKKgGUBCeyCY7VeYDKn9AXI5llISgY/AAfCZwm2cbckMHZZJciMjm+U3Q1FCF+rfhlvUcMG1VEj8r9cGpmWIRjFYVm0NmpUDDNjlC3/lUfTCOOJJyM254EUw63XxabbK6EtDN1yQe8kYRcXH//2rtEwgtMBgqHVY+OOkekzGz8Ra3EBkh6jXrAQL3zKu/GwRlK7/a1OU5MQ7dWcTjbx1AQ6Zfyjg5bZ+idqPgMbqM9Zn2+OaSby8HEEXS0QeZVooDVf/6wdYO4MQ/0A=
+    - secure: M3K3y9+D933tCda7+blW3qqVV8fA6PBDRdJoQvmQc1f0XYbWinJ+bAziFp6diKkF8sMQ+cPwLMONYJuaNT2h7/PkG+sIwF0PuUo5VVCbhGmSDrn2qOjmSnfawNs8wW31f44FQA8ICka1EFZcihohoIMf0e5xZ0tXA9jqw+ngPJiRnv4zyzC3r6t4JMAZcbS9w4KTYpIev5Yj72eCvk6lGjadSVCDVXo2sVs27tNt+BSgtMXiH6Sv8GLOnN2kFspGITgivHgB/jtU6QVtFXB+cbBJJAs3lUYnzmQZ5INecbjweYll07ilwFiCVNCX67+L15gpymKGJbQggloIGyTWrAOa2TMaB/bvblzwwQZ8wE5P3Rss5L0TFkUAcdU+3BUHM+TwV4e8F9x10v1PjgWNBRJQzd1sjKKgGUBCeyCY7VeYDKn9AXI5llISgY/AAfCZwm2cbckMHZZJciMjm+U3Q1FCF+rfhlvUcMG1VEj8r9cGpmWIRjFYVm0NmpUDDNjlC3/lUfTCOOJJyM254EUw63XxabbK6EtDN1yQe8kYRcXH//2rtEwgtMBgqHVY+OOkekzGz8Ra3EBkh6jXrAQL3zKu/GwRlK7/a1OU5MQ7dWcTjbx1AQ6Zfyjg5bZ+idqPgMbqM9Zn2+OaSby8HEEXS0QeZVooDVf/6wdYO4MQ/0A=
+    - verbose=t
 after_success:
 - openssl aes-256-cbc -K $encrypted_5a1cb914c6c9_key -iv $encrypted_5a1cb914c6c9_iv
   -in .snapcraft/travis_snapcraft.cfg -out .snapcraft/snapcraft.cfg -d

ipfs-cluster-service/lock.go

@@ -0,0 +1,58 @@
+// lock logic heavily inspired by go-ipfs/repo/fsrepo/lock/lock.go
+package main
+
+import (
+	"fmt"
+	"io"
+	"path"
+
+	filelock "go4.org/lock"
+)
+
+// The name of the file used for locking
+const lockFileName = "cluster.lock"
+
+var locker *Locker
+
+// Global file, close to relinquish lock access
+type Locker struct {
+	lockCloser io.Closer
+	path       string
+}
+
+func (locker *Locker) lock() error {
+	if locker.lockCloser != nil {
+		return fmt.Errorf("cannot acquire lock twice")
+	}
+	// set the lock file within this function
+	logger.Debug("checking lock")
+	lk, err := filelock.Lock(path.Join(locker.path, lockFileName))
+	if err != nil {
+		logger.Debug(err)
+		locker.lockCloser = nil
+		errStr := `could not obtain execution lock.  If no other process
+is running, remove ~/path/to/lock, or make sure that the config folder is 
+writable for the user running ipfs-cluster.  Run with -d for more information
+about the error`
+		logger.Error(errStr)
+		return fmt.Errorf("could not obtain execution lock.")
+	}
+	logger.Debug("Success! ipfs-cluster-service lock acquired")
+	locker.lockCloser = lk
+	return nil
+}
+
+func (locker *Locker) tryUnlock() error {
+	// Noop in the uninitialized case
+	if locker.lockCloser == nil {
+		logger.Debug("locking not initialized, unlock is noop")
+		return nil
+	}
+	err := locker.lockCloser.Close()
+	if err != nil {
+		return err
+	}
+	logger.Debug("Successfully released execution lock")
+	locker.lockCloser = nil
+	return nil
+}

ipfs-cluster-service/main.go

@@ -139,6 +139,10 @@ func out(m string, a ...interface{}) {
 func checkErr(doing string, err error) {
 	if err != nil {
 		out("error %s: %s\n", doing, err)
+		err = locker.tryUnlock()
+		if err != nil {
+			out("error releasing execution lock: %s\n", err)
+		}
 		os.Exit(1)
 	}
 }
@@ -257,7 +261,7 @@ removal, upgrade state using this command, and restart every peer.
 					Action: func(c *cli.Context) error {
 						err := upgrade()
 						checkErr("upgrading state", err)
-						return nil
+						return err
 					},
 				},
 			},
@@ -276,6 +280,9 @@ removal, upgrade state using this command, and restart every peer.
 		if c.Bool("debug") {
 			setupDebug()
 		}
+
+		locker = &Locker{path: absPath}
+
 		return nil
 	}
 
@@ -295,10 +302,16 @@ func run(c *cli.Context) error {
 func daemon(c *cli.Context) error {
 	// Load all the configurations
 	cfg, clusterCfg, apiCfg, ipfshttpCfg, consensusCfg, trackerCfg, monCfg, diskInfCfg, numpinInfCfg := makeConfigs()
+	// Execution lock
+	err := locker.lock()
+	checkErr("acquiring execution lock", err)
+	defer locker.tryUnlock()
+
+	// Load all the configurations
 	// always wait for configuration to be saved
 	defer cfg.Shutdown()
 
-	err := cfg.LoadJSONFromFile(configPath)
+	err = cfg.LoadJSONFromFile(configPath)
 	checkErr("loading configuration", err)
 
 	if a := c.String("bootstrap"); a != "" {

ipfs-cluster-service/migrate.go

@@ -12,7 +12,15 @@ import (
 func upgrade() error  {
 	// Load configs
 	cfg, clusterCfg, _, _, consensusCfg, _, _, _, _ := makeConfigs()
-	err := cfg.LoadJSONFromFile(configPath)
+	
+	// Execution lock
+	err := locker.lock()
+	if err != nil {
+		return err
+	}
+	defer locker.tryUnlock()
+
+	err = cfg.LoadJSONFromFile(configPath)
 	if err != nil {
 		return err
 	}

package.json

@@ -60,6 +60,12 @@
       "hash": "QmcWmYQEQCrezztaQ81nXzMx2jaAEow17wdesDAjjR769r",
       "name": "go-libp2p-pnet",
       "version": "2.3.1"
+    },
+    {
+      "author": "whyrusleeping",
+      "hash": "QmWi28zbQG6B1xfaaWx5cYoLn3kBFU6pQ6GWQNRV5P6dNe",
+      "name": "lock",
+      "version": "0.0.0"
     }
   ],
   "gxVersion": "0.11.0",

sharness/t0020-service-basic-commands.sh

@@ -27,4 +27,8 @@ test_expect_success "cluster-service --version succeeds and matches ctl" '
     [ "$SERV_VERSION" = "$CTL_VERSION" ]
 '
 
+test_expect_success "starting a second cluster-service process fails" '
+    test_expect_code 1 ipfs-cluster-service --config "test-config"    
+'
+
 test_done

sharness/t0050-service-state-upgrade-from-current.sh

@@ -14,7 +14,8 @@ test_expect_success IPFS,CLUSTER "cluster-service state upgrade works" '
     ipfs-cluster-ctl pin add "$cid" &&
     sleep 5 &&
     cluster_kill &&
-    ipfs-cluster-service --config "test-config" state upgrade
+    sleep 5 &&
+    ipfs-cluster-service --debug --config "test-config" state upgrade
 '
 
 # previous test kills the cluster, we need to re-start