Fix: close files. Close test files. Clean properly. Fix windows tests.

License: MIT
Signed-off-by: Hector Sanjuan <code@hector.link>

adder/adder.go

@@ -145,6 +145,7 @@ func addFile(fs files.File, ipfsAdder *ipfsadd.Adder) error {
 	if err != nil {
 		return err
 	}
+	defer f.Close()
 
 	logger.Debugf("ipfsAdder AddFile(%s)", f.FullPath())
 	return ipfsAdder.AddFile(f)

adder/adder_test.go

@@ -40,9 +40,10 @@ func (dag *mockCDagServ) Finalize(ctx context.Context, root *cid.Cid) (*cid.Cid,
 
 func TestAdder(t *testing.T) {
 	sth := test.NewShardingTestHelper()
-	defer sth.Clean()
+	defer sth.Clean(t)
 
-	mr := sth.GetTreeMultiReader(t)
+	mr, closer := sth.GetTreeMultiReader(t)
+	defer closer.Close()
 	r := multipart.NewReader(mr, mr.Boundary())
 	p := api.DefaultAddParams()
 	expectedCids := test.ShardingDirCids[:]
@@ -76,7 +77,7 @@ func TestAdder(t *testing.T) {
 
 func TestAdder_DoubleStart(t *testing.T) {
 	sth := test.NewShardingTestHelper()
-	defer sth.Clean()
+	defer sth.Clean(t)
 
 	f := sth.GetTreeSerialFile(t)
 	p := api.DefaultAddParams()
@@ -88,6 +89,7 @@ func TestAdder_DoubleStart(t *testing.T) {
 	adder := New(dags, p, nil)
 	ctx, cancel := context.WithCancel(context.Background())
 	_, err := adder.FromFiles(ctx, f)
+	f.Close()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -95,6 +97,7 @@ func TestAdder_DoubleStart(t *testing.T) {
 
 	f = sth.GetTreeSerialFile(t)
 	_, err = adder.FromFiles(ctx, f)
+	f.Close()
 	if err == nil {
 		t.Fatal("expected an error: cannot run importer twice")
 	}
@@ -102,9 +105,10 @@ func TestAdder_DoubleStart(t *testing.T) {
 
 func TestAdder_ContextCancelled(t *testing.T) {
 	sth := test.NewShardingTestHelper()
-	defer sth.Clean()
+	defer sth.Clean(t)
 
-	mr := sth.GetRandFileMultiReader(t, 50000) // 50 MB
+	mr, closer := sth.GetRandFileMultiReader(t, 50000) // 50 MB
+	defer closer.Close()
 	r := multipart.NewReader(mr, mr.Boundary())
 
 	p := api.DefaultAddParams()

adder/local/dag_service_test.go

@@ -53,8 +53,9 @@ func TestAdd(t *testing.T) {
 		add := adder.New(dags, params, nil)
 
 		sth := test.NewShardingTestHelper()
-		defer sth.Clean()
+		defer sth.Clean(t)
-		mr := sth.GetTreeMultiReader(t)
+		mr, closer := sth.GetTreeMultiReader(t)
+		defer closer.Close()
 		r := multipart.NewReader(mr, mr.Boundary())
 
 		rootCid, err := add.FromMultipart(context.Background(), r)
@@ -95,8 +96,9 @@ func TestAdd(t *testing.T) {
 		add := adder.New(dags, params, nil)
 
 		sth := test.NewShardingTestHelper()
-		defer sth.Clean()
+		defer sth.Clean(t)
-		mr := sth.GetTreeMultiReader(t)
+		mr, closer := sth.GetTreeMultiReader(t)
+		defer closer.Close()
 		r := multipart.NewReader(mr, mr.Boundary())
 
 		rootCid, err := add.FromMultipart(context.Background(), r)

adder/sharding/dag_service_test.go

@@ -17,7 +17,7 @@ import (
 )
 
 func init() {
-	logging.SetLogLevel("addshard", "INFO")
+	logging.SetLogLevel("shardingdags", "INFO")
 	logging.SetLogLevel("adder", "INFO")
 }
 
@@ -85,7 +85,7 @@ func makeAdder(t *testing.T, params *api.AddParams) (*adder.Adder, *testRPC) {
 
 func TestFromMultipart(t *testing.T) {
 	sth := test.NewShardingTestHelper()
-	defer sth.Clean()
+	defer sth.Clean(t)
 
 	t.Run("Test tree", func(t *testing.T) {
 		p := api.DefaultAddParams()
@@ -99,7 +99,8 @@ func TestFromMultipart(t *testing.T) {
 		add, rpcObj := makeAdder(t, p)
 		_ = rpcObj
 
-		mr := sth.GetTreeMultiReader(t)
+		mr, closer := sth.GetTreeMultiReader(t)
+		defer closer.Close()
 		r := multipart.NewReader(mr, mr.Boundary())
 
 		rootCid, err := add.FromMultipart(context.Background(), r)
@@ -160,7 +161,8 @@ func TestFromMultipart(t *testing.T) {
 		add, rpcObj := makeAdder(t, p)
 		_ = rpcObj
 
-		mr := sth.GetRandFileMultiReader(t, 1024*50) // 50 MB
+		mr, closer := sth.GetRandFileMultiReader(t, 1024*50) // 50 MB
+		defer closer.Close()
 		r := multipart.NewReader(mr, mr.Boundary())
 
 		rootCid, err := add.FromMultipart(context.Background(), r)
@@ -235,7 +237,7 @@ func TestFromMultipart_Errors(t *testing.T) {
 	}
 
 	sth := test.NewShardingTestHelper()
-	defer sth.Clean()
+	defer sth.Clean(t)
 	for _, tc := range tcs {
 		add, rpcObj := makeAdder(t, tc.params)
 		_ = rpcObj
@@ -248,5 +250,6 @@ func TestFromMultipart_Errors(t *testing.T) {
 		} else {
 			t.Log(tc.name, ":", err)
 		}
+		f.Close()
 	}
 }

api/rest/client/methods_test.go

@@ -437,7 +437,8 @@ func TestAddMultiFile(t *testing.T) {
 
 	testF := func(t *testing.T, c *Client) {
 		sth := test.NewShardingTestHelper()
-		mfr := sth.GetTreeMultiReader(t)
+		mfr, closer := sth.GetTreeMultiReader(t)
+		defer closer.Close()
 
 		p := &types.AddParams{
 			PinOptions: types.PinOptions{

api/rest/restapi_test.go

@@ -365,7 +365,8 @@ func TestAPIAddFileEndpointLocal(t *testing.T) {
 		fmtStr1 := "/add?shard=true&repl_min=-1&repl_max=-1"
 		localURL := url(rest) + fmtStr1
 		sth := test.NewShardingTestHelper()
-		body := sth.GetTreeMultiReader(t)
+		body, closer := sth.GetTreeMultiReader(t)
+		defer closer.Close()
 		resp := api.AddedOutput{}
 		mpContentType := "multipart/form-data; boundary=" + body.Boundary()
 		makeStreamingPost(t, rest, localURL, body, mpContentType, &resp)
@@ -379,7 +380,8 @@ func TestAPIAddFileEndpointShard(t *testing.T) {
 	defer rest.Shutdown()
 	tf := func(t *testing.T, url urlF) {
 		sth := test.NewShardingTestHelper()
-		body := sth.GetTreeMultiReader(t)
+		body, closer := sth.GetTreeMultiReader(t)
+		defer closer.Close()
 		mpContentType := "multipart/form-data; boundary=" + body.Boundary()
 		resp := api.AddedOutput{}
 		fmtStr1 := "/add?shard=true&repl_min=-1&repl_max=-1"

cluster_test.go

@@ -258,13 +258,14 @@ func TestAddFile(t *testing.T) {
 	defer cleanRaft()
 	defer cl.Shutdown()
 	sth := test.NewShardingTestHelper()
-	defer sth.Clean()
+	defer sth.Clean(t)
 
 	t.Run("local", func(t *testing.T) {
 		params := api.DefaultAddParams()
 		params.Shard = false
 		params.Name = "testlocal"
-		mfr := sth.GetTreeMultiReader(t)
+		mfr, closer := sth.GetTreeMultiReader(t)
+		defer closer.Close()
 		r := multipart.NewReader(mfr, mfr.Boundary())
 		c, err := cl.AddFile(r, params)
 		if err != nil {
@@ -292,7 +293,8 @@ func TestAddFile(t *testing.T) {
 		params := api.DefaultAddParams()
 		params.Shard = true
 		params.Name = "testshard"
-		mfr := sth.GetTreeMultiReader(t)
+		mfr, closer := sth.GetTreeMultiReader(t)
+		defer closer.Close()
 		r := multipart.NewReader(mfr, mfr.Boundary())
 		c, err := cl.AddFile(r, params)
 		if err != nil {
@@ -315,12 +317,13 @@ func TestUnpinShard(t *testing.T) {
 	defer cleanRaft()
 	defer cl.Shutdown()
 	sth := test.NewShardingTestHelper()
-	defer sth.Clean()
+	defer sth.Clean(t)
 
 	params := api.DefaultAddParams()
 	params.Shard = true
 	params.Name = "testshard"
-	mfr := sth.GetTreeMultiReader(t)
+	mfr, closer := sth.GetTreeMultiReader(t)
+	defer closer.Close()
 	r := multipart.NewReader(mfr, mfr.Boundary())
 	root, err := cl.AddFile(r, params)
 	if err != nil {

test/sharding.go

@@ -96,10 +96,10 @@ func NewShardingTestHelper() *ShardingTestHelper {
 //         * big_file (3 MB)
 //
 // The total size in ext4 is ~3420160 Bytes = ~3340 kB = ~3.4MB
-func (sth *ShardingTestHelper) GetTreeMultiReader(t *testing.T) *files.MultiFileReader {
+func (sth *ShardingTestHelper) GetTreeMultiReader(t *testing.T) (*files.MultiFileReader, io.Closer) {
 	sf := sth.GetTreeSerialFile(t)
 	slf := files.NewSliceFile("", "", []files.File{sf})
-	return files.NewMultiFileReader(slf, true)
+	return files.NewMultiFileReader(slf, true), sf
 }
 
 // GetTreeSerialFile returns a files.SerialFile pointing to the testing
@@ -116,19 +116,22 @@ func (sth *ShardingTestHelper) GetTreeSerialFile(t *testing.T) files.File {
 // GetRandFileMultiReader creates and returns a MultiFileReader for
 // a testing random file of the given size (in kbs). The random
 // file is different every time.
-func (sth *ShardingTestHelper) GetRandFileMultiReader(t *testing.T, kbs int) *files.MultiFileReader {
+func (sth *ShardingTestHelper) GetRandFileMultiReader(t *testing.T, kbs int) (*files.MultiFileReader, io.Closer) {
 	st := sth.makeRandFile(t, kbs)
 	sf, err := files.NewSerialFile("randomfile", sth.path(shardingTestFile), false, st)
 	if err != nil {
 		t.Fatal(err)
 	}
 	slf := files.NewSliceFile("", "", []files.File{sf})
-	return files.NewMultiFileReader(slf, true)
+	return files.NewMultiFileReader(slf, true), sf
 }
 
 // Clean deletes any folder and file generated by this helper.
-func (sth *ShardingTestHelper) Clean() {
+func (sth *ShardingTestHelper) Clean(t *testing.T) {
-	os.RemoveAll(shardingTestDir)
+	err := os.RemoveAll(shardingTestDir)
+	if err != nil {
+		t.Fatal(err)
+	}
 }
 
 func folderExists(t *testing.T, path string) bool {
@@ -195,7 +198,6 @@ func (sth *ShardingTestHelper) makeTree(t *testing.T) os.FileInfo {
 		[]string{p0, "B"},
 	}
 	for _, p := range paths {
-
 		makeDir(t, sth.path(p...))
 	}
 
@@ -220,16 +222,10 @@ func (sth *ShardingTestHelper) makeTree(t *testing.T) os.FileInfo {
 		f.Close()
 	}
 
-	f, err := os.Open(basepath)
+	st, err := os.Stat(basepath)
 	if err != nil {
 		t.Fatal(err)
 	}
-	f.Sync()
-	st, err := f.Stat()
-	if err != nil {
-		t.Fatal(err)
-	}
-	f.Close()
 	return st
 }
 

test/test_test.go

@@ -64,7 +64,9 @@ func TestRPCMockValid(t *testing.T) {
 // Test that testing directory is created without error
 func TestGenerateTestDirs(t *testing.T) {
 	sth := NewShardingTestHelper()
-	defer sth.Clean()
+	defer sth.Clean(t)
-	_ = sth.GetTreeMultiReader(t)
+	_, closer := sth.GetTreeMultiReader(t)
-	_ = sth.GetRandFileMultiReader(t, 2)
+	closer.Close()
+	_, closer = sth.GetRandFileMultiReader(t, 2)
+	closer.Close()
 }