ipfs-cluster/test/test_files.go

package test

import (
	"bufio"
	"io"
	"math/rand"
	"os"
	"path/filepath"
	"testing"

	"github.com/ipfs/go-ipfs-cmdkit/files"
)

const shardingDirName = "shardTesting"

// Variables related to adding the testing directory generated by tests
var (
	NumShardingDirPrints       = 15
	ShardingDirBalancedRootCID = "QmUtFSRGDUQ1mHAsX8udixU5rn8e34Lqm5pJBoUpyXPumk"
	ShardingDirTrickleRootCID  = "QmaNXYLz6LbPMrEYcm9Mot1HoHGYbsn7JvzLPk6C9ubAcp"

	// These hashes should match all the blocks produced when adding
	// the files resulting from GetShardingDir*
	// They have been obtained by adding the "shardTesting" folder
	// to go-ipfs (with wrap=true and default parameters). Then doing
	// `refs -r` on the result. It contains the wrapping folder hash.
	ShardingDirCids = [29]string{
		"QmUtFSRGDUQ1mHAsX8udixU5rn8e34Lqm5pJBoUpyXPumk",
		"QmVMmDqWhdH8d4QWyhkkVHYvQYara6ijgCg3PNpvRhbmHo",
		"QmbR4x5HwcQLiipfyHazhKYA1Z2yN9burWWdAKJBhoZnK3",
		"QmanFw3Fn96DkMc9XSuhZdvXWDk37cpLrKA6L54MniGL9Z",
		"QmdWLMAyMF23KyBfAXbNzy7sDu3zGGu27eGQRkQTPMqfoE",
		"QmRVFNBFwtUKpE7u3Bbd6Nj1QsnyHgryZSTM86bBuphPAn",
		"QmbiPudqP8264Ccia1iyTebFrtGmG3JCW85YmT5Gds1Wt9",
		"QmPZLJ3CZYgxH4K1w5jdbAdxJynXn5TCB4kHy7u8uHC3fy",
		"QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn",
		"QmQntQGk13CkPgr1b3RAseJFaTpVMqQu8zgh21ox1RnwBf",
		"QmesCMDCHRETdDYfuGjUaGVZGSE2nGxGETPoBUgwitnsCT",
		"Qmbiz4Y6ByNTrzEwE2veqy7S8gUBJNNvNqxAy6bBShpvr4",
		"QmYdHmrwb9Wd8kkjLg4yKr7EPqKNGx5vHuREU5HNc7sxnk",
		"QmNtq6PD9Ef6V1UtikhemHBcupjsvr2sDu7bu2DrBSoHWw",
		"QmavW3cdGuSfYMEQiBDfobwVtPEjUnML2Ry1q8w8X3Q8Wj",
		"QmfPHRbeerRWgbu5BzxwK7UhmJGqGvZNxuFoMCUFTuhG3H",
		"QmaYNfhw7L7KWX7LYpwWt1bh6Gq2p7z1tic35PnDRnqyBf",
		"QmWWwH1GKMh6GmFQunjq7CHjr4g4z6Q4xHyDVfuZGX7MyU",
		"QmVpHQGMF5PLsvfgj8bGo9q2YyLRPMvfu1uTb3DgREFtUc",
		"QmUrdAn4Mx4kNioX9juLgwQotwFfxeo5doUNnLJrQynBEN",
		"QmdJ86B7J8mfGq6SjQy8Jz7r5x1cLcXc9M2a7T7NmSMVZx",
		"QmS77cTMdyx8P7rP2Gij6azgYPpjp2J34EVYuhB6mfjrQh",
		"QmbsBsDspFcqi7xJ4xPxcNYnduzQ5UQDw9y6trQWZGoEHq",
		"QmakAXHMeyE6fHHaeqicSKVMM2QyuGbS2g8dgUA7ns8gSY",
		"QmTC6vGbH9ABkpXfrMmYkXbxEqH12jEVGpvGzibGZEDVHK",
		"QmRLF116yZdLLw7bLvjnHxXVHrjB2snNoJ1itpQxi8NkZP",
		"QmVorRQhT4DbND8JyhAW7HkNPd7bUzqof8XJKcfGcGmvHF",
		"QmZ2iUT3W7jh8QNnpWSiMZ1QYgpommCSQFZiPY5VdoCHyv",
		"QmR5mq8smc6zCvo3eRhS47ScgEwKpPw7b1joExysyqgbee",
	}
)

// CleanShardingDir deletes the folders generated with GetShardingDir() and
// GetShardingMultiReader.
func CleanShardingDir(t *testing.T) {
	os.RemoveAll(shardingDirName)
}

// GetShardingDirSerial returns a cmdkits serial file to the testing directory.
func GetShardingDirSerial(t *testing.T) files.File {
	if _, err := os.Stat(shardingDirName); os.IsNotExist(err) {
		err := generateShardingDirs(shardingDirName)
		if err != nil {
			t.Fatal(err)
		}
	}
	stat, err := os.Lstat(shardingDirName)
	if err != nil {
		t.Fatal(err)
	}

	if !stat.IsDir() {
		t.Fatal("testDir should be seen as directory")
	}
	f, err := files.NewSerialFile(shardingDirName, shardingDirName, false, stat)
	if err != nil {
		t.Fatal(err)
	}
	return f
}

// GetShardingDirMultiReader returns a cmdkits multifilereader to the testing
// directory.
func GetShardingDirMultiReader(t *testing.T) *files.MultiFileReader {
	file := GetShardingDirSerial(t)
	sliceFile := files.NewSliceFile("", "", []files.File{file})
	mfr := files.NewMultiFileReader(sliceFile, true)
	return mfr
}

// generateShardingDirs creates a testing directory structure on demand for testing
// leveraging random but deterministic strings.  Files are the same every run.
// Directory structure:
// - testingData
//     - A
//         - alpha
//             * small_file_0 (< 5 kB)
//         - beta
//             * small_file_1 (< 5 kB)
//         - delta
//             - empty
//         * small_file_2 (< 5 kB)
//         - gamma
//             * small_file_3 (< 5 kB)
//     - B
//         * medium_file (~.3 MB)
//         * big_file (3 MB)
//
// Take special care when modifying this function.  File data depends on order
// and each file size.  If this changes then hashes stored in test/cids.go
// recording the ipfs import hash tree must be updated manually.
func generateShardingDirs(path string) error {
	// Prepare randomness source for writing files
	src := rand.NewSource(int64(1))
	ra := rand.New(src)

	// Make top level directory
	err := os.Mkdir(path, os.ModePerm)
	if err != nil {
		return err
	}

	// Make subdir A
	aPath := filepath.Join(path, "A")
	err = os.Mkdir(aPath, os.ModePerm)
	if err != nil {
		return err
	}

	alphaPath := filepath.Join(aPath, "alpha")
	err = os.Mkdir(alphaPath, os.ModePerm)
	if err != nil {
		return err
	}

	sf0Path := filepath.Join(alphaPath, "small_file_0")
	f, err := os.Create(sf0Path)
	if err != nil {
		return err
	}
	err = writeRandFile(5, ra, f)
	if err != nil {
		return err
	}
	if err = f.Close(); err != nil {
		return err
	}

	betaPath := filepath.Join(aPath, "beta")
	err = os.Mkdir(betaPath, os.ModePerm)
	if err != nil {
		return err
	}

	sf1Path := filepath.Join(betaPath, "small_file_1")
	f, err = os.Create(sf1Path)
	if err != nil {
		return err
	}
	err = writeRandFile(5, ra, f)
	if err != nil {
		return err
	}
	if err = f.Close(); err != nil {
		return err
	}

	deltaPath := filepath.Join(aPath, "delta")
	err = os.Mkdir(deltaPath, os.ModePerm)
	if err != nil {
		return err
	}

	emptyPath := filepath.Join(deltaPath, "empty")
	err = os.Mkdir(emptyPath, os.ModePerm)
	if err != nil {
		return err
	}

	sf2Path := filepath.Join(aPath, "small_file_2")
	f, err = os.Create(sf2Path)
	if err != nil {
		return err
	}
	err = writeRandFile(5, ra, f)
	if err != nil {
		return err
	}
	if err = f.Close(); err != nil {
		return err
	}

	gammaPath := filepath.Join(aPath, "gamma")
	err = os.Mkdir(gammaPath, os.ModePerm)
	if err != nil {
		return err
	}

	sf3Path := filepath.Join(gammaPath, "small_file_3")
	f, err = os.Create(sf3Path)
	if err != nil {
		return err
	}
	err = writeRandFile(5, ra, f)
	if err != nil {
		return err
	}
	if err = f.Close(); err != nil {
		return err
	}

	// Make subdir B
	bPath := filepath.Join(path, "B")
	err = os.Mkdir(bPath, os.ModePerm)
	if err != nil {
		return err
	}

	mfPath := filepath.Join(bPath, "medium_file")
	f, err = os.Create(mfPath)
	if err != nil {
		return err
	}
	err = writeRandFile(300, ra, f)
	if err != nil {
		return err
	}
	if err = f.Close(); err != nil {
		return err
	}

	bfPath := filepath.Join(bPath, "big_file")
	f, err = os.Create(bfPath)
	if err != nil {
		return err
	}
	err = writeRandFile(3000, ra, f)
	if err != nil {
		return err
	}
	if err = f.Close(); err != nil {
		return err
	}

	return nil
}

// writeRandFile takes in a source of randomness, a file, a number of kibibytes
// and a writing buffer and writes a kibibyte at a time from the randomness to
// the file
func writeRandFile(n int, ra *rand.Rand, f io.Writer) error {
	w := bufio.NewWriter(f)
	buf := make([]byte, 1024)
	i := 0
	for i < n {
		ra.Read(buf)
		if _, err := w.Write(buf); err != nil {
			return err
		}
		i++
	}
	return nil
}