package test import ( "bufio" "io" "math/rand" "os" "path/filepath" "testing" "github.com/ipfs/go-ipfs-cmdkit/files" ) const shardingDirName = "shardTesting" // Variables related to adding the testing directory generated by tests var ( NumShardingDirPrints = 15 ShardingDirBalancedRootCID = "QmUtFSRGDUQ1mHAsX8udixU5rn8e34Lqm5pJBoUpyXPumk" ShardingDirTrickleRootCID = "QmaNXYLz6LbPMrEYcm9Mot1HoHGYbsn7JvzLPk6C9ubAcp" // These hashes should match all the blocks produced when adding // the files resulting from GetShardingDir* // They have been obtained by adding the "shardTesting" folder // to go-ipfs (with wrap=true and default parameters). Then doing // `refs -r` on the result. It contains the wrapping folder hash. ShardingDirCids = [29]string{ "QmUtFSRGDUQ1mHAsX8udixU5rn8e34Lqm5pJBoUpyXPumk", "QmVMmDqWhdH8d4QWyhkkVHYvQYara6ijgCg3PNpvRhbmHo", "QmbR4x5HwcQLiipfyHazhKYA1Z2yN9burWWdAKJBhoZnK3", "QmanFw3Fn96DkMc9XSuhZdvXWDk37cpLrKA6L54MniGL9Z", "QmdWLMAyMF23KyBfAXbNzy7sDu3zGGu27eGQRkQTPMqfoE", "QmRVFNBFwtUKpE7u3Bbd6Nj1QsnyHgryZSTM86bBuphPAn", "QmbiPudqP8264Ccia1iyTebFrtGmG3JCW85YmT5Gds1Wt9", "QmPZLJ3CZYgxH4K1w5jdbAdxJynXn5TCB4kHy7u8uHC3fy", "QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn", "QmQntQGk13CkPgr1b3RAseJFaTpVMqQu8zgh21ox1RnwBf", "QmesCMDCHRETdDYfuGjUaGVZGSE2nGxGETPoBUgwitnsCT", "Qmbiz4Y6ByNTrzEwE2veqy7S8gUBJNNvNqxAy6bBShpvr4", "QmYdHmrwb9Wd8kkjLg4yKr7EPqKNGx5vHuREU5HNc7sxnk", "QmNtq6PD9Ef6V1UtikhemHBcupjsvr2sDu7bu2DrBSoHWw", "QmavW3cdGuSfYMEQiBDfobwVtPEjUnML2Ry1q8w8X3Q8Wj", "QmfPHRbeerRWgbu5BzxwK7UhmJGqGvZNxuFoMCUFTuhG3H", "QmaYNfhw7L7KWX7LYpwWt1bh6Gq2p7z1tic35PnDRnqyBf", "QmWWwH1GKMh6GmFQunjq7CHjr4g4z6Q4xHyDVfuZGX7MyU", "QmVpHQGMF5PLsvfgj8bGo9q2YyLRPMvfu1uTb3DgREFtUc", "QmUrdAn4Mx4kNioX9juLgwQotwFfxeo5doUNnLJrQynBEN", "QmdJ86B7J8mfGq6SjQy8Jz7r5x1cLcXc9M2a7T7NmSMVZx", "QmS77cTMdyx8P7rP2Gij6azgYPpjp2J34EVYuhB6mfjrQh", "QmbsBsDspFcqi7xJ4xPxcNYnduzQ5UQDw9y6trQWZGoEHq", "QmakAXHMeyE6fHHaeqicSKVMM2QyuGbS2g8dgUA7ns8gSY", "QmTC6vGbH9ABkpXfrMmYkXbxEqH12jEVGpvGzibGZEDVHK", "QmRLF116yZdLLw7bLvjnHxXVHrjB2snNoJ1itpQxi8NkZP", "QmVorRQhT4DbND8JyhAW7HkNPd7bUzqof8XJKcfGcGmvHF", "QmZ2iUT3W7jh8QNnpWSiMZ1QYgpommCSQFZiPY5VdoCHyv", "QmR5mq8smc6zCvo3eRhS47ScgEwKpPw7b1joExysyqgbee", } ) // CleanShardingDir deletes the folders generated with GetShardingDir() and // GetShardingMultiReader. func CleanShardingDir(t *testing.T) { os.RemoveAll(shardingDirName) } // GetShardingDirSerial returns a cmdkits serial file to the testing directory. func GetShardingDirSerial(t *testing.T) files.File { if _, err := os.Stat(shardingDirName); os.IsNotExist(err) { err := generateShardingDirs(shardingDirName) if err != nil { t.Fatal(err) } } stat, err := os.Lstat(shardingDirName) if err != nil { t.Fatal(err) } if !stat.IsDir() { t.Fatal("testDir should be seen as directory") } f, err := files.NewSerialFile(shardingDirName, shardingDirName, false, stat) if err != nil { t.Fatal(err) } return f } // GetShardingDirMultiReader returns a cmdkits multifilereader to the testing // directory. func GetShardingDirMultiReader(t *testing.T) *files.MultiFileReader { file := GetShardingDirSerial(t) sliceFile := files.NewSliceFile("", "", []files.File{file}) mfr := files.NewMultiFileReader(sliceFile, true) return mfr } // generateShardingDirs creates a testing directory structure on demand for testing // leveraging random but deterministic strings. Files are the same every run. // Directory structure: // - testingData // - A // - alpha // * small_file_0 (< 5 kB) // - beta // * small_file_1 (< 5 kB) // - delta // - empty // * small_file_2 (< 5 kB) // - gamma // * small_file_3 (< 5 kB) // - B // * medium_file (~.3 MB) // * big_file (3 MB) // // Take special care when modifying this function. File data depends on order // and each file size. If this changes then hashes stored in test/cids.go // recording the ipfs import hash tree must be updated manually. func generateShardingDirs(path string) error { // Prepare randomness source for writing files src := rand.NewSource(int64(1)) ra := rand.New(src) // Make top level directory err := os.Mkdir(path, os.ModePerm) if err != nil { return err } // Make subdir A aPath := filepath.Join(path, "A") err = os.Mkdir(aPath, os.ModePerm) if err != nil { return err } alphaPath := filepath.Join(aPath, "alpha") err = os.Mkdir(alphaPath, os.ModePerm) if err != nil { return err } sf0Path := filepath.Join(alphaPath, "small_file_0") f, err := os.Create(sf0Path) if err != nil { return err } err = writeRandFile(5, ra, f) if err != nil { return err } if err = f.Close(); err != nil { return err } betaPath := filepath.Join(aPath, "beta") err = os.Mkdir(betaPath, os.ModePerm) if err != nil { return err } sf1Path := filepath.Join(betaPath, "small_file_1") f, err = os.Create(sf1Path) if err != nil { return err } err = writeRandFile(5, ra, f) if err != nil { return err } if err = f.Close(); err != nil { return err } deltaPath := filepath.Join(aPath, "delta") err = os.Mkdir(deltaPath, os.ModePerm) if err != nil { return err } emptyPath := filepath.Join(deltaPath, "empty") err = os.Mkdir(emptyPath, os.ModePerm) if err != nil { return err } sf2Path := filepath.Join(aPath, "small_file_2") f, err = os.Create(sf2Path) if err != nil { return err } err = writeRandFile(5, ra, f) if err != nil { return err } if err = f.Close(); err != nil { return err } gammaPath := filepath.Join(aPath, "gamma") err = os.Mkdir(gammaPath, os.ModePerm) if err != nil { return err } sf3Path := filepath.Join(gammaPath, "small_file_3") f, err = os.Create(sf3Path) if err != nil { return err } err = writeRandFile(5, ra, f) if err != nil { return err } if err = f.Close(); err != nil { return err } // Make subdir B bPath := filepath.Join(path, "B") err = os.Mkdir(bPath, os.ModePerm) if err != nil { return err } mfPath := filepath.Join(bPath, "medium_file") f, err = os.Create(mfPath) if err != nil { return err } err = writeRandFile(300, ra, f) if err != nil { return err } if err = f.Close(); err != nil { return err } bfPath := filepath.Join(bPath, "big_file") f, err = os.Create(bfPath) if err != nil { return err } err = writeRandFile(3000, ra, f) if err != nil { return err } if err = f.Close(); err != nil { return err } return nil } // writeRandFile takes in a source of randomness, a file, a number of kibibytes // and a writing buffer and writes a kibibyte at a time from the randomness to // the file func writeRandFile(n int, ra *rand.Rand, f io.Writer) error { w := bufio.NewWriter(f) buf := make([]byte, 1024) i := 0 for i < n { ra.Read(buf) if _, err := w.Write(buf); err != nil { return err } i++ } return nil }