2020-08-09 11:31:04 +02:00
// Package makefiles builds a directory structure with the required
// number of files in of the required size.
package makefiles
import (
"io"
2022-04-12 14:37:35 +02:00
"math"
2020-08-09 11:31:04 +02:00
"math/rand"
"os"
"path/filepath"
2021-04-10 14:42:17 +02:00
"time"
2020-08-09 11:31:04 +02:00
"github.com/rclone/rclone/cmd"
"github.com/rclone/rclone/cmd/test"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/flags"
2021-06-11 00:46:36 +02:00
"github.com/rclone/rclone/lib/file"
2020-08-09 11:31:04 +02:00
"github.com/rclone/rclone/lib/random"
2022-04-12 14:37:35 +02:00
"github.com/rclone/rclone/lib/readers"
2020-08-09 11:31:04 +02:00
"github.com/spf13/cobra"
2022-04-12 13:57:16 +02:00
"github.com/spf13/pflag"
2020-08-09 11:31:04 +02:00
)
var (
// Flags
numberOfFiles = 1000
averageFilesPerDirectory = 10
maxDepth = 10
minFileSize = fs . SizeSuffix ( 0 )
maxFileSize = fs . SizeSuffix ( 100 )
minFileNameLength = 4
maxFileNameLength = 12
2021-04-10 14:42:17 +02:00
seed = int64 ( 1 )
2022-04-12 14:37:35 +02:00
zero = false
sparse = false
ascii = false
pattern = false
2022-04-13 12:10:22 +02:00
chargen = false
2020-08-09 11:31:04 +02:00
// Globals
2021-04-10 14:42:17 +02:00
randSource * rand . Rand
2022-04-12 14:37:35 +02:00
source io . Reader
2020-08-09 11:31:04 +02:00
directoriesToCreate int
totalDirectories int
fileNames = map [ string ] struct { } { } // keep a note of which file name we've used already
)
func init ( ) {
2022-04-12 13:57:16 +02:00
test . Command . AddCommand ( makefilesCmd )
makefilesFlags := makefilesCmd . Flags ( )
2023-07-10 19:34:10 +02:00
flags . IntVarP ( makefilesFlags , & numberOfFiles , "files" , "" , numberOfFiles , "Number of files to create" , "" )
flags . IntVarP ( makefilesFlags , & averageFilesPerDirectory , "files-per-directory" , "" , averageFilesPerDirectory , "Average number of files per directory" , "" )
flags . IntVarP ( makefilesFlags , & maxDepth , "max-depth" , "" , maxDepth , "Maximum depth of directory hierarchy" , "" )
flags . FVarP ( makefilesFlags , & minFileSize , "min-file-size" , "" , "Minimum size of file to create" , "" )
flags . FVarP ( makefilesFlags , & maxFileSize , "max-file-size" , "" , "Maximum size of files to create" , "" )
flags . IntVarP ( makefilesFlags , & minFileNameLength , "min-name-length" , "" , minFileNameLength , "Minimum size of file names" , "" )
flags . IntVarP ( makefilesFlags , & maxFileNameLength , "max-name-length" , "" , maxFileNameLength , "Maximum size of file names" , "" )
2022-04-12 13:57:16 +02:00
test . Command . AddCommand ( makefileCmd )
makefileFlags := makefileCmd . Flags ( )
// Common flags to makefiles and makefile
for _ , f := range [ ] * pflag . FlagSet { makefilesFlags , makefileFlags } {
2023-07-10 19:34:10 +02:00
flags . Int64VarP ( f , & seed , "seed" , "" , seed , "Seed for the random number generator (0 for random)" , "" )
flags . BoolVarP ( f , & zero , "zero" , "" , zero , "Fill files with ASCII 0x00" , "" )
flags . BoolVarP ( f , & sparse , "sparse" , "" , sparse , "Make the files sparse (appear to be filled with ASCII 0x00)" , "" )
flags . BoolVarP ( f , & ascii , "ascii" , "" , ascii , "Fill files with random ASCII printable bytes only" , "" )
flags . BoolVarP ( f , & pattern , "pattern" , "" , pattern , "Fill files with a periodic pattern" , "" )
flags . BoolVarP ( f , & chargen , "chargen" , "" , chargen , "Fill files with a ASCII chargen pattern" , "" )
2022-04-12 13:57:16 +02:00
}
}
var makefilesCmd = & cobra . Command {
2020-08-09 11:31:04 +02:00
Use : "makefiles <dir>" ,
2021-07-20 20:37:09 +02:00
Short : ` Make a random file hierarchy in a directory ` ,
2022-11-26 23:40:49 +01:00
Annotations : map [ string ] string {
"versionIntroduced" : "v1.55" ,
} ,
2020-08-09 11:31:04 +02:00
Run : func ( command * cobra . Command , args [ ] string ) {
cmd . CheckArgs ( 1 , 1 , command , args )
2022-04-12 13:57:16 +02:00
commonInit ( )
2020-08-09 11:31:04 +02:00
outputDirectory := args [ 0 ]
directoriesToCreate = numberOfFiles / averageFilesPerDirectory
averageSize := ( minFileSize + maxFileSize ) / 2
2021-04-10 17:59:30 +02:00
start := time . Now ( )
fs . Logf ( nil , "Creating %d files of average size %v in %d directories in %q." , numberOfFiles , averageSize , directoriesToCreate , outputDirectory )
2020-08-09 11:31:04 +02:00
root := & dir { name : outputDirectory , depth : 1 }
for totalDirectories < directoriesToCreate {
root . createDirectories ( )
}
dirs := root . list ( "" , [ ] string { } )
2021-04-10 17:59:30 +02:00
totalBytes := int64 ( 0 )
2020-08-09 11:31:04 +02:00
for i := 0 ; i < numberOfFiles ; i ++ {
2021-04-10 14:42:17 +02:00
dir := dirs [ randSource . Intn ( len ( dirs ) ) ]
2022-04-12 14:44:04 +02:00
size := int64 ( minFileSize )
if maxFileSize > minFileSize {
size += randSource . Int63n ( int64 ( maxFileSize - minFileSize ) )
}
2022-04-12 13:57:16 +02:00
writeFile ( dir , fileName ( ) , size )
totalBytes += size
}
dt := time . Since ( start )
fs . Logf ( nil , "Written %vB in %v at %vB/s." , fs . SizeSuffix ( totalBytes ) , dt . Round ( time . Millisecond ) , fs . SizeSuffix ( ( totalBytes * int64 ( time . Second ) ) / int64 ( dt ) ) )
} ,
}
var makefileCmd = & cobra . Command {
2022-04-12 14:37:35 +02:00
Use : "makefile <size> [<file>]+ [flags]" ,
2022-04-12 13:57:16 +02:00
Short : ` Make files with random contents of the size given ` ,
2022-11-26 23:40:49 +01:00
Annotations : map [ string ] string {
"versionIntroduced" : "v1.59" ,
} ,
2022-04-12 13:57:16 +02:00
Run : func ( command * cobra . Command , args [ ] string ) {
cmd . CheckArgs ( 1 , 1e6 , command , args )
commonInit ( )
var size fs . SizeSuffix
err := size . Set ( args [ 0 ] )
if err != nil {
2024-08-18 16:58:35 +02:00
fs . Fatalf ( nil , "Failed to parse size %q: %v" , args [ 0 ] , err )
2022-04-12 13:57:16 +02:00
}
start := time . Now ( )
fs . Logf ( nil , "Creating %d files of size %v." , len ( args [ 1 : ] ) , size )
totalBytes := int64 ( 0 )
for _ , filePath := range args [ 1 : ] {
dir := filepath . Dir ( filePath )
name := filepath . Base ( filePath )
writeFile ( dir , name , int64 ( size ) )
totalBytes += int64 ( size )
2020-08-09 11:31:04 +02:00
}
2021-04-10 17:59:30 +02:00
dt := time . Since ( start )
2022-04-12 13:57:16 +02:00
fs . Logf ( nil , "Written %vB in %v at %vB/s." , fs . SizeSuffix ( totalBytes ) , dt . Round ( time . Millisecond ) , fs . SizeSuffix ( ( totalBytes * int64 ( time . Second ) ) / int64 ( dt ) ) )
2020-08-09 11:31:04 +02:00
} ,
}
2022-04-12 14:37:35 +02:00
func bool2int ( b bool ) int {
if b {
return 1
}
return 0
}
// common initialisation for makefiles and makefile
func commonInit ( ) {
if seed == 0 {
seed = time . Now ( ) . UnixNano ( )
fs . Logf ( nil , "Using random seed = %d" , seed )
}
randSource = rand . New ( rand . NewSource ( seed ) )
2022-04-13 12:10:22 +02:00
if bool2int ( zero ) + bool2int ( sparse ) + bool2int ( ascii ) + bool2int ( pattern ) + bool2int ( chargen ) > 1 {
2024-08-18 16:58:35 +02:00
fs . Fatal ( nil , "Can only supply one of --zero, --sparse, --ascii, --pattern or --chargen" )
2022-04-12 14:37:35 +02:00
}
switch {
case zero , sparse :
source = zeroReader { }
case ascii :
source = asciiReader { }
case pattern :
source = readers . NewPatternReader ( math . MaxInt64 )
2022-04-13 12:10:22 +02:00
case chargen :
source = & chargenReader { }
2022-04-12 14:37:35 +02:00
default :
source = randSource
}
2022-04-12 14:44:04 +02:00
if minFileSize > maxFileSize {
maxFileSize = minFileSize
}
2022-04-12 14:37:35 +02:00
}
type zeroReader struct { }
// Read a chunk of zeroes
func ( zeroReader ) Read ( p [ ] byte ) ( n int , err error ) {
for i := range p {
p [ i ] = 0
}
return len ( p ) , nil
}
type asciiReader struct { }
// Read a chunk of printable ASCII characters
func ( asciiReader ) Read ( p [ ] byte ) ( n int , err error ) {
n , err = randSource . Read ( p )
for i := range p [ : n ] {
p [ i ] = ( p [ i ] % ( 0x7F - 0x20 ) ) + 0x20
}
return n , err
}
2022-04-13 12:10:22 +02:00
type chargenReader struct {
start byte // offset from startChar to start line with
written byte // chars in line so far
}
// Read a chunk of printable ASCII characters in chargen format
func ( r * chargenReader ) Read ( p [ ] byte ) ( n int , err error ) {
const (
startChar = 0x20 // ' '
endChar = 0x7E // '~' inclusive
charsPerLine = 72
)
for i := range p {
if r . written >= charsPerLine {
r . start ++
if r . start > endChar - startChar {
r . start = 0
}
p [ i ] = '\n'
r . written = 0
} else {
c := r . start + r . written + startChar
if c > endChar {
c -= endChar - startChar + 1
}
p [ i ] = c
r . written ++
}
}
return len ( p ) , err
}
2020-08-09 11:31:04 +02:00
// fileName creates a unique random file or directory name
func fileName ( ) ( name string ) {
for {
2021-04-10 14:42:17 +02:00
length := randSource . Intn ( maxFileNameLength - minFileNameLength ) + minFileNameLength
2023-11-23 20:58:22 +01:00
name = random . StringFn ( length , randSource )
2020-08-09 11:31:04 +02:00
if _ , found := fileNames [ name ] ; ! found {
break
}
}
fileNames [ name ] = struct { } { }
return name
}
// dir is a directory in the directory hierarchy being built up
type dir struct {
name string
depth int
children [ ] * dir
parent * dir
}
// Create a random directory hierarchy under d
func ( d * dir ) createDirectories ( ) {
for totalDirectories < directoriesToCreate {
newDir := & dir {
name : fileName ( ) ,
depth : d . depth + 1 ,
parent : d ,
}
d . children = append ( d . children , newDir )
totalDirectories ++
2021-04-10 14:42:17 +02:00
switch randSource . Intn ( 4 ) {
2020-08-09 11:31:04 +02:00
case 0 :
if d . depth < maxDepth {
newDir . createDirectories ( )
}
case 1 :
return
}
}
}
// list the directory hierarchy
func ( d * dir ) list ( path string , output [ ] string ) [ ] string {
dirPath := filepath . Join ( path , d . name )
output = append ( output , dirPath )
for _ , subDir := range d . children {
output = subDir . list ( dirPath , output )
}
return output
}
// writeFile writes a random file at dir/name
2022-04-12 13:57:16 +02:00
func writeFile ( dir , name string , size int64 ) {
2021-06-11 00:46:36 +02:00
err := file . MkdirAll ( dir , 0777 )
2020-08-09 11:31:04 +02:00
if err != nil {
2024-08-18 16:58:35 +02:00
fs . Fatalf ( nil , "Failed to make directory %q: %v" , dir , err )
2020-08-09 11:31:04 +02:00
}
path := filepath . Join ( dir , name )
fd , err := os . Create ( path )
if err != nil {
2024-08-18 16:58:35 +02:00
fs . Fatalf ( nil , "Failed to open file %q: %v" , path , err )
2020-08-09 11:31:04 +02:00
}
2022-04-12 14:37:35 +02:00
if sparse {
err = fd . Truncate ( size )
} else {
_ , err = io . CopyN ( fd , source , size )
}
2020-08-09 11:31:04 +02:00
if err != nil {
2024-08-18 16:58:35 +02:00
fs . Fatalf ( nil , "Failed to write %v bytes to file %q: %v" , size , path , err )
2020-08-09 11:31:04 +02:00
}
err = fd . Close ( )
if err != nil {
2024-08-18 16:58:35 +02:00
fs . Fatalf ( nil , "Failed to close file %q: %v" , path , err )
2020-08-09 11:31:04 +02:00
}
2021-04-10 17:59:30 +02:00
fs . Infof ( path , "Written file size %v" , fs . SizeSuffix ( size ) )
2020-08-09 11:31:04 +02:00
}