diff --git a/.circleci/config.yml b/.circleci/config.yml index f70a239..0a15cc7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -62,6 +62,7 @@ jobs: - run: make - run: make vet - run: make test + - run: make lint - run: make release - store_artifacts: diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..7d99bc4 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,10 @@ +linters: + enable: + - goimports + +issues: + exclude-rules: + - path: _test\.go + linters: + - errcheck + diff --git a/.travis.yml b/.travis.yml index aec423b..7316784 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,6 +29,7 @@ matrix: - make - make vet - make test + - make lint - make artifacts/zrepl-freebsd-amd64 - make artifacts/zrepl-linux-amd64 - make artifacts/zrepl-darwin-amd64 diff --git a/Gopkg.lock b/Gopkg.lock index 7df6bf1..1e1f155 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1,6 +1,22 @@ # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. +[[projects]] + digest = "1:e4b30804a381d7603b8a344009987c1ba351c26043501b23b8c7ce21f0b67474" + name = "github.com/BurntSushi/toml" + packages = ["."] + pruneopts = "" + revision = "3012a1dbe2e4bd1391d42b32f0577cb7bbc7f005" + version = "v0.3.1" + +[[projects]] + branch = "master" + digest = "1:bf241ce6eec44d9ebca17e22af43f13c25f55267b308ce8b29b9820c0d6d1d25" + name = "github.com/OpenPeeDeeP/depguard" + packages = ["."] + pruneopts = "" + revision = "1f388ab2d81096755d25043aa729e2fb889f3dae" + [[projects]] branch = "master" digest = "1:8cf2cf1ab10480b5e0df950dac1517aaabde05d055d9d955652997ae4b9ecbbf" @@ -33,6 +49,14 @@ revision = "5b77d2a35fb0ede96d138fc9a99f5c9b6aef11b4" version = "v1.7.0" +[[projects]] + digest = "1:eb53021a8aa3f599d29c7102e65026242bdedce998a54837dc67f14b6a97c5fd" + name = "github.com/fsnotify/fsnotify" + packages = ["."] + pruneopts = "" + revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" + version = "v1.4.7" + [[projects]] branch = "master" digest = "1:5d0a2385edf4ba44f3b7b76bc0436ceb8f62bf55aa5d540a9eb9ec6c58d86809" @@ -61,6 +85,28 @@ revision = "de7e78efa4a71b3f36c7154989c529dbdf9ae623" version = "v1.1.0" +[[projects]] + digest = "1:9d3f086381a257229b34fcd5690c3e5cadcb5f365cb35757536f3c51ccbb9049" + name = "github.com/go-critic/go-critic" + packages = [ + "checkers", + "checkers/internal/lintutil", + ] + pruneopts = "" + revision = "d7b3038bc7a1c35a1d02fdd7cf4094f0f1a12001" + version = "v0.3.4" + +[[projects]] + digest = "1:50907242db0cb4c5d982ae213b995e9176b917edb269b645097af3289d9a15da" + name = "github.com/go-lintpack/lintpack" + packages = [ + ".", + "astwalk", + ] + pruneopts = "" + revision = "80adc0715ac409128d0b7212719896ad8d3444b7" + version = "v0.5.2" + [[projects]] digest = "1:6a4a01d58b227c4b6b11111b9f172ec5c17682b82724e58e6daf3f19f4faccd8" name = "github.com/go-logfmt/logfmt" @@ -69,6 +115,95 @@ revision = "390ab7935ee28ec6b286364bba9b4dd6410cb3d5" version = "v0.3.0" +[[projects]] + digest = "1:1119997895278e1b27810308d8f802590e713bf0263b13be304ca9e086bd22a7" + name = "github.com/go-toolsmith/astcast" + packages = ["."] + pruneopts = "" + revision = "a6cb19f07b66b859a53f3f2be6e4c3bba892db7e" + version = "v1.0.0" + +[[projects]] + digest = "1:bcff57ad40d16a950986eb45dae40ed142d51c702e41fad2177518071bcc3d40" + name = "github.com/go-toolsmith/astcopy" + packages = ["."] + pruneopts = "" + revision = "245af3020944a15e09072c8ad3883c1451d1fdef" + version = "v1.0.0" + +[[projects]] + digest = "1:f6629a0ef3b819e34d4634c7540edf0cb392054ade814c4a9587ab6a23685def" + name = "github.com/go-toolsmith/astequal" + packages = ["."] + pruneopts = "" + revision = "dcb477bfacd6e00a13c6d63bfc73db28dd343160" + version = "v1.0.0" + +[[projects]] + digest = "1:05a2d85ca9e1164efa7cfb988d44bff221382658e567d958a78734bb9cccb758" + name = "github.com/go-toolsmith/astfmt" + packages = ["."] + pruneopts = "" + revision = "0d74c731079884bda287cf8df9ce7b92e688af8c" + version = "v1.0.0" + +[[projects]] + digest = "1:a6f3d4784ec69928a0a54521dd2536e6bacc73edf988edefc2820230d7be7703" + name = "github.com/go-toolsmith/astp" + packages = ["."] + pruneopts = "" + revision = "6373270dee65bfb0479f2acd16d4c8e9d5db13f8" + version = "v1.0.0" + +[[projects]] + digest = "1:9ba1aaf89cddc1cfe5d9a4a83d16ff8778369eaa358278a3fa2ef97847ebdb35" + name = "github.com/go-toolsmith/strparse" + packages = ["."] + pruneopts = "" + revision = "830b6daa1241714c12a9b9a4a56849fe2f93aedc" + version = "v1.0.0" + +[[projects]] + digest = "1:b827014e6963ac236f3698ce5ca1c85ad3ccfbf722186eab207836e1d7b9d615" + name = "github.com/go-toolsmith/typep" + packages = ["."] + pruneopts = "" + revision = "cab1745ffd84a567b524317c7f90e96755b18fcf" + version = "v1.0.0" + +[[projects]] + digest = "1:9ab1b1c637d7c8f49e39d8538a650d7eb2137b076790cff69d160823b505964c" + name = "github.com/gobwas/glob" + packages = [ + ".", + "compiler", + "match", + "syntax", + "syntax/ast", + "syntax/lexer", + "util/runes", + "util/strings", + ] + pruneopts = "" + revision = "5ccd90ef52e1e632236f7326478d4faa74f99438" + version = "v0.2.3" + +[[projects]] + digest = "1:fd53b471edb4c28c7d297f617f4da0d33402755f58d6301e7ca1197ef0a90937" + name = "github.com/gogo/protobuf" + packages = ["proto"] + pruneopts = "" + revision = "ba06b47c162d49f2af050fb4c75bcbc86a159d5c" + version = "v1.2.1" + +[[projects]] + digest = "1:530233672f656641b365f8efb38ed9fba80e420baff2ce87633813ab3755ed6d" + name = "github.com/golang/mock" + packages = ["gomock"] + pruneopts = "" + revision = "51421b967af1f557f93a59e0057aaf15ca02e29c" + version = "v1.2.0" + [[projects]] digest = "1:3dd078fda7500c341bc26cfbc6c6a34614f295a2457149fc1045cab767cbcf18" name = "github.com/golang/protobuf" @@ -89,6 +224,213 @@ revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5" version = "v1.2.0" +[[projects]] + branch = "master" + digest = "1:f6a3ed95affdd867195dec281bf1d328dddb37590463eba1f7b39b44a9829e3d" + name = "github.com/golangci/check" + packages = [ + "cmd/structcheck", + "cmd/varcheck", + ] + pruneopts = "" + revision = "cfe4005ccda277a820149d44d6ededc400cc99a2" + +[[projects]] + branch = "master" + digest = "1:262000a2de14b1d0a802acb611e7ee69208b7e3e08f7d0e62226bb324f12e375" + name = "github.com/golangci/dupl" + packages = [ + ".", + "job", + "printer", + "suffixtree", + "syntax", + "syntax/golang", + ] + pruneopts = "" + revision = "3e9179ac440a0386ac7cc9a085fc44397c6b9bbc" + +[[projects]] + branch = "master" + digest = "1:2298a8780ede449cb58108de23925ac2a14cca8ac151cfae45ea5992054d6cf2" + name = "github.com/golangci/errcheck" + packages = [ + "golangci", + "internal/errcheck", + ] + pruneopts = "" + revision = "ef45e06d44b6e018d817c16c762d448990adc5e0" + +[[projects]] + branch = "master" + digest = "1:9b38ad496c9dabd1c820609c481f59c6c9597926c6125810af3d7a71bf2d649c" + name = "github.com/golangci/go-misc" + packages = ["deadcode"] + pruneopts = "" + revision = "927a3d87b613e9f6f0fb7ef8bb8de8b83c30a5a2" + +[[projects]] + branch = "master" + digest = "1:e4bbd53b867030ca8b2e0f6d7338cec2373baf14109858312daa51a144f4a091" + name = "github.com/golangci/go-tools" + packages = [ + "arg", + "callgraph", + "callgraph/static", + "config", + "deprecated", + "functions", + "internal/sharedcheck", + "lint", + "lint/lintdsl", + "lint/lintutil", + "lint/lintutil/format", + "simple", + "ssa", + "ssa/ssautil", + "ssautil", + "staticcheck", + "staticcheck/vrp", + "stylecheck", + "unused", + "version", + ] + pruneopts = "" + revision = "35a9f45a5db090b0227d692d823151104cd695fa" + +[[projects]] + branch = "master" + digest = "1:bc3387ddcbdacf135af0a16b9e9ec6ac7bf5a1f822f679d8d29c2d97cfcce205" + name = "github.com/golangci/goconst" + packages = ["."] + pruneopts = "" + revision = "041c5f2b40f3dd334a4a6ee6a3f84ca3fc70680a" + +[[projects]] + branch = "master" + digest = "1:c5c9e52a4aaca585c1ce9c79f5ea31d74d03da39dfccda0b140f93d6a1be17b7" + name = "github.com/golangci/gocyclo" + packages = ["pkg/gocyclo"] + pruneopts = "" + revision = "0a533e8fa43d6605069e94f455bf9d79d4b8ea8c" + +[[projects]] + branch = "master" + digest = "1:edccfa947bd237dcd1bceef56d1670c22930831ca196ff04f0e2c4a8483bf97b" + name = "github.com/golangci/gofmt" + packages = [ + "gofmt", + "goimports", + ] + pruneopts = "" + revision = "0b8337e80d98f7eec18e4504a4557b34423fd039" + +[[projects]] + digest = "1:0071a728673f03bd75b65863a37d4b1c5bb06ffc4a4416f1a8b6b90f36b2c5e3" + name = "github.com/golangci/golangci-lint" + packages = [ + "cmd/golangci-lint", + "pkg/commands", + "pkg/config", + "pkg/exitcodes", + "pkg/fsutils", + "pkg/golinters", + "pkg/goutil", + "pkg/lint", + "pkg/lint/astcache", + "pkg/lint/linter", + "pkg/lint/lintersdb", + "pkg/logutils", + "pkg/packages", + "pkg/printers", + "pkg/report", + "pkg/result", + "pkg/result/processors", + "pkg/timeutils", + ] + pruneopts = "" + revision = "901cf25e20f86b7e9dc6f73eaba5afbd0cbdc257" + version = "v1.15.0" + +[[projects]] + branch = "master" + digest = "1:30c45dd735f55c7dbd0ea6040e3ccc35f867532b8e1919016c0565510392417a" + name = "github.com/golangci/gosec" + packages = [ + ".", + "rules", + ] + pruneopts = "" + revision = "8afd9cbb6cfb34a3b4d4d5711bafdc6640ae892f" + +[[projects]] + branch = "master" + digest = "1:081d9ed8ba13ebbd4bd3e1f17cd703f77268416074588c38ce985d654b1fc0e1" + name = "github.com/golangci/govet" + packages = [ + ".", + "lib/cfg", + "lib/whitelist", + ] + pruneopts = "" + revision = "44ddbe260190d79165f4150b828650780405d801" + +[[projects]] + branch = "master" + digest = "1:7da7fde58cf7cf5e19f6a1c77eb153945b28cf03bab227e0d831897b7070b546" + name = "github.com/golangci/ineffassign" + packages = ["."] + pruneopts = "" + revision = "2ee8f2867dde308c46d401d6d30f6c644094b167" + +[[projects]] + branch = "master" + digest = "1:8e50794fcb5f229576cd7eda5627a6c2f20341079f0c571077a7ab807c518da9" + name = "github.com/golangci/lint-1" + packages = ["."] + pruneopts = "" + revision = "d2cdd8c0821928c61cb0903441f8b35457a98a61" + +[[projects]] + branch = "master" + digest = "1:8665edfb3c5371fbac9820d127fa0d9aed813cc2349a27a7d16064dd89fed146" + name = "github.com/golangci/maligned" + packages = ["."] + pruneopts = "" + revision = "b1d89398deca2fd3f8578e5a9551e819bd01ca5f" + +[[projects]] + digest = "1:dbf28ceee27335219701dd4c6639c767eee31e2abb61485cdb1044587a04c077" + name = "github.com/golangci/misspell" + packages = ["."] + pruneopts = "" + revision = "b90dc15cfd220ecf8bbc9043ecb928cef381f011" + version = "v0.3.4" + +[[projects]] + branch = "master" + digest = "1:045c2735b360cbebf398a0e9312aeafebf08fd38f0d51cb2aa0f9420364c3cd1" + name = "github.com/golangci/prealloc" + packages = ["."] + pruneopts = "" + revision = "215b22d4de21190b80ce05e7d8466677c1aa3223" + +[[projects]] + branch = "master" + digest = "1:c23cf3c7078c3ba927492557a40c1ee1755734d4fff0e7fbe6d2f092604dae6d" + name = "github.com/golangci/revgrep" + packages = ["."] + pruneopts = "" + revision = "276a5c0a103935ee65af49afc254a65335bf1fcf" + +[[projects]] + branch = "master" + digest = "1:c553e7c7483f2d6db1e84a27a18df144ed4041792d7556916369f86ccf5409fe" + name = "github.com/golangci/unconvert" + packages = ["."] + pruneopts = "" + revision = "28b1c447d1f4a810737ee6ab40ea6c1d0ceae4ad" + [[projects]] digest = "1:ad92aa49f34cbc3546063c7eb2cabb55ee2278b72842eda80e2a20a8a06a8d73" name = "github.com/google/uuid" @@ -97,6 +439,25 @@ revision = "0cd6bf5da1e1c83f8b45653022c74f71af0538a4" version = "v1.1.1" +[[projects]] + digest = "1:d14365c51dd1d34d5c79833ec91413bfbb166be978724f15701e17080dc06dec" + name = "github.com/hashicorp/hcl" + packages = [ + ".", + "hcl/ast", + "hcl/parser", + "hcl/printer", + "hcl/scanner", + "hcl/strconv", + "hcl/token", + "json/parser", + "json/scanner", + "json/token", + ] + pruneopts = "" + revision = "8cb6e5b959231cc1119e43259c4a608f9c51a241" + version = "v1.0.0" + [[projects]] branch = "master" digest = "1:cb09475f771b9167fb9333629f5d6a7161572602ea040f1094602b0dc8709878" @@ -105,6 +466,25 @@ pruneopts = "" revision = "db4671f3a9b8df855e993f7c94ec5ef1ffb0a23b" +[[projects]] + digest = "1:765270f95ea68ad2150f6143eb8b9c0c17b038a7e2255b46580674471af00e27" + name = "github.com/kisielk/gotool" + packages = [ + ".", + "internal/load", + ] + pruneopts = "" + revision = "80517062f582ea3340cd4baf70e86d539ae7d84d" + version = "v1.0.0" + +[[projects]] + digest = "1:0f51cee70b0d254dbc93c22666ea2abf211af81c1701a96d04e2284b408621db" + name = "github.com/konsorten/go-windows-terminal-sequences" + packages = ["."] + pruneopts = "" + revision = "f55edac94c9bbba5d6182a4be46d86a2c9b5b50e" + version = "v1.0.2" + [[projects]] branch = "master" digest = "1:1ed9eeebdf24aadfbca57eb50e6455bd1d2474525e0f0d4454de8c8e9bc7ee9a" @@ -137,6 +517,14 @@ revision = "345fbb3dbcdb252d9985ee899a84963c0fa24c82" version = "v1.0" +[[projects]] + digest = "1:961dc3b1d11f969370533390fdf203813162980c858e1dabe827b60940c909a5" + name = "github.com/magiconair/properties" + packages = ["."] + pruneopts = "" + revision = "c2353362d570a7bfa228149c62842019201cfb71" + version = "v1.8.0" + [[projects]] digest = "1:9ea83adf8e96d6304f394d40436f2eb44c1dc3250d223b74088cc253a6cd0a1c" name = "github.com/mattn/go-colorable" @@ -169,6 +557,22 @@ revision = "3247c84500bff8d9fb6d579d800f20b3e091582c" version = "v1.0.0" +[[projects]] + digest = "1:6dbb0eb72090871f2e58d1e37973fe3cb8c0f45f49459398d3fc740cb30e13bd" + name = "github.com/mitchellh/go-homedir" + packages = ["."] + pruneopts = "" + revision = "af06845cf3004701891bf4fdb884bfe4920b3727" + version = "v1.1.0" + +[[projects]] + digest = "1:bcc46a0fbd9e933087bef394871256b5c60269575bb661935874729c65bbbf60" + name = "github.com/mitchellh/mapstructure" + packages = ["."] + pruneopts = "" + revision = "3536a929edddb9a5b34bd6861dc4a9647cb459fe" + version = "v1.1.2" + [[projects]] digest = "1:4ff67dde814694496d7aa31be44b900f9717a10c8bc9136b13f49c8ef97f439a" name = "github.com/montanaflynn/stats" @@ -177,6 +581,24 @@ revision = "63fbb2597b7a13043b453a4b819945badb8f8926" version = "v0.5.0" +[[projects]] + digest = "1:9da71b9d17d6231f1486dc62d81af3f9d34535703ba9e7a60a902433c3091e3b" + name = "github.com/nbutton23/zxcvbn-go" + packages = [ + ".", + "adjacency", + "data", + "entropy", + "frequency", + "match", + "matching", + "scoring", + "utils/math", + ] + pruneopts = "" + revision = "eafdab6b0663b4b528c35975c8b0e78be6e25261" + version = "v0.1" + [[projects]] branch = "master" digest = "1:f60ff065b58bd53e641112b38bbda9d2684deb828393c7ffb89c69a1ee301d17" @@ -185,6 +607,14 @@ pruneopts = "" revision = "0fd16699aae1833640fca52a937944c6f3b1d58c" +[[projects]] + digest = "1:894aef961c056b6d85d12bac890bf60c44e99b46292888bfa66caf529f804457" + name = "github.com/pelletier/go-toml" + packages = ["."] + pruneopts = "" + revision = "c01d1270ff3e442a8a57cddc1c92dc1138598194" + version = "v1.2.0" + [[projects]] digest = "1:7365acd48986e205ccb8652cc746f09c8b7876030d53710ea6ef7d0bd0dcd7ca" name = "github.com/pkg/errors" @@ -269,6 +699,33 @@ revision = "1744e2970ca51c86172c8190fadad617561ed6e7" version = "v1.0.0" +[[projects]] + digest = "1:b73fe282e350b3ef2c71d8ff08e929e0b9670b1bb5b7fde1d3c1b4cd6e6dc8b1" + name = "github.com/sirupsen/logrus" + packages = ["."] + pruneopts = "" + revision = "dae0fa8d5b0c810a8ab733fbd5510c7cae84eca4" + version = "v1.4.0" + +[[projects]] + digest = "1:956f655c87b7255c6b1ae6c203ebb0af98cf2a13ef2507e34c9bf1c0332ac0f5" + name = "github.com/spf13/afero" + packages = [ + ".", + "mem", + ] + pruneopts = "" + revision = "588a75ec4f32903aa5e39a2619ba6a4631e28424" + version = "v1.2.2" + +[[projects]] + digest = "1:ae3493c780092be9d576a1f746ab967293ec165e8473425631f06658b6212afc" + name = "github.com/spf13/cast" + packages = ["."] + pruneopts = "" + revision = "8c9545af88b134710ab1cd196795e7f2388358d7" + version = "v1.3.0" + [[projects]] branch = "master" digest = "1:146327ce93be37e68bd3ff8541090d96da8cb3adc9e35d57570e9170a29f6bf6" @@ -277,6 +734,14 @@ pruneopts = "" revision = "b78744579491c1ceeaaa3b40205e56b0591b93a3" +[[projects]] + digest = "1:cc15ae4fbdb02ce31f3392361a70ac041f4f02e0485de8ffac92bd8033e3d26e" + name = "github.com/spf13/jwalterweatherman" + packages = ["."] + pruneopts = "" + revision = "94f6ae3ed3bceceafa716478c5fbf8d29ca601a1" + version = "v1.1.0" + [[projects]] digest = "1:261bc565833ef4f02121450d74eb88d5ae4bd74bfe5d0e862cddb8550ec35000" name = "github.com/spf13/pflag" @@ -285,6 +750,14 @@ revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66" version = "v1.0.0" +[[projects]] + digest = "1:90fe60ab6f827e308b0c8cc1e11dce8ff1e96a927c8b171271a3cb04dd517606" + name = "github.com/spf13/viper" + packages = ["."] + pruneopts = "" + revision = "9e56dacc08fbbf8c9ee2dbc717553c758ce42bc9" + version = "v1.3.2" + [[projects]] digest = "1:3926a4ec9a4ff1a072458451aa2d9b98acd059a45b38f7335d31e06c3d6a0159" name = "github.com/stretchr/testify" @@ -331,6 +804,14 @@ pruneopts = "" revision = "08227ad854131f7dfcdfb12579fb73dd8a38a03a" +[[projects]] + branch = "master" + digest = "1:36ef1d8645934b1744cc7d8726e00d3dd9d8d84c18617bf7367a3a6d532f3370" + name = "golang.org/x/crypto" + packages = ["ssh/terminal"] + pruneopts = "" + revision = "a5d413f7728c81fb97d96a2b722368945f651e78" + [[projects]] branch = "master" digest = "1:ea539c13b066dac72a940b62f37600a20ab8e88057397c78f3197c1a48475425" @@ -351,7 +832,10 @@ branch = "master" digest = "1:f358024b019f87eecaadcb098113a40852c94fe58ea670ef3c3e2d2c7bd93db1" name = "golang.org/x/sys" - packages = ["unix"] + packages = [ + "unix", + "windows", + ] pruneopts = "" revision = "4ed8d59d0b35e1e29334a206d1b3f38b1e5dfb31" @@ -375,6 +859,7 @@ "unicode/cldr", "unicode/norm", "unicode/rangetable", + "width", ] pruneopts = "" revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" @@ -384,7 +869,24 @@ branch = "master" digest = "1:4cd780b2ee42c8eac9c02bfb6e6b52dcbaef770774458c8938f5cbfb73a7b6d3" name = "golang.org/x/tools" - packages = ["cmd/stringer"] + packages = [ + "cmd/goimports", + "cmd/stringer", + "go/ast/astutil", + "go/buildutil", + "go/gcexportdata", + "go/internal/cgo", + "go/internal/gcimporter", + "go/loader", + "go/packages", + "go/ssa", + "go/ssa/ssautil", + "go/types/typeutil", + "imports", + "internal/fastwalk", + "internal/gopathwalk", + "internal/semver", + ] pruneopts = "" revision = "d0ca3933b724e6be513276cc2edb34e10d667438" @@ -436,6 +938,54 @@ revision = "df014850f6dee74ba2fc94874043a9f3f75fbfd8" version = "v1.17.0" +[[projects]] + digest = "1:cedccf16b71e86db87a24f8d4c70b0a855872eb967cb906a66b95de56aefbd0d" + name = "gopkg.in/yaml.v2" + packages = ["."] + pruneopts = "" + revision = "51d6538a90f86fe93ac480b35f37b2be17fef232" + version = "v2.2.2" + +[[projects]] + branch = "master" + digest = "1:2a6012038cdeb9851f1a71497544820e17ff2772b3cf799d24a76208cb9843b8" + name = "mvdan.cc/interfacer" + packages = ["check"] + pruneopts = "" + revision = "c20040233aedb03da82d460eca6130fcd91c629a" + +[[projects]] + branch = "master" + digest = "1:68e12be99c0d3355e04eecba6bc302876268a134a0eecd75258d8fefe44a94ed" + name = "mvdan.cc/lint" + packages = ["."] + pruneopts = "" + revision = "adc824a0674b99099789b6188a058d485eaf61c0" + +[[projects]] + branch = "master" + digest = "1:4af0788cd865cab3c8276462de56bad858e178199415241c2420e13e95d8594c" + name = "mvdan.cc/unparam" + packages = ["check"] + pruneopts = "" + revision = "1b9ccfa71afe53433971717161c9666adfc4d8c5" + +[[projects]] + digest = "1:0778809e0f18d0c0c05105a5c1e583d2253c5fd66fbd2b79b00e5f6439402491" + name = "sourcegraph.com/sourcegraph/go-diff" + packages = ["diff"] + pruneopts = "" + revision = "c613306ac97fb4807862c088149199f0dab8685a" + version = "v0.5.0" + +[[projects]] + digest = "1:ffc8cfc88692d5daab7abac1d989e9f7fc09727e42a945702f8f2d6d67f0fd6c" + name = "sourcegraph.com/sqs/pbtypes" + packages = ["."] + pruneopts = "" + revision = "688c2c2cb411327a50aae0f89119af9f38b0fc03" + version = "v1.0.0" + [solve-meta] analyzer-name = "dep" analyzer-version = 1 @@ -446,6 +996,7 @@ "github.com/go-logfmt/logfmt", "github.com/golang/protobuf/proto", "github.com/golang/protobuf/protoc-gen-go", + "github.com/golangci/golangci-lint/cmd/golangci-lint", "github.com/google/uuid", "github.com/jinzhu/copier", "github.com/kr/pretty", @@ -465,6 +1016,7 @@ "github.com/zrepl/yaml-config", "golang.org/x/net/context", "golang.org/x/sys/unix", + "golang.org/x/tools/cmd/goimports", "golang.org/x/tools/cmd/stringer", "google.golang.org/grpc", "google.golang.org/grpc/codes", diff --git a/Gopkg.toml b/Gopkg.toml index 55c0a9b..578f14b 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -5,6 +5,8 @@ ignored = [ required = [ "golang.org/x/tools/cmd/stringer", "github.com/alvaroloes/enumer", + "github.com/golangci/golangci-lint/cmd/golangci-lint", + "golang.org/x/tools/cmd/goimports", ] [[constraint]] diff --git a/Makefile b/Makefile index 42b6990..dddfd69 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: generate build test vet cover release docs docs-clean clean vendordeps +.PHONY: generate build test vet cover release docs docs-clean clean vendordeps format lint .DEFAULT_GOAL := build ARTIFACTDIR := artifacts @@ -30,6 +30,12 @@ generate: #not part of the build, must do that manually protoc -I=replication/logic/pdu --go_out=plugins=grpc:replication/logic/pdu replication/logic/pdu/pdu.proto go generate -x ./... +format: + goimports -srcdir . -local 'github.com/zrepl/zrepl' -w $(shell find . -type f -name '*.go' -not -path "./vendor/*" -not -name '*.pb.go' -not -name '*_enumer.go') + +lint: + golangci-lint run ./... + build: @echo "INFO: In case of missing dependencies, run 'make vendordeps'" $(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl" @@ -68,7 +74,7 @@ docs-clean: .PHONY: $(RELEASE_BINS) # TODO: two wildcards possible -$(RELEASE_BINS): $(ARTIFACTDIR)/zrepl-%-amd64: generate $(ARTIFACTDIR) vet test +$(RELEASE_BINS): $(ARTIFACTDIR)/zrepl-%-amd64: generate $(ARTIFACTDIR) vet test lint @echo "INFO: In case of missing dependencies, run 'make vendordeps'" GOOS=$* GOARCH=amd64 $(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl-$*-amd64" diff --git a/build/build.go b/build/build.go index ec978e4..931fc64 100644 --- a/build/build.go +++ b/build/build.go @@ -11,7 +11,8 @@ package main import ( - _ "fmt" + "fmt" + _ "github.com/alvaroloes/enumer" _ "github.com/golang/protobuf/protoc-gen-go" _ "golang.org/x/tools/cmd/stringer" diff --git a/cli/cli.go b/cli/cli.go index 6bfc313..e84affe 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -2,10 +2,12 @@ package cli import ( "fmt" + "os" + "github.com/spf13/cobra" "github.com/spf13/pflag" + "github.com/zrepl/zrepl/config" - "os" ) var rootArgs struct { @@ -23,7 +25,10 @@ var bashcompCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { if len(args) != 1 { fmt.Fprintf(os.Stderr, "specify exactly one positional agument\n") - cmd.Usage() + err := cmd.Usage() + if err != nil { + panic(err) + } os.Exit(1) } if err := rootCmd.GenBashCompletionFile(args[0]); err != nil { @@ -40,15 +45,15 @@ func init() { } type Subcommand struct { - Use string - Short string - Example string - NoRequireConfig bool - Run func(subcommand *Subcommand, args []string) error - SetupFlags func(f *pflag.FlagSet) - SetupSubcommands func() []*Subcommand + Use string + Short string + Example string + NoRequireConfig bool + Run func(subcommand *Subcommand, args []string) error + SetupFlags func(f *pflag.FlagSet) + SetupSubcommands func() []*Subcommand - config *config.Config + config *config.Config configErr error } @@ -93,8 +98,8 @@ func AddSubcommand(s *Subcommand) { func addSubcommandToCobraCmd(c *cobra.Command, s *Subcommand) { cmd := cobra.Command{ - Use: s.Use, - Short: s.Short, + Use: s.Use, + Short: s.Short, Example: s.Example, } if s.SetupSubcommands == nil { @@ -110,7 +115,6 @@ func addSubcommandToCobraCmd(c *cobra.Command, s *Subcommand) { c.AddCommand(&cmd) } - func Run() { if err := rootCmd.Execute(); err != nil { os.Exit(1) diff --git a/client/configcheck.go b/client/configcheck.go index 77d3699..10c55d5 100644 --- a/client/configcheck.go +++ b/client/configcheck.go @@ -3,39 +3,49 @@ package client import ( "encoding/json" "fmt" + "os" + "github.com/kr/pretty" "github.com/pkg/errors" "github.com/spf13/pflag" "github.com/zrepl/yaml-config" + "github.com/zrepl/zrepl/cli" "github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/daemon/job" "github.com/zrepl/zrepl/daemon/logging" "github.com/zrepl/zrepl/logger" - "os" ) var configcheckArgs struct { format string - what string + what string } var ConfigcheckCmd = &cli.Subcommand{ - Use: "configcheck", + Use: "configcheck", Short: "check if config can be parsed without errors", SetupFlags: func(f *pflag.FlagSet) { f.StringVar(&configcheckArgs.format, "format", "", "dump parsed config object [pretty|yaml|json]") f.StringVar(&configcheckArgs.what, "what", "all", "what to print [all|config|jobs|logging]") }, Run: func(subcommand *cli.Subcommand, args []string) error { - formatMap := map[string]func(interface{}) { + formatMap := map[string]func(interface{}){ "": func(i interface{}) {}, - "pretty": func(i interface{}) { pretty.Println(i) }, + "pretty": func(i interface{}) { + if _, err := pretty.Println(i); err != nil { + panic(err) + } + }, "json": func(i interface{}) { - json.NewEncoder(os.Stdout).Encode(subcommand.Config()) + if err := json.NewEncoder(os.Stdout).Encode(subcommand.Config()); err != nil { + panic(err) + } }, "yaml": func(i interface{}) { - yaml.NewEncoder(os.Stdout).Encode(subcommand.Config()) + if err := yaml.NewEncoder(os.Stdout).Encode(subcommand.Config()); err != nil { + panic(err) + } }, } @@ -71,12 +81,11 @@ var ConfigcheckCmd = &cli.Subcommand{ } } - - whatMap := map[string]func() { + whatMap := map[string]func(){ "all": func() { o := struct { - config *config.Config - jobs []job.Job + config *config.Config + jobs []job.Job logging *logger.Outlets }{ subcommand.Config(), @@ -109,4 +118,3 @@ var ConfigcheckCmd = &cli.Subcommand{ } }, } - diff --git a/client/jsonclient.go b/client/jsonclient.go index 41c272d..3152b99 100644 --- a/client/jsonclient.go +++ b/client/jsonclient.go @@ -4,10 +4,11 @@ import ( "bytes" "context" "encoding/json" - "github.com/pkg/errors" "io" "net" "net/http" + + "github.com/pkg/errors" ) func controlHttpClient(sockpath string) (client http.Client, err error) { @@ -35,7 +36,7 @@ func jsonRequestResponse(c http.Client, endpoint string, req interface{}, res in if resp.StatusCode != http.StatusOK { var msg bytes.Buffer - io.CopyN(&msg, resp.Body, 4096) + _, _ = io.CopyN(&msg, resp.Body, 4096) // ignore error, just display what we got return errors.Errorf("%s", msg.String()) } diff --git a/client/migrate.go b/client/migrate.go index 3b516bf..47d5f22 100644 --- a/client/migrate.go +++ b/client/migrate.go @@ -6,6 +6,7 @@ import ( "github.com/pkg/errors" "github.com/spf13/pflag" + "github.com/zrepl/zrepl/zfs" "github.com/zrepl/zrepl/cli" @@ -22,11 +23,6 @@ var ( } ) -type migration struct { - name string - method func(config *config.Config, args []string) error -} - var migrations = []*cli.Subcommand{ &cli.Subcommand{ Use: "0.0.X:0.1:placeholder", diff --git a/client/pprof.go b/client/pprof.go index 29cd4e5..1702e84 100644 --- a/client/pprof.go +++ b/client/pprof.go @@ -2,11 +2,12 @@ package client import ( "errors" + "log" + "os" + "github.com/zrepl/zrepl/cli" "github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/daemon" - "log" - "os" ) var pprofArgs struct { diff --git a/client/signal.go b/client/signal.go index 701849e..472c7ed 100644 --- a/client/signal.go +++ b/client/signal.go @@ -2,6 +2,7 @@ package client import ( "github.com/pkg/errors" + "github.com/zrepl/zrepl/cli" "github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/daemon" @@ -28,10 +29,10 @@ func runSignalCmd(config *config.Config, args []string) error { err = jsonRequestResponse(httpc, daemon.ControlJobEndpointSignal, struct { Name string - Op string + Op string }{ Name: args[1], - Op: args[0], + Op: args[0], }, struct{}{}, ) diff --git a/client/status.go b/client/status.go index 9abc07a..9c85a9e 100644 --- a/client/status.go +++ b/client/status.go @@ -2,15 +2,6 @@ package client import ( "fmt" - "github.com/gdamore/tcell/termbox" - "github.com/pkg/errors" - "github.com/spf13/pflag" - "github.com/zrepl/yaml-config" - "github.com/zrepl/zrepl/cli" - "github.com/zrepl/zrepl/daemon" - "github.com/zrepl/zrepl/daemon/job" - "github.com/zrepl/zrepl/daemon/pruner" - "github.com/zrepl/zrepl/replication/report" "io" "math" "net/http" @@ -19,18 +10,29 @@ import ( "strings" "sync" "time" + + "github.com/gdamore/tcell/termbox" + "github.com/pkg/errors" + "github.com/spf13/pflag" + "github.com/zrepl/yaml-config" + + "github.com/zrepl/zrepl/cli" + "github.com/zrepl/zrepl/daemon" + "github.com/zrepl/zrepl/daemon/job" + "github.com/zrepl/zrepl/daemon/pruner" + "github.com/zrepl/zrepl/replication/report" ) type byteProgressMeasurement struct { time time.Time - val int64 + val int64 } type bytesProgressHistory struct { - last *byteProgressMeasurement // pointer as poor man's optional + last *byteProgressMeasurement // pointer as poor man's optional changeCount int - lastChange time.Time - bpsAvg float64 + lastChange time.Time + bpsAvg float64 } func (p *bytesProgressHistory) Update(currentVal int64) (bytesPerSecondAvg int64, changeCount int) { @@ -38,7 +40,7 @@ func (p *bytesProgressHistory) Update(currentVal int64) (bytesPerSecondAvg int64 if p.last == nil { p.last = &byteProgressMeasurement{ time: time.Now(), - val: currentVal, + val: currentVal, } return 0, 0 } @@ -48,18 +50,17 @@ func (p *bytesProgressHistory) Update(currentVal int64) (bytesPerSecondAvg int64 p.lastChange = time.Now() } - if time.Now().Sub(p.lastChange) > 3 * time.Second { + if time.Since(p.lastChange) > 3*time.Second { p.last = nil return 0, 0 } - - deltaV := currentVal - p.last.val; - deltaT := time.Now().Sub(p.last.time) + deltaV := currentVal - p.last.val + deltaT := time.Since(p.last.time) rate := float64(deltaV) / deltaT.Seconds() factor := 0.3 - p.bpsAvg = (1-factor) * p.bpsAvg + factor * rate + p.bpsAvg = (1-factor)*p.bpsAvg + factor*rate p.last.time = time.Now() p.last.val = currentVal @@ -80,15 +81,10 @@ type tui struct { func newTui() tui { return tui{ - replicationProgress: make(map[string]*bytesProgressHistory, 0), + replicationProgress: make(map[string]*bytesProgressHistory), } } -func (t *tui) moveCursor(x, y int) { - t.x += x - t.y += y -} - const INDENT_MULTIPLIER = 4 func (t *tui) moveLine(dl int, col int) { @@ -119,7 +115,7 @@ func wrap(s string, width int) string { rem = len(s) } if idx := strings.IndexAny(s, "\n\r"); idx != -1 && idx < rem { - rem = idx+1 + rem = idx + 1 } untilNewline := strings.TrimRight(s[:rem], "\n\r") s = s[rem:] @@ -135,12 +131,12 @@ func wrap(s string, width int) string { func (t *tui) printfDrawIndentedAndWrappedIfMultiline(format string, a ...interface{}) { whole := fmt.Sprintf(format, a...) width, _ := termbox.Size() - if !strings.ContainsAny(whole, "\n\r") && t.x + len(whole) <= width { + if !strings.ContainsAny(whole, "\n\r") && t.x+len(whole) <= width { t.printf(format, a...) } else { t.addIndent(1) t.newline() - t.write(wrap(whole, width - INDENT_MULTIPLIER*t.indent)) + t.write(wrap(whole, width-INDENT_MULTIPLIER*t.indent)) t.addIndent(-1) } } @@ -159,7 +155,6 @@ func (t *tui) addIndent(indent int) { t.moveLine(0, 0) } - var statusFlags struct { Raw bool } @@ -180,14 +175,17 @@ func runStatus(s *cli.Subcommand, args []string) error { } if statusFlags.Raw { - resp, err := httpc.Get("http://unix"+daemon.ControlJobEndpointStatus) + resp, err := httpc.Get("http://unix" + daemon.ControlJobEndpointStatus) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { fmt.Fprintf(os.Stderr, "Received error response:\n") - io.CopyN(os.Stderr, resp.Body, 4096) + _, err := io.CopyN(os.Stderr, resp.Body, 4096) + if err != nil { + return err + } return errors.Errorf("exit") } if _, err := io.Copy(os.Stdout, resp.Body); err != nil { @@ -226,7 +224,7 @@ func runStatus(s *cli.Subcommand, args []string) error { ticker := time.NewTicker(500 * time.Millisecond) defer ticker.Stop() go func() { - for _ = range ticker.C { + for range ticker.C { update() } }() @@ -277,7 +275,7 @@ func (t *tui) draw() { //Iterate over map in alphabetical order keys := make([]string, len(t.report)) i := 0 - for k, _ := range t.report { + for k := range t.report { keys[i] = k i++ } @@ -363,7 +361,7 @@ func (t *tui) renderReplicationReport(rep *report.Report, history *bytesProgress t.newline() } if !rep.WaitReconnectSince.IsZero() { - delta := rep.WaitReconnectUntil.Sub(time.Now()).Round(time.Second) + delta := time.Until(rep.WaitReconnectUntil).Round(time.Second) if rep.WaitReconnectUntil.IsZero() || delta > 0 { var until string if rep.WaitReconnectUntil.IsZero() { @@ -390,7 +388,7 @@ func (t *tui) renderReplicationReport(rep *report.Report, history *bytesProgress t.newline() t.addIndent(1) for i, a := range rep.Attempts[:len(rep.Attempts)-1] { - t.printfDrawIndentedAndWrappedIfMultiline("#%d: %s (failed at %s) (ran %s)", i + 1, a.State, a.FinishAt, a.FinishAt.Sub(a.StartAt)) + t.printfDrawIndentedAndWrappedIfMultiline("#%d: %s (failed at %s) (ran %s)", i+1, a.State, a.FinishAt, a.FinishAt.Sub(a.StartAt)) t.newline() } t.addIndent(-1) @@ -462,7 +460,7 @@ func (t *tui) renderPrunerReport(r *pruner.Report) { *pruner.FSReport completed bool } - all := make([]commonFS, 0, len(r.Pending) + len(r.Completed)) + all := make([]commonFS, 0, len(r.Pending)+len(r.Completed)) for i := range r.Pending { all = append(all, commonFS{&r.Pending[i], false}) } @@ -471,7 +469,8 @@ func (t *tui) renderPrunerReport(r *pruner.Report) { } switch state { - case pruner.Plan: fallthrough + case pruner.Plan: + fallthrough case pruner.PlanErr: return } @@ -499,7 +498,7 @@ func (t *tui) renderPrunerReport(r *pruner.Report) { t.write("[") t.write(times("=", progress)) t.write(">") - t.write(times("-", 80 - progress)) + t.write(times("-", 80-progress)) t.write("]") t.printf(" %d/%d snapshots", completedDestroyCount, totalDestroyCount) t.newline() @@ -519,9 +518,9 @@ func (t *tui) renderPrunerReport(r *pruner.Report) { if fs.LastError != "" { if strings.ContainsAny(fs.LastError, "\r\n") { t.printf("ERROR:") - t.printfDrawIndentedAndWrappedIfMultiline("%s\n", fs.LastError) + t.printfDrawIndentedAndWrappedIfMultiline("%s\n", fs.LastError) } else { - t.printfDrawIndentedAndWrappedIfMultiline("ERROR: %s\n", fs.LastError) + t.printfDrawIndentedAndWrappedIfMultiline("ERROR: %s\n", fs.LastError) } t.newline() continue @@ -531,7 +530,7 @@ func (t *tui) renderPrunerReport(r *pruner.Report) { len(fs.DestroyList), len(fs.SnapshotList)) if fs.completed { - t.printf( "Completed %s\n", pruneRuleActionStr) + t.printf("Completed %s\n", pruneRuleActionStr) continue } @@ -560,14 +559,6 @@ func rightPad(str string, length int, pad string) string { return str + times(pad, length-len(str)) } - -func leftPad(str string, length int, pad string) string { - if len(str) > length { - return str[len(str)-length:] - } - return times(pad, length-len(str)) + str -} - var arrowPositions = `>\|/` // changeCount = 0 indicates stall / no progresss @@ -584,7 +575,7 @@ func (t *tui) drawBar(length int, bytes, totalBytes int64, changeCount int) { t.write("[") t.write(times("=", completedLength)) - t.write( string(arrowPositions[changeCount%len(arrowPositions)])) + t.write(string(arrowPositions[changeCount%len(arrowPositions)])) t.write(times("-", length-completedLength)) t.write("]") } diff --git a/client/stdinserver.go b/client/stdinserver.go index 5db5520..80aa1b4 100644 --- a/client/stdinserver.go +++ b/client/stdinserver.go @@ -1,13 +1,15 @@ package client import ( - "github.com/zrepl/zrepl/cli" "os" + "github.com/problame/go-netssh" + + "github.com/zrepl/zrepl/cli" + "github.com/zrepl/zrepl/config" + "context" "errors" - "github.com/problame/go-netssh" - "github.com/zrepl/zrepl/config" "log" "path" ) diff --git a/client/testcmd.go b/client/testcmd.go index 3066af9..5417c95 100644 --- a/client/testcmd.go +++ b/client/testcmd.go @@ -2,15 +2,17 @@ package client import ( "fmt" + "github.com/pkg/errors" "github.com/spf13/pflag" + "github.com/zrepl/zrepl/cli" "github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/daemon/filters" "github.com/zrepl/zrepl/zfs" ) -var TestCmd = &cli.Subcommand { +var TestCmd = &cli.Subcommand{ Use: "test", SetupSubcommands: func() []*cli.Subcommand { return []*cli.Subcommand{testFilter, testPlaceholder} @@ -18,13 +20,13 @@ var TestCmd = &cli.Subcommand { } var testFilterArgs struct { - job string - all bool + job string + all bool input string } var testFilter = &cli.Subcommand{ - Use: "filesystems --job JOB [--all | --input INPUT]", + Use: "filesystems --job JOB [--all | --input INPUT]", Short: "test filesystems filter specified in push or source job", SetupFlags: func(f *pflag.FlagSet) { f.StringVar(&testFilterArgs.job, "job", "", "the name of the push or source job") @@ -51,8 +53,10 @@ func runTestFilterCmd(subcommand *cli.Subcommand, args []string) error { return err } switch j := job.Ret.(type) { - case *config.SourceJob: confFilter = j.Filesystems - case *config.PushJob: confFilter = j.Filesystems + case *config.SourceJob: + confFilter = j.Filesystems + case *config.PushJob: + confFilter = j.Filesystems default: return fmt.Errorf("job type %T does not have filesystems filter", j) } @@ -109,10 +113,8 @@ func runTestFilterCmd(subcommand *cli.Subcommand, args []string) error { } var testPlaceholderArgs struct { - action string - ds string - plv string - all bool + ds string + all bool } var testPlaceholder = &cli.Subcommand{ diff --git a/client/version.go b/client/version.go index 9dcec05..a0dad26 100644 --- a/client/version.go +++ b/client/version.go @@ -2,23 +2,25 @@ package client import ( "fmt" + "os" + "github.com/spf13/pflag" + "github.com/zrepl/zrepl/cli" "github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/daemon" "github.com/zrepl/zrepl/version" - "os" ) var versionArgs struct { - Show string - Config *config.Config + Show string + Config *config.Config ConfigErr error } var VersionCmd = &cli.Subcommand{ - Use: "version", - Short: "print version of zrepl binary and running daemon", + Use: "version", + Short: "print version of zrepl binary and running daemon", NoRequireConfig: true, SetupFlags: func(f *pflag.FlagSet) { f.StringVar(&versionArgs.Show, "show", "", "version info to show (client|daemon)") diff --git a/config/config.go b/config/config.go index 64a8179..2cff88e 100644 --- a/config/config.go +++ b/config/config.go @@ -2,8 +2,6 @@ package config import ( "fmt" - "github.com/pkg/errors" - "github.com/zrepl/yaml-config" "io/ioutil" "log/syslog" "os" @@ -11,6 +9,9 @@ import ( "regexp" "strconv" "time" + + "github.com/pkg/errors" + "github.com/zrepl/yaml-config" ) type Config struct { @@ -34,11 +35,16 @@ type JobEnum struct { func (j JobEnum) Name() string { var name string switch v := j.Ret.(type) { - case *SnapJob: name = v.Name - case *PushJob: name = v.Name - case *SinkJob: name = v.Name - case *PullJob: name = v.Name - case *SourceJob: name = v.Name + case *SnapJob: + name = v.Name + case *PushJob: + name = v.Name + case *SinkJob: + name = v.Name + case *PullJob: + name = v.Name + case *SourceJob: + name = v.Name default: panic(fmt.Sprintf("unknown job type %T", v)) } @@ -46,38 +52,38 @@ func (j JobEnum) Name() string { } type ActiveJob struct { - Type string `yaml:"type"` - Name string `yaml:"name"` - Connect ConnectEnum `yaml:"connect"` - Pruning PruningSenderReceiver `yaml:"pruning"` - Debug JobDebugSettings `yaml:"debug,optional"` + Type string `yaml:"type"` + Name string `yaml:"name"` + Connect ConnectEnum `yaml:"connect"` + Pruning PruningSenderReceiver `yaml:"pruning"` + Debug JobDebugSettings `yaml:"debug,optional"` } type PassiveJob struct { - Type string `yaml:"type"` - Name string `yaml:"name"` - Serve ServeEnum `yaml:"serve"` - Debug JobDebugSettings `yaml:"debug,optional"` + Type string `yaml:"type"` + Name string `yaml:"name"` + Serve ServeEnum `yaml:"serve"` + Debug JobDebugSettings `yaml:"debug,optional"` } type SnapJob struct { - Type string `yaml:"type"` - Name string `yaml:"name"` - Pruning PruningLocal `yaml:"pruning"` - Debug JobDebugSettings `yaml:"debug,optional"` - Snapshotting SnapshottingEnum `yaml:"snapshotting"` - Filesystems FilesystemsFilter `yaml:"filesystems"` + Type string `yaml:"type"` + Name string `yaml:"name"` + Pruning PruningLocal `yaml:"pruning"` + Debug JobDebugSettings `yaml:"debug,optional"` + Snapshotting SnapshottingEnum `yaml:"snapshotting"` + Filesystems FilesystemsFilter `yaml:"filesystems"` } type PushJob struct { - ActiveJob `yaml:",inline"` - Snapshotting SnapshottingEnum `yaml:"snapshotting"` - Filesystems FilesystemsFilter `yaml:"filesystems"` + ActiveJob `yaml:",inline"` + Snapshotting SnapshottingEnum `yaml:"snapshotting"` + Filesystems FilesystemsFilter `yaml:"filesystems"` } type PullJob struct { ActiveJob `yaml:",inline"` - RootFS string `yaml:"root_fs"` + RootFS string `yaml:"root_fs"` Interval PositiveDurationOrManual `yaml:"interval"` } @@ -118,9 +124,9 @@ type SinkJob struct { } type SourceJob struct { - PassiveJob `yaml:",inline"` - Snapshotting SnapshottingEnum `yaml:"snapshotting"` - Filesystems FilesystemsFilter `yaml:"filesystems"` + PassiveJob `yaml:",inline"` + Snapshotting SnapshottingEnum `yaml:"snapshotting"` + Filesystems FilesystemsFilter `yaml:"filesystems"` } type FilesystemsFilter map[string]bool @@ -130,8 +136,8 @@ type SnapshottingEnum struct { } type SnapshottingPeriodic struct { - Type string `yaml:"type"` - Prefix string `yaml:"prefix"` + Type string `yaml:"type"` + Prefix string `yaml:"prefix"` Interval time.Duration `yaml:"interval,positive"` } @@ -191,7 +197,7 @@ type ConnectEnum struct { } type ConnectCommon struct { - Type string `yaml:"type"` + Type string `yaml:"type"` } type TCPConnect struct { @@ -223,8 +229,8 @@ type SSHStdinserverConnect struct { } type LocalConnect struct { - ConnectCommon `yaml:",inline"` - ListenerName string `yaml:"listener_name"` + ConnectCommon `yaml:",inline"` + ListenerName string `yaml:"listener_name"` ClientIdentity string `yaml:"client_identity"` } @@ -233,7 +239,7 @@ type ServeEnum struct { } type ServeCommon struct { - Type string `yaml:"type"` + Type string `yaml:"type"` } type TCPServe struct { @@ -253,12 +259,12 @@ type TLSServe struct { } type StdinserverServer struct { - ServeCommon `yaml:",inline"` + ServeCommon `yaml:",inline"` ClientIdentities []string `yaml:"client_identities"` } type LocalServe struct { - ServeCommon `yaml:",inline"` + ServeCommon `yaml:",inline"` ListenerName string `yaml:"listener_name"` } @@ -267,8 +273,8 @@ type PruningEnum struct { } type PruneKeepNotReplicated struct { - Type string `yaml:"type"` - KeepSnapshotAtCursor bool `yaml:"keep_snapshot_at_cursor,optional,default=true"` + Type string `yaml:"type"` + KeepSnapshotAtCursor bool `yaml:"keep_snapshot_at_cursor,optional,default=true"` } type PruneKeepLastN struct { @@ -277,8 +283,8 @@ type PruneKeepLastN struct { } type PruneKeepRegex struct { // FIXME rename to KeepRegex - Type string `yaml:"type"` - Regex string `yaml:"regex"` + Type string `yaml:"type"` + Regex string `yaml:"regex"` Negate bool `yaml:"negate,optional,default=false"` } @@ -301,7 +307,7 @@ type StdoutLoggingOutlet struct { type SyslogLoggingOutlet struct { LoggingOutletCommon `yaml:",inline"` Facility *SyslogFacility `yaml:"facility,optional,fromdefaults"` - RetryInterval time.Duration `yaml:"retry_interval,positive,default=10s"` + RetryInterval time.Duration `yaml:"retry_interval,positive,default=10s"` } type TCPLoggingOutlet struct { @@ -392,7 +398,7 @@ func (t *ConnectEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) "tcp": &TCPConnect{}, "tls": &TLSConnect{}, "ssh+stdinserver": &SSHStdinserverConnect{}, - "local": &LocalConnect{}, + "local": &LocalConnect{}, }) return } @@ -402,7 +408,7 @@ func (t *ServeEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) { "tcp": &TCPServe{}, "tls": &TLSServe{}, "stdinserver": &StdinserverServer{}, - "local" : &LocalServe{}, + "local": &LocalServe{}, }) return } @@ -420,7 +426,7 @@ func (t *PruningEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) func (t *SnapshottingEnum) UnmarshalYAML(u func(interface{}, bool) error) (err error) { t.Ret, err = enumUnmarshal(u, map[string]interface{}{ "periodic": &SnapshottingPeriodic{}, - "manual": &SnapshottingManual{}, + "manual": &SnapshottingManual{}, }) return } @@ -448,31 +454,51 @@ func (t *SyslogFacility) UnmarshalYAML(u func(interface{}, bool) error) (err err } var level syslog.Priority switch s { - case "kern": level = syslog.LOG_KERN - case "user": level = syslog.LOG_USER - case "mail": level = syslog.LOG_MAIL - case "daemon": level = syslog.LOG_DAEMON - case "auth": level = syslog.LOG_AUTH - case "syslog": level = syslog.LOG_SYSLOG - case "lpr": level = syslog.LOG_LPR - case "news": level = syslog.LOG_NEWS - case "uucp": level = syslog.LOG_UUCP - case "cron": level = syslog.LOG_CRON - case "authpriv": level = syslog.LOG_AUTHPRIV - case "ftp": level = syslog.LOG_FTP - case "local0": level = syslog.LOG_LOCAL0 - case "local1": level = syslog.LOG_LOCAL1 - case "local2": level = syslog.LOG_LOCAL2 - case "local3": level = syslog.LOG_LOCAL3 - case "local4": level = syslog.LOG_LOCAL4 - case "local5": level = syslog.LOG_LOCAL5 - case "local6": level = syslog.LOG_LOCAL6 - case "local7": level = syslog.LOG_LOCAL7 + case "kern": + level = syslog.LOG_KERN + case "user": + level = syslog.LOG_USER + case "mail": + level = syslog.LOG_MAIL + case "daemon": + level = syslog.LOG_DAEMON + case "auth": + level = syslog.LOG_AUTH + case "syslog": + level = syslog.LOG_SYSLOG + case "lpr": + level = syslog.LOG_LPR + case "news": + level = syslog.LOG_NEWS + case "uucp": + level = syslog.LOG_UUCP + case "cron": + level = syslog.LOG_CRON + case "authpriv": + level = syslog.LOG_AUTHPRIV + case "ftp": + level = syslog.LOG_FTP + case "local0": + level = syslog.LOG_LOCAL0 + case "local1": + level = syslog.LOG_LOCAL1 + case "local2": + level = syslog.LOG_LOCAL2 + case "local3": + level = syslog.LOG_LOCAL3 + case "local4": + level = syslog.LOG_LOCAL4 + case "local5": + level = syslog.LOG_LOCAL5 + case "local6": + level = syslog.LOG_LOCAL6 + case "local7": + level = syslog.LOG_LOCAL7 default: return fmt.Errorf("invalid syslog level: %q", s) } *t = SyslogFacility(level) - return nil + return nil } var ConfigFileDefaultLocations = []string{ diff --git a/config/config_global_test.go b/config/config_global_test.go index 51204b0..f782e9f 100644 --- a/config/config_global_test.go +++ b/config/config_global_test.go @@ -2,11 +2,12 @@ package config import ( "fmt" + "log/syslog" + "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/zrepl/yaml-config" - "log/syslog" - "testing" ) func testValidGlobalSection(t *testing.T, s string) *Config { @@ -24,7 +25,7 @@ jobs: ` _, err := ParseConfigBytes([]byte(jobdef)) require.NoError(t, err) - return testValidConfig(t, s + jobdef) + return testValidConfig(t, s+jobdef) } func TestOutletTypes(t *testing.T) { @@ -71,7 +72,7 @@ global: - type: prometheus listen: ':9091' `) - assert.Equal(t, ":9091", conf.Global.Monitoring[0].Ret.(*PrometheusMonitoring).Listen) + assert.Equal(t, ":9091", conf.Global.Monitoring[0].Ret.(*PrometheusMonitoring).Listen) } func TestSyslogLoggingOutletFacility(t *testing.T) { diff --git a/config/config_minimal_test.go b/config/config_minimal_test.go index 72f8752..74747bf 100644 --- a/config/config_minimal_test.go +++ b/config/config_minimal_test.go @@ -2,6 +2,7 @@ package config import ( "testing" + "github.com/stretchr/testify/assert" ) @@ -36,4 +37,4 @@ jobs: - type: last_n count: 1 `) -} \ No newline at end of file +} diff --git a/config/config_snapshotting_test.go b/config/config_snapshotting_test.go index e0f826b..1bfb7e2 100644 --- a/config/config_snapshotting_test.go +++ b/config/config_snapshotting_test.go @@ -2,9 +2,10 @@ package config import ( "fmt" - "github.com/stretchr/testify/assert" "testing" "time" + + "github.com/stretchr/testify/assert" ) func TestSnapshotting(t *testing.T) { @@ -37,7 +38,7 @@ jobs: interval: 10m ` - fillSnapshotting := func(s string) string {return fmt.Sprintf(tmpl, s)} + fillSnapshotting := func(s string) string { return fmt.Sprintf(tmpl, s) } var c *Config t.Run("manual", func(t *testing.T) { @@ -51,7 +52,7 @@ jobs: snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic) assert.Equal(t, "periodic", snp.Type) assert.Equal(t, 10*time.Minute, snp.Interval) - assert.Equal(t, "zrepl_" , snp.Prefix) + assert.Equal(t, "zrepl_", snp.Prefix) }) } diff --git a/config/config_test.go b/config/config_test.go index d51f3f7..2c17740 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -39,6 +39,7 @@ func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) { } // template must be a template/text template with a single '{{ . }}' as placehodler for val +//nolint[:deadcode,unused] func testValidConfigTemplate(t *testing.T, tmpl string, val string) *Config { tmp, err := template.New("master").Parse(tmpl) if err != nil { diff --git a/config/retentiongrid.go b/config/retentiongrid.go index 58b2ff3..e3ca38b 100644 --- a/config/retentiongrid.go +++ b/config/retentiongrid.go @@ -11,9 +11,9 @@ import ( type RetentionIntervalList []RetentionInterval type PruneGrid struct { - Type string `yaml:"type"` - Grid RetentionIntervalList `yaml:"grid"` - Regex string `yaml:"regex"` + Type string `yaml:"type"` + Grid RetentionIntervalList `yaml:"grid"` + Regex string `yaml:"regex"` } type RetentionInterval struct { @@ -31,10 +31,6 @@ func (i *RetentionInterval) KeepCount() int { const RetentionGridKeepCountAll int = -1 -type RetentionGrid struct { - intervals []RetentionInterval -} - func (t *RetentionIntervalList) UnmarshalYAML(u func(interface{}, bool) error) (err error) { var in string if err := u(&in, true); err != nil { diff --git a/daemon/control.go b/daemon/control.go index 6e60061..9c5e0cb 100644 --- a/daemon/control.go +++ b/daemon/control.go @@ -12,6 +12,7 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + "github.com/zrepl/zrepl/daemon/job" "github.com/zrepl/zrepl/daemon/nethelpers" "github.com/zrepl/zrepl/logger" @@ -43,24 +44,24 @@ func (j *controlJob) Status() *job.Status { return &job.Status{Type: job.TypeInt func (j *controlJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { return nil, false } var promControl struct { - requestBegin *prometheus.CounterVec + requestBegin *prometheus.CounterVec requestFinished *prometheus.HistogramVec } func (j *controlJob) RegisterMetrics(registerer prometheus.Registerer) { promControl.requestBegin = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: "zrepl", - Subsystem: "control", - Name: "request_begin", - Help: "number of request we started to handle", + Namespace: "zrepl", + Subsystem: "control", + Name: "request_begin", + Help: "number of request we started to handle", }, []string{"endpoint"}) promControl.requestFinished = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "zrepl", - Subsystem: "control", - Name: "request_finished", - Help: "time it took a request to finih", - Buckets: []float64{1e-6, 10e-6, 100e-6, 500e-6, 1e-3,10e-3, 100e-3, 200e-3,400e-3,800e-3, 1, 10, 20}, + Namespace: "zrepl", + Subsystem: "control", + Name: "request_finished", + Help: "time it took a request to finih", + Buckets: []float64{1e-6, 10e-6, 100e-6, 500e-6, 1e-3, 10e-3, 100e-3, 200e-3, 400e-3, 800e-3, 1, 10, 20}, }, []string{"endpoint"}) registerer.MustRegister(promControl.requestBegin) registerer.MustRegister(promControl.requestFinished) @@ -88,7 +89,7 @@ func (j *controlJob) Run(ctx context.Context) { mux := http.NewServeMux() mux.Handle(ControlJobEndpointPProf, - requestLogger{log: log, handler: jsonRequestResponder{func(decoder jsonDecoder) (interface{}, error) { + requestLogger{log: log, handler: jsonRequestResponder{log, func(decoder jsonDecoder) (interface{}, error) { var msg PprofServerControlMsg err := decoder(&msg) if err != nil { @@ -99,22 +100,22 @@ func (j *controlJob) Run(ctx context.Context) { }}}) mux.Handle(ControlJobEndpointVersion, - requestLogger{log: log, handler: jsonResponder{func() (interface{}, error) { + requestLogger{log: log, handler: jsonResponder{log, func() (interface{}, error) { return version.NewZreplVersionInformation(), nil }}}) mux.Handle(ControlJobEndpointStatus, // don't log requests to status endpoint, too spammy - jsonResponder{func() (interface{}, error) { + jsonResponder{log, func() (interface{}, error) { s := j.jobs.status() return s, nil }}) mux.Handle(ControlJobEndpointSignal, - requestLogger{log: log, handler: jsonRequestResponder{func(decoder jsonDecoder) (interface{}, error) { + requestLogger{log: log, handler: jsonRequestResponder{log, func(decoder jsonDecoder) (interface{}, error) { type reqT struct { Name string - Op string + Op string } var req reqT if decoder(&req) != nil { @@ -136,8 +137,8 @@ func (j *controlJob) Run(ctx context.Context) { server := http.Server{ Handler: mux, // control socket is local, 1s timeout should be more than sufficient, even on a loaded system - WriteTimeout: 1*time.Second, - ReadTimeout: 1*time.Second, + WriteTimeout: 1 * time.Second, + ReadTimeout: 1 * time.Second, } outer: @@ -152,7 +153,10 @@ outer: select { case <-ctx.Done(): log.WithError(ctx.Err()).Info("context done") - server.Shutdown(context.Background()) + err := server.Shutdown(context.Background()) + if err != nil { + log.WithError(err).Error("cannot shutdown server") + } break outer case err = <-served: if err != nil { @@ -166,33 +170,50 @@ outer: } type jsonResponder struct { + log Logger producer func() (interface{}, error) } func (j jsonResponder) ServeHTTP(w http.ResponseWriter, r *http.Request) { + logIoErr := func(err error) { + if err != nil { + j.log.WithError(err).Error("control handler io error") + } + } res, err := j.producer() if err != nil { + j.log.WithError(err).Error("control handler error") w.WriteHeader(http.StatusInternalServerError) - io.WriteString(w, err.Error()) + _, err = io.WriteString(w, err.Error()) + logIoErr(err) return } var buf bytes.Buffer err = json.NewEncoder(&buf).Encode(res) if err != nil { + j.log.WithError(err).Error("control handler json marshal error") w.WriteHeader(http.StatusInternalServerError) - io.WriteString(w, err.Error()) + _, err = io.WriteString(w, err.Error()) } else { - io.Copy(w, &buf) + _, err = io.Copy(w, &buf) } + logIoErr(err) } type jsonDecoder = func(interface{}) error type jsonRequestResponder struct { + log Logger producer func(decoder jsonDecoder) (interface{}, error) } func (j jsonRequestResponder) ServeHTTP(w http.ResponseWriter, r *http.Request) { + logIoErr := func(err error) { + if err != nil { + j.log.WithError(err).Error("control handler io error") + } + } + var decodeError error decoder := func(i interface{}) error { err := json.NewDecoder(r.Body).Decode(&i) @@ -204,22 +225,28 @@ func (j jsonRequestResponder) ServeHTTP(w http.ResponseWriter, r *http.Request) //If we had a decode error ignore output of producer and return error if decodeError != nil { w.WriteHeader(http.StatusBadRequest) - io.WriteString(w, decodeError.Error()) + _, err := io.WriteString(w, decodeError.Error()) + logIoErr(err) return } if producerErr != nil { + j.log.WithError(producerErr).Error("control handler error") w.WriteHeader(http.StatusInternalServerError) - io.WriteString(w, producerErr.Error()) + _, err := io.WriteString(w, producerErr.Error()) + logIoErr(err) return } var buf bytes.Buffer encodeErr := json.NewEncoder(&buf).Encode(res) if encodeErr != nil { + j.log.WithError(producerErr).Error("control handler json marhsal error") w.WriteHeader(http.StatusInternalServerError) - io.WriteString(w, encodeErr.Error()) + _, err := io.WriteString(w, encodeErr.Error()) + logIoErr(err) } else { - io.Copy(w, &buf) + _, err := io.Copy(w, &buf) + logIoErr(err) } } diff --git a/daemon/daemon.go b/daemon/daemon.go index 9f0e185..94fd7a2 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -3,8 +3,16 @@ package daemon import ( "context" "fmt" + "os" + "os/signal" + "strings" + "sync" + "syscall" + "time" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + "github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/daemon/job" "github.com/zrepl/zrepl/daemon/job/reset" @@ -12,12 +20,6 @@ import ( "github.com/zrepl/zrepl/daemon/logging" "github.com/zrepl/zrepl/logger" "github.com/zrepl/zrepl/version" - "os" - "os/signal" - "strings" - "sync" - "syscall" - "time" ) func Run(conf *config.Config) error { @@ -74,12 +76,11 @@ func Run(conf *config.Config) error { return errors.Errorf("unknown monitoring job #%d (type %T)", i, v) } if err != nil { - return errors.Wrapf(err,"cannot build monitorin gjob #%d", i) + return errors.Wrapf(err, "cannot build monitorin gjob #%d", i) } jobs.start(ctx, job, true) } - log.Info("starting daemon") // start regular jobs @@ -103,7 +104,7 @@ type jobs struct { // m protects all fields below it m sync.RWMutex wakeups map[string]wakeup.Func // by Job.Name - resets map[string]reset.Func // by Job.Name + resets map[string]reset.Func // by Job.Name jobs map[string]job.Job } @@ -116,9 +117,7 @@ func newJobs() *jobs { } const ( - logJobField string = "job" - logTaskField string = "task" - logSubsysField string = "subsystem" + logJobField string = "job" ) func (s *jobs) wait() <-chan struct{} { diff --git a/daemon/filters/fsmapfilter.go b/daemon/filters/fsmapfilter.go index b6a785d..61178c3 100644 --- a/daemon/filters/fsmapfilter.go +++ b/daemon/filters/fsmapfilter.go @@ -2,10 +2,12 @@ package filters import ( "fmt" + "strings" + "github.com/pkg/errors" + "github.com/zrepl/zrepl/endpoint" "github.com/zrepl/zrepl/zfs" - "strings" ) type DatasetMapFilter struct { diff --git a/daemon/filters/fsvfilter.go b/daemon/filters/fsvfilter.go index 3abcca8..812d258 100644 --- a/daemon/filters/fsvfilter.go +++ b/daemon/filters/fsvfilter.go @@ -1,8 +1,9 @@ package filters import ( - "github.com/zrepl/zrepl/zfs" "strings" + + "github.com/zrepl/zrepl/zfs" ) type AnyFSVFilter struct{} @@ -17,7 +18,6 @@ func (AnyFSVFilter) Filter(t zfs.VersionType, name string) (accept bool, err err return true, nil } - type PrefixFilter struct { prefix string fstype zfs.VersionType diff --git a/daemon/job/active.go b/daemon/job/active.go index a5b5a94..a878f19 100644 --- a/daemon/job/active.go +++ b/daemon/job/active.go @@ -68,8 +68,7 @@ type activeSideTasks struct { func (a *ActiveSide) updateTasks(u func(*activeSideTasks)) activeSideTasks { a.tasksMtx.Lock() defer a.tasksMtx.Unlock() - var copy activeSideTasks - copy = a.tasks + copy := a.tasks if u == nil { return copy } diff --git a/daemon/job/build_jobs.go b/daemon/job/build_jobs.go index 2e4a271..dd6707a 100644 --- a/daemon/job/build_jobs.go +++ b/daemon/job/build_jobs.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/pkg/errors" + "github.com/zrepl/zrepl/config" ) @@ -24,13 +25,13 @@ func JobsFromConfig(c *config.Config) ([]Job, error) { // receiving-side root filesystems must not overlap { - rfss := make([]string, len(js)) - for i, j := range js { + rfss := make([]string, 0, len(js)) + for _, j := range js { jrfs, ok := j.OwnedDatasetSubtreeRoot() if !ok { continue } - rfss[i] = jrfs.ToString() + rfss = append(rfss, jrfs.ToString()) } if err := validateReceivingSidesDoNotOverlap(rfss); err != nil { return nil, err diff --git a/daemon/job/job.go b/daemon/job/job.go index a472bd8..42fdfab 100644 --- a/daemon/job/job.go +++ b/daemon/job/job.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/prometheus/client_golang/prometheus" + "github.com/zrepl/zrepl/logger" "github.com/zrepl/zrepl/zfs" ) @@ -29,7 +30,6 @@ func WithLogger(ctx context.Context, l Logger) context.Context { return context.WithValue(ctx, contextKeyLog, l) } - type Job interface { Name() string Run(ctx context.Context) @@ -44,15 +44,15 @@ type Type string const ( TypeInternal Type = "internal" - TypeSnap Type = "snap" - TypePush Type = "push" - TypeSink Type = "sink" - TypePull Type = "pull" - TypeSource Type = "source" + TypeSnap Type = "snap" + TypePush Type = "push" + TypeSink Type = "sink" + TypePull Type = "pull" + TypeSource Type = "source" ) type Status struct { - Type Type + Type Type JobSpecific interface{} } @@ -65,8 +65,8 @@ func (s *Status) MarshalJSON() ([]byte, error) { if err != nil { return nil, err } - m := map[string]json.RawMessage { - "type": typeJson, + m := map[string]json.RawMessage{ + "type": typeJson, string(s.Type): jobJSON, } return json.Marshal(m) @@ -94,16 +94,21 @@ func (s *Status) UnmarshalJSON(in []byte) (err error) { var st SnapJobStatus err = json.Unmarshal(jobJSON, &st) s.JobSpecific = &st - case TypePull: fallthrough + + case TypePull: + fallthrough case TypePush: var st ActiveSideStatus err = json.Unmarshal(jobJSON, &st) s.JobSpecific = &st - case TypeSource: fallthrough + + case TypeSource: + fallthrough case TypeSink: var st PassiveStatus err = json.Unmarshal(jobJSON, &st) s.JobSpecific = &st + case TypeInternal: // internal jobs do not report specifics default: diff --git a/daemon/job/snapjob.go b/daemon/job/snapjob.go index 7eb6bef..7614336 100644 --- a/daemon/job/snapjob.go +++ b/daemon/job/snapjob.go @@ -7,6 +7,7 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + "github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/daemon/filters" "github.com/zrepl/zrepl/daemon/job/wakeup" diff --git a/daemon/logging/logging_formatters.go b/daemon/logging/logging_formatters.go index 3f3a7a8..7179676 100644 --- a/daemon/logging/logging_formatters.go +++ b/daemon/logging/logging_formatters.go @@ -4,11 +4,13 @@ import ( "bytes" "encoding/json" "fmt" + "time" + "github.com/fatih/color" "github.com/go-logfmt/logfmt" "github.com/pkg/errors" + "github.com/zrepl/zrepl/logger" - "time" ) const ( @@ -159,10 +161,16 @@ func (f *LogfmtFormatter) Format(e *logger.Entry) ([]byte, error) { enc := logfmt.NewEncoder(&buf) if f.metadataFlags&MetadataTime != 0 { - enc.EncodeKeyval(FieldTime, e.Time) + err := enc.EncodeKeyval(FieldTime, e.Time) + if err != nil { + return nil, errors.Wrap(err, "logfmt: encode time") + } } if f.metadataFlags&MetadataLevel != 0 { - enc.EncodeKeyval(FieldLevel, e.Level) + err := enc.EncodeKeyval(FieldLevel, e.Level) + if err != nil { + return nil, errors.Wrap(err, "logfmt: encode level") + } } // at least try and put job and task in front @@ -179,8 +187,10 @@ func (f *LogfmtFormatter) Format(e *logger.Entry) ([]byte, error) { prefixed[pf] = true } - enc.EncodeKeyval(FieldMessage, e.Message) - + err := enc.EncodeKeyval(FieldMessage, e.Message) + if err != nil { + return nil, errors.Wrap(err, "logfmt: encode message") + } for k, v := range e.Fields { if !prefixed[k] { if err := logfmtTryEncodeKeyval(enc, k, v); err != nil { @@ -199,7 +209,10 @@ func logfmtTryEncodeKeyval(enc *logfmt.Encoder, field, value interface{}) error case nil: // ok return nil case logfmt.ErrUnsupportedValueType: - enc.EncodeKeyval(field, fmt.Sprintf("<%T>", value)) + err := enc.EncodeKeyval(field, fmt.Sprintf("<%T>", value)) + if err != nil { + return errors.Wrap(err, "cannot encode unsuuported value type Go type") + } return nil } return errors.Wrapf(err, "cannot encode field '%s'", field) diff --git a/daemon/logging/logging_outlets.go b/daemon/logging/logging_outlets.go index b03d008..2abe3e3 100644 --- a/daemon/logging/logging_outlets.go +++ b/daemon/logging/logging_outlets.go @@ -4,12 +4,14 @@ import ( "bytes" "context" "crypto/tls" - "github.com/pkg/errors" - "github.com/zrepl/zrepl/logger" "io" "log/syslog" "net" "time" + + "github.com/pkg/errors" + + "github.com/zrepl/zrepl/logger" ) type EntryFormatter interface { @@ -28,7 +30,10 @@ func (h WriterOutlet) WriteEntry(entry logger.Entry) error { return err } _, err = h.writer.Write(bytes) - h.writer.Write([]byte("\n")) + if err != nil { + return err + } + _, err = h.writer.Write([]byte("\n")) return err } @@ -92,8 +97,10 @@ func (h *TCPOutlet) outLoop(retryInterval time.Duration) { conn = nil } } - conn.SetWriteDeadline(time.Now().Add(retryInterval)) - _, err = io.Copy(conn, msg) + err = conn.SetWriteDeadline(time.Now().Add(retryInterval)) + if err == nil { + _, err = io.Copy(conn, msg) + } if err != nil { retry = time.Now().Add(retryInterval) conn.Close() diff --git a/daemon/main.go b/daemon/main.go index fae8c12..4bad05b 100644 --- a/daemon/main.go +++ b/daemon/main.go @@ -7,7 +7,7 @@ import ( type Logger = logger.Logger -var DaemonCmd = &cli.Subcommand { +var DaemonCmd = &cli.Subcommand{ Use: "daemon", Short: "run the zrepl daemon", Run: func(subcommand *cli.Subcommand, args []string) error { diff --git a/daemon/nethelpers/helpers.go b/daemon/nethelpers/helpers.go index 994b9d2..5c1362c 100644 --- a/daemon/nethelpers/helpers.go +++ b/daemon/nethelpers/helpers.go @@ -1,10 +1,11 @@ package nethelpers import ( - "github.com/pkg/errors" "net" "os" "path/filepath" + + "github.com/pkg/errors" ) func PreparePrivateSockpath(sockpath string) error { diff --git a/daemon/pprof.go b/daemon/pprof.go index 6c96251..8a6ff1a 100644 --- a/daemon/pprof.go +++ b/daemon/pprof.go @@ -7,11 +7,12 @@ import ( "context" "net" "net/http/pprof" + + "github.com/zrepl/zrepl/daemon/job" ) type pprofServer struct { cc chan PprofServerControlMsg - state PprofServerControlMsg listener net.Listener } @@ -63,7 +64,14 @@ outer: mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile)) mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol)) mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace)) - go http.Serve(s.listener, mux) + go func() { + err := http.Serve(s.listener, mux) + if ctx.Err() != nil { + return + } else if err != nil { + job.GetLogger(ctx).WithError(err).Error("pprof server serve error") + } + }() continue } diff --git a/daemon/prometheus.go b/daemon/prometheus.go index 9db3554..3d805ed 100644 --- a/daemon/prometheus.go +++ b/daemon/prometheus.go @@ -2,15 +2,17 @@ package daemon import ( "context" + "net" + "net/http" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/daemon/job" "github.com/zrepl/zrepl/logger" "github.com/zrepl/zrepl/rpc/dataconn/frameconn" "github.com/zrepl/zrepl/zfs" - "net" - "net/http" ) type prometheusJob struct { @@ -25,7 +27,7 @@ func newPrometheusJobFromConfig(in *config.PrometheusMonitoring) (*prometheusJob } var prom struct { - taskLogEntries *prometheus.CounterVec + taskLogEntries *prometheus.CounterVec } func init() { @@ -63,17 +65,15 @@ func (j *prometheusJob) Run(ctx context.Context) { log.WithError(err).Error("cannot listen") } go func() { - select { - case <-ctx.Done(): - l.Close() - } + <-ctx.Done() + l.Close() }() mux := http.NewServeMux() mux.Handle("/metrics", promhttp.Handler()) err = http.Serve(l, mux) - if err != nil { + if err != nil && ctx.Err() == nil { log.WithError(err).Error("error while serving") } @@ -93,4 +93,3 @@ func (o prometheusJobOutlet) WriteEntry(entry logger.Entry) error { prom.taskLogEntries.WithLabelValues(o.jobName, entry.Level.String()).Inc() return nil } - diff --git a/daemon/pruner/pruner.go b/daemon/pruner/pruner.go index d81c8a4..948fbb2 100644 --- a/daemon/pruner/pruner.go +++ b/daemon/pruner/pruner.go @@ -3,17 +3,19 @@ package pruner import ( "context" "fmt" + "sort" + "strings" + "sync" + "time" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + "github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/logger" "github.com/zrepl/zrepl/pruning" "github.com/zrepl/zrepl/replication/logic/pdu" "github.com/zrepl/zrepl/util/envconst" - "sort" - "strings" - "sync" - "time" ) // Try to keep it compatible with gitub.com/zrepl/zrepl/endpoint.Endpoint @@ -53,7 +55,7 @@ type args struct { rules []pruning.KeepRule retryWait time.Duration considerSnapAtCursorReplicated bool - promPruneSecs prometheus.Observer + promPruneSecs prometheus.Observer } type Pruner struct { @@ -64,7 +66,7 @@ type Pruner struct { state State // State PlanErr - err error + err error // State Exec execQueue *execQueue @@ -75,7 +77,7 @@ type PrunerFactory struct { receiverRules []pruning.KeepRule retryWait time.Duration considerSnapAtCursorReplicated bool - promPruneSecs *prometheus.HistogramVec + promPruneSecs *prometheus.HistogramVec } type LocalPrunerFactory struct { @@ -84,19 +86,6 @@ type LocalPrunerFactory struct { promPruneSecs *prometheus.HistogramVec } -func checkContainsKeep1(rules []pruning.KeepRule) error { - if len(rules) == 0 { - return nil //No keep rules means keep all - ok - } - for _, e := range rules { - switch e.(type) { - case *pruning.KeepLastN: - return nil - } - } - return errors.New("sender keep rules must contain last_n or be empty so that the last snapshot is definitely kept") -} - func NewLocalPrunerFactory(in config.PruningLocal, promPruneSecs *prometheus.HistogramVec) (*LocalPrunerFactory, error) { rules, err := pruning.RulesFromConfig(in.Keep) if err != nil { @@ -137,11 +126,11 @@ func NewPrunerFactory(in config.PruningSenderReceiver, promPruneSecs *prometheus considerSnapAtCursorReplicated = considerSnapAtCursorReplicated || !knr.KeepSnapshotAtCursor } f := &PrunerFactory{ - senderRules: keepRulesSender, - receiverRules: keepRulesReceiver, - retryWait: envconst.Duration("ZREPL_PRUNER_RETRY_INTERVAL", 10 * time.Second), + senderRules: keepRulesSender, + receiverRules: keepRulesReceiver, + retryWait: envconst.Duration("ZREPL_PRUNER_RETRY_INTERVAL", 10*time.Second), considerSnapAtCursorReplicated: considerSnapAtCursorReplicated, - promPruneSecs: promPruneSecs, + promPruneSecs: promPruneSecs, } return f, nil } @@ -213,17 +202,17 @@ func (p *Pruner) Prune() { func (p *Pruner) prune(args args) { u := func(f func(*Pruner)) { - p.mtx.Lock() - defer p.mtx.Unlock() - f(p) - } + p.mtx.Lock() + defer p.mtx.Unlock() + f(p) + } // TODO support automatic retries // It is advisable to merge this code with package replication/driver before // That will likely require re-modelling struct fs like replication/driver.attempt, // including figuring out how to resume a plan after being interrupted by network errors // The non-retrying code in this package should move straight to replication/logic. doOneAttempt(&args, u) - } +} type Report struct { State string @@ -239,9 +228,9 @@ type FSReport struct { } type SnapshotReport struct { - Name string + Name string Replicated bool - Date time.Time + Date time.Time } func (p *Pruner) Report() *Report { @@ -250,9 +239,9 @@ func (p *Pruner) Report() *Report { r := Report{State: p.state.String()} - if p.err != nil { - r.Error = p.err.Error() - } + if p.err != nil { + r.Error = p.err.Error() + } if p.execQueue != nil { r.Pending, r.Completed = p.execQueue.Report() @@ -268,7 +257,7 @@ func (p *Pruner) State() State { } type fs struct { - path string + path string // permanent error during planning planErr error @@ -316,7 +305,7 @@ func (f *fs) Report() FSReport { if f.planErr != nil { r.LastError = f.planErr.Error() - } else if f.execErrLast != nil { + } else if f.execErrLast != nil { r.LastError = f.execErrLast.Error() } @@ -326,7 +315,7 @@ func (f *fs) Report() FSReport { } r.DestroyList = make([]SnapshotReport, len(f.destroyList)) - for i, snap := range f.destroyList{ + for i, snap := range f.destroyList { r.DestroyList[i] = snap.(snapshot).Report() } @@ -490,9 +479,9 @@ tfss_loop: }) for { - var pfs *fs + var pfs *fs u(func(pruner *Pruner) { - pfs = pruner.execQueue.Pop() + pfs = pruner.execQueue.Pop() }) if pfs == nil { break @@ -516,16 +505,15 @@ tfss_loop: hadErr := false for _, fsr := range rep.Completed { hadErr = hadErr || fsr.SkipReason.NotSkipped() && fsr.LastError != "" - } + } if hadErr { p.state = ExecErr } else { p.state = Done } }) - - } +} // attempts to exec pfs, puts it back into the queue with the result func doOneAttemptExec(a *args, u updater, pfs *fs) { @@ -558,20 +546,20 @@ func doOneAttemptExec(a *args, u updater, pfs *fs) { err = nil destroyFails := make([]*pdu.DestroySnapshotRes, 0) for _, reqDestroy := range destroyList { - res, ok := destroyResults[reqDestroy.Name] - if !ok { - err = fmt.Errorf("missing destroy-result for %s", reqDestroy.RelName()) - break - } else if res.Error != "" { - destroyFails = append(destroyFails, res) - } + res, ok := destroyResults[reqDestroy.Name] + if !ok { + err = fmt.Errorf("missing destroy-result for %s", reqDestroy.RelName()) + break + } else if res.Error != "" { + destroyFails = append(destroyFails, res) + } } if err == nil && len(destroyFails) > 0 { names := make([]string, len(destroyFails)) pairs := make([]string, len(destroyFails)) allSame := true lastMsg := destroyFails[0].Error - for i := 0; i < len(destroyFails); i++{ + for i := 0; i < len(destroyFails); i++ { allSame = allSame && destroyFails[i].Error == lastMsg relname := destroyFails[i].Snapshot.RelName() names[i] = relname diff --git a/daemon/pruner/pruner_queue.go b/daemon/pruner/pruner_queue.go index 840e93b..a824344 100644 --- a/daemon/pruner/pruner_queue.go +++ b/daemon/pruner/pruner_queue.go @@ -7,13 +7,13 @@ import ( ) type execQueue struct { - mtx sync.Mutex + mtx sync.Mutex pending, completed []*fs } func newExecQueue(cap int) *execQueue { q := execQueue{ - pending: make([]*fs, 0, cap), + pending: make([]*fs, 0, cap), completed: make([]*fs, 0, cap), } return &q @@ -55,7 +55,7 @@ func (q *execQueue) Pop() *fs { return fs } -func(q *execQueue) Put(fs *fs, err error, done bool) { +func (q *execQueue) Put(fs *fs, err error, done bool) { fs.mtx.Lock() fs.execErrLast = err if done || err != nil { @@ -79,5 +79,4 @@ func(q *execQueue) Put(fs *fs, err error, done bool) { }) q.mtx.Unlock() - -} \ No newline at end of file +} diff --git a/daemon/snapper/snapper.go b/daemon/snapper/snapper.go index 31e7b3c..a6dc268 100644 --- a/daemon/snapper/snapper.go +++ b/daemon/snapper/snapper.go @@ -1,18 +1,19 @@ package snapper import ( - "github.com/zrepl/zrepl/config" - "github.com/pkg/errors" - "time" "context" - "github.com/zrepl/zrepl/daemon/filters" "fmt" - "github.com/zrepl/zrepl/zfs" "sort" - "github.com/zrepl/zrepl/logger" "sync" -) + "time" + "github.com/pkg/errors" + + "github.com/zrepl/zrepl/config" + "github.com/zrepl/zrepl/daemon/filters" + "github.com/zrepl/zrepl/logger" + "github.com/zrepl/zrepl/zfs" +) //go:generate stringer -type=SnapState type SnapState uint @@ -28,7 +29,7 @@ type snapProgress struct { state SnapState // SnapStarted, SnapDone, SnapError - name string + name string startAt time.Time // SnapDone @@ -44,13 +45,13 @@ type args struct { prefix string interval time.Duration fsf *filters.DatasetMapFilter - snapshotsTaken chan<-struct{} + snapshotsTaken chan<- struct{} } type Snapper struct { args args - mtx sync.Mutex + mtx sync.Mutex state State // set in state Plan, used in Waiting @@ -70,7 +71,7 @@ type Snapper struct { type State uint const ( - SyncUp State = 1<&2 exit 1 fi diff --git a/logger/datastructures.go b/logger/datastructures.go index 782c8e9..474f6f7 100644 --- a/logger/datastructures.go +++ b/logger/datastructures.go @@ -4,10 +4,11 @@ import ( "bytes" "encoding/json" "fmt" - "github.com/fatih/color" - "github.com/pkg/errors" "sync" "time" + + "github.com/fatih/color" + "github.com/pkg/errors" ) type Level int @@ -66,7 +67,7 @@ func (l Level) Short() string { case Error: return "ERRO" default: - return fmt.Sprintf("%s", l) + return l.String() } } @@ -81,7 +82,7 @@ func (l Level) String() string { case Error: return "error" default: - return fmt.Sprintf("%s", string(l)) + return string(l) } } diff --git a/logger/logger.go b/logger/logger.go index b007267..a2bd647 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -66,7 +66,8 @@ func (l *loggerImpl) logInternalError(outlet Outlet, err string) { time.Now(), fields, } - l.outlets.GetLoggerErrorOutlet().WriteEntry(entry) + // ignore errors at this point (still better than panicking if the error is temporary) + _ = l.outlets.GetLoggerErrorOutlet().WriteEntry(entry) } func (l *loggerImpl) log(level Level, msg string) { diff --git a/logger/logger_test.go b/logger/logger_test.go index 62aed4d..a069244 100644 --- a/logger/logger_test.go +++ b/logger/logger_test.go @@ -2,10 +2,12 @@ package logger_test import ( "fmt" - "github.com/kr/pretty" - "github.com/zrepl/zrepl/logger" "testing" "time" + + "github.com/kr/pretty" + + "github.com/zrepl/zrepl/logger" ) type TestOutlet struct { diff --git a/logger/stderrlogger.go b/logger/stderrlogger.go index 6ca0aad..fbb234b 100644 --- a/logger/stderrlogger.go +++ b/logger/stderrlogger.go @@ -5,11 +5,7 @@ import ( "os" ) -type stderrLogger struct { - Logger -} - -type stderrLoggerOutlet struct {} +type stderrLoggerOutlet struct{} func (stderrLoggerOutlet) WriteEntry(entry Entry) error { fmt.Fprintf(os.Stderr, "%#v\n", entry) diff --git a/pruning/keep_grid.go b/pruning/keep_grid.go index 24da21a..05d61d7 100644 --- a/pruning/keep_grid.go +++ b/pruning/keep_grid.go @@ -2,12 +2,14 @@ package pruning import ( "fmt" - "github.com/pkg/errors" - "github.com/zrepl/zrepl/config" - "github.com/zrepl/zrepl/pruning/retentiongrid" "regexp" "sort" "time" + + "github.com/pkg/errors" + + "github.com/zrepl/zrepl/config" + "github.com/zrepl/zrepl/pruning/retentiongrid" ) // KeepGrid fits snapshots that match a given regex into a retentiongrid.Grid, @@ -15,7 +17,7 @@ import ( // and deletes all snapshots that do not fit the grid specification. type KeepGrid struct { retentionGrid *retentiongrid.Grid - re *regexp.Regexp + re *regexp.Regexp } func NewKeepGrid(in *config.PruneGrid) (p *KeepGrid, err error) { diff --git a/pruning/keep_helpers_test.go b/pruning/keep_helpers_test.go index bf8b399..d70f58d 100644 --- a/pruning/keep_helpers_test.go +++ b/pruning/keep_helpers_test.go @@ -1,8 +1,9 @@ package pruning import ( - "github.com/stretchr/testify/assert" "testing" + + "github.com/stretchr/testify/assert" ) func TestShallowCopySnapList(t *testing.T) { diff --git a/pruning/keep_last_n.go b/pruning/keep_last_n.go index a275154..487eaff 100644 --- a/pruning/keep_last_n.go +++ b/pruning/keep_last_n.go @@ -1,8 +1,9 @@ package pruning import ( - "github.com/pkg/errors" "sort" + + "github.com/pkg/errors" ) type KeepLastN struct { diff --git a/pruning/keep_last_n_test.go b/pruning/keep_last_n_test.go index 8ef0000..c8ba00f 100644 --- a/pruning/keep_last_n_test.go +++ b/pruning/keep_last_n_test.go @@ -1,9 +1,10 @@ package pruning import ( - "github.com/stretchr/testify/assert" "testing" "time" + + "github.com/stretchr/testify/assert" ) func TestKeepLastN(t *testing.T) { diff --git a/pruning/keep_not_replicated.go b/pruning/keep_not_replicated.go index 955d9e6..e84a5c2 100644 --- a/pruning/keep_not_replicated.go +++ b/pruning/keep_not_replicated.go @@ -1,8 +1,6 @@ package pruning -type KeepNotReplicated struct { - forceConstructor struct{} -} +type KeepNotReplicated struct{} func (*KeepNotReplicated) KeepRule(snaps []Snapshot) (destroyList []Snapshot) { return filterSnapList(snaps, func(snapshot Snapshot) bool { diff --git a/pruning/keep_regex.go b/pruning/keep_regex.go index bf45864..7618075 100644 --- a/pruning/keep_regex.go +++ b/pruning/keep_regex.go @@ -5,7 +5,7 @@ import ( ) type KeepRegex struct { - expr *regexp.Regexp + expr *regexp.Regexp negate bool } diff --git a/pruning/pruning.go b/pruning/pruning.go index 5f05ede..9f36581 100644 --- a/pruning/pruning.go +++ b/pruning/pruning.go @@ -2,9 +2,11 @@ package pruning import ( "fmt" - "github.com/pkg/errors" - "github.com/zrepl/zrepl/config" "time" + + "github.com/pkg/errors" + + "github.com/zrepl/zrepl/config" ) type KeepRule interface { @@ -20,7 +22,7 @@ type Snapshot interface { // The returned snapshot list is guaranteed to only contains elements of input parameter snaps func PruneSnapshots(snaps []Snapshot, keepRules []KeepRule) []Snapshot { - if keepRules == nil || len(keepRules) == 0 { + if len(keepRules) == 0 { return []Snapshot{} } diff --git a/pruning/retentiongrid/retentiongrid_test.go b/pruning/retentiongrid/retentiongrid_test.go index 3e5e52f..c487f13 100644 --- a/pruning/retentiongrid/retentiongrid_test.go +++ b/pruning/retentiongrid/retentiongrid_test.go @@ -2,11 +2,12 @@ package retentiongrid import ( "fmt" - "github.com/stretchr/testify/assert" "strconv" "strings" "testing" "time" + + "github.com/stretchr/testify/assert" ) type retentionIntervalStub struct { diff --git a/replication/driver/errorclass_enumer.go b/replication/driver/errorclass_enumer.go index 0a56c0e..d85c4b4 100644 --- a/replication/driver/errorclass_enumer.go +++ b/replication/driver/errorclass_enumer.go @@ -6,9 +6,9 @@ import ( "fmt" ) -const _errorClassName = "errorClassUnknownerrorClassPermanenterrorClassTemporaryConnectivityRelated" +const _errorClassName = "errorClassPermanenterrorClassTemporaryConnectivityRelated" -var _errorClassIndex = [...]uint8{0, 17, 36, 74} +var _errorClassIndex = [...]uint8{0, 19, 57} func (i errorClass) String() string { if i < 0 || i >= errorClass(len(_errorClassIndex)-1) { @@ -17,12 +17,11 @@ func (i errorClass) String() string { return _errorClassName[_errorClassIndex[i]:_errorClassIndex[i+1]] } -var _errorClassValues = []errorClass{0, 1, 2} +var _errorClassValues = []errorClass{0, 1} var _errorClassNameToValueMap = map[string]errorClass{ - _errorClassName[0:17]: 0, - _errorClassName[17:36]: 1, - _errorClassName[36:74]: 2, + _errorClassName[0:19]: 0, + _errorClassName[19:57]: 1, } // errorClassString retrieves an enum value from the enum constants string name. diff --git a/replication/driver/replication_driver.go b/replication/driver/replication_driver.go index 6a5a9ea..0c1cd8f 100644 --- a/replication/driver/replication_driver.go +++ b/replication/driver/replication_driver.go @@ -10,11 +10,12 @@ import ( "sync" "time" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "github.com/zrepl/zrepl/replication/report" "github.com/zrepl/zrepl/util/chainlock" "github.com/zrepl/zrepl/util/envconst" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" ) type interval struct { @@ -338,7 +339,7 @@ func (a *attempt) do(ctx context.Context, prev *attempt) { l := fmt.Sprintf(" %s => %v", i.cur.fs.ReportInfo().Name, prevNames) inconsistencyLines = append(inconsistencyLines, l) } - fmt.Fprintf(&msg, strings.Join(inconsistencyLines, "\n")) + fmt.Fprint(&msg, strings.Join(inconsistencyLines, "\n")) now := time.Now() a.planErr = newTimedError(errors.New(msg.String()), now) a.fss = nil @@ -551,17 +552,11 @@ func (s *step) report() *report.StepReport { return r } -type stepErrorReport struct { - err *timedError - step int -} - //go:generate enumer -type=errorClass type errorClass int const ( - errorClassUnknown errorClass = iota - errorClassPermanent + errorClassPermanent errorClass = iota errorClassTemporaryConnectivityRelated ) diff --git a/replication/driver/replication_driver_debug.go b/replication/driver/replication_driver_debug.go index 23f7ae7..6b96f09 100644 --- a/replication/driver/replication_driver_debug.go +++ b/replication/driver/replication_driver_debug.go @@ -13,6 +13,7 @@ func init() { } } +//nolint[:deadcode,unused] func debug(format string, args ...interface{}) { if debugEnabled { fmt.Fprintf(os.Stderr, "repl: driver: %s\n", fmt.Sprintf(format, args...)) @@ -21,9 +22,10 @@ func debug(format string, args ...interface{}) { type debugFunc func(format string, args ...interface{}) +//nolint[:deadcode,unused] func debugPrefix(prefixFormat string, prefixFormatArgs ...interface{}) debugFunc { prefix := fmt.Sprintf(prefixFormat, prefixFormatArgs...) return func(format string, args ...interface{}) { debug("%s: %s", prefix, fmt.Sprintf(format, args)) } -} \ No newline at end of file +} diff --git a/replication/driver/replication_driver_test.go b/replication/driver/replication_driver_test.go index 650434f..abf39f1 100644 --- a/replication/driver/replication_driver_test.go +++ b/replication/driver/replication_driver_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/stretchr/testify/require" + "github.com/zrepl/zrepl/replication/report" "github.com/stretchr/testify/assert" @@ -165,14 +166,14 @@ func TestReplication(t *testing.T) { reports := make([]*report.Report, len(fireAt)) for i := range fireAt { sleepUntil := begin.Add(fireAt[i]) - time.Sleep(sleepUntil.Sub(time.Now())) + time.Sleep(time.Until(sleepUntil)) reports[i] = getReport() // uncomment for viewing non-diffed results // t.Logf("report @ %6.4f:\n%s", fireAt[i].Seconds(), pretty.Sprint(reports[i])) } waitBegin := time.Now() wait(true) - waitDuration := time.Now().Sub(waitBegin) + waitDuration := time.Since(waitBegin) assert.True(t, waitDuration < 10*time.Millisecond, "%v", waitDuration) // and that's gratious prev, err := json.Marshal(reports[0]) diff --git a/replication/driver/replication_stepqueue_test.go b/replication/driver/replication_stepqueue_test.go index fc0f316..73f877a 100644 --- a/replication/driver/replication_stepqueue_test.go +++ b/replication/driver/replication_stepqueue_test.go @@ -96,7 +96,7 @@ func TestPqConcurrent(t *testing.T) { pos := atomic.AddUint32(&globalCtr, 1) t := time.Unix(int64(step), 0) done := q.WaitReady(fs, t) - wakeAt := time.Now().Sub(begin) + wakeAt := time.Since(begin) time.Sleep(sleepTimePerStep) done() recs = append(recs, record{fs, step, pos, wakeAt}) diff --git a/replication/logic/diff/diff_test.go b/replication/logic/diff/diff_test.go index 46200b3..931a06e 100644 --- a/replication/logic/diff/diff_test.go +++ b/replication/logic/diff/diff_test.go @@ -97,7 +97,7 @@ func TestIncrementalPath_SnapshotsOnly(t *testing.T) { }) // sender with earlier but also current version as sender is not a conflict - doTest(l("@c,3"), l("@a,1", "@b,2", "@c,3") , func(path []*FilesystemVersion, conflict error) { + doTest(l("@c,3"), l("@a,1", "@b,2", "@c,3"), func(path []*FilesystemVersion, conflict error) { t.Logf("path: %#v", path) t.Logf("conflict: %#v", conflict) assert.Empty(t, path) diff --git a/replication/logic/pdu/pdu_extras.go b/replication/logic/pdu/pdu_extras.go index 9cf52c9..c27356c 100644 --- a/replication/logic/pdu/pdu_extras.go +++ b/replication/logic/pdu/pdu_extras.go @@ -2,8 +2,9 @@ package pdu import ( "fmt" - "github.com/zrepl/zrepl/zfs" "time" + + "github.com/zrepl/zrepl/zfs" ) func (v *FilesystemVersion) RelName() string { diff --git a/replication/logic/pdu/pdu_test.go b/replication/logic/pdu/pdu_test.go index 79315a6..5abafad 100644 --- a/replication/logic/pdu/pdu_test.go +++ b/replication/logic/pdu/pdu_test.go @@ -1,9 +1,10 @@ package pdu import ( - "github.com/stretchr/testify/assert" "testing" "time" + + "github.com/stretchr/testify/assert" ) func TestFilesystemVersion_RelName(t *testing.T) { @@ -18,24 +19,24 @@ func TestFilesystemVersion_RelName(t *testing.T) { tcs := []TestCase{ { In: FilesystemVersion{ - Type: FilesystemVersion_Snapshot, - Name: "foobar", + Type: FilesystemVersion_Snapshot, + Name: "foobar", Creation: creat, }, Out: "@foobar", }, { In: FilesystemVersion{ - Type: FilesystemVersion_Bookmark, - Name: "foobar", + Type: FilesystemVersion_Bookmark, + Name: "foobar", Creation: creat, }, Out: "#foobar", }, { In: FilesystemVersion{ - Type: 2342, - Name: "foobar", + Type: 2342, + Name: "foobar", Creation: creat, }, Panic: true, @@ -58,7 +59,7 @@ func TestFilesystemVersion_RelName(t *testing.T) { func TestFilesystemVersion_ZFSFilesystemVersion(t *testing.T) { empty := &FilesystemVersion{} - _, err:= empty.ZFSFilesystemVersion() + _, err := empty.ZFSFilesystemVersion() assert.Error(t, err) dateInvalid := &FilesystemVersion{Creation: "foobar"} diff --git a/replication/logic/replication_logic.go b/replication/logic/replication_logic.go index 906b610..ed5235c 100644 --- a/replication/logic/replication_logic.go +++ b/replication/logic/replication_logic.go @@ -26,7 +26,7 @@ type Endpoint interface { ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error) ListFilesystemVersions(ctx context.Context, req *pdu.ListFilesystemVersionsReq) (*pdu.ListFilesystemVersionsRes, error) DestroySnapshots(ctx context.Context, req *pdu.DestroySnapshotsReq) (*pdu.DestroySnapshotsRes, error) - WaitForConnectivity(ctx context.Context) (error) + WaitForConnectivity(ctx context.Context) error } type Sender interface { @@ -107,7 +107,7 @@ type Filesystem struct { sender Sender receiver Receiver - Path string // compat + Path string // compat receiverFS *pdu.Filesystem promBytesReplicated prometheus.Counter // compat } diff --git a/rpc/dataconn/dataconn_client.go b/rpc/dataconn/dataconn_client.go index 8473d97..a722d79 100644 --- a/rpc/dataconn/dataconn_client.go +++ b/rpc/dataconn/dataconn_client.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/golang/protobuf/proto" + "github.com/zrepl/zrepl/replication/logic/pdu" "github.com/zrepl/zrepl/rpc/dataconn/stream" "github.com/zrepl/zrepl/transport" @@ -214,13 +215,12 @@ func (c *Client) ReqRecv(ctx context.Context, req *pdu.ReceiveReq, streamCopier return res.res, cause } - func (c *Client) ReqPing(ctx context.Context, req *pdu.PingReq) (*pdu.PingRes, error) { conn, err := c.getWire(ctx) if err != nil { return nil, err } - defer c.putWire(conn) + defer c.putWire(conn) if err := c.send(ctx, conn, EndpointPing, req, nil); err != nil { return nil, err @@ -232,4 +232,4 @@ func (c *Client) ReqPing(ctx context.Context, req *pdu.PingReq) (*pdu.PingRes, e } return &res, nil -} \ No newline at end of file +} diff --git a/rpc/dataconn/dataconn_debug.go b/rpc/dataconn/dataconn_debug.go index 3c20701..958ac80 100644 --- a/rpc/dataconn/dataconn_debug.go +++ b/rpc/dataconn/dataconn_debug.go @@ -13,6 +13,7 @@ func init() { } } +//nolint[:deadcode,unused] func debug(format string, args ...interface{}) { if debugEnabled { fmt.Fprintf(os.Stderr, "rpc/dataconn: %s\n", fmt.Sprintf(format, args...)) diff --git a/rpc/dataconn/dataconn_server.go b/rpc/dataconn/dataconn_server.go index ea6700b..fbd1a95 100644 --- a/rpc/dataconn/dataconn_server.go +++ b/rpc/dataconn/dataconn_server.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/golang/protobuf/proto" + "github.com/zrepl/zrepl/logger" "github.com/zrepl/zrepl/replication/logic/pdu" "github.com/zrepl/zrepl/rpc/dataconn/stream" @@ -137,7 +138,6 @@ func (s *Server) serveConn(nc *transport.AuthConn) { default: s.log.WithField("endpoint", endpoint).Error("unknown endpoint") handlerErr = fmt.Errorf("requested endpoint does not exist") - return } s.log.WithField("endpoint", endpoint).WithField("errType", fmt.Sprintf("%T", handlerErr)).Debug("handler returned") @@ -187,6 +187,4 @@ func (s *Server) serveConn(nc *transport.AuthConn) { s.log.WithError(err).Error("cannot write send stream") } } - - return } diff --git a/rpc/dataconn/frameconn/frameconn.go b/rpc/dataconn/frameconn/frameconn.go index 2736b98..321e6cb 100644 --- a/rpc/dataconn/frameconn/frameconn.go +++ b/rpc/dataconn/frameconn/frameconn.go @@ -1,7 +1,6 @@ package frameconn import ( - "bufio" "encoding/binary" "errors" "fmt" @@ -12,6 +11,7 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" + "github.com/zrepl/zrepl/rpc/dataconn/base2bufpool" "github.com/zrepl/zrepl/rpc/dataconn/timeoutconn" ) @@ -47,7 +47,6 @@ func (f *FrameHeader) Unmarshal(buf []byte) { type Conn struct { readMtx, writeMtx sync.Mutex nc timeoutconn.Conn - ncBuf *bufio.ReadWriter readNextValid bool readNext FrameHeader nextReadErr error diff --git a/rpc/dataconn/frameconn/frameconn_shutdown_fsm.go b/rpc/dataconn/frameconn/frameconn_shutdown_fsm.go index 980d980..f727337 100644 --- a/rpc/dataconn/frameconn/frameconn_shutdown_fsm.go +++ b/rpc/dataconn/frameconn/frameconn_shutdown_fsm.go @@ -3,24 +3,18 @@ package frameconn import "sync" type shutdownFSM struct { - mtx sync.Mutex - state shutdownFSMState + mtx sync.Mutex + state shutdownFSMState } type shutdownFSMState uint32 const ( + // zero value is important shutdownStateOpen shutdownFSMState = iota shutdownStateBegin ) -func newShutdownFSM() *shutdownFSM { - fsm := &shutdownFSM{ - state: shutdownStateOpen, - } - return fsm -} - func (f *shutdownFSM) Begin() (thisCallStartedShutdown bool) { f.mtx.Lock() defer f.mtx.Unlock() @@ -34,4 +28,3 @@ func (f *shutdownFSM) IsShuttingDown() bool { defer f.mtx.Unlock() return f.state != shutdownStateOpen } - diff --git a/rpc/dataconn/frameconn/frameconn_test.go b/rpc/dataconn/frameconn/frameconn_test.go index b070c79..d7ac057 100644 --- a/rpc/dataconn/frameconn/frameconn_test.go +++ b/rpc/dataconn/frameconn/frameconn_test.go @@ -19,4 +19,3 @@ func TestIsPublicFrameType(t *testing.T) { assert.True(t, IsPublicFrameType(255)) assert.False(t, IsPublicFrameType(rstFrameType)) } - diff --git a/rpc/dataconn/heartbeatconn/heartbeatconn.go b/rpc/dataconn/heartbeatconn/heartbeatconn.go index 2924fdc..c4a6e7a 100644 --- a/rpc/dataconn/heartbeatconn/heartbeatconn.go +++ b/rpc/dataconn/heartbeatconn/heartbeatconn.go @@ -11,9 +11,7 @@ import ( ) type Conn struct { - state state - // if not nil, opErr is returned for ReadFrame and WriteFrame (not for Close, though) - opErr atomic.Value // error + state state fc *frameconn.Conn sendInterval, timeout time.Duration stopSend chan struct{} @@ -97,7 +95,10 @@ func (c *Conn) sendHeartbeats() { debug("send heartbeat") // if the connection is in zombie mode (aka iptables DROP inbetween peers) // this call or one of its successors will block after filling up the kernel tx buffer - c.fc.WriteFrame([]byte{}, heartbeat) + err := c.fc.WriteFrame([]byte{}, heartbeat) + if err != nil { + debug("send heartbeat error: %s", err) + } // ignore errors from WriteFrame to rate-limit SendHeartbeat retries c.lastFrameSent.Store(time.Now()) }() diff --git a/rpc/dataconn/heartbeatconn/heartbeatconn_debug.go b/rpc/dataconn/heartbeatconn/heartbeatconn_debug.go index 6bdea8d..caa8884 100644 --- a/rpc/dataconn/heartbeatconn/heartbeatconn_debug.go +++ b/rpc/dataconn/heartbeatconn/heartbeatconn_debug.go @@ -13,6 +13,7 @@ func init() { } } +//nolint[:deadcode,unused] func debug(format string, args ...interface{}) { if debugEnabled { fmt.Fprintf(os.Stderr, "rpc/dataconn/heartbeatconn: %s\n", fmt.Sprintf(format, args...)) diff --git a/rpc/dataconn/heartbeatconn/heartbeatconn_test.go b/rpc/dataconn/heartbeatconn/heartbeatconn_test.go index 8b02459..897c1b9 100644 --- a/rpc/dataconn/heartbeatconn/heartbeatconn_test.go +++ b/rpc/dataconn/heartbeatconn/heartbeatconn_test.go @@ -5,6 +5,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/zrepl/zrepl/rpc/dataconn/frameconn" ) diff --git a/rpc/dataconn/stream/stream.go b/rpc/dataconn/stream/stream.go index 10e7ff0..4d3e900 100644 --- a/rpc/dataconn/stream/stream.go +++ b/rpc/dataconn/stream/stream.go @@ -28,6 +28,7 @@ func WithLogger(ctx context.Context, log Logger) context.Context { return context.WithValue(ctx, contextKeyLogger, log) } +//nolint[:deadcode,unused] func getLog(ctx context.Context) Logger { log, ok := ctx.Value(contextKeyLogger).(Logger) if !ok { diff --git a/rpc/dataconn/stream/stream_conn.go b/rpc/dataconn/stream/stream_conn.go index ce0c052..7b1b081 100644 --- a/rpc/dataconn/stream/stream_conn.go +++ b/rpc/dataconn/stream/stream_conn.go @@ -23,9 +23,8 @@ type Conn struct { // readMtx serializes read stream operations because we inherently only // support a single stream at a time over hc. - readMtx sync.Mutex - readClean bool - allowWriteStreamTo bool + readMtx sync.Mutex + readClean bool // writeMtx serializes write stream operations because we inherently only // support a single stream at a time over hc. @@ -95,7 +94,7 @@ func (c *Conn) ReadStreamedMessage(ctx context.Context, maxSize uint32, frameTyp }() err := readStream(c.frameReads, c.hc, w, frameType) c.readClean = isConnCleanAfterRead(err) - w.CloseWithError(readMessageSentinel) + _ = w.CloseWithError(readMessageSentinel) // always returns nil wg.Wait() if err != nil { return nil, err @@ -166,7 +165,7 @@ func (c *Conn) SendStream(ctx context.Context, src zfs.StreamCopier, frameType u var res writeStreamRes res.errStream, res.errConn = writeStream(ctx, c.hc, r, frameType) if w != nil { - w.CloseWithError(res.errStream) + _ = w.CloseWithError(res.errStream) // always returns nil } writeStreamErrChan <- res }() diff --git a/rpc/dataconn/stream/stream_debug.go b/rpc/dataconn/stream/stream_debug.go index c1e2ef9..e86f41a 100644 --- a/rpc/dataconn/stream/stream_debug.go +++ b/rpc/dataconn/stream/stream_debug.go @@ -13,6 +13,7 @@ func init() { } } +//nolint[:deadcode,unused] func debug(format string, args ...interface{}) { if debugEnabled { fmt.Fprintf(os.Stderr, "rpc/dataconn/stream: %s\n", fmt.Sprintf(format, args...)) diff --git a/rpc/dataconn/stream/stream_test.go b/rpc/dataconn/stream/stream_test.go index 7da4cfa..d1dabcc 100644 --- a/rpc/dataconn/stream/stream_test.go +++ b/rpc/dataconn/stream/stream_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/zrepl/zrepl/logger" "github.com/zrepl/zrepl/rpc/dataconn/heartbeatconn" "github.com/zrepl/zrepl/util/socketpair" diff --git a/rpc/dataconn/timeoutconn/internal/wireevaluator/wireevaluator.go b/rpc/dataconn/timeoutconn/internal/wireevaluator/wireevaluator.go index 65ea9cd..e57c311 100644 --- a/rpc/dataconn/timeoutconn/internal/wireevaluator/wireevaluator.go +++ b/rpc/dataconn/timeoutconn/internal/wireevaluator/wireevaluator.go @@ -11,6 +11,7 @@ import ( netssh "github.com/problame/go-netssh" "github.com/zrepl/yaml-config" + "github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/transport" transportconfig "github.com/zrepl/zrepl/transport/fromconfig" diff --git a/rpc/dataconn/timeoutconn/internal/wireevaluator/wireevaluator_closewrite.go b/rpc/dataconn/timeoutconn/internal/wireevaluator/wireevaluator_closewrite.go index cc1907e..e2dddbd 100644 --- a/rpc/dataconn/timeoutconn/internal/wireevaluator/wireevaluator_closewrite.go +++ b/rpc/dataconn/timeoutconn/internal/wireevaluator/wireevaluator_closewrite.go @@ -52,9 +52,6 @@ func (CloseWrite) sender(wire transport.Wire) { log.Printf("closeErr=%T %s", closeErr, closeErr) }() - type opResult struct { - err error - } writeDone := make(chan struct{}, 1) go func() { close(writeDone) @@ -85,7 +82,7 @@ func (CloseWrite) receiver(wire transport.Wire) { // consume half the test data, then detect an error, send it and CloseWrite - r := io.LimitReader(wire, int64(5 * len(closeWriteTestSendData)/3)) + r := io.LimitReader(wire, int64(5*len(closeWriteTestSendData)/3)) _, err := io.Copy(ioutil.Discard, r) noerror(err) @@ -103,7 +100,7 @@ func (CloseWrite) receiver(wire transport.Wire) { // io.Copy masks io.EOF to nil, and we expect io.EOF from the client's Close() call log.Panicf("unexpected error returned from reading conn: %s", err) } - + closeErr := wire.Close() log.Printf("closeErr=%T %s", closeErr, closeErr) diff --git a/rpc/dataconn/timeoutconn/timeoutconn.go b/rpc/dataconn/timeoutconn/timeoutconn.go index 9e0a3bf..e0d41c7 100644 --- a/rpc/dataconn/timeoutconn/timeoutconn.go +++ b/rpc/dataconn/timeoutconn/timeoutconn.go @@ -95,7 +95,7 @@ restart: return n, err } var nCurRead int - nCurRead, err = c.Wire.Read(p[n:len(p)]) + nCurRead, err = c.Wire.Read(p[n:]) n += nCurRead if netErr, ok := err.(net.Error); ok && netErr.Timeout() && nCurRead > 0 { err = nil @@ -111,7 +111,7 @@ restart: return n, err } var nCurWrite int - nCurWrite, err = c.Wire.Write(p[n:len(p)]) + nCurWrite, err = c.Wire.Write(p[n:]) n += nCurWrite if netErr, ok := err.(net.Error); ok && netErr.Timeout() && nCurWrite > 0 { err = nil diff --git a/rpc/dataconn/timeoutconn/timeoutconn_test.go b/rpc/dataconn/timeoutconn/timeoutconn_test.go index 708bba2..b4cce04 100644 --- a/rpc/dataconn/timeoutconn/timeoutconn_test.go +++ b/rpc/dataconn/timeoutconn/timeoutconn_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/zrepl/zrepl/util/socketpair" ) @@ -101,7 +102,7 @@ func TestNoPartialReadsDueToDeadline(t *testing.T) { // io.Copy will encounter a partial read, then wait ~50ms until the other 5 bytes are written // It is still going to fail with deadline err because it expects EOF n, err := io.Copy(&buf, bc) - readDuration := time.Now().Sub(beginRead) + readDuration := time.Since(beginRead) t.Logf("read duration=%s", readDuration) t.Logf("recv done n=%v err=%v", n, err) t.Logf("buf=%v", buf.Bytes()) @@ -152,7 +153,7 @@ func TestPartialWriteMockConn(t *testing.T) { buf := []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10} begin := time.Now() n, err := mc.Write(buf[:]) - duration := time.Now().Sub(begin) + duration := time.Since(begin) assert.NoError(t, err) assert.Equal(t, 5, n) assert.True(t, duration > 100*time.Millisecond) diff --git a/rpc/grpcclientidentity/authlistener_grpc_adaptor.go b/rpc/grpcclientidentity/authlistener_grpc_adaptor.go index 3c62cf4..267516c 100644 --- a/rpc/grpcclientidentity/authlistener_grpc_adaptor.go +++ b/rpc/grpcclientidentity/authlistener_grpc_adaptor.go @@ -18,11 +18,12 @@ import ( "net" "time" - "github.com/zrepl/zrepl/logger" - "github.com/zrepl/zrepl/transport" "google.golang.org/grpc" "google.golang.org/grpc/credentials" "google.golang.org/grpc/peer" + + "github.com/zrepl/zrepl/logger" + "github.com/zrepl/zrepl/transport" ) type Logger = logger.Logger @@ -105,7 +106,7 @@ func NewInterceptors(logger Logger, clientIdentityKey interface{}) (unary grpc.U if !ok { panic("peer.FromContext expected to return a peer in grpc.UnaryServerInterceptor") } - logger.WithField("peer_addr", fmt.Sprintf("%s", p.Addr)).Debug("peer addr") + logger.WithField("peer_addr", p.Addr.String()).Debug("peer addr") a, ok := p.AuthInfo.(*authConnAuthType) if !ok { panic(fmt.Sprintf("NewInterceptors must be used in combination with grpc.NewTransportCredentials, but got auth type %T", p.AuthInfo)) diff --git a/rpc/grpcclientidentity/example/main.go b/rpc/grpcclientidentity/example/main.go index 3813683..e69fa6b 100644 --- a/rpc/grpcclientidentity/example/main.go +++ b/rpc/grpcclientidentity/example/main.go @@ -88,6 +88,9 @@ func server() { log := logger.NewStderrDebugLogger() srv, serve, err := grpchelper.NewServer(authListenerFactory, clientIdentityKey, log) + if err != nil { + onErr(err, "new server") + } svc := &greeter{"hello "} pdu.RegisterGreeterServer(srv, svc) diff --git a/rpc/netadaptor/authlistener_netlistener_adaptor.go b/rpc/netadaptor/authlistener_netlistener_adaptor.go index cc9ae70..05f9611 100644 --- a/rpc/netadaptor/authlistener_netlistener_adaptor.go +++ b/rpc/netadaptor/authlistener_netlistener_adaptor.go @@ -25,8 +25,9 @@ package netadaptor import ( "context" "fmt" - "github.com/zrepl/zrepl/logger" "net" + + "github.com/zrepl/zrepl/logger" "github.com/zrepl/zrepl/transport" ) diff --git a/rpc/rpc_client.go b/rpc/rpc_client.go index 818cd44..d563a66 100644 --- a/rpc/rpc_client.go +++ b/rpc/rpc_client.go @@ -12,6 +12,7 @@ import ( "google.golang.org/grpc" "github.com/google/uuid" + "github.com/zrepl/zrepl/replication/logic" "github.com/zrepl/zrepl/replication/logic/pdu" "github.com/zrepl/zrepl/rpc/dataconn" @@ -158,7 +159,7 @@ func (c *Client) WaitForConnectivity(ctx context.Context) error { time.Sleep(envconst.Duration("ZREPL_RPC_DATACONN_PING_SLEEP", 1*time.Second)) continue } - // it's not a dial timeout, + // it's not a dial timeout, checkRes(data, dataErr, loggers.Data, &dataOk) return } diff --git a/rpc/rpc_debug.go b/rpc/rpc_debug.go index a31e1f2..66f3da3 100644 --- a/rpc/rpc_debug.go +++ b/rpc/rpc_debug.go @@ -13,6 +13,7 @@ func init() { } } +//nolint[:deadcode,unused] func debug(format string, args ...interface{}) { if debugEnabled { fmt.Fprintf(os.Stderr, "rpc: %s\n", fmt.Sprintf(format, args...)) diff --git a/rpc/rpc_doc.go b/rpc/rpc_doc.go index dbe7dda..c8b9400 100644 --- a/rpc/rpc_doc.go +++ b/rpc/rpc_doc.go @@ -115,4 +115,3 @@ package rpc // - remove the comments // // - vim: set virtualedit+=all // - vim: set ft=text - diff --git a/rpc/rpc_logging.go b/rpc/rpc_logging.go index e842f6e..3d52690 100644 --- a/rpc/rpc_logging.go +++ b/rpc/rpc_logging.go @@ -12,9 +12,6 @@ type contextKey int const ( contextKeyLoggers contextKey = iota - contextKeyGeneralLogger - contextKeyControlLogger - contextKeyDataLogger ) /// All fields must be non-nil diff --git a/rpc/rpc_mux.go b/rpc/rpc_mux.go index f70dde7..b3a6f66 100644 --- a/rpc/rpc_mux.go +++ b/rpc/rpc_mux.go @@ -4,8 +4,8 @@ import ( "context" "time" - "github.com/zrepl/zrepl/transport" "github.com/zrepl/zrepl/rpc/transportmux" + "github.com/zrepl/zrepl/transport" "github.com/zrepl/zrepl/util/envconst" ) diff --git a/rpc/rpc_server.go b/rpc/rpc_server.go index f0f0f6b..880c75b 100644 --- a/rpc/rpc_server.go +++ b/rpc/rpc_server.go @@ -34,8 +34,6 @@ type Server struct { dataServerServe serveFunc } -type serverContextKey int - type HandlerContextInterceptor func(ctx context.Context) context.Context // config must be valid (use its Validate function). diff --git a/rpc/transportmux/transportmux.go b/rpc/transportmux/transportmux.go index f78c1e3..90ec48c 100644 --- a/rpc/transportmux/transportmux.go +++ b/rpc/transportmux/transportmux.go @@ -7,6 +7,7 @@ package transportmux import ( "context" + "fmt" "io" "net" @@ -49,10 +50,10 @@ func (l *demuxListener) Accept(ctx context.Context) (*transport.AuthConn, error) return res.conn, res.err } -type demuxAddr struct {} +type demuxAddr struct{} func (demuxAddr) Network() string { return "demux" } -func (demuxAddr) String() string { return "demux" } +func (demuxAddr) String() string { return "demux" } func (l *demuxListener) Addr() net.Addr { return demuxAddr{} @@ -64,7 +65,7 @@ func (l *demuxListener) Close() error { return nil } // TODO // This is a protocol constant, changing it breaks the wire protocol. const LabelLen = 64 -func padLabel(out []byte, label string) (error) { +func padLabel(out []byte, label string) error { if len(label) > LabelLen { return fmt.Errorf("label %q exceeds max length (is %d, max %d)", label, len(label), LabelLen) } @@ -142,7 +143,10 @@ func Demux(ctx context.Context, rawListener transport.AuthenticatedListener, lab continue } - rawConn.SetDeadline(time.Time{}) + err = rawConn.SetDeadline(time.Time{}) + if err != nil { + getLog(ctx).WithError(err).Error("cannot reset deadline") + } // blocking is intentional demuxListener.conns <- acceptRes{conn: rawConn, err: nil} } @@ -153,7 +157,7 @@ func Demux(ctx context.Context, rawListener transport.AuthenticatedListener, lab type labeledConnecter struct { label []byte - transport.Connecter + transport.Connecter } func (c labeledConnecter) Connect(ctx context.Context) (transport.Wire, error) { @@ -169,7 +173,12 @@ func (c labeledConnecter) Connect(ctx context.Context) (transport.Wire, error) { } if dl, ok := ctx.Deadline(); ok { - defer conn.SetDeadline(time.Time{}) + defer func() { + err := conn.SetDeadline(time.Time{}) + if err != nil { + getLog(ctx).WithError(err).Error("cannot reset deadline") + } + }() if err := conn.SetDeadline(dl); err != nil { closeConn(err) return nil, err @@ -202,4 +211,3 @@ func MuxConnecter(rawConnecter transport.Connecter, labels []string, timeout tim } return ret, nil } - diff --git a/rpc/versionhandshake/versionhandshake.go b/rpc/versionhandshake/versionhandshake.go index 03835ee..01f07e3 100644 --- a/rpc/versionhandshake/versionhandshake.go +++ b/rpc/versionhandshake/versionhandshake.go @@ -17,7 +17,7 @@ import ( type HandshakeMessage struct { ProtocolVersion int - Extensions []string + Extensions []string } // A HandshakeError describes what went wrong during the handshake. @@ -25,7 +25,7 @@ type HandshakeMessage struct { type HandshakeError struct { msg string // If not nil, the underlying IO error that caused the handshake to fail. - IOError error + IOError error isAcceptError bool } @@ -36,10 +36,10 @@ func (e HandshakeError) Error() string { return e.msg } // Like with net.OpErr (Go issue 6163), a client failing to handshake // should be a temporary Accept error toward the Listener . func (e HandshakeError) Temporary() bool { - if e.isAcceptError { + if e.isAcceptError { return true } - te, ok := e.IOError.(interface{ Temporary() bool }); + te, ok := e.IOError.(interface{ Temporary() bool }) return ok && te.Temporary() } @@ -52,11 +52,11 @@ func (e HandshakeError) Timeout() bool { return false } -func hsErr(format string, args... interface{}) *HandshakeError { +func hsErr(format string, args ...interface{}) *HandshakeError { return &HandshakeError{msg: fmt.Sprintf(format, args...)} } -func hsIOErr(err error, format string, args... interface{}) *HandshakeError { +func hsIOErr(err error, format string, args ...interface{}) *HandshakeError { return &HandshakeError{IOError: err, msg: fmt.Sprintf(format, args...)} } @@ -145,7 +145,7 @@ func (m *HandshakeMessage) DecodeReader(r io.Reader, maxLen int) error { if exts[len(exts)-1] != "" { return hsErr("unexpected data trailing after last extension newline") } - m.Extensions = exts[0:len(exts)-1] + m.Extensions = exts[0 : len(exts)-1] return nil } @@ -157,18 +157,29 @@ func DoHandshakeCurrentVersion(conn net.Conn, deadline time.Time) *HandshakeErro const HandshakeMessageMaxLen = 16 * 4096 -func DoHandshakeVersion(conn net.Conn, deadline time.Time, version int) *HandshakeError { +func DoHandshakeVersion(conn net.Conn, deadline time.Time, version int) (rErr *HandshakeError) { ours := HandshakeMessage{ ProtocolVersion: version, - Extensions: nil, + Extensions: nil, } hsb, err := ours.Encode() if err != nil { return hsErr("could not encode protocol banner: %s", err) } - defer conn.SetDeadline(time.Time{}) - conn.SetDeadline(deadline) + err = conn.SetDeadline(deadline) + if err != nil { + return hsErr("could not set deadline for protocol banner handshake: %s", err) + } + defer func() { + if rErr != nil { + return + } + err := conn.SetDeadline(time.Time{}) + if err != nil { + rErr = hsErr("could not reset deadline after protocol banner handshake: %s", err) + } + }() _, err = io.Copy(conn, bytes.NewBuffer(hsb)) if err != nil { return hsErr("could not send protocol banner: %s", err) diff --git a/rpc/versionhandshake/versionhandshake_test.go b/rpc/versionhandshake/versionhandshake_test.go index dd27c9d..b45b17f 100644 --- a/rpc/versionhandshake/versionhandshake_test.go +++ b/rpc/versionhandshake/versionhandshake_test.go @@ -3,13 +3,15 @@ package versionhandshake import ( "bytes" "fmt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/zrepl/zrepl/util/socketpair" "io" "strings" "testing" "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/zrepl/zrepl/util/socketpair" ) func TestHandshakeMessage_Encode(t *testing.T) { @@ -23,8 +25,6 @@ func TestHandshakeMessage_Encode(t *testing.T) { enc := string(encB) t.Logf("enc: %s", enc) - - assert.False(t, strings.ContainsAny(enc[0:10], " ")) assert.True(t, enc[10] == ' ') @@ -45,7 +45,7 @@ func TestHandshakeMessage_Encode(t *testing.T) { func TestHandshakeMessage_Encode_InvalidProtocolVersion(t *testing.T) { - for _, pv := range []int{-1, 0, 10000, 10001} { + for _, pv := range []int{-1, 0, 10000, 10001} { t.Logf("testing invalid protocol version = %v", pv) msg := HandshakeMessage{ ProtocolVersion: pv, @@ -68,7 +68,7 @@ func TestHandshakeMessage_DecodeReader(t *testing.T) { require.NoError(t, err) out := HandshakeMessage{} - err = out.DecodeReader(bytes.NewReader([]byte(enc)), 4 * 4096) + err = out.DecodeReader(bytes.NewReader([]byte(enc)), 4*4096) assert.NoError(t, err) assert.Equal(t, 2342, out.ProtocolVersion) assert.Equal(t, 2, len(out.Extensions)) diff --git a/rpc/versionhandshake/versionhandshake_transport_wrappers.go b/rpc/versionhandshake/versionhandshake_transport_wrappers.go index 09ead7a..bc176dd 100644 --- a/rpc/versionhandshake/versionhandshake_transport_wrappers.go +++ b/rpc/versionhandshake/versionhandshake_transport_wrappers.go @@ -4,12 +4,13 @@ import ( "context" "net" "time" + "github.com/zrepl/zrepl/transport" ) type HandshakeConnecter struct { connecter transport.Connecter - timeout time.Duration + timeout time.Duration } func (c HandshakeConnecter) Connect(ctx context.Context) (transport.Wire, error) { @@ -31,17 +32,17 @@ func (c HandshakeConnecter) Connect(ctx context.Context) (transport.Wire, error) func Connecter(connecter transport.Connecter, timeout time.Duration) HandshakeConnecter { return HandshakeConnecter{ connecter: connecter, - timeout: timeout, + timeout: timeout, } } // wrapper type that performs a a protocol version handshake before returning the connection type HandshakeListener struct { - l transport.AuthenticatedListener + l transport.AuthenticatedListener timeout time.Duration } -func (l HandshakeListener) Addr() (net.Addr) { return l.l.Addr() } +func (l HandshakeListener) Addr() net.Addr { return l.l.Addr() } func (l HandshakeListener) Close() error { return l.l.Close() } diff --git a/tlsconf/tlsconf.go b/tlsconf/tlsconf.go index b1cb554..7c8241b 100644 --- a/tlsconf/tlsconf.go +++ b/tlsconf/tlsconf.go @@ -80,7 +80,9 @@ func (l *ClientAuthListener) Accept() (tcpConn *net.TCPConn, tlsConn *tls.Conn, if err = tlsConn.Handshake(); err != nil { goto CloseAndErr } - tlsConn.SetDeadline(time.Time{}) + if err = tlsConn.SetDeadline(time.Time{}); err != nil { + goto CloseAndErr + } peerCerts = tlsConn.ConnectionState().PeerCertificates if len(peerCerts) < 1 { diff --git a/transport/fromconfig/transport_fromconfig.go b/transport/fromconfig/transport_fromconfig.go index 0aa1426..def2450 100644 --- a/transport/fromconfig/transport_fromconfig.go +++ b/transport/fromconfig/transport_fromconfig.go @@ -4,7 +4,9 @@ package fromconfig import ( "fmt" + "github.com/pkg/errors" + "github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/transport" "github.com/zrepl/zrepl/transport/local" @@ -13,10 +15,10 @@ import ( "github.com/zrepl/zrepl/transport/tls" ) -func ListenerFactoryFromConfig(g *config.Global, in config.ServeEnum) (transport.AuthenticatedListenerFactory,error) { +func ListenerFactoryFromConfig(g *config.Global, in config.ServeEnum) (transport.AuthenticatedListenerFactory, error) { var ( - l transport.AuthenticatedListenerFactory + l transport.AuthenticatedListenerFactory err error ) switch v := in.Ret.(type) { @@ -35,7 +37,6 @@ func ListenerFactoryFromConfig(g *config.Global, in config.ServeEnum) (transport return l, err } - func ConnecterFromConfig(g *config.Global, in config.ConnectEnum) (transport.Connecter, error) { var ( connecter transport.Connecter diff --git a/transport/local/connect_local.go b/transport/local/connect_local.go index ba390b8..76102ea 100644 --- a/transport/local/connect_local.go +++ b/transport/local/connect_local.go @@ -3,12 +3,13 @@ package local import ( "context" "fmt" + "github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/transport" ) type LocalConnecter struct { - listenerName string + listenerName string clientIdentity string } @@ -26,4 +27,3 @@ func (c *LocalConnecter) Connect(dialCtx context.Context) (transport.Wire, error l := GetLocalListener(c.listenerName) return l.Connect(dialCtx, c.clientIdentity) } - diff --git a/transport/local/serve_local.go b/transport/local/serve_local.go index f7e42aa..3a7be66 100644 --- a/transport/local/serve_local.go +++ b/transport/local/serve_local.go @@ -3,20 +3,21 @@ package local import ( "context" "fmt" - "github.com/zrepl/zrepl/config" - "github.com/zrepl/zrepl/util/socketpair" "net" "sync" + + "github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/transport" + "github.com/zrepl/zrepl/util/socketpair" ) var localListeners struct { - m map[string]*LocalListener // listenerName -> listener + m map[string]*LocalListener // listenerName -> listener init sync.Once - mtx sync.Mutex + mtx sync.Mutex } -func GetLocalListener(listenerName string) (*LocalListener) { +func GetLocalListener(listenerName string) *LocalListener { localListeners.init.Do(func() { localListeners.m = make(map[string]*LocalListener) @@ -36,12 +37,12 @@ func GetLocalListener(listenerName string) (*LocalListener) { type connectRequest struct { clientIdentity string - callback chan connectResult + callback chan connectResult } type connectResult struct { conn transport.Wire - err error + err error } type LocalListener struct { @@ -60,7 +61,7 @@ func (l *LocalListener) Connect(dialCtx context.Context, clientIdentity string) // place request req := connectRequest{ clientIdentity: clientIdentity, - callback: make(chan connectResult), + callback: make(chan connectResult), } select { case l.connects <- req: @@ -70,7 +71,7 @@ func (l *LocalListener) Connect(dialCtx context.Context, clientIdentity string) // wait for listener response select { - case connRes := <- req.callback: + case connRes := <-req.callback: conn, err = connRes.conn, connRes.err case <-dialCtx.Done(): close(req.callback) // sending to the channel afterwards will panic, the listener has to catch this @@ -88,7 +89,7 @@ func (localAddr) Network() string { return "local" } func (a localAddr) String() string { return a.S } -func (l *LocalListener) Addr() (net.Addr) { return localAddr{""} } +func (l *LocalListener) Addr() net.Addr { return localAddr{""} } func (l *LocalListener) Accept(ctx context.Context) (*transport.AuthConn, error) { respondToRequest := func(req connectRequest, res connectResult) (err error) { @@ -163,12 +164,12 @@ func (l *LocalListener) Close() error { return nil } -func LocalListenerFactoryFromConfig(g *config.Global, in *config.LocalServe) (transport.AuthenticatedListenerFactory,error) { +func LocalListenerFactoryFromConfig(g *config.Global, in *config.LocalServe) (transport.AuthenticatedListenerFactory, error) { if in.ListenerName == "" { return nil, fmt.Errorf("ListenerName must not be empty") } listenerName := in.ListenerName - lf := func() (transport.AuthenticatedListener,error) { + lf := func() (transport.AuthenticatedListener, error) { return GetLocalListener(listenerName), nil } return lf, nil diff --git a/transport/ssh/connect_ssh.go b/transport/ssh/connect_ssh.go index d669b88..7aaea43 100644 --- a/transport/ssh/connect_ssh.go +++ b/transport/ssh/connect_ssh.go @@ -2,12 +2,14 @@ package ssh import ( "context" + "time" + "github.com/jinzhu/copier" "github.com/pkg/errors" "github.com/problame/go-netssh" + "github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/transport" - "time" ) type SSHStdinserverConnecter struct { diff --git a/transport/ssh/serve_stdinserver.go b/transport/ssh/serve_stdinserver.go index 39bfba8..03e0d51 100644 --- a/transport/ssh/serve_stdinserver.go +++ b/transport/ssh/serve_stdinserver.go @@ -1,19 +1,21 @@ package ssh import ( - "github.com/problame/go-netssh" - "github.com/zrepl/zrepl/config" - "github.com/zrepl/zrepl/daemon/nethelpers" - "github.com/zrepl/zrepl/transport" + "context" "fmt" "net" "path" - "context" - "github.com/pkg/errors" "sync/atomic" + + "github.com/pkg/errors" + "github.com/problame/go-netssh" + + "github.com/zrepl/zrepl/config" + "github.com/zrepl/zrepl/daemon/nethelpers" + "github.com/zrepl/zrepl/transport" ) -func MultiStdinserverListenerFactoryFromConfig(g *config.Global, in *config.StdinserverServer) (transport.AuthenticatedListenerFactory,error) { +func MultiStdinserverListenerFactoryFromConfig(g *config.Global, in *config.StdinserverServer) (transport.AuthenticatedListenerFactory, error) { for _, ci := range in.ClientIdentities { if err := transport.ValidateClientIdentity(ci); err != nil { @@ -24,7 +26,7 @@ func MultiStdinserverListenerFactoryFromConfig(g *config.Global, in *config.Stdi clientIdentities := in.ClientIdentities sockdir := g.Serve.StdinServer.SockDir - lf := func() (transport.AuthenticatedListener,error) { + lf := func() (transport.AuthenticatedListener, error) { return multiStdinserverListenerFromClientIdentities(sockdir, clientIdentities) } @@ -33,13 +35,13 @@ func MultiStdinserverListenerFactoryFromConfig(g *config.Global, in *config.Stdi type multiStdinserverAcceptRes struct { conn *transport.AuthConn - err error + err error } type MultiStdinserverListener struct { listeners []*stdinserverListener - accepts chan multiStdinserverAcceptRes - closed int32 + accepts chan multiStdinserverAcceptRes + closed int32 } // client identities must be validated @@ -48,7 +50,7 @@ func multiStdinserverListenerFromClientIdentities(sockdir string, cis []string) var err error for _, ci := range cis { sockpath := path.Join(sockdir, ci) - l := &stdinserverListener{clientIdentity: ci} + l := &stdinserverListener{clientIdentity: ci} if err = nethelpers.PreparePrivateSockpath(sockpath); err != nil { break } @@ -66,7 +68,7 @@ func multiStdinserverListenerFromClientIdentities(sockdir string, cis []string) return &MultiStdinserverListener{listeners: listeners}, nil } -func (m *MultiStdinserverListener) Accept(ctx context.Context) (*transport.AuthConn, error){ +func (m *MultiStdinserverListener) Accept(ctx context.Context) (*transport.AuthConn, error) { if m.accepts == nil { m.accepts = make(chan multiStdinserverAcceptRes, len(m.listeners)) @@ -80,7 +82,7 @@ func (m *MultiStdinserverListener) Accept(ctx context.Context) (*transport.AuthC } } - res := <- m.accepts + res := <-m.accepts return res.conn, res.err } @@ -116,7 +118,7 @@ func (m *MultiStdinserverListener) Close() error { // a single stdinserverListener (part of multiStinserverListener) type stdinserverListener struct { - l *netssh.Listener + l *netssh.Listener clientIdentity string } diff --git a/transport/tcp/serve_tcp.go b/transport/tcp/serve_tcp.go index a6b8107..45d3851 100644 --- a/transport/tcp/serve_tcp.go +++ b/transport/tcp/serve_tcp.go @@ -1,15 +1,17 @@ package tcp import ( - "github.com/zrepl/zrepl/config" - "net" - "github.com/pkg/errors" "context" + "net" + + "github.com/pkg/errors" + + "github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/transport" ) type ipMapEntry struct { - ip net.IP + ip net.IP ident string } @@ -25,7 +27,7 @@ func ipMapFromConfig(clients map[string]string) (*ipMap, error) { return nil, errors.Errorf("cannot parse client IP %q", clientIPString) } if err := transport.ValidateClientIdentity(clientIdent); err != nil { - return nil, errors.Wrapf(err,"invalid client identity for IP %q", clientIPString) + return nil, errors.Wrapf(err, "invalid client identity for IP %q", clientIPString) } entries = append(entries, ipMapEntry{clientIP, clientIdent}) } @@ -79,4 +81,3 @@ func (f *TCPAuthListener) Accept(ctx context.Context) (*transport.AuthConn, erro } return transport.NewAuthConn(nc, clientIdent), nil } - diff --git a/transport/tls/connect_tls.go b/transport/tls/connect_tls.go index ea578d4..ebe447f 100644 --- a/transport/tls/connect_tls.go +++ b/transport/tls/connect_tls.go @@ -6,6 +6,7 @@ import ( "net" "github.com/pkg/errors" + "github.com/zrepl/zrepl/config" "github.com/zrepl/zrepl/tlsconf" "github.com/zrepl/zrepl/transport" diff --git a/transport/tls/serve_tls.go b/transport/tls/serve_tls.go index 21aafe4..8b8360b 100644 --- a/transport/tls/serve_tls.go +++ b/transport/tls/serve_tls.go @@ -1,27 +1,22 @@ package tls import ( + "context" "crypto/tls" - "crypto/x509" "fmt" - "github.com/pkg/errors" - "github.com/zrepl/zrepl/config" - "github.com/zrepl/zrepl/transport" - "github.com/zrepl/zrepl/tlsconf" "net" "time" - "context" + + "github.com/pkg/errors" + + "github.com/zrepl/zrepl/config" + "github.com/zrepl/zrepl/tlsconf" + "github.com/zrepl/zrepl/transport" ) -type TLSListenerFactory struct { - address string - clientCA *x509.CertPool - serverCert tls.Certificate - handshakeTimeout time.Duration - clientCNs map[string]struct{} -} +type TLSListenerFactory struct{} -func TLSListenerFactoryFromConfig(c *config.Global, in *config.TLSServe) (transport.AuthenticatedListenerFactory,error) { +func TLSListenerFactoryFromConfig(c *config.Global, in *config.TLSServe) (transport.AuthenticatedListenerFactory, error) { address := in.Listen handshakeTimeout := in.HandshakeTimeout @@ -73,17 +68,24 @@ func (l tlsAuthListener) Accept(ctx context.Context) (*transport.AuthConn, error return nil, err } if _, ok := l.clientCNs[cn]; !ok { + log := transport.GetLogger(ctx) if dl, ok := ctx.Deadline(); ok { - defer tlsConn.SetDeadline(time.Time{}) - tlsConn.SetDeadline(dl) + defer func() { + err := tlsConn.SetDeadline(time.Time{}) + if err != nil { + log.WithError(err).Error("cannot clear connection deadline") + } + }() + err := tlsConn.SetDeadline(dl) + if err != nil { + log.WithError(err).WithField("deadline", dl).Error("cannot set connection deadline inherited from context") + } } if err := tlsConn.Close(); err != nil { - transport.GetLogger(ctx).WithError(err).Error("error closing connection with unauthorized common name") + log.WithError(err).Error("error closing connection with unauthorized common name") } return nil, fmt.Errorf("unauthorized client common name %q from %s", cn, tlsConn.RemoteAddr()) } adaptor := newWireAdaptor(tlsConn, tcpConn) return transport.NewAuthConn(adaptor, cn), nil } - - diff --git a/util/bytecounter/bytecounter_reader.go b/util/bytecounter/bytecounter_reader.go new file mode 100644 index 0000000..fc1f7c5 --- /dev/null +++ b/util/bytecounter/bytecounter_reader.go @@ -0,0 +1,49 @@ +package bytecounter + +import ( + "io" + "sync/atomic" + "time" +) + +type ByteCounterReader struct { + reader io.ReadCloser + + // called & accessed synchronously during Read, no external access + cb func(full int64) + cbEvery time.Duration + lastCbAt time.Time + + // set atomically because it may be read by multiple threads + bytes int64 +} + +func NewByteCounterReader(reader io.ReadCloser) *ByteCounterReader { + return &ByteCounterReader{ + reader: reader, + } +} + +func (b *ByteCounterReader) SetCallback(every time.Duration, cb func(full int64)) { + b.cbEvery = every + b.cb = cb +} + +func (b *ByteCounterReader) Close() error { + return b.reader.Close() +} + +func (b *ByteCounterReader) Read(p []byte) (n int, err error) { + n, err = b.reader.Read(p) + full := atomic.AddInt64(&b.bytes, int64(n)) + now := time.Now() + if b.cb != nil && now.Sub(b.lastCbAt) > b.cbEvery { + b.cb(full) + b.lastCbAt = now + } + return n, err +} + +func (b *ByteCounterReader) Bytes() int64 { + return atomic.LoadInt64(&b.bytes) +} diff --git a/util/bytecounter/bytecounter_streamcopier_test.go b/util/bytecounter/bytecounter_streamcopier_test.go index 29611e0..f4f8b1e 100644 --- a/util/bytecounter/bytecounter_streamcopier_test.go +++ b/util/bytecounter/bytecounter_streamcopier_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/zrepl/zrepl/zfs" ) diff --git a/util/chainedio/chainedio_reader.go b/util/chainedio/chainedio_reader.go new file mode 100644 index 0000000..a7bdea6 --- /dev/null +++ b/util/chainedio/chainedio_reader.go @@ -0,0 +1,34 @@ +package chainedio + +import "io" + +type ChainedReader struct { + Readers []io.Reader + curReader int +} + +func NewChainedReader(reader ...io.Reader) *ChainedReader { + return &ChainedReader{ + Readers: reader, + curReader: 0, + } +} + +func (c *ChainedReader) Read(buf []byte) (n int, err error) { + + n = 0 + + for c.curReader < len(c.Readers) { + n, err = c.Readers[c.curReader].Read(buf) + if err == io.EOF { + c.curReader++ + continue + } + break + } + if c.curReader == len(c.Readers) { + err = io.EOF // actually, there was no gap + } + + return +} diff --git a/util/chainlock/chainlock.go b/util/chainlock/chainlock.go index 6e7b5e5..ef1d971 100644 --- a/util/chainlock/chainlock.go +++ b/util/chainlock/chainlock.go @@ -39,4 +39,4 @@ func (l *L) NewCond() *sync.Cond { func (l *L) DropWhile(f func()) { defer l.Unlock().Lock() f() -} \ No newline at end of file +} diff --git a/util/chunking.go b/util/chunking/chunking.go similarity index 98% rename from util/chunking.go rename to util/chunking/chunking.go index bfbb59d..cceb344 100644 --- a/util/chunking.go +++ b/util/chunking/chunking.go @@ -1,4 +1,4 @@ -package util +package chunking import ( "bytes" @@ -49,7 +49,7 @@ func (c *Unchunker) Read(b []byte) (n int, err error) { } - if c.remainingChunkBytes <= 0 { + if c.remainingChunkBytes == 0 { panic("internal inconsistency: c.remainingChunkBytes must be > 0") } if len(b) <= 0 { diff --git a/util/chunking_test.go b/util/chunking/chunking_test.go similarity index 98% rename from util/chunking_test.go rename to util/chunking/chunking_test.go index 3ca48f2..7514089 100644 --- a/util/chunking_test.go +++ b/util/chunking/chunking_test.go @@ -1,13 +1,14 @@ -package util +package chunking import ( "bytes" "encoding/binary" - "github.com/stretchr/testify/assert" "io" "reflect" "testing" "testing/quick" + + "github.com/stretchr/testify/assert" ) func TestUnchunker(t *testing.T) { diff --git a/util/connlogger/connlogger.go b/util/connlogger/connlogger.go new file mode 100644 index 0000000..f355e5b --- /dev/null +++ b/util/connlogger/connlogger.go @@ -0,0 +1,67 @@ +package connlogger + +import ( + "net" + "os" +) + +type NetConnLogger struct { + net.Conn + ReadFile *os.File + WriteFile *os.File +} + +func NewNetConnLogger(conn net.Conn, readlog, writelog string) (l *NetConnLogger, err error) { + l = &NetConnLogger{ + Conn: conn, + } + flags := os.O_CREATE | os.O_WRONLY + if readlog != "" { + if l.ReadFile, err = os.OpenFile(readlog, flags, 0600); err != nil { + return + } + } + if writelog != "" { + if l.WriteFile, err = os.OpenFile(writelog, flags, 0600); err != nil { + return + } + } + return +} + +func (c *NetConnLogger) Read(buf []byte) (n int, err error) { + n, err = c.Conn.Read(buf) + if c.WriteFile != nil { + if _, writeErr := c.ReadFile.Write(buf[0:n]); writeErr != nil { + panic(writeErr) + } + } + return +} + +func (c *NetConnLogger) Write(buf []byte) (n int, err error) { + n, err = c.Conn.Write(buf) + if c.ReadFile != nil { + if _, writeErr := c.WriteFile.Write(buf[0:n]); writeErr != nil { + panic(writeErr) + } + } + return +} +func (c *NetConnLogger) Close() (err error) { + err = c.Conn.Close() + if err != nil { + return + } + if c.ReadFile != nil { + if err := c.ReadFile.Close(); err != nil { + panic(err) + } + } + if c.WriteFile != nil { + if err := c.WriteFile.Close(); err != nil { + panic(err) + } + } + return +} diff --git a/util/io.go b/util/io.go deleted file mode 100644 index 8448e9d..0000000 --- a/util/io.go +++ /dev/null @@ -1,144 +0,0 @@ -package util - -import ( - "io" - "net" - "os" - "sync/atomic" - "time" -) - -type NetConnLogger struct { - net.Conn - ReadFile *os.File - WriteFile *os.File -} - -func NewNetConnLogger(conn net.Conn, readlog, writelog string) (l *NetConnLogger, err error) { - l = &NetConnLogger{ - Conn: conn, - } - flags := os.O_CREATE | os.O_WRONLY - if readlog != "" { - if l.ReadFile, err = os.OpenFile(readlog, flags, 0600); err != nil { - return - } - } - if writelog != "" { - if l.WriteFile, err = os.OpenFile(writelog, flags, 0600); err != nil { - return - } - } - return -} - -func (c *NetConnLogger) Read(buf []byte) (n int, err error) { - n, err = c.Conn.Read(buf) - if c.WriteFile != nil { - if _, writeErr := c.ReadFile.Write(buf[0:n]); writeErr != nil { - panic(writeErr) - } - } - return -} - -func (c *NetConnLogger) Write(buf []byte) (n int, err error) { - n, err = c.Conn.Write(buf) - if c.ReadFile != nil { - if _, writeErr := c.WriteFile.Write(buf[0:n]); writeErr != nil { - panic(writeErr) - } - } - return -} -func (c *NetConnLogger) Close() (err error) { - err = c.Conn.Close() - if err != nil { - return - } - if c.ReadFile != nil { - if err := c.ReadFile.Close(); err != nil { - panic(err) - } - } - if c.WriteFile != nil { - if err := c.WriteFile.Close(); err != nil { - panic(err) - } - } - return -} - -type ChainedReader struct { - Readers []io.Reader - curReader int -} - -func NewChainedReader(reader ...io.Reader) *ChainedReader { - return &ChainedReader{ - Readers: reader, - curReader: 0, - } -} - -func (c *ChainedReader) Read(buf []byte) (n int, err error) { - - n = 0 - - for c.curReader < len(c.Readers) { - n, err = c.Readers[c.curReader].Read(buf) - if err == io.EOF { - c.curReader++ - continue - } - break - } - if c.curReader == len(c.Readers) { - err = io.EOF // actually, there was no gap - } - - return -} - -type ByteCounterReader struct { - reader io.ReadCloser - - // called & accessed synchronously during Read, no external access - cb func(full int64) - cbEvery time.Duration - lastCbAt time.Time - bytesSinceLastCb int64 - - // set atomically because it may be read by multiple threads - bytes int64 -} - -func NewByteCounterReader(reader io.ReadCloser) *ByteCounterReader { - return &ByteCounterReader{ - reader: reader, - } -} - -func (b *ByteCounterReader) SetCallback(every time.Duration, cb func(full int64)) { - b.cbEvery = every - b.cb = cb -} - -func (b *ByteCounterReader) Close() error { - return b.reader.Close() -} - -func (b *ByteCounterReader) Read(p []byte) (n int, err error) { - n, err = b.reader.Read(p) - full := atomic.AddInt64(&b.bytes, int64(n)) - now := time.Now() - if b.cb != nil && now.Sub(b.lastCbAt) > b.cbEvery { - b.cb(full) - b.lastCbAt = now - } - return n, err -} - -func (b *ByteCounterReader) Bytes() int64 { - return atomic.LoadInt64(&b.bytes) -} diff --git a/util/iocommand.go b/util/iocommand/iocommand.go similarity index 97% rename from util/iocommand.go rename to util/iocommand/iocommand.go index fe5f9a3..fa9887e 100644 --- a/util/iocommand.go +++ b/util/iocommand/iocommand.go @@ -1,21 +1,22 @@ -package util +package iocommand import ( "bytes" "context" "fmt" - "github.com/zrepl/zrepl/util/envconst" "io" "os" "os/exec" "syscall" "time" + + "github.com/zrepl/zrepl/util/envconst" ) // An IOCommand exposes a forked process's std(in|out|err) through the io.ReadWriteCloser interface. type IOCommand struct { Cmd *exec.Cmd - kill context.CancelFunc + kill context.CancelFunc Stdin io.WriteCloser Stdout io.ReadCloser StderrBuf *bytes.Buffer @@ -98,7 +99,7 @@ func (c *IOCommand) doWait(ctx context.Context) (err error) { if !ok { return } - time.Sleep(dl.Sub(time.Now())) + time.Sleep(time.Until(dl)) c.kill() c.Stdout.Close() c.Stdin.Close() diff --git a/util/contextflexibletimeout.go b/util/optionaldeadline/optionaldeadline.go similarity index 96% rename from util/contextflexibletimeout.go rename to util/optionaldeadline/optionaldeadline.go index 422318d..36f5045 100644 --- a/util/contextflexibletimeout.go +++ b/util/optionaldeadline/optionaldeadline.go @@ -1,4 +1,4 @@ -package util +package optionaldeadline import ( "context" @@ -54,7 +54,7 @@ func ContextWithOptionalDeadline(pctx context.Context) (ctx context.Context, enf } // Deadline in past? - sleepTime := deadline.Sub(time.Now()) + sleepTime := time.Until(deadline) if sleepTime <= 0 { rctx.m.Lock() rctx.err = context.DeadlineExceeded diff --git a/util/contextflexibletimeout_test.go b/util/optionaldeadline/optionaldeadline_test.go similarity index 65% rename from util/contextflexibletimeout_test.go rename to util/optionaldeadline/optionaldeadline_test.go index e6a1128..299a476 100644 --- a/util/contextflexibletimeout_test.go +++ b/util/optionaldeadline/optionaldeadline_test.go @@ -1,11 +1,14 @@ -package util +package optionaldeadline import ( "context" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "testing" "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/zrepl/zrepl/util/chainlock" ) func TestContextWithOptionalDeadline(t *testing.T) { @@ -14,19 +17,28 @@ func TestContextWithOptionalDeadline(t *testing.T) { cctx, enforceDeadline := ContextWithOptionalDeadline(ctx) begin := time.Now() - var receivedCancellation time.Time - var cancellationError error + var checker struct { + receivedCancellation time.Time + cancellationError error + timeout bool + mtx chainlock.L + } go func() { select { case <-cctx.Done(): - receivedCancellation = time.Now() - cancellationError = cctx.Err() + defer checker.mtx.Lock().Unlock() + checker.receivedCancellation = time.Now() + checker.cancellationError = cctx.Err() case <-time.After(600 * time.Millisecond): - t.Fatalf("should have been cancelled by deadline") + defer checker.mtx.Lock().Unlock() + checker.timeout = true } }() - time.Sleep(100 * time.Millisecond) - if !receivedCancellation.IsZero() { + defer checker.mtx.Lock().Unlock() + checker.mtx.DropWhile(func() { + time.Sleep(100 * time.Millisecond) + }) + if !checker.receivedCancellation.IsZero() { t.Fatalf("no enforcement means no cancellation") } require.Nil(t, cctx.Err(), "no error while not cancelled") @@ -37,11 +49,15 @@ func TestContextWithOptionalDeadline(t *testing.T) { // second call must be ignored, i.e. we expect the deadline to be at begin+200ms, not begin+400ms enforceDeadline(begin.Add(400 * time.Millisecond)) - time.Sleep(300 * time.Millisecond) // 100ms margin for scheduler - if receivedCancellation.Sub(begin) > 250*time.Millisecond { - t.Fatalf("cancellation is beyond acceptable scheduler latency") + checker.mtx.DropWhile(func() { + time.Sleep(300 * time.Millisecond) // 100ms margin for scheduler + }) + assert.False(t, checker.timeout, "test timeout") + receivedCancellationAfter := checker.receivedCancellation.Sub(begin) + if receivedCancellationAfter > 250*time.Millisecond { + t.Fatalf("cancellation is beyond acceptable scheduler latency: %s", receivedCancellationAfter) } - require.Equal(t, context.DeadlineExceeded, cancellationError) + require.Equal(t, context.DeadlineExceeded, checker.cancellationError) } func TestContextWithOptionalDeadlineNegativeDeadline(t *testing.T) { diff --git a/zfs/datasetpath_visitor.go b/zfs/datasetpath_visitor.go index b8c8e05..7facec9 100644 --- a/zfs/datasetpath_visitor.go +++ b/zfs/datasetpath_visitor.go @@ -108,8 +108,7 @@ func (t *datasetPathTree) WalkTopDown(parent []string, visitor DatasetPathsVisit func newDatasetPathTree(initialComps []string) (t *datasetPathTree) { t = &datasetPathTree{} - var cur *datasetPathTree - cur = t + cur := t for i, comp := range initialComps { cur.Component = comp cur.FilledIn = true diff --git a/zfs/datasetpath_visitor_test.go b/zfs/datasetpath_visitor_test.go index 7e348f2..731d5a6 100644 --- a/zfs/datasetpath_visitor_test.go +++ b/zfs/datasetpath_visitor_test.go @@ -1,8 +1,9 @@ package zfs import ( - "github.com/stretchr/testify/assert" "testing" + + "github.com/stretchr/testify/assert" ) func TestNewDatasetPathTree(t *testing.T) { diff --git a/zfs/mapping.go b/zfs/mapping.go index dd874e2..ca2c794 100644 --- a/zfs/mapping.go +++ b/zfs/mapping.go @@ -13,7 +13,8 @@ type DatasetFilter interface { func NoFilter() DatasetFilter { return noFilter{} } -type noFilter struct {} + +type noFilter struct{} var _ DatasetFilter = noFilter{} diff --git a/zfs/replication_history.go b/zfs/replication_history.go index d420047..36663c3 100644 --- a/zfs/replication_history.go +++ b/zfs/replication_history.go @@ -2,8 +2,9 @@ package zfs import ( "fmt" - "github.com/pkg/errors" "strconv" + + "github.com/pkg/errors" ) const ReplicationCursorBookmarkName = "zrepl_replication_cursor" @@ -30,6 +31,9 @@ func ZFSSetReplicationCursor(fs *DatasetPath, snapname string) (guid uint64, err return 0, errors.Wrap(err, "zfs: replication cursor: get snapshot createtxg") } snapGuid, err := strconv.ParseUint(propsSnap.Get("guid"), 10, 64) + if err != nil { + return 0, errors.Wrap(err, "zfs: replication cursor: parse snapshot guid") + } bookmarkPath := fmt.Sprintf("%s#%s", fs.ToString(), ReplicationCursorBookmarkName) propsBookmark, err := zfsGet(bookmarkPath, []string{"createtxg"}, sourceAny) _, bookmarkNotExistErr := err.(*DatasetDoesNotExist) diff --git a/zfs/resume_token_test.go b/zfs/resume_token_test.go index 4a35595..d4acd6f 100644 --- a/zfs/resume_token_test.go +++ b/zfs/resume_token_test.go @@ -2,9 +2,11 @@ package zfs_test import ( "context" - "github.com/stretchr/testify/assert" - "github.com/zrepl/zrepl/zfs" "testing" + + "github.com/stretchr/testify/assert" + + "github.com/zrepl/zrepl/zfs" ) type ResumeTokenTest struct { diff --git a/zfs/versions.go b/zfs/versions.go index 29303c0..86850c0 100644 --- a/zfs/versions.go +++ b/zfs/versions.go @@ -3,20 +3,21 @@ package zfs import ( "bytes" "context" - "errors" "fmt" - "github.com/prometheus/client_golang/prometheus" "io" "strconv" "strings" "time" + + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" ) type VersionType string const ( Bookmark VersionType = "bookmark" - Snapshot = "snapshot" + Snapshot VersionType = "snapshot" ) func (t VersionType) DelimiterChar() string { @@ -36,14 +37,14 @@ func (t VersionType) String() string { func DecomposeVersionString(v string) (fs string, versionType VersionType, name string, err error) { if len(v) < 3 { - err = errors.New(fmt.Sprintf("snapshot or bookmark name implausibly short: %s", v)) + err = fmt.Errorf("snapshot or bookmark name implausibly short: %s", v) return } snapSplit := strings.SplitN(v, "@", 2) bookmarkSplit := strings.SplitN(v, "#", 2) if len(snapSplit)*len(bookmarkSplit) != 2 { - err = errors.New(fmt.Sprintf("dataset cannot be snapshot and bookmark at the same time: %s", v)) + err = fmt.Errorf("dataset cannot be snapshot and bookmark at the same time: %s", v) return } @@ -121,12 +122,12 @@ func ZFSListFilesystemVersions(fs *DatasetPath, filter FilesystemVersionFilter) } if v.Guid, err = strconv.ParseUint(line[1], 10, 64); err != nil { - err = errors.New(fmt.Sprintf("cannot parse GUID: %s", err.Error())) + err = errors.Wrap(err, "cannot parse GUID") return } if v.CreateTXG, err = strconv.ParseUint(line[2], 10, 64); err != nil { - err = errors.New(fmt.Sprintf("cannot parse CreateTXG: %s", err.Error())) + err = errors.Wrap(err, "cannot parse CreateTXG") return } @@ -159,16 +160,9 @@ func ZFSDestroyFilesystemVersion(filesystem *DatasetPath, version *FilesystemVer datasetPath := version.ToAbsPath(filesystem) // Sanity check... - if strings.IndexAny(datasetPath, "@#") == -1 { - return fmt.Errorf("sanity check failed: no @ character found in dataset path: %s", datasetPath) + if !strings.ContainsAny(datasetPath, "@#") { + return fmt.Errorf("sanity check failed: no @ or # character found in %q", datasetPath) } - err = ZFSDestroy(datasetPath) - if err == nil { - return - } - - // Check for EBUSY, special meaning to us - return - + return ZFSDestroy(datasetPath) } diff --git a/zfs/zfs.go b/zfs/zfs.go index 40b1434..fd40917 100644 --- a/zfs/zfs.go +++ b/zfs/zfs.go @@ -15,9 +15,11 @@ import ( "time" "context" - "github.com/prometheus/client_golang/prometheus" "regexp" "strconv" + + "github.com/prometheus/client_golang/prometheus" + "github.com/zrepl/zrepl/util/envconst" ) @@ -65,7 +67,6 @@ func (p *DatasetPath) TrimPrefix(prefix *DatasetPath) { for i := 0; i < newlen; i++ { p.comps[i] = oldcomps[prelen+i] } - return } func (p *DatasetPath) TrimNPrefixComps(n int) { @@ -249,7 +250,9 @@ func ZFSListChan(ctx context.Context, out chan ZFSListResult, properties []strin return } defer func() { - cmd.Wait() + // discard the error, this defer is only relevant if we return while parsing the output + // in which case we'll return an 'unexpected output' error and not the exit status + _ = cmd.Wait() }() s := bufio.NewScanner(stdout) @@ -281,7 +284,6 @@ func ZFSListChan(ctx context.Context, out chan ZFSListResult, properties []strin sendResult(nil, s.Err()) return } - return } func validateRelativeZFSVersion(s string) error { @@ -351,7 +353,7 @@ type readErrRecorder struct { type sendStreamCopierError struct { isReadErr bool // if false, it's a write error - err error + err error } func (e sendStreamCopierError) Error() string { @@ -362,7 +364,7 @@ func (e sendStreamCopierError) Error() string { } } -func (e sendStreamCopierError) IsReadError() bool { return e.isReadErr } +func (e sendStreamCopierError) IsReadError() bool { return e.isReadErr } func (e sendStreamCopierError) IsWriteError() bool { return !e.isReadErr } func (r *readErrRecorder) Read(p []byte) (n int, err error) { @@ -410,13 +412,12 @@ func pipeWithCapacityHint(capacity int) (r, w *os.File, err error) { } type sendStream struct { - cmd *exec.Cmd + cmd *exec.Cmd kill context.CancelFunc - closeMtx sync.Mutex + closeMtx sync.Mutex stdoutReader *os.File - opErr error - + opErr error } func (s *sendStream) Read(p []byte) (n int, err error) { @@ -484,7 +485,7 @@ func (s *sendStream) killAndWait(precedingReadErr error) error { if closePipeErr == nil { // avoid double-closes in case anything below doesn't work // and someone calls Close again - s.stdoutReader = nil + s.stdoutReader = nil } else { return closePipeErr } @@ -493,7 +494,7 @@ func (s *sendStream) killAndWait(precedingReadErr error) error { // we managed to tear things down, no let's give the user some pretty *ZFSError if exitErr != nil { s.opErr = &ZFSError{ - Stderr: exitErr.Stderr, + Stderr: exitErr.Stderr, WaitErr: exitErr, } } else { @@ -545,15 +546,14 @@ func ZFSSend(ctx context.Context, fs string, from, to string, token string) (str stdoutWriter.Close() stream := &sendStream{ - cmd: cmd, - kill: cancel, + cmd: cmd, + kill: cancel, stdoutReader: stdoutReader, } return newSendStreamCopier(stream), err } - type DrySendType string const ( @@ -563,25 +563,27 @@ const ( func DrySendTypeFromString(s string) (DrySendType, error) { switch s { - case string(DrySendTypeFull): return DrySendTypeFull, nil - case string(DrySendTypeIncremental): return DrySendTypeIncremental, nil + case string(DrySendTypeFull): + return DrySendTypeFull, nil + case string(DrySendTypeIncremental): + return DrySendTypeIncremental, nil default: return "", fmt.Errorf("unknown dry send type %q", s) } } type DrySendInfo struct { - Type DrySendType - Filesystem string // parsed from To field - From, To string // direct copy from ZFS output - SizeEstimate int64 // -1 if size estimate is not possible + Type DrySendType + Filesystem string // parsed from To field + From, To string // direct copy from ZFS output + SizeEstimate int64 // -1 if size estimate is not possible } var ( // keep same number of capture groups for unmarshalInfoLine homogenity sendDryRunInfoLineRegexFull = regexp.MustCompile(`^(full)\t()([^\t]+@[^\t]+)\t([0-9]+)$`) - // cannot enforce '[#@]' in incremental source, see test cases + // cannot enforce '[#@]' in incremental source, see test cases sendDryRunInfoLineRegexIncremental = regexp.MustCompile(`^(incremental)\t([^\t]+)\t([^\t]+@[^\t]+)\t([0-9]+)$`) ) @@ -602,7 +604,6 @@ func (s *DrySendInfo) unmarshalZFSOutput(output []byte) (err error) { return fmt.Errorf("no match for info line (regex1 %s) (regex2 %s)", sendDryRunInfoLineRegexFull, sendDryRunInfoLineRegexIncremental) } - // unmarshal info line, looks like this: // full zroot/test/a@1 5389768 // incremental zroot/test/a@1 zroot/test/a@2 5383936 @@ -653,19 +654,19 @@ func ZFSSendDry(fs string, from, to string, token string) (_ *DrySendInfo, err e * Redacted send & recv will bring this functionality, see * https://github.com/openzfs/openzfs/pull/484 */ - fromAbs, err := absVersion(fs, from) - if err != nil { - return nil, fmt.Errorf("error building abs version for 'from': %s", err) - } - toAbs, err := absVersion(fs, to) - if err != nil { - return nil, fmt.Errorf("error building abs version for 'to': %s", err) - } - return &DrySendInfo{ - Type: DrySendTypeIncremental, - Filesystem: fs, - From: fromAbs, - To: toAbs, + fromAbs, err := absVersion(fs, from) + if err != nil { + return nil, fmt.Errorf("error building abs version for 'from': %s", err) + } + toAbs, err := absVersion(fs, to) + if err != nil { + return nil, fmt.Errorf("error building abs version for 'to': %s", err) + } + return &DrySendInfo{ + Type: DrySendTypeIncremental, + Filesystem: fs, + From: fromAbs, + To: toAbs, SizeEstimate: -1}, nil } @@ -730,7 +731,7 @@ func ZFSRecv(ctx context.Context, fs string, streamCopier StreamCopier, opts Rec { vs, err := ZFSListFilesystemVersions(fsdp, nil) if err != nil { - err = fmt.Errorf("cannot list versions to rollback is required: %s", err) + return fmt.Errorf("cannot list versions for rollback for forced receive: %s", err) } for _, v := range vs { if v.Type == Snapshot { @@ -784,7 +785,7 @@ func ZFSRecv(ctx context.Context, fs string, streamCopier StreamCopier, opts Rec if err != nil { return err } - + cmd.Stdin = stdin if err = cmd.Start(); err != nil { @@ -794,7 +795,7 @@ func ZFSRecv(ctx context.Context, fs string, streamCopier StreamCopier, opts Rec } stdin.Close() defer stdinWriter.Close() - + pid := cmd.Process.Pid debug := func(format string, args ...interface{}) { debug("recv: pid=%v: %s", pid, fmt.Sprintf(format, args...)) @@ -823,7 +824,7 @@ func ZFSRecv(ctx context.Context, fs string, streamCopier StreamCopier, opts Rec copierErr := <-copierErrChan debug("copierErr: %T %s", copierErr, copierErr) if copierErr != nil { - cancelCmd() + cancelCmd() } waitErr := <-waitErrChan @@ -838,7 +839,7 @@ func ZFSRecv(ctx context.Context, fs string, streamCopier StreamCopier, opts Rec type ClearResumeTokenError struct { ZFSOutput []byte - CmdError error + CmdError error } func (e ClearResumeTokenError) Error() string { @@ -947,13 +948,27 @@ const ( func (s zfsPropertySource) zfsGetSourceFieldPrefixes() []string { prefixes := make([]string, 0, 7) - if s&sourceLocal != 0 {prefixes = append(prefixes, "local")} - if s&sourceDefault != 0 {prefixes = append(prefixes, "default")} - if s&sourceInherited != 0 {prefixes = append(prefixes, "inherited")} - if s&sourceNone != 0 {prefixes = append(prefixes, "-")} - if s&sourceTemporary != 0 { prefixes = append(prefixes, "temporary")} - if s&sourceReceived != 0 { prefixes = append(prefixes, "received")} - if s == sourceAny { prefixes = append(prefixes, "") } + if s&sourceLocal != 0 { + prefixes = append(prefixes, "local") + } + if s&sourceDefault != 0 { + prefixes = append(prefixes, "default") + } + if s&sourceInherited != 0 { + prefixes = append(prefixes, "inherited") + } + if s&sourceNone != 0 { + prefixes = append(prefixes, "-") + } + if s&sourceTemporary != 0 { + prefixes = append(prefixes, "temporary") + } + if s&sourceReceived != 0 { + prefixes = append(prefixes, "received") + } + if s == sourceAny { + prefixes = append(prefixes, "") + } return prefixes } @@ -992,7 +1007,7 @@ func zfsGet(path string, props []string, allowedSources zfsPropertySource) (*ZFS return nil, fmt.Errorf("zfs get did not return property,value,source tuples") } for _, p := range allowedPrefixes { - if strings.HasPrefix(fields[2],p) { + if strings.HasPrefix(fields[2], p) { res.m[fields[0]] = fields[1] break } @@ -1010,8 +1025,10 @@ func ZFSDestroy(dataset string) (err error) { filesystem = dataset } else { switch dataset[idx] { - case '@': dstype = "snapshot" - case '#': dstype = "bookmark" + case '@': + dstype = "snapshot" + case '#': + dstype = "bookmark" } filesystem = dataset[:idx] } diff --git a/zfs/zfs_debug.go b/zfs/zfs_debug.go index 32846e4..575bb31 100644 --- a/zfs/zfs_debug.go +++ b/zfs/zfs_debug.go @@ -13,6 +13,7 @@ func init() { } } +//nolint[:deadcode,unused] func debug(format string, args ...interface{}) { if debugEnabled { fmt.Fprintf(os.Stderr, "zfs: %s\n", fmt.Sprintf(format, args...)) diff --git a/zfs/zfs_test.go b/zfs/zfs_test.go index 217d765..f9ea860 100644 --- a/zfs/zfs_test.go +++ b/zfs/zfs_test.go @@ -1,8 +1,9 @@ package zfs import ( - "github.com/stretchr/testify/assert" "testing" + + "github.com/stretchr/testify/assert" ) func TestZFSListHandlesProducesZFSErrorOnNonZeroExit(t *testing.T) { @@ -33,8 +34,8 @@ func TestDatasetPathTrimNPrefixComps(t *testing.T) { func TestZFSPropertySource(t *testing.T) { - tcs := []struct{ - in zfsPropertySource + tcs := []struct { + in zfsPropertySource exp []string }{ { @@ -43,11 +44,11 @@ func TestZFSPropertySource(t *testing.T) { exp: []string{"local", "default", "inherited", "-", "temporary", "received", ""}, }, { - in: sourceTemporary, + in: sourceTemporary, exp: []string{"temporary"}, }, { - in: sourceLocal|sourceInherited, + in: sourceLocal | sourceInherited, exp: []string{"local", "inherited"}, }, } @@ -137,9 +138,9 @@ size 10518512 incrementalWithSpacesInIntermediateComponent := "\nincremental\tblaffoo\tpool1/otherjob/another ds with spaces/childfs@blaffoo2\t624\nsize\t624\n" type tc struct { - name string - in string - exp *DrySendInfo + name string + in string + exp *DrySendInfo expErr bool } @@ -147,10 +148,10 @@ size 10518512 { name: "fullSend", in: fullSend, exp: &DrySendInfo{ - Type: DrySendTypeFull, - Filesystem: "zroot/test/a", - From: "", - To: "zroot/test/a@1", + Type: DrySendTypeFull, + Filesystem: "zroot/test/a", + From: "", + To: "zroot/test/a@1", SizeEstimate: 5389768, }, }, @@ -158,7 +159,7 @@ size 10518512 name: "incSend", in: incSend, exp: &DrySendInfo{ Type: DrySendTypeIncremental, - Filesystem: "zroot/test/a", + Filesystem: "zroot/test/a", From: "zroot/test/a@1", To: "zroot/test/a@2", SizeEstimate: 5383936, @@ -168,16 +169,16 @@ size 10518512 name: "incSendBookmark", in: incSendBookmark, exp: &DrySendInfo{ Type: DrySendTypeIncremental, - Filesystem: "zroot/test/a", + Filesystem: "zroot/test/a", From: "zroot/test/a#1", To: "zroot/test/a@2", SizeEstimate: 5383312, }, }, - { + { name: "incNoToken", in: incNoToken, exp: &DrySendInfo{ - Type: DrySendTypeIncremental, + Type: DrySendTypeIncremental, Filesystem: "zroot/test/a", // as can be seen in the string incNoToken, // we cannot infer whether the incremental source is a snapshot or bookmark @@ -189,10 +190,10 @@ size 10518512 { name: "fullNoToken", in: fullNoToken, exp: &DrySendInfo{ - Type: DrySendTypeFull, - Filesystem: "zroot/test/a", - From: "", - To: "zroot/test/a@3", + Type: DrySendTypeFull, + Filesystem: "zroot/test/a", + From: "", + To: "zroot/test/a@3", SizeEstimate: 10518512, }, },