diff --git a/.travis.yml b/.travis.yml index ba05546..c4245c2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,16 +8,13 @@ git: # TODO: use docker service to start influxdb etc. and create database like oltpbench # https://github.com/benchhub/oltpbench/blob/config/cleanup-v2/.travis.yml go: - - 1.9.x - - "1.10" + - "1.13" - tip +# FIXME: enable test when we no longer rely on local go modules + install: - - go get -u github.com/golang/dep/cmd/dep - - dep version - - dep ensure - - pwd + - go version script: - - make install - - make test \ No newline at end of file + - go version \ No newline at end of file diff --git a/Gopkg.lock b/Gopkg.lock deleted file mode 100644 index fede080..0000000 --- a/Gopkg.lock +++ /dev/null @@ -1,99 +0,0 @@ -# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. - - -[[projects]] - branch = "master" - name = "github.com/at15/go.ice" - packages = [ - "ice/cli", - "ice/util/logutil" - ] - revision = "1eae51f02b6b9b0e8adf781c6dc570458c18db77" - -[[projects]] - branch = "master" - name = "github.com/dyweb/gommon" - packages = [ - "cast", - "config", - "errors", - "log", - "log/handlers/cli", - "requests", - "structure", - "util", - "util/color", - "util/hashutil", - "util/logutil", - "util/runtimeutil" - ] - revision = "ae7701f9d583472626a924dd9ce761157fc7ee05" - -[[projects]] - name = "github.com/gogo/protobuf" - packages = [ - "gogoproto", - "proto", - "protoc-gen-gogo/descriptor" - ] - revision = "1adfc126b41513cc696b209667c8656ea7aac67c" - version = "v1.0.0" - -[[projects]] - name = "github.com/inconshreveable/mousetrap" - packages = ["."] - revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" - version = "v1.0" - -[[projects]] - branch = "master" - name = "github.com/libtsdb/libtsdb-go" - packages = [ - "libtsdb", - "libtsdb/client/akumuliw", - "libtsdb/client/genericw", - "libtsdb/client/graphitew", - "libtsdb/client/influxdbw", - "libtsdb/client/kairosdbw", - "libtsdb/common", - "libtsdb/common/akumuli", - "libtsdb/common/graphite", - "libtsdb/common/influxdb", - "libtsdb/common/kairosdb", - "libtsdb/config", - "libtsdb/libtsdbpb", - "libtsdb/util/bytesutil", - "libtsdb/util/logutil" - ] - revision = "419c24436f340441f068a4af150b01ec34eb51ba" - -[[projects]] - name = "github.com/spf13/cobra" - packages = ["."] - revision = "7b2c5ac9fc04fc5efafb60700713d4fa609b777b" - version = "v0.0.1" - -[[projects]] - name = "github.com/spf13/pflag" - packages = ["."] - revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66" - version = "v1.0.0" - -[[projects]] - branch = "master" - name = "golang.org/x/net" - packages = ["proxy"] - revision = "24dd3780ca4f75fed9f321890729414a4b5d3f13" - -[[projects]] - name = "gopkg.in/yaml.v2" - packages = ["."] - revision = "7f97868eec74b32b0982dd158a51a446d1da7eb5" - version = "v2.1.1" - -[solve-meta] - analyzer-name = "dep" - analyzer-version = 1 - inputs-digest = "5c940ed0dc8a22730cf28d94a7f00ff21413b72d6711abc2bcefa72f77faaf84" - solver-name = "gps-cdcl" - solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml deleted file mode 100644 index 8b9a6d1..0000000 --- a/Gopkg.toml +++ /dev/null @@ -1,41 +0,0 @@ -# Gopkg.toml example -# -# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md -# for detailed Gopkg.toml documentation. -# -# required = ["github.com/user/thing/cmd/thing"] -# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] -# -# [[constraint]] -# name = "github.com/user/project" -# version = "1.0.0" -# -# [[constraint]] -# name = "github.com/user/project2" -# branch = "dev" -# source = "github.com/myfork/project2" -# -# [[override]] -# name = "github.com/x/y" -# version = "2.4.0" -# -# [prune] -# non-go = false -# go-tests = true -# unused-packages = true - -[[constraint]] - name = "github.com/at15/go.ice" - branch = "master" - -[[constraint]] - name = "github.com/libtsdb/libtsdb-go" - branch = "master" - -[[constraint]] - name = "github.com/dyweb/gommon" - branch = "master" - -[prune] - go-tests = true - unused-packages = true diff --git a/Makefile b/Makefile index 5753de7..54bb581 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,6 @@ FLAGS = -X main.version=$(VERSION) -X main.commit=$(BUILD_COMMIT) -X main.buildT .PHONY: install install: - go install -ldflags "$(FLAGS)" ./cmd/xab go install -ldflags "$(FLAGS)" ./cmd/xb .PHONY: test @@ -16,7 +15,7 @@ test: .PHONY: fmt fmt: - gofmt -d -l -w ./pkg ./cmd + goimports -d -l -w ./pkg ./cmd .PHONY: generate generate: diff --git a/README.md b/README.md index 82cc5e8..be7e448 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ A time series database benchmark tool. NOTE: it is under major rewrite -- status: Under major rewrite, after libtsdb-go +- status: Under major rewrite, along with [libtsdb-go](https://github.com/libtsdb/libtsdb-go) - [Documentation](doc) - [Slide: Introduce Xephon-B](http://www.slideshare.net/ssuser7e134a/intoduce-xephonb) @@ -32,16 +32,13 @@ A time series database benchmark tool. NOTE: it is under major rewrite MIT - [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fxephonhq%2Fxephon-b.svg?type=large)](https://app.fossa.io/projects/git%2Bgithub.com%2Fxephonhq%2Fxephon-b?ref=badge_large) ## Authors -- [Pinglei Guo](https://at15.github.io) [@at15](https://github.com/at15), [linkedin](https://www.linkedin.com/in/at1510086), [twitter](https://twitter.com/at1510086) -- [Zheyuan Chen](http://czheo.github.io/) [@czheo](https://github.com/czheo), [linkedin](https://www.linkedin.com/in/zheyuan-chen), [twitter](https://twitter.com/czheo) +- [Pinglei Guo](https://at15.github.io) [@at15](https://github.com/at15), [twitter](https://twitter.com/at1510086) +- [Zheyuan Chen](http://czheo.github.io/) [@czheo](https://github.com/czheo), [twitter](https://twitter.com/czheo) ## About B is for benchmark and Xephon comes from the animation [RahXephon](https://en.wikipedia.org/wiki/RahXephon) - -It was merged into Xephon-K and now split out from it as a standalone project again \ No newline at end of file diff --git a/cmd/README.md b/cmd/README.md index 557a8d3..6b1d6ca 100644 --- a/cmd/README.md +++ b/cmd/README.md @@ -1,7 +1,2 @@ -# Cli +# Xephon-B binaries -## Legacy - -- tsdb-proxy -- xb is alias of xephon-b -- xephon-b \ No newline at end of file diff --git a/cmd/xab/main.go b/cmd/xab/main.go deleted file mode 100644 index fe93e6f..0000000 --- a/cmd/xab/main.go +++ /dev/null @@ -1,65 +0,0 @@ -package main - -import ( - "fmt" - "os" - "runtime" - - icli "github.com/at15/go.ice/ice/cli" - "github.com/spf13/cobra" - "github.com/xephonhq/xephon-b/pkg/util/logutil" - "net/http" -) - -const ( - myname = "xab" -) - -// TODO: we should not use lib log as top registry, but they are same struct, so it should work -var log = logutil.Registry - -var ( - version string - commit string - buildTime string - buildUser string - goVersion = runtime.Version() -) - -var buildInfo = icli.BuildInfo{Version: version, Commit: commit, BuildTime: buildTime, BuildUser: buildUser, GoVersion: goVersion} - -var testSeverCmd = &cobra.Command{ - Use: "test-server", - Short: "start test server", - Long: "Start test http server on 8080 with /ping /empty", - Run: func(cmd *cobra.Command, args []string) { - mux := http.NewServeMux() - mux.HandleFunc("/ping", func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte("pong")) - }) - addr := "localhost:8080" - srv := http.Server{ - Addr: addr, - } - srv.Handler = mux - log.Infof("listen on %s try %s/ping in browser", addr, addr) - if err := srv.ListenAndServe(); err != nil { - log.Fatal(err) - } - }, -} - -func main() { - cli := icli.New( - icli.Name(myname), - icli.Description("Xephon-B version of http benchmark"), - icli.Version(buildInfo), - icli.LogRegistry(log), - ) - root := cli.Command() - root.AddCommand(testSeverCmd) - if err := root.Execute(); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } -} diff --git a/cmd/xb/main.go b/cmd/xb/main.go index 1fc81af..a689990 100644 --- a/cmd/xb/main.go +++ b/cmd/xb/main.go @@ -5,19 +5,17 @@ import ( "os" "runtime" - icli "github.com/at15/go.ice/ice/cli" - goicelog "github.com/at15/go.ice/ice/util/logutil" - + icli "github.com/dyweb/go.ice/cli" + dlog "github.com/dyweb/gommon/log" "github.com/xephonhq/xephon-b/pkg/config" - "github.com/xephonhq/xephon-b/pkg/util/logutil" ) const ( myname = "xb" ) -// FIXME: debug logging is not working .... -var log = logutil.Registry +var logReg = dlog.NewRegistry() +var log = logReg.Logger() var ( version string @@ -37,7 +35,6 @@ func main() { icli.Name(myname), icli.Description("Xephon-B Time Series Benchmark cli"), icli.Version(buildInfo), - icli.LogRegistry(log), ) root := cli.Command() root.AddCommand(runCmd) @@ -48,12 +45,7 @@ func main() { } func mustLoadConfig() { - // TODO: go.ice should use UnmarshalStrict, and might just use direct load ... if err := cli.LoadConfigTo(&cfg); err != nil { log.Fatal(err) } } - -func init() { - log.AddChild(goicelog.Registry) -} diff --git a/cmd/xb/run.go b/cmd/xb/run.go index c86f607..1d8a467 100644 --- a/cmd/xb/run.go +++ b/cmd/xb/run.go @@ -2,6 +2,7 @@ package main import ( "context" + "github.com/spf13/cobra" "github.com/xephonhq/xephon-b/pkg/runner" ) diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..47cea6d --- /dev/null +++ b/go.mod @@ -0,0 +1,14 @@ +module github.com/xephonhq/xephon-b + +go 1.13 + +require ( + github.com/dyweb/go.ice v0.0.3 + github.com/dyweb/gommon v0.0.13 + github.com/libtsdb/libtsdb-go v0.0.0-20180319021657-419c24436f34 + github.com/spf13/cobra v0.0.5 +) + +replace github.com/dyweb/go.ice => ../../dyweb/go.ice + +replace github.com/libtsdb/libtsdb-go => ../../libtsdb/libtsdb-go diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..cc2fd28 --- /dev/null +++ b/go.sum @@ -0,0 +1,51 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dyweb/gommon v0.0.13 h1:SwEex1UakZJVpO37GqiTc0QQ7iOe5Rz+6hQnGh1UePo= +github.com/dyweb/gommon v0.0.13/go.mod h1:cdTMuWn9B/9F87Jza4nwI6Ka8z+uT5+WcGDWWKFhnqM= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/ab/pkg.go b/pkg/ab/pkg.go deleted file mode 100644 index 73ff81a..0000000 --- a/pkg/ab/pkg.go +++ /dev/null @@ -1 +0,0 @@ -package ab diff --git a/pkg/config/ab.go b/pkg/config/ab.go deleted file mode 100644 index d912156..0000000 --- a/pkg/config/ab.go +++ /dev/null @@ -1 +0,0 @@ -package config diff --git a/pkg/config/xb.go b/pkg/config/xb.go index f156709..485d856 100644 --- a/pkg/config/xb.go +++ b/pkg/config/xb.go @@ -3,19 +3,19 @@ package config import ( "time" - "github.com/libtsdb/libtsdb-go/libtsdb/config" + "github.com/libtsdb/libtsdb-go/libtsdb" ) type XephonBConfig struct { - Limit string `yaml:"limit"` - Duration time.Duration `yaml:"duration"` - Worker WorkerConfig `yaml:"worker"` - Workload string `yaml:"workload"` - Workloads []WorkloadConfig `yaml:"workloads"` - Database string `yaml:"database"` - Databases []DatabaseConfig `yaml:"databases"` - Reporter string `yaml:"reporter"` - Reporters []ReporterConfig `yaml:"reporters"` + Limit string `yaml:"limit"` + Duration time.Duration `yaml:"duration"` + Worker WorkerConfig `yaml:"worker"` + Workload string `yaml:"workload"` + Workloads []WorkloadConfig `yaml:"workloads"` + Database string `yaml:"database"` + Databases []libtsdb.DatabaseConfig `yaml:"databases"` + Reporter string `yaml:"reporter"` + Reporters []ReporterConfig `yaml:"reporters"` } type WorkerConfig struct { @@ -65,23 +65,14 @@ type RandomValueGeneratorConfig struct { Max float64 `yaml:"max"` } -type DatabaseConfig struct { - Name string `yaml:"name"` - Type string `yaml:"type"` - Akumuli *config.AkumuliClientConfig `yaml:"akumuli"` - Graphite *config.GraphiteClientConfig `yaml:"graphite"` - Influxdb *config.InfluxdbClientConfig `yaml:"influxdb"` - Kairosdb *config.KairosdbClientConfig `yaml:"kairosdb"` -} - type CounterReporterConfig struct { Foo string `yaml:"foo"` } type TSDBReporterConfig struct { // Sample is 1 from n samples, will ignore everything if n < 1 - Sample int `yaml:"sample"` - Database DatabaseConfig `yaml:"database"` + Sample int `yaml:"sample"` + Database libtsdb.DatabaseConfig `yaml:"database"` } type ReporterConfig struct { diff --git a/pkg/generator/pkg.go b/pkg/generator/pkg.go index a5f53f6..a64391f 100644 --- a/pkg/generator/pkg.go +++ b/pkg/generator/pkg.go @@ -1,14 +1,15 @@ package generator import ( - pb "github.com/libtsdb/libtsdb-go/libtsdb/libtsdbpb" - "github.com/xephonhq/xephon-b/pkg/util/logutil" + dlog "github.com/dyweb/gommon/log" + "github.com/libtsdb/libtsdb-go/tspb" ) -var log = logutil.NewPackageLogger() +var logReg = dlog.NewRegistry() +var log = logReg.Logger() type SeriesGenerator interface { - NextSeries() pb.EmptySeries + NextSeries() tspb.EmptySeries } type TimeGenerator interface { diff --git a/pkg/generator/series.go b/pkg/generator/series.go index d0955d3..7b1255a 100644 --- a/pkg/generator/series.go +++ b/pkg/generator/series.go @@ -6,8 +6,7 @@ import ( "time" "github.com/dyweb/gommon/errors" - pb "github.com/libtsdb/libtsdb-go/libtsdb/libtsdbpb" - + "github.com/libtsdb/libtsdb-go/tspb" "github.com/xephonhq/xephon-b/pkg/config" ) @@ -16,7 +15,7 @@ var _ SeriesGenerator = (*GenericSeries)(nil) // TODO: libtsdb does not have definition for series type GenericSeries struct { cfg config.SeriesConfig - series []pb.EmptySeries + series []tspb.EmptySeries r *rand.Rand i int len int @@ -40,14 +39,14 @@ func NewGenericSeries(cfg config.SeriesConfig) (*GenericSeries, error) { } func (g *GenericSeries) gen() { - var series []pb.EmptySeries + var series []tspb.EmptySeries r := g.r.Intn(1000) for i := 0; i < g.cfg.Num; i++ { - s := pb.EmptySeries{Name: fmt.Sprintf("%s.series.%d.%d", g.cfg.Prefix, r, i)} + s := tspb.EmptySeries{Name: fmt.Sprintf("%s.series.%d.%d", g.cfg.Prefix, r, i)} for j := 0; j < g.cfg.NumTags; j++ { - t := pb.Tag{ - K: fmt.Sprintf("key%d%d", r, j), - V: fmt.Sprintf("val%d%d", r, j), + t := tspb.Tag{ + Key: fmt.Sprintf("key%d%d", r, j), + Value: fmt.Sprintf("val%d%d", r, j), } s.Tags = append(s.Tags, t) } @@ -73,7 +72,7 @@ func (g *GenericSeries) cur() int { } } -func (g *GenericSeries) NextSeries() pb.EmptySeries { +func (g *GenericSeries) NextSeries() tspb.EmptySeries { // generate a set of new series to simulate series churn if g.cfg.Churn && time.Now().After(g.churnTime) { g.gen() diff --git a/pkg/generator/series_test.go b/pkg/generator/series_test.go index 5c341cc..b513bad 100644 --- a/pkg/generator/series_test.go +++ b/pkg/generator/series_test.go @@ -1,9 +1,10 @@ package generator import ( - "github.com/xephonhq/xephon-b/pkg/config" "testing" "time" + + "github.com/xephonhq/xephon-b/pkg/config" ) func TestGenericSeries_NextSeries(t *testing.T) { diff --git a/pkg/generator/time.go b/pkg/generator/time.go index 18582a9..856d7df 100644 --- a/pkg/generator/time.go +++ b/pkg/generator/time.go @@ -1,8 +1,9 @@ package generator import ( - "github.com/dyweb/gommon/errors" "time" + + "github.com/dyweb/gommon/errors" ) var _ TimeGenerator = (*FixIntervalTime)(nil) diff --git a/pkg/metrics/response.go b/pkg/metrics/response.go index 0cd5fb6..053a67a 100644 --- a/pkg/metrics/response.go +++ b/pkg/metrics/response.go @@ -1,7 +1,7 @@ package metrics -import "github.com/libtsdb/libtsdb-go/libtsdb" +import "github.com/libtsdb/libtsdb-go/protocol" type Response interface { - libtsdb.Trace + protocol.Trace } diff --git a/pkg/reporter/gommon_generated.go b/pkg/reporter/gommon_generated.go index 18310ae..be5992a 100644 --- a/pkg/reporter/gommon_generated.go +++ b/pkg/reporter/gommon_generated.go @@ -12,7 +12,7 @@ func (c *Counter) GetLogger() *dlog.Logger { return c.log } -func (c *Counter) LoggerIdentity(justCallMe func() *dlog.Identity) *dlog.Identity { +func (c *Counter) LoggerIdentity(justCallMe func() dlog.Identity) dlog.Identity { return justCallMe() } @@ -24,6 +24,6 @@ func (d *TSDB) GetLogger() *dlog.Logger { return d.log } -func (d *TSDB) LoggerIdentity(justCallMe func() *dlog.Identity) *dlog.Identity { +func (d *TSDB) LoggerIdentity(justCallMe func() dlog.Identity) dlog.Identity { return justCallMe() } diff --git a/pkg/reporter/pkg.go b/pkg/reporter/pkg.go index dcc5eba..bec532c 100644 --- a/pkg/reporter/pkg.go +++ b/pkg/reporter/pkg.go @@ -2,11 +2,13 @@ package reporter import ( "context" + + dlog "github.com/dyweb/gommon/log" "github.com/xephonhq/xephon-b/pkg/metrics" - "github.com/xephonhq/xephon-b/pkg/util/logutil" ) -var log = logutil.NewPackageLogger() +var logReg = dlog.NewRegistry() +var log = logReg.Logger() type Sink interface { // Run drain from response channel until it is closed or the context is canceled diff --git a/pkg/reporter/tsdb.go b/pkg/reporter/tsdb.go index 140de75..f7a933d 100644 --- a/pkg/reporter/tsdb.go +++ b/pkg/reporter/tsdb.go @@ -6,13 +6,14 @@ import ( "fmt" "time" + "github.com/libtsdb/libtsdb-go/libtsdb" + "github.com/dyweb/gommon/errors" dlog "github.com/dyweb/gommon/log" - "github.com/libtsdb/libtsdb-go/libtsdb" - pb "github.com/libtsdb/libtsdb-go/libtsdb/libtsdbpb" + "github.com/libtsdb/libtsdb-go/database" + "github.com/libtsdb/libtsdb-go/tspb" "github.com/xephonhq/xephon-b/pkg/config" "github.com/xephonhq/xephon-b/pkg/metrics" - "github.com/xephonhq/xephon-b/pkg/util/tsdbutil" ) var _ Sink = (*TSDB)(nil) @@ -21,7 +22,7 @@ var _ Sink = (*TSDB)(nil) type TSDB struct { cfg config.TSDBReporterConfig globalConfig config.XephonBConfig - c libtsdb.WriteClient + c database.WriteClient precision time.Duration bufferedPoints int @@ -32,7 +33,7 @@ type TSDB struct { } func NewTSDB(cfg config.TSDBReporterConfig, gcfg config.XephonBConfig) (*TSDB, error) { - c, err := tsdbutil.CreateClient(cfg.Database) + c, err := libtsdb.CreateClient(cfg.Database) if err != nil { return nil, errors.Wrap(err, "need tsdb client") } @@ -80,17 +81,17 @@ func (d *TSDB) Record(res metrics.Response) { t = t / 1000 } // TODO: distinguish worker ... - p := pb.PointIntTagged{ + p := tspb.PointIntTagged{ Name: "latency", - Tags: []pb.Tag{ - {K: "db", V: d.globalConfig.Database}, + Tags: []tspb.Tag{ + {Key: "db", Value: d.globalConfig.Database}, // TODO: actually response should pass worker id so they are stored in different series - {K: "totalWorker", V: fmt.Sprintf("%d", d.globalConfig.Worker.Num)}, - {K: "workload", V: d.globalConfig.Workload}, + {Key: "totalWorker", Value: fmt.Sprintf("%d", d.globalConfig.Worker.Num)}, + {Key: "workload", Value: d.globalConfig.Workload}, }, - Point: pb.PointInt{ - T: t, - V: res.GetEndTime() - res.GetStartTime(), + Point: tspb.PointInt{ + Time: t, + Value: res.GetEndTime() - res.GetStartTime(), }, } d.c.WriteIntPoint(&p) diff --git a/pkg/runner/gommon_generated.go b/pkg/runner/gommon_generated.go index a35a0a3..b39b24d 100644 --- a/pkg/runner/gommon_generated.go +++ b/pkg/runner/gommon_generated.go @@ -12,7 +12,7 @@ func (m *Manager) GetLogger() *dlog.Logger { return m.log } -func (m *Manager) LoggerIdentity(justCallMe func() *dlog.Identity) *dlog.Identity { +func (m *Manager) LoggerIdentity(justCallMe func() dlog.Identity) dlog.Identity { return justCallMe() } @@ -24,6 +24,6 @@ func (w *Worker) GetLogger() *dlog.Logger { return w.log } -func (w *Worker) LoggerIdentity(justCallMe func() *dlog.Identity) *dlog.Identity { +func (w *Worker) LoggerIdentity(justCallMe func() dlog.Identity) dlog.Identity { return justCallMe() } diff --git a/pkg/runner/manager.go b/pkg/runner/manager.go index b95c063..fc21721 100644 --- a/pkg/runner/manager.go +++ b/pkg/runner/manager.go @@ -3,12 +3,14 @@ package runner import ( "context" "fmt" + "sync" + "github.com/dyweb/gommon/errors" dlog "github.com/dyweb/gommon/log" + "github.com/libtsdb/libtsdb-go/libtsdb" "github.com/xephonhq/xephon-b/pkg/config" "github.com/xephonhq/xephon-b/pkg/metrics" "github.com/xephonhq/xephon-b/pkg/reporter" - "sync" ) type Manager struct { @@ -41,7 +43,7 @@ func (m *Manager) Run(ctx context.Context) error { // read config var ( - dbcfg config.DatabaseConfig + dbcfg libtsdb.DatabaseConfig wlcfg config.WorkloadConfig rpcfg config.ReporterConfig err error @@ -112,14 +114,14 @@ func (m *Manager) Run(ctx context.Context) error { return nil } -func (m *Manager) selectDatabase() (config.DatabaseConfig, error) { +func (m *Manager) selectDatabase() (libtsdb.DatabaseConfig, error) { for _, c := range m.cfg.Databases { if c.Name == m.cfg.Database { m.log.Infof("target database is %s type %s", c.Name, c.Type) return c, nil } } - return config.DatabaseConfig{}, + return libtsdb.DatabaseConfig{}, errors.Errorf("database %s does not have config, check name in databases section", m.cfg.Database) } diff --git a/pkg/runner/pkg.go b/pkg/runner/pkg.go index 35a86d1..365083b 100644 --- a/pkg/runner/pkg.go +++ b/pkg/runner/pkg.go @@ -1,7 +1,8 @@ package runner import ( - "github.com/xephonhq/xephon-b/pkg/util/logutil" + dlog "github.com/dyweb/gommon/log" ) -var log = logutil.NewPackageLogger() +var logReg = dlog.NewRegistry() +var log = logReg.Logger() diff --git a/pkg/runner/worker.go b/pkg/runner/worker.go index 8cc7cc8..343d467 100644 --- a/pkg/runner/worker.go +++ b/pkg/runner/worker.go @@ -6,22 +6,19 @@ import ( "github.com/dyweb/gommon/errors" dlog "github.com/dyweb/gommon/log" - + "github.com/libtsdb/libtsdb-go/database" "github.com/libtsdb/libtsdb-go/libtsdb" - - pb "github.com/libtsdb/libtsdb-go/libtsdb/libtsdbpb" - + "github.com/libtsdb/libtsdb-go/tspb" "github.com/xephonhq/xephon-b/pkg/config" "github.com/xephonhq/xephon-b/pkg/generator" "github.com/xephonhq/xephon-b/pkg/metrics" - "github.com/xephonhq/xephon-b/pkg/util/tsdbutil" ) type Worker struct { id int wcfg config.WorkloadConfig - dcfg config.DatabaseConfig - c libtsdb.TracedWriteClient + dcfg libtsdb.DatabaseConfig + c database.TracedWriteClient // generator sGen generator.SeriesGenerator @@ -35,13 +32,13 @@ type Worker struct { } func NewWorker(id int, - wcfg config.WorkloadConfig, dcfg config.DatabaseConfig, + wcfg config.WorkloadConfig, dcfg libtsdb.DatabaseConfig, resChan chan<- metrics.Response) (*Worker, error) { // check workload config if wcfg.Batch.Series <= 0 || wcfg.Batch.Points <= 0 { return nil, errors.Errorf("invalid batch series %d or points %d", wcfg.Batch.Series, wcfg.Batch.Points) } - c, err := tsdbutil.CreateClient(dcfg) + c, err := libtsdb.CreateClient(dcfg) if err != nil { return nil, err } @@ -87,7 +84,6 @@ func (w *Worker) Run(ctx context.Context) error { w.resChan <- w.c.Trace() } } - return nil } func (w *Worker) genBatch() { @@ -97,12 +93,12 @@ func (w *Worker) genBatch() { for j := 0; j < w.wcfg.Batch.Points; j++ { // FIXME: we hardcoded to use float, should allow mix them or at least pick one ... v := w.vGen.NextDouble() - p := pb.PointDoubleTagged{ + p := tspb.PointDoubleTagged{ Name: sMeta.Name, Tags: sMeta.Tags, - Point: pb.PointDouble{ - T: t, - V: v, + Point: tspb.PointDouble{ + Time: t, + Value: v, }, } // TODO: we should put many points in a series for tsdb that supports this, i.e. KairosDB, OpenTSDB, it is not supported by libtsdb yet diff --git a/pkg/util/logutil/pkg.go b/pkg/util/logutil/pkg.go deleted file mode 100644 index 6781b86..0000000 --- a/pkg/util/logutil/pkg.go +++ /dev/null @@ -1,22 +0,0 @@ -package logutil - -import ( - "github.com/dyweb/gommon/log" - gommonlog "github.com/dyweb/gommon/util/logutil" - libtsdblog "github.com/libtsdb/libtsdb-go/libtsdb/util/logutil" -) - -var Registry = log.NewLibraryLogger() - -func NewPackageLogger() *log.Logger { - l := log.NewPackageLoggerWithSkip(1) - Registry.AddChild(l) - return l -} - -func init() { - // gain control of important libraries, NOTE: there could be duplicate and cycle when various library is involved - // thus gommon/log would keep track of visited logger when doing recursive version of SetLevel and SetHandler - Registry.AddChild(gommonlog.Registry) - Registry.AddChild(libtsdblog.Registry) -} diff --git a/pkg/util/tsdbutil/client.go b/pkg/util/tsdbutil/client.go deleted file mode 100644 index 9221251..0000000 --- a/pkg/util/tsdbutil/client.go +++ /dev/null @@ -1,42 +0,0 @@ -package tsdbutil - -import ( - "github.com/dyweb/gommon/errors" - - "github.com/libtsdb/libtsdb-go/libtsdb" - "github.com/libtsdb/libtsdb-go/libtsdb/client/akumuliw" - "github.com/libtsdb/libtsdb-go/libtsdb/client/graphitew" - "github.com/libtsdb/libtsdb-go/libtsdb/client/influxdbw" - "github.com/libtsdb/libtsdb-go/libtsdb/client/kairosdbw" - "github.com/xephonhq/xephon-b/pkg/config" -) - -func CreateClient(cfg config.DatabaseConfig) (libtsdb.TracedWriteClient, error) { - switch cfg.Type { - case "akumuli": - if cfg.Akumuli == nil { - return nil, errors.New("akumuli is selected but no config") - } - return akumuliw.New(*cfg.Akumuli) - case "graphite": - if cfg.Graphite == nil { - return nil, errors.New("graphite is selected but no config") - } - return graphitew.New(*cfg.Graphite) - case "influxdb": - if cfg.Influxdb == nil { - return nil, errors.New("influxdb is selected but no config") - } - return influxdbw.New(*cfg.Influxdb) - case "kairosdb": - if cfg.Kairosdb == nil { - return nil, errors.New("kairosdb is selected but no config") - } - if cfg.Kairosdb.Telnet { - return kairosdbw.NewTcp(*cfg.Kairosdb) - } - return kairosdbw.NewHttp(*cfg.Kairosdb) - default: - return nil, errors.Errorf("unknown database %s", cfg.Type) - } -} diff --git a/script/influxdb/novolume.sh b/script/influxdb/novolume.sh index 7d43b22..5126353 100755 --- a/script/influxdb/novolume.sh +++ b/script/influxdb/novolume.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -docker run -p 8086:8086 influxdb:1.3.9 \ No newline at end of file +docker run -p 8086:8086 influxdb:1.7.9 \ No newline at end of file