diff --git a/Makefile b/Makefile index 35beb727f..1f87202ab 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,9 @@ GOTEST := CGO_ENABLED=1 $(GO) test -p 3 PACKAGES := $$(go list ./... | grep -vE 'vendor') FILES := $$(find . -name '*.go' -type f | grep -vE 'vendor') VENDOR_TIDB := vendor/github.com/pingcap/tidb - +PACKAGE_LIST := go list ./... +PACKAGES := $$($(PACKAGE_LIST)) +FAIL_ON_STDOUT := awk '{ print } END { if (NR > 0) { exit 1 } }' build: prepare check importer checker dump_region binlogctl sync_diff_inspector ddl_checker finish @@ -58,9 +60,8 @@ fmt: check: #go get github.com/golang/lint/golint @echo "vet" - @ go tool vet $(FILES) 2>&1 | awk '{print} END{if(NR>0) {exit 1}}' - @echo "vet --shadow" - @ go tool vet --shadow $(FILES) 2>&1 | awk '{print} END{if(NR>0) {exit 1}}' + @$(GO) vet -composites=false $(PACKAGES) + @$(GO) vet -vettool=$(CURDIR)/bin/shadow $(PACKAGES) || true #@echo "golint" #@ golint ./... 2>&1 | grep -vE '\.pb\.go' | grep -vE 'vendor' | awk '{print} END{if(NR>0) {exit 1}}' @echo "gofmt (simplify)" diff --git a/go.mod1 b/go.mod1 index d267ec815..dd076978b 100644 --- a/go.mod1 +++ b/go.mod1 @@ -2,6 +2,7 @@ module github.com/pingcap/tidb-tools require ( github.com/BurntSushi/toml v0.3.1 + github.com/DATA-DOG/go-sqlmock v1.3.3 github.com/Shopify/sarama v1.19.0 github.com/Shopify/toxiproxy v2.1.3+incompatible // indirect github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect diff --git a/go.sum1 b/go.sum1 index ea7dea73f..b459c82d7 100644 --- a/go.sum1 +++ b/go.sum1 @@ -1,6 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08= +github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/Shopify/sarama v1.19.0 h1:9oksLxC6uxVPHPVYUmq6xhr1BOF/hHobWH2UzO67z1s= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.3+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= @@ -11,7 +13,7 @@ github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/bbolt v1.3.1-coreos.6 h1:uTXKg9gY70s9jMAKdfljFQcuh4e/BXOM+V+d00KFj3A= github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible h1:KjVWqrZ5U0wa3CxY2AxlH6/UcB+PK2td1DcsYhA+HRs= +github.com/coreos/etcd v3.3.10+incompatible h1:jFneRYjIvLMLhDLCzuTuU4rSJUjRplcJQ7pD7MnhC04= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -43,7 +45,6 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff h1:kOkM9whyQYodu09SJ6W3NCsHG7crFaJILQ22Gozp3lg= github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -57,7 +58,7 @@ github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v1.2.0 h1:VJtLvh6VQym50czpZzx07z/kw9EgAxI3x1ZB8taTMQQ= github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:BWIsLfhgKhV5g/oF34aRjniBHLTZe5DNekSjbAjIS6c= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= @@ -69,7 +70,6 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e h1:+lIPJOWl+jSiJOc70QXJ07+2eg2Jy2EC7Mi11BWujeM= github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v0.0.0-20180402223658-b729f2633dfe/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -91,8 +91,6 @@ github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pingcap/check v0.0.0-20171206051426-1c287c953996 h1:ZBdiJCMan6GSo/aPAM7gywcUKa0z58gczVrnG6TQnAQ= github.com/pingcap/check v0.0.0-20171206051426-1c287c953996/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= -github.com/pingcap/errors v0.0.0-20181012004132-a4583d0a56ea h1:g2k+8WR7cHch4g0tBDhfiEvAp7fXxTNBiD1oC1Oxj3E= -github.com/pingcap/errors v0.0.0-20181012004132-a4583d0a56ea/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= github.com/pingcap/errors v0.11.0 h1:DCJQB8jrHbQ1VVlMFIrbj2ApScNNotVmkSNplu2yUt4= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e h1:P73/4dPCL96rGrobssy1nVy2VaVpNCuLpCbr+FEaTA8= @@ -103,17 +101,8 @@ github.com/pingcap/parser v0.0.0-20181024082006-53ac409ed043 h1:P9Osi8lei5j2fiRg github.com/pingcap/parser v0.0.0-20181024082006-53ac409ed043/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v2.1.0-rc.4+incompatible h1:/buwGk04aHO5odk/+O8ZOXGs4qkUjYTJ2UpCJXna8NE= github.com/pingcap/pd v2.1.0-rc.4+incompatible/go.mod h1:nD3+EoYes4+aNNODO99ES59V83MZSI+dFbhyr667a0E= -github.com/pingcap/tidb v0.0.0-20181101063854-2ccb08f7b3f4 h1:mjvCmA21qVEmXCEg7XnWs0if9RQ93MEj4m6wj3tEOoM= -github.com/pingcap/tidb v0.0.0-20181101063854-2ccb08f7b3f4/go.mod h1:I8C6jrPINP2rrVunTRd7C9fRRhQrtR43S1/CL5ix/yQ= -github.com/pingcap/tidb v2.0.8+incompatible h1:4G85C71eFTQRJ0Icwul/z3gJfR0u0aWXq1t/f4O8R40= -github.com/pingcap/tidb v2.0.8+incompatible/go.mod h1:I8C6jrPINP2rrVunTRd7C9fRRhQrtR43S1/CL5ix/yQ= -github.com/pingcap/tidb v2.1.0-rc.3.0.20181031081318-c23a30c1bb53+incompatible h1:SYX09K6nRZ/i5+fztEwmbmrZU0/BfiWcx81rTKhwuG4= -github.com/pingcap/tidb v2.1.0-rc.3.0.20181031081318-c23a30c1bb53+incompatible/go.mod h1:I8C6jrPINP2rrVunTRd7C9fRRhQrtR43S1/CL5ix/yQ= -github.com/pingcap/tidb v2.1.0-rc.3.0.20181101063854-2ccb08f7b3f4+incompatible h1:ydHc4nc7/SS4HanVuOzbdm5ucZ15m7R0iJExrRkwVsA= -github.com/pingcap/tidb v2.1.0-rc.3.0.20181101063854-2ccb08f7b3f4+incompatible/go.mod h1:I8C6jrPINP2rrVunTRd7C9fRRhQrtR43S1/CL5ix/yQ= github.com/pingcap/tidb v2.1.0-rc.3.0.20181101080907-32b1dbd8d59f+incompatible h1:CIZHlDunFGiLzenh8z5Rh05JgrK6DW2f5/nTRGiyF6I= github.com/pingcap/tidb v2.1.0-rc.3.0.20181101080907-32b1dbd8d59f+incompatible/go.mod h1:I8C6jrPINP2rrVunTRd7C9fRRhQrtR43S1/CL5ix/yQ= -github.com/pingcap/tidb-tools v0.0.0-20181101060435-7083cb45066d/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tipb v0.0.0-20181012112600-11e33c750323 h1:mRKKzRjDNaUNPnAkPAHnRqpNmwNWBX1iA+hxlmvQ93I= github.com/pingcap/tipb v0.0.0-20181012112600-11e33c750323/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= @@ -179,7 +168,6 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2I golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.2.0 h1:S0iUepdCWODXRvtE+gcRDd15L+k+k1AiHlMiMjefH24= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2 h1:67iHsV9djwGdZpdZNbLuQj6FOzCaZe3w+vhLjn5AcFA= diff --git a/pkg/dbutil/common.go b/pkg/dbutil/common.go index ca350235c..d6a513c25 100644 --- a/pkg/dbutil/common.go +++ b/pkg/dbutil/common.go @@ -156,6 +156,7 @@ func GetRowCount(ctx context.Context, db *sql.DB, schemaName string, tableName s if len(where) > 0 { query += fmt.Sprintf(" WHERE %s", where) } + log.Debugf("get row count by sql %s", query) var cnt sql.NullInt64 err := db.QueryRowContext(ctx, query).Scan(&cnt) diff --git a/pkg/diff/chunk.go b/pkg/diff/chunk.go index 431a53e5d..50475cca5 100644 --- a/pkg/diff/chunk.go +++ b/pkg/diff/chunk.go @@ -258,7 +258,6 @@ func (s *randomSpliter) splitRange(db *sql.DB, chunk *chunkRange, count int, sch symbolMin = gte symbolMax = lte - } splitValues := make([]string, 0, count) @@ -269,7 +268,7 @@ func (s *randomSpliter) splitRange(db *sql.DB, chunk *chunkRange, count int, sch if err != nil { return nil, errors.Trace(err) } - log.Infof("split chunk %v, get split values from GetRandomValues: %v", chunk, randomValues) + log.Debugf("split chunk %v, get split values from GetRandomValues: %v", chunk, randomValues) /* for examples: @@ -319,8 +318,17 @@ func (s *randomSpliter) splitRange(db *sql.DB, chunk *chunkRange, count int, sch the splitValues is [1, 1, 1, 1]; the chunk [`a` = 2] will split to [`a` = 2 AND `b` < 'x'], [`a` = 2 AND `b` >= 'x' AND `b` < 'y'] and [`a` = 2 AND `b` >= 'y'] */ - var lower, upper, lowerSymbol, upperSymbol string + + lowerSymbol := symbolMin + upperSymbol := lt + for i := 0; i < len(splitValues); i++ { + if i == 0 && useNewColumn { + // create chunk less than min + newChunk := chunk.copyAndUpdate(splitCol, "", "", splitValues[i], lt) + chunks = append(chunks, newChunk) + } + if valueCounts[i] > 1 { // means should split it newChunk := chunk.copyAndUpdate(splitCol, splitValues[i], equal, "", "") @@ -332,45 +340,28 @@ func (s *randomSpliter) splitRange(db *sql.DB, chunk *chunkRange, count int, sch // already have the chunk [column = value], so next chunk should start with column > value lowerSymbol = gt - } else { - if i == 0 { - if useNewColumn { - lower = "" - lowerSymbol = "" - } else { - lower = splitValues[i] - lowerSymbol = symbolMin - } - } else { - lower = splitValues[i] - } - - if i == len(splitValues)-2 { - if useNewColumn && valueCounts[len(valueCounts)-1] == 1 { - upper = "" - upperSymbol = "" - } else { - upper = splitValues[i+1] - upperSymbol = symbolMax - } - } else { - if i == len(splitValues)-1 { - continue - } + } - upper = splitValues[i+1] - upperSymbol = lt + if i == len(splitValues)-2 && valueCounts[i+1] == 1 { + upperSymbol = symbolMax + } - } + if i < len(splitValues)-1 { + newChunk := chunk.copyAndUpdate(splitCol, splitValues[i], lowerSymbol, splitValues[i+1], upperSymbol) + chunks = append(chunks, newChunk) } - newChunk := chunk.copyAndUpdate(splitCol, lower, lowerSymbol, upper, upperSymbol) - chunks = append(chunks, newChunk) + if i == len(splitValues)-1 && useNewColumn { + // create chunk greater than max + newChunk := chunk.copyAndUpdate(splitCol, splitValues[i], gt, "", "") + chunks = append(chunks, newChunk) + } lowerSymbol = gte } log.Debugf("getChunksForTable cut table: cnt=%d min=%s max=%s chunk=%d", count, min, max, len(chunks)) + return chunks, nil } diff --git a/pkg/diff/chunk_test.go b/pkg/diff/chunk_test.go index a43f41a5d..76cae680e 100644 --- a/pkg/diff/chunk_test.go +++ b/pkg/diff/chunk_test.go @@ -200,5 +200,4 @@ func (*testChunkSuite) TestChunkToString(c *C) { for i, arg := range args { c.Assert(arg, Equals, expectArgs[i]) } - } diff --git a/pkg/diff/diff_test.go b/pkg/diff/diff_test.go index aab94f583..f2ffa4fbf 100644 --- a/pkg/diff/diff_test.go +++ b/pkg/diff/diff_test.go @@ -39,11 +39,11 @@ func (*testDiffSuite) TestGenerateSQLs(c *C) { c.Assert(err, IsNil) rowsData := map[string]*dbutil.ColumnData{ - "id": {[]byte("1"), false}, - "name": {[]byte("xxx"), false}, - "birthday": {[]byte("2018-01-01 00:00:00"), false}, - "update_time": {[]byte("10:10:10"), false}, - "money": {[]byte("11.1111"), false}, + "id": {Data: []byte("1"), IsNull: false}, + "name": {Data: []byte("xxx"), IsNull: false}, + "birthday": {Data: []byte("2018-01-01 00:00:00"), IsNull: false}, + "update_time": {Data: []byte("10:10:10"), IsNull: false}, + "money": {Data: []byte("11.1111"), IsNull: false}, } _, orderKeyCols := dbutil.SelectUniqueOrderKey(tableInfo) @@ -63,13 +63,13 @@ func (*testDiffSuite) TestGenerateSQLs(c *C) { c.Assert(deleteSQL, Equals, "DELETE FROM `test`.`atest` WHERE `id` = 1 AND `name` = 'xxx';") // test value is nil - rowsData["name"] = &dbutil.ColumnData{[]byte(""), true} + rowsData["name"] = &dbutil.ColumnData{Data: []byte(""), IsNull: true} replaceSQL = generateDML("replace", rowsData, orderKeyCols, tableInfo, "test") deleteSQL = generateDML("delete", rowsData, orderKeyCols, tableInfo, "test") c.Assert(replaceSQL, Equals, "REPLACE INTO `test`.`atest`(`id`,`name`,`birthday`,`update_time`,`money`) VALUES (1,NULL,'2018-01-01 00:00:00','10:10:10',11.1111);") c.Assert(deleteSQL, Equals, "DELETE FROM `test`.`atest` WHERE `id` = 1;") - rowsData["id"] = &dbutil.ColumnData{[]byte(""), true} + rowsData["id"] = &dbutil.ColumnData{Data: []byte(""), IsNull: true} replaceSQL = generateDML("replace", rowsData, orderKeyCols, tableInfo, "test") deleteSQL = generateDML("delete", rowsData, orderKeyCols, tableInfo, "test") c.Assert(replaceSQL, Equals, "REPLACE INTO `test`.`atest`(`id`,`name`,`birthday`,`update_time`,`money`) VALUES (NULL,NULL,'2018-01-01 00:00:00','10:10:10',11.1111);") diff --git a/pkg/diff/merge_test.go b/pkg/diff/merge_test.go index a7006a44a..4f6391be6 100644 --- a/pkg/diff/merge_test.go +++ b/pkg/diff/merge_test.go @@ -46,9 +46,9 @@ func (s *testMergerSuite) TestMerge(c *C) { heap.Init(rowDatas) for i, id := range ids { data := map[string]*dbutil.ColumnData{ - "id": {[]byte(strconv.Itoa(id)), false}, - "name": {[]byte(names[i]), false}, - "age": {[]byte(strconv.Itoa(ages[i])), false}, + "id": {Data: []byte(strconv.Itoa(id)), IsNull: false}, + "name": {Data: []byte(names[i]), IsNull: false}, + "age": {Data: []byte(strconv.Itoa(ages[i])), IsNull: false}, } heap.Push(rowDatas, RowData{ Data: data, diff --git a/pkg/diff/spliter_test.go b/pkg/diff/spliter_test.go new file mode 100644 index 000000000..21dc28f7d --- /dev/null +++ b/pkg/diff/spliter_test.go @@ -0,0 +1,319 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package diff + +import ( + "fmt" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + . "github.com/pingcap/check" + "github.com/pingcap/tidb-tools/pkg/dbutil" +) + +var _ = Suite(&testSpliterSuite{}) + +type testSpliterSuite struct{} + +func (s *testSpliterSuite) TestRandomSpliter(c *C) { + db, mock, err := sqlmock.New() + c.Assert(err, IsNil) + + createTableSQL := "create table `test`.`test`(`a` int, `b` varchar(10), `c` float, `d` datetime, primary key(`a`, `b`))" + tableInfo, err := dbutil.GetTableInfoBySQL(createTableSQL) + c.Assert(err, IsNil) + + tableInstance := &TableInstance{ + Conn: db, + Schema: "test", + Table: "test", + info: tableInfo, + } + + testCases := []struct { + count int + aMin int + aMax int + bMin string + bMax string + aRandomValues []int + bRandomValues []string + aRandomValueCounts []int + }{ + // only use column a + {10, 0, 10, "x", "z", []int{1, 2, 3, 4}, nil, []int{1, 1, 1, 1}}, + // will use column b + {10, 0, 10, "x", "z", []int{1, 2, 3, 4}, []string{"y"}, []int{1, 2, 1, 1}}, + // will split the max value by column b + {10, 0, 10, "x", "z", []int{1, 2, 3, 10}, []string{"y"}, []int{1, 1, 1, 1}}, + // will split the min value by column b + {10, 0, 10, "x", "z", []int{0, 2, 3, 4}, []string{"y"}, []int{1, 1, 1, 1}}, + // will split the min value and max value by column b + {4, 0, 10, "x", "z", []int{0, 10}, []string{"y"}, []int{1, 1}}, + // will split all values by column b + {4, 0, 10, "x", "z", []int{0, 5, 10}, []string{"y"}, []int{1, 2, 1}}, + } + + expectResult := [][]struct { + chunkStr string + args []string + }{ + { + {"`a` < ?", []string{"0"}}, + {"`a` >= ? AND `a` < ?", []string{"0", "1"}}, + {"`a` >= ? AND `a` < ?", []string{"1", "2"}}, + {"`a` >= ? AND `a` < ?", []string{"2", "3"}}, + {"`a` >= ? AND `a` < ?", []string{"3", "4"}}, + {"`a` >= ? AND `a` <= ?", []string{"4", "10"}}, + {"`a` > ?", []string{"10"}}, + }, + { + {"`a` < ?", []string{"0"}}, + {"`a` >= ? AND `a` < ?", []string{"0", "1"}}, + {"`a` >= ? AND `a` < ?", []string{"1", "2"}}, + {"`a` = ? AND `b` < ?", []string{"2", "x"}}, + {"`a` = ? AND `b` >= ? AND `b` < ?", []string{"2", "x", "y"}}, + {"`a` = ? AND `b` >= ? AND `b` <= ?", []string{"2", "y", "z"}}, + {"`a` = ? AND `b` > ?", []string{"2", "z"}}, + {"`a` > ? AND `a` < ?", []string{"2", "3"}}, + {"`a` >= ? AND `a` < ?", []string{"3", "4"}}, + {"`a` >= ? AND `a` <= ?", []string{"4", "10"}}, + {"`a` > ?", []string{"10"}}, + }, + { + {"`a` < ?", []string{"0"}}, + {"`a` >= ? AND `a` < ?", []string{"0", "1"}}, + {"`a` >= ? AND `a` < ?", []string{"1", "2"}}, + {"`a` >= ? AND `a` < ?", []string{"2", "3"}}, + {"`a` >= ? AND `a` < ?", []string{"3", "10"}}, + {"`a` = ? AND `b` < ?", []string{"10", "x"}}, + {"`a` = ? AND `b` >= ? AND `b` < ?", []string{"10", "x", "y"}}, + {"`a` = ? AND `b` >= ? AND `b` <= ?", []string{"10", "y", "z"}}, + {"`a` = ? AND `b` > ?", []string{"10", "z"}}, + {"`a` > ?", []string{"10"}}, + }, + { + {"`a` < ?", []string{"0"}}, + {"`a` = ? AND `b` < ?", []string{"0", "x"}}, + {"`a` = ? AND `b` >= ? AND `b` < ?", []string{"0", "x", "y"}}, + {"`a` = ? AND `b` >= ? AND `b` <= ?", []string{"0", "y", "z"}}, + {"`a` = ? AND `b` > ?", []string{"0", "z"}}, + {"`a` > ? AND `a` < ?", []string{"0", "2"}}, + {"`a` >= ? AND `a` < ?", []string{"2", "3"}}, + {"`a` >= ? AND `a` < ?", []string{"3", "4"}}, + {"`a` >= ? AND `a` <= ?", []string{"4", "10"}}, + {"`a` > ?", []string{"10"}}, + }, + { + {"`a` < ?", []string{"0"}}, + {"`a` = ? AND `b` < ?", []string{"0", "x"}}, + {"`a` = ? AND `b` >= ? AND `b` < ?", []string{"0", "x", "y"}}, + {"`a` = ? AND `b` >= ? AND `b` <= ?", []string{"0", "y", "z"}}, + {"`a` = ? AND `b` > ?", []string{"0", "z"}}, + {"`a` > ? AND `a` < ?", []string{"0", "10"}}, + {"`a` = ? AND `b` < ?", []string{"10", "x"}}, + {"`a` = ? AND `b` >= ? AND `b` < ?", []string{"10", "x", "y"}}, + {"`a` = ? AND `b` >= ? AND `b` <= ?", []string{"10", "y", "z"}}, + {"`a` = ? AND `b` > ?", []string{"10", "z"}}, + {"`a` > ?", []string{"10"}}, + }, + { + {"`a` < ?", []string{"0"}}, + {"`a` = ? AND `b` < ?", []string{"0", "x"}}, + {"`a` = ? AND `b` >= ? AND `b` < ?", []string{"0", "x", "y"}}, + {"`a` = ? AND `b` >= ? AND `b` <= ?", []string{"0", "y", "z"}}, + {"`a` = ? AND `b` > ?", []string{"0", "z"}}, + {"`a` > ? AND `a` < ?", []string{"0", "5"}}, + {"`a` = ? AND `b` < ?", []string{"5", "x"}}, + {"`a` = ? AND `b` >= ? AND `b` < ?", []string{"5", "x", "y"}}, + {"`a` = ? AND `b` >= ? AND `b` <= ?", []string{"5", "y", "z"}}, + {"`a` = ? AND `b` > ?", []string{"5", "z"}}, + {"`a` > ? AND `a` < ?", []string{"5", "10"}}, + {"`a` = ? AND `b` < ?", []string{"10", "x"}}, + {"`a` = ? AND `b` >= ? AND `b` < ?", []string{"10", "x", "y"}}, + {"`a` = ? AND `b` >= ? AND `b` <= ?", []string{"10", "y", "z"}}, + {"`a` = ? AND `b` > ?", []string{"10", "z"}}, + {"`a` > ?", []string{"10"}}, + }, + } + + for i, t := range testCases { + createFakeResultForRandomSplit(mock, t.count, t.aMin, t.aMax, t.bMin, t.bMax, t.aRandomValues, t.bRandomValues, t.aRandomValueCounts) + + rSpliter := new(randomSpliter) + chunks, err := rSpliter.split(tableInstance, tableInfo.Columns, 2, "TRUE", "") + c.Assert(err, IsNil) + + for j, chunk := range chunks { + chunkStr, args := chunk.toString(normalMode, "") + c.Assert(chunkStr, Equals, expectResult[i][j].chunkStr) + c.Assert(args, DeepEquals, expectResult[i][j].args) + } + } + + // test case for split a range use same column + // split (0, 10) to (0, 5) and [5, 10) + randomRows := sqlmock.NewRows([]string{"a", "count"}).AddRow("5", 1) + mock.ExpectQuery("ORDER BY RAND()").WillReturnRows(randomRows) + + expectChunks := []struct { + chunkStr string + args []string + }{ + {"`a` > ? AND `a` < ?", []string{"0", "5"}}, + {"`a` >= ? AND `a` < ?", []string{"5", "10"}}, + } + + r := &randomSpliter{ + table: tableInstance, + } + + oriChunk := newChunkRange().copyAndUpdate("a", "0", gt, "10", lt) + chunks, err := r.splitRange(db, oriChunk, 2, "test", "test", tableInfo.Columns) + c.Assert(err, IsNil) + for i, chunk := range chunks { + chunkStr, args := chunk.toString(normalMode, "") + c.Assert(chunkStr, Equals, expectChunks[i].chunkStr) + c.Assert(args, DeepEquals, expectChunks[i].args) + } +} + +func createFakeResultForRandomSplit(mock sqlmock.Sqlmock, count, aMin, aMax int, bMin, bMax string, aRandomValues []int, bRandomValues []string, aRandomValueCounts []int) { + // generate fake result for get the row count of this table + countRows := sqlmock.NewRows([]string{"cnt"}).AddRow(count) + mock.ExpectQuery("SELECT COUNT.*").WillReturnRows(countRows) + + // generate fake result for get min and max value for column a + aMinMaxRows := sqlmock.NewRows([]string{"MIN", "MAX"}).AddRow(aMin, aMax) + mock.ExpectQuery("SELECT .* MIN(.+a.+) as MIN, .*").WillReturnRows(aMinMaxRows) + + // generate fake result for get random value for column a + aRandomRows := sqlmock.NewRows([]string{"a", "count"}) + for i, randomValue := range aRandomValues { + aRandomRows.AddRow(randomValue, aRandomValueCounts[i]) + } + mock.ExpectQuery("ORDER BY RAND()").WillReturnRows(aRandomRows) + + if len(bRandomValues) == 0 { + return + } + + num := 0 + splitValues := make(map[int]int) + for i, value := range aRandomValues { + splitValues[value] = aRandomValueCounts[i] + } + splitValues[aMin]++ + splitValues[aMax]++ + for _, count := range splitValues { + if count > 1 { + num += (count - 1) + } + } + + // means need split more num times for column b + for i := 0; i < num; i++ { + // generate fake result for get min and max value for column b + bMinMaxRows := sqlmock.NewRows([]string{"MIN", "MAX"}).AddRow(bMin, bMax) + mock.ExpectQuery("SELECT .* MIN(.+b.+) as MIN, .*").WillReturnRows(bMinMaxRows) + + // generate fake result for get random value for column b + bRandomRows := sqlmock.NewRows([]string{"b", "count"}) + for _, randomValue := range bRandomValues { + bRandomRows.AddRow(randomValue, 1) + } + mock.ExpectQuery("SELECT b, COUNT.*").WillReturnRows(bRandomRows) + } + + return +} + +func (s *testSpliterSuite) TestBucketSpliter(c *C) { + db, mock, err := sqlmock.New() + c.Assert(err, IsNil) + + createTableSQL := "create table `test`.`test`(`a` int, `b` varchar(10), `c` float, `d` datetime, primary key(`a`, `b`))" + tableInfo, err := dbutil.GetTableInfoBySQL(createTableSQL) + c.Assert(err, IsNil) + + chunkSizes := []int{2, 64, 127, 128} + expectResult := [][]struct { + chunkStr string + args []string + }{ + { + {"(`a` <= ?) OR (`a` = ? AND `b` <= ?)", []string{"63", "63", "11"}}, + {"((`a` > ?) OR (`a` = ? AND `b` > ?)) AND ((`a` <= ?) OR (`a` = ? AND `b` <= ?))", []string{"63", "63", "11", "127", "127", "23"}}, + {"((`a` > ?) OR (`a` = ? AND `b` > ?)) AND ((`a` <= ?) OR (`a` = ? AND `b` <= ?))", []string{"127", "127", "23", "191", "191", "35"}}, + {"((`a` > ?) OR (`a` = ? AND `b` > ?)) AND ((`a` <= ?) OR (`a` = ? AND `b` <= ?))", []string{"191", "191", "35", "255", "255", "47"}}, + {"(`a` > ?) OR (`a` = ? AND `b` > ?)", []string{"255", "255", "47"}}, + }, + { + {"(`a` <= ?) OR (`a` = ? AND `b` <= ?)", []string{"127", "127", "23"}}, + {"((`a` > ?) OR (`a` = ? AND `b` > ?)) AND ((`a` <= ?) OR (`a` = ? AND `b` <= ?))", []string{"127", "127", "23", "255", "255", "47"}}, + {"(`a` > ?) OR (`a` = ? AND `b` > ?)", []string{"255", "255", "47"}}, + }, + { + {"(`a` <= ?) OR (`a` = ? AND `b` <= ?)", []string{"127", "127", "23"}}, + {"((`a` > ?) OR (`a` = ? AND `b` > ?)) AND ((`a` <= ?) OR (`a` = ? AND `b` <= ?))", []string{"127", "127", "23", "255", "255", "47"}}, + {"(`a` > ?) OR (`a` = ? AND `b` > ?)", []string{"255", "255", "47"}}, + }, + { + {"(`a` <= ?) OR (`a` = ? AND `b` <= ?)", []string{"191", "191", "35"}}, + {"(`a` > ?) OR (`a` = ? AND `b` > ?)", []string{"191", "191", "35"}}, + }, + } + + tableInstance := &TableInstance{ + Conn: db, + Schema: "test", + Table: "test", + info: tableInfo, + } + + for i, chunkSize := range chunkSizes { + fmt.Println(i) + createFakeResultForBucketSplit(mock) + bSpliter := new(bucketSpliter) + chunks, err := bSpliter.split(tableInstance, tableInfo.Columns, chunkSize, "TRUE", "") + c.Assert(err, IsNil) + + for j, chunk := range chunks { + chunkStr, args := chunk.toString(bucketMode, "") + c.Assert(chunkStr, Equals, expectResult[i][j].chunkStr) + c.Assert(args, DeepEquals, expectResult[i][j].args) + } + } + +} + +func createFakeResultForBucketSplit(mock sqlmock.Sqlmock) { + /* + +---------+------------+-------------+----------+-----------+-------+---------+-------------+-------------+ + | Db_name | Table_name | Column_name | Is_index | Bucket_id | Count | Repeats | Lower_Bound | Upper_Bound | + +---------+------------+-------------+----------+-----------+-------+---------+-------------+-------------+ + | test | test | PRIMARY | 1 | 0 | 64 | 1 | (0, 0) | (63, 11) | + | test | test | PRIMARY | 1 | 1 | 128 | 1 | (64, 12) | (127, 23) | + | test | test | PRIMARY | 1 | 2 | 192 | 1 | (128, 24) | (191, 35) | + | test | test | PRIMARY | 1 | 3 | 256 | 1 | (192, 36) | (255, 47) | + | test | test | PRIMARY | 1 | 4 | 320 | 1 | (256, 48) | (319, 59) | + */ + + statsRows := sqlmock.NewRows([]string{"Db_name", "Table_name", "Column_name", "Is_index", "Bucket_id", "Count", "Repeats", "Lower_Bound", "Upper_Bound"}) + for i := 0; i < 5; i++ { + statsRows.AddRow("test", "test", "PRIMARY", 1, (i+1)*64, (i+1)*64, 1, fmt.Sprintf("(%d, %d)", i*64, i*12), fmt.Sprintf("(%d, %d)", (i+1)*64-1, (i+1)*12-1)) + } + mock.ExpectQuery("SHOW STATS_BUCKETS").WillReturnRows(statsRows) + + return +}