diff: add unit test for chunk split (#209) (#216)

* diff: add unit test for chunk split (#209)

Makefile

@@ -21,7 +21,9 @@ GOTEST   := CGO_ENABLED=1 $(GO) test -p 3
 PACKAGES := $$(go list ./... | grep -vE 'vendor')
 FILES     := $$(find . -name '*.go' -type f | grep -vE 'vendor')
 VENDOR_TIDB := vendor/github.com/pingcap/tidb
-
+PACKAGE_LIST  := go list ./...
+PACKAGES  := $$($(PACKAGE_LIST))
+FAIL_ON_STDOUT := awk '{ print } END { if (NR > 0) { exit 1 } }'
 
 build: prepare check importer checker dump_region binlogctl sync_diff_inspector ddl_checker finish
 
@@ -58,9 +60,8 @@ fmt:
 check:
 	#go get github.com/golang/lint/golint
 	@echo "vet"
-	@ go tool vet $(FILES) 2>&1 | awk '{print} END{if(NR>0) {exit 1}}'
-	@echo "vet --shadow"
-	@ go tool vet --shadow $(FILES) 2>&1 | awk '{print} END{if(NR>0) {exit 1}}'
+	@$(GO) vet -composites=false $(PACKAGES)
+	@$(GO) vet -vettool=$(CURDIR)/bin/shadow $(PACKAGES) || true
 	#@echo "golint"
 	#@ golint ./... 2>&1 | grep -vE '\.pb\.go' | grep -vE 'vendor' | awk '{print} END{if(NR>0) {exit 1}}'
 	@echo "gofmt (simplify)"

go.mod1

@@ -2,6 +2,7 @@ module github.com/pingcap/tidb-tools
 
 require (
 	github.com/BurntSushi/toml v0.3.1
+	github.com/DATA-DOG/go-sqlmock v1.3.3
 	github.com/Shopify/sarama v1.19.0
 	github.com/Shopify/toxiproxy v2.1.3+incompatible // indirect
 	github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect

go.sum1

@@ -1,6 +1,8 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08=
+github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
 github.com/Shopify/sarama v1.19.0 h1:9oksLxC6uxVPHPVYUmq6xhr1BOF/hHobWH2UzO67z1s=
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
 github.com/Shopify/toxiproxy v2.1.3+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
@@ -11,7 +13,7 @@ github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/coreos/bbolt v1.3.1-coreos.6 h1:uTXKg9gY70s9jMAKdfljFQcuh4e/BXOM+V+d00KFj3A=
 github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
-github.com/coreos/etcd v3.3.10+incompatible h1:KjVWqrZ5U0wa3CxY2AxlH6/UcB+PK2td1DcsYhA+HRs=
+github.com/coreos/etcd v3.3.10+incompatible h1:jFneRYjIvLMLhDLCzuTuU4rSJUjRplcJQ7pD7MnhC04=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
@@ -43,7 +45,6 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG
 github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff h1:kOkM9whyQYodu09SJ6W3NCsHG7crFaJILQ22Gozp3lg=
 github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -57,7 +58,7 @@ github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51
 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/websocket v1.2.0 h1:VJtLvh6VQym50czpZzx07z/kw9EgAxI3x1ZB8taTMQQ=
 github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:BWIsLfhgKhV5g/oF34aRjniBHLTZe5DNekSjbAjIS6c=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
@@ -69,7 +70,6 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
 github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e h1:+lIPJOWl+jSiJOc70QXJ07+2eg2Jy2EC7Mi11BWujeM=
 github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/konsorten/go-windows-terminal-sequences v0.0.0-20180402223658-b729f2633dfe/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -91,8 +91,6 @@ github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pingcap/check v0.0.0-20171206051426-1c287c953996 h1:ZBdiJCMan6GSo/aPAM7gywcUKa0z58gczVrnG6TQnAQ=
 github.com/pingcap/check v0.0.0-20171206051426-1c287c953996/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ=
-github.com/pingcap/errors v0.0.0-20181012004132-a4583d0a56ea h1:g2k+8WR7cHch4g0tBDhfiEvAp7fXxTNBiD1oC1Oxj3E=
-github.com/pingcap/errors v0.0.0-20181012004132-a4583d0a56ea/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q=
 github.com/pingcap/errors v0.11.0 h1:DCJQB8jrHbQ1VVlMFIrbj2ApScNNotVmkSNplu2yUt4=
 github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
 github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e h1:P73/4dPCL96rGrobssy1nVy2VaVpNCuLpCbr+FEaTA8=
@@ -103,17 +101,8 @@ github.com/pingcap/parser v0.0.0-20181024082006-53ac409ed043 h1:P9Osi8lei5j2fiRg
 github.com/pingcap/parser v0.0.0-20181024082006-53ac409ed043/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA=
 github.com/pingcap/pd v2.1.0-rc.4+incompatible h1:/buwGk04aHO5odk/+O8ZOXGs4qkUjYTJ2UpCJXna8NE=
 github.com/pingcap/pd v2.1.0-rc.4+incompatible/go.mod h1:nD3+EoYes4+aNNODO99ES59V83MZSI+dFbhyr667a0E=
-github.com/pingcap/tidb v0.0.0-20181101063854-2ccb08f7b3f4 h1:mjvCmA21qVEmXCEg7XnWs0if9RQ93MEj4m6wj3tEOoM=
-github.com/pingcap/tidb v0.0.0-20181101063854-2ccb08f7b3f4/go.mod h1:I8C6jrPINP2rrVunTRd7C9fRRhQrtR43S1/CL5ix/yQ=
-github.com/pingcap/tidb v2.0.8+incompatible h1:4G85C71eFTQRJ0Icwul/z3gJfR0u0aWXq1t/f4O8R40=
-github.com/pingcap/tidb v2.0.8+incompatible/go.mod h1:I8C6jrPINP2rrVunTRd7C9fRRhQrtR43S1/CL5ix/yQ=
-github.com/pingcap/tidb v2.1.0-rc.3.0.20181031081318-c23a30c1bb53+incompatible h1:SYX09K6nRZ/i5+fztEwmbmrZU0/BfiWcx81rTKhwuG4=
-github.com/pingcap/tidb v2.1.0-rc.3.0.20181031081318-c23a30c1bb53+incompatible/go.mod h1:I8C6jrPINP2rrVunTRd7C9fRRhQrtR43S1/CL5ix/yQ=
-github.com/pingcap/tidb v2.1.0-rc.3.0.20181101063854-2ccb08f7b3f4+incompatible h1:ydHc4nc7/SS4HanVuOzbdm5ucZ15m7R0iJExrRkwVsA=
-github.com/pingcap/tidb v2.1.0-rc.3.0.20181101063854-2ccb08f7b3f4+incompatible/go.mod h1:I8C6jrPINP2rrVunTRd7C9fRRhQrtR43S1/CL5ix/yQ=
 github.com/pingcap/tidb v2.1.0-rc.3.0.20181101080907-32b1dbd8d59f+incompatible h1:CIZHlDunFGiLzenh8z5Rh05JgrK6DW2f5/nTRGiyF6I=
 github.com/pingcap/tidb v2.1.0-rc.3.0.20181101080907-32b1dbd8d59f+incompatible/go.mod h1:I8C6jrPINP2rrVunTRd7C9fRRhQrtR43S1/CL5ix/yQ=
-github.com/pingcap/tidb-tools v0.0.0-20181101060435-7083cb45066d/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
 github.com/pingcap/tipb v0.0.0-20181012112600-11e33c750323 h1:mRKKzRjDNaUNPnAkPAHnRqpNmwNWBX1iA+hxlmvQ93I=
 github.com/pingcap/tipb v0.0.0-20181012112600-11e33c750323/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI=
 github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
@@ -179,7 +168,6 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2I
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.2.0 h1:S0iUepdCWODXRvtE+gcRDd15L+k+k1AiHlMiMjefH24=
 google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2 h1:67iHsV9djwGdZpdZNbLuQj6FOzCaZe3w+vhLjn5AcFA=

pkg/dbutil/common.go

@@ -156,6 +156,7 @@ func GetRowCount(ctx context.Context, db *sql.DB, schemaName string, tableName s
 	if len(where) > 0 {
 		query += fmt.Sprintf(" WHERE %s", where)
 	}
+	log.Debugf("get row count by sql %s", query)
 
 	var cnt sql.NullInt64
 	err := db.QueryRowContext(ctx, query).Scan(&cnt)

pkg/diff/chunk.go

@@ -258,7 +258,6 @@ func (s *randomSpliter) splitRange(db *sql.DB, chunk *chunkRange, count int, sch
 
 		symbolMin = gte
 		symbolMax = lte
-
 	}
 
 	splitValues := make([]string, 0, count)
@@ -269,7 +268,7 @@ func (s *randomSpliter) splitRange(db *sql.DB, chunk *chunkRange, count int, sch
 	if err != nil {
 		return nil, errors.Trace(err)
 	}
-	log.Infof("split chunk %v, get split values from GetRandomValues: %v", chunk, randomValues)
+	log.Debugf("split chunk %v, get split values from GetRandomValues: %v", chunk, randomValues)
 
 	/*
 		for examples:
@@ -319,8 +318,17 @@ func (s *randomSpliter) splitRange(db *sql.DB, chunk *chunkRange, count int, sch
 		the splitValues is [1, 1, 1, 1];
 		the chunk [`a` = 2] will split to [`a` = 2 AND `b` < 'x'], [`a` = 2 AND `b` >= 'x' AND `b` < 'y'] and [`a` = 2 AND `b` >= 'y']
 	*/
-	var lower, upper, lowerSymbol, upperSymbol string
+
+	lowerSymbol := symbolMin
+	upperSymbol := lt
+
 	for i := 0; i < len(splitValues); i++ {
+		if i == 0 && useNewColumn {
+			// create chunk less than min
+			newChunk := chunk.copyAndUpdate(splitCol, "", "", splitValues[i], lt)
+			chunks = append(chunks, newChunk)
+		}
+
 		if valueCounts[i] > 1 {
 			// means should split it
 			newChunk := chunk.copyAndUpdate(splitCol, splitValues[i], equal, "", "")
@@ -332,45 +340,28 @@ func (s *randomSpliter) splitRange(db *sql.DB, chunk *chunkRange, count int, sch
 
 			// already have the chunk [column = value], so next chunk should start with column > value
 			lowerSymbol = gt
-		} else {
-			if i == 0 {
-				if useNewColumn {
-					lower = ""
-					lowerSymbol = ""
-				} else {
-					lower = splitValues[i]
-					lowerSymbol = symbolMin
-				}
-			} else {
-				lower = splitValues[i]
-			}
-
-			if i == len(splitValues)-2 {
-				if useNewColumn && valueCounts[len(valueCounts)-1] == 1 {
-					upper = ""
-					upperSymbol = ""
-				} else {
-					upper = splitValues[i+1]
-					upperSymbol = symbolMax
-				}
-			} else {
-				if i == len(splitValues)-1 {
-					continue
-				}
+		}
 
-				upper = splitValues[i+1]
-				upperSymbol = lt
+		if i == len(splitValues)-2 && valueCounts[i+1] == 1 {
+			upperSymbol = symbolMax
+		}
 
-			}
+		if i < len(splitValues)-1 {
+			newChunk := chunk.copyAndUpdate(splitCol, splitValues[i], lowerSymbol, splitValues[i+1], upperSymbol)
+			chunks = append(chunks, newChunk)
 		}
 
-		newChunk := chunk.copyAndUpdate(splitCol, lower, lowerSymbol, upper, upperSymbol)
-		chunks = append(chunks, newChunk)
+		if i == len(splitValues)-1 && useNewColumn {
+			// create chunk greater than max
+			newChunk := chunk.copyAndUpdate(splitCol, splitValues[i], gt, "", "")
+			chunks = append(chunks, newChunk)
+		}
 
 		lowerSymbol = gte
 	}
 
 	log.Debugf("getChunksForTable cut table: cnt=%d min=%s max=%s chunk=%d", count, min, max, len(chunks))
+
 	return chunks, nil
 }
 

pkg/diff/chunk_test.go

@@ -200,5 +200,4 @@ func (*testChunkSuite) TestChunkToString(c *C) {
 	for i, arg := range args {
 		c.Assert(arg, Equals, expectArgs[i])
 	}
-
 }

pkg/diff/diff_test.go

@@ -39,11 +39,11 @@ func (*testDiffSuite) TestGenerateSQLs(c *C) {
 	c.Assert(err, IsNil)
 
 	rowsData := map[string]*dbutil.ColumnData{
-		"id":          {[]byte("1"), false},
-		"name":        {[]byte("xxx"), false},
-		"birthday":    {[]byte("2018-01-01 00:00:00"), false},
-		"update_time": {[]byte("10:10:10"), false},
-		"money":       {[]byte("11.1111"), false},
+		"id":          {Data: []byte("1"), IsNull: false},
+		"name":        {Data: []byte("xxx"), IsNull: false},
+		"birthday":    {Data: []byte("2018-01-01 00:00:00"), IsNull: false},
+		"update_time": {Data: []byte("10:10:10"), IsNull: false},
+		"money":       {Data: []byte("11.1111"), IsNull: false},
 	}
 
 	_, orderKeyCols := dbutil.SelectUniqueOrderKey(tableInfo)
@@ -63,13 +63,13 @@ func (*testDiffSuite) TestGenerateSQLs(c *C) {
 	c.Assert(deleteSQL, Equals, "DELETE FROM `test`.`atest` WHERE `id` = 1 AND `name` = 'xxx';")
 
 	// test value is nil
-	rowsData["name"] = &dbutil.ColumnData{[]byte(""), true}
+	rowsData["name"] = &dbutil.ColumnData{Data: []byte(""), IsNull: true}
 	replaceSQL = generateDML("replace", rowsData, orderKeyCols, tableInfo, "test")
 	deleteSQL = generateDML("delete", rowsData, orderKeyCols, tableInfo, "test")
 	c.Assert(replaceSQL, Equals, "REPLACE INTO `test`.`atest`(`id`,`name`,`birthday`,`update_time`,`money`) VALUES (1,NULL,'2018-01-01 00:00:00','10:10:10',11.1111);")
 	c.Assert(deleteSQL, Equals, "DELETE FROM `test`.`atest` WHERE `id` = 1;")
 
-	rowsData["id"] = &dbutil.ColumnData{[]byte(""), true}
+	rowsData["id"] = &dbutil.ColumnData{Data: []byte(""), IsNull: true}
 	replaceSQL = generateDML("replace", rowsData, orderKeyCols, tableInfo, "test")
 	deleteSQL = generateDML("delete", rowsData, orderKeyCols, tableInfo, "test")
 	c.Assert(replaceSQL, Equals, "REPLACE INTO `test`.`atest`(`id`,`name`,`birthday`,`update_time`,`money`) VALUES (NULL,NULL,'2018-01-01 00:00:00','10:10:10',11.1111);")

pkg/diff/merge_test.go

@@ -46,9 +46,9 @@ func (s *testMergerSuite) TestMerge(c *C) {
 	heap.Init(rowDatas)
 	for i, id := range ids {
 		data := map[string]*dbutil.ColumnData{
-			"id":   {[]byte(strconv.Itoa(id)), false},
-			"name": {[]byte(names[i]), false},
-			"age":  {[]byte(strconv.Itoa(ages[i])), false},
+			"id":   {Data: []byte(strconv.Itoa(id)), IsNull: false},
+			"name": {Data: []byte(names[i]), IsNull: false},
+			"age":  {Data: []byte(strconv.Itoa(ages[i])), IsNull: false},
 		}
 		heap.Push(rowDatas, RowData{
 			Data: data,