fix: resolve some bugs

- store failed when persistent is mysql
- log gorm sql with zap
- more detail message when task failed
- sort tasks by updatetime
- task duration always be  0s while persistant is mysql
- do not save sshpassowrd when AuthenticateType is SshPasswordNotSave or SshPasswordUsePubkey

business/ck_archive.go

@@ -76,7 +76,6 @@ func (this *ArchiveHDFS) InitConns() (err error) {
 		}
 		_, err = common.ConnectClickHouse(host, this.Port, this.Database, this.User, this.Password)
 		if err != nil {
-			err = errors.Wrapf(err, "")
 			return
 		}
 		log.Logger.Infof("initialized clickhouse connection to %s", host)

business/ck_purger.go

@@ -3,6 +3,7 @@ package business
 import (
 	"database/sql"
 	"fmt"
+
 	"github.com/housepower/ckman/common"
 	"github.com/housepower/ckman/log"
 	"github.com/pkg/errors"
@@ -38,7 +39,6 @@ func (this *PurgerRange) InitConns() (err error) {
 		}
 		_, err = common.ConnectClickHouse(host, this.Port, this.Database, this.User, this.Password)
 		if err != nil {
-			err = errors.Wrapf(err, "")
 			return
 		}
 		log.Logger.Infof("initialized clickhouse connection to %s", host)
@@ -147,4 +147,3 @@ func (this *PurgerRange) PurgeTable(table string) (err error) {
 	}
 	return
 }
-

business/ck_rebalance.go

@@ -50,7 +50,6 @@ func (this *CKRebalance) InitCKConns() (err error) {
 	for _, host := range this.Hosts {
 		_, err = common.ConnectClickHouse(host, this.Port, model.ClickHouseDefaultDB, this.User, this.Password)
 		if err != nil {
-			err = errors.Wrapf(err, "")
 			return
 		}
 		log.Logger.Infof("initialized clickhouse connection to %s", host)
@@ -66,7 +65,6 @@ func (this *CKRebalance) GetTables() (err error) {
 		return fmt.Errorf("can't get connection: %s", host)
 	}
 	if this.Databases, this.DBTables, err = common.GetMergeTreeTables("MergeTree", db); err != nil {
-		err = errors.Wrapf(err, "")
 		return
 	}
 	return
@@ -123,7 +121,6 @@ func (this *CKRebalance) InitSshConns(database string) (err error) {
 			}
 			var out string
 			if out, err = common.RemoteExecute(sshOpts, cmd); err != nil {
-				err = errors.Wrapf(err, "output: %s", out)
 				return
 			}
 			log.Logger.Debugf("host: %s, output: %s", srcHost, out)
@@ -296,7 +293,6 @@ func (this *CKRebalance) ExecutePlan(database string, tbl *TblPartitions) (err e
 		}
 		var out string
 		if out, err = common.RemoteExecute(sshOpts, strings.Join(cmds, ";")); err != nil {
-			err = errors.Wrapf(err, "output: %s", out)
 			lock.Unlock()
 			return
 		}
@@ -338,7 +334,7 @@ func (this *CKRebalance) DoRebalance() (err error) {
 			var tbls []*TblPartitions
 			if tbls, err = this.GetState(database, table); err != nil {
 				log.Logger.Errorf("got error %+v", err)
-				return errors.Wrap(err, "")
+				return err
 			}
 			this.GeneratePlan(fmt.Sprintf("%s.%s", database, table), tbls)
 
@@ -356,7 +352,7 @@ func (this *CKRebalance) DoRebalance() (err error) {
 			}
 			common.Pool.Wait()
 			if gotError {
-				return errors.Wrap(err, "")
+				return err
 			}
 			log.Logger.Infof("table %s rebalance done", table)
 		}

business/ck_schema.go

@@ -64,12 +64,10 @@ func GetCreateReplicaObjects(db *sql.DB, host, user, password string) (names, st
 
 	names1, statements1, err := GetObjectListFromClickHouse(db, sqlDBs)
 	if err != nil {
-		err = errors.Wrapf(err, "")
 		return
 	}
 	names2, statements2, err := GetObjectListFromClickHouse(db, sqlTables)
 	if err != nil {
-		err = errors.Wrapf(err, "")
 		return
 	}
 	names = append(names1, names2...)
@@ -145,7 +143,7 @@ INNER JOIN
 			"logictbl":    logictbl,
 		}
 		if err = common.ReplaceTemplateString(&localsql, replaceTmpl); err != nil {
-			return nil, errors.Wrap(err, "")
+			return nil, err
 		}
 		localsqls = append(localsqls, localsql)
 
@@ -154,7 +152,7 @@ INNER JOIN
 		distsqls = append(distsqls, distsql)
 
 		if err = common.ReplaceTemplateString(&logicsql, replaceTmpl); err != nil {
-			return nil, errors.Wrap(err, "")
+			return nil, err
 		}
 		logicsqls = append(logicsqls, logicsql)
 	}

ckconfig/custom.go

@@ -8,7 +8,6 @@ import (
 	"github.com/housepower/ckman/common"
 	"github.com/housepower/ckman/model"
 	"github.com/imdario/mergo"
-	"github.com/pkg/errors"
 )
 
 func root(conf *model.CKManClickHouseConfig, ipv6Enable bool) map[string]interface{} {
@@ -168,7 +167,7 @@ func GenerateCustomXML(filename string, conf *model.CKManClickHouseConfig, ipv6E
 	xml.Merge(custom)
 	xml.End(rootTag)
 	if err := xml.Dump(); err != nil {
-		return filename, errors.Wrap(err, "")
+		return filename, err
 	}
 	return filename, nil
 }

ckconfig/hosts.go

@@ -3,10 +3,9 @@ package ckconfig
 import (
 	"github.com/housepower/ckman/common"
 	"github.com/housepower/ckman/model"
-	"github.com/pkg/errors"
 )
 
-func GenerateHostXML(filename string, conf *model.CKManClickHouseConfig, host string)(string, error){
+func GenerateHostXML(filename string, conf *model.CKManClickHouseConfig, host string) (string, error) {
 	shardIndex := 0
 	for i, shard := range conf.Shards {
 		for _, replica := range shard.Replicas {
@@ -29,7 +28,7 @@ func GenerateHostXML(filename string, conf *model.CKManClickHouseConfig, host st
 	xml.End("yandex")
 	err := xml.Dump()
 	if err != nil {
-		return "", errors.Wrap(err, "")
+		return "", err
 	}
 	return filename, nil
 }

ckconfig/metrika.go

@@ -3,10 +3,9 @@ package ckconfig
 import (
 	"github.com/housepower/ckman/common"
 	"github.com/housepower/ckman/model"
-	"github.com/pkg/errors"
 )
 
-func GenerateMetrikaXML(filename string, conf *model.CKManClickHouseConfig)(string, error){
+func GenerateMetrikaXML(filename string, conf *model.CKManClickHouseConfig) (string, error) {
 	xml := common.NewXmlFile(filename)
 	xml.Begin("yandex")
 	xml.Append(GenZookeeperMetrika(xml.GetIndent(), conf))
@@ -16,12 +15,12 @@ func GenerateMetrikaXML(filename string, conf *model.CKManClickHouseConfig)(stri
 	xml.End("yandex")
 	err := xml.Dump()
 	if err != nil {
-		return "", errors.Wrap(err, "")
+		return "", err
 	}
 	return filename, nil
 }
 
-func GenerateMetrikaXMLwithLogic(filename string, conf *model.CKManClickHouseConfig, logicMrtrika string)(string, error){
+func GenerateMetrikaXMLwithLogic(filename string, conf *model.CKManClickHouseConfig, logicMrtrika string) (string, error) {
 	xml := common.NewXmlFile(filename)
 	xml.Begin("yandex")
 	xml.Append(GenZookeeperMetrika(xml.GetIndent(), conf))
@@ -32,7 +31,7 @@ func GenerateMetrikaXMLwithLogic(filename string, conf *model.CKManClickHouseCon
 	xml.End("yandex")
 	err := xml.Dump()
 	if err != nil {
-		return "", errors.Wrap(err, "")
+		return "", err
 	}
 	return filename, nil
 }
@@ -42,7 +41,7 @@ func GenZookeeperMetrika(indent int, conf *model.CKManClickHouseConfig) string {
 	xml.SetIndent(indent)
 	xml.Begin("zookeeper")
 	for index, zk := range conf.ZkNodes {
-		xml.BeginwithAttr("node",  []common.XMLAttr{{Key: "index", Value:index+1}})
+		xml.BeginwithAttr("node", []common.XMLAttr{{Key: "index", Value: index + 1}})
 		xml.Write("host", zk)
 		xml.Write("port", conf.ZkPort)
 		xml.End("node")
@@ -51,12 +50,12 @@ func GenZookeeperMetrika(indent int, conf *model.CKManClickHouseConfig) string {
 	return xml.GetContext()
 }
 
-func GenLocalMetrika(indent int, conf *model.CKManClickHouseConfig)string {
+func GenLocalMetrika(indent int, conf *model.CKManClickHouseConfig) string {
 	xml := common.NewXmlFile("")
 	xml.SetIndent(indent)
 	xml.Begin(conf.Cluster)
 	secret := true
-	if common.CompareClickHouseVersion(conf.Version, "20.10.3.30") < 0{
+	if common.CompareClickHouseVersion(conf.Version, "20.10.3.30") < 0 {
 		secret = false
 	}
 	if secret {
@@ -99,4 +98,3 @@ func GenLocalMetrika(indent int, conf *model.CKManClickHouseConfig)string {
 	xml.End(conf.Cluster)
 	return xml.GetContext()
 }
-

ckconfig/users.go

@@ -4,7 +4,6 @@ import (
 	"github.com/housepower/ckman/common"
 	"github.com/housepower/ckman/model"
 	"github.com/imdario/mergo"
-	"github.com/pkg/errors"
 )
 
 type HostInfo struct {
@@ -152,7 +151,7 @@ func GenerateUsersXML(filename string, conf *model.CKManClickHouseConfig, info H
 	xml.Merge(userconf)
 	xml.End(rootTag)
 	if err := xml.Dump(); err != nil {
-		return filename, errors.Wrap(err, "")
+		return filename, err
 	}
 	return filename, nil
 }

common/ck.go

@@ -98,7 +98,7 @@ func GetMergeTreeTables(engine string, db *sql.DB) ([]string, map[string][]strin
 	log.Logger.Debugf("query: %s", query)
 	if rows, err = db.Query(query); err != nil {
 		err = errors.Wrapf(err, "")
-		return nil, nil, errors.Wrap(err, "")
+		return nil, nil, err
 	}
 	defer rows.Close()
 	var tables []string
@@ -107,7 +107,7 @@ func GetMergeTreeTables(engine string, db *sql.DB) ([]string, map[string][]strin
 		var database, name string
 		if err = rows.Scan(&database, &name); err != nil {
 			err = errors.Wrapf(err, "")
-			return nil, nil, errors.Wrap(err, "")
+			return nil, nil, err
 		}
 		if database != predbname {
 			if predbname != "" {
@@ -137,7 +137,7 @@ func GetShardAvaliableHosts(conf *model.CKManClickHouseConfig) ([]string, error)
 				hosts = append(hosts, replica.Ip)
 				break
 			} else {
-				lastErr = errors.Wrap(err, "")
+				lastErr = err
 			}
 		}
 	}

common/gosypt.go

@@ -1,9 +1,10 @@
 package common
 
 import (
-	"github.com/pkg/errors"
 	"reflect"
 	"strings"
+
+	"github.com/pkg/errors"
 )
 
 const (
@@ -58,25 +59,25 @@ func (gsypt *Gosypt) Unmarshal(v interface{}) error {
 	if rt.Kind() == reflect.Struct {
 		v, err := gsypt.structHandle(rt, rv)
 		if err != nil {
-			return errors.Wrap(err, "")
+			return err
 		}
 		rv.Set(v)
 	} else if rt.Kind() == reflect.Slice || rt.Kind() == reflect.Array {
 		v, err := gsypt.sliceHandle(rt, rv)
 		if err != nil {
-			return errors.Wrap(err, "")
+			return err
 		}
 		rv.Set(v)
 	} else if rt.Kind() == reflect.Map {
 		v, err := gsypt.mapHandle(rt, rv)
 		if err != nil {
-			return errors.Wrap(err, "")
+			return err
 		}
 		rv.Set(v)
 	} else if rt.Kind() == reflect.Interface {
 		v, err := gsypt.interfaceHandle(rt, rv)
 		if err != nil {
-			return errors.Wrap(err, "")
+			return err
 		}
 		rv.Set(v)
 	} else if rt.Kind() == reflect.String {
@@ -92,7 +93,7 @@ func (gsypt *Gosypt) sliceHandle(rt reflect.Type, rv reflect.Value) (reflect.Val
 			rv.Index(j).Set(gsypt.stringHandle(rv.Index(j)))
 		} else {
 			if err := gsypt.Unmarshal(rv.Index(j).Addr().Interface()); err != nil {
-				return rv, errors.Wrap(err, "")
+				return rv, err
 			}
 		}
 	}
@@ -108,7 +109,7 @@ func (gsypt *Gosypt) mapHandle(rt reflect.Type, rv reflect.Value) (reflect.Value
 		} else {
 			v := rv.MapIndex(key).Interface()
 			if err := gsypt.Unmarshal(&v); err != nil {
-				return rv, errors.Wrap(err, "")
+				return rv, err
 			}
 			rv.SetMapIndex(key, reflect.ValueOf(v))
 		}
@@ -135,14 +136,14 @@ func (gsypt *Gosypt) structHandle(rt reflect.Type, rv reflect.Value) (reflect.Va
 			rv.Field(i).Set(gsypt.stringHandle(rvf))
 		} else {
 			if err := gsypt.Unmarshal(rvf.Addr().Interface()); err != nil {
-				return rv, errors.Wrap(err, "")
+				return rv, err
 			}
 		}
 	}
 	return rv, nil
 }
 
-func (gsypt *Gosypt)stringHandle(rv reflect.Value)reflect.Value {
+func (gsypt *Gosypt) stringHandle(rv reflect.Value) reflect.Value {
 	rv.SetString(gsypt.ensurePassword(rv.String()))
 	return rv
-}
+}