diff --git a/business/ck_archive.go b/business/ck_archive.go index bf0decc9..0f85da79 100644 --- a/business/ck_archive.go +++ b/business/ck_archive.go @@ -76,7 +76,6 @@ func (this *ArchiveHDFS) InitConns() (err error) { } _, err = common.ConnectClickHouse(host, this.Port, this.Database, this.User, this.Password) if err != nil { - err = errors.Wrapf(err, "") return } log.Logger.Infof("initialized clickhouse connection to %s", host) diff --git a/business/ck_purger.go b/business/ck_purger.go index 0d4aec53..9af73766 100644 --- a/business/ck_purger.go +++ b/business/ck_purger.go @@ -3,6 +3,7 @@ package business import ( "database/sql" "fmt" + "github.com/housepower/ckman/common" "github.com/housepower/ckman/log" "github.com/pkg/errors" @@ -38,7 +39,6 @@ func (this *PurgerRange) InitConns() (err error) { } _, err = common.ConnectClickHouse(host, this.Port, this.Database, this.User, this.Password) if err != nil { - err = errors.Wrapf(err, "") return } log.Logger.Infof("initialized clickhouse connection to %s", host) @@ -147,4 +147,3 @@ func (this *PurgerRange) PurgeTable(table string) (err error) { } return } - diff --git a/business/ck_rebalance.go b/business/ck_rebalance.go index 1274f2c0..b8cb9ef7 100644 --- a/business/ck_rebalance.go +++ b/business/ck_rebalance.go @@ -50,7 +50,6 @@ func (this *CKRebalance) InitCKConns() (err error) { for _, host := range this.Hosts { _, err = common.ConnectClickHouse(host, this.Port, model.ClickHouseDefaultDB, this.User, this.Password) if err != nil { - err = errors.Wrapf(err, "") return } log.Logger.Infof("initialized clickhouse connection to %s", host) @@ -66,7 +65,6 @@ func (this *CKRebalance) GetTables() (err error) { return fmt.Errorf("can't get connection: %s", host) } if this.Databases, this.DBTables, err = common.GetMergeTreeTables("MergeTree", db); err != nil { - err = errors.Wrapf(err, "") return } return @@ -123,7 +121,6 @@ func (this *CKRebalance) InitSshConns(database string) (err error) { } var out string if out, err = common.RemoteExecute(sshOpts, cmd); err != nil { - err = errors.Wrapf(err, "output: %s", out) return } log.Logger.Debugf("host: %s, output: %s", srcHost, out) @@ -296,7 +293,6 @@ func (this *CKRebalance) ExecutePlan(database string, tbl *TblPartitions) (err e } var out string if out, err = common.RemoteExecute(sshOpts, strings.Join(cmds, ";")); err != nil { - err = errors.Wrapf(err, "output: %s", out) lock.Unlock() return } @@ -338,7 +334,7 @@ func (this *CKRebalance) DoRebalance() (err error) { var tbls []*TblPartitions if tbls, err = this.GetState(database, table); err != nil { log.Logger.Errorf("got error %+v", err) - return errors.Wrap(err, "") + return err } this.GeneratePlan(fmt.Sprintf("%s.%s", database, table), tbls) @@ -356,7 +352,7 @@ func (this *CKRebalance) DoRebalance() (err error) { } common.Pool.Wait() if gotError { - return errors.Wrap(err, "") + return err } log.Logger.Infof("table %s rebalance done", table) } diff --git a/business/ck_schema.go b/business/ck_schema.go index bbc1918b..fd9ab09e 100644 --- a/business/ck_schema.go +++ b/business/ck_schema.go @@ -64,12 +64,10 @@ func GetCreateReplicaObjects(db *sql.DB, host, user, password string) (names, st names1, statements1, err := GetObjectListFromClickHouse(db, sqlDBs) if err != nil { - err = errors.Wrapf(err, "") return } names2, statements2, err := GetObjectListFromClickHouse(db, sqlTables) if err != nil { - err = errors.Wrapf(err, "") return } names = append(names1, names2...) @@ -145,7 +143,7 @@ INNER JOIN "logictbl": logictbl, } if err = common.ReplaceTemplateString(&localsql, replaceTmpl); err != nil { - return nil, errors.Wrap(err, "") + return nil, err } localsqls = append(localsqls, localsql) @@ -154,7 +152,7 @@ INNER JOIN distsqls = append(distsqls, distsql) if err = common.ReplaceTemplateString(&logicsql, replaceTmpl); err != nil { - return nil, errors.Wrap(err, "") + return nil, err } logicsqls = append(logicsqls, logicsql) } diff --git a/ckconfig/custom.go b/ckconfig/custom.go index 9b0c6d47..8b5aba59 100644 --- a/ckconfig/custom.go +++ b/ckconfig/custom.go @@ -8,7 +8,6 @@ import ( "github.com/housepower/ckman/common" "github.com/housepower/ckman/model" "github.com/imdario/mergo" - "github.com/pkg/errors" ) func root(conf *model.CKManClickHouseConfig, ipv6Enable bool) map[string]interface{} { @@ -168,7 +167,7 @@ func GenerateCustomXML(filename string, conf *model.CKManClickHouseConfig, ipv6E xml.Merge(custom) xml.End(rootTag) if err := xml.Dump(); err != nil { - return filename, errors.Wrap(err, "") + return filename, err } return filename, nil } diff --git a/ckconfig/hosts.go b/ckconfig/hosts.go index bc49a8b9..0cf94a9c 100644 --- a/ckconfig/hosts.go +++ b/ckconfig/hosts.go @@ -3,10 +3,9 @@ package ckconfig import ( "github.com/housepower/ckman/common" "github.com/housepower/ckman/model" - "github.com/pkg/errors" ) -func GenerateHostXML(filename string, conf *model.CKManClickHouseConfig, host string)(string, error){ +func GenerateHostXML(filename string, conf *model.CKManClickHouseConfig, host string) (string, error) { shardIndex := 0 for i, shard := range conf.Shards { for _, replica := range shard.Replicas { @@ -29,7 +28,7 @@ func GenerateHostXML(filename string, conf *model.CKManClickHouseConfig, host st xml.End("yandex") err := xml.Dump() if err != nil { - return "", errors.Wrap(err, "") + return "", err } return filename, nil } diff --git a/ckconfig/metrika.go b/ckconfig/metrika.go index c628c59c..19c5c041 100644 --- a/ckconfig/metrika.go +++ b/ckconfig/metrika.go @@ -3,10 +3,9 @@ package ckconfig import ( "github.com/housepower/ckman/common" "github.com/housepower/ckman/model" - "github.com/pkg/errors" ) -func GenerateMetrikaXML(filename string, conf *model.CKManClickHouseConfig)(string, error){ +func GenerateMetrikaXML(filename string, conf *model.CKManClickHouseConfig) (string, error) { xml := common.NewXmlFile(filename) xml.Begin("yandex") xml.Append(GenZookeeperMetrika(xml.GetIndent(), conf)) @@ -16,12 +15,12 @@ func GenerateMetrikaXML(filename string, conf *model.CKManClickHouseConfig)(stri xml.End("yandex") err := xml.Dump() if err != nil { - return "", errors.Wrap(err, "") + return "", err } return filename, nil } -func GenerateMetrikaXMLwithLogic(filename string, conf *model.CKManClickHouseConfig, logicMrtrika string)(string, error){ +func GenerateMetrikaXMLwithLogic(filename string, conf *model.CKManClickHouseConfig, logicMrtrika string) (string, error) { xml := common.NewXmlFile(filename) xml.Begin("yandex") xml.Append(GenZookeeperMetrika(xml.GetIndent(), conf)) @@ -32,7 +31,7 @@ func GenerateMetrikaXMLwithLogic(filename string, conf *model.CKManClickHouseCon xml.End("yandex") err := xml.Dump() if err != nil { - return "", errors.Wrap(err, "") + return "", err } return filename, nil } @@ -42,7 +41,7 @@ func GenZookeeperMetrika(indent int, conf *model.CKManClickHouseConfig) string { xml.SetIndent(indent) xml.Begin("zookeeper") for index, zk := range conf.ZkNodes { - xml.BeginwithAttr("node", []common.XMLAttr{{Key: "index", Value:index+1}}) + xml.BeginwithAttr("node", []common.XMLAttr{{Key: "index", Value: index + 1}}) xml.Write("host", zk) xml.Write("port", conf.ZkPort) xml.End("node") @@ -51,12 +50,12 @@ func GenZookeeperMetrika(indent int, conf *model.CKManClickHouseConfig) string { return xml.GetContext() } -func GenLocalMetrika(indent int, conf *model.CKManClickHouseConfig)string { +func GenLocalMetrika(indent int, conf *model.CKManClickHouseConfig) string { xml := common.NewXmlFile("") xml.SetIndent(indent) xml.Begin(conf.Cluster) secret := true - if common.CompareClickHouseVersion(conf.Version, "20.10.3.30") < 0{ + if common.CompareClickHouseVersion(conf.Version, "20.10.3.30") < 0 { secret = false } if secret { @@ -99,4 +98,3 @@ func GenLocalMetrika(indent int, conf *model.CKManClickHouseConfig)string { xml.End(conf.Cluster) return xml.GetContext() } - diff --git a/ckconfig/users.go b/ckconfig/users.go index 1665aac7..5ab9fb8d 100644 --- a/ckconfig/users.go +++ b/ckconfig/users.go @@ -4,7 +4,6 @@ import ( "github.com/housepower/ckman/common" "github.com/housepower/ckman/model" "github.com/imdario/mergo" - "github.com/pkg/errors" ) type HostInfo struct { @@ -152,7 +151,7 @@ func GenerateUsersXML(filename string, conf *model.CKManClickHouseConfig, info H xml.Merge(userconf) xml.End(rootTag) if err := xml.Dump(); err != nil { - return filename, errors.Wrap(err, "") + return filename, err } return filename, nil } diff --git a/common/ck.go b/common/ck.go index 1271ca94..c7728b0b 100644 --- a/common/ck.go +++ b/common/ck.go @@ -98,7 +98,7 @@ func GetMergeTreeTables(engine string, db *sql.DB) ([]string, map[string][]strin log.Logger.Debugf("query: %s", query) if rows, err = db.Query(query); err != nil { err = errors.Wrapf(err, "") - return nil, nil, errors.Wrap(err, "") + return nil, nil, err } defer rows.Close() var tables []string @@ -107,7 +107,7 @@ func GetMergeTreeTables(engine string, db *sql.DB) ([]string, map[string][]strin var database, name string if err = rows.Scan(&database, &name); err != nil { err = errors.Wrapf(err, "") - return nil, nil, errors.Wrap(err, "") + return nil, nil, err } if database != predbname { if predbname != "" { @@ -137,7 +137,7 @@ func GetShardAvaliableHosts(conf *model.CKManClickHouseConfig) ([]string, error) hosts = append(hosts, replica.Ip) break } else { - lastErr = errors.Wrap(err, "") + lastErr = err } } } diff --git a/common/gosypt.go b/common/gosypt.go index 72adc594..26ce383f 100644 --- a/common/gosypt.go +++ b/common/gosypt.go @@ -1,9 +1,10 @@ package common import ( - "github.com/pkg/errors" "reflect" "strings" + + "github.com/pkg/errors" ) const ( @@ -58,25 +59,25 @@ func (gsypt *Gosypt) Unmarshal(v interface{}) error { if rt.Kind() == reflect.Struct { v, err := gsypt.structHandle(rt, rv) if err != nil { - return errors.Wrap(err, "") + return err } rv.Set(v) } else if rt.Kind() == reflect.Slice || rt.Kind() == reflect.Array { v, err := gsypt.sliceHandle(rt, rv) if err != nil { - return errors.Wrap(err, "") + return err } rv.Set(v) } else if rt.Kind() == reflect.Map { v, err := gsypt.mapHandle(rt, rv) if err != nil { - return errors.Wrap(err, "") + return err } rv.Set(v) } else if rt.Kind() == reflect.Interface { v, err := gsypt.interfaceHandle(rt, rv) if err != nil { - return errors.Wrap(err, "") + return err } rv.Set(v) } else if rt.Kind() == reflect.String { @@ -92,7 +93,7 @@ func (gsypt *Gosypt) sliceHandle(rt reflect.Type, rv reflect.Value) (reflect.Val rv.Index(j).Set(gsypt.stringHandle(rv.Index(j))) } else { if err := gsypt.Unmarshal(rv.Index(j).Addr().Interface()); err != nil { - return rv, errors.Wrap(err, "") + return rv, err } } } @@ -108,7 +109,7 @@ func (gsypt *Gosypt) mapHandle(rt reflect.Type, rv reflect.Value) (reflect.Value } else { v := rv.MapIndex(key).Interface() if err := gsypt.Unmarshal(&v); err != nil { - return rv, errors.Wrap(err, "") + return rv, err } rv.SetMapIndex(key, reflect.ValueOf(v)) } @@ -135,14 +136,14 @@ func (gsypt *Gosypt) structHandle(rt reflect.Type, rv reflect.Value) (reflect.Va rv.Field(i).Set(gsypt.stringHandle(rvf)) } else { if err := gsypt.Unmarshal(rvf.Addr().Interface()); err != nil { - return rv, errors.Wrap(err, "") + return rv, err } } } return rv, nil } -func (gsypt *Gosypt)stringHandle(rv reflect.Value)reflect.Value { +func (gsypt *Gosypt) stringHandle(rv reflect.Value) reflect.Value { rv.SetString(gsypt.ensurePassword(rv.String())) return rv -} \ No newline at end of file +} diff --git a/common/iprange.go b/common/iprange.go index d24c0ab8..09632f53 100644 --- a/common/iprange.go +++ b/common/iprange.go @@ -1,35 +1,36 @@ package common import ( - "github.com/pkg/errors" "math" "net" "strings" + + "github.com/pkg/errors" ) -func ParseHosts(hosts []string)([]string, error) { +func ParseHosts(hosts []string) ([]string, error) { var allHosts []string - for _, host := range hosts{ + for _, host := range hosts { ips, err := ParseIPRange(host) if err != nil { - return allHosts, errors.Wrap(err, "") + return allHosts, err } allHosts = append(allHosts, ips...) } - return allHosts, nil + return allHosts, nil } -func ParseIPRange(s string)([]string, error){ +func ParseIPRange(s string) ([]string, error) { var ips []string var err error if strings.Contains(s, "-") { if ips, err = ipRangeParse(s); err != nil { - return ips, errors.Wrap(err, "") + return ips, err } } else if strings.Contains(s, "/") { ips, err = ipCIDR(s) if err != nil { - return ips, errors.Wrap(err, "") + return ips, err } } else { ips = append(ips, s) @@ -63,7 +64,7 @@ func InetNtoA(i uint) (string, error) { return ip.String(), nil } -func ipRangeParse(s string)([]string, error) { +func ipRangeParse(s string) ([]string, error) { var ips []string ipAddrs := strings.Split(s, "-") if len(ipAddrs) != 2 { @@ -71,11 +72,11 @@ func ipRangeParse(s string)([]string, error) { } begin, err := InetAtoN(ipAddrs[0]) if err != nil { - return ips, errors.Wrap(err, "") + return ips, err } end, err := InetAtoN(ipAddrs[1]) if err != nil { - return ips, errors.Wrap(err, "") + return ips, err } if begin > end { @@ -92,8 +93,7 @@ func ipRangeParse(s string)([]string, error) { return ips, nil } - -func ipCIDR(s string)([]string, error){ +func ipCIDR(s string) ([]string, error) { ip, ipnet, err := net.ParseCIDR(s) if err != nil { return nil, errors.Wrap(err, "") @@ -113,4 +113,4 @@ func inc(ip net.IP) { break } } -} \ No newline at end of file +} diff --git a/common/rsa.go b/common/rsa.go index 16c47fdb..9436aa62 100644 --- a/common/rsa.go +++ b/common/rsa.go @@ -7,8 +7,9 @@ import ( "encoding/base64" "encoding/pem" "fmt" - "github.com/pkg/errors" "io" + + "github.com/pkg/errors" ) /* @@ -80,7 +81,7 @@ func (encry RSAEncryption) Decode(encode []byte, publicKey string) ([]byte, erro output := bytes.NewBuffer(nil) err := pubKeyIO(pubkey, bytes.NewReader(databs), output, false) if err != nil { - return []byte(""), errors.Wrap(err, "") + return []byte(""), err } return io.ReadAll(output) } @@ -94,7 +95,7 @@ func (encry RSAEncryption) Encode(decode []byte, privateKey string) ([]byte, err output := bytes.NewBuffer(nil) err := priKeyIO(prikey, bytes.NewReader(decode), output, true) if err != nil { - return []byte(""), errors.Wrap(err, "") + return []byte(""), err } rsadata, err := io.ReadAll(output) if err != nil { diff --git a/common/rsa_ext.go b/common/rsa_ext.go index aadd873b..e5da3b33 100644 --- a/common/rsa_ext.go +++ b/common/rsa_ext.go @@ -3,11 +3,11 @@ package common import ( "crypto/rand" "crypto/rsa" - "github.com/pkg/errors" "io" "math/big" -) + "github.com/pkg/errors" +) var ( ErrDataToLarge = errors.New("message too long for RSA public key size") @@ -19,10 +19,9 @@ var ( ErrPrivateKey = errors.New("get private key error") ) - var ( bigZero = big.NewInt(0) - bigOne = big.NewInt(1) + bigOne = big.NewInt(1) ) func pubKeyIO(pub *rsa.PublicKey, in io.Reader, out io.Writer, isEncrytp bool) (err error) { @@ -140,7 +139,7 @@ func priKeyEncrypt(rand io.Reader, priv *rsa.PrivateKey, hashed []byte) ([]byte, m := new(big.Int).SetBytes(em) c, err := decrypt(rand, priv, m) if err != nil { - return nil, errors.Wrap(err, "") + return nil, err } copyWithLeftPad(em, c.Bytes()) return em, nil @@ -219,7 +218,6 @@ func copyWithLeftPad(dest, src []byte) { copy(dest[numPaddingBytes:], src) } - func leftPad(input []byte, size int) (out []byte) { n := len(input) if n > size { diff --git a/common/ssh.go b/common/ssh.go index 233f0cfa..af2c16aa 100644 --- a/common/ssh.go +++ b/common/ssh.go @@ -79,7 +79,7 @@ func SSHConnect(opts SshOptions) (*ssh.Client, error) { clientConfig, err = sshConnectwithPassword(opts.User, opts.Password) } if err != nil { - return nil, errors.Wrap(err, "") + return nil, err } // connet to ssh @@ -97,7 +97,7 @@ func ScpConnect(opts SshOptions) (*scp.Client, *ssh.Client, error) { if err != nil { return nil, nil, err } - // create sftp client + // create scp client var client scp.Client if client, err = scp.NewClientBySSH(sshClient); err != nil { err = errors.Wrapf(err, "") @@ -126,7 +126,7 @@ func ScpUpload(client *scp.Client, localFilePath, remoteFilePath string) error { defer f.Close() perm, err := GetFilePerm(localFilePath) if err != nil { - return errors.Wrapf(err, "") + return err } err = client.CopyFromFile(context.Background(), *f, remoteFilePath, perm) if err != nil { @@ -137,6 +137,7 @@ func ScpUpload(client *scp.Client, localFilePath, remoteFilePath string) error { } func ScpDownload(client *scp.Client, remoteFilePath, localFilePath string) error { + _ = os.Remove(localFilePath) //truncate local file first f, err := os.Create(localFilePath) if err != nil { err = errors.Wrapf(err, "") @@ -237,7 +238,7 @@ func ScpUploadFiles(files []string, remotePath string, opts SshOptions) error { remoteFile := path.Join(remotePath, path.Base(file)) err := ScpUploadFile(file, remoteFile, opts) if err != nil { - return errors.Wrap(err, "") + return err } } return nil @@ -246,7 +247,7 @@ func ScpUploadFiles(files []string, remotePath string, opts SshOptions) error { func ScpUploadFile(localFile, remoteFile string, opts SshOptions) error { client, sshClient, err := ScpConnect(opts) if err != nil { - return errors.Wrap(err, "") + return err } defer client.Close() defer sshClient.Close() @@ -254,19 +255,19 @@ func ScpUploadFile(localFile, remoteFile string, opts SshOptions) error { cmd := fmt.Sprintf("rm -rf %s", path.Join(TmpWorkDirectory, path.Base(remoteFile))) _, err = RemoteExecute(opts, cmd) if err != nil { - return errors.Wrap(err, "") + return err } err = ScpUpload(client, localFile, path.Join(TmpWorkDirectory, path.Base(remoteFile))) if err != nil { - return errors.Wrap(err, "") + return err } if path.Dir(remoteFile) != TmpWorkDirectory { cmd = fmt.Sprintf("cp %s %s", path.Join(TmpWorkDirectory, path.Base(remoteFile)), remoteFile) _, err = RemoteExecute(opts, cmd) if err != nil { - return errors.Wrap(err, "") + return err } } @@ -276,7 +277,7 @@ func ScpUploadFile(localFile, remoteFile string, opts SshOptions) error { func ScpDownloadFiles(files []string, localPath string, opts SshOptions) error { client, sshClient, err := ScpConnect(opts) if err != nil { - return errors.Wrap(err, "") + return err } defer client.Close() defer sshClient.Close() @@ -285,7 +286,7 @@ func ScpDownloadFiles(files []string, localPath string, opts SshOptions) error { baseName := path.Base(file) err = ScpDownload(client, file, path.Join(localPath, baseName)) if err != nil { - return errors.Wrap(err, "") + return err } } return nil @@ -294,14 +295,14 @@ func ScpDownloadFiles(files []string, localPath string, opts SshOptions) error { func ScpDownloadFile(remoteFile, localFile string, opts SshOptions) error { client, sshClient, err := ScpConnect(opts) if err != nil { - return errors.Wrap(err, "") + return err } defer client.Close() defer sshClient.Close() err = ScpDownload(client, remoteFile, localFile) if err != nil { - return errors.Wrap(err, "") + return err } return nil } diff --git a/controller/clickhouse.go b/controller/clickhouse.go index a4702103..ab9a1e2e 100644 --- a/controller/clickhouse.go +++ b/controller/clickhouse.go @@ -91,7 +91,7 @@ func (ck *ClickHouseController) ImportCluster(c *gin.Context) { if req.LogicCluster != "" { physics, err := repository.Ps.GetLogicClusterbyName(req.LogicCluster) if err != nil { - if err == repository.ErrRecordNotFound { + if errors.Is(err, repository.ErrRecordNotFound) { physics = []string{req.Cluster} if err1 := repository.Ps.CreateLogicCluster(req.LogicCluster, physics); err1 != nil { model.WrapMsg(c, model.IMPORT_CK_CLUSTER_FAIL, err1) @@ -1632,9 +1632,6 @@ func (ck *ClickHouseController) GetConfig(c *gin.Context) { return } cluster.Normalize() - if cluster.AuthenticateType == model.SshPasswordNotSave { - cluster.SshPassword = "" - } data, err := params.MarshalConfig(cluster) if err != nil { model.WrapMsg(c, model.GET_CK_CLUSTER_INFO_FAIL, nil) @@ -1919,7 +1916,7 @@ func mergeClickhouseConfig(conf *model.CKManClickHouseConfig) (bool, error) { cluster.UsersConf = conf.UsersConf cluster.LogicCluster = conf.LogicCluster if err = common.DeepCopyByGob(conf, cluster); err != nil { - return false, errors.Wrap(err, "") + return false, err } return restart, nil } diff --git a/controller/deploy.go b/controller/deploy.go index 35c6049a..24078a61 100644 --- a/controller/deploy.go +++ b/controller/deploy.go @@ -100,11 +100,11 @@ func checkDeployParams(conf *model.CKManClickHouseConfig) error { return errors.Errorf("can't find any host") } if conf.Hosts, err = common.ParseHosts(conf.Hosts); err != nil { - return errors.Wrap(err, "") + return err } if err = MatchingPlatfrom(conf); err != nil { - return errors.Wrap(err, "") + return err } //if conf.IsReplica && len(conf.Hosts)%2 == 1 { // return errors.Errorf("When supporting replica, the number of nodes must be even") @@ -156,7 +156,7 @@ func checkDeployParams(conf *model.CKManClickHouseConfig) error { return errors.Errorf(fmt.Sprintf("path %s must end with '/'", disk.DiskLocal.Path)) } if err = checkAccess(disk.DiskLocal.Path, conf); err != nil { - return errors.Wrap(err, "") + return err } localPath = append(localPath, disk.DiskLocal.Path) case "hdfs": @@ -177,13 +177,13 @@ func checkDeployParams(conf *model.CKManClickHouseConfig) error { } } if err = EnsurePathNonPrefix(localPath); err != nil { - return errors.Wrap(err, "") + return err } if err = EnsurePathNonPrefix(hdfsEndpoints); err != nil { - return errors.Wrap(err, "") + return err } if err = EnsurePathNonPrefix(s3Endpoints); err != nil { - return errors.Wrap(err, "") + return err } for _, policy := range conf.Storage.Policies { for _, vol := range policy.Volumns { @@ -294,7 +294,7 @@ func checkAccess(localPath string, conf *model.CKManClickHouseConfig) error { } output, err := common.RemoteExecute(sshOpts, cmd) if err != nil { - return errors.Wrap(err, "") + return err } access := strings.Trim(output, "\n") if access != "0" { diff --git a/controller/task.go b/controller/task.go index 30d62c90..27a72b83 100644 --- a/controller/task.go +++ b/controller/task.go @@ -2,12 +2,14 @@ package controller import ( "fmt" + "sort" + "strings" + "github.com/gin-gonic/gin" "github.com/go-errors/errors" "github.com/housepower/ckman/common" "github.com/housepower/ckman/model" "github.com/housepower/ckman/repository" - "strings" ) const ( @@ -64,7 +66,7 @@ func (t *TaskController) TasksList(c *gin.Context) { model.WrapMsg(c, model.GET_TASK_FAIL, err) return } - var resps []model.TaskResp + var resps model.TaskResps for _, task := range tasks { typ := strings.Split(task.TaskType, ".")[0] option := model.TaskOptionMap[task.TaskType] @@ -82,8 +84,11 @@ func (t *TaskController) TasksList(c *gin.Context) { resps = append(resps, resp) } if len(resps) == 0 { - resps = []model.TaskResp{} + resps = model.TaskResps{} } + + //sort by updateTime + sort.Sort(resps) model.WrapMsg(c, model.SUCCESS, resps) } @@ -130,7 +135,6 @@ func (t *TaskController) DeleteTask(c *gin.Context) { model.WrapMsg(c, model.SUCCESS, nil) } - // @Summary StopTask // @Description stop task by taskid // @version 1.0 @@ -163,4 +167,4 @@ func (t *TaskController) StopTask(c *gin.Context) { } model.WrapMsg(c, model.SUCCESS, nil) -} \ No newline at end of file +} diff --git a/deploy/ck.go b/deploy/ck.go index 220268c8..086f9c95 100644 --- a/deploy/ck.go +++ b/deploy/ck.go @@ -97,7 +97,7 @@ func (d *CKDeploy) Init() error { } common.Pool.Wait() if lastError != nil { - return errors.Wrap(lastError, "") + return lastError } clusterNodeNum := 0 @@ -134,7 +134,7 @@ func (d *CKDeploy) Init() error { } common.Pool.Wait() if lastError != nil { - return errors.Wrap(lastError, "") + return lastError } if len(HostNameMap) != clusterNodeNum { @@ -172,7 +172,7 @@ func (d *CKDeploy) Prepare() error { } common.Pool.Wait() if lastError != nil { - return errors.Wrap(lastError, "") + return lastError } log.Logger.Infof("prepare done") return nil @@ -234,7 +234,7 @@ func (d *CKDeploy) Install() error { } common.Pool.Wait() if lastError != nil { - return errors.Wrap(lastError, "") + return lastError } log.Logger.Infof("install done") return nil @@ -273,7 +273,7 @@ func (d *CKDeploy) Uninstall() error { } common.Pool.Wait() if lastError != nil { - return errors.Wrap(lastError, "") + return lastError } log.Logger.Infof("uninstall done") return nil @@ -305,7 +305,7 @@ func (d *CKDeploy) Upgrade() error { } common.Pool.Wait() if lastError != nil { - return errors.Wrap(lastError, "") + return lastError } log.Logger.Infof("upgrade done") return nil @@ -318,14 +318,14 @@ func (d *CKDeploy) Config() error { if d.Conf.LogicCluster == nil { metrika, err := ckconfig.GenerateMetrikaXML(path.Join(config.GetWorkDirectory(), "package", "metrika.xml"), d.Conf) if err != nil { - return errors.Wrap(err, "") + return err } confFiles = append(confFiles, metrika) } custom, err := ckconfig.GenerateCustomXML(path.Join(config.GetWorkDirectory(), "package", "custom.xml"), d.Conf, d.Ext.Ipv6Enable) if err != nil { - return errors.Wrap(err, "") + return err } confFiles = append(confFiles, custom) @@ -408,7 +408,7 @@ func (d *CKDeploy) Config() error { } common.Pool.Wait() if lastError != nil { - return errors.Wrap(lastError, "") + return lastError } if d.Conf.LogicCluster != nil { logicMetrika, deploys := GenLogicMetrika(d) @@ -450,7 +450,7 @@ func (d *CKDeploy) Config() error { } common.Pool.Wait() if lastError != nil { - return errors.Wrap(lastError, "") + return lastError } } } @@ -484,7 +484,7 @@ func (d *CKDeploy) Start() error { } common.Pool.Wait() if lastError != nil { - return errors.Wrap(lastError, "") + return lastError } log.Logger.Infof("start done") return nil @@ -516,7 +516,7 @@ func (d *CKDeploy) Stop() error { } common.Pool.Wait() if lastError != nil { - return errors.Wrap(lastError, "") + return lastError } log.Logger.Infof("stop done") return nil @@ -548,7 +548,7 @@ func (d *CKDeploy) Restart() error { } common.Pool.Wait() if lastError != nil { - return errors.Wrap(lastError, "") + return lastError } log.Logger.Infof("restart done") return nil @@ -591,7 +591,7 @@ func (d *CKDeploy) Check(timeout int) error { common.Pool.Wait() if lastError != nil { - return errors.Wrap(lastError, "") + return lastError } log.Logger.Infof("check done") return nil @@ -614,10 +614,10 @@ func StartCkCluster(conf *model.CKManClickHouseConfig) error { deploy.Conf.Hosts = chHosts if err := deploy.Start(); err != nil { - return errors.Wrap(err, "") + return err } if err := deploy.Check(model.MaxTimeOut); err != nil { - return errors.Wrap(err, "") + return err } return nil } @@ -653,7 +653,7 @@ func ConfigLogicOtherCluster(clusterName string) error { deploy.Conf.Normalize() logicFile, err := common.NewTempFile(path.Join(config.GetWorkDirectory(), "package"), "metrika") if err != nil { - return errors.Wrap(err, "") + return err } defer os.Remove(logicFile.FullName) m, _ := ckconfig.GenerateMetrikaXMLwithLogic(logicFile.FullName, deploy.Conf, metrika) @@ -691,7 +691,7 @@ func ConfigLogicOtherCluster(clusterName string) error { } common.Pool.Wait() if lastError != nil { - return errors.Wrap(lastError, "") + return lastError } } return nil @@ -769,7 +769,7 @@ func ClearLogicCluster(cluster, logic string, reconf bool) error { if reconf { for _, newLogic := range newPhysics { if err = ConfigLogicOtherCluster(newLogic); err != nil { - return errors.Wrap(err, "") + return err } } } diff --git a/deploy/task.go b/deploy/task.go index 2be6851f..8f5bb1a4 100644 --- a/deploy/task.go +++ b/deploy/task.go @@ -2,13 +2,14 @@ package deploy import ( "fmt" + "time" + "github.com/go-basic/uuid" "github.com/housepower/ckman/common" "github.com/housepower/ckman/log" "github.com/housepower/ckman/model" "github.com/housepower/ckman/repository" "github.com/pkg/errors" - "time" ) func CreateNewTask(clusterName, taskType string, deploy interface{}) (string, error) { @@ -52,7 +53,7 @@ func CreateNewTask(clusterName, taskType string, deploy interface{}) (string, er } err := repository.Ps.CreateTask(task) if err != nil { - return "", errors.Wrap(err, "") + return "", err } return task.TaskId, nil } @@ -63,7 +64,7 @@ func hasEffectiveTasks(clusterName string) bool { return false } for _, task := range tasks { - if task.Status == model.TaskStatusFailed || task.Status == model.TaskStatusSuccess || task.Status == model.TaskStatusStopped{ + if task.Status == model.TaskStatusFailed || task.Status == model.TaskStatusSuccess || task.Status == model.TaskStatusStopped { continue } if clusterName == task.ClusterName { diff --git a/go.mod b/go.mod index f960ef55..ce797a3b 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,6 @@ require ( github.com/nacos-group/nacos-sdk-go v1.0.8 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 - github.com/pkg/sftp v1.13.0 github.com/prometheus/client_golang v1.10.0 github.com/prometheus/common v0.27.0 github.com/spf13/cobra v1.1.3 @@ -30,7 +29,7 @@ require ( github.com/swaggo/swag v1.7.1 github.com/txn2/txeh v1.3.0 github.com/valyala/fastjson v1.6.3 - go.uber.org/zap v1.17.0 + go.uber.org/zap v1.21.0 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 golang.org/x/term v0.0.0-20210503060354-a79de5458b56 gopkg.in/natefinch/lumberjack.v2 v2.0.0 @@ -46,6 +45,7 @@ require ( gorm.io/driver/mysql v1.3.3 gorm.io/driver/postgres v1.3.4 gorm.io/gorm v1.23.1 + moul.io/zapgorm2 v1.1.3 ) require ( @@ -86,7 +86,6 @@ require ( github.com/jinzhu/now v1.1.4 // indirect github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/kr/fs v0.1.0 // indirect github.com/leodido/go-urn v1.2.0 // indirect github.com/lestrrat/go-file-rotatelogs v0.0.0-20180223000712-d3151e2a480f // indirect github.com/lestrrat/go-strftime v0.0.0-20180220042222-ba3bf9c1d042 // indirect @@ -100,11 +99,11 @@ require ( github.com/toolkits/concurrent v0.0.0-20150624120057-a4371d70e3e3 // indirect github.com/ugorji/go/codec v1.1.13 // indirect go.uber.org/atomic v1.7.0 // indirect - go.uber.org/multierr v1.6.0 // indirect + go.uber.org/multierr v1.7.0 // indirect golang.org/x/net v0.0.0-20210525063256-abc453219eb5 // indirect golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect golang.org/x/text v0.3.7 // indirect - golang.org/x/tools v0.1.0 // indirect + golang.org/x/tools v0.1.5 // indirect google.golang.org/protobuf v1.25.0 // indirect gopkg.in/ini.v1 v1.51.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index b62482bc..39f11276 100644 --- a/go.sum +++ b/go.sum @@ -72,6 +72,8 @@ github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6l github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -422,8 +424,6 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -534,8 +534,6 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= -github.com/pkg/sftp v1.13.0 h1:Riw6pgOKK41foc1I1Uu03CjvbLZDXeGpInycM4shXoI= -github.com/pkg/sftp v1.13.0/go.mod h1:41g+FIPlQUTDCveupEmEA65IoiQFrtgCeDopC4ajGIM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= @@ -663,6 +661,7 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -680,18 +679,21 @@ go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= +go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= -go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= +go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -705,7 +707,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -740,8 +741,9 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -782,6 +784,7 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210525063256-abc453219eb5 h1:wjuX4b5yYQnEQHzd+CBcrcC6OVR2J1CN6mUy0oSxIPo= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -801,6 +804,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -852,7 +856,9 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -925,8 +931,9 @@ golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1070,6 +1077,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +moul.io/zapgorm2 v1.1.3 h1:PP9224dk0l2f56KE1anr3vcS2HzKV9PusKUE6UT9ncI= +moul.io/zapgorm2 v1.1.3/go.mod h1:HTO6sXgHhQD0s2D9HA4xcnJ+qxFRFwsCUxIeFDnKtq0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/log/log.go b/log/log.go index fd6f8a97..2a5c4dbf 100644 --- a/log/log.go +++ b/log/log.go @@ -1,14 +1,16 @@ package log import ( + "strings" + "github.com/housepower/ckman/config" "go.uber.org/zap" "go.uber.org/zap/zapcore" "gopkg.in/natefinch/lumberjack.v2" - "strings" ) var Logger *zap.SugaredLogger +var ZapLog *zap.Logger func InitLogger(path string, config *config.CKManLogConfig) { errPath := strings.TrimSuffix(path, ".log") + ".err.log" @@ -20,11 +22,11 @@ func InitLogger(path string, config *config.CKManLogConfig) { infocore := zapcore.NewCore(encoder, writeSyncer, level) errcore := zapcore.NewCore(encoder, errSyncer, zapcore.ErrorLevel) core := zapcore.NewTee(infocore, errcore) - logger := zap.New(core, zap.AddCaller(), zap.AddStacktrace(zap.ErrorLevel)) - Logger = logger.Sugar() + ZapLog = zap.New(core, zap.AddCaller(), zap.AddStacktrace(zap.ErrorLevel)) + Logger = ZapLog.Sugar() } -func InitLoggerConsole(){ +func InitLoggerConsole() { cfg := zap.NewProductionConfig() cfg.Encoding = "console" cfg.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder diff --git a/model/task.go b/model/task.go index f782b592..4fbdefba 100644 --- a/model/task.go +++ b/model/task.go @@ -1,8 +1,9 @@ package model import ( - "github.com/pkg/errors" "time" + + "github.com/pkg/errors" ) var ( @@ -118,3 +119,13 @@ type TaskResp struct { UpdateTime time.Time Duration string } + +type TaskResps []TaskResp + +func SortResps(v1, v2 TaskResp) bool { + return v1.UpdateTime.After(v2.UpdateTime) +} + +func (v TaskResps) Len() int { return len(v) } +func (v TaskResps) Swap(i, j int) { v[i], v[j] = v[j], v[i] } +func (v TaskResps) Less(i, j int) bool { return SortResps(v[i], v[j]) } diff --git a/repository/local/local.go b/repository/local/local.go index 5d253ac1..99cb82c5 100644 --- a/repository/local/local.go +++ b/repository/local/local.go @@ -3,17 +3,18 @@ package local import ( "encoding/json" "fmt" + "os" + "path" + "sort" + "sync" + "time" + "github.com/housepower/ckman/common" "github.com/housepower/ckman/log" "github.com/housepower/ckman/model" "github.com/housepower/ckman/repository" "github.com/pkg/errors" "gopkg.in/yaml.v3" - "os" - "path" - "sort" - "sync" - "time" ) type LocalPersistent struct { @@ -218,7 +219,7 @@ func (lp *LocalPersistent) GetAllQueryHistory() (map[string]model.QueryHistory, for k, v := range lp.Data.QueryHistory { historys[k] = v } - return historys,nil + return historys, nil } func (lp *LocalPersistent) GetQueryHistoryByCluster(cluster string) ([]model.QueryHistory, error) { @@ -231,15 +232,15 @@ func (lp *LocalPersistent) GetQueryHistoryByCluster(cluster string) ([]model.Que } } sort.Sort(sort.Reverse(historys)) - return historys,nil + return historys, nil } -func (lp *LocalPersistent) GetQueryHistoryByCheckSum(checksum string)(model.QueryHistory, error){ +func (lp *LocalPersistent) GetQueryHistoryByCheckSum(checksum string) (model.QueryHistory, error) { lp.lock.RLock() defer lp.lock.RUnlock() history, ok := lp.Data.QueryHistory[checksum] if !ok { - return model.QueryHistory{},repository.ErrRecordNotFound + return model.QueryHistory{}, repository.ErrRecordNotFound } return history, nil } @@ -350,11 +351,10 @@ func (lp *LocalPersistent) DeleteTask(id string) error { func (lp *LocalPersistent) GetAllTasks() ([]model.Task, error) { lp.lock.RLock() defer lp.lock.RUnlock() - var tasks Tasks + var tasks []model.Task for _, value := range lp.Data.Task { tasks = append(tasks, value) } - sort.Sort(tasks) return tasks, nil } @@ -387,7 +387,7 @@ func (lp *LocalPersistent) GetTaskbyTaskId(id string) (model.Task, error) { defer lp.lock.RUnlock() task, ok := lp.Data.Task[id] if !ok { - return model.Task{},repository.ErrRecordNotFound + return model.Task{}, repository.ErrRecordNotFound } return task, nil } diff --git a/repository/local/model.go b/repository/local/model.go index 1db855b1..c2415599 100644 --- a/repository/local/model.go +++ b/repository/local/model.go @@ -20,13 +20,3 @@ func CompareHistory(v1, v2 model.QueryHistory) bool { func (v Historys) Len() int { return len(v) } func (v Historys) Swap(i, j int) { v[i], v[j] = v[j], v[i] } func (v Historys) Less(i, j int) bool { return CompareHistory(v[i], v[j]) } - -type Tasks []model.Task - -func (v Tasks) Len() int { return len(v) } -func (v Tasks) Swap(i, j int) { v[i], v[j] = v[j], v[i] } -func (v Tasks) Less(i, j int) bool { - return func(v1, v2 model.Task) bool { - return v1.Status < v2.Status - }(v[i], v[j]) -} diff --git a/repository/mysql/mysql.go b/repository/mysql/mysql.go index ab33537a..93682ba9 100644 --- a/repository/mysql/mysql.go +++ b/repository/mysql/mysql.go @@ -3,14 +3,16 @@ package mysql import ( "encoding/json" "fmt" + "strings" + "time" + "github.com/housepower/ckman/log" "github.com/housepower/ckman/model" "github.com/housepower/ckman/repository" "github.com/pkg/errors" driver "gorm.io/driver/mysql" "gorm.io/gorm" - "strings" - "time" + "moul.io/zapgorm2" ) type MysqlPersistent struct { @@ -33,7 +35,11 @@ func (mp *MysqlPersistent) Init(config interface{}) error { mp.Config.DataBase) log.Logger.Debugf("mysql dsn:%s", dsn) - db, err := gorm.Open(driver.Open(dsn), &gorm.Config{}) + logger := zapgorm2.New(log.ZapLog) + logger.SetAsDefault() + db, err := gorm.Open(driver.Open(dsn), &gorm.Config{ + Logger: logger, + }) if err != nil { return errors.Wrap(err, "") } @@ -367,6 +373,7 @@ func (mp *MysqlPersistent) CreateTask(task model.Task) error { } func (mp *MysqlPersistent) UpdateTask(task model.Task) error { + task.UpdateTime = time.Now() config, err := json.Marshal(task) if err != nil { return errors.Wrap(err, "") @@ -387,9 +394,9 @@ func (mp *MysqlPersistent) DeleteTask(id string) error { func (mp *MysqlPersistent) GetAllTasks() ([]model.Task, error) { var tables []TblTask - tx := mp.Client.Find(&tables).Order("status") - if tx.Error != nil && tx.Error != gorm.ErrRecordNotFound { - return nil, errors.Wrap(tx.Error, "") + tx := mp.Client.Find(&tables) + if tx.Error != nil && !errors.Is(tx.Error, gorm.ErrRecordNotFound) { + return nil, tx.Error } var tasks []model.Task var err error @@ -447,10 +454,10 @@ func (mp *MysqlPersistent) GetTaskbyTaskId(id string) (model.Task, error) { } func wrapError(err error) error { - if err == gorm.ErrRecordNotFound { + if errors.Is(err, gorm.ErrRecordNotFound) { err = repository.ErrRecordNotFound } - return errors.Wrap(err, "") + return err } func NewMysqlPersistent() *MysqlPersistent { diff --git a/repository/postgres/postgres.go b/repository/postgres/postgres.go index 424e93ea..6d95b71d 100644 --- a/repository/postgres/postgres.go +++ b/repository/postgres/postgres.go @@ -3,15 +3,16 @@ package postgres import ( "encoding/json" "fmt" + "strings" + "time" + "github.com/housepower/ckman/log" "github.com/housepower/ckman/model" "github.com/housepower/ckman/repository" "github.com/pkg/errors" driver "gorm.io/driver/postgres" "gorm.io/gorm" - "gorm.io/gorm/logger" - "strings" - "time" + "moul.io/zapgorm2" ) type PostgresPersistent struct { @@ -34,8 +35,10 @@ func (mp *PostgresPersistent) Init(config interface{}) error { mp.Config.Password) log.Logger.Debugf("postgres dsn:%s", dsn) + logger := zapgorm2.New(log.ZapLog) + logger.SetAsDefault() db, err := gorm.Open(driver.Open(dsn), &gorm.Config{ - Logger: logger.Default.LogMode(logger.Info), + Logger: logger, }) if err != nil { return errors.Wrap(err, "") @@ -369,6 +372,7 @@ func (mp *PostgresPersistent) CreateTask(task model.Task) error { } func (mp *PostgresPersistent) UpdateTask(task model.Task) error { + task.UpdateTime = time.Now() config, err := json.Marshal(task) if err != nil { return errors.Wrap(err, "") @@ -389,9 +393,9 @@ func (mp *PostgresPersistent) DeleteTask(id string) error { func (mp *PostgresPersistent) GetAllTasks() ([]model.Task, error) { var tables []TblTask - tx := mp.Client.Find(&tables).Order("status") - if tx.Error != nil && tx.Error != gorm.ErrRecordNotFound { - return nil, errors.Wrap(tx.Error, "") + tx := mp.Client.Find(&tables) + if tx.Error != nil && !errors.Is(tx.Error, gorm.ErrRecordNotFound) { + return nil, tx.Error } var tasks []model.Task var err error @@ -449,10 +453,10 @@ func (mp *PostgresPersistent) GetTaskbyTaskId(id string) (model.Task, error) { } func wrapError(err error) error { - if err == gorm.ErrRecordNotFound { + if errors.Is(err, gorm.ErrRecordNotFound) { err = repository.ErrRecordNotFound } - return errors.Wrap(err, "") + return err } func NewPostgresPersistent() *PostgresPersistent { diff --git a/service/clickhouse/clickhouse_service.go b/service/clickhouse/clickhouse_service.go index b4a14a26..c35f6afe 100644 --- a/service/clickhouse/clickhouse_service.go +++ b/service/clickhouse/clickhouse_service.go @@ -66,7 +66,7 @@ func (ck *CkService) InitCkService() error { } } if !hasConnect { - return errors.Wrap(lastError, "") + return lastError } return nil } @@ -117,11 +117,10 @@ func GetCkClusterConfig(conf *model.CKManClickHouseConfig) error { value, err := service.QueryInfo(fmt.Sprintf("SELECT cluster, shard_num, replica_num, host_name, host_address FROM system.clusters WHERE cluster='%s' ORDER BY cluster, shard_num, replica_num", conf.Cluster)) if err != nil { - return errors.Wrap(err, "") + return err } if len(value) == 1 { - err = fmt.Errorf("cluster %s is not exist, or hosts %v is not in cluster %s", conf.Cluster, hosts, conf.Cluster) - return errors.Wrap(err, "") + return errors.Errorf("cluster %s is not exist, or hosts %v is not in cluster %s", conf.Cluster, hosts, conf.Cluster) } shardNum := uint32(0) for i := 1; i < len(value); i++ { @@ -154,7 +153,7 @@ func GetCkClusterConfig(conf *model.CKManClickHouseConfig) error { value, err = service.QueryInfo("SELECT version()") if err != nil { - return errors.Wrap(err, "") + return err } conf.Version = value[1][0].(string) @@ -423,10 +422,10 @@ func (ck *CkService) AlterTable(params *model.AlterCkTableParams) error { LogicCluster: *conf.LogicCluster, } if err := ck.DeleteDistTblOnLogic(&distParams); err != nil { - return errors.Wrap(err, "") + return err } if err := ck.CreateDistTblOnLogic(&distParams); err != nil { - return errors.Wrap(err, "") + return err } } @@ -443,7 +442,7 @@ func (ck *CkService) DescTable(params *model.DescCkTableParams) ([]model.CkColum log.Logger.Debugf(desc) rows, err := ck.DB.Query(desc) if err != nil { - return attrs, errors.Wrap(err, "") + return attrs, err } defer rows.Close() @@ -531,7 +530,7 @@ func (ck *CkService) QueryInfo(query string) ([][]interface{}, error) { func (ck *CkService) FetchSchemerFromOtherNode(host, password string) error { names, statements, err := business.GetCreateReplicaObjects(ck.DB, host, model.ClickHouseDefaultUser, password) if err != nil { - return errors.Wrap(err, "") + return err } num := len(names) @@ -549,13 +548,13 @@ func GetCkTableMetrics(conf *model.CKManClickHouseConfig) (map[string]*model.CkT chHosts, err := common.GetShardAvaliableHosts(conf) if err != nil { - return nil, errors.Wrap(err, "") + return nil, err } for _, host := range chHosts { service, err := GetCkNodeService(conf.Cluster, host) if err != nil { - return nil, errors.Wrap(err, "") + return nil, err } // get table names @@ -586,7 +585,7 @@ func GetCkTableMetrics(conf *model.CKManClickHouseConfig) (map[string]*model.CkT log.Logger.Infof("host: %s, query: %s", host, query) value, err = service.QueryInfo(query) if err != nil { - return nil, errors.Wrap(err, "") + return nil, err } for i := 1; i < len(value); i++ { table := value[i][0].(string) @@ -602,7 +601,7 @@ func GetCkTableMetrics(conf *model.CKManClickHouseConfig) (map[string]*model.CkT log.Logger.Infof("host: %s, query: %s", host, query) value, err = service.QueryInfo(query) if err != nil { - return nil, errors.Wrap(err, "") + return nil, err } for i := 1; i < len(value); i++ { table := value[i][0].(string) @@ -620,7 +619,7 @@ func GetCkTableMetrics(conf *model.CKManClickHouseConfig) (map[string]*model.CkT query = fmt.Sprintf("select table, is_readonly from system.replicas where database = '%s'", database) value, err = service.QueryInfo(query) if err != nil { - return nil, errors.Wrap(err, "") + return nil, err } for i := 1; i < len(value); i++ { table := value[i][0].(string) @@ -638,7 +637,7 @@ func GetCkTableMetrics(conf *model.CKManClickHouseConfig) (map[string]*model.CkT log.Logger.Infof("host: %s, query: %s", host, query) value, err = service.QueryInfo(query) if err != nil { - return nil, errors.Wrap(err, "") + return nil, err } for i := 1; i < len(value); i++ { table := value[i][0].(string) @@ -658,7 +657,7 @@ func GetCkTableMetrics(conf *model.CKManClickHouseConfig) (map[string]*model.CkT log.Logger.Infof("host: %s, query: %s", host, query) value, err = service.QueryInfo(query) if err != nil { - return nil, errors.Wrap(err, "") + return nil, err } for i := 1; i < len(value); i++ { table := value[i][0].(string) @@ -693,7 +692,7 @@ func getHostSessions(service *CkService, query, host string) ([]*model.CkSession value, err := service.QueryInfo(query) if err != nil { - return nil, errors.Wrap(err, "") + return nil, err } for i := 1; i < len(value); i++ { session := new(model.CkSessionInfo) @@ -733,7 +732,7 @@ func getCkSessions(conf *model.CKManClickHouseConfig, limit int, query string) ( } common.Pool.Wait() if lastError != nil { - return nil, errors.Wrap(lastError, "") + return nil, lastError } sort.Sort(model.SessionList(list)) @@ -753,7 +752,7 @@ func GetCkOpenSessions(conf *model.CKManClickHouseConfig, limit int) ([]*model.C func KillCkOpenSessions(conf *model.CKManClickHouseConfig, host, queryId string) error { db, err := common.ConnectClickHouse(host, conf.Port, clickhouse.DefaultDatabase, conf.User, conf.Password) if err != nil { - return errors.Wrap(err, "") + return err } query := fmt.Sprintf("KILL QUERY WHERE query_id = '%s'", queryId) _, err = db.Exec(query) @@ -774,12 +773,12 @@ func GetReplicaZkPath(conf *model.CKManClickHouseConfig) error { service := NewCkService(conf) if err = service.InitCkService(); err != nil { log.Logger.Errorf("all hosts not available, can't get zoopath") - return errors.Wrap(err, "") + return err } databases, dbtables, err := common.GetMergeTreeTables("Replicated\\\\w*MergeTree", service.DB) if err != nil { - return errors.Wrap(err, "") + return err } // clear and reload again @@ -789,7 +788,7 @@ func GetReplicaZkPath(conf *model.CKManClickHouseConfig) error { for _, table := range tables { path, err := getReplicaZkPath(service.DB, database, table) if err != nil { - return errors.Wrap(err, "") + return err } tableName := fmt.Sprintf("%s.%s", database, table) conf.ZooPath[tableName] = path @@ -889,7 +888,7 @@ func (ck *CkService) ShowCreateTable(tbname, database string) (string, error) { query := fmt.Sprintf("SELECT create_table_query FROM system.tables WHERE database = '%s' AND name = '%s'", database, tbname) value, err := ck.QueryInfo(query) if err != nil { - return "", errors.Wrap(err, "") + return "", err } schema := value[1][0].(string) return schema, nil @@ -932,7 +931,7 @@ ORDER BY tblLists[database] = tblMapping } } - return tblLists, errors.Wrap(err, "") + return tblLists, err } func GetCKVersion(conf *model.CKManClickHouseConfig, host string) (string, error) { @@ -940,11 +939,11 @@ func GetCKVersion(conf *model.CKManClickHouseConfig, host string) (string, error tmp.Hosts = []string{host} service, err := GetCkService(conf.Cluster) if err != nil { - return "", errors.Wrap(err, "") + return "", err } value, err := service.QueryInfo("SELECT version()") if err != nil { - return "", errors.Wrap(err, "") + return "", err } version := value[1][0].(string) return version, nil diff --git a/service/cron/clickhouse.go b/service/cron/clickhouse.go index d0feefd6..275e0f7f 100644 --- a/service/cron/clickhouse.go +++ b/service/cron/clickhouse.go @@ -62,11 +62,11 @@ func syncLogicbyTable(clusters []string, database, localTable string) error { for _, cluster := range clusters { conf, err := repository.Ps.GetClusterbyName(cluster) if err != nil { - return errors.Wrap(err, "") + return err } ckService := clickhouse.NewCkService(&conf) if err = ckService.InitCkService(); err != nil { - return errors.Wrap(err, "") + return err } query := fmt.Sprintf("SELECT name, type FROM system.columns WHERE database = '%s' AND table = '%s'", database, localTable) log.Logger.Debugf("query: %s", query) @@ -102,11 +102,11 @@ func syncLogicbyTable(clusters []string, database, localTable string) error { needAdds := allCols.Difference(cols).(common.Map) conf, err := repository.Ps.GetClusterbyName(cluster) if err != nil { - return errors.Wrap(err, "") + return err } ckService := clickhouse.NewCkService(&conf) if err = ckService.InitCkService(); err != nil { - return errors.Wrap(err, "") + return err } for k, v := range needAdds { query := fmt.Sprintf("ALTER TABLE `%s`.`%s` ON CLUSTER `%s` ADD COLUMN `%s` %s", database, localTable, cluster, k, v) @@ -140,10 +140,10 @@ func syncLogicbyTable(clusters []string, database, localTable string) error { LogicCluster: *conf.LogicCluster, } if err = ckService.DeleteDistTblOnLogic(&distParams); err != nil { - return errors.Wrap(err, "") + return err } if err = ckService.CreateDistTblOnLogic(&distParams); err != nil { - return errors.Wrap(err, "") + return err } } } diff --git a/service/runner/ck.go b/service/runner/ck.go index dd6cd98e..a4b744be 100644 --- a/service/runner/ck.go +++ b/service/runner/ck.go @@ -15,32 +15,32 @@ import ( func DeployCkCluster(task *model.Task, d deploy.CKDeploy) error { deploy.SetNodeStatus(task, model.NodeStatusInit, model.ALL_NODES_DEFAULT) if err := d.Init(); err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusInit.EN) } deploy.SetNodeStatus(task, model.NodeStatusPrepare, model.ALL_NODES_DEFAULT) if err := d.Prepare(); err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusPrepare.EN) } deploy.SetNodeStatus(task, model.NodeStatusInstall, model.ALL_NODES_DEFAULT) if err := d.Install(); err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusInstall.EN) } deploy.SetNodeStatus(task, model.NodeStatusConfig, model.ALL_NODES_DEFAULT) if err := d.Config(); err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusConfig.EN) } deploy.SetNodeStatus(task, model.NodeStatusStart, model.ALL_NODES_DEFAULT) if err := d.Start(); err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusStart.EN) } deploy.SetNodeStatus(task, model.NodeStatusCheck, model.ALL_NODES_DEFAULT) if err := d.Check(10); err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusCheck.EN) } return nil } @@ -51,25 +51,25 @@ func DestroyCkCluster(task *model.Task, d deploy.CKDeploy, conf *model.CKManClic deploy.SetNodeStatus(task, model.NodeStatusUninstall, model.ALL_NODES_DEFAULT) if err := d.Uninstall(); err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusInstall.EN) } //clear zkNode deploy.SetNodeStatus(task, model.NodeStatusClearData, model.ALL_NODES_DEFAULT) service, err := zookeeper.NewZkService(conf.ZkNodes, conf.ZkPort) if err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusClearData.EN) } //delete from standard path stdZooPath := fmt.Sprintf("/clickhouse/tables/%s", conf.Cluster) if err = service.DeleteAll(stdZooPath); err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusClearData.EN) } zooPaths := clickhouse.ConvertZooPath(conf) if len(zooPaths) > 0 { for _, zooPath := range zooPaths { if err = service.DeleteAll(zooPath); err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusClearData.EN) } } } @@ -107,7 +107,7 @@ func DeleteCkClusterNode(task *model.Task, conf *model.CKManClickHouseConfig, ip service, err := zookeeper.NewZkService(conf.ZkNodes, conf.ZkPort) if err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusClearData.EN) } _ = clickhouse.GetReplicaZkPath(conf) var zooPaths []string @@ -123,7 +123,7 @@ func DeleteCkClusterNode(task *model.Task, conf *model.CKManClickHouseConfig, ip shardNode := fmt.Sprintf("%d", shardNum+1) err = service.DeletePathUntilNode(path, shardNode) if err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusClearData.EN) } } else { // delete replica path @@ -132,7 +132,7 @@ func DeleteCkClusterNode(task *model.Task, conf *model.CKManClickHouseConfig, ip log.Logger.Debugf("replicaPath: %s", replicaPath) err = service.DeleteAll(replicaPath) if err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusClearData.EN) } } } @@ -179,10 +179,10 @@ func DeleteCkClusterNode(task *model.Task, conf *model.CKManClickHouseConfig, ip d.Conf.Hosts = hosts d.Conf.Shards = shards if err = d.Init(); err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusConfigExt.EN) } if err = d.Config(); err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusConfigExt.EN) } conf.Hosts = hosts @@ -193,32 +193,32 @@ func DeleteCkClusterNode(task *model.Task, conf *model.CKManClickHouseConfig, ip func AddCkClusterNode(task *model.Task, conf *model.CKManClickHouseConfig, d *deploy.CKDeploy) error { deploy.SetNodeStatus(task, model.NodeStatusInit, model.ALL_NODES_DEFAULT) if err := d.Init(); err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusInit.EN) } deploy.SetNodeStatus(task, model.NodeStatusPrepare, model.ALL_NODES_DEFAULT) if err := d.Prepare(); err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusPrepare.EN) } deploy.SetNodeStatus(task, model.NodeStatusInstall, model.ALL_NODES_DEFAULT) if err := d.Install(); err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusInstall.EN) } deploy.SetNodeStatus(task, model.NodeStatusConfig, model.ALL_NODES_DEFAULT) if err := d.Config(); err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusConfig.EN) } deploy.SetNodeStatus(task, model.NodeStatusStart, model.ALL_NODES_DEFAULT) if err := d.Start(); err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusStart.EN) } deploy.SetNodeStatus(task, model.NodeStatusCheck, model.ALL_NODES_DEFAULT) if err := d.Check(10); err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusCheck.EN) } // update other nodes config @@ -226,10 +226,10 @@ func AddCkClusterNode(task *model.Task, conf *model.CKManClickHouseConfig, d *de d2 := deploy.NewCkDeploy(*conf) d2.Conf.Shards = d.Conf.Shards if err := d2.Init(); err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusConfigExt.EN) } if err := d2.Config(); err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusConfig.EN) } conf.Shards = d.Conf.Shards @@ -243,13 +243,13 @@ func UpgradeCkCluster(task *model.Task, d deploy.CKDeploy) error { for _, host := range d.Conf.Hosts { d.Conf.Hosts = []string{host} if err := upgradePackage(task, d, model.MaxTimeOut); err != nil { - return errors.Wrap(err, "") + return err } } case model.UpgradePolicyFull: err := upgradePackage(task, d, 10) if err != model.CheckTimeOutErr { - return errors.Wrap(err, "") + return err } default: return fmt.Errorf("not support policy %s yet", d.Ext.UpgradePolicy) @@ -268,7 +268,7 @@ func upgradePackage(task *model.Task, d deploy.CKDeploy, timeout int) error { deploy.SetNodeStatus(task, model.NodeStatusInit, node) if err := d.Init(); err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusInit.EN) } deploy.SetNodeStatus(task, model.NodeStatusStop, node) if err := d.Stop(); err != nil { @@ -277,27 +277,27 @@ func upgradePackage(task *model.Task, d deploy.CKDeploy, timeout int) error { deploy.SetNodeStatus(task, model.NodeStatusPrepare, node) if err := d.Prepare(); err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusPrepare.EN) } deploy.SetNodeStatus(task, model.NodeStatusUpgrade, node) if err := d.Upgrade(); err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusUpgrade.EN) } deploy.SetNodeStatus(task, model.NodeStatusConfig, node) if err := d.Config(); err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusConfig.EN) } deploy.SetNodeStatus(task, model.NodeStatusStart, node) if err := d.Start(); err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusStart.EN) } deploy.SetNodeStatus(task, model.NodeStatusCheck, node) if err := d.Check(timeout); err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusCheck.EN) } deploy.SetNodeStatus(task, model.NodeStatusDone, node) @@ -307,17 +307,17 @@ func upgradePackage(task *model.Task, d deploy.CKDeploy, timeout int) error { func ConfigCkCluster(task *model.Task, d deploy.CKDeploy) error { deploy.SetNodeStatus(task, model.NodeStatusInit, model.ALL_NODES_DEFAULT) if err := d.Init(); err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusInit.EN) } deploy.SetNodeStatus(task, model.NodeStatusConfig, model.ALL_NODES_DEFAULT) if err := d.Config(); err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusConfig.EN) } if d.Ext.Restart { deploy.SetNodeStatus(task, model.NodeStatusRestart, model.ALL_NODES_DEFAULT) if err := d.Restart(); err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusRestart.EN) } _ = d.Check(10) } diff --git a/service/runner/handle.go b/service/runner/handle.go index 2e93acc4..8482e09b 100644 --- a/service/runner/handle.go +++ b/service/runner/handle.go @@ -41,11 +41,11 @@ func UnmarshalConfig(config interface{}, v interface{}) error { func CKDeployHandle(task *model.Task) error { var d deploy.CKDeploy if err := UnmarshalConfig(task.DeployConfig, &d); err != nil { - return errors.Wrap(err, "") + return err } if err := DeployCkCluster(task, d); err != nil { - return errors.Wrap(err, "") + return err } // sync table schema when logic cluster exists @@ -62,25 +62,27 @@ func CKDeployHandle(task *model.Task) error { } } } - if err := repository.Ps.Begin(); err != nil { return err } + if d.Conf.AuthenticateType != model.SshPasswordSave { + d.Conf.SshPassword = "" + } if err := repository.Ps.CreateCluster(*d.Conf); err != nil { _ = repository.Ps.Rollback() - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusStore.EN) } if d.Conf.LogicCluster != nil { logics, err := repository.Ps.GetLogicClusterbyName(*d.Conf.LogicCluster) if err != nil { - if err == repository.ErrRecordNotFound { + if errors.Is(err, repository.ErrRecordNotFound) { if !common.ArraySearch(d.Conf.Cluster, logics) { logics = append(logics, d.Conf.Cluster) } _ = repository.Ps.CreateLogicCluster(*d.Conf.LogicCluster, logics) } else { _ = repository.Ps.Rollback() - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusStore.EN) } } else { if !common.ArraySearch(d.Conf.Cluster, logics) { @@ -97,7 +99,7 @@ func CKDeployHandle(task *model.Task) error { func CKDestoryHandle(task *model.Task) error { var d deploy.CKDeploy if err := UnmarshalConfig(task.DeployConfig, &d); err != nil { - return errors.Wrap(err, "") + return err } conf, err := repository.Ps.GetClusterbyName(d.Conf.Cluster) @@ -107,7 +109,7 @@ func CKDestoryHandle(task *model.Task) error { common.CloseConns(conf.Hosts) if err = DestroyCkCluster(task, d, &conf); err != nil { - return errors.Wrap(err, "") + return err } deploy.SetNodeStatus(task, model.NodeStatusStore, model.ALL_NODES_DEFAULT) @@ -117,7 +119,7 @@ func CKDestoryHandle(task *model.Task) error { if conf.LogicCluster != nil { if err = deploy.ClearLogicCluster(conf.Cluster, *conf.LogicCluster, true); err != nil { _ = repository.Ps.Rollback() - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusStore.EN) } } @@ -144,7 +146,7 @@ func CKDestoryHandle(task *model.Task) error { if err = repository.Ps.DeleteCluster(conf.Cluster); err != nil { _ = repository.Ps.Rollback() - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusStore.EN) } _ = repository.Ps.Commit() @@ -166,13 +168,13 @@ func CKDeleteNodeHandle(task *model.Task) error { err = DeleteCkClusterNode(task, &conf, ip) if err != nil { - return errors.Wrap(err, "") + return err } common.CloseConns([]string{ip}) deploy.SetNodeStatus(task, model.NodeStatusStore, model.ALL_NODES_DEFAULT) if err = repository.Ps.UpdateCluster(conf); err != nil { - return err + return errors.Wrapf(err, "[%s]", model.NodeStatusStore.EN) } deploy.SetNodeStatus(task, model.NodeStatusDone, model.ALL_NODES_DEFAULT) return nil @@ -191,7 +193,7 @@ func CKAddNodeHandle(task *model.Task) error { err = AddCkClusterNode(task, &conf, &d) if err != nil { - return errors.Wrap(err, "") + return err } deploy.SetNodeStatus(task, model.NodeStatusConfigExt, model.ALL_NODES_DEFAULT) @@ -205,15 +207,15 @@ func CKAddNodeHandle(task *model.Task) error { service := clickhouse.NewCkService(tmp) if err := service.InitCkService(); err != nil { - return err + return errors.Wrapf(err, "[%s]", model.NodeStatusConfigExt.EN) } if err := service.FetchSchemerFromOtherNode(conf.Hosts[0], conf.Password); err != nil { - return err + return errors.Wrapf(err, "[%s]", model.NodeStatusConfigExt.EN) } deploy.SetNodeStatus(task, model.NodeStatusStore, model.ALL_NODES_DEFAULT) if err = repository.Ps.UpdateCluster(conf); err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusStore.EN) } deploy.SetNodeStatus(task, model.NodeStatusDone, model.ALL_NODES_DEFAULT) @@ -223,7 +225,7 @@ func CKAddNodeHandle(task *model.Task) error { func CKUpgradeHandle(task *model.Task) error { var d deploy.CKDeploy if err := UnmarshalConfig(task.DeployConfig, &d); err != nil { - return errors.Wrap(err, "") + return err } conf, err := repository.Ps.GetClusterbyName(d.Conf.Cluster) @@ -233,12 +235,12 @@ func CKUpgradeHandle(task *model.Task) error { err = UpgradeCkCluster(task, d) if err != nil { - return errors.Wrap(err, "") + return err } conf.Version = d.Conf.Version if err = repository.Ps.UpdateCluster(conf); err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusStore.EN) } deploy.SetNodeStatus(task, model.NodeStatusDone, model.ALL_NODES_DEFAULT) @@ -248,11 +250,11 @@ func CKUpgradeHandle(task *model.Task) error { func CKSettingHandle(task *model.Task) error { var d deploy.CKDeploy if err := UnmarshalConfig(task.DeployConfig, &d); err != nil { - return errors.Wrap(err, "") + return err } if err := ConfigCkCluster(task, d); err != nil { - return errors.Wrap(err, "") + return err } deploy.SetNodeStatus(task, model.NodeStatusStore, model.ALL_NODES_DEFAULT) @@ -273,21 +275,24 @@ func CKSettingHandle(task *model.Task) error { } if err := repository.Ps.Begin(); err != nil { - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusStore.EN) + } + if d.Conf.AuthenticateType != model.SshPasswordSave { + d.Conf.SshPassword = "" } if err := repository.Ps.UpdateCluster(*d.Conf); err != nil { _ = repository.Ps.Rollback() - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusStore.EN) } if d.Conf.LogicCluster != nil { logics, err := repository.Ps.GetLogicClusterbyName(*d.Conf.LogicCluster) if err != nil { - if err == repository.ErrRecordNotFound { + if errors.Is(err, repository.ErrRecordNotFound) { logics = append(logics, d.Conf.Cluster) _ = repository.Ps.CreateLogicCluster(*d.Conf.LogicCluster, logics) } else { _ = repository.Ps.Rollback() - return errors.Wrap(err, "") + return errors.Wrapf(err, "[%s]", model.NodeStatusStore.EN) } } else { logics = append(logics, d.Conf.Cluster) diff --git a/service/runner/runner.go b/service/runner/runner.go index 0f0ad114..9ee1252f 100644 --- a/service/runner/runner.go +++ b/service/runner/runner.go @@ -10,7 +10,6 @@ import ( "github.com/housepower/ckman/log" "github.com/housepower/ckman/model" "github.com/housepower/ckman/repository" - "github.com/pkg/errors" ) type RunnerService struct { @@ -81,7 +80,7 @@ func (runner *RunnerService) ProcesswithTaskType(task model.Task) error { if err := TaskHandleFunc[task.TaskType](&task); err != nil { deploy.SetNodeStatus(&task, model.NodeStatusFailed, model.ALL_NODES_DEFAULT) _ = deploy.SetTaskStatus(&task, model.TaskStatusFailed, err.Error()) - return errors.Wrap(err, "") + return err } return deploy.SetTaskStatus(&task, model.TaskStatusSuccess, model.TaskStatusMap[model.TaskStatusSuccess]) } diff --git a/service/zookeeper/zookeeper_service.go b/service/zookeeper/zookeeper_service.go index d7c7ae69..14ef7157 100644 --- a/service/zookeeper/zookeeper_service.go +++ b/service/zookeeper/zookeeper_service.go @@ -58,7 +58,7 @@ func GetZkService(clusterName string) (*ZkService, error) { if err == nil { service, err := NewZkService(conf.ZkNodes, conf.ZkPort) if err != nil { - return nil, errors.Wrap(err, "") + return nil, err } ZkServiceCache.SetDefault(clusterName, service) return service, nil @@ -71,7 +71,7 @@ func GetZkService(clusterName string) (*ZkService, error) { func (z *ZkService) GetReplicatedTableStatus(conf *model.CKManClickHouseConfig) ([]model.ZkReplicatedTableStatus, error) { err := clickhouse.GetReplicaZkPath(conf) if err != nil { - return nil, errors.Wrap(err, "") + return nil, err } tableStatus := make([]model.ZkReplicatedTableStatus, len(conf.ZooPath)) @@ -134,7 +134,7 @@ func (z *ZkService) GetReplicatedTableStatus(conf *model.CKManClickHouseConfig) func (z *ZkService) DeleteAll(node string) (err error) { children, stat, err := z.Conn.Children(node) - if err == zk.ErrNoNode { + if errors.Is(err, zk.ErrNoNode) { return nil } else if err != nil { err = errors.Wrap(err, "delete zk node: ")