diff --git a/ckconfig/macros.go b/ckconfig/hosts.go similarity index 68% rename from ckconfig/macros.go rename to ckconfig/hosts.go index 1e57fae4..a0141e0c 100644 --- a/ckconfig/macros.go +++ b/ckconfig/hosts.go @@ -5,14 +5,12 @@ import ( "github.com/housepower/ckman/model" ) -func GenerateMacrosXML(filename string, conf *model.CkDeployConfig, host string)(string, error){ +func GenerateHostXML(filename string, conf *model.CkDeployConfig, host string)(string, error){ shardIndex := 0 - hostName := "" for i, shard := range conf.Shards { for _, replica := range shard.Replicas { if host == replica.Ip { shardIndex = i + 1 - hostName = replica.HostName break } } @@ -20,10 +18,12 @@ func GenerateMacrosXML(filename string, conf *model.CkDeployConfig, host string) xml := common.NewXmlFile(filename) xml.Begin("yandex") + xml.Comment("This xml file contains every node's special configuration self.") + xml.Write("interserver_http_host", host) xml.Begin("macros") xml.Write("cluster", conf.ClusterName) xml.Write("shard", shardIndex) - xml.Write("replica", hostName) + xml.Write("replica", host) xml.End("macros") xml.End("yandex") err := xml.Dump() diff --git a/ckconfig/metrika.go b/ckconfig/metrika.go index 9aab82ad..e030571c 100644 --- a/ckconfig/metrika.go +++ b/ckconfig/metrika.go @@ -59,7 +59,7 @@ func GenLocalMetrika(indent int, conf *model.CkDeployConfig)string { xml.Write("internal_replication", conf.IsReplica) for _, replica := range shard.Replicas { xml.Begin("replica") - xml.Write("host", replica.HostName) + xml.Write("host", replica.Ip) xml.Write("port", conf.CkTcpPort) xml.End("replica") } diff --git a/common/xml.go b/common/xml.go index e243f46c..64dfdaa9 100644 --- a/common/xml.go +++ b/common/xml.go @@ -74,7 +74,7 @@ func (xml *XMLFile) WritewithAttr(tag string, value interface{}, attrs []XMLAttr } xml.context += fmt.Sprintf("%s=\"%v\"", attr.Key, finalValue(attr.Value)) } - xml.context += fmt.Sprintf("%v\n", tag, value) + xml.context += fmt.Sprintf(">%v\n", value, tag) } func (xml *XMLFile) Begin(tag string) { diff --git a/controller/deploy.go b/controller/deploy.go index 1af83b05..a3c702ff 100644 --- a/controller/deploy.go +++ b/controller/deploy.go @@ -55,7 +55,7 @@ func DeployPackage(d deploy.Deploy) (string, error) { } log.Logger.Infof("start check service") - if err := d.Check(); err != nil { + if err := d.Check(5); err != nil { return model.CHECK_PACKAGE_FAIL, err } diff --git a/deploy/ck.go b/deploy/ck.go index 6e939b43..0f783cf4 100644 --- a/deploy/ck.go +++ b/deploy/ck.go @@ -34,6 +34,10 @@ type CkUpdateNodeParam struct { Op int } +var ( + CheckTimeOutErr = errors.New("check clickhouse timeout error") +) + func (d *CKDeploy) Init() error { d.Conf.Normalize() d.HostInfos = make([]ckconfig.HostInfo, len(d.Hosts)) @@ -278,18 +282,20 @@ func (d *CKDeploy) Config() error { } userFiles = append(userFiles, profiles) - macrosFile, err := common.NewTempFile(path.Join(config.GetWorkDirectory(), "package"), "macros") + hostFile, err := common.NewTempFile(path.Join(config.GetWorkDirectory(), "package"), "host") if err != nil { lastError = err return } - defer os.Remove(macrosFile.FullName) - macros, err := ckconfig.GenerateMacrosXML(macrosFile.FullName, d.Conf, innerHost) + defer os.Remove(hostFile.FullName) + hostXml, err := ckconfig.GenerateHostXML(hostFile.FullName, d.Conf, innerHost) if err != nil { lastError = err return } - confFiles = append(confFiles, macros) + confFiles = append(confFiles, hostXml) + + _, _ = common.RemoteExecute(d.User, d.Password, host, d.Port, "rm -rf /etc/clickhouse-server/config.d/* /etc/clickhouse-server/users.d/*") if err := common.ScpUploadFiles(confFiles, "/etc/clickhouse-server/config.d/", d.User, d.Password, innerHost, d.Port); err != nil { lastError = err @@ -301,7 +307,7 @@ func (d *CKDeploy) Config() error { } cmds := make([]string, 0) - cmds = append(cmds, fmt.Sprintf("mv /etc/clickhouse-server/config.d/%s /etc/clickhouse-server/config.d/macros.xml", macrosFile.BaseName)) + cmds = append(cmds, fmt.Sprintf("mv /etc/clickhouse-server/config.d/%s /etc/clickhouse-server/config.d/host.xml", hostFile.BaseName)) cmds = append(cmds, fmt.Sprintf("mv /etc/clickhouse-server/users.d/%s /etc/clickhouse-server/users.d/profiles.xml", profilesFile.BaseName)) cmds = append(cmds, "chown -R clickhouse:clickhouse /etc/clickhouse-server") @@ -415,23 +421,31 @@ func (d *CKDeploy) Restart() error { return nil } -func (d *CKDeploy) Check() error { - time.Sleep(5 * time.Second) - +func (d *CKDeploy) Check(timeout int) error { var lastError error for _, host := range d.Hosts { innerHost := host _ = d.Pool.Submit(func() { - db, err := common.ConnectClickHouse(innerHost, d.Conf.CkTcpPort, model.ClickHouseDefaultDB, d.Conf.User, d.Conf.Password) - if err != nil { - lastError = err - return - } - if err = db.Ping(); err != nil { - lastError = err - return + ticker := time.NewTicker(5*time.Second) + for { + select { + case <-ticker.C: + db, err := common.ConnectClickHouse(innerHost, d.Conf.CkTcpPort, model.ClickHouseDefaultDB, d.Conf.User, d.Conf.Password) + if err != nil { + continue + } + if err = db.Ping(); err != nil { + continue + } + if err == nil { + log.Logger.Debugf("host %s check done", innerHost) + return + } + case <-time.After(time.Duration(timeout)*time.Second): + lastError = CheckTimeOutErr + return + } } - log.Logger.Debugf("host %s check done", innerHost) }) } @@ -470,12 +484,15 @@ func UpgradeCkCluster(conf *model.CKManClickHouseConfig, req model.CkUpgradeCkRe switch req.Policy { case model.UpgradePolicyRolling: for _, host := range chHosts { - if err := upgradePackage(conf, []string{host}, packages); err != nil { + if err := upgradePackage(conf, []string{host}, packages, model.MaxTimeOut); err != nil { return err } } case model.UpgradePolicyFull: - return upgradePackage(conf, chHosts, packages) + err := upgradePackage(conf, chHosts, packages, 5) + if err != CheckTimeOutErr { + return err + } default: return fmt.Errorf("not support policy %s yet", req.Policy) } @@ -483,7 +500,7 @@ func UpgradeCkCluster(conf *model.CKManClickHouseConfig, req model.CkUpgradeCkRe return nil } -func upgradePackage(conf *model.CKManClickHouseConfig, hosts []string, packages []string) error { +func upgradePackage(conf *model.CKManClickHouseConfig, hosts []string, packages []string, timeout int) error { deploy := ConvertCKDeploy(conf) deploy.Hosts = hosts deploy.Packages = packages @@ -510,7 +527,7 @@ func upgradePackage(conf *model.CKManClickHouseConfig, hosts []string, packages return err } log.Logger.Infof("cluster start succeed ") - if err := deploy.Check(); err != nil { + if err := deploy.Check(timeout); err != nil { return err } log.Logger.Infof("cluster checked succeed ") @@ -537,7 +554,7 @@ func StartCkCluster(conf *model.CKManClickHouseConfig) error { if err := deploy.Start(); err != nil { return err } - if err := deploy.Check(); err != nil { + if err := deploy.Check(model.MaxTimeOut); err != nil { return err } return nil @@ -667,7 +684,7 @@ func AddCkClusterNode(conf *model.CKManClickHouseConfig, req *model.AddNodeReq) if err := deploy.Start(); err != nil { return err } - if err := deploy.Check(); err != nil { + if err := deploy.Check(5); err != nil { return err } @@ -821,9 +838,7 @@ func ConfigCkCluster(conf *model.CKManClickHouseConfig, restart bool)error { if err := d.Restart(); err != nil { return err } - if err := d.Check(); err != nil { - return err - } + _ = d.Check(5) } return nil } diff --git a/deploy/deploy.go b/deploy/deploy.go index 6f49c67b..77090786 100644 --- a/deploy/deploy.go +++ b/deploy/deploy.go @@ -8,5 +8,5 @@ type Deploy interface { Start() error Stop() error Restart() error - Check() error + Check(timeout int) error } diff --git a/deploy/zk.go b/deploy/zk.go index 72434d5b..49735838 100644 --- a/deploy/zk.go +++ b/deploy/zk.go @@ -34,6 +34,6 @@ func (d *ZKDeploy) Restart() error { return nil } -func (d *ZKDeploy) Check() error { +func (d *ZKDeploy) Check(timeout int) error { return nil } diff --git a/docs/guide/deploy.md b/docs/guide/deploy.md index b2709d09..b0972f1c 100644 --- a/docs/guide/deploy.md +++ b/docs/guide/deploy.md @@ -1,8 +1,8 @@ # 部署依赖 - `prometheus`(非必需) - `node_exporter`(非必需) -- `nacos`(>`1.4`，目前不支持`2.0`)(非必需) -- `zookeeper`(>`3.5.0`, 推荐 ) +- `nacos`(>`1.4`)(非必需) +- `zookeeper`(>`3.6.0`, 推荐 ) # 监控配置(可选，不影响ckman核心功能) @@ -14,7 +14,7 @@ ```ini metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider -metricsProvider.httpPort=7070 #暴露给promethues的监控端口 +metricsProvider.httpPort=7000 #暴露给promethues的监控端口 admin.enableServer=true admin.serverPort=8080 #暴露给四字命令如mntr等的监控端口，3.5.0以上版本支持 ``` diff --git a/docs/img/image-20210907094043322.png b/docs/img/image-20210907094043322.png new file mode 100644 index 00000000..fb520fa7 Binary files /dev/null and b/docs/img/image-20210907094043322.png differ diff --git a/model/deploy_ck.go b/model/deploy_ck.go index 7ba14345..93290d59 100644 --- a/model/deploy_ck.go +++ b/model/deploy_ck.go @@ -23,6 +23,8 @@ const ( SshPasswordSave int = 0 SshPasswordNotSave int = 1 SshPasswordUsePubkey int = 2 + + MaxTimeOut int = 3600 ) type CkDeployConfig struct {