feat: use scp instead of sftp

common/ssh.go

@@ -2,6 +2,7 @@ package common
 
 import (
 	"bufio"
+	"context"
 	"fmt"
 	"io"
 	"net"
@@ -11,13 +12,13 @@ import (
 	"sync"
 	"time"
 
+	scp "github.com/bramvdbogaerde/go-scp"
 	"github.com/housepower/ckman/model"
 
 	"github.com/housepower/ckman/config"
 	"github.com/housepower/ckman/log"
 
 	"github.com/pkg/errors"
-	"github.com/pkg/sftp"
 	"golang.org/x/crypto/ssh"
 )
 
@@ -91,92 +92,63 @@ func SSHConnect(opts SshOptions) (*ssh.Client, error) {
 	return client, nil
 }
 
-func SFTPConnect(opts SshOptions) (*sftp.Client, *ssh.Client, error) {
-	var (
-		addr         string
-		clientConfig *ssh.ClientConfig
-		sshClient    *ssh.Client
-		sftpClient   *sftp.Client
-		err          error
-	)
-
-	if opts.AuthenticateType == model.SshPasswordUsePubkey {
-		clientConfig, err = sshConnectwithPublickKey(opts.User)
-	} else {
-		clientConfig, err = sshConnectwithPassword(opts.User, opts.Password)
-	}
+func ScpConnect(opts SshOptions) (*scp.Client, *ssh.Client, error) {
+	sshClient, err := SSHConnect(opts)
 	if err != nil {
-		return nil, nil, errors.Wrap(err, "")
-	}
-
-	// connet to ssh
-	addr = net.JoinHostPort(opts.Host, fmt.Sprintf("%d", opts.Port))
-
-	if sshClient, err = ssh.Dial("tcp", addr, clientConfig); err != nil {
-		err = errors.Wrapf(err, "")
 		return nil, nil, err
 	}
-
 	// create sftp client
-	if sftpClient, err = sftp.NewClient(sshClient); err != nil {
+	var client scp.Client
+	if client, err = scp.NewClientBySSH(sshClient); err != nil {
 		err = errors.Wrapf(err, "")
 		sshClient.Close()
 		return nil, nil, err
 	}
 
-	return sftpClient, sshClient, nil
+	return &client, sshClient, nil
 }
 
-func SFTPUpload(sftpClient *sftp.Client, localFilePath, remoteFilePath string) error {
-	srcFile, err := os.Open(localFilePath)
+// https://stackoverflow.com/questions/41259439/how-to-convert-filemode-to-int
+func GetFilePerm(file string) (string, error) {
+	fileInfo, err := os.Stat(file)
 	if err != nil {
-		err = errors.Wrapf(err, "")
-		return err
+		return "", errors.Wrapf(err, "GetFilePerm")
 	}
-	defer srcFile.Close()
+	perm := fileInfo.Mode().Perm()
+	return fmt.Sprintf("%04o", perm), nil
+}
 
-	dstFile, err := sftpClient.Create(remoteFilePath)
+func ScpUpload(client *scp.Client, localFilePath, remoteFilePath string) error {
+	f, err := os.Open(localFilePath)
 	if err != nil {
-		err = errors.Wrapf(err, "")
-		return err
+		return errors.Wrapf(err, "os.Open")
 	}
-	defer dstFile.Close()
-
-	buf := make([]byte, 1024*1024)
-	for {
-		n, _ := srcFile.Read(buf)
-		if n == 0 {
-			break
-		}
-		_, _ = dstFile.Write(buf[0:n])
+	defer f.Close()
+	perm, err := GetFilePerm(localFilePath)
+	if err != nil {
+		return errors.Wrapf(err, "")
+	}
+	err = client.CopyFromFile(context.Background(), *f, remoteFilePath, perm)
+	if err != nil {
+		err = errors.Wrapf(err, "CopyFromFile")
+		return err
 	}
-
 	return nil
 }
 
-func SFTPDownload(sftpClient *sftp.Client, remoteFilePath, localFilePath string) error {
-	dstFile, err := os.Create(localFilePath)
+func ScpDownload(client *scp.Client, remoteFilePath, localFilePath string) error {
+	f, err := os.Create(localFilePath)
 	if err != nil {
 		err = errors.Wrapf(err, "")
 		return err
 	}
-	defer dstFile.Close()
+	defer f.Close()
 
-	srcFile, err := sftpClient.Open(remoteFilePath)
+	err = client.CopyFromRemote(context.Background(), f, remoteFilePath)
 	if err != nil {
 		err = errors.Wrapf(err, "")
 		return err
 	}
-	defer srcFile.Close()
-
-	buf := make([]byte, 1024*1024)
-	for {
-		n, _ := srcFile.Read(buf)
-		if n == 0 {
-			break
-		}
-		_, _ = dstFile.Write(buf[0:n])
-	}
 
 	return nil
 }
@@ -258,18 +230,12 @@ func SSHRun(client *ssh.Client, password, shell string) (result string, err erro
 }
 
 func ScpUploadFiles(files []string, remotePath string, opts SshOptions) error {
-	sftpClient, sshClient, err := SFTPConnect(opts)
-	if err != nil {
-		return errors.Wrap(err, "")
-	}
-	defer sftpClient.Close()
-	defer sshClient.Close()
 	for _, file := range files {
 		if file == "" {
 			continue
 		}
 		remoteFile := path.Join(remotePath, path.Base(file))
-		err = ScpUploadFile(file, remoteFile, opts)
+		err := ScpUploadFile(file, remoteFile, opts)
 		if err != nil {
 			return errors.Wrap(err, "")
 		}
@@ -278,11 +244,11 @@ func ScpUploadFiles(files []string, remotePath string, opts SshOptions) error {
 }
 
 func ScpUploadFile(localFile, remoteFile string, opts SshOptions) error {
-	sftpClient, sshClient, err := SFTPConnect(opts)
+	client, sshClient, err := ScpConnect(opts)
 	if err != nil {
 		return errors.Wrap(err, "")
 	}
-	defer sftpClient.Close()
+	defer client.Close()
 	defer sshClient.Close()
 	// delete remote file first, beacuse maybe the remote file exists and created by root
 	cmd := fmt.Sprintf("rm -rf %s", path.Join(TmpWorkDirectory, path.Base(remoteFile)))
@@ -291,7 +257,7 @@ func ScpUploadFile(localFile, remoteFile string, opts SshOptions) error {
 		return errors.Wrap(err, "")
 	}
 
-	err = SFTPUpload(sftpClient, localFile, path.Join(TmpWorkDirectory, path.Base(remoteFile)))
+	err = ScpUpload(client, localFile, path.Join(TmpWorkDirectory, path.Base(remoteFile)))
 	if err != nil {
 		return errors.Wrap(err, "")
 	}
@@ -308,16 +274,16 @@ func ScpUploadFile(localFile, remoteFile string, opts SshOptions) error {
 }
 
 func ScpDownloadFiles(files []string, localPath string, opts SshOptions) error {
-	sftpClient, sshClient, err := SFTPConnect(opts)
+	client, sshClient, err := ScpConnect(opts)
 	if err != nil {
 		return errors.Wrap(err, "")
 	}
-	defer sftpClient.Close()
+	defer client.Close()
 	defer sshClient.Close()
 
 	for _, file := range files {
 		baseName := path.Base(file)
-		err = SFTPDownload(sftpClient, file, path.Join(localPath, baseName))
+		err = ScpDownload(client, file, path.Join(localPath, baseName))
 		if err != nil {
 			return errors.Wrap(err, "")
 		}
@@ -326,14 +292,14 @@ func ScpDownloadFiles(files []string, localPath string, opts SshOptions) error {
 }
 
 func ScpDownloadFile(remoteFile, localFile string, opts SshOptions) error {
-	sftpClient, sshClient, err := SFTPConnect(opts)
+	client, sshClient, err := ScpConnect(opts)
 	if err != nil {
 		return errors.Wrap(err, "")
 	}
-	defer sftpClient.Close()
+	defer client.Close()
 	defer sshClient.Close()
 
-	err = SFTPDownload(sftpClient, remoteFile, localFile)
+	err = ScpDownload(client, remoteFile, localFile)
 	if err != nil {
 		return errors.Wrap(err, "")
 	}

controller/clickhouse.go

@@ -1632,6 +1632,9 @@ func (ck *ClickHouseController) GetConfig(c *gin.Context) {
 		return
 	}
 	cluster.Normalize()
+	if cluster.AuthenticateType == model.SshPasswordNotSave {
+		cluster.SshPassword = ""
+	}
 	data, err := params.MarshalConfig(cluster)
 	if err != nil {
 		model.WrapMsg(c, model.GET_CK_CLUSTER_INFO_FAIL, nil)

go.mod

@@ -38,6 +38,7 @@ require (
 )
 
 require (
+	github.com/bramvdbogaerde/go-scp v1.2.0
 	github.com/go-basic/uuid v1.0.0
 	github.com/imdario/mergo v0.3.12
 	github.com/robfig/cron/v3 v3.0.1

go.sum

@@ -80,6 +80,8 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB
 github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk=
 github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
+github.com/bramvdbogaerde/go-scp v1.2.0 h1:mNF1lCXQ6jQcxCBBuc2g/CQwVy/4QONaoD5Aqg9r+Zg=
+github.com/bramvdbogaerde/go-scp v1.2.0/go.mod h1:s4ZldBoRAOgUg8IrRP2Urmq5qqd2yPXQTPshACY8vQ0=
 github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23 h1:D21IyuvjDCshj1/qq+pCNd3VZOAEI9jy6Bi131YlXgI=
 github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
@@ -704,6 +706,7 @@ golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
 golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg=
@@ -850,6 +853,7 @@ golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=