Feat/kube metrics

.github/workflows/build.yaml

@@ -22,6 +22,11 @@ jobs:
         with:
           go-version: ${{ env.GO_VERSION }}
 
+      - name: Install UPX
+        uses: crazy-max/ghaction-upx@v3
+        with:
+          install-only: true
+
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v3
         with:

.goreleaser.yaml

@@ -3,10 +3,11 @@ project_name: k8spider
 before:
   hooks:
     - go mod tidy
-    - go generate ./...
 builds:
   - env:
       - CGO_ENABLED=0
+    ldflags:
+      - -s -w
     goos:
       - darwin
       - windows
@@ -23,6 +24,13 @@ archives:
       - goos: windows
         format: zip
 
+upx:
+  - enabled: true
+    compress: best
+    brute: true
+    lzma: true
+
+
 checksum:
   name_template: "checksums.txt"
 snapshot:

Makefile

@@ -3,14 +3,18 @@ BUILD_DIR = bin
 
 MAIN_PROGRAM_NAME = k8spider
 
-default: build build-static
+default: build build-static check-size
 
 # build
 build:
 	go	build -o $(BUILD_DIR)/$(MAIN_PROGRAM_NAME) main.go
 
 build-static:
 	GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o $(BUILD_DIR)/$(MAIN_PROGRAM_NAME)-linux-static main.go
-	upx $(BUILD_DIR)/$(MAIN_PROGRAM_NAME)-linux-static
+	upx --lzma --brute  $(BUILD_DIR)/$(MAIN_PROGRAM_NAME)-linux-static
+
+check-size:
+	ls -alh $(BUILD_DIR)/$(MAIN_PROGRAM_NAME)*
+
 clean:
 	rm -rf $(BUILD_DIR)

cmd/all/all.go

@@ -48,13 +48,15 @@ var AllCmd = &cobra.Command{
 			log.Warnf("ParseStringToIPNet failed: %v", err)
 			return
 		}
+
 		var finalRecord define.Records
 		if command.Opts.MultiThreadingMode {
 			finalRecord = RunMultiThread(ipNets, command.Opts.ThreadingNum)
 		} else {
 			finalRecord = Run(ipNets)
 		}
 		printer.PrintResult(finalRecord, command.Opts.OutputFile)
+
 		PostRun(finalRecord)
 	},
 }
@@ -78,6 +80,9 @@ func RunMultiThread(net *net.IPNet, count int) (finalRecord define.Records) {
 }
 
 func PostRun(finalRecord define.Records) {
+	if finalRecord == nil || len(finalRecord) == 0 {
+		return
+	}
 	log.Info("Extract Namespaces: ")
 	list := post.RecordsDumpNameSpace(finalRecord, command.Opts.Zone)
 	for _, ns := range list {

cmd/dnsutils/dns.go

@@ -0,0 +1,46 @@
+package dnsutils
+
+import (
+	"strings"
+
+	command "github.com/esonhugh/k8spider/cmd"
+	"github.com/esonhugh/k8spider/pkg"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+var queryType string
+
+func init() {
+	DNSCmd.PersistentFlags().StringVarP(&queryType, "type", "t", "A", "query type")
+	command.RootCmd.AddCommand(DNSCmd)
+}
+
+var DNSCmd = &cobra.Command{
+	Use:     "dns",
+	Aliases: []string{"dig"},
+	Short:   "dns is a command to query dns server",
+	Run: func(cmd *cobra.Command, args []string) {
+		var querier pkg.DnsQuery
+		switch strings.ToLower(queryType) {
+		case "a", "aaaa":
+			querier = pkg.QueryA
+		case "ptr":
+			querier = pkg.QueryPTR
+		case "srv":
+			querier = pkg.QuerySRV
+		case "txt":
+			querier = pkg.QueryTXT
+		default:
+			querier = pkg.QueryA
+		}
+		for _, query := range args {
+			res, err := querier(query)
+			if err != nil {
+				log.Warnf("Query %s failed: %v", query, err)
+				continue
+			}
+			log.Infof("Query [%d] %s: %v", queryType, query, res)
+		}
+	},
+}

cmd/metrics/parser.go

@@ -0,0 +1,94 @@
+package metrics
+
+import (
+	"bufio"
+	"io"
+	"net/http"
+	"os"
+	"strings"
+
+	cmdx "github.com/esonhugh/k8spider/cmd"
+	"github.com/esonhugh/k8spider/pkg/metrics"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+var MetricOpt struct {
+	From string
+}
+
+func init() {
+	cmdx.RootCmd.AddCommand(MetricCmd)
+	MetricCmd.PersistentFlags().StringVarP(&MetricOpt.From, "metric", "m", "", "metrics from (file / remote url)")
+
+}
+
+var MetricCmd = &cobra.Command{
+	Use:   "metric",
+	Short: "parse kube stat metrics to readable resource",
+	Run: func(cmd *cobra.Command, args []string) {
+		if MetricOpt.From == "" {
+			return
+		}
+		log.Debugf("parse metrics from %v", MetricOpt.From)
+		rule := metrics.DefaultMatchRules()
+		if err := rule.Compile(); err != nil {
+			log.Fatalf("compile rule failed: %v", err)
+		}
+		log.Debugf("compiled rules completed, start to get resource \n")
+
+		ot := output()
+
+		var r io.Reader
+		if strings.HasPrefix("http://", MetricOpt.From) || strings.HasPrefix("https://", MetricOpt.From) {
+			resp, err := http.Get(MetricOpt.From)
+			if err != nil {
+				log.Fatalf("get metrics from %v failed: %v", MetricOpt.From, err)
+			}
+			defer resp.Body.Close()
+			r = resp.Body
+		} else {
+			f, err := os.OpenFile(MetricOpt.From, os.O_RDONLY, 0666)
+			if err != nil {
+				log.Fatalf("open file %v failed: %v", MetricOpt.From, err)
+			}
+			defer f.Close()
+			r = f
+		}
+		log.Debugf("start to parse metrics line by line\n")
+
+		var rx []*metrics.MetricMatcher
+
+		scanner := bufio.NewScanner(r)
+		for scanner.Scan() {
+			line := scanner.Text()
+			res, err := rule.Match(line)
+			if err != nil {
+				continue
+			} else {
+				log.Debugf("matched: %s", res.DumpString())
+				rx = append(rx, res.CopyData())
+			}
+		}
+		if err := scanner.Err(); err != nil {
+			log.Warnf("scan metrics failed and break out, reason: %v", err)
+		}
+		var res metrics.ResourceList = metrics.ConvertToResource(rx)
+		log.Debugf("parse metrics completed, start to print result\n")
+
+		res.Print(ot)
+	},
+}
+
+func output() io.WriteCloser {
+	if cmdx.Opts.OutputFile != "" {
+		f, err := os.OpenFile(cmdx.Opts.OutputFile, os.O_CREATE|os.O_WRONLY, 0644)
+		if err != nil {
+			log.Warnf("create output file failed: %v", err)
+			return nil
+		}
+		return f
+	} else {
+		return os.Stdout
+	}
+}

go.mod

@@ -1,18 +1,21 @@
 module github.com/esonhugh/k8spider
 
-go 1.19
+go 1.21.0
+
+toolchain go1.23.2
 
 require (
+	github.com/elastic/go-grok v0.3.1
 	github.com/miekg/dns v1.1.58
 	github.com/sirupsen/logrus v1.9.0
 	github.com/spf13/cobra v1.5.0
 )
 
 require (
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/magefile/mage v1.15.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/stretchr/testify v1.8.0 // indirect
-	golang.org/x/mod v0.14.0 // indirect
+	golang.org/x/mod v0.17.0 // indirect
 	golang.org/x/net v0.20.0 // indirect
 	golang.org/x/sys v0.16.0 // indirect
 	golang.org/x/tools v0.17.0 // indirect

go.sum

@@ -2,8 +2,12 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/elastic/go-grok v0.3.1 h1:WEhUxe2KrwycMnlvMimJXvzRa7DoByJB4PVUIE1ZD/U=
+github.com/elastic/go-grok v0.3.1/go.mod h1:n38ls8ZgOboZRgKcjMY8eFeZFMmcL9n2lP0iHhIDk64=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg=
+github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
 github.com/miekg/dns v1.1.58 h1:ca2Hdkz+cDg/7eNF6V56jjzuZ4aCAE+DbVkILdQWG/4=
 github.com/miekg/dns v1.1.58/go.mod h1:Ypv+3b/KadlvW9vJfXOTf300O4UqaHFzFCuHz+rPkBY=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -16,16 +20,15 @@ github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJ
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
-github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
-golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
+golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
 golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
-golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
 golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=

main.go

@@ -4,6 +4,8 @@ import (
 	"github.com/esonhugh/k8spider/cmd"
 	_ "github.com/esonhugh/k8spider/cmd/all"
 	_ "github.com/esonhugh/k8spider/cmd/axfr"
+	_ "github.com/esonhugh/k8spider/cmd/dnsutils"
+	_ "github.com/esonhugh/k8spider/cmd/metrics"
 	_ "github.com/esonhugh/k8spider/cmd/neighbor"
 	_ "github.com/esonhugh/k8spider/cmd/service"
 	_ "github.com/esonhugh/k8spider/cmd/subnet"

pkg/metrics/matcher.go

@@ -0,0 +1,60 @@
+package metrics
+
+import "errors"
+
+type MatchRules []*MetricMatcher
+
+func GenerateMatchRules() MatchRules {
+	return make(MatchRules, 0)
+}
+
+func DefaultMatchRules() MatchRules {
+	return []*MetricMatcher{
+		NewMetricMatcher("configmap").AddLabel("namespace").AddLabel("configmap"),
+		NewMetricMatcher("secret").AddLabel("namespace").AddLabel("secret"),
+
+		NewMetricMatcher("node").AddLabel("node").AddLabel("kernel_version").
+			AddLabel("os_image").AddLabel("container_runtime_version").
+			AddLabel("provider_id").AddLabel("internal_ip"),
+
+		NewMetricMatcher("pod").AddLabel("namespace").AddLabel("pod").AddLabel("node").
+			AddLabel("host_ip").AddLabel("pod_ip"),
+		NewMetricMatcher("container").SetHeader("kube_pod_container_info").AddLabel("namespace").
+			AddLabel("pod").AddLabel("container").AddLabel("image"),
+		NewMetricMatcher("cronjob").AddLabel("namespace").AddLabel("cronjob").
+			AddLabel("schedule"),
+
+		NewMetricMatcher("service_account").SetHeader("kube_pod_service_account").
+			AddLabel("namespace").AddLabel("pod").AddLabel("service_account"),
+
+		NewMetricMatcher("service").AddLabel("namespace").AddLabel("service").
+			AddLabel("cluster_ip").AddLabel("external_name").AddLabel("load_balancer_ip"),
+		NewMetricMatcher("endpoint_address").SetHeader("kube_endpoint_address").
+			AddLabel("namespace").AddLabel("endpoint").AddLabel("ip"),
+		NewMetricMatcher("endpoint_port").SetHeader("kube_endpoint_ports").
+			AddLabel("namespace").AddLabel("endpoint").AddLabel("port_number"),
+	}
+}
+
+func (m MatchRules) Compile() error {
+	var err error = nil
+	for i := range m {
+		e := m[i].Compile()
+		if e != nil {
+			err = errors.Join(err, e)
+		}
+	}
+	return err
+}
+
+func (m MatchRules) Match(target string) (*MetricMatcher, error) {
+	for _, r := range m {
+		_, e := r.Match(target)
+		if e != nil {
+			continue
+		} else {
+			return r, nil
+		}
+	}
+	return nil, errors.New("no match found")
+}