feat: use doublestar Glob

go.mod

@@ -4,6 +4,7 @@ go 1.23.1
 
 require (
 	dario.cat/mergo v1.0.1
+	github.com/bmatcuk/doublestar v1.3.4
 	github.com/fatih/color v1.14.1
 	github.com/flant/addon-operator v1.5.0
 	github.com/go-openapi/spec v0.21.0

go.sum

@@ -39,6 +39,8 @@ github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:o
 github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0=
+github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk=

internal/werf/files.go

@@ -3,6 +3,8 @@ package werf
 import (
 	"os"
 	"path/filepath"
+
+	"github.com/bmatcuk/doublestar"
 )
 
 type files struct {
@@ -25,14 +27,14 @@ func (f files) Get(relPath string) string {
 
 func (f files) doGlob(pattern string) (map[string]any, error) {
 	res := map[string]any{}
-	matches, err := filepath.Glob(filepath.Join(f.rootDir, pattern))
+	matches, err := doublestar.Glob(filepath.Join(f.rootDir, pattern))
 	if err != nil {
 		return nil, err
 	}
 	for _, path := range matches {
 		data, readErr := os.ReadFile(path)
 		if readErr != nil {
-			panic(readErr.Error())
+			return nil, readErr
 		}
 		res[path] = string(data)
 	}

internal/werf/testdata/.werf/werf-release-channel.yaml

@@ -1,44 +1,37 @@
 ---
 image: release-channel-version-prebuild
-from: {{ .Images.BASE_ALPINE }}
+from: { { .Images.BASE_ALPINE } }
 dependencies:
-- image: dev
-  before: setup
-  imports:
-  - type: ImageDigest
-    targetEnv: deckhouseImageDigest
+  - image: dev
+    before: setup
+    imports:
+      - type: ImageDigest
+        targetEnv: deckhouseImageDigest
 import:
-- image: common/yq4-artifact
-  add: /usr/local/bin/yq
-  to: /usr/local/bin/yq
-  before: install
+  - image: common/yq4-artifact
+    add: /usr/local/bin/yq
+    to: /usr/local/bin/yq
+    before: install
 shell:
   install:
-  - |
-    export version="{{ env "CI_COMMIT_REF_NAME" }}"
-    yq eval '.version = env(version)' /deckhouse/release.yaml | yq eval -j > version.json
-    # changelog exists only for tags, we have to skip it for branches
-    {{- $changelog := index (.Files.Glob "CHANGELOG/CHANGELOG-*") (printf "CHANGELOG/CHANGELOG-%s.yml" (env "CI_COMMIT_REF_NAME")) }}
-    {{ if $changelog }}
-    cat <<"EOF" > /changelog.yaml
-    {{ $changelog | nindent 6 }}
-    EOF
-    {{ end }}
+    - |
+      export version="{{ env "CI_COMMIT_REF_NAME" }}"
 git:
-- add: /
-  to: /deckhouse
-  includePaths:
-  - release.yaml
+  - add: /
+    to: /deckhouse
+    includePaths:
+      - release.yaml
 
 ---
 image: release-channel-version
-from: {{ .Images.BASE_SCRATCH }}
+from: { { .Images.BASE_SCRATCH } }
 import:
-- image: release-channel-version-prebuild
-  add: /
-  to: /
-  after: install
-  includePaths:
-  - version.json
-  - changelog.yaml
+  - image: release-channel-version-prebuild
+    add: /
+    to: /
+    after: install
+    includePaths:
+      - version.json
+      - changelog.yaml
 ---
+

internal/werf/testdata/modules/021-cni-cilium/images/agent/werf.inc.yaml

@@ -0,0 +1,240 @@
+{{- $iptables_version := "1.8.9" }}
+{{- $iptables_image_version := $iptables_version | replace "." "-" }}
+---
+# #####################################################################
+# Final image of cilium-agent (used in helm-templates)
+# Based on https://github.com/cilium/cilium/blob/v1.14.14/images/runtime/Dockerfile
+# and https://github.com/cilium/cilium/blob/v1.14.14/images/runtime/install-runtime-deps.sh
+# and https://github.com/cilium/cilium/blob/v1.14.14/images/cilium/Dockerfile (release stage)
+---
+# #####################################################################
+# List of binary files used by agent
+# from base install script
+## ln -snf /usr/share/zoneinfo/Etc/UTC /etc/localtime
+{{ $binariesFromALT := "/etc/localtime" }}
+# shell-scripts dependencies
+{{ $binariesFromALT := cat $binariesFromALT "/bin/bash /bin/sh /bin/echo /usr/bin/printf /bin/sed /bin/awk /usr/bin/nsenter /bin/mount /bin/mkdir /bin/basename" }}
+{{ $binariesFromALT := cat $binariesFromALT "/bin/cat /bin/head /bin/cut /usr/bin/od /bin/grep /bin/cp /bin/mv /bin/rm /bin/ln /bin/wc /bin/find" }}
+# kmod and dependencies
+{{ $binariesFromALT := cat $binariesFromALT "/bin/kmod /bin/lsmod /sbin/depmod /sbin/insmod /sbin/lsmod /sbin/modinfo /sbin/modprobe /sbin/rmmod" }}
+# iproute2 and dependencies
+{{ $binariesFromALT := cat $binariesFromALT "/sbin/ip /usr/sbin/ss /usr/sbin/bridge /sbin/dcb /sbin/devlink /usr/sbin/rtacct /sbin/rtmon /sbin/tc /usr/sbin/tipc /sbin/vdpa /usr/bin/lnstat" }}
+{{ $binariesFromALT := cat $binariesFromALT "/usr/bin/nstat /sbin/rdma /usr/sbin/routef /usr/sbin/routel /usr/sbin/ctstat /usr/sbin/rtstat /usr/sbin/genl" }}
+# {{ $binariesFromALT := cat $binariesFromALT "/usr/sbin/arpd" }}
+# ipset and dependencies
+{{ $binariesFromALT := cat $binariesFromALT "/sbin/ipset" }}
+# clang dependencies
+{{ $binariesFromALT := cat $binariesFromALT "/usr/lib64/libstdc++.so.6" }}
+# bash-completion
+{{ $binariesFromALT := cat $binariesFromALT "/etc/bashrc.d/bash_completion.sh" }}
+{{ $binariesFromALT := cat $binariesFromALT "/usr/share/bash-completion/bash_completion" }}
+# groups
+{{ $binariesFromALT := cat $binariesFromALT "/usr/bin/groups" }}
+# for prepull
+{{ $binariesFromALT := cat $binariesFromALT "/bin/true" }}
+# for cilium-cli sysdump
+{{ $binariesFromALT := cat $binariesFromALT "/bin/tar /usr/bin/timeout /usr/bin/stat /bin/tail" }}
+# for debug
+{{ $binariesFromALT := cat $binariesFromALT "/bin/sleep /bin/ls /usr/bin/curl /usr/bin/ldd /usr/bin/strace /usr/bin/env" }}
+#####################################################################
+{{ $selfBuiltBinaries := "" }}
+# from llvm-artifact
+{{ $selfBuiltBinaries := cat $selfBuiltBinaries "/usr/local/bin/clang /usr/local/bin/llc" }}
+# from bpftool-artifact
+{{ $selfBuiltBinaries := cat $selfBuiltBinaries "/usr/local/bin/bpftool" }}
+# from cilium-envoy-artifact
+{{ $selfBuiltBinaries := cat $selfBuiltBinaries "/usr/bin/cilium-envoy /usr/lib/libcilium.so" }}
+# from cni-plugins-artifact
+{{ $selfBuiltBinaries := cat $selfBuiltBinaries "/cni/loopback" }}
+# from gops-artifact
+{{ $selfBuiltBinaries := cat $selfBuiltBinaries "/bin/gops" }}
+# from hubble-artifact
+{{ $selfBuiltBinaries := cat $selfBuiltBinaries "/usr/bin/hubble /etc/bash_completion.d/hubble" }}
+# from cilium-artifact
+{{ $selfBuiltBinaries := cat $selfBuiltBinaries "/cni-uninstall.sh /init-container.sh /install-plugin.sh" }}
+{{ $selfBuiltBinaries := cat $selfBuiltBinaries "/LICENSE.all /etc/bash_completion.d/cilium" }}
+{{ $selfBuiltBinaries := cat $selfBuiltBinaries "/opt/cni/bin/cilium-cni /usr/bin/cilium*" }}
+# from common/pause
+{{ $selfBuiltBinaries := cat $selfBuiltBinaries "/pause" }}
+# from /jq
+{{ $selfBuiltBinaries := cat $selfBuiltBinaries "/usr/bin/jq" }}
+# for check and cleaning unnecessary iptables
+{{ $selfBuiltBinaries := cat $selfBuiltBinaries "/check-n-cleaning-iptables.sh" }}
+# iptables and dependencies
+{{ $selfBuiltBinaries := cat $selfBuiltBinaries "/sbin/xtables*" }}
+{{ $selfBuiltBinaries := cat $selfBuiltBinaries "/usr/sbin/iptables-wrapper" }}
+# #####################################################################
+# Binaries artifact for distroless agent (based on Ubuntu)
+---
+artifact: {{ $.ModuleName }}/agent-binaries-artifact
+fromImage: {{ $.ModuleName }}/base-cilium-dev
+git:
+- add: /{{ $.ModulePath }}modules/{{ $.ModulePriority }}-{{ $.ModuleName }}/images/{{ $.ImageName }}
+  to: /
+  includePaths:
+  - binary_replace.sh
+  - check-n-cleaning-iptables.sh
+  stageDependencies:
+    install:
+    - "**/*.sh"
+import:
+- artifact: {{ $.ModuleName }}/llvm-artifact
+  add: /usr/local/bin/
+  to: /usr/local/bin
+  before: install
+  includePaths:
+  - clang
+  - llc
+- artifact: {{ $.ModuleName }}/bpftool-artifact
+  add: /usr/local/bin/bpftool
+  to: /usr/local/bin/bpftool
+  before: install
+- artifact: {{ $.ModuleName }}/cni-plugins-artifact
+  add: /out/linux/amd64/bin/loopback
+  to: /cni/loopback
+  before: install
+- artifact: {{ $.ModuleName }}/gops-artifact
+  add: /out/linux/amd64/bin/gops
+  to: /bin/gops
+  before: install
+- artifact: registrypackages/iptables-artifact-{{ $iptables_image_version }}
+  add: /
+  to: /sbin
+  includePaths:
+  - xtables-legacy-multi
+  - xtables-nft-multi
+  before: install
+- image: common/iptables-wrapper
+  add: /iptables-wrapper
+  to: /usr/sbin/iptables-wrapper
+  before: install
+- artifact: {{ $.ModuleName }}/cilium-artifact
+  add: /tmp/install
+  to: /
+  before: install
+  includePaths:
+  - cni-uninstall.sh
+  - init-container.sh
+  - install-plugin.sh
+  - LICENSE.all
+  - etc/bash_completion.d/cilium
+  - opt/cni/bin/cilium-cni
+  - usr/bin/cilium*
+  - var/lib/cilium/bpf
+- artifact: {{ $.ModuleName }}/cilium-envoy-artifact
+  add: /tmp/install/usr
+  to: /usr
+  before: install
+  includePaths:
+  - bin/cilium-envoy
+  - bin/cilium-envoy-starter
+  - lib/libcilium.so
+- artifact: {{ $.ModuleName }}/hubble-artifact
+  add: /hubble
+  to: /usr/bin/hubble
+  before: install
+- artifact: {{ $.ModuleName }}/hubble-artifact
+  add: /bash_completion
+  to: /etc/bash_completion.d/hubble
+  before: install
+- image: common/pause
+  add: /pause
+  to: /pause
+  before: install
+- image: common/jq-artifact
+  add: /usr/bin/jq
+  to: /usr/bin/jq
+  before: install
+- image: common/distroless
+  add: /etc/group
+  to: /from_common_distroless/group
+  before: setup
+shell:
+  install:
+  - chown root:root /usr/sbin/iptables-wrapper
+  - chmod 755 /usr/sbin/iptables-wrapper
+  #
+  - chmod +x /check-n-cleaning-iptables.sh
+  beforeSetup:
+  # common relocate
+  - chmod +x /binary_replace.sh
+  - mkdir -p /relocate
+  # copy base binaries and deps from ALT
+  - ln -snf /usr/share/zoneinfo/Etc/UTC /etc/localtime
+  - /binary_replace.sh -i "{{ $binariesFromALT }}" -o /relocate
+  # copy self built binaries and deps
+  - /binary_replace.sh -i "{{ $selfBuiltBinaries }}" -o /relocate
+  # additional relocate for iptables
+  - |
+    for cmd in iptables iptables-save iptables-restore ip6tables ip6tables-save ip6tables-restore; do
+      rm -f "/relocate/sbin/${cmd}"
+      ln -f -s /usr/sbin/iptables-wrapper "/relocate/sbin/${cmd}"
+    done
+    # broken symlinks are not imported from the artifact
+    touch /usr/sbin/iptables-wrapper
+  - |
+    for mode in legacy nft; do
+      for basecmd in iptables ip6tables; do
+        for cmd in ${basecmd}-${mode} ${basecmd}-${mode}-save ${basecmd}-${mode}-restore; do
+          ln -sf /sbin/xtables-${mode}-multi "/relocate/sbin/${cmd}"
+        done
+      done
+    done
+  - |
+    for basecmd in ebtables arptables; do
+      for cmd in ${basecmd}-nft ${basecmd}-nft-save ${basecmd}-nft-restore; do
+        ln -sf /sbin/xtables-nft-multi "/relocate/sbin/${cmd}"
+      done
+    done
+  # additional relocate from cilium
+  - mkdir -p /relocate/var/lib/cilium
+  - cp -a /var/lib/cilium/bpf /relocate/var/lib/cilium
+  - echo ". /etc/bashrc.d/bash_completion.sh" >> /root/.bashrc
+  - mkdir -p /relocate/root && cp -a /root/.bashrc /relocate/root
+  setup:
+  # prepare final fs
+  - mkdir -p /relocate/usr/sbin
+  - if [ -d "/relocate/sbin" ]; then cp -a /relocate/sbin/* /relocate/usr/sbin/ && rm -rf /relocate/sbin; fi
+  - ln -f -s usr/sbin "/relocate/sbin"
+  - mkdir -p /relocate/usr/bin
+  - if [ -d "/relocate/bin" ]; then cp -a /relocate/bin/* /relocate/usr/bin/ && rm -rf /relocate/bin; fi
+  - ln -f -s usr/bin "/relocate/bin"
+  - mkdir -p /relocate/usr/lib
+  - if [ -d "/relocate/lib" ]; then cp -a /relocate/lib/* /relocate/usr/lib/ && rm -rf /relocate/lib; fi
+  - ln -f -s usr/lib "/relocate/lib"
+  - mkdir -p /relocate/usr/lib32
+  - if [ -d "/relocate/lib32" ]; then cp -a /relocate/lib32/* /relocate/usr/lib32/ && rm -rf /relocate/lib32; fi
+  - ln -f -s usr/lib32 "/relocate/lib32"
+  - mkdir -p /relocate/usr/lib64
+  - if [ -d "/relocate/lib64" ]; then cp -a /relocate/lib64/* /relocate/usr/lib64/ && rm -rf /relocate/lib64; fi
+  - ln -f -s usr/lib64 "/relocate/lib64"
+  - mkdir -p /relocate/usr/libx32
+  - if [ -d "/relocate/libx32" ]; then cp -a /relocate/libx32/* /relocate/usr/libx32/ && rm -rf /relocate/libx32; fi
+  - ln -f -s usr/libx32 "/relocate/libx32"
+  - mkdir -p /relocate/var /relocate/run /relocate/run/lock
+  - ln -f -s /run "/relocate/var/run"
+  - ln -f -s /run/lock "/relocate/var/lock"
+  #
+  - mkdir -p /relocate/home/cilium
+  - mkdir -p /relocate/etc
+  - cp -a /from_common_distroless/group /relocate/etc/group
+  - echo "cilium:x:1000:" >> /relocate/etc/group
+---
+# #####################################################################
+# New Main Agent Image (Distroless)
+---
+image: {{ $.ModuleName }}/agent-distroless
+fromImage: common/distroless
+import:
+- artifact: {{ $.ModuleName }}/agent-binaries-artifact
+  add: /relocate
+  to: /
+  before: install
+docker:
+  ENV:
+    HUBBLE_SERVER: "unix:///var/run/cilium/hubble.sock"
+    INITSYSTEM: SYSTEMD
+    HUBBLE_COMPAT: legacy-json-output
+    PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+  WORKDIR: "/home/cilium"
+  CMD: ["/usr/bin/cilium"]