From 4740fbe18decc35456f9c06e81092042926a68cc Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Thu, 17 Oct 2024 16:08:45 +0300 Subject: [PATCH] refactor --- .buildkite/hooks/pre-command | 4 +- .../scripts/agentbeat/setup_agentbeat.py | 69 +++++++++++-------- .../x-pack/pipeline.xpack.agentbeat.yml | 57 +++++++++------ 3 files changed, 79 insertions(+), 51 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index cb5bb51eb80..5cb0722edd8 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -19,7 +19,9 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" && "$BUILDKITE_STEP fi if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-agentbeat" && "$BUILDKITE_STEP_KEY" == *"agentbeat-it"* ]]; then - AGENTBEAT_PATH=$(.buildkite/scripts/agentbeat/setup_agentbeat.py | tail -n 1) + out=$(.buildkite/scripts/agentbeat/setup_agentbeat.py) + echo "$out" + AGENTBEAT_PATH=$(echo "$out" | tail -n 1) export AGENTBEAT_PATH fi diff --git a/.buildkite/scripts/agentbeat/setup_agentbeat.py b/.buildkite/scripts/agentbeat/setup_agentbeat.py index 35139ec2278..02634649e78 100755 --- a/.buildkite/scripts/agentbeat/setup_agentbeat.py +++ b/.buildkite/scripts/agentbeat/setup_agentbeat.py @@ -6,6 +6,26 @@ import tarfile PATH = 'x-pack/agentbeat/build/distributions' +PLATFORMS = { + 'windows': { + 'amd64': 'x86_64', + }, + 'linux': { + 'x86_64': 'x86_64', + 'aarch64': 'arm64', + }, + 'darwin': { + 'x86_64': 'x86_64', + 'arm64': 'aarch64', + } + } + + +class Archive: + def __init__(self, os, arch, ext): + self.os = os + self.arch = arch + self.ext = ext def log(msg): @@ -18,38 +38,29 @@ def log_err(msg): sys.stderr.flush() -def get_os() -> str: - return platform.system().lower() +def get_archive_params() -> Archive: + system = platform.system().lower() + machine = platform.machine().lower() + arch = PLATFORMS.get(system, {}).get(machine) + ext = get_artifact_extension(system) + return Archive(system, arch, ext) -def get_arch() -> str: - arch = platform.machine().lower() - if arch == 'amd64': - return 'x86_64' - else: - if get_os() == 'darwin': - return 'aarch64' - else: - return arch - - -def get_artifact_extension(agent_os) -> str: - if agent_os == 'windows': +def get_artifact_extension(system) -> str: + if system == 'windows': return 'zip' else: return 'tar.gz' -def get_artifact_pattern() -> str: - agent_os = get_os() - agent_arch = get_arch() - extension = get_artifact_extension(agent_os) - print('Artifact params: ' + agent_os + ' ' + agent_arch + ' ' + extension) - return f'{PATH}/agentbeat-*-{agent_os}-{agent_arch}.{extension}' +def get_artifact_pattern(archive_obj) -> str: + return f'{PATH}/agentbeat-*-{archive_obj.os}-{archive_obj.arch}.{archive_obj.ext}' -def download_agentbeat(pattern, path) -> str: +def download_agentbeat(archive_obj) -> str: + pattern = get_artifact_pattern(archive_obj) + log('--- Downloading Agentbeat artifact by pattern: ' + pattern) try: subprocess.run( ['buildkite-agent', 'artifact', 'download', pattern, '.', @@ -59,15 +70,14 @@ def download_agentbeat(pattern, path) -> str: except subprocess.CalledProcessError: exit(1) - return get_filename(path) + return get_full_filename() -def get_filename(path) -> str: +def get_full_filename() -> str: try: out = subprocess.run( - ['ls', '-p', path], + ['ls', '-p', PATH], check=True, capture_output=True, text=True) - print("--- ls -p: " + out.stdout) return out.stdout.strip() except subprocess.CalledProcessError: exit(1) @@ -75,6 +85,7 @@ def get_filename(path) -> str: def extract_agentbeat(filename): filepath = PATH + '/' + filename + log('Extracting Agentbeat artifact: ' + filepath) if filepath.endswith('.zip'): unzip_agentbeat(filepath) @@ -109,10 +120,10 @@ def get_path_to_executable(filepath) -> str: path = f'../../{match.group(1)}/agentbeat' return path else: - log_err("No agentbeat executable found") + log_err('No agentbeat executable found') exit(1) -artifact_pattern = get_artifact_pattern() -archive = download_agentbeat(artifact_pattern, PATH) +archive_params = get_archive_params() +archive = download_agentbeat(archive_params) extract_agentbeat(archive) log(get_path_to_executable(archive)) diff --git a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml index a5375bd1e3d..1687aa25d92 100644 --- a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml @@ -18,26 +18,26 @@ env: AGENTBEAT_SPEC: "./agentbeat.spec.yml" steps: -# - group: "Check/Update" -# key: "x-pack-agentbeat-check-update" -# -# steps: -# - label: "agentbeat: Run pre-commit" -# command: "pre-commit run --all-files" -# agents: -# image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}" -# memory: "2Gi" -# useCustomGlobalHooks: true -# notify: -# - github_commit_status: -# context: "agentbeat: pre-commit" -# -# - wait: ~ -# # with PRs, we want to run mandatory tests only if check/update step succeed -# # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests -# # this allows building DRA artifacts even if there is flakiness in check/update step -# if: build.env("BUILDKITE_PULL_REQUEST") != "false" -# depends_on: "x-pack-agentbeat-check-update" + - group: "Check/Update" + key: "x-pack-agentbeat-check-update" + + steps: + - label: "agentbeat: Run pre-commit" + command: "pre-commit run --all-files" + agents: + image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}" + memory: "2Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "agentbeat: pre-commit" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "x-pack-agentbeat-check-update" - group: "Agentbeat tests" key: "agentbeat-mandatory-tests" @@ -85,6 +85,9 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" + notify: + - github_commit_status: + context: "agentbeat: Ubuntu x86_64 Spec tests" - label: ":ubuntu: x-pack/agentbeat: Ubuntu arm64 Spec tests" key: "agentbeat-it-linux-arm64" @@ -99,6 +102,9 @@ steps: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" + notify: + - github_commit_status: + context: "agentbeat: Ubuntu arm64 Spec tests" - label: ":windows: x-pack/agentbeat: Windows x86_64 Spec tests" key: "agentbeat-it-windows" @@ -115,6 +121,9 @@ steps: machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_size: 200 disk_type: "pd-ssd" + notify: + - github_commit_status: + context: "agentbeat: Windows x86_64 Spec tests" - label: ":macos: x-pack/agentbeat: macOS x86_64 Spec tests" key: "agentbeat-it-macos-x86-64" @@ -130,8 +139,11 @@ steps: agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" + notify: + - github_commit_status: + context: "agentbeat: macOS x86_64 Spec tests" - - label: ":macos: Agentbeat/Integration tests macOS arm64" + - label: ":macos: x-pack/agentbeat: macOS arm64 Spec tests" key: "agentbeat-it-macos-arm64" depends_on: - agentbeat-package-linux @@ -145,3 +157,6 @@ steps: agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_ARM}" + notify: + - github_commit_status: + context: "agentbeat: macOS arm64 Spec tests"