Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use dataclass to define build configs #255

Merged
merged 1 commit into from
Dec 13, 2023
Merged

Use dataclass to define build configs #255

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
256 changes: 145 additions & 111 deletions .github/scripts/matrix.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,25 @@
#!/usr/bin/env python3

from json import dumps
from enum import Enum
import os
import dataclasses
import json

from enum import Enum
from typing import Any, Dict, List, Final, Set, Union

MANAGED_OWNER: Final[str] = "kernel-patches"
MANAGED_REPOS: Final[Set[str]] = {
f"{MANAGED_OWNER}/bpf",
f"{MANAGED_OWNER}/vmtest",
}
# We need to run on ubuntu 20.04 because our rootfs is based on debian buster and we
# otherwise get library versioning issue such as
# `./test_verifier: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.34' not found (required by ./test_verifier)`
DEFAULT_RUNNER: Final[str] = "ubuntu-20.04"
DEFAULT_LLVM_VERSION: Final[int] = 17


class Arch(Enum):
class Arch(str, Enum):
"""
CPU architecture supported by CI.
"""
Expand All @@ -15,18 +29,102 @@ class Arch(Enum):
X86_64 = "x86_64"


class Compiler(str, Enum):
GCC = "gcc"
LLVM = "llvm"


@dataclasses.dataclass
class Toolchain:
compiler: Compiler
# This is relevant ONLY for LLVM and should not be required for GCC
version: int

@property
def short_name(self) -> str:
return str(self.compiler.value)

@property
def full_name(self) -> str:
if self.compiler == Compiler.GCC:
return self.short_name

return f"{self.short_name}-{self.version}"

def to_dict(self) -> Dict[str, Union[str, int]]:
return {
"name": self.short_name,
"fullname": self.full_name,
"version": self.version,
}


@dataclasses.dataclass
class BuildConfig:
arch: Arch
toolchain: Toolchain
kernel: str = "LATEST"
run_veristat: bool = False
parallel_tests: bool = False
build_release: bool = False

@property
def runs_on(self) -> List[str]:
if is_managed_repo():
return ["self-hosted", self.arch.value]
return [DEFAULT_RUNNER]

@property
def tests(self) -> Dict[str, Any]:
tests_list = [
"test_progs",
"test_progs_parallel",
"test_progs_no_alu32",
"test_progs_no_alu32_parallel",
"test_maps",
"test_verifier",
]

if self.toolchain.version >= 18:
tests_list.append("test_progs_cpuv4")

if not self.parallel_tests:
tests_list = [test for test in tests_list if not test.endswith("parallel")]

return {"include": [generate_test_config(test) for test in tests_list]}

def to_dict(self) -> Dict[str, Any]:
return {
"arch": self.arch.value,
"toolchain": self.toolchain.to_dict(),
"kernel": self.kernel,
"run_veristat": self.run_veristat,
"parallel_tests": self.parallel_tests,
"build_release": self.build_release,
"runs_on": self.runs_on,
"tests": self.tests,
}


def is_managed_repo() -> bool:
return (
os.environ["GITHUB_REPOSITORY_OWNER"] == MANAGED_OWNER
and os.environ["GITHUB_REPOSITORY"] in MANAGED_REPOS
)


def set_output(name, value):
"""Write an output variable to the GitHub output file."""
with open(os.getenv("GITHUB_OUTPUT"), "a", encoding="utf-8") as file:
file.write(f"{name}={value}\n")


def generate_test_config(test):
def generate_test_config(test: str) -> Dict[str, Union[str, int]]:
"""Create the configuration for the provided test."""
experimental = test.endswith("_parallel")
is_parallel = test.endswith("_parallel")
config = {
"test": test,
"continue_on_error": experimental,
"continue_on_error": is_parallel,
# While in experimental mode, parallel jobs may get stuck
# anywhere, including in user space where the kernel won't detect
# a problem and panic. We add a second layer of (smaller) timeouts
Expand All @@ -36,114 +134,50 @@ def generate_test_config(test):
# non-experimental jobs, 360 is the default which will be
# superseded by the overall workflow timeout (but we need to
# specify something).
"timeout_minutes": 30 if experimental else 360,
"timeout_minutes": 30 if is_parallel else 360,
}
return config


def get_tests(config):
tests = [
"test_progs",
"test_progs_parallel",
"test_progs_no_alu32",
"test_progs_no_alu32_parallel",
"test_maps",
"test_verifier",
if __name__ == "__main__":
matrix = [
BuildConfig(
arch=Arch.X86_64,
toolchain=Toolchain(compiler=Compiler.GCC, version=DEFAULT_LLVM_VERSION),
run_veristat=True,
parallel_tests=True,
),
BuildConfig(
arch=Arch.X86_64,
toolchain=Toolchain(compiler=Compiler.LLVM, version=DEFAULT_LLVM_VERSION),
build_release=True,
),
BuildConfig(
arch=Arch.X86_64,
toolchain=Toolchain(compiler=Compiler.LLVM, version=18),
build_release=True,
),
BuildConfig(
arch=Arch.AARCH64,
toolchain=Toolchain(compiler=Compiler.GCC, version=DEFAULT_LLVM_VERSION),
),
# BuildConfig(
# arch=Arch.AARCH64,
# toolchain=Toolchain(
# compiler=Compiler.LLVM,
# version=DEFAULT_LLVM_VERSION
# ),
# ),
BuildConfig(
arch=Arch.S390X,
toolchain=Toolchain(compiler=Compiler.GCC, version=DEFAULT_LLVM_VERSION),
),
]
if int(config.get("llvm-version", 0)) >= 18:
tests.append("test_progs_cpuv4")
if config.get("parallel_tests", False):
return tests
return [test for test in tests if not test.endswith("parallel")]


matrix = [
{
"kernel": "LATEST",
"runs_on": [],
"arch": Arch.X86_64.value,
"toolchain": "gcc",
"llvm-version": "17",
"run_veristat": True,
"parallel_tests": True,
},
{
"kernel": "LATEST",
"runs_on": [],
"arch": Arch.X86_64.value,
"toolchain": "llvm",
"llvm-version": "17",
"build_release": True,
},
{
"kernel": "LATEST",
"runs_on": [],
"arch": Arch.X86_64.value,
"toolchain": "llvm",
"llvm-version": "18",
"build_release": True,
},
{
"kernel": "LATEST",
"runs_on": [],
"arch": Arch.AARCH64.value,
"toolchain": "gcc",
"llvm-version": "17",
},
# {
# "kernel": "LATEST",
# "runs_on": [],
# "arch": Arch.AARCH64.value,
# "toolchain": "llvm",
# "llvm-version": "16",
# },
{
"kernel": "LATEST",
"runs_on": [],
"arch": Arch.S390X.value,
"toolchain": "gcc",
"llvm-version": "17",
},
]
self_hosted_repos = [
"kernel-patches/bpf",
"kernel-patches/vmtest",
]

for idx in range(len(matrix) - 1, -1, -1):
if matrix[idx]["toolchain"] == "gcc":
matrix[idx]["toolchain_full"] = "gcc"
else:
matrix[idx]["toolchain_full"] = "llvm-" + matrix[idx]["llvm-version"]

# Set run_veristat to false by default.
matrix[idx]["run_veristat"] = matrix[idx].get("run_veristat", False)
# Set build_release to false by default.
matrix[idx]["build_release"] = matrix[idx].get("build_release", False)
# Feel in the tests
matrix[idx]["tests"] = {
"include": [generate_test_config(test) for test in get_tests(matrix[idx])]
}

# Only a few repository within "kernel-patches" use self-hosted runners.
if (
os.environ["GITHUB_REPOSITORY_OWNER"] != "kernel-patches"
or os.environ["GITHUB_REPOSITORY"] not in self_hosted_repos
):
# Outside of those repositories, we only run on x86_64 GH hosted runners (ubuntu-20.04)
# We need to run on ubuntu 20.04 because our rootfs is based on debian buster and we
# otherwise get library versioning issue such as
# `./test_verifier: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.34' not found (required by ./test_verifier)`
for idx in range(len(matrix) - 1, -1, -1):
if matrix[idx]["arch"] != Arch.X86_64.value:
del matrix[idx]
else:
matrix[idx]["runs_on"] = ["ubuntu-20.04"]
else:
# Otherwise, run on (self-hosted, arch) runners
for idx in range(len(matrix) - 1, -1, -1):
matrix[idx]["runs_on"].extend(["self-hosted", matrix[idx]["arch"]])

build_matrix = {"include": matrix}

set_output("build_matrix", dumps(build_matrix))
# Outside of those repositories we only run on x86_64
if not is_managed_repo():
matrix = [config for config in matrix if config.arch == Arch.X86_64]

json_matrix = json.dumps({"include": [config.to_dict() for config in matrix]})
print(json_matrix)
set_output("build_matrix", json_matrix)
8 changes: 4 additions & 4 deletions .github/workflows/test.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ jobs:
build-and-test:
# Setting name to arch-compiler here to avoid lengthy autogenerated names due to matrix
# e.g build-and-test x86_64-gcc / test (test_progs_parallel, true, 30) / test_progs_parallel on x86_64 with gcc
name: "${{ matrix.arch }}-${{ matrix.toolchain_full }}"
name: "${{ matrix.arch }}-${{ matrix.toolchain.fullname }}"
uses: ./.github/workflows/kernel-build-test.yml
needs: [set-matrix]
permissions:
Expand All @@ -41,10 +41,10 @@ jobs:
matrix: ${{ fromJSON(needs.set-matrix.outputs.build-matrix) }}
with:
arch: ${{ matrix.arch }}
toolchain_full: ${{ matrix.toolchain_full }}
toolchain: ${{ matrix.toolchain }}
toolchain_full: ${{ matrix.toolchain.fullname }}
toolchain: ${{ matrix.toolchain.name }}
runs_on: ${{ toJSON(matrix.runs_on) }}
llvm-version: ${{ matrix.llvm-version }}
llvm-version: ${{ matrix.toolchain.version }}
kernel: ${{ matrix.kernel }}
tests: ${{ toJSON(matrix.tests) }}
run_veristat: ${{ matrix.run_veristat }}
Expand Down
Loading