From a328104c4b7a7f305aa0b56d3057b4dc8bc23a53 Mon Sep 17 00:00:00 2001 From: Arne Tarara Date: Thu, 25 Jul 2024 12:56:35 +0200 Subject: [PATCH] Rapl energy filtering check (#836) * Security fix for python interpreter replacement * RAPL energy filtering check * Also added killall to check_file_permissions * Reversed variable trueness --- install_linux.sh | 85 +++++++++++++------ install_mac.sh | 20 +++-- lib/hardware_info_root_original.py | 14 ++- lib/system_checks.py | 12 ++- .../psu/energy/ac/xgboost/machine/model | 2 +- 5 files changed, 97 insertions(+), 36 deletions(-) diff --git a/install_linux.sh b/install_linux.sh index f2869d8b9..6ac01c1e0 100755 --- a/install_linux.sh +++ b/install_linux.sh @@ -49,14 +49,19 @@ function check_file_permissions() { db_pw='' api_url='' metrics_url='' -no_build=false -no_python=false -no_hosts=false +build_docker_containers=true +install_python_packages=true +modify_hosts=true ask_tmpfs=true - +install_ipmi=true +install_sensors=true +install_msr_tools=true +# The system site packages are only an option to choose if you are in temporary VMs anyway +# Not recommended for classical developer system +use_system_site_packages=false reboot_echo_flag=false -while getopts "p:a:m:nhtb" o; do +while getopts "p:a:m:nhtbisyr" o; do case "$o" in p) db_pw=${OPTARG} @@ -68,17 +73,30 @@ while getopts "p:a:m:nhtb" o; do metrics_url=${OPTARG} ;; b) - no_build=true + build_docker_containers=false ;; h) - no_hosts=true + modify_hosts=false ;; n) - no_python=true + install_python_packages=false ;; t) ask_tmpfs=false ;; + i) + install_ipmi=false + ;; + s) + install_sensors=false + # currently unused + ;; + r) + install_msr_tools=false + ;; + y) + use_system_site_packages=true + ;; esac done @@ -173,7 +191,11 @@ while IFS= read -r subdir; do done print_message "Setting up python venv" -python3 -m venv venv +if [[ $use_system_site_packages == true ]] ; then + python3 -m venv venv --system_site_packages +else + python3 -m venv venv +fi source venv/bin/activate print_message "Setting GMT in include path for python via .pth file" @@ -189,6 +211,7 @@ check_file_permissions "/usr/bin/python3" # Please note the -m as here we will later call python3 without venv. It must understand the .lib imports # and not depend on venv installed packages echo "ALL ALL=(ALL) NOPASSWD:/usr/bin/python3 -m lib.hardware_info_root" | sudo tee /etc/sudoers.d/green-coding-hardware-info +echo "ALL ALL=(ALL) NOPASSWD:/usr/bin/python3 -m lib.hardware_info_root --read-rapl-energy-filtering" | sudo tee -a /etc/sudoers.d/green-coding-hardware-info sudo chmod 500 /etc/sudoers.d/green-coding-hardware-info # remove old file name sudo rm -f /etc/sudoers.d/green_coding_hardware_info @@ -204,23 +227,37 @@ sudo chown root:root $PWD/tools/cluster/cleanup.sh sudo chmod 755 $PWD/tools/cluster/cleanup.sh sudo chmod +x $PWD/tools/cluster/cleanup.sh -print_message "Installing IPMI tools" -if lsb_release -is | grep -q "Fedora"; then - sudo dnf -y install ipmitool -else - sudo apt-get install -y freeipmi-tools ipmitool -fi +if [[ $install_msr_tools == true ]] ; then + print_message "Installing msr-tools" + print_message "Important: If this step fails it means msr-tools is not available on you system" + print_message "If you do not plan to use RAPL you can skip the installation by appending '-r'" + if lsb_release -is | grep -q "Fedora"; then + sudo dnf -y install msr-tools + else + sudo apt-get install -y msr-tools + fi +fi -print_message "Adding IPMI to sudoers file" -check_file_permissions "/usr/sbin/ipmi-dcmi" -echo "ALL ALL=(ALL) NOPASSWD:/usr/sbin/ipmi-dcmi --get-system-power-statistics" | sudo tee /etc/sudoers.d/green-coding-ipmi-get-machine-energy-stat -sudo chmod 500 /etc/sudoers.d/green-coding-ipmi-get-machine-energy-stat -# remove old file name -sudo rm -f /etc/sudoers.d/ipmi_get_machine_energy_stat +if [[ $install_ipmi == true ]] ; then + print_message "Installing IPMI tools" + print_message "Important: If this step fails it means ipmitool is not available on you system" + print_message "If you do not plan to use IPMI you can skip the installation by appending '-i'" + if lsb_release -is | grep -q "Fedora"; then + sudo dnf -y install ipmitool + else + sudo apt-get install -y freeipmi-tools ipmitool + fi + print_message "Adding IPMI to sudoers file" + check_file_permissions "/usr/sbin/ipmi-dcmi" + echo "ALL ALL=(ALL) NOPASSWD:/usr/sbin/ipmi-dcmi --get-system-power-statistics" | sudo tee /etc/sudoers.d/green-coding-ipmi-get-machine-energy-stat + sudo chmod 500 /etc/sudoers.d/green-coding-ipmi-get-machine-energy-stat + # remove old file name + sudo rm -f /etc/sudoers.d/ipmi_get_machine_energy_stat +fi -if [[ $no_hosts != true ]] ; then +if [[ $modify_hosts == true ]] ; then etc_hosts_line_1="127.0.0.1 green-coding-postgres-container" etc_hosts_line_2="127.0.0.1 ${host_api_url} ${host_metrics_url}" @@ -244,7 +281,7 @@ if [[ $no_hosts != true ]] ; then fi fi -if [[ $no_build != true ]] ; then +if [[ $build_docker_containers == true ]] ; then print_message "Building / Updating docker containers" if docker info 2>/dev/null | grep rootless; then print_message "Docker is running in rootless mode. Using non-sudo call ..." @@ -259,7 +296,7 @@ if [[ $no_build != true ]] ; then fi fi -if [[ $no_python != true ]] ; then +if [[ $install_python_packages == true ]] ; then print_message "Updating python requirements" python3 -m pip install --upgrade pip python3 -m pip install -r requirements.txt diff --git a/install_mac.sh b/install_mac.sh index af9d53048..c24789ea3 100755 --- a/install_mac.sh +++ b/install_mac.sh @@ -49,9 +49,9 @@ function check_file_permissions() { db_pw='' api_url='' metrics_url='' -no_build=false -no_python=false -no_hosts=false +build_docker_containers=true +install_python_packages=true +modify_hosts=true while getopts "p:a:m:nhtb" o; do @@ -63,13 +63,13 @@ while getopts "p:a:m:nhtb" o; do api_url=${OPTARG} ;; b) - no_build=true + build_docker_containers=false ;; h) - no_hosts=true + modify_hosts=false ;; n) - no_python=true + install_python_packages=false ;; m) metrics_url=${OPTARG} @@ -140,6 +140,7 @@ find venv -type d -name "site-packages" -exec sh -c 'echo $PWD > "$0/gmt-lib.pth print_message "Adding python3 lib.hardware_info_root to sudoers file" check_file_permissions "/usr/bin/python3" echo "ALL ALL=(ALL) NOPASSWD:/usr/bin/python3 -m lib.hardware_info_root" | sudo tee /etc/sudoers.d/green_coding_hardware_info +echo "ALL ALL=(ALL) NOPASSWD:/usr/bin/python3 -m lib.hardware_info_root --read-rapl-energy-filtering" | sudo tee -a /etc/sudoers.d/green-coding-hardware-info # remove old file name sudo rm -f /etc/sudoers.d/green_coding_hardware_info @@ -151,11 +152,12 @@ sudo chmod 755 $PWD/lib/hardware_info_root.py print_message "Adding powermetrics to sudoers file" check_file_permissions "/usr/bin/powermetrics" +check_file_permissions "/usr/bin/killall" echo "ALL ALL=(ALL) NOPASSWD:/usr/bin/powermetrics" | sudo tee /etc/sudoers.d/green_coding_powermetrics echo "ALL ALL=(ALL) NOPASSWD:/usr/bin/killall powermetrics" | sudo tee /etc/sudoers.d/green_coding_kill_powermetrics echo "ALL ALL=(ALL) NOPASSWD:/usr/bin/killall -9 powermetrics" | sudo tee /etc/sudoers.d/green_coding_kill_powermetrics_sigkill -if [[ $no_hosts != true ]] ; then +if [[ $modify_hosts == true ]] ; then print_message "Writing to /etc/hosts file..." etc_hosts_line_1="127.0.0.1 green-coding-postgres-container" etc_hosts_line_2="127.0.0.1 ${host_api_url} ${host_metrics_url}" @@ -196,14 +198,14 @@ while IFS= read -r subdir; do fi done -if [[ $no_build != true ]] ; then +if [[ $build_docker_containers == true ]] ; then print_message "Building / Updating docker containers" docker compose -f docker/compose.yml down docker compose -f docker/compose.yml build docker compose -f docker/compose.yml pull fi -if [[ $no_python != true ]] ; then +if [[ $install_python_packages == true ]] ; then print_message "Updating python requirements" python3 -m pip install --upgrade pip python3 -m pip install -r requirements.txt diff --git a/lib/hardware_info_root_original.py b/lib/hardware_info_root_original.py index 7b5b1f3a1..ff6712ad8 100755 --- a/lib/hardware_info_root_original.py +++ b/lib/hardware_info_root_original.py @@ -69,6 +69,9 @@ def get_values(list_of_tasks): return {x[1]: x[0](*x[2:]) for x in list_of_tasks} +def read_rapl_energy_filtering(): + return read_process_with_regex('rdmsr -d 0xbc', r'(?P.*)', re.IGNORECASE | re.DOTALL) + # Defining shortcuts to make the lines shorter rfwr = read_file_with_regex rpwr = read_process_with_regex @@ -79,6 +82,7 @@ def get_values(list_of_tasks): [rdr, 'Power Limits', '/sys/devices/virtual/powercap/intel-rapl'], [rdr, 'CPU Scheduling', '/sys/kernel/debug/sched'], [rpwr, 'Hardware Details', 'lshw', r'(?P.*)', re.IGNORECASE | re.DOTALL], + [cf, 'RAPL Energy Filtering', read_rapl_energy_filtering], ] def get_root_list(): @@ -92,4 +96,12 @@ def get_root_list(): if platform.system() == 'Darwin': print('{}') else: - print(json.dumps(get_values(get_root_list()))) + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--read-rapl-energy-filtering', action='store_true', help='Read RAPL energy filtering') + args = parser.parse_args() + + if args.read_rapl_energy_filtering is True: + print(read_rapl_energy_filtering(), end='') + else: + print(json.dumps(get_values(get_root_list()))) diff --git a/lib/system_checks.py b/lib/system_checks.py index 7076759a2..ddf1c4c24 100644 --- a/lib/system_checks.py +++ b/lib/system_checks.py @@ -22,6 +22,8 @@ from lib.terminal_colors import TerminalColors from lib.configuration_check_error import ConfigurationCheckError, Status +CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) + GMT_Resources = { 'free_disk': 1024 ** 3, # 1GB in bytes 'free_memory': 1024 ** 3, # 1GB in bytes @@ -54,6 +56,14 @@ def check_free_disk(): def check_free_memory(): return psutil.virtual_memory().available >= GMT_Resources['free_memory'] +def check_energy_filtering(): + result = subprocess.run(['sudo', 'python3', '-m', 'lib.hardware_info_root', '--read-rapl-energy-filtering'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=os.path.abspath(os.path.join(CURRENT_DIR, '..')), + check=True, encoding='UTF-8') + return "1" != result.stdout.strip() + def check_containers_running(): result = subprocess.run(['docker', 'ps', '--format', '{{.Names}}'], stdout=subprocess.PIPE, @@ -83,7 +93,7 @@ def check_utf_encoding(): (check_docker_daemon, Status.ERROR, 'docker daemon', 'The docker daemon could not be reached. Are you running in rootless mode or have added yourself to the docker group? See installation: [See https://docs.green-coding.io/docs/installation/]'), (check_containers_running, Status.WARN, 'running containers', 'You have other containers running on the system. This is usually what you want in local development, but for undisturbed measurements consider going for a measurement cluster [See https://docs.green-coding.io/docs/installation/installation-cluster/].'), (check_utf_encoding, Status.ERROR, 'utf file encoding', 'Your system encoding is not set to utf-8. This is needed as we need to parse console output.'), - + (check_energy_filtering, Status.ERROR, 'rapl energy filtering', 'RAPL Energy filtering is active!'), ] diff --git a/metric_providers/psu/energy/ac/xgboost/machine/model b/metric_providers/psu/energy/ac/xgboost/machine/model index 1873c06b8..317c92cdf 160000 --- a/metric_providers/psu/energy/ac/xgboost/machine/model +++ b/metric_providers/psu/energy/ac/xgboost/machine/model @@ -1 +1 @@ -Subproject commit 1873c06b8fb32996d37a7c4952b72c78e4f3d5f2 +Subproject commit 317c92cdfa161316e18f27adbcaa143207bcb2a2