Merge branch 'EESSI:main' into autobuild_bootstrap_container

ansible/playbooks/roles/compatibility_layer/defaults/main.yml

@@ -1,6 +1,6 @@
 # Defaults file for the compatibility layer role.
 ---
-eessi_version: "2021.03"
+eessi_version: "2021.06"
 
 custom_overlays:
   - name: eessi
@@ -17,17 +17,15 @@ gentoo_prefix_path: /cvmfs/{{ cvmfs_repository }}/{{ eessi_version }}/compat/{{
 
 # How to build the prefix
 prefix_required_space: 15 GB
-prefix_snapshot_url: http://cvmfs-s0.eessi-hpc.org/snapshots
-prefix_snapshot_version: 20210322
-prefix_python_targets: python3_8
+prefix_snapshot_url: https://eessi-gentoo-snapshot.s3-eu-west-1.amazonaws.com
+prefix_snapshot_version: 20210607
 prefix_user_defined_trusted_dirs:
   - "/cvmfs/{{ cvmfs_repository }}/host_injections/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib"
 prefix_singularity_command: "singularity run -B {{ gentoo_prefix_path }}:{{ gentoo_prefix_path }}"
 prefix_source: "docker://ghcr.io/eessi/bootstrap-prefix:centos8"
 prefix_source_options: "{{ gentoo_prefix_path }} noninteractive"
 prefix_install: >-
     SINGULARITYENV_USE_CPU_CORES={{ ansible_processor_vcpus }}
-    SINGULARITYENV_PYTHON_TARGETS="{{ prefix_python_targets }}"
     SINGULARITYENV_CUSTOM_SNAPSHOT_URL="{{ prefix_snapshot_url }}"
     SINGULARITYENV_CUSTOM_SNAPSHOT_VERSION="{{ prefix_snapshot_version }}"
     {{ prefix_singularity_command }} {{ prefix_source }} {{ prefix_source_options }}
@@ -64,6 +62,9 @@ symlinks_to_host:
   # required to use the DNS resolver from the host (should be done automatically)
   - /etc/resolv.conf
 
+  # required to pick up the right timezone from the host
+  - /etc/localtime
+
   # required if Centrify is used in nsswitch.conf
   - /lib64/libnss_centrifydc.so.2
 

ansible/playbooks/roles/compatibility_layer/tasks/install_packages.yml

@@ -1,27 +1,5 @@
 # Install a specified list of sets and packages.
 ---
-- name: Find all strings in libc library
-  command: "strings {{ gentoo_prefix_path }}/usr/lib64/libc.a"
-  register: libc_strings
-  when: eessi_host_os == "linux"
-
-- name: Find user defined trusted dirs in libc strings output
-  set_fact: match='{{ libc_strings.stdout | regex_search("\n" + item + "/?\n") | default('', True) | string | length>0 }}'
-  with_items: "{{ prefix_user_defined_trusted_dirs }}"
-  register: trusted_dirs_in_libc
-
-- name: (Re)install glibc with the user-defined-trusted-dirs option
-  portage:
-    package: sys-libs/glibc
-    noreplace: no
-    oneshot: yes
-  become: no
-  environment:
-    EXTRA_EMAKE: "user-defined-trusted-dirs={{ prefix_user_defined_trusted_dirs | join(':') }}"
-  when:
-    - eessi_host_os == "linux"
-    - trusted_dirs_in_libc.results | selectattr('ansible_facts.match', 'equalto', False) | list | length>0
-
 - name: Install package set {{ package_sets }}
   portage:
     package: "@{{ item }}"

ansible/playbooks/roles/compatibility_layer/tasks/main.yml

@@ -24,8 +24,8 @@
     args:
       apply:
         become: False
-        environment:
-          PYTHON_TARGETS: "{{ prefix_python_targets }}"
+
+  - include_tasks: set_glibc_trusted_dirs.yml
 
   - include_tasks: install_packages.yml
 

ansible/playbooks/roles/compatibility_layer/tasks/set_glibc_trusted_dirs.yml

@@ -0,0 +1,45 @@
+# Make sure that glibc is always compiled with a user-defined-trusted-dirs option
+---
+- name: Find all strings in libc library
+  command: "strings {{ gentoo_prefix_path }}/usr/lib64/libc.a"
+  register: libc_strings
+  when: eessi_host_os == "linux"
+
+- name: Find user defined trusted dirs in libc strings output
+  set_fact: match='{{ libc_strings.stdout | regex_search("\n" + item + "/?\n") | default('', True) | string | length>0 }}'
+  with_items: "{{ prefix_user_defined_trusted_dirs }}"
+  register: trusted_dirs_in_libc
+
+- name: (Re)install glibc with the user-defined-trusted-dirs option
+  portage:
+    package: sys-libs/glibc
+    noreplace: no
+    oneshot: yes
+  become: no
+  environment:
+    EXTRA_EMAKE: "user-defined-trusted-dirs={{ prefix_user_defined_trusted_dirs | join(':') }}"
+  when:
+    - eessi_host_os == "linux"
+    - trusted_dirs_in_libc.results | selectattr('ansible_facts.match', 'equalto', False) | list | length>0
+
+- name: Create portage env directory
+  file:
+    path: "{{ gentoo_prefix_path }}/etc/portage/env"
+    state: directory
+    mode: 0755
+
+- name: Add env file for glibc to make sure the user-defined-trusted-dirs is always used
+  copy:
+    dest: "{{ gentoo_prefix_path }}/etc/portage/env/glibc-user-defined-trusted-dirs.conf"
+    mode: 0644
+    content: |
+      EXTRA_EMAKE="user-defined-trusted-dirs={{ prefix_user_defined_trusted_dirs | join(':') }}"
+
+- name: Add glibc env file to package.env
+  lineinfile:
+    path: "{{ gentoo_prefix_path }}/etc/portage/package.env"
+    create: yes
+    mode: 0644
+    line: sys-libs/glibc glibc-user-defined-trusted-dirs.conf
+    state: present
+

bootstrap-prefix.sh

@@ -282,7 +282,7 @@ configure_toolchain() {
 			;;
 	esac
 
-	return
+	return 0
 }
 
 bootstrap_setup() {
@@ -450,6 +450,12 @@ bootstrap_setup() {
 	ACCEPT_KEYWORDS="~${ARCH}-linux"
 	EOF
 
+	# bug #788613 avoid gcc-11 during stage 2/3 prior sync/emerge -e
+	is-rap && cat >> "${ROOT}"/etc/portage/make.profile/package.mask <<-EOF
+	# during bootstrap mask, bug #788613
+	>=sys-devel/gcc-11
+	EOF
+
 	# Use package.use to disable in the portage tree to be shared between
 	# stage2 and stage3. The hack will be undone during tree sync in stage3.
 	cat >> "${ROOT}"/etc/portage/make.profile/package.use <<-EOF
@@ -1355,8 +1361,8 @@ bootstrap_libressl() {
 
 bootstrap_stage_host_gentoo() {
 	if ! is-rap ; then
-		einfo "Shortcut only supports prefix-standalone, but we are bootstrapping"
-		einfo "prefix-rpath. Do nothing."
+		einfo "Shortcut only supports prefix-standalone, but we "
+		einfo "are bootstrapping prefix-rpath.  Do nothing."
 		return 0
 	fi
 
@@ -1447,6 +1453,7 @@ bootstrap_stage1() {
 	# we're working with now, bug #650284
 	[[ -x ${ROOT}/tmp/usr/bin/bash ]] \
 		|| (bootstrap_bash) || return 1
+
 	# Some host tools need to be wrapped to be useful for us.
 	# We put them in tmp/usr/local/bin, to not accidentally
 	# be identified as stage1-installed like in bug #615410.
@@ -1475,10 +1482,14 @@ bootstrap_stage1() {
 				# We need to direct the system gcc to find the system binutils.
 				cat >> "${ROOT}"/tmp/usr/local/bin/gcc <<-EOF
 					#! /bin/sh
-					PATH="${STAGE1_PATH}" export PATH
-					exec "\${0##*/}" "\$@"
+					PATH="${ORIGINAL_PATH}" export PATH
+					exec "$(type -P gcc)" "\$@"
+				EOF
+				cat >> "${ROOT}"/tmp/usr/local/bin/g++ <<-EOF
+					#! /bin/sh
+					PATH="${ORIGINAL_PATH}" export PATH
+					exec "$(type -P g++)" "\$@"
 				EOF
-				cp "${ROOT}"/tmp/usr/local/bin/g{cc,++}
 				chmod 755 "${ROOT}"/tmp/usr/local/bin/g{cc,++}
 			fi
 			;;
@@ -1905,7 +1916,11 @@ bootstrap_stage3() {
 	}
 
 	# pre_emerge_pkgs relies on stage 2 portage.
-	pre_emerge_pkgs() { is-rap && without_stack_emerge_pkgs "$@" || with_stack_emerge_pkgs "$@"; }
+	pre_emerge_pkgs() {
+		is-rap \
+			&& without_stack_emerge_pkgs "$@" \
+			|| with_stack_emerge_pkgs "$@"
+	}
 
 	# Some packages fail to properly depend on sys-apps/texinfo.
 	# We don't really need that package, so we fake it instead,
@@ -2768,7 +2783,6 @@ EOF
 		# location seems ok
 		break
 	done
-	export STAGE1_PATH=${PATH}
 	export PATH="$EPREFIX/usr/bin:$EPREFIX/bin:$EPREFIX/tmp/usr/bin:$EPREFIX/tmp/bin:$EPREFIX/tmp/usr/local/bin:${PATH}"
 
 	cat << EOF
@@ -3078,9 +3092,6 @@ case ${CHOST} in
 	powerpc-*darwin*)
 		DARWIN_USE_GCC=1  # must use GCC, Clang is impossible
 		;;
-#	arm64-*darwin*)
-#		DARWIN_USE_GCC=0  # cannot use GCC yet (needs silicon support)
-#		;;
 	*-darwin*)
 		# normalise value of DARWIN_USE_GCC
 		case ${DARWIN_USE_GCC} in
@@ -3116,6 +3127,9 @@ case ${CHOST}:${LC_ALL}:${LANG} in
 	;;
 esac
 
+# save original path, need this before interactive, #788334
+ORIGINAL_PATH="${PATH}"
+
 # Just guessing a prefix is kind of scary.  Hence, to make it a bit less
 # scary, we force the user to give the prefix location here.  This also
 # makes the script a bit less dangerous as it will die when just run to

test/compat_layer.py

@@ -182,3 +182,28 @@ def __init__(self):
             sn.assert_eq(self.exit_code, 0),
             sn.assert_found(f'\n/{self.symlink_to_host}\n', self.stdout),
         ])
+
+@rfm.simple_test
+class GlibcEnvFileTest(RunInGentooPrefixTest):
+    def __init__(self):
+        # the glibc env file was added in 2021.06
+        self.skip_if(self.eessi_version == '2021.03')
+
+        super().__init__()
+        self.descr = 'Verify that the env file for sys-libs/glibc was created and is picked up by emerge.'
+        self.command = 'equery has --package glibc EXTRA_EMAKE'
+
+        trusted_dir = os.path.join(
+            self.eessi_repo_dir,
+            'host_injections',
+            self.eessi_version,
+            'compat',
+            self.eessi_os,
+            self.eessi_arch,
+            'lib'
+        )
+
+        self.sanity_patterns = sn.assert_found(
+            f'user-defined-trusted-dirs={trusted_dir}',
+            self.stdout
+        )