Skip to content

Commit

Permalink
Merge pull request #106 from bedroge/gentoo_git
Browse files Browse the repository at this point in the history
Use git instead of rsync for the Gentoo overlay
  • Loading branch information
boegel authored Jun 21, 2021
2 parents 6c8105c + 254a1b8 commit 8541357
Show file tree
Hide file tree
Showing 9 changed files with 180 additions and 1 deletion.
2 changes: 1 addition & 1 deletion .github/actions/install_compatibility_layer/entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ cat << EOF > hosts
127.0.0.1 eessi_host_arch=$(uname -m) eessi_host_os=linux
EOF

ansible-playbook --connection=local --inventory=hosts -e ansible_python_interpreter=python3 -e gentoo_prefix_path=$1 ${GITHUB_WORKSPACE}/ansible/playbooks/install.yml
ansible-playbook -v --connection=local --inventory=hosts -e ansible_python_interpreter=python3 -e gentoo_prefix_path=$1 ${GITHUB_WORKSPACE}/ansible/playbooks/install.yml

# A successful installation should at least have Lmod and archspec,
# so let's check if we can use them.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
#!/bin/bash

repository_name="${1}"
repository_path="${3}"

[[ ${repository_name} == "gentoo" ]] || exit 0

source $(portageq envvar EPREFIX)/lib/gentoo/functions.sh

# Number of jobs for egencache, default is number or processors.
parallel_jobs="$(nproc)"

if [[ -f ${repository_path}/metadata/timestamp.x ]]; then
portage_current_timestamp=$(cut -f 1 -d " " "${repository_path}/metadata/timestamp.x" )
else
portage_current_timestamp=0
fi

ebegin "Fetching metadata timestamp for ${repository_name}"
rsync -aq rsync://rsync.gentoo.org/gentoo-portage/metadata/timestamp.x "${repository_path}"/metadata/timestamp.x
eend $?
portage_new_timestamp=$(cut -f 1 -d " " "${repository_path}/metadata/timestamp.x" )

if [[ ${portage_current_timestamp} -lt ${portage_new_timestamp} ]]; then
ebegin "Fetching pre-generated metadata cache for ${repository_name}"
rsync -aq rsync://rsync.gentoo.org/gentoo-portage/metadata/md5-cache/ "${repository_path}"/metadata/md5-cache/
eend $?
else
einfo "Metadata cache for ${repository_name} already recent, no need to fetch it :-)"
fi

ebegin "Updating metadata cache for ${repository_name}"
egencache --jobs="${parallel_jobs}" --repo="${repository_name}" --update --update-use-local-desc
eend $?
17 changes: 17 additions & 0 deletions ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_dtd
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
#!/bin/bash

repository_name="${1}"
repository_path="${3}"

[[ ${repository_name} == "gentoo" ]] || exit 0

source $(portageq envvar EPREFIX)/lib/gentoo/functions.sh

DTDDIR="${repository_path}"/metadata/dtd
ebegin "Updating DTDs"
if [[ -e ${DTDDIR} ]]; then
git -C "${DTDDIR}" pull -q --ff-only
else
git clone -q https://anongit.gentoo.org/git/data/dtd.git "${DTDDIR}"
fi
eend "$?"
17 changes: 17 additions & 0 deletions ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_glsa
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
#!/bin/bash

repository_name="${1}"
repository_path="${3}"

[[ ${repository_name} == "gentoo" ]] || exit 0

source $(portageq envvar EPREFIX)/lib/gentoo/functions.sh

GLSADIR="${repository_path}"/metadata/glsa
ebegin "Updating GLSAs"
if [[ -e ${GLSADIR} ]]; then
git -C "${GLSADIR}" pull -q --ff-only
else
git clone -q https://anongit.gentoo.org/git/data/glsa.git "${GLSADIR}"
fi
eend "$?"
17 changes: 17 additions & 0 deletions ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_news
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
#!/bin/bash

repository_name="${1}"
repository_path="${3}"

[[ ${repository_name} == "gentoo" ]] || exit 0

source $(portageq envvar EPREFIX)/lib/gentoo/functions.sh

NEWSDIR="${repository_path}"/metadata/news
ebegin "Updating news items"
if [[ -e ${NEWSDIR} ]]; then
git -C "${NEWSDIR}" pull -q --ff-only
else
git clone -q https://anongit.gentoo.org/git/data/gentoo-news.git "${NEWSDIR}"
fi
eend $? "Try to remove ${NEWSDIR}"
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/bash

repository_name="${1}"
repository_path="${3}"

[[ ${repository_name} == "gentoo" ]] || exit 0

source $(portageq envvar EPREFIX)/lib/gentoo/functions.sh

ebegin "Updating projects.xml"
wget -q -P "${repository_path}"/metadata/ -N https://api.gentoo.org/metastructure/projects.xml
eend $?
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
#!/bin/bash

repository_name="${1}"

[[ ${repository_name} == "gentoo" ]] && exit 0

source $(portageq envvar EPREFIX)/lib/gentoo/functions.sh

# Number of jobs for egencache, default is number or processors.
parallel_jobs="$(nproc)"

ebegin "Updating metadata cache for ${repository_name}"
egencache --jobs="${parallel_jobs}" --repo="${repository_name}" --update --update-use-local-desc
eend $?
48 changes: 48 additions & 0 deletions ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,46 @@
selectattr('eclass-overrides', 'equalto', True) | map(attribute='name') | join(' ')
}}
- name: Use git instead of rsync for the Gentoo overlay
copy:
dest: "{{ gentoo_prefix_path }}/etc/portage/repos.conf/gentoo.conf"
mode: "0644"
content: |
[DEFAULT]
main-repo = gentoo
sync-git-pull-extra-opts = --quiet
[gentoo]
priority = 1
location = {{ gentoo_prefix_path }}/var/db/repos/gentoo
sync-uri = https://github.com/gentoo/gentoo.git
sync-type = git
auto-sync = Yes
clone-depth = 1
- name: Create repo.postsync.d directory
file:
path: "{{ gentoo_prefix_path }}/etc/portage/repo.postsync.d"
state: directory
mode: '0755'

- name: Add post-sync hooks for Gentoo overlay
copy:
src: "{{ item }}"
dest: "{{ gentoo_prefix_path }}/etc/portage/repo.postsync.d/{{ item }}"
mode: 0755
with_items:
- sync_gentoo_cache
- sync_gentoo_dtd
- sync_gentoo_glsa
- sync_gentoo_news
- sync_overlay_cache

- name: Remove the old Gentoo overlay directory
file:
state: absent
path: "{{ gentoo_prefix_path }}/var/db/repos/gentoo"

- name: Sync the repositories
portage:
sync: 'yes'
Expand All @@ -77,3 +117,11 @@
force: yes
with_items:
"{{ find_configs.results | rejectattr('files', 'equalto', []) | map(attribute='files') | list }}"

- name: Update system set if we are running in an existing Prefix installation
portage:
changed_use: yes
deep: yes
package: "@world"
update: yes
when: startprefix.stat.exists
20 changes: 20 additions & 0 deletions test/compat_layer.py
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,26 @@ def __init__(self):
sn.assert_found(f'\n/{self.symlink_to_host}\n', self.stdout),
])


@rfm.simple_test
class GentooOverlayGitTest(RunInGentooPrefixTest):
def __init__(self):
# the switch to git was made in pilot version 2021.06
self.skip_if(self.eessi_version == '2021.03')

super().__init__()
self.descr = 'Verify that the Gentoo overlay is synced using git.'
self.command = f'emerge --info'

gentoo_repo_dir = os.path.join(self.compat_dir, 'var', 'db', 'repos', 'gentoo')
gentoo_git_repo_info = '''gentoo
location: %s
sync-type: git
sync-uri: https://github.com/gentoo/gentoo.git''' % gentoo_repo_dir

self.sanity_patterns = sn.assert_found(gentoo_git_repo_info, self.stdout)


@rfm.simple_test
class GlibcEnvFileTest(RunInGentooPrefixTest):
def __init__(self):
Expand Down

0 comments on commit 8541357

Please sign in to comment.