From 7258477f6e022bef2d6bb71f454eaf85fc173e61 Mon Sep 17 00:00:00 2001 From: Rob Woolley Date: Wed, 11 Dec 2024 10:35:58 -0500 Subject: [PATCH 01/25] Fix code style guide problems for PEP8 and Flake8 The CI system picked up non-conformance issues with recent changes to the ebuild generator. Fix lines that exceed 79 characters and remove an used chunk_count variable. Signed-off-by: Rob Woolley --- superflore/generators/ebuild/gen_packages.py | 10 +++++++--- superflore/generators/ebuild/overlay_instance.py | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/superflore/generators/ebuild/gen_packages.py b/superflore/generators/ebuild/gen_packages.py index d002b35b..87032d60 100644 --- a/superflore/generators/ebuild/gen_packages.py +++ b/superflore/generators/ebuild/gen_packages.py @@ -156,7 +156,8 @@ def _gen_metadata_for_package( warn("fetch metadata for package {}".format(pkg_name)) return pkg_metadata_xml package_condition_context = _package_condition_context(distro.name) - pkg = PackageMetadata(pkg_xml, evaluate_condition_context=package_condition_context) + pkg = PackageMetadata(pkg_xml, + evaluate_condition_context=package_condition_context) pkg_metadata_xml.upstream_email = pkg.upstream_email pkg_metadata_xml.upstream_name = pkg.upstream_name pkg_metadata_xml.longdescription = pkg.longdescription @@ -174,7 +175,9 @@ def _gen_ebuild_for_package( pkg_ebuild.src_uri = pkg_rosinstall[0]['tar']['uri'] pkg_names = get_package_names(distro) package_condition_context = _package_condition_context(distro.name) - pkg_dep_walker = DependencyWalker(distro, evaluate_condition_context=package_condition_context) + pkg_dep_walker = DependencyWalker( + distro, + evaluate_condition_context=package_condition_context) pkg_buildtool_deps = pkg_dep_walker.get_depends(pkg_name, "buildtool") pkg_build_deps = pkg_dep_walker.get_depends(pkg_name, "build") @@ -209,7 +212,8 @@ def _gen_ebuild_for_package( except Exception: warn("fetch metadata for package {}".format(pkg_name)) return pkg_ebuild - pkg = PackageMetadata(pkg_xml, evaluate_condition_context=package_condition_context) + pkg = PackageMetadata(pkg_xml, + evaluate_condition_context=package_condition_context) pkg_ebuild.upstream_license = pkg.upstream_license pkg_ebuild.description = pkg.description pkg_ebuild.homepage = pkg.homepage diff --git a/superflore/generators/ebuild/overlay_instance.py b/superflore/generators/ebuild/overlay_instance.py index 382ba6db..cd3ee03a 100644 --- a/superflore/generators/ebuild/overlay_instance.py +++ b/superflore/generators/ebuild/overlay_instance.py @@ -82,7 +82,6 @@ def regenerate_manifests( dock.map_directory(self.repo.repo_dir, '/tmp/ros-overlay') for distro in regen_dict.keys(): chunk_list = [] - chunk_count = 0 pkg_list = regen_dict[distro] while len(pkg_list) > 0: current_chunk = list() @@ -96,7 +95,8 @@ def regenerate_manifests( info("key_lists: '%s'" % chunk_list) for chunk in chunk_list: for pkg in chunk: - pkg_dir = '/tmp/ros-overlay/ros-{0}/{1}'.format(distro, pkg) + pkg_dir = '/tmp/ros-overlay/ros-{0}/{1}'.format(distro, + pkg) dock.add_bash_command('cd {0}'.format(pkg_dir)) dock.add_bash_command('repoman manifest') try: From e373e3a79da6630e247e1fdf5fdeb00d93149d39 Mon Sep 17 00:00:00 2001 From: Ben Wolsieffer Date: Mon, 1 Apr 2019 18:41:52 -0400 Subject: [PATCH 02/25] Add support for Nix package generation. --- README.md | 1 + setup.py | 1 + superflore/generators/nix/__init__.py | 3 + superflore/generators/nix/gen_packages.py | 97 +++++++++ superflore/generators/nix/nix_derivation.py | 176 ++++++++++++++++ superflore/generators/nix/nix_package.py | 134 +++++++++++++ superflore/generators/nix/nix_package_set.py | 31 +++ superflore/generators/nix/nix_ros_overlay.py | 43 ++++ superflore/generators/nix/run.py | 199 +++++++++++++++++++ superflore/utils.py | 24 +++ 10 files changed, 709 insertions(+) create mode 100644 superflore/generators/nix/__init__.py create mode 100644 superflore/generators/nix/gen_packages.py create mode 100644 superflore/generators/nix/nix_derivation.py create mode 100644 superflore/generators/nix/nix_package.py create mode 100644 superflore/generators/nix/nix_package_set.py create mode 100644 superflore/generators/nix/nix_ros_overlay.py create mode 100644 superflore/generators/nix/run.py diff --git a/README.md b/README.md index bb32844f..912582ec 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ Supported Platforms: -------------------- * Gentoo * OpenEmbedded + * Nix Installation: ============= diff --git a/setup.py b/setup.py index 0dc34dcd..0e61a3f9 100755 --- a/setup.py +++ b/setup.py @@ -86,6 +86,7 @@ def append_local_version_label(public_version): 'console_scripts': [ 'superflore-gen-ebuilds = superflore.generators.ebuild:main', 'superflore-gen-oe-recipes = superflore.generators.bitbake:main', + 'superflore-gen-nix = superflore.generators.nix:main', 'superflore-check-ebuilds = superflore.test_integration.gentoo:main', ] } diff --git a/superflore/generators/nix/__init__.py b/superflore/generators/nix/__init__.py new file mode 100644 index 00000000..5e40da76 --- /dev/null +++ b/superflore/generators/nix/__init__.py @@ -0,0 +1,3 @@ +from superflore.generators.nix.run import main +if __name__ == '__main__': + main() diff --git a/superflore/generators/nix/gen_packages.py b/superflore/generators/nix/gen_packages.py new file mode 100644 index 00000000..b4113a9c --- /dev/null +++ b/superflore/generators/nix/gen_packages.py @@ -0,0 +1,97 @@ +# Copyright 2019 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from typing import Iterable + +from rosdistro import DistributionFile +from rosinstall_generator.distro import get_package_names + +from superflore.exceptions import NoPkgXml +from superflore.exceptions import UnresolvedDependency +from superflore.generators.nix.nix_package import NixPackage +from superflore.generators.nix.nix_package_set import NixPackageSet +from superflore.utils import err +from superflore.utils import make_dir +from superflore.utils import ok + +org = "Open Source Robotics Foundation" +org_license = "BSD" + + +def regenerate_pkg(overlay, pkg: str, distro: DistributionFile, + preserve_existing: bool, tar_dir: str, sha256_cache): + pkg_names = get_package_names(distro)[0] + + if pkg not in pkg_names: + raise RuntimeError("Unknown package '{}'".format(pkg)) + + normalized_pkg = NixPackage.normalize_name(pkg) + + package_dir = os.path.join(overlay.repo.repo_dir, distro.name, + normalized_pkg) + package_file = os.path.join(package_dir, 'default.nix') + make_dir(package_dir) + + # check for an existing package + existing = os.path.exists(package_file) + + if preserve_existing and existing: + ok("derivation for package '{}' up to date, skipping...".format(pkg)) + return None, [] + + try: + current = NixPackage(pkg, distro, tar_dir, sha256_cache) + except Exception as e: + err('Failed to generate derivation for package {}!'.format(pkg)) + raise e + + try: + derivation_text = current.derivation.get_text(org, org_license) + except UnresolvedDependency: + err("'Failed to resolve required dependencies for package {}!" + .format(pkg)) + unresolved = current.unresolved_dependencies + for dep in unresolved: + err(" unresolved: \"{}\"".format(dep)) + return None, unresolved + except NoPkgXml: + err("Could not fetch pkg!") + return None, [] + except Exception as e: + err('Failed to generate derivation for package {}!'.format(pkg)) + raise e + + ok("Successfully generated derivation for package '{}'.".format(pkg)) + try: + with open('{0}'.format(package_file), "w") as recipe_file: + recipe_file.write(derivation_text) + except Exception as e: + err("Failed to write derivation to disk!") + raise e + return current, [] + + +def regenerate_pkg_set(overlay, distro_name: str, pkg_names: Iterable[str]): + distro_dir = os.path.join(overlay.repo.repo_dir, distro_name) + overlay_file = os.path.join(distro_dir, 'generated.nix') + make_dir(distro_dir) + + package_set = NixPackageSet(pkg_names) + + try: + with open(overlay_file, "w") as recipe_file: + recipe_file.write(package_set.get_text(org, org_license)) + except Exception as e: + err("Failed to write derivation to disk!") + raise e diff --git a/superflore/generators/nix/nix_derivation.py b/superflore/generators/nix/nix_derivation.py new file mode 100644 index 00000000..b0c13686 --- /dev/null +++ b/superflore/generators/nix/nix_derivation.py @@ -0,0 +1,176 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) 2016 David Bensoussan, Synapticon GmbH +# Copyright (c) 2019 Open Source Robotics Foundation, Inc. +# +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +from textwrap import dedent +from time import gmtime, strftime +from typing import Iterable + +from superflore.exceptions import UnknownLicense +from superflore.utils import get_license + + +class NixLicense: + """ + Generates + """ + + _LICENSE_MAP = { + 'Apache-2.0': 'asl20', + 'BSD': 'bsdOriginal', + 'BSD-2': 'bsd2', + 'LGPL-2': 'lgpl2', + 'LGPL-2.1': 'lgpl21', + 'LGPL-3': 'lgpl3', + 'GPL-1': 'gpl1', + 'GPL-2': 'gpl2', + 'GPL-3': 'gpl3', + 'MPL-1.0': 'mpl10', + 'MPL-1.1': 'mpl11', + 'MPL-2.0': 'mpl20', + 'MIT': 'mit', + 'CC-BY-NC-SA-4.0': 'cc-by-nc-sa-40', + 'Boost-1.0': 'boost', + 'public_domain': 'publicDomain' + } + + def __init__(self, name): + try: + name = get_license(name) + self.name = self._LICENSE_MAP[name] + self.custom = False + except (KeyError, UnknownLicense): + self.name = name + self.custom = True + + def nix_code(self) -> str: + if self.custom: + return '"{}"'.format(self.name) + else: + return self.name + + +class NixDerivation: + def __init__(self, name: str, version: str, src_uri: str, src_sha256: str, + description: str, licenses: Iterable[NixLicense], + distro_name: str, + build_inputs: Iterable[str] = tuple(), + propagated_build_inputs: Iterable[str] = tuple(), + check_inputs: Iterable[str] = tuple(), + native_build_inputs: Iterable[str] = tuple(), + propagated_native_build_inputs: Iterable[str] = tuple() + ) -> None: + self.name = name + self.version = version + self.src_uri = src_uri + self.src_sha256 = src_sha256 + + self.description = description + self.licenses = licenses + self.distro_name = distro_name + + self.build_inputs = set(build_inputs) + self.propagated_build_inputs = set(propagated_build_inputs) + self.check_inputs = set(check_inputs) + self.native_build_inputs = set(native_build_inputs) + self.propagated_native_build_inputs = \ + set(propagated_native_build_inputs) + + @staticmethod + def _to_nix_list(it: Iterable[str]) -> str: + return '[ ' + ' '.join(it) + ' ]' + + @staticmethod + def _to_nix_parameter(dep: str) -> str: + return dep.split('.')[0] + + def get_text(self, distributor: str, license_name: str) -> str: + """ + Generate the Nix derivation, given the distributor line + and the license text. + """ + + ret = [] + ret += dedent(''' + # Copyright {} {} + # Distributed under the terms of the {} license + + ''').format( + strftime("%Y", gmtime()), distributor, + license_name) + + ret += '{ lib, buildRosPackage, fetchurl, ' + \ + ', '.join(set(map(self._to_nix_parameter, + self.build_inputs | + self.check_inputs | + self.propagated_build_inputs | + self.native_build_inputs | + self.propagated_native_build_inputs))) + ' }:' + + ret += dedent(''' + buildRosPackage {{ + pname = "ros-{}-{}"; + version = "{}"; + + src = fetchurl {{ + url = {}; + sha256 = "{}"; + }}; + + ''').format( + self.distro_name, self.name, + self.version, + self.src_uri, + self.src_sha256) + + if self.build_inputs: + ret += " buildInputs = {};\n" \ + .format(self._to_nix_list(self.build_inputs)) + + if self.check_inputs: + ret += " checkInputs = {};\n" \ + .format(self._to_nix_list(self.check_inputs)) + + if self.propagated_build_inputs: + ret += " propagatedBuildInputs = {};\n" \ + .format(self._to_nix_list(self.propagated_build_inputs)) + + if self.native_build_inputs: + ret += " nativeBuildInputs = {};\n" \ + .format(self._to_nix_list(self.native_build_inputs)) + + if self.propagated_native_build_inputs: + ret += " propagatedNativeBuildInputs = {};\n".format( + self._to_nix_list(self.propagated_native_build_inputs)) + + ret += dedent(''' + meta = {{ + description = ''{}''; + license = with lib.licenses; {}; + }}; + }} + ''').format(self.description, + self._to_nix_list(map(NixLicense.nix_code, self.licenses))) + + return ''.join(ret) diff --git a/superflore/generators/nix/nix_package.py b/superflore/generators/nix/nix_package.py new file mode 100644 index 00000000..441df059 --- /dev/null +++ b/superflore/generators/nix/nix_package.py @@ -0,0 +1,134 @@ +import hashlib +import itertools +import os +import tarfile +from typing import Dict, Iterable +from urllib.request import urlretrieve + +from rosdistro import DistributionFile +from rosdistro.dependency_walker import DependencyWalker +from rosdistro.rosdistro import RosPackage +from rosinstall_generator.distro import _generate_rosinstall, get_package_names + +from superflore.PackageMetadata import PackageMetadata +from superflore.exceptions import UnresolvedDependency +from superflore.generators.nix.nix_derivation import NixDerivation, NixLicense +from superflore.utils import info, get_pkg_version, warn, resolve_dep, \ + get_distro_condition_context + + +class NixPackage: + """ + Retrieves the required metadata to define a Nix package derivation. + """ + + def __init__(self, name: str, distro: DistributionFile, tar_dir: str, + sha256_cache: Dict[str, str]) -> None: + self.distro = distro + + pkg = distro.release_packages[name] + repo = distro.repositories[pkg.repository_name].release_repository + ros_pkg = RosPackage(name, repo) + + rosinstall = _generate_rosinstall(name, repo.url, + repo.get_release_tag(name), True) + + normalized_name = NixPackage.normalize_name(name) + version = get_pkg_version(distro, name) + src_uri = rosinstall[0]['tar']['uri'] + + archive_path = os.path.join(tar_dir, '{}-{}-{}.tar.gz' + .format(self.normalize_name(name), + version, distro.name)) + + downloaded_archive = False + if os.path.exists(archive_path): + info("using cached archive for package '{}'...".format(name)) + else: + info("downloading archive version for package '{}'...".format(name)) + urlretrieve(src_uri, archive_path) + downloaded_archive = True + + if downloaded_archive or archive_path not in sha256_cache: + sha256_cache[archive_path] = hashlib.sha256( + open(archive_path, 'rb').read()).hexdigest() + src_sha256 = sha256_cache[archive_path] + + # We already have the archive, so try to extract package.xml from it. + # This is much faster than downloading it from GitHub. + package_xml = None + archive = tarfile.open(archive_path, 'r') + while True: + file_info = archive.next() + if file_info is None: + break + if '/' not in file_info.name: + root = file_info.name + package_xml = archive.extractfile(root + '/package.xml').read() + break + # Fallback to the standard method of fetching package.xml + if package_xml is None: + warn("failed to extract package.xml from archive: {}".format(e)) + package_xml = ros_pkg.get_package_xml(distro.name) + + metadata = PackageMetadata(package_xml) + + dep_walker = DependencyWalker(distro, + get_distro_condition_context(distro.name)) + + buildtool_deps = dep_walker.get_depends(pkg.name, "buildtool") + build_deps = dep_walker.get_depends(pkg.name, "build") + # TODO: do we need exec depends as well + run_deps = dep_walker.get_depends(pkg.name, "run") + test_deps = dep_walker.get_depends(pkg.name, "test") + + self.unresolved_dependencies = set() + + build_inputs = self._resolve_dependencies(build_deps) + propagated_build_inputs = self._resolve_dependencies(run_deps) + check_inputs = self._resolve_dependencies(test_deps) + native_build_inputs = self._resolve_dependencies(buildtool_deps) + + self._derivation = NixDerivation( + name=normalized_name, + version=version, + src_uri=src_uri, + src_sha256=src_sha256, + description=metadata.description, + licenses=map(NixLicense, metadata.upstream_license), + distro_name=distro.name, + build_inputs=build_inputs, + propagated_build_inputs=propagated_build_inputs, + check_inputs=check_inputs, + native_build_inputs=native_build_inputs) + + def _resolve_dependencies(self, deps: Iterable[str]) -> Iterable[str]: + return itertools.chain.from_iterable( + map(self._resolve_dependency, deps)) + + def _resolve_dependency(self, d: str) -> Iterable[str]: + try: + return (self.normalize_name(d),) \ + if d in self.distro.release_packages \ + else resolve_dep(d, 'nix')[0] + except UnresolvedDependency: + self.unresolved_dependencies.add(d) + return tuple() + + @staticmethod + def normalize_name(name: str) -> str: + """ + Convert underscores to dashes to match normal Nix package naming + conventions. + + :param name: original package name + :return: normalized package name + """ + return name.replace('_', '-') + + @property + def derivation(self): + if self.unresolved_dependencies: + raise UnresolvedDependency("failed to resolve dependencies!") + + return self._derivation diff --git a/superflore/generators/nix/nix_package_set.py b/superflore/generators/nix/nix_package_set.py new file mode 100644 index 00000000..1bae511f --- /dev/null +++ b/superflore/generators/nix/nix_package_set.py @@ -0,0 +1,31 @@ +from textwrap import dedent +from time import strftime, gmtime +from typing import Iterable + +from superflore.generators.nix.nix_package import NixPackage + + +class NixPackageSet: + """ + Code generator for Nix overlay package set. Each package in the set must + be defined as a function in a default.nix file within a directory named + after the package. The package functions are called with callPackage. + """ + + def __init__(self, pkg_names: Iterable[str]): + self.pkg_names = pkg_names + + def get_text(self, distributor: str, license_name: str) -> str: + ret = [] + ret += dedent(''' + # Copyright {} {} + # Distributed under the terms of the {} license + + self: super: {{ + + ''').format(strftime("%Y", gmtime()), distributor, license_name) + ret.extend((" {0} = self.callPackage ./{0} {{}};\n\n" + .format(NixPackage.normalize_name(pkg_name)) + for pkg_name in self.pkg_names)) + ret += "}\n" + return ''.join(ret) diff --git a/superflore/generators/nix/nix_ros_overlay.py b/superflore/generators/nix/nix_ros_overlay.py new file mode 100644 index 00000000..ab2a5f99 --- /dev/null +++ b/superflore/generators/nix/nix_ros_overlay.py @@ -0,0 +1,43 @@ +# Copyright 2019 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +from superflore.repo_instance import RepoInstance +from superflore.utils import info +from superflore.utils import rand_ascii_str + + +class NixRosOverlay(object): + def __init__(self, repo_dir, do_clone, org='lopsided98', repo='nix-ros-overlay'): + self.repo = RepoInstance(org, repo, repo_dir, do_clone) + self.branch_name = 'nix-bot-%s' % rand_ascii_str() + info('Creating new branch {0}...'.format(self.branch_name)) + self.repo.create_branch(self.branch_name) + + def commit_changes(self, distro): + info('Adding changes...') + if distro == 'all': + commit_msg = 'regenerate all distros, {0}' + self.repo.git.add('*/*/default.nix') + else: + commit_msg = 'regenerate ros-{1}, {0}' + self.repo.git.add(distro) + commit_msg = commit_msg.format(time.ctime(), distro) + info('Committing to branch {0}...'.format(self.branch_name)) + self.repo.git.commit(m=commit_msg) + + def pull_request(self, message, distro=None): + pr_title = 'rosdistro sync, {0}'.format(time.ctime()) + self.repo.pull_request(message, pr_title, branch=distro) diff --git a/superflore/generators/nix/run.py b/superflore/generators/nix/run.py new file mode 100644 index 00000000..31150d88 --- /dev/null +++ b/superflore/generators/nix/run.py @@ -0,0 +1,199 @@ +# Copyright 2019 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +from rosinstall_generator.distro import get_distro + +from superflore.CacheManager import CacheManager +from superflore.TempfileManager import TempfileManager +from superflore.generate_installers import generate_installers +from superflore.generators.nix.gen_packages import regenerate_pkg, \ + regenerate_pkg_set +from superflore.generators.nix.nix_ros_overlay import NixRosOverlay +from superflore.parser import get_parser +from superflore.repo_instance import RepoInstance +from superflore.utils import clean_up, get_distros_by_status +from superflore.utils import err +from superflore.utils import file_pr +from superflore.utils import gen_delta_msg +from superflore.utils import gen_missing_deps_msg +from superflore.utils import info +from superflore.utils import load_pr +from superflore.utils import ok +from superflore.utils import save_pr +from superflore.utils import url_to_repo_org +from superflore.utils import warn + + +def main(): + preserve_existing = True + parser = get_parser('Deploy ROS packages using Nix') + parser.add_argument( + '--tar-archive-dir', + help='location to store archived packages', + type=str + ) + args = parser.parse_args(sys.argv[1:]) + pr_comment = args.pr_comment + skip_keys = args.skip_keys or [] + selected_targets = None + if args.pr_only: + if args.dry_run: + parser.error('Invalid args! cannot dry-run and file PR') + if not args.output_repository_path: + parser.error('Invalid args! no repository specified') + try: + prev_overlay = RepoInstance(args.output_repository_path, False) + msg, title = load_pr() + prev_overlay.pull_request(msg, title) + clean_up() + sys.exit(0) + except Exception as e: + err('Failed to file PR!') + err('reason: {0}'.format(e)) + sys.exit(1) + elif args.all: + warn('"All" mode detected... This may take a while!') + preserve_existing = False + elif args.ros_distro: + warn('"{0}" distro detected...'.format(args.ros_distro)) + selected_targets = [args.ros_distro] + preserve_existing = False + elif args.only: + parser.error('Invalid args! --only requires specifying --ros-distro') + if not selected_targets: + selected_targets = get_distros_by_status('active') + repo_org = 'lopsided98' + repo_name = 'nix-ros-overlay' + if args.upstream_repo: + repo_org, repo_name = url_to_repo_org(args.upstream_repo) + with TempfileManager(args.output_repository_path) as _repo: + if not args.output_repository_path: + # give our group write permissions to the temp dir + os.chmod(_repo, 17407) + # clone if args.output_repository_path is None + overlay = NixRosOverlay( + _repo, + not args.output_repository_path, + org=repo_org, + repo=repo_name + ) + if not preserve_existing and not args.only: + pr_comment = pr_comment or ( + 'Superflore Nix generator began regeneration of all packages ' + ' from ROS distro %s from nix-ros-overlay commit %s.' % ( + selected_targets, + overlay.repo.get_last_hash() + ) + ) + elif not args.only: + pr_comment = pr_comment or ( + 'Superflore Nix generator ran update from nix-ros-overlay ' + + 'commit %s.' % (overlay.repo.get_last_hash()) + ) + # generate installers + total_installers = dict() + total_broken = set() + total_changes = dict() + if args.tar_archive_dir: + sha256_filename = '%s/sha256_cache.pickle' % args.tar_archive_dir + else: + sha256_filename = None + with TempfileManager(args.tar_archive_dir) as tar_dir, \ + CacheManager(sha256_filename) as sha256_cache: + if args.only: + for pkg in args.only: + if pkg in skip_keys: + warn("Package '%s' is in skip-keys list, skipping..." + % pkg) + continue + info("Regenerating package '%s'..." % pkg) + try: + regenerate_pkg( + overlay, + pkg, + get_distro(args.ros_distro), + preserve_existing, + tar_dir, + sha256_cache + ) + except KeyError: + err("No package to satisfy key '%s'" % pkg) + sys.exit(1) + # Commit changes and file pull request + regen_dict = dict() + regen_dict[args.ros_distro] = args.only + overlay.commit_changes(args.ros_distro) + if args.dry_run: + save_pr(overlay, args.only, '', pr_comment) + sys.exit(0) + delta = "Regenerated: '%s'\n" % args.only + file_pr(overlay, delta, '', pr_comment, distro=args.ros_distro) + ok('Successfully synchronized repositories!') + sys.exit(0) + + for distro in selected_targets: + distro_installers, distro_broken, distro_changes = \ + generate_installers( + get_distro(distro), + overlay, + regenerate_pkg, + preserve_existing, + tar_dir, + sha256_cache, + skip_keys=skip_keys, + ) + for key in distro_broken.keys(): + for pkg in distro_broken[key]: + total_broken.add(pkg) + + total_changes[distro] = distro_changes + total_installers[distro] = distro_installers + + # If we are just updating a few packages using --only, then + # leave the package set alone. This means that new packages will + # not be added, but it is still useful for updates. + if not preserve_existing: + regenerate_pkg_set(overlay, distro, distro_installers) + ok('Generated package set for distro \'{}\''.format(distro)) + + num_changes = 0 + for distro_name in total_changes: + num_changes += len(total_changes[distro_name]) + + if num_changes == 0: + info('ROS distro is up to date.') + info('Exiting...') + clean_up() + sys.exit(0) + + # remove duplicates + delta = gen_delta_msg(total_changes) + missing_deps = gen_missing_deps_msg(total_broken) + + # Commit changes and file pull request + overlay.commit_changes('all' if args.all else args.ros_distro) + + if args.dry_run: + info('Running in dry mode, not filing PR') + save_pr( + overlay, delta, missing_deps=missing_deps, comment=pr_comment + ) + sys.exit(0) + file_pr(overlay, delta, missing_deps, comment=pr_comment) + + clean_up() + ok('Successfully synchronized repositories!') diff --git a/superflore/utils.py b/superflore/utils.py index 8fb5cc71..1245c20c 100644 --- a/superflore/utils.py +++ b/superflore/utils.py @@ -20,6 +20,7 @@ import string import sys import time +from typing import Dict from pkg_resources import DistributionNotFound, get_distribution from superflore.exceptions import UnknownPlatform @@ -700,6 +701,8 @@ def resolve_dep(pkg, os, distro=None): return resolve_rosdep_key(pkg, 'openembedded', '', distro) elif os == 'gentoo': return resolve_rosdep_key(pkg, 'gentoo', '2.4.0') + elif os == 'nix': + return resolve_rosdep_key(pkg, 'nixos', '') else: msg = "Unknown target platform '{0}'".format(os) raise UnknownPlatform(msg) @@ -715,6 +718,27 @@ def get_distros_by_status(status='active'): if t[1].get('distribution_status') == status] +def get_distro_condition_context(distro_name: str) -> Dict[str, str]: + """ + Get the condition context for a particular ROS distro. This context is used + to evaluate conditions in package.xml format 3. + + This allows superflore to support packages that are designed to work with + both ROS 1 and 2, which have conditional dependencies on catkin and ament. + + :param distro_name: name of the ROS distro + :return: dictionary containing context keys and their values + """ + index = get_cached_index() + distro = index.distributions[distro_name] + context = {'ROS_DISTRO': distro_name} + if distro['distribution_type'] == 'ros1': + context['ROS_VERSION'] = '1' + elif distro['distribution_type'] == 'ros2': + context['ROS_VERSION'] = '2' + return context + + def gen_delta_msg(total_changes, markup='*'): """Return string of changes for the PR message.""" delta = '' From d7848b1246bf645a49d1eefa41ffc57e4a540d49 Mon Sep 17 00:00:00 2001 From: Ben Wolsieffer Date: Sat, 4 May 2019 23:56:53 -0400 Subject: [PATCH 03/25] Simple performance optimizations based on profiling. This makes Nix package generation nearly 4 times faster, if tarballs are already cached. Some of the optimizations also apply to other distros, but the performance impact there was not tested. --- superflore/generators/nix/gen_packages.py | 11 ++++++----- superflore/generators/nix/nix_package.py | 23 +++++++++++------------ superflore/rosdep_support.py | 13 +++++++------ 3 files changed, 24 insertions(+), 23 deletions(-) diff --git a/superflore/generators/nix/gen_packages.py b/superflore/generators/nix/gen_packages.py index b4113a9c..6ba10040 100644 --- a/superflore/generators/nix/gen_packages.py +++ b/superflore/generators/nix/gen_packages.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import os -from typing import Iterable +from typing import Iterable, Dict, Set from rosdistro import DistributionFile from rosinstall_generator.distro import get_package_names @@ -30,10 +30,11 @@ def regenerate_pkg(overlay, pkg: str, distro: DistributionFile, - preserve_existing: bool, tar_dir: str, sha256_cache): - pkg_names = get_package_names(distro)[0] + preserve_existing: bool, tar_dir: str, + sha256_cache: Dict[str, str]): + all_pkgs = set(get_package_names(distro)[0]) - if pkg not in pkg_names: + if pkg not in all_pkgs: raise RuntimeError("Unknown package '{}'".format(pkg)) normalized_pkg = NixPackage.normalize_name(pkg) @@ -51,7 +52,7 @@ def regenerate_pkg(overlay, pkg: str, distro: DistributionFile, return None, [] try: - current = NixPackage(pkg, distro, tar_dir, sha256_cache) + current = NixPackage(pkg, distro, tar_dir, sha256_cache, all_pkgs) except Exception as e: err('Failed to generate derivation for package {}!'.format(pkg)) raise e diff --git a/superflore/generators/nix/nix_package.py b/superflore/generators/nix/nix_package.py index 441df059..616820a5 100644 --- a/superflore/generators/nix/nix_package.py +++ b/superflore/generators/nix/nix_package.py @@ -1,8 +1,9 @@ import hashlib import itertools import os +import re import tarfile -from typing import Dict, Iterable +from typing import Dict, Iterable, Set from urllib.request import urlretrieve from rosdistro import DistributionFile @@ -23,8 +24,9 @@ class NixPackage: """ def __init__(self, name: str, distro: DistributionFile, tar_dir: str, - sha256_cache: Dict[str, str]) -> None: + sha256_cache: Dict[str, str], all_pkgs: Set[str]) -> None: self.distro = distro + self._all_pkgs = all_pkgs pkg = distro.release_packages[name] repo = distro.repositories[pkg.repository_name].release_repository @@ -56,19 +58,16 @@ def __init__(self, name: str, distro: DistributionFile, tar_dir: str, # We already have the archive, so try to extract package.xml from it. # This is much faster than downloading it from GitHub. + package_xml_regex = re.compile(r'^[^/]+/package\.xml$') package_xml = None - archive = tarfile.open(archive_path, 'r') - while True: - file_info = archive.next() - if file_info is None: - break - if '/' not in file_info.name: - root = file_info.name - package_xml = archive.extractfile(root + '/package.xml').read() + archive = tarfile.open(archive_path, 'r|*') + for file in archive: + if package_xml_regex.match(file.name): + package_xml = archive.extractfile(file).read() break # Fallback to the standard method of fetching package.xml if package_xml is None: - warn("failed to extract package.xml from archive: {}".format(e)) + warn("failed to extract package.xml from archive") package_xml = ros_pkg.get_package_xml(distro.name) metadata = PackageMetadata(package_xml) @@ -109,7 +108,7 @@ def _resolve_dependencies(self, deps: Iterable[str]) -> Iterable[str]: def _resolve_dependency(self, d: str) -> Iterable[str]: try: return (self.normalize_name(d),) \ - if d in self.distro.release_packages \ + if d in self._all_pkgs \ else resolve_dep(d, 'nix')[0] except UnresolvedDependency: self.unresolved_dependencies.add(d) diff --git a/superflore/rosdep_support.py b/superflore/rosdep_support.py index 5ab8f1e2..ff0bab00 100644 --- a/superflore/rosdep_support.py +++ b/superflore/rosdep_support.py @@ -53,6 +53,9 @@ def get_view(os_name, os_version, ros_distro): return view_cache[key] +_installer_ctx = create_default_installer_context() + + def resolve_more_for_os(rosdep_key, view, installer, os_name, os_version): """ Resolve rosdep key to dependencies and installer key. @@ -64,9 +67,8 @@ def resolve_more_for_os(rosdep_key, view, installer, os_name, os_version): :raises: :exc:`rosdep2.ResolutionError` """ d = view.lookup(rosdep_key) - ctx = create_default_installer_context() - os_installers = ctx.get_os_installer_keys(os_name) - default_os_installer = ctx.get_default_os_installer_key(os_name) + os_installers = _installer_ctx.get_os_installer_keys(os_name) + default_os_installer = _installer_ctx.get_default_os_installer_key(os_name) inst_key, rule = d.get_rule_for_platform(os_name, os_version, os_installers, default_os_installer) @@ -82,15 +84,14 @@ def resolve_rosdep_key( ignored=None ): ignored = ignored or [] - ctx = create_default_installer_context() try: - installer_key = ctx.get_default_os_installer_key(os_name) + installer_key = _installer_ctx.get_default_os_installer_key(os_name) except KeyError: raise UnresolvedDependency( "could not resolve package {} for os {}." .format(key, os_name) ) - installer = ctx.get_installer(installer_key) + installer = _installer_ctx.get_installer(installer_key) ros_distro = ros_distro or DEFAULT_ROS_DISTRO view = get_view(os_name, os_version, ros_distro) try: From 34ec0576af06b39bec057d4556fa118b058efa19 Mon Sep 17 00:00:00 2001 From: Ben Wolsieffer Date: Sun, 5 May 2019 01:59:41 -0400 Subject: [PATCH 04/25] nix: add tests and fix code style --- superflore/generators/nix/gen_packages.py | 2 +- superflore/generators/nix/nix_derivation.py | 12 +-- superflore/generators/nix/nix_package.py | 9 +- superflore/generators/nix/nix_ros_overlay.py | 3 +- superflore/generators/nix/run.py | 7 +- superflore/test_integration/nix/__init__.py | 4 + superflore/test_integration/nix/build_base.py | 53 ++++++++++++ superflore/test_integration/nix/main.py | 86 +++++++++++++++++++ tests/test_nix.py | 28 ++++++ 9 files changed, 190 insertions(+), 14 deletions(-) create mode 100644 superflore/test_integration/nix/__init__.py create mode 100644 superflore/test_integration/nix/build_base.py create mode 100644 superflore/test_integration/nix/main.py create mode 100644 tests/test_nix.py diff --git a/superflore/generators/nix/gen_packages.py b/superflore/generators/nix/gen_packages.py index 6ba10040..d696239a 100644 --- a/superflore/generators/nix/gen_packages.py +++ b/superflore/generators/nix/gen_packages.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import os -from typing import Iterable, Dict, Set +from typing import Iterable, Dict from rosdistro import DistributionFile from rosinstall_generator.distro import get_package_names diff --git a/superflore/generators/nix/nix_derivation.py b/superflore/generators/nix/nix_derivation.py index b0c13686..81979f62 100644 --- a/superflore/generators/nix/nix_derivation.py +++ b/superflore/generators/nix/nix_derivation.py @@ -22,7 +22,7 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # - +from operator import attrgetter from textwrap import dedent from time import gmtime, strftime from typing import Iterable @@ -64,6 +64,7 @@ def __init__(self, name): self.name = name self.custom = True + @property def nix_code(self) -> str: if self.custom: return '"{}"'.format(self.name) @@ -115,7 +116,7 @@ def get_text(self, distributor: str, license_name: str) -> str: ret += dedent(''' # Copyright {} {} # Distributed under the terms of the {} license - + ''').format( strftime("%Y", gmtime()), distributor, license_name) @@ -132,12 +133,12 @@ def get_text(self, distributor: str, license_name: str) -> str: buildRosPackage {{ pname = "ros-{}-{}"; version = "{}"; - + src = fetchurl {{ url = {}; sha256 = "{}"; }}; - + ''').format( self.distro_name, self.name, self.version, @@ -171,6 +172,7 @@ def get_text(self, distributor: str, license_name: str) -> str: }}; }} ''').format(self.description, - self._to_nix_list(map(NixLicense.nix_code, self.licenses))) + self._to_nix_list(map(attrgetter('nix_code'), + self.licenses))) return ''.join(ret) diff --git a/superflore/generators/nix/nix_package.py b/superflore/generators/nix/nix_package.py index 616820a5..b539051a 100644 --- a/superflore/generators/nix/nix_package.py +++ b/superflore/generators/nix/nix_package.py @@ -9,7 +9,7 @@ from rosdistro import DistributionFile from rosdistro.dependency_walker import DependencyWalker from rosdistro.rosdistro import RosPackage -from rosinstall_generator.distro import _generate_rosinstall, get_package_names +from rosinstall_generator.distro import _generate_rosinstall from superflore.PackageMetadata import PackageMetadata from superflore.exceptions import UnresolvedDependency @@ -47,7 +47,8 @@ def __init__(self, name: str, distro: DistributionFile, tar_dir: str, if os.path.exists(archive_path): info("using cached archive for package '{}'...".format(name)) else: - info("downloading archive version for package '{}'...".format(name)) + info("downloading archive version for package '{}'..." + .format(name)) urlretrieve(src_uri, archive_path) downloaded_archive = True @@ -73,11 +74,11 @@ def __init__(self, name: str, distro: DistributionFile, tar_dir: str, metadata = PackageMetadata(package_xml) dep_walker = DependencyWalker(distro, - get_distro_condition_context(distro.name)) + get_distro_condition_context( + distro.name)) buildtool_deps = dep_walker.get_depends(pkg.name, "buildtool") build_deps = dep_walker.get_depends(pkg.name, "build") - # TODO: do we need exec depends as well run_deps = dep_walker.get_depends(pkg.name, "run") test_deps = dep_walker.get_depends(pkg.name, "test") diff --git a/superflore/generators/nix/nix_ros_overlay.py b/superflore/generators/nix/nix_ros_overlay.py index ab2a5f99..94ed685a 100644 --- a/superflore/generators/nix/nix_ros_overlay.py +++ b/superflore/generators/nix/nix_ros_overlay.py @@ -20,7 +20,8 @@ class NixRosOverlay(object): - def __init__(self, repo_dir, do_clone, org='lopsided98', repo='nix-ros-overlay'): + def __init__(self, repo_dir, do_clone, org='lopsided98', + repo='nix-ros-overlay'): self.repo = RepoInstance(org, repo, repo_dir, do_clone) self.branch_name = 'nix-bot-%s' % rand_ascii_str() info('Creating new branch {0}...'.format(self.branch_name)) diff --git a/superflore/generators/nix/run.py b/superflore/generators/nix/run.py index 31150d88..c7274984 100644 --- a/superflore/generators/nix/run.py +++ b/superflore/generators/nix/run.py @@ -164,11 +164,12 @@ def main(): total_installers[distro] = distro_installers # If we are just updating a few packages using --only, then - # leave the package set alone. This means that new packages will - # not be added, but it is still useful for updates. + # leave the package set alone. This means that new packages + # will not be added, but it is still useful for updates. if not preserve_existing: regenerate_pkg_set(overlay, distro, distro_installers) - ok('Generated package set for distro \'{}\''.format(distro)) + ok('Generated package set for distro \'{}\'' + .format(distro)) num_changes = 0 for distro_name in total_changes: diff --git a/superflore/test_integration/nix/__init__.py b/superflore/test_integration/nix/__init__.py new file mode 100644 index 00000000..b95c42b8 --- /dev/null +++ b/superflore/test_integration/nix/__init__.py @@ -0,0 +1,4 @@ +from superflore.test_integration.nix.main import main + +if __name__ == '__main__': + main() diff --git a/superflore/test_integration/nix/build_base.py b/superflore/test_integration/nix/build_base.py new file mode 100644 index 00000000..e3e38cee --- /dev/null +++ b/superflore/test_integration/nix/build_base.py @@ -0,0 +1,53 @@ +# Copyright 2019 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from docker.errors import ContainerError + +from superflore.docker import Docker +from superflore.generators.nix.nix_package import NixPackage +from superflore.utils import err +from superflore.utils import info +from superflore.utils import ok + + +class NixBuilder: + def __init__(self, image_owner='nixos', image_name='nix'): + self.container = Docker() + self.container.pull(image_owner, image_name) + self.package_list = dict() + + def add_target(self, ros_distro, pkg): + pkg = NixPackage.normalize_name(pkg) + self.package_list[ + 'rosPackages.{}.{}'.format(ros_distro, pkg)] = 'unknown' + + def run(self, verbose=True, log_file=None): + info('testing Nix package integrity') + + nix_ros_overlay_url = 'https://github.com/lopsided98/nix-ros-overlay' \ + '/archive/master.tar.gz' + for pkg in sorted(self.package_list.keys()): + self.container.add_sh_command( + 'nix-build {} -A {}'.format(nix_ros_overlay_url, pkg)) + try: + self.container.run(rm=True, show_cmd=True, log_file=log_file) + self.package_list[pkg] = 'building' + ok(" '%s': building" % pkg) + except ContainerError: + self.package_list[pkg] = 'failing' + err(" '%s': failing" % pkg) + if verbose: + print(self.container.log) + self.container.clear_commands() + return self.package_list diff --git a/superflore/test_integration/nix/main.py b/superflore/test_integration/nix/main.py new file mode 100644 index 00000000..1901def0 --- /dev/null +++ b/superflore/test_integration/nix/main.py @@ -0,0 +1,86 @@ +# Copyright 2019 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import sys + +import yaml + +from superflore.test_integration.nix.build_base import NixBuilder +from superflore.utils import get_distros_by_status + + +def main(): + """ + Test Nix builds in a Docker container. Each package build is run in a new + container, so everything is built from scratch each time. + """ + + tester = NixBuilder() + parser = argparse.ArgumentParser( + 'Check if ROS packages are building with Nix' + ) + parser.add_argument( + '--ros-distro', + help='distro(s) to check', + type=str, + nargs="+", + default=get_distros_by_status('active') + ) + parser.add_argument( + '--pkgs', + help='packages to build', + type=str, + nargs='+' + ) + parser.add_argument( + '-f', + help='build packages specified by the input file', + type=str + ) + parser.add_argument( + '-v', + '--verbose', + help='show output from docker', + action="store_true" + ) + parser.add_argument( + '--log-file', + help='location to store the log file', + type=str + ) + args = parser.parse_args(sys.argv[1:]) + + if args.f: + # load the yaml file holding the test files + with open(args.f, 'r') as test_file: + test_dict = yaml.load(test_file) + for distro, pkg_list in test_dict.items(): + for pkg in pkg_list: + tester.add_target(distro, pkg) + elif args.pkgs: + # use passed-in arguments to test + for distro in args.ros_distro: + for pkg in args.pkgs: + tester.add_target(distro, pkg) + else: + parser.error('Invalid args! You must supply a package list.') + sys.exit(1) + results = tester.run(args.verbose, args.log_file) + failures = 0 + for test_case in results.keys(): + if results[test_case] == 'failing': + failures = failures + 1 + # set exit status to the number of failures + sys.exit(failures) diff --git a/tests/test_nix.py b/tests/test_nix.py new file mode 100644 index 00000000..f32105e1 --- /dev/null +++ b/tests/test_nix.py @@ -0,0 +1,28 @@ +# Copyright 2019 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from superflore.generators.nix.nix_derivation import NixLicense + + +class TestNixLicense(unittest.TestCase): + + def test_known_license(self): + l = NixLicense('GPL 3') + self.assertEqual(l.nix_code, 'gpl3') + + def test_unknown_license(self): + l = NixLicense("some license") + self.assertEqual(l.nix_code, '"some license"') From e45c9483e779298f4586d894d22e5640d815096c Mon Sep 17 00:00:00 2001 From: Ben Wolsieffer Date: Sun, 8 Dec 2019 16:29:51 -0500 Subject: [PATCH 05/25] nix: update to account for upstream changes --- superflore/generators/nix/nix_derivation.py | 3 +-- superflore/generators/nix/nix_package.py | 5 +++-- superflore/generators/nix/run.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/superflore/generators/nix/nix_derivation.py b/superflore/generators/nix/nix_derivation.py index 81979f62..e6d94701 100644 --- a/superflore/generators/nix/nix_derivation.py +++ b/superflore/generators/nix/nix_derivation.py @@ -27,7 +27,6 @@ from time import gmtime, strftime from typing import Iterable -from superflore.exceptions import UnknownLicense from superflore.utils import get_license @@ -60,7 +59,7 @@ def __init__(self, name): name = get_license(name) self.name = self._LICENSE_MAP[name] self.custom = False - except (KeyError, UnknownLicense): + except KeyError: self.name = name self.custom = True diff --git a/superflore/generators/nix/nix_package.py b/superflore/generators/nix/nix_package.py index b539051a..19d282b5 100644 --- a/superflore/generators/nix/nix_package.py +++ b/superflore/generators/nix/nix_package.py @@ -15,7 +15,7 @@ from superflore.exceptions import UnresolvedDependency from superflore.generators.nix.nix_derivation import NixDerivation, NixLicense from superflore.utils import info, get_pkg_version, warn, resolve_dep, \ - get_distro_condition_context + get_distro_condition_context, retry_on_exception class NixPackage: @@ -69,7 +69,8 @@ def __init__(self, name: str, distro: DistributionFile, tar_dir: str, # Fallback to the standard method of fetching package.xml if package_xml is None: warn("failed to extract package.xml from archive") - package_xml = ros_pkg.get_package_xml(distro.name) + package_xml = retry_on_exception(ros_pkg.get_package_xml, + distro.name) metadata = PackageMetadata(package_xml) diff --git a/superflore/generators/nix/run.py b/superflore/generators/nix/run.py index c7274984..f8b50d77 100644 --- a/superflore/generators/nix/run.py +++ b/superflore/generators/nix/run.py @@ -58,7 +58,7 @@ def main(): try: prev_overlay = RepoInstance(args.output_repository_path, False) msg, title = load_pr() - prev_overlay.pull_request(msg, title) + prev_overlay.pull_request(msg, title=title) clean_up() sys.exit(0) except Exception as e: From 65066d7257678db065fa8185b200ab76ecc87975 Mon Sep 17 00:00:00 2001 From: Ben Wolsieffer Date: Thu, 7 Nov 2019 12:14:19 -0500 Subject: [PATCH 06/25] nix: add ROS 2 support --- superflore/generators/nix/nix_derivation.py | 36 ++++++++++++++------- superflore/generators/nix/nix_package.py | 9 ++++-- 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/superflore/generators/nix/nix_derivation.py b/superflore/generators/nix/nix_derivation.py index e6d94701..02665585 100644 --- a/superflore/generators/nix/nix_derivation.py +++ b/superflore/generators/nix/nix_derivation.py @@ -22,6 +22,8 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # +import os +import urllib.parse from operator import attrgetter from textwrap import dedent from time import gmtime, strftime @@ -32,11 +34,12 @@ class NixLicense: """ - Generates + Converts a ROS license to the correct Nix license attribute. """ _LICENSE_MAP = { 'Apache-2.0': 'asl20', + 'ASL 2.0': 'asl20', 'BSD': 'bsdOriginal', 'BSD-2': 'bsd2', 'LGPL-2': 'lgpl2', @@ -72,9 +75,11 @@ def nix_code(self) -> str: class NixDerivation: - def __init__(self, name: str, version: str, src_uri: str, src_sha256: str, + def __init__(self, name: str, version: str, + src_url: str, src_sha256: str, description: str, licenses: Iterable[NixLicense], distro_name: str, + build_type: str, build_inputs: Iterable[str] = tuple(), propagated_build_inputs: Iterable[str] = tuple(), check_inputs: Iterable[str] = tuple(), @@ -83,12 +88,16 @@ def __init__(self, name: str, version: str, src_uri: str, src_sha256: str, ) -> None: self.name = name self.version = version - self.src_uri = src_uri + self.src_url = src_url self.src_sha256 = src_sha256 + # fetchurl's naming logic cannot account for URL parameters + self.src_name = os.path.basename( + urllib.parse.urlparse(self.src_url).path) self.description = description self.licenses = licenses self.distro_name = distro_name + self.build_type = build_type self.build_inputs = set(build_inputs) self.propagated_build_inputs = set(propagated_build_inputs) @@ -130,19 +139,24 @@ def get_text(self, distributor: str, license_name: str) -> str: ret += dedent(''' buildRosPackage {{ - pname = "ros-{}-{}"; - version = "{}"; + pname = "ros-{distro_name}-{name}"; + version = "{version}"; src = fetchurl {{ - url = {}; - sha256 = "{}"; + url = "{src_url}"; + name = "{src_name}"; + sha256 = "{src_sha256}"; }}; + buildType = "{build_type}"; ''').format( - self.distro_name, self.name, - self.version, - self.src_uri, - self.src_sha256) + distro_name=self.distro_name, + name=self.name, + version=self.version, + src_url=self.src_url, + src_name=self.src_name, + src_sha256=self.src_sha256, + build_type=self.build_type) if self.build_inputs: ret += " buildInputs = {};\n" \ diff --git a/superflore/generators/nix/nix_package.py b/superflore/generators/nix/nix_package.py index 19d282b5..c70b7752 100644 --- a/superflore/generators/nix/nix_package.py +++ b/superflore/generators/nix/nix_package.py @@ -79,25 +79,28 @@ def __init__(self, name: str, distro: DistributionFile, tar_dir: str, distro.name)) buildtool_deps = dep_walker.get_depends(pkg.name, "buildtool") + buildtool_export_deps = dep_walker.get_depends(pkg.name, "buildtool_export") build_deps = dep_walker.get_depends(pkg.name, "build") - run_deps = dep_walker.get_depends(pkg.name, "run") + build_export_deps = dep_walker.get_depends(pkg.name, "build_export") + exec_deps = dep_walker.get_depends(pkg.name, "exec") test_deps = dep_walker.get_depends(pkg.name, "test") self.unresolved_dependencies = set() build_inputs = self._resolve_dependencies(build_deps) - propagated_build_inputs = self._resolve_dependencies(run_deps) + propagated_build_inputs = self._resolve_dependencies(exec_deps | buildtool_export_deps | build_export_deps) check_inputs = self._resolve_dependencies(test_deps) native_build_inputs = self._resolve_dependencies(buildtool_deps) self._derivation = NixDerivation( name=normalized_name, version=version, - src_uri=src_uri, + src_url=src_uri, src_sha256=src_sha256, description=metadata.description, licenses=map(NixLicense, metadata.upstream_license), distro_name=distro.name, + build_type=metadata.build_type, build_inputs=build_inputs, propagated_build_inputs=propagated_build_inputs, check_inputs=check_inputs, From ce860ad745af2a0f6eaa40d9c58d7ca834611e91 Mon Sep 17 00:00:00 2001 From: Ben Wolsieffer Date: Thu, 7 Nov 2019 19:20:28 -0500 Subject: [PATCH 07/25] Add ROS_PYTHON_VERSION to distro condition context. --- superflore/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/superflore/utils.py b/superflore/utils.py index 1245c20c..71acf24a 100644 --- a/superflore/utils.py +++ b/superflore/utils.py @@ -731,7 +731,10 @@ def get_distro_condition_context(distro_name: str) -> Dict[str, str]: """ index = get_cached_index() distro = index.distributions[distro_name] - context = {'ROS_DISTRO': distro_name} + context = { + 'ROS_DISTRO': distro_name, + 'ROS_PYTHON_VERSION': distro['python_version'] + } if distro['distribution_type'] == 'ros1': context['ROS_VERSION'] = '1' elif distro['distribution_type'] == 'ros2': From b9f020f44771eae35330a8ac0a122873d070f8c9 Mon Sep 17 00:00:00 2001 From: Ben Wolsieffer Date: Sun, 8 Dec 2019 17:21:54 -0500 Subject: [PATCH 08/25] nix: update based on upstream changes --- superflore/generators/nix/gen_packages.py | 8 ++-- superflore/generators/nix/nix_ros_overlay.py | 41 ++++++++++++++------ superflore/generators/nix/run.py | 8 +++- 3 files changed, 41 insertions(+), 16 deletions(-) diff --git a/superflore/generators/nix/gen_packages.py b/superflore/generators/nix/gen_packages.py index d696239a..328c6eb4 100644 --- a/superflore/generators/nix/gen_packages.py +++ b/superflore/generators/nix/gen_packages.py @@ -49,7 +49,7 @@ def regenerate_pkg(overlay, pkg: str, distro: DistributionFile, if preserve_existing and existing: ok("derivation for package '{}' up to date, skipping...".format(pkg)) - return None, [] + return None, [], None try: current = NixPackage(pkg, distro, tar_dir, sha256_cache, all_pkgs) @@ -65,10 +65,10 @@ def regenerate_pkg(overlay, pkg: str, distro: DistributionFile, unresolved = current.unresolved_dependencies for dep in unresolved: err(" unresolved: \"{}\"".format(dep)) - return None, unresolved + return None, unresolved, None except NoPkgXml: err("Could not fetch pkg!") - return None, [] + return None, [], None except Exception as e: err('Failed to generate derivation for package {}!'.format(pkg)) raise e @@ -80,7 +80,7 @@ def regenerate_pkg(overlay, pkg: str, distro: DistributionFile, except Exception as e: err("Failed to write derivation to disk!") raise e - return current, [] + return current, [], normalized_pkg def regenerate_pkg_set(overlay, distro_name: str, pkg_names: Iterable[str]): diff --git a/superflore/generators/nix/nix_ros_overlay.py b/superflore/generators/nix/nix_ros_overlay.py index 94ed685a..440ffcc7 100644 --- a/superflore/generators/nix/nix_ros_overlay.py +++ b/superflore/generators/nix/nix_ros_overlay.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import time from superflore.repo_instance import RepoInstance @@ -21,24 +22,42 @@ class NixRosOverlay(object): def __init__(self, repo_dir, do_clone, org='lopsided98', - repo='nix-ros-overlay'): - self.repo = RepoInstance(org, repo, repo_dir, do_clone) - self.branch_name = 'nix-bot-%s' % rand_ascii_str() - info('Creating new branch {0}...'.format(self.branch_name)) - self.repo.create_branch(self.branch_name) + repo='nix-ros-overlay', from_branch='', new_branch=True): + self.repo = RepoInstance(org, repo, repo_dir, do_clone, + from_branch=from_branch) + if new_branch: + self.branch_name = 'nix-bot-%s' % rand_ascii_str() + info('Creating new branch {0}...'.format(self.branch_name)) + self.repo.create_branch(self.branch_name) + else: + self.branch_name = None def commit_changes(self, distro): info('Adding changes...') if distro == 'all': commit_msg = 'regenerate all distros, {0}' self.repo.git.add('*/*/default.nix') + self.repo.git.add('*/generated.nix') else: commit_msg = 'regenerate ros-{1}, {0}' self.repo.git.add(distro) - commit_msg = commit_msg.format(time.ctime(), distro) - info('Committing to branch {0}...'.format(self.branch_name)) - self.repo.git.commit(m=commit_msg) + if self.repo.git.status('--porcelain') == '': + info('Nothing changed; no commit done') + else: + timestamp = os.getenv( + 'SUPERFLORE_GENERATION_DATETIME', + time.ctime()) + commit_msg = commit_msg.format(timestamp, distro) + if self.branch_name: + info('Committing to branch {0}...'.format(self.branch_name)) + else: + info('Committing to current branch') + self.repo.git.commit(m=commit_msg) - def pull_request(self, message, distro=None): - pr_title = 'rosdistro sync, {0}'.format(time.ctime()) - self.repo.pull_request(message, pr_title, branch=distro) + def pull_request(self, message, distro=None, title=''): + if not title: + timestamp = os.getenv( + 'SUPERFLORE_GENERATION_DATETIME', + time.ctime()) + title = 'rosdistro sync, {0}'.format(timestamp) + self.repo.pull_request(message, title, branch=self.branch_name) diff --git a/superflore/generators/nix/run.py b/superflore/generators/nix/run.py index f8b50d77..d2d83e10 100644 --- a/superflore/generators/nix/run.py +++ b/superflore/generators/nix/run.py @@ -18,6 +18,7 @@ from rosinstall_generator.distro import get_distro from superflore.CacheManager import CacheManager +from superflore.exceptions import NoGitHubAuthToken from superflore.TempfileManager import TempfileManager from superflore.generate_installers import generate_installers from superflore.generators.nix.gen_packages import regenerate_pkg, \ @@ -50,6 +51,9 @@ def main(): pr_comment = args.pr_comment skip_keys = args.skip_keys or [] selected_targets = None + if not args.dry_run: + if 'SUPERFLORE_GITHUB_TOKEN' not in os.environ: + raise NoGitHubAuthToken() if args.pr_only: if args.dry_run: parser.error('Invalid args! cannot dry-run and file PR') @@ -89,7 +93,9 @@ def main(): _repo, not args.output_repository_path, org=repo_org, - repo=repo_name + repo=repo_name, + from_branch=args.upstream_branch, + new_branch=(not args.no_branch) ) if not preserve_existing and not args.only: pr_comment = pr_comment or ( From cdfe6f6f83cc75bdc6481dfaad290b0a28745747 Mon Sep 17 00:00:00 2001 From: Ben Wolsieffer Date: Sun, 8 Dec 2019 23:25:08 -0500 Subject: [PATCH 09/25] nix: fix submitting pull requests. --- superflore/generators/bitbake/run.py | 8 ++++--- superflore/generators/nix/gen_packages.py | 18 ++++++++++++++- superflore/generators/nix/nix_ros_overlay.py | 5 +++-- superflore/repo_instance.py | 23 ++++++++++---------- superflore/utils.py | 18 ++++++++------- tests/test_utils.py | 2 +- 6 files changed, 48 insertions(+), 26 deletions(-) diff --git a/superflore/generators/bitbake/run.py b/superflore/generators/bitbake/run.py index 88c9c4cd..e1ab7129 100644 --- a/superflore/generators/bitbake/run.py +++ b/superflore/generators/bitbake/run.py @@ -150,8 +150,9 @@ def main(): delta = "Regenerated: '%s'\n" % args.only overlay.add_generated_files(args.ros_distro) commit_msg = '\n'.join([get_pr_text( - title + '\n' + pr_comment.replace( - '**superflore**', 'superflore'), markup=''), delta]) + comment=title + '\n' + pr_comment.replace( + '**superflore**', 'superflore'), + markup=''), delta]) overlay.commit_changes(args.ros_distro, commit_msg) if args.dry_run: save_pr(overlay, args.only, '', pr_comment, title=title) @@ -216,7 +217,8 @@ def main(): args.ros_distro, now) commit_msg = '\n'.join([get_pr_text( - title + '\n' + pr_comment.replace('**superflore**', 'superflore'), + comment=title + '\n' + + pr_comment.replace('**superflore**', 'superflore'), markup=''), delta]) overlay.commit_changes(args.ros_distro, commit_msg) delta = gen_delta_msg(total_changes) diff --git a/superflore/generators/nix/gen_packages.py b/superflore/generators/nix/gen_packages.py index 328c6eb4..283f341e 100644 --- a/superflore/generators/nix/gen_packages.py +++ b/superflore/generators/nix/gen_packages.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import os +import re from typing import Iterable, Dict from rosdistro import DistributionFile @@ -24,10 +25,12 @@ from superflore.utils import err from superflore.utils import make_dir from superflore.utils import ok +from superflore.utils import warn org = "Open Source Robotics Foundation" org_license = "BSD" +_version_regex = re.compile(r"version\s*=\s*\"([^\"]*)\"") def regenerate_pkg(overlay, pkg: str, distro: DistributionFile, preserve_existing: bool, tar_dir: str, @@ -46,11 +49,24 @@ def regenerate_pkg(overlay, pkg: str, distro: DistributionFile, # check for an existing package existing = os.path.exists(package_file) + previous_version = None if preserve_existing and existing: ok("derivation for package '{}' up to date, skipping...".format(pkg)) return None, [], None + if existing: + with open(package_file, 'r') as f: + existing_derivation = f.read() + version_match = _version_regex.search(existing_derivation) + if version_match: + try: + previous_version = version_match.group(1) + except IndexError: + pass + if not previous_version: + warn("Failed to extract previous package version") + try: current = NixPackage(pkg, distro, tar_dir, sha256_cache, all_pkgs) except Exception as e: @@ -80,7 +96,7 @@ def regenerate_pkg(overlay, pkg: str, distro: DistributionFile, except Exception as e: err("Failed to write derivation to disk!") raise e - return current, [], normalized_pkg + return current, previous_version, normalized_pkg def regenerate_pkg_set(overlay, distro_name: str, pkg_names: Iterable[str]): diff --git a/superflore/generators/nix/nix_ros_overlay.py b/superflore/generators/nix/nix_ros_overlay.py index 440ffcc7..43662079 100644 --- a/superflore/generators/nix/nix_ros_overlay.py +++ b/superflore/generators/nix/nix_ros_overlay.py @@ -39,7 +39,7 @@ def commit_changes(self, distro): self.repo.git.add('*/*/default.nix') self.repo.git.add('*/generated.nix') else: - commit_msg = 'regenerate ros-{1}, {0}' + commit_msg = 'regenerate rosPackages.{1}, {0}' self.repo.git.add(distro) if self.repo.git.status('--porcelain') == '': info('Nothing changed; no commit done') @@ -60,4 +60,5 @@ def pull_request(self, message, distro=None, title=''): 'SUPERFLORE_GENERATION_DATETIME', time.ctime()) title = 'rosdistro sync, {0}'.format(timestamp) - self.repo.pull_request(message, title, branch=self.branch_name) + self.repo.pull_request(message, title, branch=self.branch_name, + fork=False) diff --git a/superflore/repo_instance.py b/superflore/repo_instance.py index 1f732410..74e91d34 100644 --- a/superflore/repo_instance.py +++ b/superflore/repo_instance.py @@ -92,30 +92,31 @@ def rebase(self, target): """ self.git.rebase(i=target) - def pull_request(self, message, title, branch='master', remote='origin'): - info('Forking repository if a fork does not exist...') + def pull_request(self, message, title, branch='master', fork=True): self.github = Github(os.environ['SUPERFLORE_GITHUB_TOKEN']) self.gh_user = self.github.get_user() self.gh_upstream = self.github.get_repo( - '%s/%s' % ( - self.repo_owner, self.repo_name - ) + '{}/{}'.format(self.repo_owner, self.repo_name) ) - # TODO(allenh1): Don't fork if you're authorized for repo - forked_repo = self.gh_user.create_fork(self.gh_upstream) - info('Pushing changes to fork...') - self.git.remote('add', 'github', forked_repo.html_url) + if fork: + info('Forking repository if a fork does not exist...') + pr_repo = self.gh_user.create_fork(self.gh_upstream) + pr_head = '{}:{}'.format(self.gh_user.login, self.branch or branch) + else: + pr_repo = self.gh_upstream + pr_head = self.branch or branch + info('Pushing changes to repo...') + self.git.remote('add', 'github', pr_repo.html_url) retry_on_exception( self.git.push, '-u', 'github', self.branch or branch, retry_msg='Could not push', error_msg='Error during push', sleep_secs=0.0, ) info('Filing pull-request...') - pr_head = '%s:%s' % (self.gh_user.login, self.branch) pr = self.gh_upstream.create_pull( title=title, body=message, - base=self.from_branch or branch, + base=self.from_branch, head=pr_head ) ok('Successfully filed a pull request.') diff --git a/superflore/utils.py b/superflore/utils.py index 71acf24a..0e49401a 100644 --- a/superflore/utils.py +++ b/superflore/utils.py @@ -44,24 +44,27 @@ def info(string): # pragma: no cover print(colored('>>>> {0}'.format(string), 'cyan')) -def get_pr_text(comment=None, markup='```'): +def get_pr_text(delta=None, missing_deps=None, comment=None, markup='```'): msg = '' if comment: - msg += '%s\n' % comment + msg += comment + '\n' msg += 'This pull request was generated by running the following command:' msg += '\n\n' args = sys.argv args[0] = args[0].split('/')[-1] msg += '{1}\n{0}\n{1}\n'.format(' '.join(args), markup) + if delta: + msg += '\n' + delta + '\n' + if missing_deps: + msg += missing_deps + '\n' return msg -def save_pr(overlay, delta, missing_deps, comment, - title='rosdistro sync, {0}\n'.format(time.ctime())): +def save_pr(overlay, delta, missing_deps, comment, title=''): with open('.pr-title.tmp', 'w') as title_file: title_file.write(title) with open('.pr-message.tmp', 'w') as pr_msg_file: - pr_msg_file.write('%s\n' % get_pr_text(comment)) + pr_msg_file.write(get_pr_text(delta, missing_deps, comment)) def load_pr(): @@ -84,9 +87,8 @@ def load_pr(): def file_pr(overlay, delta, missing_deps, comment, distro=None, title=''): try: - msg = get_pr_text(comment) - overlay.pull_request('{}\n{}\n{}'.format(msg, delta, missing_deps), - distro, title=title) + msg = get_pr_text(delta, missing_deps, comment) + overlay.pull_request(msg, distro, title=title) except Exception as e: err( 'Failed to file PR with the %s/%s repo!' % ( diff --git a/tests/test_utils.py b/tests/test_utils.py index 74a3c7ab..f5f3eb1a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -183,7 +183,7 @@ def test_get_pr_text(self): expected = 'sample\n'\ 'This pull request was generated by running the following command:\n\n'\ '```\na b c d\n```\n' - self.assertEqual(expected, get_pr_text('sample')) + self.assertEqual(expected, get_pr_text(comment='sample')) def test_cleanup(self): """Test PR dry run cleanup""" From 2bad38b16e981b028df378254c60db0b9d35b5f4 Mon Sep 17 00:00:00 2001 From: Ben Wolsieffer Date: Tue, 10 Dec 2019 14:58:13 -0500 Subject: [PATCH 10/25] nix: make dependency ordering deterministic --- superflore/generators/nix/nix_derivation.py | 38 ++++++++++----------- superflore/generators/nix/nix_package.py | 18 +++++++--- 2 files changed, 32 insertions(+), 24 deletions(-) diff --git a/superflore/generators/nix/nix_derivation.py b/superflore/generators/nix/nix_derivation.py index 02665585..cdf2e33a 100644 --- a/superflore/generators/nix/nix_derivation.py +++ b/superflore/generators/nix/nix_derivation.py @@ -27,7 +27,7 @@ from operator import attrgetter from textwrap import dedent from time import gmtime, strftime -from typing import Iterable +from typing import Iterable, Set from superflore.utils import get_license @@ -80,11 +80,11 @@ def __init__(self, name: str, version: str, description: str, licenses: Iterable[NixLicense], distro_name: str, build_type: str, - build_inputs: Iterable[str] = tuple(), - propagated_build_inputs: Iterable[str] = tuple(), - check_inputs: Iterable[str] = tuple(), - native_build_inputs: Iterable[str] = tuple(), - propagated_native_build_inputs: Iterable[str] = tuple() + build_inputs: Set[str] = set(), + propagated_build_inputs: Set[str] = set(), + check_inputs: Set[str] = set(), + native_build_inputs: Set[str] = set(), + propagated_native_build_inputs: Set[str] = set() ) -> None: self.name = name self.version = version @@ -99,12 +99,12 @@ def __init__(self, name: str, version: str, self.distro_name = distro_name self.build_type = build_type - self.build_inputs = set(build_inputs) - self.propagated_build_inputs = set(propagated_build_inputs) - self.check_inputs = set(check_inputs) - self.native_build_inputs = set(native_build_inputs) + self.build_inputs = build_inputs + self.propagated_build_inputs = propagated_build_inputs + self.check_inputs = check_inputs + self.native_build_inputs = native_build_inputs self.propagated_native_build_inputs = \ - set(propagated_native_build_inputs) + propagated_native_build_inputs @staticmethod def _to_nix_list(it: Iterable[str]) -> str: @@ -130,12 +130,12 @@ def get_text(self, distributor: str, license_name: str) -> str: license_name) ret += '{ lib, buildRosPackage, fetchurl, ' + \ - ', '.join(set(map(self._to_nix_parameter, + ', '.join(sorted(set(map(self._to_nix_parameter, self.build_inputs | - self.check_inputs | self.propagated_build_inputs | + self.check_inputs | self.native_build_inputs | - self.propagated_native_build_inputs))) + ' }:' + self.propagated_native_build_inputs)))) + ' }:' ret += dedent(''' buildRosPackage {{ @@ -160,23 +160,23 @@ def get_text(self, distributor: str, license_name: str) -> str: if self.build_inputs: ret += " buildInputs = {};\n" \ - .format(self._to_nix_list(self.build_inputs)) + .format(self._to_nix_list(sorted(self.build_inputs))) if self.check_inputs: ret += " checkInputs = {};\n" \ - .format(self._to_nix_list(self.check_inputs)) + .format(self._to_nix_list(sorted(self.check_inputs))) if self.propagated_build_inputs: ret += " propagatedBuildInputs = {};\n" \ - .format(self._to_nix_list(self.propagated_build_inputs)) + .format(self._to_nix_list(sorted(self.propagated_build_inputs))) if self.native_build_inputs: ret += " nativeBuildInputs = {};\n" \ - .format(self._to_nix_list(self.native_build_inputs)) + .format(self._to_nix_list(sorted(self.native_build_inputs))) if self.propagated_native_build_inputs: ret += " propagatedNativeBuildInputs = {};\n".format( - self._to_nix_list(self.propagated_native_build_inputs)) + self._to_nix_list(sorted(self.propagated_native_build_inputs))) ret += dedent(''' meta = {{ diff --git a/superflore/generators/nix/nix_package.py b/superflore/generators/nix/nix_package.py index c70b7752..074b8d67 100644 --- a/superflore/generators/nix/nix_package.py +++ b/superflore/generators/nix/nix_package.py @@ -87,9 +87,17 @@ def __init__(self, name: str, distro: DistributionFile, tar_dir: str, self.unresolved_dependencies = set() - build_inputs = self._resolve_dependencies(build_deps) - propagated_build_inputs = self._resolve_dependencies(exec_deps | buildtool_export_deps | build_export_deps) + build_inputs = set(self._resolve_dependencies(build_deps)) + # buildtool_export_depends should probably be + # propagatedNativeBuildInputs, but that causes many build failures. + # Either ROS packages don't use it correctly or it doesn't map well to + # Nix. + propagated_build_inputs = self._resolve_dependencies(exec_deps | build_export_deps | buildtool_export_deps) + build_inputs -= propagated_build_inputs + check_inputs = self._resolve_dependencies(test_deps) + check_inputs -= build_inputs + native_build_inputs = self._resolve_dependencies(buildtool_deps) self._derivation = NixDerivation( @@ -106,9 +114,9 @@ def __init__(self, name: str, distro: DistributionFile, tar_dir: str, check_inputs=check_inputs, native_build_inputs=native_build_inputs) - def _resolve_dependencies(self, deps: Iterable[str]) -> Iterable[str]: - return itertools.chain.from_iterable( - map(self._resolve_dependency, deps)) + def _resolve_dependencies(self, deps: Iterable[str]) -> Set[str]: + return set(itertools.chain.from_iterable( + map(self._resolve_dependency, deps))) def _resolve_dependency(self, d: str) -> Iterable[str]: try: From 3d8c1c05e0932c79ecb6338be59f989cdd5b64ff Mon Sep 17 00:00:00 2001 From: Ben Wolsieffer Date: Fri, 17 Jan 2020 21:29:15 -0500 Subject: [PATCH 11/25] nix: put generated files in a subdirectory --- superflore/generators/nix/gen_packages.py | 4 ++-- superflore/generators/nix/nix_ros_overlay.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/superflore/generators/nix/gen_packages.py b/superflore/generators/nix/gen_packages.py index 283f341e..0d420be2 100644 --- a/superflore/generators/nix/gen_packages.py +++ b/superflore/generators/nix/gen_packages.py @@ -42,7 +42,7 @@ def regenerate_pkg(overlay, pkg: str, distro: DistributionFile, normalized_pkg = NixPackage.normalize_name(pkg) - package_dir = os.path.join(overlay.repo.repo_dir, distro.name, + package_dir = os.path.join(overlay.repo.repo_dir, 'distros', distro.name, normalized_pkg) package_file = os.path.join(package_dir, 'default.nix') make_dir(package_dir) @@ -100,7 +100,7 @@ def regenerate_pkg(overlay, pkg: str, distro: DistributionFile, def regenerate_pkg_set(overlay, distro_name: str, pkg_names: Iterable[str]): - distro_dir = os.path.join(overlay.repo.repo_dir, distro_name) + distro_dir = os.path.join(overlay.repo.repo_dir, 'distros', distro_name) overlay_file = os.path.join(distro_dir, 'generated.nix') make_dir(distro_dir) diff --git a/superflore/generators/nix/nix_ros_overlay.py b/superflore/generators/nix/nix_ros_overlay.py index 43662079..c0ad8564 100644 --- a/superflore/generators/nix/nix_ros_overlay.py +++ b/superflore/generators/nix/nix_ros_overlay.py @@ -36,11 +36,11 @@ def commit_changes(self, distro): info('Adding changes...') if distro == 'all': commit_msg = 'regenerate all distros, {0}' - self.repo.git.add('*/*/default.nix') - self.repo.git.add('*/generated.nix') + self.repo.git.add('distros/*/*/default.nix') + self.repo.git.add('distros/*/generated.nix') else: commit_msg = 'regenerate rosPackages.{1}, {0}' - self.repo.git.add(distro) + self.repo.git.add('distros/' + distro) if self.repo.git.status('--porcelain') == '': info('Nothing changed; no commit done') else: From f1a4bd0714a2de25571ae25953a8c790c1ade9f0 Mon Sep 17 00:00:00 2001 From: Ben Wolsieffer Date: Fri, 1 May 2020 20:23:10 -0400 Subject: [PATCH 12/25] nix: fix downloading archives from GitLab GitLab returns a 403 error if the default urllib user agent is used. --- superflore/generators/nix/gen_packages.py | 3 --- superflore/generators/nix/nix_package.py | 12 +++++++----- superflore/utils.py | 15 +++++++++++++++ 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/superflore/generators/nix/gen_packages.py b/superflore/generators/nix/gen_packages.py index 0d420be2..62de8999 100644 --- a/superflore/generators/nix/gen_packages.py +++ b/superflore/generators/nix/gen_packages.py @@ -82,9 +82,6 @@ def regenerate_pkg(overlay, pkg: str, distro: DistributionFile, for dep in unresolved: err(" unresolved: \"{}\"".format(dep)) return None, unresolved, None - except NoPkgXml: - err("Could not fetch pkg!") - return None, [], None except Exception as e: err('Failed to generate derivation for package {}!'.format(pkg)) raise e diff --git a/superflore/generators/nix/nix_package.py b/superflore/generators/nix/nix_package.py index 074b8d67..4e74b12c 100644 --- a/superflore/generators/nix/nix_package.py +++ b/superflore/generators/nix/nix_package.py @@ -4,18 +4,18 @@ import re import tarfile from typing import Dict, Iterable, Set -from urllib.request import urlretrieve from rosdistro import DistributionFile from rosdistro.dependency_walker import DependencyWalker from rosdistro.rosdistro import RosPackage from rosinstall_generator.distro import _generate_rosinstall -from superflore.PackageMetadata import PackageMetadata from superflore.exceptions import UnresolvedDependency from superflore.generators.nix.nix_derivation import NixDerivation, NixLicense -from superflore.utils import info, get_pkg_version, warn, resolve_dep, \ - get_distro_condition_context, retry_on_exception +from superflore.PackageMetadata import PackageMetadata +from superflore.utils import (download_file, get_distro_condition_context, + get_pkg_version, info, resolve_dep, + retry_on_exception, warn) class NixPackage: @@ -49,7 +49,9 @@ def __init__(self, name: str, distro: DistributionFile, tar_dir: str, else: info("downloading archive version for package '{}'..." .format(name)) - urlretrieve(src_uri, archive_path) + retry_on_exception(download_file, src_uri, archive_path, + retry_msg="network error downloading '{}'".format(src_uri), + error_msg="failed to download archive for '{}'".format(name)) downloaded_archive = True if downloaded_archive or archive_path not in sha256_cache: diff --git a/superflore/utils.py b/superflore/utils.py index 0e49401a..0f3760bd 100644 --- a/superflore/utils.py +++ b/superflore/utils.py @@ -17,10 +17,12 @@ import os import random import re +import shutil import string import sys import time from typing import Dict +import urllib.request from pkg_resources import DistributionNotFound, get_distribution from superflore.exceptions import UnknownPlatform @@ -789,6 +791,19 @@ def url_to_repo_org(url): return url[0], url[1] +def download_file(url, filename): + # GitLab returns 403 when using the default urllib User-Agent + request = urllib.request.Request(url, headers={ + 'User-Agent': 'superflore/{}'.format(get_superflore_version()) + }) + try: + with urllib.request.urlopen(request) as response, open(filename, 'wb') as file: + shutil.copyfileobj(response, file) + except Exception as e: + os.remove(filename) + raise e + + def retry_on_exception(callback, *args, max_retries=5, num_retry=0, retry_msg='', error_msg='', sleep_secs=0.125): try: From 6c49e84bae8ba5939b3ba0c4df8f0c571b4019fe Mon Sep 17 00:00:00 2001 From: Ben Wolsieffer Date: Mon, 24 Aug 2020 18:18:09 -0400 Subject: [PATCH 13/25] nix: fix integration test ArgumentParser description --- superflore/test_integration/nix/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superflore/test_integration/nix/main.py b/superflore/test_integration/nix/main.py index 1901def0..96090f6a 100644 --- a/superflore/test_integration/nix/main.py +++ b/superflore/test_integration/nix/main.py @@ -29,7 +29,7 @@ def main(): tester = NixBuilder() parser = argparse.ArgumentParser( - 'Check if ROS packages are building with Nix' + description='Check if ROS packages are building with Nix' ) parser.add_argument( '--ros-distro', From 0f1e781ca821075f96c2cc4c14771b962219c00c Mon Sep 17 00:00:00 2001 From: Ben Wolsieffer Date: Mon, 24 Aug 2020 18:57:29 -0400 Subject: [PATCH 14/25] utils: ignore error removing downloaded file on error --- superflore/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/superflore/utils.py b/superflore/utils.py index 0f3760bd..a295e2ca 100644 --- a/superflore/utils.py +++ b/superflore/utils.py @@ -800,7 +800,10 @@ def download_file(url, filename): with urllib.request.urlopen(request) as response, open(filename, 'wb') as file: shutil.copyfileobj(response, file) except Exception as e: - os.remove(filename) + try: + os.remove(filename) + except Exception as re: + warn('failed to remove: {}: {}'.format(filename, re)) raise e From d2e11090f269a8ea206b74bc64482e7da9d52997 Mon Sep 17 00:00:00 2001 From: Ben Wolsieffer Date: Mon, 28 Mar 2022 21:15:23 -0400 Subject: [PATCH 15/25] nix: sort license dictionary --- superflore/generators/nix/nix_derivation.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/superflore/generators/nix/nix_derivation.py b/superflore/generators/nix/nix_derivation.py index cdf2e33a..7905e73e 100644 --- a/superflore/generators/nix/nix_derivation.py +++ b/superflore/generators/nix/nix_derivation.py @@ -40,21 +40,21 @@ class NixLicense: _LICENSE_MAP = { 'Apache-2.0': 'asl20', 'ASL 2.0': 'asl20', - 'BSD': 'bsdOriginal', + 'Boost-1.0': 'boost', 'BSD-2': 'bsd2', - 'LGPL-2': 'lgpl2', - 'LGPL-2.1': 'lgpl21', - 'LGPL-3': 'lgpl3', + 'BSD': 'bsdOriginal', + 'CC-BY-NC-SA-4.0': 'cc-by-nc-sa-40', 'GPL-1': 'gpl1', 'GPL-2': 'gpl2', 'GPL-3': 'gpl3', + 'LGPL-2.1': 'lgpl21', + 'LGPL-2': 'lgpl2', + 'LGPL-3': 'lgpl3', + 'MIT': 'mit', 'MPL-1.0': 'mpl10', 'MPL-1.1': 'mpl11', 'MPL-2.0': 'mpl20', - 'MIT': 'mit', - 'CC-BY-NC-SA-4.0': 'cc-by-nc-sa-40', - 'Boost-1.0': 'boost', - 'public_domain': 'publicDomain' + 'public_domain': 'publicDomain', } def __init__(self, name): From 34f7f3945a1fcccf46231c21135d4544da696e84 Mon Sep 17 00:00:00 2001 From: Ben Wolsieffer Date: Mon, 28 Mar 2022 21:20:00 -0400 Subject: [PATCH 16/25] nix: add more license mappings --- superflore/generators/nix/nix_derivation.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/superflore/generators/nix/nix_derivation.py b/superflore/generators/nix/nix_derivation.py index 7905e73e..65d49d93 100644 --- a/superflore/generators/nix/nix_derivation.py +++ b/superflore/generators/nix/nix_derivation.py @@ -42,13 +42,16 @@ class NixLicense: 'ASL 2.0': 'asl20', 'Boost-1.0': 'boost', 'BSD-2': 'bsd2', + 'BSD-3-Clause': 'bsd3', 'BSD': 'bsdOriginal', 'CC-BY-NC-SA-4.0': 'cc-by-nc-sa-40', 'GPL-1': 'gpl1', 'GPL-2': 'gpl2', + 'GPL-3.0-only': 'gpl3Only', 'GPL-3': 'gpl3', 'LGPL-2.1': 'lgpl21', 'LGPL-2': 'lgpl2', + 'LGPL-3.0-only': 'lgpl3Only', 'LGPL-3': 'lgpl3', 'MIT': 'mit', 'MPL-1.0': 'mpl10', From 3fd5aa968cfc4823851a9caa0310d5c12219675a Mon Sep 17 00:00:00 2001 From: Ben Wolsieffer Date: Mon, 28 Mar 2022 21:27:35 -0400 Subject: [PATCH 17/25] nix: provide condition context to package metadata parser This allows conditions in package.xml to be evaluated correctly. Mostly copied from the Bitbake generator. --- superflore/generators/nix/nix_package.py | 25 ++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/superflore/generators/nix/nix_package.py b/superflore/generators/nix/nix_package.py index 4e74b12c..7cb21367 100644 --- a/superflore/generators/nix/nix_package.py +++ b/superflore/generators/nix/nix_package.py @@ -14,7 +14,7 @@ from superflore.generators.nix.nix_derivation import NixDerivation, NixLicense from superflore.PackageMetadata import PackageMetadata from superflore.utils import (download_file, get_distro_condition_context, - get_pkg_version, info, resolve_dep, + get_distros, get_pkg_version, info, resolve_dep, retry_on_exception, warn) @@ -74,7 +74,8 @@ def __init__(self, name: str, distro: DistributionFile, tar_dir: str, package_xml = retry_on_exception(ros_pkg.get_package_xml, distro.name) - metadata = PackageMetadata(package_xml) + metadata = PackageMetadata( + package_xml, NixPackage._get_condition_context(distro.name)) dep_walker = DependencyWalker(distro, get_distro_condition_context( @@ -129,6 +130,26 @@ def _resolve_dependency(self, d: str) -> Iterable[str]: self.unresolved_dependencies.add(d) return tuple() + @staticmethod + def _get_ros_version(distro): + distros = get_distros() + return 2 if distro not in distros \ + else int(distros[distro]['distribution_type'][len('ros'):]) + + @staticmethod + def _get_ros_python_version(distro): + return 2 if distro in ['melodic'] else 3 + + @staticmethod + def _get_condition_context(distro): + context = dict() + context["ROS_OS_OVERRIDE"] = "nixos" + context["ROS_DISTRO"] = distro + context["ROS_VERSION"] = str(NixPackage._get_ros_version(distro)) + context["ROS_PYTHON_VERSION"] = str( + NixPackage._get_ros_python_version(distro)) + return context + @staticmethod def normalize_name(name: str) -> str: """ From f0144375780c9e52f27bbf659ca40d7068e8ff16 Mon Sep 17 00:00:00 2001 From: Ben Wolsieffer Date: Tue, 20 Sep 2022 17:09:38 -0400 Subject: [PATCH 18/25] nix: add buildtool_depends to both buildInputs and nativeBuildInputs This is required to make strictDeps/cross-compilation work. --- superflore/generators/nix/nix_package.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/superflore/generators/nix/nix_package.py b/superflore/generators/nix/nix_package.py index 7cb21367..ddd81ac1 100644 --- a/superflore/generators/nix/nix_package.py +++ b/superflore/generators/nix/nix_package.py @@ -90,18 +90,20 @@ def __init__(self, name: str, distro: DistributionFile, tar_dir: str, self.unresolved_dependencies = set() - build_inputs = set(self._resolve_dependencies(build_deps)) - # buildtool_export_depends should probably be - # propagatedNativeBuildInputs, but that causes many build failures. - # Either ROS packages don't use it correctly or it doesn't map well to - # Nix. + # buildtool_depends are added to buildInputs and nativeBuildInputs. Some + # (such as CMake) have binaries that need to run at build time (and + # therefore need to be in nativeBuildInputs. Others (such as + # ament_cmake_*) need to be added to CMAKE_PREFIX_PATH and therefore + # need to be in buildInputs. There is no easy way to distinguish these + # two cases, so they are added to both, which generally works fine. + build_inputs = set(self._resolve_dependencies(build_deps | buildtool_deps)) propagated_build_inputs = self._resolve_dependencies(exec_deps | build_export_deps | buildtool_export_deps) build_inputs -= propagated_build_inputs check_inputs = self._resolve_dependencies(test_deps) check_inputs -= build_inputs - native_build_inputs = self._resolve_dependencies(buildtool_deps) + native_build_inputs = self._resolve_dependencies(buildtool_deps | buildtool_export_deps) self._derivation = NixDerivation( name=normalized_name, From a85af4ffbc2666bcdbcc438f89f62632b9a97b3f Mon Sep 17 00:00:00 2001 From: Ben Wolsieffer Date: Fri, 9 Dec 2022 18:00:39 -0500 Subject: [PATCH 19/25] nix: fix formatting to satisfy pycodestyle and flake8 --- superflore/generators/nix/gen_packages.py | 5 ++-- superflore/generators/nix/nix_derivation.py | 18 +++++++------ superflore/generators/nix/nix_package.py | 25 +++++++++++-------- superflore/generators/nix/nix_package_set.py | 2 +- superflore/generators/nix/nix_ros_overlay.py | 4 +-- superflore/generators/nix/run.py | 3 +-- superflore/test_integration/nix/build_base.py | 1 - superflore/test_integration/nix/main.py | 3 +-- superflore/utils.py | 3 ++- 9 files changed, 34 insertions(+), 30 deletions(-) diff --git a/superflore/generators/nix/gen_packages.py b/superflore/generators/nix/gen_packages.py index 62de8999..06eb4a68 100644 --- a/superflore/generators/nix/gen_packages.py +++ b/superflore/generators/nix/gen_packages.py @@ -13,12 +13,10 @@ # limitations under the License. import os import re -from typing import Iterable, Dict +from typing import Dict, Iterable from rosdistro import DistributionFile from rosinstall_generator.distro import get_package_names - -from superflore.exceptions import NoPkgXml from superflore.exceptions import UnresolvedDependency from superflore.generators.nix.nix_package import NixPackage from superflore.generators.nix.nix_package_set import NixPackageSet @@ -32,6 +30,7 @@ _version_regex = re.compile(r"version\s*=\s*\"([^\"]*)\"") + def regenerate_pkg(overlay, pkg: str, distro: DistributionFile, preserve_existing: bool, tar_dir: str, sha256_cache: Dict[str, str]): diff --git a/superflore/generators/nix/nix_derivation.py b/superflore/generators/nix/nix_derivation.py index 65d49d93..31fa311b 100644 --- a/superflore/generators/nix/nix_derivation.py +++ b/superflore/generators/nix/nix_derivation.py @@ -22,12 +22,12 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # -import os -import urllib.parse from operator import attrgetter +import os from textwrap import dedent from time import gmtime, strftime from typing import Iterable, Set +import urllib.parse from superflore.utils import get_license @@ -134,11 +134,12 @@ def get_text(self, distributor: str, license_name: str) -> str: ret += '{ lib, buildRosPackage, fetchurl, ' + \ ', '.join(sorted(set(map(self._to_nix_parameter, - self.build_inputs | - self.propagated_build_inputs | - self.check_inputs | - self.native_build_inputs | - self.propagated_native_build_inputs)))) + ' }:' + self.build_inputs | + self.propagated_build_inputs | + self.check_inputs | + self.native_build_inputs | + self.propagated_native_build_inputs))) + ) + ' }:' ret += dedent(''' buildRosPackage {{ @@ -171,7 +172,8 @@ def get_text(self, distributor: str, license_name: str) -> str: if self.propagated_build_inputs: ret += " propagatedBuildInputs = {};\n" \ - .format(self._to_nix_list(sorted(self.propagated_build_inputs))) + .format(self._to_nix_list(sorted( + self.propagated_build_inputs))) if self.native_build_inputs: ret += " nativeBuildInputs = {};\n" \ diff --git a/superflore/generators/nix/nix_package.py b/superflore/generators/nix/nix_package.py index ddd81ac1..3f482ccb 100644 --- a/superflore/generators/nix/nix_package.py +++ b/superflore/generators/nix/nix_package.py @@ -9,7 +9,6 @@ from rosdistro.dependency_walker import DependencyWalker from rosdistro.rosdistro import RosPackage from rosinstall_generator.distro import _generate_rosinstall - from superflore.exceptions import UnresolvedDependency from superflore.generators.nix.nix_derivation import NixDerivation, NixLicense from superflore.PackageMetadata import PackageMetadata @@ -50,8 +49,10 @@ def __init__(self, name: str, distro: DistributionFile, tar_dir: str, info("downloading archive version for package '{}'..." .format(name)) retry_on_exception(download_file, src_uri, archive_path, - retry_msg="network error downloading '{}'".format(src_uri), - error_msg="failed to download archive for '{}'".format(name)) + retry_msg="network error downloading '{}'" + .format(src_uri), + error_msg="failed to download archive for '{}'" + .format(name)) downloaded_archive = True if downloaded_archive or archive_path not in sha256_cache: @@ -82,7 +83,8 @@ def __init__(self, name: str, distro: DistributionFile, tar_dir: str, distro.name)) buildtool_deps = dep_walker.get_depends(pkg.name, "buildtool") - buildtool_export_deps = dep_walker.get_depends(pkg.name, "buildtool_export") + buildtool_export_deps = dep_walker.get_depends( + pkg.name, "buildtool_export") build_deps = dep_walker.get_depends(pkg.name, "build") build_export_deps = dep_walker.get_depends(pkg.name, "build_export") exec_deps = dep_walker.get_depends(pkg.name, "exec") @@ -90,20 +92,23 @@ def __init__(self, name: str, distro: DistributionFile, tar_dir: str, self.unresolved_dependencies = set() - # buildtool_depends are added to buildInputs and nativeBuildInputs. Some - # (such as CMake) have binaries that need to run at build time (and - # therefore need to be in nativeBuildInputs. Others (such as + # buildtool_depends are added to buildInputs and nativeBuildInputs. + # Some (such as CMake) have binaries that need to run at build time + # (and therefore need to be in nativeBuildInputs. Others (such as # ament_cmake_*) need to be added to CMAKE_PREFIX_PATH and therefore # need to be in buildInputs. There is no easy way to distinguish these # two cases, so they are added to both, which generally works fine. - build_inputs = set(self._resolve_dependencies(build_deps | buildtool_deps)) - propagated_build_inputs = self._resolve_dependencies(exec_deps | build_export_deps | buildtool_export_deps) + build_inputs = set(self._resolve_dependencies( + build_deps | buildtool_deps)) + propagated_build_inputs = self._resolve_dependencies( + exec_deps | build_export_deps | buildtool_export_deps) build_inputs -= propagated_build_inputs check_inputs = self._resolve_dependencies(test_deps) check_inputs -= build_inputs - native_build_inputs = self._resolve_dependencies(buildtool_deps | buildtool_export_deps) + native_build_inputs = self._resolve_dependencies( + buildtool_deps | buildtool_export_deps) self._derivation = NixDerivation( name=normalized_name, diff --git a/superflore/generators/nix/nix_package_set.py b/superflore/generators/nix/nix_package_set.py index 1bae511f..f89093c7 100644 --- a/superflore/generators/nix/nix_package_set.py +++ b/superflore/generators/nix/nix_package_set.py @@ -1,5 +1,5 @@ from textwrap import dedent -from time import strftime, gmtime +from time import gmtime, strftime from typing import Iterable from superflore.generators.nix.nix_package import NixPackage diff --git a/superflore/generators/nix/nix_ros_overlay.py b/superflore/generators/nix/nix_ros_overlay.py index c0ad8564..19f3abeb 100644 --- a/superflore/generators/nix/nix_ros_overlay.py +++ b/superflore/generators/nix/nix_ros_overlay.py @@ -24,7 +24,7 @@ class NixRosOverlay(object): def __init__(self, repo_dir, do_clone, org='lopsided98', repo='nix-ros-overlay', from_branch='', new_branch=True): self.repo = RepoInstance(org, repo, repo_dir, do_clone, - from_branch=from_branch) + from_branch=from_branch) if new_branch: self.branch_name = 'nix-bot-%s' % rand_ascii_str() info('Creating new branch {0}...'.format(self.branch_name)) @@ -61,4 +61,4 @@ def pull_request(self, message, distro=None, title=''): time.ctime()) title = 'rosdistro sync, {0}'.format(timestamp) self.repo.pull_request(message, title, branch=self.branch_name, - fork=False) + fork=False) diff --git a/superflore/generators/nix/run.py b/superflore/generators/nix/run.py index d2d83e10..2131b41a 100644 --- a/superflore/generators/nix/run.py +++ b/superflore/generators/nix/run.py @@ -16,16 +16,15 @@ import sys from rosinstall_generator.distro import get_distro - from superflore.CacheManager import CacheManager from superflore.exceptions import NoGitHubAuthToken -from superflore.TempfileManager import TempfileManager from superflore.generate_installers import generate_installers from superflore.generators.nix.gen_packages import regenerate_pkg, \ regenerate_pkg_set from superflore.generators.nix.nix_ros_overlay import NixRosOverlay from superflore.parser import get_parser from superflore.repo_instance import RepoInstance +from superflore.TempfileManager import TempfileManager from superflore.utils import clean_up, get_distros_by_status from superflore.utils import err from superflore.utils import file_pr diff --git a/superflore/test_integration/nix/build_base.py b/superflore/test_integration/nix/build_base.py index e3e38cee..b94dba79 100644 --- a/superflore/test_integration/nix/build_base.py +++ b/superflore/test_integration/nix/build_base.py @@ -13,7 +13,6 @@ # limitations under the License. from docker.errors import ContainerError - from superflore.docker import Docker from superflore.generators.nix.nix_package import NixPackage from superflore.utils import err diff --git a/superflore/test_integration/nix/main.py b/superflore/test_integration/nix/main.py index 96090f6a..186012da 100644 --- a/superflore/test_integration/nix/main.py +++ b/superflore/test_integration/nix/main.py @@ -15,10 +15,9 @@ import argparse import sys -import yaml - from superflore.test_integration.nix.build_base import NixBuilder from superflore.utils import get_distros_by_status +import yaml def main(): diff --git a/superflore/utils.py b/superflore/utils.py index a295e2ca..65ad499a 100644 --- a/superflore/utils.py +++ b/superflore/utils.py @@ -797,7 +797,8 @@ def download_file(url, filename): 'User-Agent': 'superflore/{}'.format(get_superflore_version()) }) try: - with urllib.request.urlopen(request) as response, open(filename, 'wb') as file: + with urllib.request.urlopen(request) as response, \ + open(filename, 'wb') as file: shutil.copyfileobj(response, file) except Exception as e: try: From d45c3edb755a1bd0ea1dee53f5e3bbaa29c014c5 Mon Sep 17 00:00:00 2001 From: Ben Wolsieffer Date: Fri, 9 Dec 2022 18:14:04 -0500 Subject: [PATCH 20/25] nix: rename NixDerivation to NixExpression This name better represents what this class does, which is to generate the actual text of a Nix expression for a package. --- .../generators/nix/{nix_derivation.py => nix_expression.py} | 4 ++-- superflore/generators/nix/nix_package.py | 4 ++-- tests/test_nix.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) rename superflore/generators/nix/{nix_derivation.py => nix_expression.py} (98%) diff --git a/superflore/generators/nix/nix_derivation.py b/superflore/generators/nix/nix_expression.py similarity index 98% rename from superflore/generators/nix/nix_derivation.py rename to superflore/generators/nix/nix_expression.py index 31fa311b..ace918fe 100644 --- a/superflore/generators/nix/nix_derivation.py +++ b/superflore/generators/nix/nix_expression.py @@ -77,7 +77,7 @@ def nix_code(self) -> str: return self.name -class NixDerivation: +class NixExpression: def __init__(self, name: str, version: str, src_url: str, src_sha256: str, description: str, licenses: Iterable[NixLicense], @@ -119,7 +119,7 @@ def _to_nix_parameter(dep: str) -> str: def get_text(self, distributor: str, license_name: str) -> str: """ - Generate the Nix derivation, given the distributor line + Generate the Nix expression, given the distributor line and the license text. """ diff --git a/superflore/generators/nix/nix_package.py b/superflore/generators/nix/nix_package.py index 3f482ccb..95df620b 100644 --- a/superflore/generators/nix/nix_package.py +++ b/superflore/generators/nix/nix_package.py @@ -10,7 +10,7 @@ from rosdistro.rosdistro import RosPackage from rosinstall_generator.distro import _generate_rosinstall from superflore.exceptions import UnresolvedDependency -from superflore.generators.nix.nix_derivation import NixDerivation, NixLicense +from superflore.generators.nix.nix_expression import NixExpression, NixLicense from superflore.PackageMetadata import PackageMetadata from superflore.utils import (download_file, get_distro_condition_context, get_distros, get_pkg_version, info, resolve_dep, @@ -110,7 +110,7 @@ def __init__(self, name: str, distro: DistributionFile, tar_dir: str, native_build_inputs = self._resolve_dependencies( buildtool_deps | buildtool_export_deps) - self._derivation = NixDerivation( + self._derivation = NixExpression( name=normalized_name, version=version, src_url=src_uri, diff --git a/tests/test_nix.py b/tests/test_nix.py index f32105e1..bdd13658 100644 --- a/tests/test_nix.py +++ b/tests/test_nix.py @@ -14,7 +14,7 @@ import unittest -from superflore.generators.nix.nix_derivation import NixLicense +from superflore.generators.nix.nix_expression import NixLicense class TestNixLicense(unittest.TestCase): From 6865dfd98d800a8715a8b2ac2c3fd0a2bf02597d Mon Sep 17 00:00:00 2001 From: Ben Wolsieffer Date: Fri, 9 Dec 2022 18:19:54 -0500 Subject: [PATCH 21/25] nix: fix license test and public domain license The license translation logic has changed, and adds a hyphen to unknown licenses. Also, add a test for the public domain license. --- superflore/generators/nix/nix_expression.py | 2 +- tests/test_nix.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/superflore/generators/nix/nix_expression.py b/superflore/generators/nix/nix_expression.py index ace918fe..391aa244 100644 --- a/superflore/generators/nix/nix_expression.py +++ b/superflore/generators/nix/nix_expression.py @@ -57,7 +57,7 @@ class NixLicense: 'MPL-1.0': 'mpl10', 'MPL-1.1': 'mpl11', 'MPL-2.0': 'mpl20', - 'public_domain': 'publicDomain', + 'PD': 'publicDomain', } def __init__(self, name): diff --git a/tests/test_nix.py b/tests/test_nix.py index bdd13658..58775cb9 100644 --- a/tests/test_nix.py +++ b/tests/test_nix.py @@ -25,4 +25,8 @@ def test_known_license(self): def test_unknown_license(self): l = NixLicense("some license") - self.assertEqual(l.nix_code, '"some license"') + self.assertEqual(l.nix_code, '"some-license"') + + def test_public_domain(self): + l = NixLicense("Public Domain") + self.assertEqual(l.nix_code, 'publicDomain') From d49c52da3d7e4c8deaa3c7381717b144c024c3f8 Mon Sep 17 00:00:00 2001 From: Ben Wolsieffer Date: Fri, 9 Dec 2022 18:51:44 -0500 Subject: [PATCH 22/25] nix: add rolling to --all mode When --all is specified, generate for rolling as well. --- superflore/generators/nix/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/superflore/generators/nix/run.py b/superflore/generators/nix/run.py index 2131b41a..c1f6d4bf 100644 --- a/superflore/generators/nix/run.py +++ b/superflore/generators/nix/run.py @@ -78,7 +78,8 @@ def main(): elif args.only: parser.error('Invalid args! --only requires specifying --ros-distro') if not selected_targets: - selected_targets = get_distros_by_status('active') + selected_targets = get_distros_by_status('active') + \ + get_distros_by_status('rolling') repo_org = 'lopsided98' repo_name = 'nix-ros-overlay' if args.upstream_repo: From adcd250c47988737fce00ab361ab91090aaf9fc1 Mon Sep 17 00:00:00 2001 From: koalp Date: Mon, 19 Feb 2024 23:04:02 +0100 Subject: [PATCH 23/25] nix: fix: escape special characters in license The license string in the generated nix package file was broken when the license name contained quote. It has been fixed. Backslash and the sequence "dollar opening bracket" have been escaped too. --- superflore/generators/nix/nix_expression.py | 4 +++- tests/test_nix.py | 9 ++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/superflore/generators/nix/nix_expression.py b/superflore/generators/nix/nix_expression.py index 391aa244..c0c6ca92 100644 --- a/superflore/generators/nix/nix_expression.py +++ b/superflore/generators/nix/nix_expression.py @@ -31,6 +31,8 @@ from superflore.utils import get_license +def _sanitize_nix_string(string: str): + return string.replace("\\", "\\\\").replace("${", r"\${").replace('"', r'\"') class NixLicense: """ @@ -72,7 +74,7 @@ def __init__(self, name): @property def nix_code(self) -> str: if self.custom: - return '"{}"'.format(self.name) + return '"{}"'.format(_sanitize_nix_string(self.name)) else: return self.name diff --git a/tests/test_nix.py b/tests/test_nix.py index 58775cb9..3b4d8c33 100644 --- a/tests/test_nix.py +++ b/tests/test_nix.py @@ -16,7 +16,6 @@ from superflore.generators.nix.nix_expression import NixLicense - class TestNixLicense(unittest.TestCase): def test_known_license(self): @@ -30,3 +29,11 @@ def test_unknown_license(self): def test_public_domain(self): l = NixLicense("Public Domain") self.assertEqual(l.nix_code, 'publicDomain') + + def test_escape_quote(self): + l = NixLicense(r'license with "quotes" and \backslash" '); + self.assertEqual(l.nix_code, r'"license-with-\"quotes\"-and-\\backslash"') + + def test_escape_quote(self): + l = NixLicense('some license with the "${" sequence'); + self.assertEqual(l.nix_code, r'"some-license-with-the-\"\${\"-sequence"') From 6d5327e23c3145ae574e1703a76b31cb13db853e Mon Sep 17 00:00:00 2001 From: Ben Wolsieffer Date: Fri, 22 Mar 2024 22:15:08 -0400 Subject: [PATCH 24/25] nix: escape description string Use the same function to escape the description string as is used for the license string. --- superflore/generators/nix/nix_expression.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/superflore/generators/nix/nix_expression.py b/superflore/generators/nix/nix_expression.py index c0c6ca92..e096e6ec 100644 --- a/superflore/generators/nix/nix_expression.py +++ b/superflore/generators/nix/nix_expression.py @@ -31,8 +31,12 @@ from superflore.utils import get_license -def _sanitize_nix_string(string: str): - return string.replace("\\", "\\\\").replace("${", r"\${").replace('"', r'\"') + +def _escape_nix_string(string: str): + return '"{}"'.format(string.replace("\\", "\\\\") + .replace("${", r"\${") + .replace('"', r"\"")) + class NixLicense: """ @@ -74,7 +78,7 @@ def __init__(self, name): @property def nix_code(self) -> str: if self.custom: - return '"{}"'.format(_sanitize_nix_string(self.name)) + return _escape_nix_string(self.name) else: return self.name @@ -187,11 +191,11 @@ def get_text(self, distributor: str, license_name: str) -> str: ret += dedent(''' meta = {{ - description = ''{}''; + description = {}; license = with lib.licenses; {}; }}; }} - ''').format(self.description, + ''').format(_escape_nix_string(self.description), self._to_nix_list(map(attrgetter('nix_code'), self.licenses))) From 80631400180cf5412bb1953ba1eeabb0d165de95 Mon Sep 17 00:00:00 2001 From: Ben Wolsieffer Date: Fri, 22 Mar 2024 22:54:27 -0400 Subject: [PATCH 25/25] Fix trailing whitespace --- tests/test_nix.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_nix.py b/tests/test_nix.py index 3b4d8c33..c4dabea0 100644 --- a/tests/test_nix.py +++ b/tests/test_nix.py @@ -30,10 +30,10 @@ def test_public_domain(self): l = NixLicense("Public Domain") self.assertEqual(l.nix_code, 'publicDomain') - def test_escape_quote(self): + def test_escape_quote(self): l = NixLicense(r'license with "quotes" and \backslash" '); self.assertEqual(l.nix_code, r'"license-with-\"quotes\"-and-\\backslash"') - - def test_escape_quote(self): + + def test_escape_quote(self): l = NixLicense('some license with the "${" sequence'); self.assertEqual(l.nix_code, r'"some-license-with-the-\"\${\"-sequence"')