Allow to derive from multiple containers

kiwi/builder/container.py

@@ -65,9 +65,9 @@ def __init__(
             xml_state.build_type.get_metadata_path()
 
         if xml_state.get_derived_from_image_uri() and not self.delta_root:
-            # The base image is expected to be unpacked by the kiwi
-            # prepare step and stored inside of the root_dir/image directory.
-            # In addition a md5 file of the image is expected too
+            # The base image(all derived imports) is expected to be unpacked
+            # by the kiwi prepare step and stored inside of the root_dir/image
+            # directory. In addition a md5 file of the image is expected too
             self.base_image = Defaults.get_imported_root_image(
                 self.root_dir
             )

kiwi/system/prepare.py

@@ -82,12 +82,12 @@ def __init__(
             root_dir, allow_existing
         )
         root.create()
-        image_uri = xml_state.get_derived_from_image_uri()
+        image_uris = xml_state.get_derived_from_image_uri()
         delta_root = xml_state.build_type.get_delta_root()
 
-        if image_uri:
+        if image_uris:
             self.root_import = RootImport.new(
-                root_dir, image_uri, xml_state.build_type.get_image()
+                root_dir, image_uris, xml_state.build_type.get_image()
             )
             if delta_root:
                 self.root_import.overlay_data()

kiwi/system/root_import/__init__.py

@@ -17,6 +17,7 @@
 #
 import logging
 import importlib
+from typing import List
 from abc import (
     ABCMeta,
     abstractmethod
@@ -49,7 +50,7 @@ def __init__(self) -> None:
         return None  # pragma: no cover
 
     @staticmethod
-    def new(root_dir: str, image_uri: Uri, image_type: str):
+    def new(root_dir: str, image_uri: List[Uri], image_type: str):
         name_map = {
             'docker': 'OCI',
             'oci': 'OCI'

kiwi/system/root_import/base.py

@@ -18,13 +18,18 @@
 import os
 import logging
 import pathlib
+from typing import (
+    Dict, List, Optional
+)
 
 # project
 from kiwi.xml_state import XMLState
 from kiwi.defaults import Defaults
 from kiwi.system.setup import SystemSetup
 from kiwi.path import Path
+from kiwi.system.uri import Uri
 from kiwi.command import Command
+from kiwi.mount_manager import MountManager
 from kiwi.utils.checksum import Checksum
 from kiwi.exceptions import (
     KiwiRootImportError,
@@ -41,38 +46,43 @@ class RootImportBase:
     * :attr:`root_dir`
         root directory path name
 
-    * :attr:`image_uri`
-        Uri object to store source location
+    * :attr:`image_uris`
+        Uri object(s) to store source location(s)
 
     * :attr:`custom_args`
         Dictonary to set specialized class specific configuration values
     """
-    def __init__(self, root_dir, image_uri, custom_args=None):
-        self.unknown_uri = None
-        self.overlay = None
+    def __init__(
+        self, root_dir: str, image_uris: List[Uri],
+        custom_args: Dict[str, str] = {}
+    ):
+        self.raw_urls: List[str] = []
+        self.image_files: List[str] = []
+        self.overlay: Optional[MountManager] = None
         self.root_dir = root_dir
-        try:
-            if image_uri.is_remote():
-                raise KiwiRootImportError(
-                    'Only local imports are supported'
-                )
+        for image_uri in image_uris:
+            try:
+                if image_uri.is_remote():
+                    raise KiwiRootImportError(
+                        'Only local imports are supported'
+                    )
 
-            self.image_file = image_uri.translate()
+                image_file = image_uri.translate()
+                self.image_files.append(image_file)
 
-            if not os.path.exists(self.image_file):
-                raise KiwiRootImportError(
-                    'Could not stat base image file: {0}'.format(
-                        self.image_file
+                if not os.path.exists(image_file):
+                    raise KiwiRootImportError(
+                        f'Could not stat base image file: {image_file}'
                     )
+            except KiwiUriTypeUnknown:
+                # Let specialized class handle unknown uri schemes
+                raw_url = image_uri.uri
+                log.warning(
+                    f'Unkown URI type for the base image: {raw_url}'
                 )
-        except KiwiUriTypeUnknown:
-            # Let specialized class handle unknown uri schemes
-            log.warning(
-                'Unkown URI type for the base image: %s', image_uri.uri
-            )
-            self.unknown_uri = image_uri.uri
-        finally:
-            self.post_init(custom_args)
+                self.raw_urls.append(raw_url)
+
+        self.post_init(custom_args)
 
     def post_init(self, custom_args):
         """

kiwi/system/root_import/oci.py

@@ -18,6 +18,9 @@
 import os
 import logging
 import pathlib
+from typing import (
+    Dict, List
+)
 
 # project
 from kiwi.system.root_import.base import RootImportBase
@@ -35,65 +38,69 @@ class RootImportOCI(RootImportBase):
     Implements the base class for importing a root system from
     a oci image tarball file.
     """
-    def post_init(self, custom_args):
+    def post_init(self, custom_args: Dict[str, str]):
         self.archive_transport = custom_args['archive_transport']
 
     def sync_data(self):
         """
         Synchronize data from the given base image to the target root
         directory.
         """
-        image_uri = self._get_image_uri()
+        for image_url in self._get_image_urls():
+            with OCI.new() as oci:
+                oci.import_container_image(image_url)
+                oci.unpack()
+                oci.import_rootfs(self.root_dir)
 
-        with OCI.new() as oci:
-            oci.import_container_image(image_uri)
-            oci.unpack()
-            oci.import_rootfs(self.root_dir)
+        # A copy of the uncompressed image(all derived imports) and
+        # its checksum are kept inside the root_dir in order to ensure
+        # the later steps i.e. system create are atomic and don't need
+        # any third party archive.
+        image_copy = Defaults.get_imported_root_image(self.root_dir)
 
-            # A copy of the uncompressed image and its checksum are
-            # kept inside the root_dir in order to ensure the later steps
-            # i.e. system create are atomic and don't need any third
-            # party archive.
-            image_copy = Defaults.get_imported_root_image(self.root_dir)
-            Path.create(os.path.dirname(image_copy))
-            oci.export_container_image(
-                image_copy, 'oci-archive',
-                Defaults.get_container_base_image_tag()
-            )
-            self._make_checksum(image_copy)
+        Path.create(os.path.dirname(image_copy))
+        oci.export_container_image(
+            image_copy, 'oci-archive',
+            Defaults.get_container_base_image_tag()
+        )
+        self._make_checksum(image_copy)
 
     def overlay_data(self) -> None:
         """
         Synchronize data from the given base image to the target root
         directory as an overlayfs mounted target.
         """
-        image_uri = self._get_image_uri()
-
         root_dir_ro = f'{self.root_dir}_ro'
 
-        with OCI.new() as oci:
-            oci.import_container_image(image_uri)
-            oci.unpack()
-            oci.import_rootfs(self.root_dir)
-            log.debug("renaming %s -> %s", self.root_dir, root_dir_ro)
-            pathlib.Path(self.root_dir).replace(root_dir_ro)
-            Path.create(self.root_dir)
+        for image_url in self._get_image_urls():
+            with OCI.new() as oci:
+                oci.import_container_image(image_url)
+                oci.unpack()
+                oci.import_rootfs(self.root_dir)
+
+        log.debug("renaming %s -> %s", self.root_dir, root_dir_ro)
+        pathlib.Path(self.root_dir).replace(root_dir_ro)
+        Path.create(self.root_dir)
 
-            self.overlay = MountManager(device='', mountpoint=self.root_dir)
-            self.overlay.overlay_mount(root_dir_ro)
+        self.overlay = MountManager(device='', mountpoint=self.root_dir)
+        self.overlay.overlay_mount(root_dir_ro)
 
-    def _get_image_uri(self) -> str:
-        if not self.unknown_uri:
-            self.compressor = Compress(self.image_file)
-            if self.compressor.get_format():
-                self.compressor.uncompress(True)
-                self.uncompressed_image = self.compressor.uncompressed_filename
-            else:
-                self.uncompressed_image = self.image_file
-            image_uri = '{0}:{1}'.format(
-                self.archive_transport, self.uncompressed_image
-            )
+    def _get_image_urls(self) -> List[str]:
+        if not self.raw_urls:
+            image_urls: List[str] = []
+            for image_file in self.image_files:
+                compressor = Compress(image_file)
+                if compressor.get_format():
+                    compressor.uncompress(True)
+                    uncompressed_image = compressor.uncompressed_filename
+                else:
+                    uncompressed_image = image_file
+                image_urls.append(
+                    '{0}:{1}'.format(
+                        self.archive_transport, uncompressed_image
+                    )
+                )
+            return image_urls
         else:
             log.warning('Bypassing base image URI to OCI tools')
-            image_uri = self.unknown_uri
-        return image_uri
+            return self.raw_urls

kiwi/xml_state.py

@@ -2781,22 +2781,26 @@ def get_luks_format_options(self) -> List[str]:
             result.append(luks_pbkdf)
         return result
 
-    def get_derived_from_image_uri(self) -> Optional[Uri]:
+    def get_derived_from_image_uri(self) -> List[Uri]:
         """
-        Uri object of derived image if configured
+        Uri object(s) of derived image if configured
 
-        Specific image types can be based on a master image.
-        This method returns the location of this image when
-        configured in the XML description
+        Specific image types can be based on one ore more derived
+        images. This method returns the location of this image(s)
+        when configured in the XML description
 
-        :return: Instance of Uri
+        :return: List of Uri instances
 
-        :rtype: object
+        :rtype: list
         """
-        derived_image = self.build_type.get_derived_from()
-        if derived_image:
-            return Uri(derived_image, repo_type='container')
-        return None
+        image_uris = []
+        derived_images = self.build_type.get_derived_from()
+        if derived_images:
+            for derived_image in derived_images.split(','):
+                image_uris.append(
+                    Uri(derived_image, repo_type='container')
+                )
+        return image_uris
 
     def set_derived_from_image_uri(self, uri: str) -> None:
         """

test/unit/system/root_import/base_test.py

@@ -18,32 +18,32 @@ def test_init(self, mock_buildservice, mock_path):
         mock_buildservice.return_value = False
         mock_path.return_value = True
         with patch.dict('os.environ', {'HOME': '../data'}):
-            RootImportBase('root_dir', Uri('file:///image.tar.xz'))
+            RootImportBase('root_dir', [Uri('file:///image.tar.xz')])
         assert call('/image.tar.xz') in mock_path.call_args_list
 
     def test_init_remote_uri(self):
         with raises(KiwiRootImportError):
-            RootImportBase('root_dir', Uri('http://example.com/image.tar.xz'))
+            RootImportBase('root_dir', [Uri('http://example.com/image.tar.xz')])
 
     @patch('kiwi.system.root_import.base.log.warning')
     def test_init_unknown_uri(self, mock_log_warn):
-        root = RootImportBase('root_dir', Uri('docker://opensuse:leap'))
-        assert root.unknown_uri == 'docker://opensuse:leap'
+        root = RootImportBase('root_dir', [Uri('docker://opensuse:leap')])
+        assert root.raw_urls == ['docker://opensuse:leap']
         assert mock_log_warn.called
 
     @patch('os.path.exists')
     def test_init_non_existing(self, mock_path):
         mock_path.return_value = False
         with patch.dict('os.environ', {'HOME': '../data'}):
             with raises(KiwiRootImportError):
-                RootImportBase('root_dir', Uri('file:///image.tar.xz'))
+                RootImportBase('root_dir', [Uri('file:///image.tar.xz')])
 
     @patch('os.path.exists')
     def test_data_sync(self, mock_path):
         mock_path.return_value = True
         with patch.dict('os.environ', {'HOME': '../data'}):
             root_import = RootImportBase(
-                'root_dir', Uri('file:///image.tar.xz')
+                'root_dir', [Uri('file:///image.tar.xz')]
             )
         with raises(NotImplementedError):
             root_import.sync_data()
@@ -53,7 +53,7 @@ def test_overlay_data(self, mock_path):
         mock_path.return_value = True
         with patch.dict('os.environ', {'HOME': '../data'}):
             root_import = RootImportBase(
-                'root_dir', Uri('docker://opensuse:leap')
+                'root_dir', [Uri('docker://opensuse:leap')]
             )
         with raises(NotImplementedError):
             root_import.overlay_data()
@@ -77,7 +77,7 @@ def test_overlay_finalize(
             xml_state = Mock()
             mock_Command_run.return_value.output = '/file_a\n/file_b'
             with patch.dict('os.environ', {'HOME': '../data'}):
-                root = RootImportBase('root_dir', Uri('docker://opensuse:leap'))
+                root = RootImportBase('root_dir', [Uri('docker://opensuse:leap')])
                 root.overlay = Mock()
                 mock_pathlib.Path = Mock()
                 root_overlay_path_mock = Mock()

test/unit/system/root_import/oci_test.py

@@ -22,10 +22,10 @@ def setup(self, mock_path):
         mock_path.return_value = True
         with patch.dict('os.environ', {'HOME': '../data'}):
             self.oci_import = RootImportOCI(
-                'root_dir', Uri('file:///image.tar'),
+                'root_dir', [Uri('file:///image.tar')],
                 {'archive_transport': 'oci-archive'}
             )
-        assert self.oci_import.image_file == '/image.tar'
+        assert self.oci_import.image_files == ['/image.tar']
 
     @patch('os.path.exists')
     def setup_method(self, cls, mock_path):
@@ -36,7 +36,7 @@ def test_failed_init(self, mock_path):
         mock_path.return_value = False
         with raises(KiwiRootImportError):
             RootImportOCI(
-                'root_dir', Uri('file:///image.tar.xz'),
+                'root_dir', [Uri('file:///image.tar.xz')],
                 {'archive_transport': 'oci-archive'}
             )
 
@@ -137,7 +137,7 @@ def test_sync_data_unknown_uri(
         mock_md5.return_value = Mock()
         with patch.dict('os.environ', {'HOME': '../data'}):
             oci_import = RootImportOCI(
-                'root_dir', Uri('docker:image:tag'),
+                'root_dir', [Uri('docker:image:tag')],
                 {'archive_transport': 'docker-archive'}
             )
 

test/unit/xml_state_test.py

@@ -1105,14 +1105,14 @@ def test_get_build_type_format_options(self):
     def test_get_derived_from_image_uri(self):
         xml_data = self.description.load()
         state = XMLState(xml_data, ['derivedContainer'], 'docker')
-        assert state.get_derived_from_image_uri().uri == \
+        assert state.get_derived_from_image_uri()[0].uri == \
             'obs://project/repo/image#mytag'
 
     def test_set_derived_from_image_uri(self):
         xml_data = self.description.load()
         state = XMLState(xml_data, ['derivedContainer'], 'docker')
         state.set_derived_from_image_uri('file:///new_uri')
-        assert state.get_derived_from_image_uri().translate() == '/new_uri'
+        assert state.get_derived_from_image_uri()[0].translate() == '/new_uri'
 
     def test_set_derived_from_image_uri_not_applied(self):
         with self._caplog.at_level(logging.WARNING):