Merge pull request #1773 from dbungert/zfsutils-linux

install zfsutils-linux if needed

examples/autoinstall/interactive.yaml

@@ -26,5 +26,8 @@ packages:
 snaps:
   - name: etcd
     channel: 3.2/stable
+storage:
+  layout:
+    name: zfs
 debconf-selections: |
   wtf wtf

scripts/installdeps.sh

@@ -3,7 +3,10 @@ set -eux
 cd "$(dirname ${BASH_SOURCE:0})/.."
 
 apt-get update
-DEBIAN_FRONTEND=noninteractive apt-get -o APT::Get::Always-Include-Phased-Updates=true -y dist-upgrade
+DEBIAN_FRONTEND=noninteractive apt-get \
+    -o Dpkg::Options::=--force-confnew \
+    -o APT::Get::Always-Include-Phased-Updates=true \
+    -y dist-upgrade
 mkdir -p /etc/systemd/system/zfs-mount.service.d/
 cat >/etc/systemd/system/zfs-mount.service.d/override.conf <<EOF
 [Unit]

subiquity/models/filesystem.py

@@ -24,7 +24,7 @@ import pathlib
 import platform
 import tempfile
 from abc import ABC, abstractmethod
-from typing import List, Optional, Set, Union
+from typing import List, Optional, Set, Tuple, Union
 
 import attr
 import more_itertools
@@ -1275,6 +1275,7 @@ class FilesystemModel(object):
         self.storage_version = 1
         self._probe_data = None
         self.dd_target: Optional[Disk] = None
+        self.reset_partition: Optional[Partition] = None
         self.reset()
 
     def reset(self):
@@ -2037,3 +2038,12 @@ class FilesystemModel(object):
         )
         self._actions.append(zpool)
         return zpool
+
+    async def live_packages(self) -> Tuple[Set, Set]:
+        before = set()
+        during = set()
+        if self._one(type="zpool") is not None:
+            before.add("zfsutils-linux")
+        if self.reset_partition is not None:
+            during.add("efibootmgr")
+        return (before, during)

subiquity/models/subiquity.py

@@ -20,7 +20,7 @@ import logging
 import os
 import uuid
 from collections import OrderedDict
-from typing import Any, Dict, Set
+from typing import Any, Dict, Set, Tuple
 
 import yaml
 from cloudinit.config.schema import (
@@ -390,6 +390,21 @@ class SubiquityModel:
                 packages.extend(await meth())
         return packages
 
+    async def live_packages(self) -> Tuple[Set, Set]:
+        """return a tuple of sets of packages to install into the live environment.
+        The first set must be installed before partitioning, the second set may be
+        allowed to run in parallel with partitioning.
+        """
+        before = set()
+        during = set()
+        for model_name in self._install_model_names.all():
+            meth = getattr(getattr(self, model_name), "live_packages", None)
+            if meth is not None:
+                packages = await meth()
+                before |= packages[0]
+                during |= packages[1]
+        return (before, during)
+
     def _cloud_init_files(self):
         # TODO, this should be moved to the in-target cloud-config seed so on
         # first boot of the target, it reconfigures datasource_list to none

subiquity/models/tests/test_filesystem.py

@@ -1526,3 +1526,35 @@ class TestRootfs(SubiTestCase):
         m = make_model()
         make_zpool(model=m, mountpoint="/srv")
         self.assertFalse(m.is_root_mounted())
+
+
+class TestLivePackages(SubiTestCase):
+    async def test_defaults(self):
+        m = make_model()
+        (before, during) = await m.live_packages()
+        self.assertEqual(set(), before)
+        self.assertEqual(set(), during)
+
+    async def test_zfs(self):
+        m = make_model()
+        make_zpool(model=m, mountpoint="/")
+        (before, during) = await m.live_packages()
+        self.assertEqual(set(["zfsutils-linux"]), before)
+        self.assertEqual(set(), during)
+
+    async def test_efibootmgr(self):
+        m = make_model()
+        d = make_disk(m)
+        m.reset_partition = make_partition(m, d)
+        (before, during) = await m.live_packages()
+        self.assertEqual(set(), before)
+        self.assertEqual(set(["efibootmgr"]), during)
+
+    async def test_both(self):
+        m = make_model()
+        d = make_disk(m)
+        make_zpool(model=m, mountpoint="/")
+        m.reset_partition = make_partition(m, d)
+        (before, during) = await m.live_packages()
+        self.assertEqual(set(["zfsutils-linux"]), before)
+        self.assertEqual(set(["efibootmgr"]), during)

subiquity/server/controllers/filesystem.py

@@ -62,9 +62,7 @@ from subiquity.models.filesystem import (
     ArbitraryDevice,
 )
 from subiquity.models.filesystem import Disk as ModelDisk
-from subiquity.models.filesystem import MiB
+from subiquity.models.filesystem import MiB, Raid, _Device, align_down, align_up
-from subiquity.models.filesystem import Partition as ModelPartition
-from subiquity.models.filesystem import Raid, _Device, align_down, align_up
 from subiquity.server import snapdapi
 from subiquity.server.controller import SubiquityController
 from subiquity.server.mounter import Mounter
@@ -244,7 +242,6 @@ class FilesystemController(SubiquityController, FilesystemManipulator):
         # If probe data come in while we are doing partitioning, store it in
         # this variable. It will be picked up on next reset.
         self.queued_probe_data: Optional[Dict[str, Any]] = None
-        self.reset_partition: Optional[ModelPartition] = None
         self.reset_partition_only: bool = False
 
     def is_core_boot_classic(self):
@@ -635,7 +632,7 @@ class FilesystemController(SubiquityController, FilesystemManipulator):
                 reset_size = int(cp.stdout.strip().split()[0])
                 reset_size = align_up(int(reset_size * 1.10), 256 * MiB)
             reset_gap, gap = gap.split(reset_size)
-            self.reset_partition = self.create_partition(
+            self.model.reset_partition = self.create_partition(
                 device=reset_gap.device,
                 gap=reset_gap,
                 spec={"fstype": "fat32"},

subiquity/server/controllers/install.py

@@ -413,7 +413,7 @@ class InstallController(SubiquityController):
             # really write recovery_system={snapd_system_label} to
             # {target}/var/lib/snapd/modeenv to get snapd to pick it up on
             # first boot. But not needed for now.
-        rp = fs_controller.reset_partition
+        rp = fs_controller.model.reset_partition
         if rp is not None:
             mounter = Mounter(self.app)
             rp_target = os.path.join(self.app.root, "factory-reset")
@@ -597,6 +597,19 @@ class InstallController(SubiquityController):
         with open(self.tpath("etc/fstab"), "w") as fp:
             fp.write("/run/mnt/ubuntu-boot/EFI/ubuntu /boot/grub none bind\n")
 
+    async def install_live_packages(self, *, context):
+        before, during = await self.model.live_packages()
+        if len(before) < 1 and len(during) < 1:
+            return
+
+        with context.child("live-packages", "installing packages to live system"):
+            for package in before:
+                state = await self.app.package_installer.install_pkg(package)
+                if state != PackageInstallState.DONE:
+                    raise RuntimeError(f"could not install {package}")
+            for package in during:
+                self.app.package_installer.start_installing_pkg(package)
+
     @with_context()
     async def install(self, *, context):
         context.set("is-install-context", True)
@@ -627,8 +640,7 @@ class InstallController(SubiquityController):
                 fsc = self.app.controllers.Filesystem
                 for_install_path = self.model.source.get_source(fsc._info.name)
 
-            if self.app.controllers.Filesystem.reset_partition:
+            await self.install_live_packages(context=context)
-                self.app.package_installer.start_installing_pkg("efibootmgr")
 
             if self.model.target is not None:
                 if os.path.exists(self.model.target):

subiquity/server/pkghelper.py

@@ -34,9 +34,10 @@ class PackageInstaller:
     by the server installer.
     """
 
-    def __init__(self):
+    def __init__(self, app):
         self.pkgs: Dict[str, asyncio.Task] = {}
         self._cache: Optional[apt.Cache] = None
+        self.app = app
 
     @property
     def cache(self):
@@ -70,6 +71,10 @@ class PackageInstaller:
         if binpkg.installed:
             log.debug("%s already installed", pkgname)
             return PackageInstallState.DONE
+        if self.app.opts.dry_run:
+            log.debug("dry-run apt-get install %s", pkgname)
+            await asyncio.sleep(2 / self.app.scale_factor)
+            return PackageInstallState.DONE
         if not binpkg.candidate.uri.startswith("cdrom:"):
             log.debug(
                 "%s not available from cdrom (rather %s)", pkgname, binpkg.candidate.uri

subiquity/server/server.py

@@ -291,7 +291,7 @@ class SubiquityServer(Application):
         self.event_syslog_id = "subiquity_event.{}".format(os.getpid())
         self.log_syslog_id = "subiquity_log.{}".format(os.getpid())
         self.command_runner = get_command_runner(self)
-        self.package_installer = PackageInstaller()
+        self.package_installer = PackageInstaller(self)
 
         self.error_reporter = ErrorReporter(
             self.context.child("ErrorReporter"), self.opts.dry_run, self.root