From ff990df628526725c4a4af0d59827ee87de00530 Mon Sep 17 00:00:00 2001
From: Dan Bungert <daniel.bungert@canonical.com>
Date: Fri, 25 Aug 2023 13:28:07 -0600
Subject: [PATCH 1/7] storage: move reset_partition to model

---
 subiquity/models/filesystem.py             | 1 +
 subiquity/server/controllers/filesystem.py | 7 ++-----
 subiquity/server/controllers/install.py    | 4 ++--
 3 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/subiquity/models/filesystem.py b/subiquity/models/filesystem.py
index 0f53e548..57440049 100644
--- a/subiquity/models/filesystem.py
+++ b/subiquity/models/filesystem.py
@@ -1275,6 +1275,7 @@ class FilesystemModel(object):
         self.storage_version = 1
         self._probe_data = None
         self.dd_target: Optional[Disk] = None
+        self.reset_partition: Optional[Partition] = None
         self.reset()
 
     def reset(self):
diff --git a/subiquity/server/controllers/filesystem.py b/subiquity/server/controllers/filesystem.py
index af92df4b..c0fbf1a6 100644
--- a/subiquity/server/controllers/filesystem.py
+++ b/subiquity/server/controllers/filesystem.py
@@ -62,9 +62,7 @@ from subiquity.models.filesystem import (
     ArbitraryDevice,
 )
 from subiquity.models.filesystem import Disk as ModelDisk
-from subiquity.models.filesystem import MiB
-from subiquity.models.filesystem import Partition as ModelPartition
-from subiquity.models.filesystem import Raid, _Device, align_down, align_up
+from subiquity.models.filesystem import MiB, Raid, _Device, align_down, align_up
 from subiquity.server import snapdapi
 from subiquity.server.controller import SubiquityController
 from subiquity.server.mounter import Mounter
@@ -244,7 +242,6 @@ class FilesystemController(SubiquityController, FilesystemManipulator):
         # If probe data come in while we are doing partitioning, store it in
         # this variable. It will be picked up on next reset.
         self.queued_probe_data: Optional[Dict[str, Any]] = None
-        self.reset_partition: Optional[ModelPartition] = None
         self.reset_partition_only: bool = False
 
     def is_core_boot_classic(self):
@@ -635,7 +632,7 @@ class FilesystemController(SubiquityController, FilesystemManipulator):
                 reset_size = int(cp.stdout.strip().split()[0])
                 reset_size = align_up(int(reset_size * 1.10), 256 * MiB)
             reset_gap, gap = gap.split(reset_size)
-            self.reset_partition = self.create_partition(
+            self.model.reset_partition = self.create_partition(
                 device=reset_gap.device,
                 gap=reset_gap,
                 spec={"fstype": "fat32"},
diff --git a/subiquity/server/controllers/install.py b/subiquity/server/controllers/install.py
index 865b5d4f..aeee3ff9 100644
--- a/subiquity/server/controllers/install.py
+++ b/subiquity/server/controllers/install.py
@@ -413,7 +413,7 @@ class InstallController(SubiquityController):
             # really write recovery_system={snapd_system_label} to
             # {target}/var/lib/snapd/modeenv to get snapd to pick it up on
             # first boot. But not needed for now.
-        rp = fs_controller.reset_partition
+        rp = fs_controller.model.reset_partition
         if rp is not None:
             mounter = Mounter(self.app)
             rp_target = os.path.join(self.app.root, "factory-reset")
@@ -627,7 +627,7 @@ class InstallController(SubiquityController):
                 fsc = self.app.controllers.Filesystem
                 for_install_path = self.model.source.get_source(fsc._info.name)
 
-            if self.app.controllers.Filesystem.reset_partition:
+            if self.app.base_model.filesystem.reset_partition:
                 self.app.package_installer.start_installing_pkg("efibootmgr")
 
             if self.model.target is not None:

From aa27b338201406b2a731d2f19f53153289f8a457 Mon Sep 17 00:00:00 2001
From: Dan Bungert <daniel.bungert@canonical.com>
Date: Fri, 25 Aug 2023 16:16:52 -0600
Subject: [PATCH 2/7] generalize live system package install

Add live_packages model method where install models may specify
packages needed in the live system.  Move efibootmgr there.
---
 subiquity/models/filesystem.py          | 6 ++++++
 subiquity/models/subiquity.py           | 8 ++++++++
 subiquity/server/controllers/install.py | 8 ++++++--
 3 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/subiquity/models/filesystem.py b/subiquity/models/filesystem.py
index 57440049..93a6ccbc 100644
--- a/subiquity/models/filesystem.py
+++ b/subiquity/models/filesystem.py
@@ -2038,3 +2038,9 @@ class FilesystemModel(object):
         )
         self._actions.append(zpool)
         return zpool
+
+    async def live_packages(self):
+        r = []
+        if self.reset_partition is not None:
+            r.append("efibootmgr")
+        return r
diff --git a/subiquity/models/subiquity.py b/subiquity/models/subiquity.py
index d97dba61..a3077439 100644
--- a/subiquity/models/subiquity.py
+++ b/subiquity/models/subiquity.py
@@ -390,6 +390,14 @@ class SubiquityModel:
                 packages.extend(await meth())
         return packages
 
+    async def live_packages(self):
+        packages = []
+        for model_name in self._install_model_names.all():
+            meth = getattr(getattr(self, model_name), "live_packages", None)
+            if meth is not None:
+                packages.extend(await meth())
+        return packages
+
     def _cloud_init_files(self):
         # TODO, this should be moved to the in-target cloud-config seed so on
         # first boot of the target, it reconfigures datasource_list to none
diff --git a/subiquity/server/controllers/install.py b/subiquity/server/controllers/install.py
index aeee3ff9..d5e62c68 100644
--- a/subiquity/server/controllers/install.py
+++ b/subiquity/server/controllers/install.py
@@ -597,6 +597,11 @@ class InstallController(SubiquityController):
         with open(self.tpath("etc/fstab"), "w") as fp:
             fp.write("/run/mnt/ubuntu-boot/EFI/ubuntu /boot/grub none bind\n")
 
+    @with_context(description="installing packages to live system")
+    async def install_live_packages(self, *, context):
+        for package in await self.model.live_packages():
+            await self.app.package_installer.install_pkg(package)
+
     @with_context()
     async def install(self, *, context):
         context.set("is-install-context", True)
@@ -627,8 +632,7 @@ class InstallController(SubiquityController):
                 fsc = self.app.controllers.Filesystem
                 for_install_path = self.model.source.get_source(fsc._info.name)
 
-            if self.app.base_model.filesystem.reset_partition:
-                self.app.package_installer.start_installing_pkg("efibootmgr")
+            await self.install_live_packages()
 
             if self.model.target is not None:
                 if os.path.exists(self.model.target):

From 96c0c91d8a5a21d8e3e190051f3557c7fd2f3829 Mon Sep 17 00:00:00 2001
From: Dan Bungert <daniel.bungert@canonical.com>
Date: Fri, 25 Aug 2023 16:18:43 -0600
Subject: [PATCH 3/7] storage: trigger zfsutils-linux install if zpools

pre-Mantic installs can now zfsroot without autoinstall workarounds.
---
 subiquity/models/filesystem.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/subiquity/models/filesystem.py b/subiquity/models/filesystem.py
index 93a6ccbc..b8d4de21 100644
--- a/subiquity/models/filesystem.py
+++ b/subiquity/models/filesystem.py
@@ -2043,4 +2043,6 @@ class FilesystemModel(object):
         r = []
         if self.reset_partition is not None:
             r.append("efibootmgr")
+        if self._one(type="zpool") is not None:
+            r.append("zfsutils-linux")
         return r

From 67102261c38946bef096ba400366a7e2eb1f38ef Mon Sep 17 00:00:00 2001
From: Dan Bungert <daniel.bungert@canonical.com>
Date: Mon, 28 Aug 2023 10:12:09 -0600
Subject: [PATCH 4/7] install: live pkgs can be installed in parallel

Split packages that must be done early from ones that can be done
in parallel with early install steps.
---
 subiquity/models/filesystem.py            | 15 ++++++-----
 subiquity/models/subiquity.py             | 17 ++++++++----
 subiquity/models/tests/test_filesystem.py | 32 +++++++++++++++++++++++
 subiquity/server/controllers/install.py   | 16 +++++++++---
 4 files changed, 64 insertions(+), 16 deletions(-)

diff --git a/subiquity/models/filesystem.py b/subiquity/models/filesystem.py
index b8d4de21..f8bbe60a 100644
--- a/subiquity/models/filesystem.py
+++ b/subiquity/models/filesystem.py
@@ -24,7 +24,7 @@ import pathlib
 import platform
 import tempfile
 from abc import ABC, abstractmethod
-from typing import List, Optional, Set, Union
+from typing import List, Optional, Set, Tuple, Union
 
 import attr
 import more_itertools
@@ -2039,10 +2039,11 @@ class FilesystemModel(object):
         self._actions.append(zpool)
         return zpool
 
-    async def live_packages(self):
-        r = []
-        if self.reset_partition is not None:
-            r.append("efibootmgr")
+    async def live_packages(self) -> Tuple[Set, Set]:
+        before = set()
+        during = set()
         if self._one(type="zpool") is not None:
-            r.append("zfsutils-linux")
-        return r
+            before.add("zfsutils-linux")
+        if self.reset_partition is not None:
+            during.add("efibootmgr")
+        return (before, during)
diff --git a/subiquity/models/subiquity.py b/subiquity/models/subiquity.py
index a3077439..0ff2283a 100644
--- a/subiquity/models/subiquity.py
+++ b/subiquity/models/subiquity.py
@@ -20,7 +20,7 @@ import logging
 import os
 import uuid
 from collections import OrderedDict
-from typing import Any, Dict, Set
+from typing import Any, Dict, Set, Tuple
 
 import yaml
 from cloudinit.config.schema import (
@@ -390,13 +390,20 @@ class SubiquityModel:
                 packages.extend(await meth())
         return packages
 
-    async def live_packages(self):
-        packages = []
+    async def live_packages(self) -> Tuple[Set, Set]:
+        """return a tuple of sets of packages to install into the live environment.
+        The first set must be installed before partitioning, the second set may be
+        allowed to run in parallel with partitioning.
+        """
+        before = set()
+        during = set()
         for model_name in self._install_model_names.all():
             meth = getattr(getattr(self, model_name), "live_packages", None)
             if meth is not None:
-                packages.extend(await meth())
-        return packages
+                packages = await meth()
+                before |= packages[0]
+                during |= packages[1]
+        return (before, during)
 
     def _cloud_init_files(self):
         # TODO, this should be moved to the in-target cloud-config seed so on
diff --git a/subiquity/models/tests/test_filesystem.py b/subiquity/models/tests/test_filesystem.py
index f60aa4fd..b5e18c22 100644
--- a/subiquity/models/tests/test_filesystem.py
+++ b/subiquity/models/tests/test_filesystem.py
@@ -1526,3 +1526,35 @@ class TestRootfs(SubiTestCase):
         m = make_model()
         make_zpool(model=m, mountpoint="/srv")
         self.assertFalse(m.is_root_mounted())
+
+
+class TestLivePackages(SubiTestCase):
+    async def test_defaults(self):
+        m = make_model()
+        (before, during) = await m.live_packages()
+        self.assertEqual(set(), before)
+        self.assertEqual(set(), during)
+
+    async def test_zfs(self):
+        m = make_model()
+        make_zpool(model=m, mountpoint="/")
+        (before, during) = await m.live_packages()
+        self.assertEqual(set(["zfsutils-linux"]), before)
+        self.assertEqual(set(), during)
+
+    async def test_efibootmgr(self):
+        m = make_model()
+        d = make_disk(m)
+        m.reset_partition = make_partition(m, d)
+        (before, during) = await m.live_packages()
+        self.assertEqual(set(), before)
+        self.assertEqual(set(["efibootmgr"]), during)
+
+    async def test_both(self):
+        m = make_model()
+        d = make_disk(m)
+        make_zpool(model=m, mountpoint="/")
+        m.reset_partition = make_partition(m, d)
+        (before, during) = await m.live_packages()
+        self.assertEqual(set(["zfsutils-linux"]), before)
+        self.assertEqual(set(["efibootmgr"]), during)
diff --git a/subiquity/server/controllers/install.py b/subiquity/server/controllers/install.py
index d5e62c68..7eccf77e 100644
--- a/subiquity/server/controllers/install.py
+++ b/subiquity/server/controllers/install.py
@@ -597,10 +597,18 @@ class InstallController(SubiquityController):
         with open(self.tpath("etc/fstab"), "w") as fp:
             fp.write("/run/mnt/ubuntu-boot/EFI/ubuntu /boot/grub none bind\n")
 
-    @with_context(description="installing packages to live system")
     async def install_live_packages(self, *, context):
-        for package in await self.model.live_packages():
-            await self.app.package_installer.install_pkg(package)
+        before, during = await self.model.live_packages()
+        if len(before) < 1 and len(during) < 1:
+            return
+
+        with context.child("live-packages", "installing packages to live system"):
+            for package in before:
+                state = await self.app.package_installer.install_pkg(package)
+                if state != PackageInstallState.DONE:
+                    raise RuntimeError(f"could not install {package}")
+            for package in during:
+                self.app.package_installer.start_installing_pkg(package)
 
     @with_context()
     async def install(self, *, context):
@@ -632,7 +640,7 @@ class InstallController(SubiquityController):
                 fsc = self.app.controllers.Filesystem
                 for_install_path = self.model.source.get_source(fsc._info.name)
 
-            await self.install_live_packages()
+            await self.install_live_packages(context=context)
 
             if self.model.target is not None:
                 if os.path.exists(self.model.target):

From 18710aef489028b55ec99a422fc0bc0e60869968 Mon Sep 17 00:00:00 2001
From: Dan Bungert <daniel.bungert@canonical.com>
Date: Mon, 28 Aug 2023 10:48:31 -0600
Subject: [PATCH 5/7] runtests: +zfs

---
 examples/autoinstall/interactive.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/examples/autoinstall/interactive.yaml b/examples/autoinstall/interactive.yaml
index 56e20bde..67036276 100644
--- a/examples/autoinstall/interactive.yaml
+++ b/examples/autoinstall/interactive.yaml
@@ -26,5 +26,8 @@ packages:
 snaps:
   - name: etcd
     channel: 3.2/stable
+storage:
+  layout:
+    name: zfs
 debconf-selections: |
   wtf wtf

From ad881dc9487f17a5ce5e1a78890720d3c925a44a Mon Sep 17 00:00:00 2001
From: Dan Bungert <daniel.bungert@canonical.com>
Date: Mon, 28 Aug 2023 10:48:38 -0600
Subject: [PATCH 6/7] pkghelper: dry run handling

---
 subiquity/server/pkghelper.py | 7 ++++++-
 subiquity/server/server.py    | 2 +-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/subiquity/server/pkghelper.py b/subiquity/server/pkghelper.py
index bd2c270e..d27521d5 100644
--- a/subiquity/server/pkghelper.py
+++ b/subiquity/server/pkghelper.py
@@ -34,9 +34,10 @@ class PackageInstaller:
     by the server installer.
     """
 
-    def __init__(self):
+    def __init__(self, app):
         self.pkgs: Dict[str, asyncio.Task] = {}
         self._cache: Optional[apt.Cache] = None
+        self.app = app
 
     @property
     def cache(self):
@@ -70,6 +71,10 @@ class PackageInstaller:
         if binpkg.installed:
             log.debug("%s already installed", pkgname)
             return PackageInstallState.DONE
+        if self.app.opts.dry_run:
+            log.debug("dry-run apt-get install %s", pkgname)
+            await asyncio.sleep(2 / self.app.scale_factor)
+            return PackageInstallState.DONE
         if not binpkg.candidate.uri.startswith("cdrom:"):
             log.debug(
                 "%s not available from cdrom (rather %s)", pkgname, binpkg.candidate.uri
diff --git a/subiquity/server/server.py b/subiquity/server/server.py
index 509fa948..5550b8e5 100644
--- a/subiquity/server/server.py
+++ b/subiquity/server/server.py
@@ -291,7 +291,7 @@ class SubiquityServer(Application):
         self.event_syslog_id = "subiquity_event.{}".format(os.getpid())
         self.log_syslog_id = "subiquity_log.{}".format(os.getpid())
         self.command_runner = get_command_runner(self)
-        self.package_installer = PackageInstaller()
+        self.package_installer = PackageInstaller(self)
 
         self.error_reporter = ErrorReporter(
             self.context.child("ErrorReporter"), self.opts.dry_run, self.root

From 0bbec68bd9419220e2c727d7fb979c2d620f3ae9 Mon Sep 17 00:00:00 2001
From: Dan Bungert <daniel.bungert@canonical.com>
Date: Mon, 28 Aug 2023 12:07:28 -0600
Subject: [PATCH 7/7] installdeps: use confnew to workaround upgrade

---
 scripts/installdeps.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scripts/installdeps.sh b/scripts/installdeps.sh
index 759f6087..753b019a 100755
--- a/scripts/installdeps.sh
+++ b/scripts/installdeps.sh
@@ -3,7 +3,10 @@ set -eux
 cd "$(dirname ${BASH_SOURCE:0})/.."
 
 apt-get update
-DEBIAN_FRONTEND=noninteractive apt-get -o APT::Get::Always-Include-Phased-Updates=true -y dist-upgrade
+DEBIAN_FRONTEND=noninteractive apt-get \
+    -o Dpkg::Options::=--force-confnew \
+    -o APT::Get::Always-Include-Phased-Updates=true \
+    -y dist-upgrade
 mkdir -p /etc/systemd/system/zfs-mount.service.d/
 cat >/etc/systemd/system/zfs-mount.service.d/override.conf <<EOF
 [Unit]