codecs: skip installation when running an offline install

ubuntu-restricted-addons is a multiverse package and is not included in
the pool. Therefore, trying to get it installed when offline leads to an
obvious error.

Instead of making the whole Ubuntu installation fail, we now warn and
skip installation of the package when performing an offline install.
In a perfect world, we should not have offered to install the package in
the first place, but in practice, we can run an offline installation as
the result of failed mirror testing (bad network for instance).

Signed-off-by: Olivier Gayot <olivier.gayot@canonical.com>
(cherry picked from commit 01ec1da86f)

examples/autoinstall/fallback-offline.yaml

@@ -0,0 +1,21 @@
+version: 1
+locale: en_GB.UTF-8
+network:
+  version: 2
+  ethernets:
+    all-eth:
+      match:
+        name: "en*"
+      dhcp6: yes
+apt:
+  mirror-selection:
+    primary:
+    - uri: http://localhost/failed
+  fallback: offline-install
+codecs:
+  install: true
+identity:
+  realname: ''
+  username: ubuntu
+  password: '$6$wdAcoXrU039hKYPd$508Qvbe7ObUnxoj15DRCkzC3qO7edjH0VV7BPNRDYK4QR8ofJaEEF2heacn0QgD.f8pO8SNp83XNdWG6tocBM1'
+  hostname: ubuntu

scripts/runtests.sh

@@ -68,6 +68,9 @@ validate () {
                     apt.security[1].uri='"http://ports.ubuntu.com/ubuntu-ports"'
                 ;;
         esac
+        if [ "$testname" == autoinstall-fallback-offline ]; then
+            grep -F -- 'skipping installation of package ubuntu-restricted-addons' "$tmpdir"/subiquity-server-debug.log
+        fi
         netplan generate --root $tmpdir
     elif [ "${mode}" = "system_setup" ]; then
         setup_mode="$2"
@@ -319,6 +322,18 @@ LANG=C.UTF-8 timeout --foreground 60 \
     --source-catalog examples/sources/install.yaml
 validate install
 
+clean
+testname=autoinstall-fallback-offline
+LANG=C.UTF-8 timeout --foreground 60 \
+    python3 -m subiquity.cmd.tui \
+    --dry-run \
+    --output-base "$tmpdir" \
+    --machine-config examples/machines/simple.json \
+    --autoinstall examples/autoinstall/fallback-offline.yaml \
+    --kernel-cmdline autoinstall \
+    --source-catalog examples/sources/install.yaml
+validate install
+
 # The OOBE doesn't exist in WSL < 20.04
 if [ "${RELEASE%.*}" -ge 20 ]; then
     # Test TCP connectivity (system_setup only)

subiquity/common/pkg.py

@@ -0,0 +1,25 @@
+# Copyright 2023 Canonical, Ltd.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import attr
+
+
+@attr.s(auto_attribs=True)
+class TargetPkg:
+    name: str
+    # Some packages are not present in the pool and require a working network
+    # connection to be downloaded. By marking them with "skip_when_offline", we
+    # can skip them when running an offline install.
+    skip_when_offline: bool

subiquity/models/ad.py

@@ -14,8 +14,9 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import logging
-from typing import Optional
+from typing import List, Optional
 
+from subiquity.common.pkg import TargetPkg
 from subiquity.common.types import AdConnectionInfo
 
 log = logging.getLogger("subiquity.models.ad")
@@ -42,10 +43,14 @@ class AdModel:
         else:
             self.conn_info = AdConnectionInfo(domain_name=domain)
 
-    async def target_packages(self):
+    async def target_packages(self) -> List[TargetPkg]:
         # NOTE Those packages must be present in the target system to allow
         # joining to a domain.
         if self.do_join:
-            return ["adcli", "realmd", "sssd"]
+            return [
+                TargetPkg(name="adcli", skip_when_offline=False),
+                TargetPkg(name="realmd", skip_when_offline=False),
+                TargetPkg(name="sssd", skip_when_offline=False),
+            ]
 
         return []

subiquity/models/codecs.py

@@ -14,6 +14,9 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import logging
+from typing import List
+
+from subiquity.common.pkg import TargetPkg
 
 log = logging.getLogger("subiquity.models.codecs")
 
@@ -21,9 +24,12 @@ log = logging.getLogger("subiquity.models.codecs")
 class CodecsModel:
     do_install = False
 
-    async def target_packages(self):
+    async def target_packages(self) -> List[TargetPkg]:
         # NOTE currently, ubuntu-restricted-addons is an empty package that
         # pulls relevant packages through Recommends: Ideally, we should make
         # sure to run the APT command for this package with the
         # --install-recommends option.
-        return ["ubuntu-restricted-addons"] if self.do_install else []
+        if not self.do_install:
+            return []
+
+        return [TargetPkg(name="ubuntu-restricted-addons", skip_when_offline=True)]

subiquity/models/locale.py

@@ -16,7 +16,9 @@
 import locale
 import logging
 import subprocess
+from typing import List
 
+from subiquity.common.pkg import TargetPkg
 from subiquitycore.utils import arun_command, split_cmd_output
 
 log = logging.getLogger("subiquity.models.locale")
@@ -60,7 +62,7 @@ class LocaleModel:
             locale += ".UTF-8"
         return {"locale": locale}
 
-    async def target_packages(self):
+    async def target_packages(self) -> List[TargetPkg]:
         if self.selected_language is None:
             return []
         if self.locale_support != "langpack":
@@ -69,6 +71,7 @@ class LocaleModel:
         if lang == "C":
             return []
 
-        return await split_cmd_output(
+        pkgs = await split_cmd_output(
             self.chroot_prefix + ["check-language-support", "-l", lang], None
         )
+        return [TargetPkg(name=pkg, skip_when_offline=False) for pkg in pkgs]

subiquity/models/network.py

@@ -15,8 +15,10 @@
 
 import logging
 import subprocess
+from typing import List
 
 from subiquity import cloudinit
+from subiquity.common.pkg import TargetPkg
 from subiquitycore.models.network import NetworkModel as CoreNetworkModel
 from subiquitycore.utils import arun_command
 
@@ -93,12 +95,12 @@ class NetworkModel(CoreNetworkModel):
                 }
         return r
 
-    async def target_packages(self):
+    async def target_packages(self) -> List[TargetPkg]:
-        if self.needs_wpasupplicant:
+        if not self.needs_wpasupplicant:
-            return ["wpasupplicant"]
-        else:
             return []
 
+        return [TargetPkg(name="wpasupplicant", skip_when_offline=False)]
+
     async def is_nm_enabled(self):
         try:
             cp = await arun_command(("nmcli", "networking"), check=True)

subiquity/models/ssh.py

@@ -16,6 +16,8 @@
 import logging
 from typing import List
 
+from subiquity.common.pkg import TargetPkg
+
 log = logging.getLogger("subiquity.models.ssh")
 
 
@@ -29,8 +31,8 @@ class SSHModel:
         # we go back to it.
         self.ssh_import_id = ""
 
-    async def target_packages(self):
+    async def target_packages(self) -> List[TargetPkg]:
-        if self.install_server:
+        if not self.install_server:
-            return ["openssh-server"]
-        else:
             return []
+
+        return [TargetPkg(name="openssh-server", skip_when_offline=False)]

subiquity/models/subiquity.py

@@ -20,7 +20,7 @@ import logging
 import os
 import uuid
 from collections import OrderedDict
-from typing import Any, Dict, Set, Tuple
+from typing import Any, Dict, List, Set, Tuple
 
 import yaml
 from cloudinit.config.schema import (
@@ -39,6 +39,7 @@ except ImportError:
 
 from curtin.config import merge_config
 
+from subiquity.common.pkg import TargetPkg
 from subiquity.common.resources import get_users_and_groups
 from subiquity.server.types import InstallerChannels
 from subiquitycore.file_util import generate_timestamped_header, write_file
@@ -177,6 +178,9 @@ class SubiquityModel:
             self.chroot_prefix = []
 
         self.active_directory = AdModel()
+        # List of packages that will be installed using cloud-init on first
+        # boot.
+        self.cloud_init_packages: List[str] = []
         self.codecs = CodecsModel()
         self.debconf_selections = DebconfSelectionsModel()
         self.drivers = DriversModel()
@@ -189,7 +193,7 @@ class SubiquityModel:
         self.mirror = MirrorModel()
         self.network = NetworkModel()
         self.oem = OEMModel()
-        self.packages = []
+        self.packages: List[TargetPkg] = []
         self.proxy = ProxyModel()
         self.snaplist = SnapListModel()
         self.ssh = SSHModel()
@@ -376,13 +380,15 @@ class SubiquityModel:
             model = getattr(self, model_name)
             if getattr(model, "make_cloudconfig", None):
                 merge_config(config, model.make_cloudconfig())
+        for package in self.cloud_init_packages:
+            merge_config(config, {"packages": list(self.cloud_init_packages)})
         merge_cloud_init_config(config, self.userdata)
         if lsb_release()["release"] not in ("20.04", "22.04"):
             config.setdefault("write_files", []).append(CLOUDINIT_DISABLE_AFTER_INSTALL)
         self.validate_cloudconfig_schema(data=config, data_source="system install")
         return config
 
-    async def target_packages(self):
+    async def target_packages(self) -> List[TargetPkg]:
         packages = list(self.packages)
         for model_name in self._postinstall_model_names.all():
             meth = getattr(getattr(self, model_name), "target_packages", None)

subiquity/server/controllers/install.py

@@ -34,6 +34,7 @@ from curtin.config import merge_config
 from curtin.util import get_efibootmgr, is_uefi_bootable
 
 from subiquity.common.errorreport import ErrorReportKind
+from subiquity.common.pkg import TargetPkg
 from subiquity.common.types import ApplicationState, PackageInstallState
 from subiquity.journald import journald_listen
 from subiquity.models.filesystem import ActionRenderMode, Partition
@@ -689,11 +690,22 @@ class InstallController(SubiquityController):
             {"autoinstall": self.app.make_autoinstall()}
         )
         write_file(autoinstall_path, autoinstall_config)
-        await self.configure_cloud_init(context=context)
+        try:
             if self.supports_apt():
                 packages = await self.get_target_packages(context=context)
                 for package in packages:
-                await self.install_package(context=context, package=package)
+                    if package.skip_when_offline and not self.model.network.has_network:
+                        log.warning(
+                            "skipping installation of package %s when"
+                            " performing an offline install.",
+                            package.name,
+                        )
+                        continue
+                    await self.install_package(context=context, package=package.name)
+        finally:
+            await self.configure_cloud_init(context=context)
+
+        if self.supports_apt():
             if self.model.drivers.do_install:
                 with context.child(
                     "ubuntu-drivers-install", "installing third-party drivers"
@@ -720,7 +732,7 @@ class InstallController(SubiquityController):
         await run_in_thread(self.model.configure_cloud_init)
 
     @with_context(description="calculating extra packages to install")
-    async def get_target_packages(self, context):
+    async def get_target_packages(self, context) -> List[TargetPkg]:
         return await self.app.base_model.target_packages()
 
     @with_context(name="install_{package}", description="installing {package}")

subiquity/server/controllers/package.py

@@ -13,6 +13,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from subiquity.common.pkg import TargetPkg
 from subiquity.server.controller import NonInteractiveController
 
 
@@ -25,7 +26,7 @@ class PackageController(NonInteractiveController):
     }
 
     def load_autoinstall_data(self, data):
-        self.model[:] = data
+        self.model[:] = [TargetPkg(name=pkg, skip_when_offline=False) for pkg in data]
 
     def make_autoinstall(self):
-        return self.model
+        return [pkg.name for pkg in self.model]

subiquity/tests/api/test_api.py

@@ -2133,7 +2133,7 @@ class TestActiveDirectory(TestAPI):
         to be installed in the target system and whether they were
         referred to or not in the server log."""
         expected_packages = await self.target_packages()
-        packages_lookup = {p: False for p in expected_packages}
+        packages_lookup = {p.name: False for p in expected_packages}
         log_path = os.path.join(log_dir, "subiquity-server-debug.log")
         find_start = "finish: subiquity/Install/install/postinstall/install_{}:"
         log_status = " SUCCESS: installing {}"