Merge pull request #1649 from dbungert/lunar-2023-04-12

Lunar 2023 04 12

scripts/slimy-update-snap.sh

@@ -35,7 +35,9 @@ add_overlay() {
         local upper="$(mktemp -dp "${tmpdir}")"
     fi
     chmod go+rx "${work}" "${upper}"
-    do_mount -t overlay overlay -o lowerdir="${lower}",upperdir="${upper}",workdir="${work}" "${mountpoint}"
+    do_mount -t overlay overlay \
+        -o lowerdir="${lower}",upperdir="${upper}",workdir="${work}" \
+        "${mountpoint}"
 }
 
 
@@ -53,11 +55,22 @@ do_mount $old old
 add_overlay old new
 
 rm -rf new/lib/python3.10/site-packages/curtin
-rm -rf new/lib/python3.10/site-packages/subiquity
+
-rm -rf new/lib/python3.10/site-packages/subiquitycore
+if [ -d new/lib/python3.10/site-packages/subiquity ] ; then
+    subiquity_dest=new/lib/python3.10/site-packages
+elif [ -d new/bin/subiquity/subiquity ] ; then
+    subiquity_dest=new/bin/subiquity
+else
+    echo "unrecognized snap" >&2
+    exit 1
+fi
+
+rm -rf "${subiquity_dest}/subiquity"
+rm -rf "${subiquity_dest}/subiquitycore"
 
 (cd "${src}" && ./scripts/update-part.py curtin)
 
-rsync -a --chown 0:0 $src/subiquity $src/subiquitycore $src/curtin/curtin new/lib/python3.10/site-packages
+rsync -a --chown 0:0 $src/curtin/curtin new/lib/python3.10/site-packages
+rsync -a --chown 0:0 $src/subiquity $src/subiquitycore $subiquity_dest
 
 snapcraft pack new --output $new

snapcraft.yaml

@@ -216,7 +216,7 @@ parts:
 
     source: https://github.com/canonical/probert.git
     source-type: git
-    source-commit: 40479d08fce0370a0c41a140e6d322d2846c2a8f
+    source-commit: fdeb0908064ad8185bfa26a8cc881ea9bc2bf483
 
     override-build: *pyinstall
 

subiquity/cmd/server.py

@@ -172,6 +172,10 @@ def main():
             "InstallerServerLog", logfiles['debug'])
         server.note_file_for_apport(
             "InstallerServerLogInfo", logfiles['info'])
+        server.note_file_for_apport(
+            "UdiLog",
+            os.path.realpath(
+                "/var/log/installer/ubuntu_desktop_installer.log"))
         await server.run()
 
     asyncio.run(run_with_loop())

subiquity/server/controllers/install.py

@@ -34,6 +34,7 @@ from subiquitycore.async_helpers import (
     )
 from subiquitycore.context import with_context
 from subiquitycore.file_util import write_file, generate_config_yaml
+from subiquitycore.utils import log_process_streams
 
 from subiquity.common.errorreport import ErrorReportKind
 from subiquity.common.types import (
@@ -480,7 +481,11 @@ class InstallController(SubiquityController):
                 self.app, context, "in-target", "-t", self.tpath(),
                 "--", "unattended-upgrades", "-v",
                 private_mounts=True)
-            await self.unattended_upgrades_cmd.wait()
+            try:
+                await self.unattended_upgrades_cmd.wait()
+            except subprocess.CalledProcessError as cpe:
+                log_process_streams(logging.ERROR, cpe, 'Unattended upgrades')
+                context.description = f"FAILED to apply {policy} updates"
             self.unattended_upgrades_cmd = None
             self.unattended_upgrades_ctx = None
 

subiquity/server/controllers/mirror.py

@@ -155,6 +155,7 @@ class MirrorController(SubiquityController):
             self._promote_mirror)
         self.apt_configurer = None
         self.mirror_check: Optional[MirrorCheck] = None
+        self.autoinstall_apply_started = False
 
     def load_autoinstall_data(self, data):
         if data is None:
@@ -273,6 +274,7 @@ class MirrorController(SubiquityController):
 
     @with_context()
     async def apply_autoinstall_config(self, context):
+        self.autoinstall_apply_started = True
         await self.run_mirror_selection_or_fallback(context)
 
     def on_geoip(self):
@@ -281,6 +283,12 @@ class MirrorController(SubiquityController):
         self.cc_event.set()
 
     def on_source(self):
+        if self.autoinstall_apply_started:
+            # Alternatively, we should cancel and restart the
+            # apply_autoinstall_config but this is out of scope.
+            raise RuntimeError("source model has changed but autoinstall"
+                               " configuration is already being applied")
+
         # FIXME disabled until we can sort out umount
         # if self.apt_configurer is not None:
         #     await self.apt_configurer.cleanup()
@@ -323,8 +331,14 @@ class MirrorController(SubiquityController):
 
     async def run_mirror_testing(self, output: io.StringIO) -> None:
         await self.source_configured_event.wait()
-        await self.apt_configurer.apply_apt_config(self.context, final=False)
+        # If the source model changes at the wrong time, there is a chance that
-        await self.apt_configurer.run_apt_config_check(output)
+        # self.apt_configurer will be replaced between the call to
+        # apply_apt_config and run_apt_config_check. Just make sure we still
+        # use the original one.
+        configurer = self.apt_configurer
+        await configurer.apply_apt_config(
+            self.context, final=False)
+        await configurer.run_apt_config_check(output)
 
     async def wait_config(self):
         await self._apply_apt_config_task.wait()

subiquity/server/runner.py

@@ -81,7 +81,8 @@ class LoggedCommandRunner:
         # .communicate() forces returncode to be set to a value
         assert proc.returncode is not None
         if proc.returncode != 0:
-            raise subprocess.CalledProcessError(proc.returncode, proc.args)
+            raise subprocess.CalledProcessError(
+                proc.returncode, proc.args, output=stdout, stderr=stderr)
         else:
             return subprocess.CompletedProcess(
                 proc.args, proc.returncode, stdout=stdout, stderr=stderr)

subiquity/tests/api/test_api.py

@@ -265,13 +265,14 @@ async def start_server_factory(factory, *args, **kwargs):
 
 
 @contextlib.asynccontextmanager
-async def start_server(*args, **kwargs):
+async def start_server(*args, set_first_source=True, **kwargs):
     async with start_server_factory(Server, *args, **kwargs) as instance:
-        sources = await instance.get('/source')
+        if set_first_source:
-        if sources is None:
+            sources = await instance.get('/source')
-            raise Exception('unexpected /source response')
+            if sources is None:
-        await instance.post(
+                raise Exception('unexpected /source response')
-            '/source', source_id=sources['sources'][0]['id'])
+            await instance.post(
+                '/source', source_id=sources['sources'][0]['id'])
         while True:
             resp = await instance.get('/storage/v2')
             print(resp)
@@ -1653,7 +1654,8 @@ class TestAutoinstallServer(TestAPI):
             '--autoinstall', 'examples/autoinstall-short.yaml',
             '--source-catalog', 'examples/install-sources.yaml',
         ]
-        async with start_server(cfg, extra_args=extra) as inst:
+        async with start_server(cfg, extra_args=extra,
+                                set_first_source=False) as inst:
             view_request_unspecified, resp = await inst.get(
                     '/locale',
                     full_response=True)
@@ -1798,7 +1800,8 @@ class TestActiveDirectory(TestAPI):
             '--kernel-cmdline', 'autoinstall',
         ]
         try:
-            async with start_server(cfg, extra_args=extra) as inst:
+            async with start_server(cfg, extra_args=extra,
+                                    set_first_source=False) as inst:
                 endpoint = '/active_directory'
                 logdir = inst.output_base()
                 self.assertIsNotNone(logdir)

subiquitycore/utils.py

@@ -142,6 +142,26 @@ def start_command(cmd: Sequence[str], *,
                             env=_clean_env(env, locale=clean_locale), **kw)
 
 
+def _log_stream(level: int, stream, name: str):
+    if stream:
+        log.log(level, f'{name}: ------------------------------------------')
+        for line in stream.splitlines():
+            log.log(level, line)
+    elif stream is None:
+        log.log(level, f'<{name} is None>')
+    else:
+        log.log(level, f'<{name} is empty>')
+
+
+def log_process_streams(level: int,
+                        cpe: subprocess.CalledProcessError,
+                        command_msg: str):
+    log.log(level, f'{command_msg} exited with result: {cpe.returncode}')
+    _log_stream(level, cpe.stdout, 'stdout')
+    _log_stream(level, cpe.stderr, 'stderr')
+    log.log(level, '--------------------------------------------------')
+
+
 # FIXME: replace with passlib and update package deps
 def crypt_password(passwd, algo='SHA-512'):
     # encryption algo - id pairs for crypt()