create a helper class for invoking curtin

scripts/runtests.sh

@@ -16,7 +16,7 @@ validate () {
     fi
 
     if [ "${mode}" = "install" ]; then
-        python3 scripts/validate-yaml.py .subiquity/subiquity-curtin-install.conf
+        python3 scripts/validate-yaml.py .subiquity/var/log/installer/subiquity-curtin-install.conf
         if [ ! -e .subiquity/subiquity-client-debug.log ] || [ ! -e .subiquity/subiquity-server-debug.log ]; then
             echo "log file not created"
             exit 1
@@ -85,7 +85,7 @@ validate () {
 }
 
 clean () {
-    rm -f .subiquity/subiquity-curtin-install.conf
+    rm -rf .subiquity/var/log/
     rm -f .subiquity/subiquity-*.log
     rm -f "$testschema"
     rm -rf .subiquity/run/
@@ -144,7 +144,7 @@ timeout --foreground 60 sh -c "LANG=C.UTF-8 python3 -m subiquity.cmd.tui --autoi
                                --kernel-cmdline 'autoinstall' \
                                --source-catalog=examples/install-sources.yaml"
 validate
-python3 scripts/check-yaml-fields.py .subiquity/subiquity-curtin-install.conf \
+python3 scripts/check-yaml-fields.py .subiquity/var/log/installer/subiquity-curtin-install.conf \
         debconf_selections.subiquity='"eek"' \
         storage.config[-1].options='"errors=remount-ro"'
 python3 scripts/check-yaml-fields.py <(python3 scripts/check-yaml-fields.py .subiquity/etc/cloud/cloud.cfg.d/99-installer.cfg datasource.None.userdata_raw) \

subiquity/journald.py

@@ -38,10 +38,10 @@ def journald_listen(loop, identifiers, callback, seek=False):
 
 
 @contextlib.contextmanager
-def journald_subscriptions(loop, ids_callbacks):
+def journald_subscriptions(loop, ids_callbacks, seek=False):
     fds = set()
     for id, callback in ids_callbacks:
-        fds.add(journald_listen(loop, [id], callback))
+        fds.add(journald_listen(loop, [id], callback, seek=seek))
     try:
         yield
     finally:

subiquity/server/controllers/install.py

@@ -15,6 +15,7 @@
 
 import asyncio
 import datetime
+import json
 import logging
 import os
 import re
@@ -45,7 +46,9 @@ from subiquity.server.controller import (
 from subiquity.common.types import (
     ApplicationState,
     )
-from subiquity.journald import journald_subscriptions
+from subiquity.journald import (
+    journald_listen,
+    )
 
 log = logging.getLogger("subiquity.server.controllers.install")
 
@@ -108,35 +111,17 @@ class DryRunCommandRunner(LoggedCommandRunner):
         return proc
 
 
-class InstallController(SubiquityController):
+class CurtinCommandRunner:
 
-    def __init__(self, app):
+    def __init__(self, runner, event_syslog_id, config_location):
-        super().__init__(app)
+        self.runner = runner
-        self.model = app.base_model
+        self.event_syslog_id = event_syslog_id
+        self.config_location = config_location
+        self._event_contexts = {}
+        journald_listen(
+            asyncio.get_event_loop(), [event_syslog_id], self._event)
 
-        self.unattended_upgrades_proc = None
+    def _event(self, event):
-        self.unattended_upgrades_ctx = None
-        self._event_syslog_id = 'curtin_event.%s' % (os.getpid(),)
-        self.tb_extractor = TracebackExtractor()
-        self.curtin_event_contexts = {}
-        if self.app.opts.dry_run:
-            self.command_runner = DryRunCommandRunner(
-                self.app.log_syslog_id, 2/self.app.scale_factor)
-        else:
-            self.command_runner = LoggedCommandRunner(self.app.log_syslog_id)
-
-    def stop_uu(self):
-        if self.app.state == ApplicationState.UU_RUNNING:
-            self.app.update_state(ApplicationState.UU_CANCELLING)
-            self.app.aio_loop.create_task(self.stop_unattended_upgrades())
-
-    def start(self):
-        self.install_task = self.app.aio_loop.create_task(self.install())
-
-    def tpath(self, *path):
-        return os.path.join(self.model.target, *path)
-
-    def curtin_event(self, event):
         e = {
             "EVENT_TYPE": "???",
             "MESSAGE": "???",
@@ -156,82 +141,131 @@ class InstallController(SubiquityController):
 
             curtin_ctx = None
             for pre, post in p(e["NAME"]):
-                if pre in self.curtin_event_contexts:
+                if pre in self._event_contexts:
-                    parent = self.curtin_event_contexts[pre]
+                    parent = self._event_contexts[pre]
                     curtin_ctx = parent.child(post, e["MESSAGE"])
-                    self.curtin_event_contexts[e["NAME"]] = curtin_ctx
+                    self._event_contexts[e["NAME"]] = curtin_ctx
                     break
             if curtin_ctx:
                 curtin_ctx.enter()
         if event_type == 'finish':
             status = getattr(Status, e["RESULT"], Status.WARN)
-            curtin_ctx = self.curtin_event_contexts.pop(e["NAME"], None)
+            curtin_ctx = self._event_contexts.pop(e["NAME"], None)
             if curtin_ctx is not None:
                 curtin_ctx.exit(result=status)
 
+    def make_command(self, command, *args, **conf):
+        cmd = [
+            sys.executable, '-m', 'curtin', '--showtrace',
+            '-c', self.config_location,
+            ]
+        for k, v in conf.items():
+            cmd.extend(['--set', 'json:' + k + '=' + json.dumps(v)])
+        cmd.append(command)
+        cmd.extend(args)
+        return cmd
+
+    async def run(self, context, command, *args, **conf):
+        self._event_contexts[''] = context
+        await self.runner.run(self.make_command(command, *args, **conf))
+        waited = 0.0
+        while len(self._event_contexts) > 1 and waited < 5.0:
+            await asyncio.sleep(0.1)
+            waited += 0.1
+            log.debug("waited %s seconds for events to drain", waited)
+        self._event_contexts.pop('', None)
+
+
+class DryRunCurtinCommandRunner(CurtinCommandRunner):
+
+    event_file = 'examples/curtin-events.json'
+
+    def make_command(self, command, *args, **conf):
+        if command == 'install':
+            return [
+                sys.executable, "scripts/replay-curtin-log.py",
+                self.event_file, self.event_syslog_id,
+                '.subiquity' + INSTALL_LOG,
+                ]
+        else:
+            return super().make_command(command, *args, **conf)
+
+
+class FailingDryRunCurtinCommandRunner(DryRunCurtinCommandRunner):
+
+    event_file = 'examples/curtin-events-fail.json'
+
+
+class InstallController(SubiquityController):
+
+    def __init__(self, app):
+        super().__init__(app)
+        self.model = app.base_model
+
+        self.unattended_upgrades_proc = None
+        self.unattended_upgrades_ctx = None
+        self._event_syslog_id = 'curtin_event.%s' % (os.getpid(),)
+        self.tb_extractor = TracebackExtractor()
+        if self.app.opts.dry_run:
+            self.command_runner = DryRunCommandRunner(
+                self.app.log_syslog_id, 2/self.app.scale_factor)
+        else:
+            self.command_runner = LoggedCommandRunner(self.app.log_syslog_id)
+        self.curtin_runner = None
+
+    def stop_uu(self):
+        if self.app.state == ApplicationState.UU_RUNNING:
+            self.app.update_state(ApplicationState.UU_CANCELLING)
+            self.app.aio_loop.create_task(self.stop_unattended_upgrades())
+
+    def start(self):
+        journald_listen(
+            self.app.aio_loop, [self.app.log_syslog_id], self.log_event)
+        self.install_task = self.app.aio_loop.create_task(self.install())
+
+    def tpath(self, *path):
+        return os.path.join(self.model.target, *path)
+
     def log_event(self, event):
         self.tb_extractor.feed(event['MESSAGE'])
 
-    def _write_config(self, path, config):
+    def make_curtin_command_runner(self):
-        with open(path, 'w') as conf:
+        config = self.model.render(syslog_identifier=self._event_syslog_id)
+        config_location = '/var/log/installer/subiquity-curtin-install.conf'
+        log_location = INSTALL_LOG
+        if self.app.opts.dry_run:
+            config_location = '.subiquity' + config_location
+            log_location = '.subiquity' + INSTALL_LOG
+        os.makedirs(os.path.dirname(config_location), exist_ok=True)
+        os.makedirs(os.path.dirname(log_location), exist_ok=True)
+        with open(config_location, 'w') as conf:
             datestr = '# Autogenerated by Subiquity: {} UTC\n'.format(
                 str(datetime.datetime.utcnow()))
             conf.write(datestr)
             conf.write(yaml.dump(config))
-
-    def _get_curtin_command(self):
-        config_file_name = 'subiquity-curtin-install.conf'
-
-        if self.app.opts.dry_run:
-            config_location = os.path.join('.subiquity/', config_file_name)
-            log_location = '.subiquity/install.log'
-            event_file = "examples/curtin-events.json"
-            if 'install-fail' in self.app.debug_flags:
-                event_file = "examples/curtin-events-fail.json"
-            curtin_cmd = [
-                sys.executable, "scripts/replay-curtin-log.py", event_file,
-                self._event_syslog_id, log_location,
-                ]
-        else:
-            config_location = os.path.join('/var/log/installer',
-                                           config_file_name)
-            curtin_cmd = [sys.executable, '-m', 'curtin', '--showtrace', '-c',
-                          config_location, 'install']
-            log_location = INSTALL_LOG
-
-        ident = self._event_syslog_id
-        self._write_config(config_location,
-                           self.model.render(syslog_identifier=ident))
-
         self.app.note_file_for_apport("CurtinConfig", config_location)
-        self.app.note_file_for_apport("CurtinLog", log_location)
         self.app.note_file_for_apport("CurtinErrors", ERROR_TARFILE)
-
+        self.app.note_file_for_apport("CurtinLog", log_location)
-        return curtin_cmd
+        if self.app.opts.dry_run:
+            if 'install-fail' in self.app.debug_flags:
+                cls = FailingDryRunCurtinCommandRunner
+            else:
+                cls = DryRunCurtinCommandRunner
+        else:
+            cls = CurtinCommandRunner
+        self.curtin_runner = cls(
+            self.command_runner, self._event_syslog_id, config_location)
 
     @with_context(description="umounting /target dir")
     async def unmount_target(self, *, context, target):
-        cmd = [
+        await self.curtin_runner.run(context, 'unmount', '-t', target)
-            sys.executable, '-m', 'curtin', 'unmount',
-            '-t', target,
-            ]
-        await self.command_runner.run(cmd)
         if not self.app.opts.dry_run:
             shutil.rmtree(target)
 
     @with_context(
         description="installing system", level="INFO", childlevel="DEBUG")
     async def curtin_install(self, *, context):
-        log.debug('curtin_install')
+        await self.curtin_runner.run(context, 'install')
-        self.curtin_event_contexts[''] = context
-
-        curtin_cmd = self._get_curtin_command()
-
-        log.debug('curtin install cmd: {}'.format(curtin_cmd))
-
-        cp = await self.command_runner.run(curtin_cmd)
-
-        log.debug('curtin_install completed: %s', cp.returncode)
 
     @with_context()
     async def install(self, *, context):
@@ -253,16 +287,13 @@ class InstallController(SubiquityController):
 
             self.app.update_state(ApplicationState.RUNNING)
 
+            self.make_curtin_command_runner()
+
             if os.path.exists(self.model.target):
                 await self.unmount_target(
                     context=context, target=self.model.target)
 
-            with journald_subscriptions(
-                    self.app.aio_loop,
-                    [(self.app.log_syslog_id, self.log_event),
-                     (self._event_syslog_id, self.curtin_event)]):
             await self.curtin_install(context=context)
-                await self.drain_curtin_events(context=context)
 
             self.app.update_state(ApplicationState.POST_WAIT)
 
@@ -281,14 +312,6 @@ class InstallController(SubiquityController):
                 ErrorReportKind.INSTALL_FAIL, "install failed", **kw)
             raise
 
-    async def drain_curtin_events(self, *, context):
-        waited = 0.0
-        while len(self.curtin_event_contexts) > 1 and waited < 5.0:
-            await asyncio.sleep(0.1)
-            waited += 0.1
-        log.debug("waited %s seconds for events to drain", waited)
-        self.curtin_event_contexts.pop('', None)
-
     @with_context(
         description="final system configuration", level="INFO",
         childlevel="DEBUG")
@@ -321,33 +344,21 @@ class InstallController(SubiquityController):
         name="install_{package}",
         description="installing {package}")
     async def install_package(self, *, context, package):
-        cmd = [
+        await self.curtin_runner.run(context, 'system-install', '--', package)
-            sys.executable, "-m", "curtin", "system-install", "-t",
-            "/target", "--", package,
-            ]
-        await self.command_runner.run(cmd)
 
     @with_context(description="restoring apt configuration")
     async def restore_apt_config(self, context):
-        cmds = [
+        await self.command_runner.run(["umount", self.tpath('etc/apt')])
-            ["umount", self.tpath('etc/apt')],
-            ]
         if self.model.network.has_network:
-            cmds.append([
+            await self.curtin_runner.run(
-                sys.executable, "-m", "curtin", "in-target", "-t",
+                context, "in-target", "-t", self.tpath(),
-                "/target", "--", "apt-get", "update",
+                "--", "apt-get", "update")
-                ])
         else:
-            cmds.append(["umount", self.tpath('var/lib/apt/lists')])
+            await self.command_runner.run(
-        for cmd in cmds:
+                ["umount", self.tpath('var/lib/apt/lists')])
-            await self.command_runner.run(cmd)
 
     @with_context(description="downloading and installing {policy} updates")
     async def run_unattended_upgrades(self, context, policy):
-        command = [
-            sys.executable, "-m", "curtin", "in-target",
-            "-t", "/target", "--", "unattended-upgrades", "-v",
-            ]
         if self.app.opts.dry_run:
             aptdir = self.tpath("tmp")
         else:
@@ -364,7 +375,9 @@ class InstallController(SubiquityController):
             apt_conf.close()
             self.unattended_upgrades_ctx = context
             self.unattended_upgrades_proc = await self.command_runner.start(
-                command)
+                self.curtin_runner.make_command(
+                    "in-target", "-t", self.tpath(),
+                    "--", "unattended-upgrades", "-v"))
             await self.unattended_upgrades_proc.communicate()
             self.unattended_upgrades_proc = None
             self.unattended_upgrades_ctx = None
@@ -374,7 +387,7 @@ class InstallController(SubiquityController):
                 "stop_unattended_upgrades",
                 "cancelling update"):
             await self.command_runner.run([
-                'chroot', '/target',
+                'chroot', self.tpath(),
                 '/usr/share/unattended-upgrades/'
                 'unattended-upgrade-shutdown',
                 '--stop-only',