split subiquity.core into subiquity.client.client and subiquity.server.server

po/POTFILES.in

@@ -1,4 +1,7 @@
 [encoding: UTF-8]
+subiquity/client/client.py
+subiquity/client/__init__.py
+subiquity/client/keycodes.py
 subiquity/cmd/common.py
 subiquity/cmd/__init__.py
 subiquity/cmd/schema.py
@@ -61,7 +64,6 @@ subiquitycore/models/network.py
 subiquitycore/netplan.py
 subiquitycore/palette.py
 subiquitycore/prober.py
-subiquity/core.py
 subiquitycore/screen.py
 subiquitycore/signals.py
 subiquitycore/snapd.py
@@ -100,7 +102,6 @@ subiquitycore/view.py
 subiquity/__init__.py
 subiquity/journald.py
 subiquity/keyboard.py
-subiquity/keycodes.py
 subiquity/lockfile.py
 subiquity/__main__.py
 subiquity/models/filesystem.py

subiquity/client/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2020 Canonical, Ltd.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.

subiquity/client/client.py

@@ -0,0 +1,455 @@
+# Copyright 2015 Canonical, Ltd.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import asyncio
+import logging
+import os
+import signal
+import sys
+import traceback
+
+import aiohttp
+
+from subiquitycore.async_helpers import (
+    run_in_thread,
+    )
+from subiquitycore.screen import is_linux_tty
+from subiquitycore.tuicontroller import Skip
+from subiquitycore.tui import TuiApplication
+from subiquitycore.view import BaseView
+
+from subiquity.client.controller import Confirm
+from subiquity.client.keycodes import (
+    DummyKeycodesFilter,
+    KeyCodesFilter,
+    )
+from subiquity.common.api.client import make_client_for_conn
+from subiquity.common.apidef import API
+from subiquity.common.errorreport import (
+    ErrorReporter,
+    )
+from subiquity.common.serialize import from_json
+from subiquity.common.types import (
+    ApplicationState,
+    ErrorReportKind,
+    ErrorReportRef,
+    InstallState,
+    )
+from subiquity.journald import journald_listen
+from subiquity.ui.frame import SubiquityUI
+from subiquity.ui.views.error import ErrorReportStretchy
+from subiquity.ui.views.help import HelpMenu
+from subiquity.ui.views.installprogress import (
+    InstallConfirmation,
+    )
+
+
+log = logging.getLogger('subiquity.client.client')
+
+
+class Abort(Exception):
+    def __init__(self, error_report_ref):
+        self.error_report_ref = error_report_ref
+
+
+DEBUG_SHELL_INTRO = _("""\
+Installer shell session activated.
+
+This shell session is running inside the installer environment.  You
+will be returned to the installer when this shell is exited, for
+example by typing Control-D or 'exit'.
+
+Be aware that this is an ephemeral environment.  Changes to this
+environment will not survive a reboot. If the install has started, the
+installed system will be mounted at /target.""")
+
+
+class SubiquityClient(TuiApplication):
+
+    snapd_socket_path = '/run/snapd.socket'
+
+    from subiquity.client import controllers as controllers_mod
+    project = "subiquity"
+
+    def make_model(self):
+        return None
+
+    def make_ui(self):
+        return SubiquityUI(self, self.help_menu)
+
+    controllers = []
+
+    def __init__(self, opts):
+        if is_linux_tty():
+            self.input_filter = KeyCodesFilter()
+        else:
+            self.input_filter = DummyKeycodesFilter()
+
+        self.help_menu = HelpMenu(self)
+        super().__init__(opts)
+        self.interactive = None
+        self.server_updated = None
+        self.restarting_server = False
+        self.global_overlays = []
+
+        try:
+            self.our_tty = os.ttyname(0)
+        except OSError:
+            self.our_tty = "not a tty"
+
+        self.conn = aiohttp.UnixConnector(self.opts.socket)
+        self.client = make_client_for_conn(API, self.conn, self.resp_hook)
+
+        self.error_reporter = ErrorReporter(
+            self.context.child("ErrorReporter"), self.opts.dry_run, self.root,
+            self.client)
+
+        self.note_data_for_apport("SnapUpdated", str(self.updated))
+        self.note_data_for_apport("UsingAnswers", str(bool(self.answers)))
+
+    async def _restart_server(self):
+        log.debug("_restart_server")
+        try:
+            await self.client.meta.restart.POST()
+        except aiohttp.ServerDisconnectedError:
+            pass
+        self.restart(remove_last_screen=False)
+
+    def restart(self, remove_last_screen=True, restart_server=False):
+        log.debug(f"restart {remove_last_screen} {restart_server}")
+        if remove_last_screen:
+            self._remove_last_screen()
+        if restart_server:
+            self.restarting_server = True
+            self.ui.block_input = True
+            self.aio_loop.create_task(self._restart_server())
+            return
+        if self.urwid_loop is not None:
+            self.urwid_loop.stop()
+        cmdline = ['snap', 'run', 'subiquity']
+        if self.opts.dry_run:
+            cmdline = [
+                sys.executable, '-m', 'subiquity.cmd.tui',
+                ] + sys.argv[1:] + ['--socket', self.opts.socket]
+            if self.opts.server_pid is not None:
+                cmdline.extend(['--server-pid', self.opts.server_pid])
+            log.debug("restarting %r", cmdline)
+
+        os.execvp(cmdline[0], cmdline)
+
+    def resp_hook(self, response):
+        headers = response.headers
+        if 'x-updated' in headers:
+            if self.server_updated is None:
+                self.server_updated = headers['x-updated']
+            elif self.server_updated != headers['x-updated']:
+                self.restart(remove_last_screen=False)
+        status = headers.get('x-status')
+        if status == 'skip':
+            raise Skip
+        elif status == 'confirm':
+            raise Confirm
+        if headers.get('x-error-report') is not None:
+            ref = from_json(ErrorReportRef, headers['x-error-report'])
+            raise Abort(ref)
+        try:
+            response.raise_for_status()
+        except aiohttp.ClientError:
+            report = self.error_reporter.make_apport_report(
+                ErrorReportKind.SERVER_REQUEST_FAIL,
+                "request to {}".format(response.url.path))
+            raise Abort(report.ref())
+        return response
+
+    async def noninteractive_confirmation(self):
+        await asyncio.sleep(1)
+        yes = _('yes')
+        no = _('no')
+        answer = no
+        print(_("Confirmation is required to continue."))
+        print(_("Add 'autoinstall' to your kernel command line to avoid this"))
+        print()
+        prompt = "\n\n{} ({}|{})".format(
+            _("Continue with autoinstall?"), yes, no)
+        while answer != yes:
+            print(prompt)
+            answer = await run_in_thread(input)
+        await self.confirm_install()
+
+    async def noninteractive_watch_install_state(self):
+        install_state = None
+        confirm_task = None
+        while True:
+            try:
+                install_status = await self.client.install.status.GET(
+                    cur=install_state)
+                install_state = install_status.state
+            except aiohttp.ClientError:
+                await asyncio.sleep(1)
+                continue
+            if install_state == InstallState.NEEDS_CONFIRMATION:
+                if confirm_task is not None:
+                    confirm_task = self.aio_loop.create_task(
+                        self.noninteractive_confirmation())
+            elif confirm_task is not None:
+                confirm_task.cancel()
+                confirm_task = None
+
+    def subiquity_event_noninteractive(self, event):
+        if event['SUBIQUITY_EVENT_TYPE'] == 'start':
+            print('start: ' + event["MESSAGE"])
+        elif event['SUBIQUITY_EVENT_TYPE'] == 'finish':
+            print('finish: ' + event["MESSAGE"])
+            context_name = event.get('SUBIQUITY_CONTEXT_NAME', '')
+            if context_name == 'subiquity/Reboot/reboot':
+                self.exit()
+
+    async def connect(self):
+        print("connecting...", end='', flush=True)
+        while True:
+            try:
+                status = await self.client.meta.status.GET()
+            except aiohttp.ClientError:
+                await asyncio.sleep(1)
+                print(".", end='', flush=True)
+            else:
+                break
+        print()
+        self.event_syslog_id = status.event_syslog_id
+        if status.state == ApplicationState.STARTING:
+            print("server is starting...", end='', flush=True)
+            while status.state == ApplicationState.STARTING:
+                await asyncio.sleep(1)
+                print(".", end='', flush=True)
+                status = await self.client.meta.status.GET()
+            print()
+        if status.state == ApplicationState.EARLY_COMMANDS:
+            print("running early commands...")
+            fd = journald_listen(
+                self.aio_loop,
+                [status.early_commands_syslog_id],
+                lambda e: print(e['MESSAGE']))
+            status.state = await self.client.meta.status.GET(cur=status.state)
+            await asyncio.sleep(0.5)
+            self.aio_loop.remove_reader(fd)
+        return status
+
+    async def start(self):
+        status = await self.connect()
+        if status.state == ApplicationState.INTERACTIVE:
+            self.interactive = True
+            await super().start()
+            journald_listen(
+                self.aio_loop,
+                [status.event_syslog_id],
+                self.controllers.Progress.event)
+            journald_listen(
+                self.aio_loop,
+                [status.log_syslog_id],
+                self.controllers.Progress.log_line)
+            self.error_reporter.load_reports()
+            for report in self.error_reporter.reports:
+                if report.kind == ErrorReportKind.UI and not report.seen:
+                    self.show_error_report(report.ref())
+                    break
+        else:
+            self.interactive = False
+            if self.opts.run_on_serial:
+                # Thanks to the fact that we are launched with agetty's
+                # --skip-login option, on serial lines we can end up starting
+                # with some strange terminal settings (see the docs for
+                # --skip-login in agetty(8)). For an interactive install this
+                # does not matter as the settings will soon be clobbered but
+                # for a non-interactive one we need to clear things up or the
+                # prompting for confirmation will be confusing.
+                os.system('stty sane')
+            journald_listen(
+                self.aio_loop,
+                [status.event_syslog_id],
+                self.subiquity_event_noninteractive,
+                seek=True)
+            self.aio_loop.create_task(
+                self.noninteractive_watch_install_state())
+
+    def _exception_handler(self, loop, context):
+        exc = context.get('exception')
+        if self.restarting_server:
+            log.debug('ignoring %s %s during restart', exc, type(exc))
+            return
+        if isinstance(exc, Abort):
+            self.show_error_report(exc.error_report_ref)
+            return
+        super()._exception_handler(loop, context)
+
+    def extra_urwid_loop_args(self):
+        return dict(input_filter=self.input_filter.filter)
+
+    def run(self):
+        try:
+            super().run()
+        except Exception:
+            print("generating crash report")
+            try:
+                report = self.make_apport_report(
+                    ErrorReportKind.UI, "Installer UI", interrupt=False,
+                    wait=True)
+                if report is not None:
+                    print("report saved to {path}".format(path=report.path))
+            except Exception:
+                print("report generation failed")
+                traceback.print_exc()
+            Error = getattr(self.controllers, "Error", None)
+            if Error is not None and Error.cmds:
+                new_loop = asyncio.new_event_loop()
+                asyncio.set_event_loop(new_loop)
+                new_loop.run_until_complete(Error.run())
+            if self.interactive:
+                self._remove_last_screen()
+                raise
+            else:
+                traceback.print_exc()
+                signal.pause()
+        finally:
+            if self.opts.server_pid:
+                print('killing server {}'.format(self.opts.server_pid))
+                pid = int(self.opts.server_pid)
+                os.kill(pid, 2)
+                os.waitpid(pid, 0)
+
+    async def confirm_install(self):
+        await self.client.meta.confirm.POST(self.our_tty)
+
+    def add_global_overlay(self, overlay):
+        self.global_overlays.append(overlay)
+        if isinstance(self.ui.body, BaseView):
+            self.ui.body.show_stretchy_overlay(overlay)
+
+    def remove_global_overlay(self, overlay):
+        self.global_overlays.remove(overlay)
+        if isinstance(self.ui.body, BaseView):
+            self.ui.body.remove_overlay(overlay)
+
+    def _remove_last_screen(self):
+        last_screen = self.state_path('last-screen')
+        if os.path.exists(last_screen):
+            os.unlink(last_screen)
+
+    def exit(self):
+        self._remove_last_screen()
+        super().exit()
+
+    def select_initial_screen(self):
+        last_screen = None
+        if self.updated:
+            state_path = self.state_path('last-screen')
+            if os.path.exists(state_path):
+                with open(state_path) as fp:
+                    last_screen = fp.read().strip()
+        index = 0
+        if last_screen:
+            for i, controller in enumerate(self.controllers.instances):
+                if controller.name == last_screen:
+                    index = i
+        self.aio_loop.create_task(self._select_initial_screen(index))
+
+    async def _select_initial_screen(self, index):
+        endpoint_names = []
+        for c in self.controllers.instances[:index]:
+            if c.endpoint_name:
+                endpoint_names.append(c.endpoint_name)
+        if endpoint_names:
+            await self.client.meta.mark_configured.POST(endpoint_names)
+        self.controllers.index = index - 1
+        self.next_screen()
+
+    async def move_screen(self, increment, coro):
+        try:
+            await super().move_screen(increment, coro)
+        except Confirm:
+            self.show_confirm_install()
+
+    def show_confirm_install(self):
+        log.debug("showing InstallConfirmation over %s", self.ui.body)
+        self.add_global_overlay(InstallConfirmation(self))
+
+    async def make_view_for_controller(self, new):
+        view = await super().make_view_for_controller(new)
+        if new.answers:
+            self.aio_loop.call_soon(new.run_answers)
+        with open(self.state_path('last-screen'), 'w') as fp:
+            fp.write(new.name)
+        return view
+
+    def show_progress(self):
+        self.ui.set_body(self.controllers.Progress.progress_view)
+
+    def unhandled_input(self, key):
+        if key == 'f1':
+            if not self.ui.right_icon.current_help:
+                self.ui.right_icon.open_pop_up()
+        elif key in ['ctrl z', 'f2']:
+            self.debug_shell()
+        elif self.opts.dry_run:
+            self.unhandled_input_dry_run(key)
+        else:
+            super().unhandled_input(key)
+
+    def unhandled_input_dry_run(self, key):
+        if key in ['ctrl e', 'ctrl r']:
+            interrupt = key == 'ctrl e'
+            try:
+                1/0
+            except ZeroDivisionError:
+                self.make_apport_report(
+                    ErrorReportKind.UNKNOWN, "example", interrupt=interrupt)
+        elif key == 'ctrl u':
+            1/0
+        elif key == 'ctrl b':
+            self.aio_loop.create_task(self.client.dry_run.crash.GET())
+        else:
+            super().unhandled_input(key)
+
+    def debug_shell(self, after_hook=None):
+
+        def _before():
+            os.system("clear")
+            print(DEBUG_SHELL_INTRO)
+
+        self.run_command_in_foreground(
+            ["bash"], before_hook=_before, after_hook=after_hook, cwd='/')
+
+    def note_file_for_apport(self, key, path):
+        self.error_reporter.note_file_for_apport(key, path)
+
+    def note_data_for_apport(self, key, value):
+        self.error_reporter.note_data_for_apport(key, value)
+
+    def make_apport_report(self, kind, thing, *, interrupt, wait=False, **kw):
+        report = self.error_reporter.make_apport_report(
+            kind, thing, wait=wait, **kw)
+
+        if report is not None and interrupt:
+            self.show_error_report(report.ref())
+
+        return report
+
+    def show_error_report(self, error_ref):
+        log.debug("show_error_report %r", error_ref.base)
+        if isinstance(self.ui.body, BaseView):
+            w = getattr(self.ui.body._w, 'stretchy', None)
+            if isinstance(w, ErrorReportStretchy):
+                # Don't show an error if already looking at one.
+                return
+        self.add_global_overlay(ErrorReportStretchy(self, error_ref))

subiquity/client/keycodes.py

@@ -19,7 +19,7 @@ import os
 import struct
 import sys
 
-log = logging.getLogger('subiquity.keycodes')
+log = logging.getLogger('subiquity.client.keycodes')
 
 # /usr/include/linux/kd.h
 K_RAW = 0x00

subiquity/cmd/server.py

@@ -34,6 +34,35 @@ def make_server_args_parser():
                         dest='dry_run',
                         help='menu-only, do not call installer function')
     parser.add_argument('--socket')
+    parser.add_argument('--machine-config', metavar='CONFIG',
+                        dest='machine_config',
+                        help="Don't Probe. Use probe data file")
+    parser.add_argument('--source', default=[], action='append',
+                        dest='sources', metavar='URL',
+                        help='install from url instead of default.')
+    parser.add_argument('--bootloader',
+                        choices=['none', 'bios', 'prep', 'uefi'],
+                        help='Override style of bootloader to use')
+    parser.add_argument('--autoinstall', action='store')
+    with open('/proc/cmdline') as fp:
+        cmdline = fp.read()
+    parser.add_argument('--kernel-cmdline', action='store', default=cmdline)
+    parser.add_argument(
+        '--snaps-from-examples', action='store_const', const=True,
+        dest="snaps_from_examples",
+        help=("Load snap details from examples/snaps instead of store. "
+              "Default in dry-run mode.  "
+              "See examples/snaps/README.md for more."))
+    parser.add_argument(
+        '--no-snaps-from-examples', action='store_const', const=False,
+        dest="snaps_from_examples",
+        help=("Load snap details from store instead of examples. "
+              "Default in when not in dry-run mode.  "
+              "See examples/snaps/README.md for more."))
+    parser.add_argument(
+        '--snap-section', action='store', default='server',
+        help=("Show snaps from this section of the store in the snap "
+              "list screen."))
     return parser
 
 
@@ -47,6 +76,8 @@ def main():
     opts = parser.parse_args(sys.argv[1:])
     logdir = LOGDIR
     if opts.dry_run:
+        if opts.snaps_from_examples is None:
+            opts.snaps_from_examples = True
         logdir = ".subiquity"
     if opts.socket is None:
         if opts.dry_run:
@@ -55,6 +86,17 @@ def main():
             opts.socket = '/run/subiquity/socket'
     os.makedirs(os.path.basename(opts.socket), exist_ok=True)
 
+    block_log_dir = os.path.join(logdir, "block")
+    os.makedirs(block_log_dir, exist_ok=True)
+    handler = logging.FileHandler(os.path.join(block_log_dir, 'discover.log'))
+    handler.setLevel('DEBUG')
+    handler.setFormatter(
+        logging.Formatter("%(asctime)s %(name)s:%(lineno)d %(message)s"))
+    logging.getLogger('probert').addHandler(handler)
+    handler.addFilter(lambda rec: rec.name != 'probert.network')
+    logging.getLogger('curtin').addHandler(handler)
+    logging.getLogger('block-discover').addHandler(handler)
+
     logfiles = setup_logger(dir=logdir, base='subiquity-server')
 
     logger = logging.getLogger('subiquity')
@@ -62,14 +104,14 @@ def main():
     logger.info("Starting Subiquity server revision {}".format(version))
     logger.info("Arguments passed: {}".format(sys.argv))
 
-    subiquity_interface = SubiquityServer(opts)
+    server = SubiquityServer(opts, block_log_dir)
 
-    subiquity_interface.note_file_for_apport(
+    server.note_file_for_apport(
         "InstallerServerLog", logfiles['debug'])
-    subiquity_interface.note_file_for_apport(
+    server.note_file_for_apport(
         "InstallerServerLogInfo", logfiles['info'])
 
-    subiquity_interface.run()
+    server.run()
 
 
 if __name__ == '__main__':

subiquity/cmd/tui.py

@@ -64,12 +64,6 @@ def make_client_args_parser():
     parser.add_argument('--unicode', action='store_false',
                         dest='ascii',
                         help='Run the installer in unicode mode.')
-    parser.add_argument('--machine-config', metavar='CONFIG',
-                        dest='machine_config',
-                        help="Don't Probe. Use probe data file")
-    parser.add_argument('--bootloader',
-                        choices=['none', 'bios', 'prep', 'uefi'],
-                        help='Override style of bootloader to use')
     parser.add_argument('--screens', action='append', dest='screens',
                         default=[])
     parser.add_argument('--script', metavar="SCRIPT", action='append',
@@ -79,29 +73,6 @@ def make_client_args_parser():
     parser.add_argument('--click', metavar="PAT", action=ClickAction,
                         help='Synthesize a click on a button matching PAT')
     parser.add_argument('--answers')
-    parser.add_argument('--autoinstall', action='store')
-    with open('/proc/cmdline') as fp:
-        cmdline = fp.read()
-    parser.add_argument('--kernel-cmdline', action='store', default=cmdline)
-    parser.add_argument('--source', default=[], action='append',
-                        dest='sources', metavar='URL',
-                        help='install from url instead of default.')
-    parser.add_argument(
-        '--snaps-from-examples', action='store_const', const=True,
-        dest="snaps_from_examples",
-        help=("Load snap details from examples/snaps instead of store. "
-              "Default in dry-run mode.  "
-              "See examples/snaps/README.md for more."))
-    parser.add_argument(
-        '--no-snaps-from-examples', action='store_const', const=False,
-        dest="snaps_from_examples",
-        help=("Load snap details from store instead of examples. "
-              "Default in when not in dry-run mode.  "
-              "See examples/snaps/README.md for more."))
-    parser.add_argument(
-        '--snap-section', action='store', default='server',
-        help=("Show snaps from this section of the store in the snap "
-              "list screen."))
     parser.add_argument('--server-pid')
     return parser
 
@@ -113,7 +84,7 @@ def main():
     setup_environment()
     # setup_environment sets $APPORT_DATA_DIR which must be set before
     # apport is imported, which is done by this import:
-    from subiquity.core import Subiquity
+    from subiquity.client.client import SubiquityClient
     parser = make_client_args_parser()
     args = sys.argv[1:]
     if '--dry-run' in args:
@@ -143,10 +114,8 @@ def main():
     os.makedirs(os.path.basename(opts.socket), exist_ok=True)
     logdir = LOGDIR
     if opts.dry_run:
-        if opts.snaps_from_examples is None:
-            opts.snaps_from_examples = True
         logdir = ".subiquity"
-    logfiles = setup_logger(dir=logdir, base='subiquity')
+    logfiles = setup_logger(dir=logdir, base='subiquity-client')
 
     logger = logging.getLogger('subiquity')
     version = os.environ.get("SNAP_REVISION", "unknown")
@@ -178,17 +147,6 @@ def main():
                 "cloud-init status: %r, assumed disabled",
                 status_txt)
 
-    block_log_dir = os.path.join(logdir, "block")
-    os.makedirs(block_log_dir, exist_ok=True)
-    handler = logging.FileHandler(os.path.join(block_log_dir, 'discover.log'))
-    handler.setLevel('DEBUG')
-    handler.setFormatter(
-        logging.Formatter("%(asctime)s %(name)s:%(lineno)d %(message)s"))
-    logging.getLogger('probert').addHandler(handler)
-    handler.addFilter(lambda rec: rec.name != 'probert.network')
-    logging.getLogger('curtin').addHandler(handler)
-    logging.getLogger('block-discover').addHandler(handler)
-
     if opts.ssh:
         from subiquity.ui.views.help import (
             ssh_help_texts, get_installer_password)
@@ -219,7 +177,7 @@ def main():
             opts.answers.close()
             opts.answers = None
 
-    subiquity_interface = Subiquity(opts, block_log_dir)
+    subiquity_interface = SubiquityClient(opts)
 
     subiquity_interface.note_file_for_apport(
         "InstallerLog", logfiles['debug'])

subiquity/common/apidef.py

@@ -13,9 +13,12 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from typing import List, Optional
+
 from subiquity.common.api.defs import api
 from subiquity.common.types import (
     ApplicationState,
+    ApplicationStatus,
     ErrorReportRef,
     )
 
@@ -26,9 +29,18 @@ class API:
 
     class meta:
         class status:
-            def GET() -> ApplicationState:
+            def GET(cur: Optional[ApplicationState] = None) \
+              -> ApplicationStatus:
                 """Get the installer state."""
 
+        class mark_configured:
+            def POST(endpoint_names: List[str]) -> None:
+                """Mark the controllers for endpoint_names as configured."""
+
+        class confirm:
+            def POST(tty: str) -> None:
+                """Confirm that the installation should proceed."""
+
         class restart:
             def POST() -> None:
                 """Restart the server process."""

subiquity/common/types.py

@@ -27,6 +27,17 @@ import attr
 
 class ApplicationState(enum.Enum):
     STARTING = enum.auto()
+    EARLY_COMMANDS = enum.auto()
+    INTERACTIVE = enum.auto()
+    NON_INTERACTIVE = enum.auto()
+
+
+@attr.s(auto_attribs=True)
+class ApplicationStatus:
+    state: ApplicationState
+    early_commands_syslog_id: str
+    log_syslog_id: str
+    event_syslog_id: str
 
 
 class ErrorReportState(enum.Enum):

subiquity/core.py

@@ -1,648 +0,0 @@
-# Copyright 2015 Canonical, Ltd.
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import asyncio
-import logging
-import os
-import shlex
-import signal
-import sys
-import traceback
-
-import aiohttp
-
-import jsonschema
-
-from systemd import journal
-
-import yaml
-
-from subiquitycore.async_helpers import (
-    run_in_thread,
-    schedule_task,
-    )
-from subiquitycore.prober import Prober
-from subiquitycore.screen import is_linux_tty
-from subiquitycore.tuicontroller import Skip
-from subiquitycore.tui import TuiApplication
-from subiquitycore.snapd import (
-    AsyncSnapd,
-    FakeSnapdConnection,
-    SnapdConnection,
-    )
-from subiquitycore.view import BaseView
-
-from subiquity.common.api.client import make_client_for_conn
-from subiquity.common.apidef import API
-from subiquity.common.errorreport import (
-    ErrorReporter,
-    )
-from subiquity.common.serialize import from_json
-from subiquity.common.types import (
-    ErrorReportKind,
-    ErrorReportRef,
-    )
-from subiquity.controller import Confirm
-from subiquity.journald import journald_listen
-from subiquity.keycodes import (
-    DummyKeycodesFilter,
-    KeyCodesFilter,
-    )
-from subiquity.lockfile import Lockfile
-from subiquity.models.subiquity import SubiquityModel
-from subiquity.ui.frame import SubiquityUI
-from subiquity.ui.views.error import ErrorReportStretchy
-from subiquity.ui.views.help import HelpMenu
-
-
-log = logging.getLogger('subiquity.core')
-
-
-class Abort(Exception):
-    def __init__(self, error_report_ref):
-        self.error_report_ref = error_report_ref
-
-
-DEBUG_SHELL_INTRO = _("""\
-Installer shell session activated.
-
-This shell session is running inside the installer environment.  You
-will be returned to the installer when this shell is exited, for
-example by typing Control-D or 'exit'.
-
-Be aware that this is an ephemeral environment.  Changes to this
-environment will not survive a reboot. If the install has started, the
-installed system will be mounted at /target.""")
-
-
-class Subiquity(TuiApplication):
-
-    snapd_socket_path = '/run/snapd.socket'
-
-    base_schema = {
-        'type': 'object',
-        'properties': {
-            'version': {
-                'type': 'integer',
-                'minimum': 1,
-                'maximum': 1,
-                },
-            },
-        'required': ['version'],
-        'additionalProperties': True,
-        }
-
-    from subiquity import controllers as controllers_mod
-    project = "subiquity"
-
-    def make_model(self):
-        root = '/'
-        if self.opts.dry_run:
-            root = os.path.abspath('.subiquity')
-        return SubiquityModel(root, self.opts.sources)
-
-    def make_ui(self):
-        return SubiquityUI(self, self.help_menu)
-
-    controllers = [
-        "Early",
-        "Reporting",
-        "Error",
-        "Userdata",
-        "Package",
-        "Debconf",
-        "Welcome",
-        "Refresh",
-        "Keyboard",
-        "Zdev",
-        "Network",
-        "Proxy",
-        "Mirror",
-        "Refresh",
-        "Filesystem",
-        "Identity",
-        "SSH",
-        "SnapList",
-        "InstallProgress",
-        "Late",
-        "Reboot",
-    ]
-
-    def __init__(self, opts, block_log_dir):
-        if is_linux_tty():
-            self.input_filter = KeyCodesFilter()
-        else:
-            self.input_filter = DummyKeycodesFilter()
-
-        self.help_menu = HelpMenu(self)
-        super().__init__(opts)
-
-        self.event_syslog_id = 'subiquity_event.{}'.format(os.getpid())
-        self.log_syslog_id = 'subiquity_log.{}'.format(os.getpid())
-
-        self.server_updated = None
-        self.restarting_server = False
-        self.prober = Prober(opts.machine_config, self.debug_flags)
-        journald_listen(
-            self.aio_loop, ["subiquity"], self.subiquity_event, seek=True)
-        self.event_listeners = []
-        self.install_lock_file = Lockfile(self.state_path("installing"))
-        self.global_overlays = []
-        self.block_log_dir = block_log_dir
-        self.kernel_cmdline = shlex.split(opts.kernel_cmdline)
-        if opts.snaps_from_examples:
-            connection = FakeSnapdConnection(
-                os.path.join(
-                    os.path.dirname(
-                        os.path.dirname(__file__)),
-                    "examples", "snaps"),
-                self.scale_factor)
-        else:
-            connection = SnapdConnection(self.root, self.snapd_socket_path)
-        self.snapd = AsyncSnapd(connection)
-        self.signal.connect_signals([
-            ('network-proxy-set', lambda: schedule_task(self._proxy_set())),
-            ('network-change', self._network_change),
-            ])
-
-        self.conn = aiohttp.UnixConnector(self.opts.socket)
-        self.client = make_client_for_conn(API, self.conn, self.resp_hook)
-
-        self.autoinstall_config = {}
-        self.error_reporter = ErrorReporter(
-            self.context.child("ErrorReporter"), self.opts.dry_run, self.root,
-            self.client)
-
-        self.note_data_for_apport("SnapUpdated", str(self.updated))
-        self.note_data_for_apport("UsingAnswers", str(bool(self.answers)))
-
-    def subiquity_event(self, event):
-        if event["MESSAGE"] == "starting install":
-            if event["_PID"] == os.getpid():
-                return
-            if not self.install_lock_file.is_exclusively_locked():
-                return
-            from subiquity.ui.views.installprogress import (
-                InstallRunning,
-                )
-            tty = self.install_lock_file.read_content()
-            install_running = InstallRunning(self.ui.body, self, tty)
-            self.add_global_overlay(install_running)
-            schedule_task(self._hide_install_running(install_running))
-
-    async def _hide_install_running(self, install_running):
-        # Wait until the install has completed...
-        async with self.install_lock_file.shared():
-            # And remove the overlay.
-            self.remove_global_overlay(install_running)
-
-    async def _restart_server(self):
-        log.debug("_restart_server")
-        try:
-            await self.client.meta.restart.POST()
-        except aiohttp.ServerDisconnectedError:
-            pass
-        self.restart(remove_last_screen=False)
-
-    def restart(self, remove_last_screen=True, restart_server=False):
-        log.debug(f"restart {remove_last_screen} {restart_server}")
-        if remove_last_screen:
-            self._remove_last_screen()
-        if restart_server:
-            self.restarting_server = True
-            self.ui.block_input = True
-            self.aio_loop.create_task(self._restart_server())
-            return
-        if self.urwid_loop is not None:
-            self.urwid_loop.stop()
-        cmdline = ['snap', 'run', 'subiquity']
-        if self.opts.dry_run:
-            cmdline = [
-                sys.executable, '-m', 'subiquity.cmd.tui',
-                ] + sys.argv[1:] + ['--socket', self.opts.socket]
-            if self.opts.server_pid is not None:
-                cmdline.extend(['--server-pid', self.opts.server_pid])
-            log.debug("restarting %r", cmdline)
-
-        os.execvp(cmdline[0], cmdline)
-
-    def get_primary_tty(self):
-        tty = '/dev/tty1'
-        for work in self.kernel_cmdline:
-            if work.startswith('console='):
-                tty = '/dev/' + work[len('console='):].split(',')[0]
-        return tty
-
-    async def load_autoinstall_config(self):
-        with open(self.opts.autoinstall) as fp:
-            self.autoinstall_config = yaml.safe_load(fp)
-        primary_tty = self.get_primary_tty()
-        try:
-            our_tty = os.ttyname(0)
-        except OSError:
-            # This is a gross hack for testing in travis.
-            our_tty = "/dev/not a tty"
-        if not self.interactive() and our_tty != primary_tty:
-            while True:
-                print(
-                    _("the installer running on {tty} will perform the "
-                      "autoinstall").format(tty=primary_tty))
-                print()
-                print(_("press enter to start a shell"))
-                input()
-                os.system("cd / && bash")
-        self.controllers.Reporting.start()
-        with self.context.child("core_validation", level="INFO"):
-            jsonschema.validate(self.autoinstall_config, self.base_schema)
-        self.controllers.Reporting.setup_autoinstall()
-        self.controllers.Early.setup_autoinstall()
-        self.controllers.Error.setup_autoinstall()
-        if self.controllers.Early.cmds:
-            stamp_file = self.state_path("early-commands")
-            if our_tty != primary_tty:
-                print(
-                    _("waiting for installer running on {tty} to run early "
-                      "commands").format(tty=primary_tty))
-                while not os.path.exists(stamp_file):
-                    await asyncio.sleep(1)
-            elif not os.path.exists(stamp_file):
-                await self.controllers.Early.run()
-                open(stamp_file, 'w').close()
-            with open(self.opts.autoinstall) as fp:
-                self.autoinstall_config = yaml.safe_load(fp)
-            with self.context.child("core_validation", level="INFO"):
-                jsonschema.validate(self.autoinstall_config, self.base_schema)
-        for controller in self.controllers.instances:
-            controller.setup_autoinstall()
-        if not self.interactive() and self.opts.run_on_serial:
-            # Thanks to the fact that we are launched with agetty's
-            # --skip-login option, on serial lines we can end up starting with
-            # some strange terminal settings (see the docs for --skip-login in
-            # agetty(8)). For an interactive install this does not matter as
-            # the settings will soon be clobbered but for a non-interactive
-            # one we need to clear things up or the prompting for confirmation
-            # in next_screen below will be confusing.
-            os.system('stty sane')
-
-    def resp_hook(self, response):
-        headers = response.headers
-        if 'x-updated' in headers:
-            if self.server_updated is None:
-                self.server_updated = headers['x-updated']
-            elif self.server_updated != headers['x-updated']:
-                self.restart(remove_last_screen=False)
-        status = response.headers.get('x-status')
-        if status == 'skip':
-            raise Skip
-        elif status == 'confirm':
-            raise Confirm
-        if headers.get('x-error-report') is not None:
-            ref = from_json(ErrorReportRef, headers['x-error-report'])
-            raise Abort(ref)
-        try:
-            response.raise_for_status()
-        except aiohttp.ClientError:
-            report = self.error_reporter.make_apport_report(
-                ErrorReportKind.SERVER_REQUEST_FAIL,
-                "request to {}".format(response.url.path))
-            raise Abort(report.ref())
-        return response
-
-    def subiquity_event_noninteractive(self, event):
-        if event['SUBIQUITY_EVENT_TYPE'] == 'start':
-            print('start: ' + event["MESSAGE"])
-        elif event['SUBIQUITY_EVENT_TYPE'] == 'finish':
-            print('finish: ' + event["MESSAGE"])
-            context_name = event.get('SUBIQUITY_CONTEXT_NAME', '')
-            if context_name == 'subiquity/Reboot/reboot':
-                self.exit()
-
-    async def connect(self):
-        print("connecting...", end='', flush=True)
-        while True:
-            try:
-                await self.client.meta.status.GET()
-            except aiohttp.ClientError:
-                await asyncio.sleep(1)
-                print(".", end='', flush=True)
-            else:
-                print()
-                break
-
-    def load_serialized_state(self):
-        for controller in self.controllers.instances:
-            controller.load_state()
-
-    async def start(self):
-        self.controllers.load_all()
-        self.load_serialized_state()
-        await self.connect()
-        if self.opts.autoinstall is not None:
-            await self.load_autoinstall_config()
-            if not self.interactive() and not self.opts.dry_run:
-                open('/run/casper-no-prompt', 'w').close()
-        interactive = self.interactive()
-        if interactive:
-            journald_listen(
-                self.aio_loop,
-                [self.event_syslog_id],
-                self.controllers.InstallProgress.event)
-            journald_listen(
-                self.aio_loop,
-                [self.log_syslog_id],
-                self.controllers.InstallProgress.log_line)
-        else:
-            journald_listen(
-                self.aio_loop,
-                [self.event_syslog_id],
-                self.subiquity_event_noninteractive,
-                seek=True)
-            await asyncio.sleep(1)
-        await super().start(start_urwid=interactive)
-        if not interactive:
-            self.select_initial_screen()
-
-    def _exception_handler(self, loop, context):
-        exc = context.get('exception')
-        if self.restarting_server:
-            log.debug('ignoring %s %s during restart', exc, type(exc))
-            return
-        if isinstance(exc, Abort):
-            self.show_error_report(exc.error_report_ref)
-            return
-        super()._exception_handler(loop, context)
-
-    def _remove_last_screen(self):
-        last_screen = self.state_path('last-screen')
-        if os.path.exists(last_screen):
-            os.unlink(last_screen)
-
-    def exit(self):
-        self._remove_last_screen()
-        super().exit()
-
-    def extra_urwid_loop_args(self):
-        return dict(input_filter=self.input_filter.filter)
-
-    def run(self):
-        try:
-            super().run()
-        except Exception:
-            print("generating crash report")
-            try:
-                report = self.make_apport_report(
-                    ErrorReportKind.UI, "Installer UI", interrupt=False,
-                    wait=True)
-                if report is not None:
-                    print("report saved to {path}".format(path=report.path))
-            except Exception:
-                print("report generation failed")
-                traceback.print_exc()
-            Error = getattr(self.controllers, "Error", None)
-            if Error is not None and Error.cmds:
-                new_loop = asyncio.new_event_loop()
-                asyncio.set_event_loop(new_loop)
-                new_loop.run_until_complete(Error.run())
-            if self.interactive():
-                self._remove_last_screen()
-                raise
-            else:
-                traceback.print_exc()
-                signal.pause()
-        finally:
-            if self.opts.server_pid:
-                print('killing server {}'.format(self.opts.server_pid))
-                pid = int(self.opts.server_pid)
-                os.kill(pid, 2)
-                os.waitpid(pid, 0)
-
-    def add_event_listener(self, listener):
-        self.event_listeners.append(listener)
-
-    def report_start_event(self, context, description):
-        for listener in self.event_listeners:
-            listener.report_start_event(context, description)
-        self._maybe_push_to_journal('start', context, description)
-
-    def report_finish_event(self, context, description, status):
-        for listener in self.event_listeners:
-            listener.report_finish_event(context, description, status)
-        self._maybe_push_to_journal('finish', context, description)
-
-    def _maybe_push_to_journal(self, event_type, context, description):
-        if not context.get('is-install-context') and self.interactive():
-            controller = context.get('controller')
-            if controller is None or controller.interactive():
-                return
-        if context.get('request'):
-            return
-        indent = context.full_name().count('/') - 2
-        if context.get('is-install-context') and self.interactive():
-            indent -= 1
-            msg = context.description
-        else:
-            msg = context.full_name()
-            if description:
-                msg += ': ' + description
-        msg = '  ' * indent + msg
-        if context.parent:
-            parent_id = str(context.parent.id)
-        else:
-            parent_id = ''
-        journal.send(
-            msg,
-            PRIORITY=context.level,
-            SYSLOG_IDENTIFIER=self.event_syslog_id,
-            SUBIQUITY_CONTEXT_NAME=context.full_name(),
-            SUBIQUITY_EVENT_TYPE=event_type,
-            SUBIQUITY_CONTEXT_ID=str(context.id),
-            SUBIQUITY_CONTEXT_PARENT_ID=parent_id)
-
-    async def confirm_install(self):
-        self.base_model.confirm()
-
-    def interactive(self):
-        if not self.autoinstall_config:
-            return True
-        return bool(self.autoinstall_config.get('interactive-sections'))
-
-    def add_global_overlay(self, overlay):
-        self.global_overlays.append(overlay)
-        if isinstance(self.ui.body, BaseView):
-            self.ui.body.show_stretchy_overlay(overlay)
-
-    def remove_global_overlay(self, overlay):
-        self.global_overlays.remove(overlay)
-        if isinstance(self.ui.body, BaseView):
-            self.ui.body.remove_overlay(overlay)
-
-    def initial_controller_index(self):
-        if not self.updated:
-            return 0
-        state_path = self.state_path('last-screen')
-        if not os.path.exists(state_path):
-            return 0
-        with open(state_path) as fp:
-            last_screen = fp.read().strip()
-        controller_index = 0
-        for i, controller in enumerate(self.controllers.instances):
-            if controller.name == last_screen:
-                controller_index = i
-        return controller_index
-
-    def select_initial_screen(self):
-        self.error_reporter.load_reports()
-        for report in self.error_reporter.reports:
-            if report.kind == ErrorReportKind.UI and not report.seen:
-                self.show_error_report(report.ref())
-                break
-        index = self.initial_controller_index()
-        for controller in self.controllers.instances[:index]:
-            controller.configured()
-        self.controllers.index = index - 1
-        self.next_screen()
-
-    async def move_screen(self, increment, coro):
-        try:
-            await super().move_screen(increment, coro)
-        except Confirm:
-            if self.interactive():
-                log.debug("showing InstallConfirmation over %s", self.ui.body)
-                from subiquity.ui.views.installprogress import (
-                    InstallConfirmation,
-                    )
-                self.add_global_overlay(InstallConfirmation(self))
-            else:
-                yes = _('yes')
-                no = _('no')
-                answer = no
-                if 'autoinstall' in self.kernel_cmdline:
-                    answer = yes
-                else:
-                    print(_("Confirmation is required to continue."))
-                    print(_("Add 'autoinstall' to your kernel command line to"
-                            " avoid this"))
-                    print()
-                prompt = "\n\n{} ({}|{})".format(
-                    _("Continue with autoinstall?"), yes, no)
-                while answer != yes:
-                    print(prompt)
-                    answer = input()
-                self.next_screen(self.confirm_install())
-
-    async def make_view_for_controller(self, new):
-        if self.base_model.needs_confirmation(new.model_name):
-            raise Confirm
-        if new.interactive():
-            view = await super().make_view_for_controller(new)
-            if new.answers:
-                self.aio_loop.call_soon(new.run_answers)
-            with open(self.state_path('last-screen'), 'w') as fp:
-                fp.write(new.name)
-            return view
-        else:
-            if self.autoinstall_config and not new.autoinstall_applied:
-                await new.apply_autoinstall_config()
-                new.autoinstall_applied = True
-            new.configured()
-            raise Skip
-
-    def show_progress(self):
-        self.ui.set_body(self.controllers.InstallProgress.progress_view)
-
-    def _network_change(self):
-        self.signal.emit_signal('snapd-network-change')
-
-    async def _proxy_set(self):
-        await run_in_thread(
-            self.snapd.connection.configure_proxy, self.base_model.proxy)
-        self.signal.emit_signal('snapd-network-change')
-
-    def unhandled_input(self, key):
-        if key == 'f1':
-            if not self.ui.right_icon.current_help:
-                self.ui.right_icon.open_pop_up()
-        elif key in ['ctrl z', 'f2']:
-            self.debug_shell()
-        elif self.opts.dry_run:
-            self.unhandled_input_dry_run(key)
-        else:
-            super().unhandled_input(key)
-
-    def unhandled_input_dry_run(self, key):
-        if key == 'ctrl g':
-            from systemd import journal
-
-            async def mock_install():
-                async with self.install_lock_file.exclusive():
-                    self.install_lock_file.write_content("nowhere")
-                    journal.send(
-                        "starting install", SYSLOG_IDENTIFIER="subiquity")
-                    await asyncio.sleep(5)
-            schedule_task(mock_install())
-        elif key in ['ctrl e', 'ctrl r']:
-            interrupt = key == 'ctrl e'
-            try:
-                1/0
-            except ZeroDivisionError:
-                self.make_apport_report(
-                    ErrorReportKind.UNKNOWN, "example", interrupt=interrupt)
-        elif key == 'ctrl u':
-            1/0
-        elif key == 'ctrl b':
-            self.aio_loop.create_task(self.client.dry_run.crash.GET())
-        else:
-            super().unhandled_input(key)
-
-    def debug_shell(self, after_hook=None):
-
-        def _before():
-            os.system("clear")
-            print(DEBUG_SHELL_INTRO)
-
-        self.run_command_in_foreground(
-            ["bash"], before_hook=_before, after_hook=after_hook, cwd='/')
-
-    def note_file_for_apport(self, key, path):
-        self.error_reporter.note_file_for_apport(key, path)
-
-    def note_data_for_apport(self, key, value):
-        self.error_reporter.note_data_for_apport(key, value)
-
-    def make_apport_report(self, kind, thing, *, interrupt, wait=False, **kw):
-        report = self.error_reporter.make_apport_report(
-            kind, thing, wait=wait, **kw)
-
-        if report is not None and interrupt and self.interactive():
-            self.show_error_report(report.ref())
-
-        return report
-
-    def show_error_report(self, error_ref):
-        log.debug("show_error_report %r", error_ref.base)
-        if isinstance(self.ui.body, BaseView):
-            w = getattr(self.ui.body._w, 'stretchy', None)
-            if isinstance(w, ErrorReportStretchy):
-                # Don't show an error if already looking at one.
-                return
-        self.add_global_overlay(ErrorReportStretchy(self, error_ref))
-
-    def make_autoinstall(self):
-        config = {'version': 1}
-        for controller in self.controllers.instances:
-            controller_conf = controller.make_autoinstall()
-            if controller_conf:
-                config[controller.autoinstall_key] = controller_conf
-        return config

subiquity/models/subiquity.py

@@ -136,28 +136,23 @@ class SubiquityModel:
         if model_name in INSTALL_MODEL_NAMES:
             unconfigured = {
                 mn for mn in INSTALL_MODEL_NAMES
-                if not self.is_configured(mn)
+                if not self._events[model_name].is_set()
                 }
         elif model_name in POSTINSTALL_MODEL_NAMES:
             unconfigured = {
                 mn for mn in POSTINSTALL_MODEL_NAMES
-                if not self.is_configured(mn)
+                if not self._events[model_name].is_set()
                 }
         log.debug("model %s is configured, to go %s", model_name, unconfigured)
 
-    def needs_confirmation(self, model_name):
+    def needs_configuration(self, model_name):
         if model_name is None:
             return False
-        if not all(e.is_set() for e in self.install_events):
+        return not self._events[model_name].is_set()
-            return None
-        return not self.confirmation.is_set()
 
     def confirm(self):
         self.confirmation.set()
 
-    def is_configured(self, model_name):
-        return self._events[model_name].is_set()
-
     def get_target_groups(self):
         command = ['chroot', self.target, 'getent', 'group']
         if self.root != '/':

subiquity/server/server.py

@@ -13,13 +13,25 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import asyncio
 import logging
 import os
+import shlex
 import sys
+from typing import List, Optional
 
 from aiohttp import web
 
+import jsonschema
+
+from systemd import journal
+
+import yaml
+
+from subiquitycore.async_helpers import run_in_thread, schedule_task
+from subiquitycore.context import with_context
 from subiquitycore.core import Application
+from subiquitycore.prober import Prober
 
 from subiquity.common.api.server import (
     bind,
@@ -33,10 +45,18 @@ from subiquity.common.errorreport import (
 from subiquity.common.serialize import to_json
 from subiquity.common.types import (
     ApplicationState,
+    ApplicationStatus,
     ErrorReportRef,
+    InstallState,
     )
 from subiquity.server.controller import SubiquityController
+from subiquity.models.subiquity import SubiquityModel
 from subiquity.server.errors import ErrorController
+from subiquitycore.snapd import (
+    AsyncSnapd,
+    FakeSnapdConnection,
+    SnapdConnection,
+    )
 
 
 log = logging.getLogger('subiquity.server.server')
@@ -48,28 +68,146 @@ class MetaController:
         self.app = app
         self.context = app.context.child("Meta")
 
-    async def status_GET(self) -> ApplicationState:
+    async def status_GET(self, cur: Optional[ApplicationState] = None) \
-        return self.app.status
+            -> ApplicationStatus:
+        if cur == self.app.state:
+            await self.app.state_event.wait()
+        return ApplicationStatus(
+            self.app.state,
+            early_commands_syslog_id=self.app.early_commands_syslog_id,
+            event_syslog_id=self.app.event_syslog_id,
+            log_syslog_id=self.app.log_syslog_id)
+
+    async def confirm_POST(self, tty: str) -> None:
+        self.app.confirming_tty = tty
+        self.app.base_model.confirm()
 
     async def restart_POST(self) -> None:
         self.app.restart()
 
+    async def mark_configured_POST(self, endpoint_names: List[str]) -> None:
+        endpoints = {getattr(API, en, None) for en in endpoint_names}
+        for controller in self.app.controllers.instances:
+            if controller.endpoint in endpoints:
+                controller.configured()
+
 
 class SubiquityServer(Application):
 
+    snapd_socket_path = '/run/snapd.socket'
+
+    base_schema = {
+        'type': 'object',
+        'properties': {
+            'version': {
+                'type': 'integer',
+                'minimum': 1,
+                'maximum': 1,
+                },
+            },
+        'required': ['version'],
+        'additionalProperties': True,
+        }
+
     project = "subiquity"
     from subiquity.server import controllers as controllers_mod
     controllers = []
 
     def make_model(self):
-        return None
+        root = '/'
+        if self.opts.dry_run:
+            root = os.path.abspath('.subiquity')
+        return SubiquityModel(root, self.opts.sources)
 
-    def __init__(self, opts):
+    def __init__(self, opts, block_log_dir):
         super().__init__(opts)
-        self.status = ApplicationState.STARTING
+        self.block_log_dir = block_log_dir
-        self.server_proc = None
+        self._state = ApplicationState.STARTING
+        self.state_event = asyncio.Event()
+        self.confirming_tty = ''
+
+        self.early_commands_syslog_id = 'subiquity_commands.{}'.format(
+            os.getpid())
+        self.event_syslog_id = 'subiquity_event.{}'.format(os.getpid())
+        self.log_syslog_id = 'subiquity_log.{}'.format(os.getpid())
+
         self.error_reporter = ErrorReporter(
             self.context.child("ErrorReporter"), self.opts.dry_run, self.root)
+        self.prober = Prober(opts.machine_config, self.debug_flags)
+        self.kernel_cmdline = shlex.split(opts.kernel_cmdline)
+        if opts.snaps_from_examples:
+            connection = FakeSnapdConnection(
+                os.path.join(
+                    os.path.dirname(
+                        os.path.dirname(
+                            os.path.dirname(__file__))),
+                    "examples", "snaps"),
+                self.scale_factor)
+        else:
+            connection = SnapdConnection(self.root, self.snapd_socket_path)
+        self.snapd = AsyncSnapd(connection)
+        self.note_data_for_apport("SnapUpdated", str(self.updated))
+        self.event_listeners = []
+        self.autoinstall_config = None
+        self.signal.connect_signals([
+            ('network-proxy-set', lambda: schedule_task(self._proxy_set())),
+            ('network-change', self._network_change),
+            ])
+
+    def load_serialized_state(self):
+        for controller in self.controllers.instances:
+            controller.load_state()
+
+    def add_event_listener(self, listener):
+        self.event_listeners.append(listener)
+
+    def _maybe_push_to_journal(self, event_type, context, description):
+        if not context.get('is-install-context') and self.interactive():
+            controller = context.get('controller')
+            if controller is None or controller.interactive():
+                return
+        if context.get('request'):
+            return
+        indent = context.full_name().count('/') - 2
+        if context.get('is-install-context') and self.interactive():
+            indent -= 1
+            msg = context.description
+        else:
+            msg = context.full_name()
+            if description:
+                msg += ': ' + description
+        msg = '  ' * indent + msg
+        if context.parent:
+            parent_id = str(context.parent.id)
+        else:
+            parent_id = ''
+        journal.send(
+            msg,
+            PRIORITY=context.level,
+            SYSLOG_IDENTIFIER=self.event_syslog_id,
+            SUBIQUITY_CONTEXT_NAME=context.full_name(),
+            SUBIQUITY_EVENT_TYPE=event_type,
+            SUBIQUITY_CONTEXT_ID=str(context.id),
+            SUBIQUITY_CONTEXT_PARENT_ID=parent_id)
+
+    def report_start_event(self, context, description):
+        for listener in self.event_listeners:
+            listener.report_start_event(context, description)
+        self._maybe_push_to_journal('start', context, description)
+
+    def report_finish_event(self, context, description, status):
+        for listener in self.event_listeners:
+            listener.report_finish_event(context, description, status)
+        self._maybe_push_to_journal('finish', context, description)
+
+    @property
+    def state(self):
+        return self._state
+
+    def update_state(self, state):
+        self._state = state
+        self.state_event.set()
+        self.state_event.clear()
 
     def note_file_for_apport(self, key, path):
         self.error_reporter.note_file_for_apport(key, path)
@@ -81,22 +219,30 @@ class SubiquityServer(Application):
         return self.error_reporter.make_apport_report(
             kind, thing, wait=wait, **kw)
 
+    def interactive(self):
+        if not self.autoinstall_config:
+            return True
+        return bool(self.autoinstall_config.get('interactive-sections'))
+
     @web.middleware
     async def middleware(self, request, handler):
-        if self.updated:
+        override_status = None
-            updated = 'yes'
-        else:
-            updated = 'no'
         controller = await controller_for_request(request)
         if isinstance(controller, SubiquityController):
+            install_state = self.controllers.Install.install_state
             if not controller.interactive():
-                return web.Response(
+                override_status = 'skip'
-                    headers={'x-status': 'skip', 'x-updated': updated})
+            elif install_state == InstallState.NEEDS_CONFIRMATION:
-            elif self.base_model.needs_confirmation(controller.model_name):
+                if self.base_model.needs_configuration(controller.model_name):
-                return web.Response(
+                    override_status = 'confirm'
-                    headers={'x-status': 'confirm', 'x-updated': updated})
+        if override_status is not None:
+            resp = web.Response(headers={'x-status': override_status})
+        else:
             resp = await handler(request)
-        resp.headers['x-updated'] = updated
+        if self.updated:
+            resp.headers['x-updated'] = 'yes'
+        else:
+            resp.headers['x-updated'] = 'no'
         if resp.get('exception'):
             exc = resp['exception']
             log.debug(
@@ -109,6 +255,34 @@ class SubiquityServer(Application):
                 ErrorReportRef, report.ref())
         return resp
 
+    @with_context()
+    async def apply_autoinstall_config(self, context):
+        for controller in self.controllers.instances:
+            if controller.interactive():
+                log.debug(
+                    "apply_autoinstall_config: skipping %s as interactive",
+                    controller.name)
+                continue
+            await controller.apply_autoinstall_config()
+            controller.configured()
+
+    def load_autoinstall_config(self, only_early):
+        log.debug("load_autoinstall_config only_early %s", only_early)
+        if self.opts.autoinstall is None:
+            return
+        with open(self.opts.autoinstall) as fp:
+            self.autoinstall_config = yaml.safe_load(fp)
+        if only_early:
+            self.controllers.Reporting.setup_autoinstall()
+            self.controllers.Reporting.start()
+            self.controllers.Error.setup_autoinstall()
+            with self.context.child("core_validation", level="INFO"):
+                jsonschema.validate(self.autoinstall_config, self.base_schema)
+            self.controllers.Early.setup_autoinstall()
+        else:
+            for controller in self.controllers.instances:
+                controller.setup_autoinstall()
+
     async def start_api_server(self):
         app = web.Application(middlewares=[self.middleware])
         bind(app.router, API.meta, MetaController(self))
@@ -116,14 +290,42 @@ class SubiquityServer(Application):
         if self.opts.dry_run:
             from .dryrun import DryRunController
             bind(app.router, API.dry_run, DryRunController(self))
+        for controller in self.controllers.instances:
+            controller.add_routes(app)
         runner = web.AppRunner(app)
         await runner.setup()
         site = web.UnixSite(runner, self.opts.socket)
         await site.start()
 
     async def start(self):
-        await super().start()
+        self.controllers.load_all()
         await self.start_api_server()
+        self.load_autoinstall_config(only_early=True)
+        if self.autoinstall_config and self.controllers.Early.cmds:
+            stamp_file = self.state_path("early-commands")
+            if not os.path.exists(stamp_file):
+                self.update_state(ApplicationState.EARLY_COMMANDS)
+                await self.controllers.Early.run()
+                open(stamp_file, 'w').close()
+        self.load_autoinstall_config(only_early=False)
+        if not self.interactive() and not self.opts.dry_run:
+            open('/run/casper-no-prompt', 'w').close()
+        self.load_serialized_state()
+        if self.interactive():
+            self.update_state(ApplicationState.INTERACTIVE)
+        else:
+            self.update_state(ApplicationState.NON_INTERACTIVE)
+            await asyncio.sleep(1)
+        await super().start()
+        await self.apply_autoinstall_config()
+
+    def _network_change(self):
+        self.signal.emit_signal('snapd-network-change')
+
+    async def _proxy_set(self):
+        await run_in_thread(
+            self.snapd.connection.configure_proxy, self.base_model.proxy)
+        self.signal.emit_signal('snapd-network-change')
 
     def restart(self):
         cmdline = ['snap', 'run', 'subiquity']
@@ -132,3 +334,11 @@ class SubiquityServer(Application):
                 sys.executable, '-m', 'subiquity.cmd.server',
                 ] + sys.argv[1:]
         os.execvp(cmdline[0], cmdline)
+
+    def make_autoinstall(self):
+        config = {'version': 1}
+        for controller in self.controllers.instances:
+            controller_conf = controller.make_autoinstall()
+            if controller_conf:
+                config[controller.autoinstall_key] = controller_conf
+        return config

subiquitycore/core.py

@@ -129,12 +129,8 @@ class Application:
 
     def run(self):
         self.base_model = self.make_model()
-        try:
         self.aio_loop.create_task(self.start())
         self.aio_loop.run_forever()
-        finally:
-            self.aio_loop.run_until_complete(
-                self.aio_loop.shutdown_asyncgens())
         if self._exc:
             exc, self._exc = self._exc, None
             raise exc