add a minimal server process

run it automatically in dry-run mode

console_conf/core.py

@@ -15,6 +15,7 @@
 
 import logging
 
+from subiquitycore.prober import Prober
 from subiquitycore.tui import TuiApplication
 
 from console_conf.models.console_conf import ConsoleConfModel
@@ -60,6 +61,7 @@ class RecoveryChooser(TuiApplication):
         )
 
         super().__init__(opts)
+        self.prober = Prober(opts.machine_config, self.debug_flags)
 
     def respond(self, choice):
         """Produce a response to the parent process"""

examples/answers.yaml

@@ -21,7 +21,7 @@ Identity:
   hostname: ubuntu-server
   # ubuntu
   password: '$6$wdAcoXrU039hKYPd$508Qvbe7ObUnxoj15DRCkzC3qO7edjH0VV7BPNRDYK4QR8ofJaEEF2heacn0QgD.f8pO8SNp83XNdWG6tocBM1'
-  ssh-import-id: lp:mwhudson
+  ssh-import-id: gh:mwhudson
 SnapList:
   snaps:
     hello:

subiquity/cmd/common.py

@@ -0,0 +1,38 @@
+# Copyright 2020 Canonical, Ltd.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import locale
+import os
+
+LOGDIR = "/var/log/installer/"
+
+
+def setup_environment():
+    # Python 3.7+ does more or less this by default, but we need to
+    # work with the Python 3.6 in bionic.
+    try:
+        locale.setlocale(locale.LC_ALL, "")
+    except locale.Error:
+        locale.setlocale(locale.LC_CTYPE, "C.UTF-8")
+
+    # Prefer utils from $SNAP, over system-wide
+    snap = os.environ.get('SNAP')
+    if snap:
+        os.environ['PATH'] = os.pathsep.join([
+            os.path.join(snap, 'bin'),
+            os.path.join(snap, 'usr', 'bin'),
+            os.environ['PATH'],
+        ])
+        os.environ["APPORT_DATA_DIR"] = os.path.join(snap, 'share/apport')

subiquity/cmd/server.py

@@ -0,0 +1,76 @@
+# Copyright 2020 Canonical, Ltd.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import argparse
+import logging
+import os
+import sys
+
+from subiquitycore.log import setup_logger
+
+from .common import (
+    LOGDIR,
+    setup_environment,
+    )
+
+
+def make_server_args_parser():
+    parser = argparse.ArgumentParser(
+        description='SUbiquity - Ubiquity for Servers',
+        prog='subiquity')
+    parser.add_argument('--dry-run', action='store_true',
+                        dest='dry_run',
+                        help='menu-only, do not call installer function')
+    parser.add_argument('--socket')
+    return parser
+
+
+def main():
+    print('starting server')
+    setup_environment()
+    # setup_environment sets $APPORT_DATA_DIR which must be set before
+    # apport is imported, which is done by this import:
+    from subiquity.server.server import SubiquityServer
+    parser = make_server_args_parser()
+    opts = parser.parse_args(sys.argv[1:])
+    logdir = LOGDIR
+    if opts.dry_run:
+        logdir = ".subiquity"
+    if opts.socket is None:
+        if opts.dry_run:
+            opts.socket = '.subiquity/socket'
+        else:
+            opts.socket = '/run/subiquity/socket'
+    os.makedirs(os.path.basename(opts.socket), exist_ok=True)
+
+    logfiles = setup_logger(dir=logdir, base='subiquity-server')
+
+    logger = logging.getLogger('subiquity')
+    version = os.environ.get("SNAP_REVISION", "unknown")
+    logger.info("Starting Subiquity server revision {}".format(version))
+    logger.info("Arguments passed: {}".format(sys.argv))
+
+    subiquity_interface = SubiquityServer(opts)
+
+    subiquity_interface.note_file_for_apport(
+        "InstallerServerLog", logfiles['debug'])
+    subiquity_interface.note_file_for_apport(
+        "InstallerServerLogInfo", logfiles['info'])
+
+    subiquity_interface.run()
+
+
+if __name__ == '__main__':
+    sys.exit(main())

subiquity/cmd/tui.py

@@ -15,10 +15,10 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import argparse
-import locale
 import logging
 import os
 import fcntl
+import subprocess
 import sys
 import time
 
@@ -27,13 +27,19 @@ from cloudinit import atomic_helper, safeyaml, stages
 from subiquitycore.log import setup_logger
 from subiquitycore.utils import run_command
 
+from .common import (
+    LOGDIR,
+    setup_environment,
+    )
+from .server import make_server_args_parser
+
 
 class ClickAction(argparse.Action):
     def __call__(self, parser, namespace, values, option_string=None):
         namespace.scripts.append("c(" + repr(values) + ")")
 
 
-def parse_options(argv):
+def make_client_args_parser():
     parser = argparse.ArgumentParser(
         description='SUbiquity - Ubiquity for Servers',
         prog='subiquity')
@@ -45,6 +51,7 @@ def parse_options(argv):
     parser.add_argument('--dry-run', action='store_true',
                         dest='dry_run',
                         help='menu-only, do not call installer function')
+    parser.add_argument('--socket')
     parser.add_argument('--serial', action='store_true',
                         dest='run_on_serial',
                         help='Run the installer over serial console.')
@@ -95,40 +102,48 @@ def parse_options(argv):
         '--snap-section', action='store', default='server',
         help=("Show snaps from this section of the store in the snap "
               "list screen."))
-    return parser.parse_args(argv)
+    return parser
 
 
-LOGDIR = "/var/log/installer/"
-
 AUTO_ANSWERS_FILE = "/subiquity_config/answers.yaml"
 
 
 def main():
-    # Python 3.7+ does more or less this by default, but we need to
-    # work with the Python 3.6 in bionic.
-    try:
-        locale.setlocale(locale.LC_ALL, "")
-    except locale.Error:
-        locale.setlocale(locale.LC_CTYPE, "C.UTF-8")
-
-    # Prefer utils from $SNAP, over system-wide
-    snap = os.environ.get('SNAP')
-    if snap:
-        os.environ['PATH'] = os.pathsep.join([
-            os.path.join(snap, 'bin'),
-            os.path.join(snap, 'usr', 'bin'),
-            os.environ['PATH'],
-        ])
-        os.environ["APPORT_DATA_DIR"] = os.path.join(snap, 'share/apport')
-    # This must come after setting $APPORT_DATA_DIR.
+    setup_environment()
+    # setup_environment sets $APPORT_DATA_DIR which must be set before
+    # apport is imported, which is done by this import:
     from subiquity.core import Subiquity
-    opts = parse_options(sys.argv[1:])
-    global LOGDIR
+    parser = make_client_args_parser()
+    args = sys.argv[1:]
+    server_proc = None
+    if '--dry-run' in args:
+        opts, unknown = parser.parse_known_args(args)
+        if opts.socket is None:
+            os.makedirs('.subiquity', exist_ok=True)
+            sock_path = '.subiquity/socket'
+            opts.socket = sock_path
+            server_args = ['--dry-run', '--socket=' + sock_path] + unknown
+            server_parser = make_server_args_parser()
+            server_parser.parse_args(server_args)  # just to check
+            server_output = open('.subiquity/server-output', 'w')
+            server_cmd = [sys.executable, '-m', 'subiquity.cmd.server'] + \
+                server_args
+            server_proc = subprocess.Popen(
+                server_cmd, stdout=server_output, stderr=subprocess.STDOUT)
+            print("running server pid {}".format(server_proc.pid))
+        else:
+            opts = parser.parse_args(args)
+    else:
+        opts = parser.parse_args(args)
+        if opts.socket is None:
+            opts.socket = '/run/subiquity/socket'
+    os.makedirs(os.path.basename(opts.socket), exist_ok=True)
+    logdir = LOGDIR
     if opts.dry_run:
-        LOGDIR = ".subiquity"
         if opts.snaps_from_examples is None:
             opts.snaps_from_examples = True
-    logfiles = setup_logger(dir=LOGDIR)
+        logdir = ".subiquity"
+    logfiles = setup_logger(dir=logdir, base='subiquity')
 
     logger = logging.getLogger('subiquity')
     version = os.environ.get("SNAP_REVISION", "unknown")
@@ -160,7 +175,7 @@ def main():
                 "cloud-init status: %r, assumed disabled",
                 status_txt)
 
-    block_log_dir = os.path.join(LOGDIR, "block")
+    block_log_dir = os.path.join(logdir, "block")
     os.makedirs(block_log_dir, exist_ok=True)
     handler = logging.FileHandler(os.path.join(block_log_dir, 'discover.log'))
     handler.setLevel('DEBUG')
@@ -202,13 +217,20 @@ def main():
             opts.answers = None
 
     subiquity_interface = Subiquity(opts, block_log_dir)
+    subiquity_interface.server_proc = server_proc
 
     subiquity_interface.note_file_for_apport(
         "InstallerLog", logfiles['debug'])
     subiquity_interface.note_file_for_apport(
         "InstallerLogInfo", logfiles['info'])
 
-    subiquity_interface.run()
+    try:
+        subiquity_interface.run()
+    finally:
+        if server_proc is not None:
+            print('killing server {}'.format(server_proc.pid))
+            server_proc.send_signal(2)
+            server_proc.wait()
 
 
 if __name__ == '__main__':

subiquity/common/apidef.py

@@ -0,0 +1,29 @@
+# Copyright 2020 Canonical, Ltd.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from subiquity.common.api.defs import api
+from subiquity.common.types import (
+    ApplicationState,
+    )
+
+
+@api
+class API:
+    """The API offered by the subiquity installer process."""
+
+    class meta:
+        class status:
+            def GET() -> ApplicationState:
+                """Get the installer state."""

subiquity/common/types.py

@@ -25,6 +25,10 @@ from typing import List, Optional
 import attr
 
 
+class ApplicationState(enum.Enum):
+    STARTING = enum.auto()
+
+
 class ErrorReportState(enum.Enum):
     INCOMPLETE = enum.auto()
     LOADING = enum.auto()

subiquity/core.py

@@ -23,6 +23,8 @@ import sys
 import traceback
 import urwid
 
+import aiohttp
+
 import jsonschema
 
 import yaml
@@ -31,6 +33,7 @@ from subiquitycore.async_helpers import (
     run_in_thread,
     schedule_task,
     )
+from subiquitycore.prober import Prober
 from subiquitycore.screen import is_linux_tty
 from subiquitycore.tuicontroller import Skip
 from subiquitycore.tui import TuiApplication
@@ -41,6 +44,8 @@ from subiquitycore.snapd import (
     )
 from subiquitycore.view import BaseView
 
+from subiquity.common.api.client import make_client_for_conn
+from subiquity.common.apidef import API
 from subiquity.common.errorreport import (
     ErrorReporter,
     ErrorReportKind,
@@ -137,6 +142,7 @@ class Subiquity(TuiApplication):
 
         self.help_menu = HelpMenu(self)
         super().__init__(opts)
+        self.prober = Prober(opts.machine_config, self.debug_flags)
         journald_listen(
             self.aio_loop, ["subiquity"], self.subiquity_event, seek=True)
         self.event_listeners = []
@@ -159,6 +165,9 @@ class Subiquity(TuiApplication):
             ('network-change', self._network_change),
             ])
 
+        self.conn = aiohttp.UnixConnector(self.opts.socket)
+        self.client = make_client_for_conn(API, self.conn)
+
         self.autoinstall_config = {}
         self.report_to_show = None
         self.show_progress_handle = None
@@ -272,7 +281,20 @@ class Subiquity(TuiApplication):
             # in next_screen below will be confusing.
             os.system('stty sane')
 
+    async def connect(self):
+        print("connecting...", end='', flush=True)
+        while True:
+            try:
+                await self.client.meta.status.GET()
+            except aiohttp.ClientError:
+                await asyncio.sleep(1)
+                print(".", end='', flush=True)
+            else:
+                print()
+                break
+
     async def start(self):
+        await self.connect()
         if self.opts.autoinstall is not None:
             await self.load_autoinstall_config()
             if not self.interactive() and not self.opts.dry_run:

subiquity/server/controllers/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2020 Canonical, Ltd.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.

subiquity/server/server.py

@@ -0,0 +1,81 @@
+# Copyright 2020 Canonical, Ltd.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+
+from aiohttp import web
+
+from subiquitycore.core import Application
+
+from subiquity.common.api.server import bind
+from subiquity.common.apidef import API
+from subiquity.common.errorreport import (
+    ErrorReporter,
+    )
+from subiquity.common.types import (
+    ApplicationState,
+    )
+
+
+log = logging.getLogger('subiquity.server.server')
+
+
+class MetaController:
+
+    def __init__(self, app):
+        self.app = app
+        self.context = app.context.child("Meta")
+
+    async def status_GET(self) -> ApplicationState:
+        return self.app.status
+
+
+class SubiquityServer(Application):
+
+    project = "subiquity"
+    from subiquity.server import controllers as controllers_mod
+    controllers = []
+
+    def make_model(self):
+        return None
+
+    def __init__(self, opts):
+        super().__init__(opts)
+        self.status = ApplicationState.STARTING
+        self.server_proc = None
+        self.error_reporter = ErrorReporter(
+            self.context.child("ErrorReporter"), self.opts.dry_run, self.root)
+
+    def note_file_for_apport(self, key, path):
+        self.error_reporter.note_file_for_apport(key, path)
+
+    def note_data_for_apport(self, key, value):
+        self.error_reporter.note_data_for_apport(key, value)
+
+    def make_apport_report(self, kind, thing, *, wait=False, **kw):
+        return self.error_reporter.make_apport_report(
+            kind, thing, wait=wait, **kw)
+
+    async def start_api_server(self):
+        app = web.Application()
+        bind(app.router, API.meta, MetaController(self))
+        runner = web.AppRunner(app)
+        await runner.setup()
+        site = web.UnixSite(runner, self.opts.socket)
+        await site.start()
+
+    async def start(self):
+        await super().start()
+        await self.start_api_server()

subiquitycore/core.py

@@ -22,7 +22,6 @@ from subiquitycore.context import (
     Context,
     )
 from subiquitycore.controllerset import ControllerSet
-from subiquitycore.prober import Prober
 from subiquitycore.signals import Signal
 
 log = logging.getLogger('subiquitycore.core')
@@ -58,8 +57,6 @@ class Application:
             #    subiquity/controllers/installprogress.py
             self.debug_flags = os.environ.get('SUBIQUITY_DEBUG', '').split(',')
 
-        prober = Prober(opts.machine_config, self.debug_flags)
-
         self.opts = opts
         opts.project = self.project
 
@@ -73,7 +70,6 @@ class Application:
             os.environ.get('SUBIQUITY_REPLAY_TIMESCALE', "1"))
         self.updated = os.path.exists(self.state_path('updating'))
         self.signal = Signal()
-        self.prober = prober
         self.aio_loop = asyncio.get_event_loop()
         self.aio_loop.set_exception_handler(self._exception_handler)
         self.controllers = ControllerSet(

subiquitycore/log.py

@@ -17,7 +17,7 @@ import logging
 import os
 
 
-def setup_logger(dir):
+def setup_logger(dir, base='subiquity'):
     os.makedirs(dir, exist_ok=True)
 
     logger = logging.getLogger("")
@@ -26,7 +26,7 @@ def setup_logger(dir):
     r = {}
 
     for level in 'info', 'debug':
-        nopid_file = os.path.join(dir, "subiquity-{}.log".format(level))
+        nopid_file = os.path.join(dir, "{}-{}.log".format(base, level))
         logfile = "{}.{}".format(nopid_file, os.getpid())
         handler = logging.FileHandler(logfile, mode='w')
         # os.symlink cannot replace an existing file or symlink so create