diff --git a/console_conf/core.py b/console_conf/core.py index 588baaf3..b7c916ec 100644 --- a/console_conf/core.py +++ b/console_conf/core.py @@ -15,6 +15,7 @@ import logging +from subiquitycore.prober import Prober from subiquitycore.tui import TuiApplication from console_conf.models.console_conf import ConsoleConfModel @@ -60,6 +61,7 @@ class RecoveryChooser(TuiApplication): ) super().__init__(opts) + self.prober = Prober(opts.machine_config, self.debug_flags) def respond(self, choice): """Produce a response to the parent process""" diff --git a/examples/answers.yaml b/examples/answers.yaml index c5df51e9..e8274506 100644 --- a/examples/answers.yaml +++ b/examples/answers.yaml @@ -21,7 +21,7 @@ Identity: hostname: ubuntu-server # ubuntu password: '$6$wdAcoXrU039hKYPd$508Qvbe7ObUnxoj15DRCkzC3qO7edjH0VV7BPNRDYK4QR8ofJaEEF2heacn0QgD.f8pO8SNp83XNdWG6tocBM1' - ssh-import-id: lp:mwhudson + ssh-import-id: gh:mwhudson SnapList: snaps: hello: diff --git a/subiquity/cmd/common.py b/subiquity/cmd/common.py new file mode 100644 index 00000000..5fbc1788 --- /dev/null +++ b/subiquity/cmd/common.py @@ -0,0 +1,38 @@ +# Copyright 2020 Canonical, Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import locale +import os + +LOGDIR = "/var/log/installer/" + + +def setup_environment(): + # Python 3.7+ does more or less this by default, but we need to + # work with the Python 3.6 in bionic. + try: + locale.setlocale(locale.LC_ALL, "") + except locale.Error: + locale.setlocale(locale.LC_CTYPE, "C.UTF-8") + + # Prefer utils from $SNAP, over system-wide + snap = os.environ.get('SNAP') + if snap: + os.environ['PATH'] = os.pathsep.join([ + os.path.join(snap, 'bin'), + os.path.join(snap, 'usr', 'bin'), + os.environ['PATH'], + ]) + os.environ["APPORT_DATA_DIR"] = os.path.join(snap, 'share/apport') diff --git a/subiquity/cmd/server.py b/subiquity/cmd/server.py new file mode 100644 index 00000000..34966c71 --- /dev/null +++ b/subiquity/cmd/server.py @@ -0,0 +1,76 @@ +# Copyright 2020 Canonical, Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import argparse +import logging +import os +import sys + +from subiquitycore.log import setup_logger + +from .common import ( + LOGDIR, + setup_environment, + ) + + +def make_server_args_parser(): + parser = argparse.ArgumentParser( + description='SUbiquity - Ubiquity for Servers', + prog='subiquity') + parser.add_argument('--dry-run', action='store_true', + dest='dry_run', + help='menu-only, do not call installer function') + parser.add_argument('--socket') + return parser + + +def main(): + print('starting server') + setup_environment() + # setup_environment sets $APPORT_DATA_DIR which must be set before + # apport is imported, which is done by this import: + from subiquity.server.server import SubiquityServer + parser = make_server_args_parser() + opts = parser.parse_args(sys.argv[1:]) + logdir = LOGDIR + if opts.dry_run: + logdir = ".subiquity" + if opts.socket is None: + if opts.dry_run: + opts.socket = '.subiquity/socket' + else: + opts.socket = '/run/subiquity/socket' + os.makedirs(os.path.basename(opts.socket), exist_ok=True) + + logfiles = setup_logger(dir=logdir, base='subiquity-server') + + logger = logging.getLogger('subiquity') + version = os.environ.get("SNAP_REVISION", "unknown") + logger.info("Starting Subiquity server revision {}".format(version)) + logger.info("Arguments passed: {}".format(sys.argv)) + + subiquity_interface = SubiquityServer(opts) + + subiquity_interface.note_file_for_apport( + "InstallerServerLog", logfiles['debug']) + subiquity_interface.note_file_for_apport( + "InstallerServerLogInfo", logfiles['info']) + + subiquity_interface.run() + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/subiquity/cmd/tui.py b/subiquity/cmd/tui.py index c5835fc5..c20d6fa9 100755 --- a/subiquity/cmd/tui.py +++ b/subiquity/cmd/tui.py @@ -15,10 +15,10 @@ # along with this program. If not, see . import argparse -import locale import logging import os import fcntl +import subprocess import sys import time @@ -27,13 +27,19 @@ from cloudinit import atomic_helper, safeyaml, stages from subiquitycore.log import setup_logger from subiquitycore.utils import run_command +from .common import ( + LOGDIR, + setup_environment, + ) +from .server import make_server_args_parser + class ClickAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): namespace.scripts.append("c(" + repr(values) + ")") -def parse_options(argv): +def make_client_args_parser(): parser = argparse.ArgumentParser( description='SUbiquity - Ubiquity for Servers', prog='subiquity') @@ -45,6 +51,7 @@ def parse_options(argv): parser.add_argument('--dry-run', action='store_true', dest='dry_run', help='menu-only, do not call installer function') + parser.add_argument('--socket') parser.add_argument('--serial', action='store_true', dest='run_on_serial', help='Run the installer over serial console.') @@ -95,40 +102,48 @@ def parse_options(argv): '--snap-section', action='store', default='server', help=("Show snaps from this section of the store in the snap " "list screen.")) - return parser.parse_args(argv) + return parser -LOGDIR = "/var/log/installer/" - AUTO_ANSWERS_FILE = "/subiquity_config/answers.yaml" def main(): - # Python 3.7+ does more or less this by default, but we need to - # work with the Python 3.6 in bionic. - try: - locale.setlocale(locale.LC_ALL, "") - except locale.Error: - locale.setlocale(locale.LC_CTYPE, "C.UTF-8") - - # Prefer utils from $SNAP, over system-wide - snap = os.environ.get('SNAP') - if snap: - os.environ['PATH'] = os.pathsep.join([ - os.path.join(snap, 'bin'), - os.path.join(snap, 'usr', 'bin'), - os.environ['PATH'], - ]) - os.environ["APPORT_DATA_DIR"] = os.path.join(snap, 'share/apport') - # This must come after setting $APPORT_DATA_DIR. + setup_environment() + # setup_environment sets $APPORT_DATA_DIR which must be set before + # apport is imported, which is done by this import: from subiquity.core import Subiquity - opts = parse_options(sys.argv[1:]) - global LOGDIR + parser = make_client_args_parser() + args = sys.argv[1:] + server_proc = None + if '--dry-run' in args: + opts, unknown = parser.parse_known_args(args) + if opts.socket is None: + os.makedirs('.subiquity', exist_ok=True) + sock_path = '.subiquity/socket' + opts.socket = sock_path + server_args = ['--dry-run', '--socket=' + sock_path] + unknown + server_parser = make_server_args_parser() + server_parser.parse_args(server_args) # just to check + server_output = open('.subiquity/server-output', 'w') + server_cmd = [sys.executable, '-m', 'subiquity.cmd.server'] + \ + server_args + server_proc = subprocess.Popen( + server_cmd, stdout=server_output, stderr=subprocess.STDOUT) + print("running server pid {}".format(server_proc.pid)) + else: + opts = parser.parse_args(args) + else: + opts = parser.parse_args(args) + if opts.socket is None: + opts.socket = '/run/subiquity/socket' + os.makedirs(os.path.basename(opts.socket), exist_ok=True) + logdir = LOGDIR if opts.dry_run: - LOGDIR = ".subiquity" if opts.snaps_from_examples is None: opts.snaps_from_examples = True - logfiles = setup_logger(dir=LOGDIR) + logdir = ".subiquity" + logfiles = setup_logger(dir=logdir, base='subiquity') logger = logging.getLogger('subiquity') version = os.environ.get("SNAP_REVISION", "unknown") @@ -160,7 +175,7 @@ def main(): "cloud-init status: %r, assumed disabled", status_txt) - block_log_dir = os.path.join(LOGDIR, "block") + block_log_dir = os.path.join(logdir, "block") os.makedirs(block_log_dir, exist_ok=True) handler = logging.FileHandler(os.path.join(block_log_dir, 'discover.log')) handler.setLevel('DEBUG') @@ -202,13 +217,20 @@ def main(): opts.answers = None subiquity_interface = Subiquity(opts, block_log_dir) + subiquity_interface.server_proc = server_proc subiquity_interface.note_file_for_apport( "InstallerLog", logfiles['debug']) subiquity_interface.note_file_for_apport( "InstallerLogInfo", logfiles['info']) - subiquity_interface.run() + try: + subiquity_interface.run() + finally: + if server_proc is not None: + print('killing server {}'.format(server_proc.pid)) + server_proc.send_signal(2) + server_proc.wait() if __name__ == '__main__': diff --git a/subiquity/common/apidef.py b/subiquity/common/apidef.py new file mode 100644 index 00000000..1079bc3c --- /dev/null +++ b/subiquity/common/apidef.py @@ -0,0 +1,29 @@ +# Copyright 2020 Canonical, Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from subiquity.common.api.defs import api +from subiquity.common.types import ( + ApplicationState, + ) + + +@api +class API: + """The API offered by the subiquity installer process.""" + + class meta: + class status: + def GET() -> ApplicationState: + """Get the installer state.""" diff --git a/subiquity/common/types.py b/subiquity/common/types.py index 35fcba18..ab553ef6 100644 --- a/subiquity/common/types.py +++ b/subiquity/common/types.py @@ -25,6 +25,10 @@ from typing import List, Optional import attr +class ApplicationState(enum.Enum): + STARTING = enum.auto() + + class ErrorReportState(enum.Enum): INCOMPLETE = enum.auto() LOADING = enum.auto() diff --git a/subiquity/core.py b/subiquity/core.py index a9e54d80..abc65865 100644 --- a/subiquity/core.py +++ b/subiquity/core.py @@ -23,6 +23,8 @@ import sys import traceback import urwid +import aiohttp + import jsonschema import yaml @@ -31,6 +33,7 @@ from subiquitycore.async_helpers import ( run_in_thread, schedule_task, ) +from subiquitycore.prober import Prober from subiquitycore.screen import is_linux_tty from subiquitycore.tuicontroller import Skip from subiquitycore.tui import TuiApplication @@ -41,6 +44,8 @@ from subiquitycore.snapd import ( ) from subiquitycore.view import BaseView +from subiquity.common.api.client import make_client_for_conn +from subiquity.common.apidef import API from subiquity.common.errorreport import ( ErrorReporter, ErrorReportKind, @@ -137,6 +142,7 @@ class Subiquity(TuiApplication): self.help_menu = HelpMenu(self) super().__init__(opts) + self.prober = Prober(opts.machine_config, self.debug_flags) journald_listen( self.aio_loop, ["subiquity"], self.subiquity_event, seek=True) self.event_listeners = [] @@ -159,6 +165,9 @@ class Subiquity(TuiApplication): ('network-change', self._network_change), ]) + self.conn = aiohttp.UnixConnector(self.opts.socket) + self.client = make_client_for_conn(API, self.conn) + self.autoinstall_config = {} self.report_to_show = None self.show_progress_handle = None @@ -272,7 +281,20 @@ class Subiquity(TuiApplication): # in next_screen below will be confusing. os.system('stty sane') + async def connect(self): + print("connecting...", end='', flush=True) + while True: + try: + await self.client.meta.status.GET() + except aiohttp.ClientError: + await asyncio.sleep(1) + print(".", end='', flush=True) + else: + print() + break + async def start(self): + await self.connect() if self.opts.autoinstall is not None: await self.load_autoinstall_config() if not self.interactive() and not self.opts.dry_run: diff --git a/subiquity/server/controllers/__init__.py b/subiquity/server/controllers/__init__.py new file mode 100644 index 00000000..8e549e25 --- /dev/null +++ b/subiquity/server/controllers/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2020 Canonical, Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . diff --git a/subiquity/server/server.py b/subiquity/server/server.py new file mode 100644 index 00000000..6e1dfe7b --- /dev/null +++ b/subiquity/server/server.py @@ -0,0 +1,81 @@ +# Copyright 2020 Canonical, Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging + +from aiohttp import web + +from subiquitycore.core import Application + +from subiquity.common.api.server import bind +from subiquity.common.apidef import API +from subiquity.common.errorreport import ( + ErrorReporter, + ) +from subiquity.common.types import ( + ApplicationState, + ) + + +log = logging.getLogger('subiquity.server.server') + + +class MetaController: + + def __init__(self, app): + self.app = app + self.context = app.context.child("Meta") + + async def status_GET(self) -> ApplicationState: + return self.app.status + + +class SubiquityServer(Application): + + project = "subiquity" + from subiquity.server import controllers as controllers_mod + controllers = [] + + def make_model(self): + return None + + def __init__(self, opts): + super().__init__(opts) + self.status = ApplicationState.STARTING + self.server_proc = None + self.error_reporter = ErrorReporter( + self.context.child("ErrorReporter"), self.opts.dry_run, self.root) + + def note_file_for_apport(self, key, path): + self.error_reporter.note_file_for_apport(key, path) + + def note_data_for_apport(self, key, value): + self.error_reporter.note_data_for_apport(key, value) + + def make_apport_report(self, kind, thing, *, wait=False, **kw): + return self.error_reporter.make_apport_report( + kind, thing, wait=wait, **kw) + + async def start_api_server(self): + app = web.Application() + bind(app.router, API.meta, MetaController(self)) + runner = web.AppRunner(app) + await runner.setup() + site = web.UnixSite(runner, self.opts.socket) + await site.start() + + async def start(self): + await super().start() + await self.start_api_server() diff --git a/subiquitycore/core.py b/subiquitycore/core.py index e84ade70..f2f704ef 100644 --- a/subiquitycore/core.py +++ b/subiquitycore/core.py @@ -22,7 +22,6 @@ from subiquitycore.context import ( Context, ) from subiquitycore.controllerset import ControllerSet -from subiquitycore.prober import Prober from subiquitycore.signals import Signal log = logging.getLogger('subiquitycore.core') @@ -58,8 +57,6 @@ class Application: # subiquity/controllers/installprogress.py self.debug_flags = os.environ.get('SUBIQUITY_DEBUG', '').split(',') - prober = Prober(opts.machine_config, self.debug_flags) - self.opts = opts opts.project = self.project @@ -73,7 +70,6 @@ class Application: os.environ.get('SUBIQUITY_REPLAY_TIMESCALE', "1")) self.updated = os.path.exists(self.state_path('updating')) self.signal = Signal() - self.prober = prober self.aio_loop = asyncio.get_event_loop() self.aio_loop.set_exception_handler(self._exception_handler) self.controllers = ControllerSet( diff --git a/subiquitycore/log.py b/subiquitycore/log.py index 3cac07d8..dfdfe689 100644 --- a/subiquitycore/log.py +++ b/subiquitycore/log.py @@ -17,7 +17,7 @@ import logging import os -def setup_logger(dir): +def setup_logger(dir, base='subiquity'): os.makedirs(dir, exist_ok=True) logger = logging.getLogger("") @@ -26,7 +26,7 @@ def setup_logger(dir): r = {} for level in 'info', 'debug': - nopid_file = os.path.join(dir, "subiquity-{}.log".format(level)) + nopid_file = os.path.join(dir, "{}-{}.log".format(base, level)) logfile = "{}.{}".format(nopid_file, os.getpid()) handler = logging.FileHandler(logfile, mode='w') # os.symlink cannot replace an existing file or symlink so create