From a6270cbaa1cd25cfeb62b70b61908f56dd02cbc3 Mon Sep 17 00:00:00 2001 From: Michael Hudson-Doyle Date: Thu, 19 Aug 2021 16:47:03 +1200 Subject: [PATCH] server side implementation of source selection --- po/POTFILES.in | 5 + subiquity/cmd/server.py | 2 + subiquity/common/apidef.py | 5 + subiquity/common/types.py | 16 +++ subiquity/models/source.py | 92 ++++++++++++++ subiquity/models/subiquity.py | 7 +- subiquity/models/tests/test_source.py | 117 ++++++++++++++++++ subiquity/server/controllers/__init__.py | 2 + subiquity/server/controllers/source.py | 83 +++++++++++++ .../server/controllers/tests/__init__.py | 1 + .../server/controllers/tests/test_source.py | 65 ++++++++++ subiquity/server/server.py | 10 +- subiquitycore/tests/util.py | 6 + 13 files changed, 406 insertions(+), 5 deletions(-) create mode 100644 subiquity/models/source.py create mode 100644 subiquity/models/tests/test_source.py create mode 100644 subiquity/server/controllers/source.py create mode 100644 subiquity/server/controllers/tests/__init__.py create mode 100644 subiquity/server/controllers/tests/test_source.py diff --git a/po/POTFILES.in b/po/POTFILES.in index 6efa40b2..387aa76b 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -118,11 +118,13 @@ subiquity/models/mirror.py subiquity/models/network.py subiquity/models/proxy.py subiquity/models/snaplist.py +subiquity/models/source.py subiquity/models/ssh.py subiquity/models/subiquity.py subiquity/models/tests/__init__.py subiquity/models/tests/test_filesystem.py subiquity/models/tests/test_mirror.py +subiquity/models/tests/test_source.py subiquity/models/tests/test_subiquity.py subiquity/models/timezone.py subiquity/models/updates.py @@ -144,8 +146,11 @@ subiquity/server/controllers/refresh.py subiquity/server/controllers/reporting.py subiquity/server/controllers/shutdown.py subiquity/server/controllers/snaplist.py +subiquity/server/controllers/source.py subiquity/server/controllers/ssh.py +subiquity/server/controllers/tests/__init__.py subiquity/server/controllers/tests/test_keyboard.py +subiquity/server/controllers/tests/test_source.py subiquity/server/controllers/timezone.py subiquity/server/controllers/updates.py subiquity/server/controllers/userdata.py diff --git a/subiquity/cmd/server.py b/subiquity/cmd/server.py index 54244f4c..7e5c88d1 100644 --- a/subiquity/cmd/server.py +++ b/subiquity/cmd/server.py @@ -61,6 +61,8 @@ def make_server_args_parser(): '--snap-section', action='store', default='server', help=("Show snaps from this section of the store in the snap " "list screen.")) + parser.add_argument( + '--source-catalog', dest='source_catalog', action='store') return parser diff --git a/subiquity/common/apidef.py b/subiquity/common/apidef.py index e8400bc4..e1096ea2 100644 --- a/subiquity/common/apidef.py +++ b/subiquity/common/apidef.py @@ -41,6 +41,7 @@ from subiquity.common.types import ( SnapInfo, SnapListResponse, SnapSelection, + SourceSelectionAndSetting, SSHData, LiveSessionSSHInfo, StorageResponse, @@ -123,6 +124,10 @@ class API: class steps: def GET(index: Optional[str]) -> AnyStep: ... + class source: + def GET() -> SourceSelectionAndSetting: ... + def POST(source_id: str) -> None: ... + class zdev: def GET() -> List[ZdevInfo]: ... diff --git a/subiquity/common/types.py b/subiquity/common/types.py index 4000e459..db507887 100644 --- a/subiquity/common/types.py +++ b/subiquity/common/types.py @@ -168,6 +168,22 @@ class KeyboardSetup: layouts: List[KeyboardLayout] +@attr.s(auto_attribs=True) +class SourceSelection: + name: str + description: str + id: str + size: int + variant: str + default: bool + + +@attr.s(auto_attribs=True) +class SourceSelectionAndSetting: + sources: List[SourceSelection] + current_id: str + + @attr.s(auto_attribs=True) class ZdevInfo: id: str diff --git a/subiquity/models/source.py b/subiquity/models/source.py new file mode 100644 index 00000000..99df435e --- /dev/null +++ b/subiquity/models/source.py @@ -0,0 +1,92 @@ +# Copyright 2021 Canonical, Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging +import os +import typing +import yaml + +import attr + +log = logging.getLogger('subiquity.models.source') + + +@attr.s(auto_attribs=True) +class CatalogEntry: + variant: str + id: str + name: typing.Dict[str, str] + description: typing.Dict[str, str] + path: str + size: int + type: str + default: bool = False + + +fake_entries = { + 'server': CatalogEntry( + variant='server', + id='synthesized', + name={'en': 'Ubuntu Server'}, + description={'en': 'the default'}, + path='/media/filesystem', + type='cp', + default=True, + size=2 << 30), + 'desktop': CatalogEntry( + variant='desktop', + id='synthesized', + name={'en': 'Ubuntu Desktop'}, + description={'en': 'the default'}, + path='/media/filesystem', + type='cp', + default=True, + size=5 << 30), + } + + +class SourceModel: + + def __init__(self): + self._dir = '/cdrom/casper' + self.current = fake_entries['server'] + self.sources = [self.current] + + def load_from_file(self, fp): + self._dir = os.path.dirname(fp.name) + self.sources = [] + self.current = None + entries = yaml.safe_load(fp) + for entry in entries: + kw = {} + for field in attr.fields(CatalogEntry): + if field.name in entry: + kw[field.name] = entry[field.name] + c = CatalogEntry(**kw) + self.sources.append(c) + if c.default: + self.current = c + log.debug("loaded %d sources from %r", len(self.sources), fp.name) + if self.current is None: + self.current = self.sources[0] + + def render(self): + path = os.path.join(self._dir, self.current.path) + scheme = self.current.type + return { + 'sources': { + 'ubuntu00': f'{scheme}://{path}' + }, + } diff --git a/subiquity/models/subiquity.py b/subiquity/models/subiquity.py index 99de443c..1bb43c81 100644 --- a/subiquity/models/subiquity.py +++ b/subiquity/models/subiquity.py @@ -39,6 +39,7 @@ from .mirror import MirrorModel from .network import NetworkModel from .proxy import ProxyModel from .snaplist import SnapListModel +from .source import SourceModel from .ssh import SSHModel from .timezone import TimeZoneModel from .updates import UpdatesModel @@ -116,6 +117,7 @@ class SubiquityModel: self.proxy = ProxyModel() self.snaplist = SnapListModel() self.ssh = SSHModel() + self.source = SourceModel() self.timezone = TimeZoneModel() self.updates = UpdatesModel() self.userdata = {} @@ -307,16 +309,13 @@ class SubiquityModel: config = { 'stages': stages, - 'sources': { - 'ubuntu00': 'cp:///media/filesystem' - }, - 'curthooks_commands': { '001-configure-apt': [ resource_path('bin/subiquity-configure-apt'), sys.executable, str(self.network.has_network).lower(), ], }, + 'grub': { 'terminal': 'unmodified', 'probe_additional_os': True diff --git a/subiquity/models/tests/test_source.py b/subiquity/models/tests/test_source.py new file mode 100644 index 00000000..1663fb52 --- /dev/null +++ b/subiquity/models/tests/test_source.py @@ -0,0 +1,117 @@ +# Copyright 2021 Canonical, Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os +import random +import shutil +import tempfile +import unittest +import yaml + +from subiquitycore.tests.util import random_string + +from subiquity.models.source import ( + SourceModel, + ) + + +def make_entry(**fields): + raw = { + 'name': { + 'en': random_string(), + }, + 'description': { + 'en': random_string(), + }, + 'id': random_string(), + 'type': random_string(), + 'variant': random_string(), + 'path': random_string(), + 'size': random.randint(1000000, 2000000), + } + for k, v in fields.items(): + raw[k] = v + return raw + + +class TestMirrorModel(unittest.TestCase): + + def tdir(self): + tdir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tdir) + return tdir + + def write_and_load_entries(self, model, entries, dir=None): + if dir is None: + dir = self.tdir() + cat_path = os.path.join(dir, 'catalog.yaml') + with open(cat_path, 'w') as fp: + yaml.dump(entries, fp) + with open(cat_path) as fp: + model.load_from_file(fp) + + def test_initially_server(self): + model = SourceModel() + self.assertEqual(model.current.variant, 'server') + + def test_load_from_file_simple(self): + entry = make_entry(id='id1') + model = SourceModel() + self.write_and_load_entries(model, [entry]) + self.assertEqual(model.current.id, 'id1') + + def test_load_from_file_ignores_extra_keys(self): + entry = make_entry(id='id1', foobarbaz=random_string()) + model = SourceModel() + self.write_and_load_entries(model, [entry]) + self.assertEqual(model.current.id, 'id1') + + def test_default(self): + entries = [ + make_entry(id='id1'), + make_entry(id='id2', default=True), + make_entry(id='id3'), + ] + model = SourceModel() + self.write_and_load_entries(model, entries) + self.assertEqual(model.current.id, 'id2') + + def test_render_absolute(self): + entry = make_entry( + type='scheme', + path='/foo/bar/baz', + ) + model = SourceModel() + self.write_and_load_entries(model, [entry]) + self.assertEqual( + model.render(), {'sources': {'ubuntu00': 'scheme:///foo/bar/baz'}}) + + def test_render_relative(self): + dir = self.tdir() + entry = make_entry( + type='scheme', + path='foo/bar/baz', + ) + model = SourceModel() + self.write_and_load_entries(model, [entry], dir) + self.assertEqual( + model.render(), + {'sources': {'ubuntu00': f'scheme://{dir}/foo/bar/baz'}}) + + def test_render_initial(self): + model = SourceModel() + self.assertEqual( + model.render(), + {'sources': {'ubuntu00': 'cp:///media/filesystem'}}) diff --git a/subiquity/server/controllers/__init__.py b/subiquity/server/controllers/__init__.py index 3edbc7d4..d875238b 100644 --- a/subiquity/server/controllers/__init__.py +++ b/subiquity/server/controllers/__init__.py @@ -29,6 +29,7 @@ from .refresh import RefreshController from .reporting import ReportingController from .shutdown import ShutdownController from .snaplist import SnapListController +from .source import SourceController from .ssh import SSHController from .timezone import TimeZoneController from .updates import UpdatesController @@ -54,6 +55,7 @@ __all__ = [ 'ReportingController', 'ShutdownController', 'SnapListController', + 'SourceController', 'SSHController', 'TimeZoneController', 'UpdatesController', diff --git a/subiquity/server/controllers/source.py b/subiquity/server/controllers/source.py new file mode 100644 index 00000000..a9fe65e2 --- /dev/null +++ b/subiquity/server/controllers/source.py @@ -0,0 +1,83 @@ +# Copyright 2021 Canonical, Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os + +from subiquity.common.apidef import API +from subiquity.common.types import ( + SourceSelection, + SourceSelectionAndSetting, + ) +from subiquity.server.controller import SubiquityController + + +def _translate(d, lang): + if lang: + for lang in lang, lang.split('_', 1)[0]: + if lang in d: + return d[lang] + return _(d['en']) + + +def convert_source(source, lang): + return SourceSelection( + name=_translate(source.name, lang), + description=_translate(source.description, lang), + id=source.id, + size=source.size, + variant=source.variant, + default=source.default) + + +class SourceController(SubiquityController): + + model_name = "source" + + endpoint = API.source + + def start(self): + path = '/cdrom/casper/install-sources.yaml' + if self.app.opts.source_catalog is not None: + path = self.app.opts.source_catalog + if not os.path.exists(path): + return + with open(path) as fp: + self.model.load_from_file(fp) + + def interactive(self): + if len(self.model.sources) <= 1: + return False + return super().interactive() + + async def GET(self) -> SourceSelectionAndSetting: + cur_lang = self.app.base_model.locale.selected_language + cur_lang = cur_lang.rsplit('.', 1)[0] + + return SourceSelectionAndSetting( + [ + convert_source(source, cur_lang) + for source in self.model.sources + ], + self.model.current.id) + + def configured(self): + super().configured() + self.app.base_model.set_source_variant(self.model.current.variant) + + async def POST(self, source_id: str) -> None: + for source in self.model.sources: + if source.id == source_id: + self.model.current = source + self.configured() diff --git a/subiquity/server/controllers/tests/__init__.py b/subiquity/server/controllers/tests/__init__.py new file mode 100644 index 00000000..792d6005 --- /dev/null +++ b/subiquity/server/controllers/tests/__init__.py @@ -0,0 +1 @@ +# diff --git a/subiquity/server/controllers/tests/test_source.py b/subiquity/server/controllers/tests/test_source.py new file mode 100644 index 00000000..fbe03323 --- /dev/null +++ b/subiquity/server/controllers/tests/test_source.py @@ -0,0 +1,65 @@ +# Copyright 2021 Canonical, Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import random +import unittest + +import attr + +from subiquitycore.tests.util import random_string + +from subiquity.models.source import CatalogEntry +from subiquity.server.controllers.source import convert_source + + +def make_entry(**kw): + fields = { + 'default': False, + 'name': {'en': random_string()}, + 'description': {'en': random_string()}, + 'size': random.randint(1000000, 2000000), + } + for field in attr.fields(CatalogEntry): + if field.name not in fields: + fields[field.name] = random_string() + for k, v in kw.items(): + fields[k] = v + return CatalogEntry(**fields) + + +class TestSubiquityModel(unittest.TestCase): + + def test_convert_source(self): + entry = make_entry() + source = convert_source(entry, 'C') + self.assertEqual(source.id, entry.id) + + def test_convert_translations(self): + entry = make_entry( + name={ + 'en': 'English', + 'fr': 'French', + 'fr_CA': 'French Canadian', + }) + self.assertEqual( + convert_source(entry, 'C').name, "English") + self.assertEqual( + convert_source(entry, 'en').name, "English") + self.assertEqual( + convert_source(entry, 'fr').name, "French") + self.assertEqual( + convert_source(entry, 'fr_CA').name, "French Canadian") + self.assertEqual( + convert_source(entry, 'fr_BE').name, "French") diff --git a/subiquity/server/server.py b/subiquity/server/server.py index e833367d..db326499 100644 --- a/subiquity/server/server.py +++ b/subiquity/server/server.py @@ -115,7 +115,13 @@ class MetaController: async def client_variant_POST(self, variant: str) -> None: if variant not in self.app.supported_variants: raise ValueError(f'unrecognized client variant {variant}') - self.app.base_model.set_source_variant(variant) + from subiquity.models.source import fake_entries + if variant in fake_entries: + if self.app.base_model.source.current.variant != variant: + self.app.base_model.source.current = fake_entries[variant] + self.app.controllers.Source.configured() + else: + self.app.base_model.set_source_variant(variant) async def ssh_info_GET(self) -> Optional[LiveSessionSSHInfo]: ips = [] @@ -169,6 +175,7 @@ INSTALL_MODEL_NAMES = ModelNames({ "mirror", "network", "proxy", + "source", }) POSTINSTALL_MODEL_NAMES = ModelNames({ @@ -213,6 +220,7 @@ class SubiquityServer(Application): "Kernel", "Keyboard", "Zdev", + "Source", "Network", "Proxy", "Mirror", diff --git a/subiquitycore/tests/util.py b/subiquitycore/tests/util.py index f69f6bc0..3deab2d8 100644 --- a/subiquitycore/tests/util.py +++ b/subiquitycore/tests/util.py @@ -14,7 +14,13 @@ # along with this program. If not, see . import asyncio +import random +import string def run_coro(coro): return asyncio.get_event_loop().run_until_complete(coro) + + +def random_string(): + return ''.join(random.choice(string.ascii_letters) for _ in range(8))