From 3896ebbdde28bc4ad07e82bc00b268268ada125e Mon Sep 17 00:00:00 2001 From: Olivier Gayot Date: Tue, 17 Jan 2023 19:31:46 +0100 Subject: [PATCH] mirror: add different dry-run mechanisms for mirror checks In dry-run mode, we can now define rules to control if the mirror check should: * succeed (the output of APT is faked) * fail (the output of APT is faked) * randomly succeed or fail (the output of APT is faked) * run on the host (this will effectively bypass the apt parameters supplied) The default rules are as follows: * https://archive.ubuntu.com/ubuntu will succeed * https://.archive.ubuntu.com/ubuntu will succeed * any URL that ends in /success will succeed * any URL that ends in /fail or /failed will fail * any URL that ends in /rand or /random will randomly succeed or fail * any URL that ends in /host will run on the host (bypassing the URL configured) * anything else with run on the host Different rules can be provided by supplying a --dry-run-config YAML file. Signed-off-by: Olivier Gayot --- subiquity/server/apt.py | 100 ++++++++++++++++++++++++- subiquity/server/controllers/mirror.py | 5 +- subiquity/server/dryrun.py | 22 ++++++ subiquity/server/tests/test_apt.py | 66 ++++++++++++++++ 4 files changed, 191 insertions(+), 2 deletions(-) diff --git a/subiquity/server/apt.py b/subiquity/server/apt.py index bdb0f728..5912fd8c 100644 --- a/subiquity/server/apt.py +++ b/subiquity/server/apt.py @@ -15,10 +15,13 @@ import asyncio import contextlib +import enum import io import logging import os import pathlib +import random +import re import shutil import subprocess from typing import Optional @@ -164,7 +167,6 @@ class AptConfigurer: apt_cmd.append( f"-oAcquire::IndexTargets::{target}::DefaultEnabled=false") - proc = await astart_command(apt_cmd, stderr=subprocess.STDOUT) async def _reader(): @@ -274,6 +276,12 @@ class AptConfigurer: class DryRunAptConfigurer(AptConfigurer): + class MirrorCheckStrategy(enum.Enum): + SUCCESS = "success" + FAILURE = "failure" + RANDOM = "random" + + RUN_ON_HOST = "run-on-host" @contextlib.asynccontextmanager async def overlay(self): @@ -282,6 +290,96 @@ class DryRunAptConfigurer(AptConfigurer): async def deconfigure(self, context, target): await self.cleanup() + def get_mirror_check_strategy(self, url: str) -> "MirrorCheckStrategy": + """ For a given mirror URL, return the strategy that we should use to + perform mirror checking. """ + for known in self.app.dr_cfg.apt_mirrors_known: + if "url" in known: + if known["url"] != url: + continue + elif "pattern" in known: + if not re.search(known["pattern"], url): + continue + else: + assert False + + return self.MirrorCheckStrategy(known["strategy"]) + + return self.MirrorCheckStrategy( + self.app.dr_cfg.apt_mirror_check_default_strategy) + + async def apt_config_check_failure(self, output: io.StringIO) -> None: + """ Pretend that the execution of the apt-get update command results in + a failure. """ + url = self.app.base_model.mirror.get_mirror() + release = lsb_release(dry_run=True)["codename"] + host = url.split("/")[2] + + output.write(f"""\ +Ign:1 {url} {release} InRelease +Ign:2 {url} {release}-updates InRelease +Ign:3 {url} {release}-backports InRelease +Ign:4 {url} {release}-security InRelease +Ign:2 {url} {release} InRelease +Ign:3 {url} {release}-updates InRelease +Err:1 {url} kinetic InRelease + Temporary failure resolving '{host}' +Err:2 {url} kinetic-updates InRelease + Temporary failure resolving '{host}' +Err:3 {url} kinetic-backports InRelease + Temporary failure resolving '{host}' +Err:4 {url} kinetic-security InRelease + Temporary failure resolving '{host}' +Reading package lists... +E: Failed to fetch {url}/dists/{release}/InRelease\ + Temporary failure resolving '{host}' +E: Failed to fetch {url}/dists/{release}-updates/InRelease\ + Temporary failure resolving '{host}' +E: Failed to fetch {url}/dists/{release}-backports/InRelease\ + Temporary failure resolving '{host}' +E: Failed to fetch {url}/dists/{release}-security/InRelease\ + Temporary failure resolving '{host}' +E: Some index files failed to download. They have been ignored, + or old ones used instead. +""") + raise AptConfigCheckError + + async def apt_config_check_success(self, output: io.StringIO) -> None: + """ Pretend that the execution of the apt-get update command results in + a success. """ + url = self.app.base_model.mirror.get_mirror() + release = lsb_release(dry_run=True)["codename"] + + output.write(f"""\ +Get:1 {url} {release} InRelease [267 kB] +Get:2 {url} {release}-updates InRelease [109 kB] +Get:3 {url} {release}-backports InRelease [99.9 kB] +Get:4 {url} {release}-security InRelease [109 kB] +Fetched 585 kB in 1s (1057 kB/s) +Reading package lists... +""") + + async def run_apt_config_check(self, output: io.StringIO) -> None: + """ Dry-run implementation of the Apt config check. + The strategy used is based on the URL of the primary mirror. The + apt-get command can either run on the host or be faked entirely. """ + assert self.configured_tree is not None + + failure = self.apt_config_check_failure + success = self.apt_config_check_success + + strategies = { + self.MirrorCheckStrategy.RUN_ON_HOST: super().run_apt_config_check, + self.MirrorCheckStrategy.FAILURE: failure, + self.MirrorCheckStrategy.SUCCESS: success, + self.MirrorCheckStrategy.RANDOM: random.choice([failure, success]), + } + mirror_url = self.app.base_model.mirror.get_mirror() + + strategy = strategies[self.get_mirror_check_strategy(mirror_url)] + + await strategy(output) + def get_apt_configurer(app, source: str): if app.opts.dry_run: diff --git a/subiquity/server/controllers/mirror.py b/subiquity/server/controllers/mirror.py index b718f4a9..fce8ca9b 100644 --- a/subiquity/server/controllers/mirror.py +++ b/subiquity/server/controllers/mirror.py @@ -189,7 +189,8 @@ class MirrorController(SubiquityController): log.debug(data) self.model.disabled_components = set(data) - async def check_mirror_start_POST(self, cancel_ongoing: bool = False) -> None: + async def check_mirror_start_POST( + self, cancel_ongoing: bool = False) -> None: if self.mirror_check is not None and not self.mirror_check.task.done(): if cancel_ongoing: await self.check_mirror_abort_POST() @@ -206,6 +207,8 @@ class MirrorController(SubiquityController): return None if self.mirror_check.task.done(): if self.mirror_check.task.exception(): + log.warning("Mirror check failed: %s", + self.mirror_check.task.exception()) status = MirrorCheckStatus.FAILED else: status = MirrorCheckStatus.OK diff --git a/subiquity/server/dryrun.py b/subiquity/server/dryrun.py index 17c88ac0..0e27c199 100644 --- a/subiquity/server/dryrun.py +++ b/subiquity/server/dryrun.py @@ -13,6 +13,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from typing import List, TypedDict import yaml import attr @@ -28,6 +29,15 @@ class DryRunController: 1/0 +class KnownMirror(TypedDict, total=False): + """ Dictionary type hints for a known mirror. Either url or pattern should + be specified. """ + url: str + pattern: str + + strategy: str + + @attr.s(auto_attribs=True) class DRConfig: """ Configuration for dry-run-only executions. @@ -43,6 +53,18 @@ class DRConfig: # ua-contrats. pro_ua_contracts_url: str = "https://contracts.staging.canonical.com" + apt_mirror_check_default_strategy: str = "run-on-host" + apt_mirrors_known: List[KnownMirror] = [ + {"pattern": r"https?://archive\.ubuntu\.com/ubuntu/?", + "strategy": "success"}, + {"pattern": r"https?://[a-z]{2,}\.archive\.ubuntu\.com/ubuntu/?", + "strategy": "success"}, + {"pattern": r"/success/?$", "strategy": "success"}, + {"pattern": r"/rand(om)?/?$", "strategy": "random"}, + {"pattern": r"/host/?$", "strategy": "run-on-host"}, + {"pattern": r"/fail(ed)?/?$", "strategy": "failure"}, + ] + @classmethod def load(cls, stream): data = yaml.safe_load(stream) diff --git a/subiquity/server/tests/test_apt.py b/subiquity/server/tests/test_apt.py index 47198b6a..20e86f4b 100644 --- a/subiquity/server/tests/test_apt.py +++ b/subiquity/server/tests/test_apt.py @@ -22,9 +22,11 @@ from subiquitycore.tests.mocks import make_app from subiquitycore.utils import astart_command from subiquity.server.apt import ( AptConfigurer, + DryRunAptConfigurer, AptConfigCheckError, OverlayMountpoint, ) +from subiquity.server.dryrun import DRConfig from subiquity.models.mirror import MirrorModel from subiquity.models.proxy import ProxyModel @@ -126,3 +128,67 @@ class TestAptConfigurer(SubiTestCase): with patch(self.astart_sym, side_effect=astart_failure): with self.assertRaises(AptConfigCheckError): await self.configurer.run_apt_config_check(output) + + +class TestDRAptConfigurer(SubiTestCase): + def setUp(self): + self.model = Mock() + self.model.mirror = MirrorModel() + self.app = make_app(self.model) + self.app.dr_cfg = DRConfig() + self.app.dr_cfg.apt_mirror_check_default_strategy = "failure" + self.app.dr_cfg.apt_mirrors_known = [ + {"url": "http://success", "strategy": "success"}, + {"url": "http://failure", "strategy": "failure"}, + {"url": "http://run-on-host", "strategy": "run-on-host"}, + {"pattern": "/random$", "strategy": "random"}, + ] + self.configurer = DryRunAptConfigurer(self.app, AsyncMock(), '') + self.configurer.configured_tree = OverlayMountpoint( + upperdir="upperdir-install-tree", + lowers=["lowers1-install-tree"], + mountpoint="mountpoint-install-tree", + ) + + def test_get_mirror_check_strategy(self): + Strategy = DryRunAptConfigurer.MirrorCheckStrategy + self.assertEqual( + Strategy.SUCCESS, + self.configurer.get_mirror_check_strategy("http://success")) + self.assertEqual( + Strategy.FAILURE, + self.configurer.get_mirror_check_strategy("http://failure")) + self.assertEqual(Strategy.RUN_ON_HOST, + self.configurer.get_mirror_check_strategy( + "http://run-on-host")) + self.assertEqual(Strategy.RANDOM, + self.configurer.get_mirror_check_strategy( + "http://mirror/random")) + self.assertEqual( + Strategy.FAILURE, + self.configurer.get_mirror_check_strategy("http://default")) + + async def test_run_apt_config_check_success(self): + output = io.StringIO() + self.app.dr_cfg.apt_mirror_check_default_strategy = "success" + self.model.mirror.set_mirror("http://default") + await self.configurer.run_apt_config_check(output) + + async def test_run_apt_config_check_failed(self): + output = io.StringIO() + self.app.dr_cfg.apt_mirror_check_default_strategy = "failure" + self.model.mirror.set_mirror("http://default") + with self.assertRaises(AptConfigCheckError): + await self.configurer.run_apt_config_check(output) + + async def test_run_apt_config_check_random(self): + output = io.StringIO() + self.app.dr_cfg.apt_mirror_check_default_strategy = "random" + self.model.mirror.set_mirror("http://default") + with patch("subiquity.server.apt.random.choice", + return_value=self.configurer.apt_config_check_success): + await self.configurer.run_apt_config_check(output) + with patch("subiquity.server.apt.random.choice", + return_value=self.configurer.apt_config_check_failure): + with self.assertRaises(AptConfigCheckError): + await self.configurer.run_apt_config_check(output)