mirror: add different dry-run mechanisms for mirror checks
In dry-run mode, we can now define rules to control if the mirror check should: * succeed (the output of APT is faked) * fail (the output of APT is faked) * randomly succeed or fail (the output of APT is faked) * run on the host (this will effectively bypass the apt parameters supplied) The default rules are as follows: * https://archive.ubuntu.com/ubuntu will succeed * https://<country-code>.archive.ubuntu.com/ubuntu will succeed * any URL that ends in /success will succeed * any URL that ends in /fail or /failed will fail * any URL that ends in /rand or /random will randomly succeed or fail * any URL that ends in /host will run on the host (bypassing the URL configured) * anything else with run on the host Different rules can be provided by supplying a --dry-run-config YAML file. Signed-off-by: Olivier Gayot <olivier.gayot@canonical.com>
This commit is contained in:
Olivier Gayot
parent
0d24ce3fc2
commit
3896ebbdde
|
|
@ -15,10 +15,13 @@
|
|||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import enum
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
import random
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
from typing import Optional
|
||||
|
|
@ -164,7 +167,6 @@ class AptConfigurer:
|
|||
apt_cmd.append(
|
||||
f"-oAcquire::IndexTargets::{target}::DefaultEnabled=false")
|
||||
|
||||
|
||||
proc = await astart_command(apt_cmd, stderr=subprocess.STDOUT)
|
||||
|
||||
async def _reader():
|
||||
|
|
@ -274,6 +276,12 @@ class AptConfigurer:
|
|||
|
||||
|
||||
class DryRunAptConfigurer(AptConfigurer):
|
||||
class MirrorCheckStrategy(enum.Enum):
|
||||
SUCCESS = "success"
|
||||
FAILURE = "failure"
|
||||
RANDOM = "random"
|
||||
|
||||
RUN_ON_HOST = "run-on-host"
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def overlay(self):
|
||||
|
|
@ -282,6 +290,96 @@ class DryRunAptConfigurer(AptConfigurer):
|
|||
async def deconfigure(self, context, target):
|
||||
await self.cleanup()
|
||||
|
||||
def get_mirror_check_strategy(self, url: str) -> "MirrorCheckStrategy":
|
||||
""" For a given mirror URL, return the strategy that we should use to
|
||||
perform mirror checking. """
|
||||
for known in self.app.dr_cfg.apt_mirrors_known:
|
||||
if "url" in known:
|
||||
if known["url"] != url:
|
||||
continue
|
||||
elif "pattern" in known:
|
||||
if not re.search(known["pattern"], url):
|
||||
continue
|
||||
else:
|
||||
assert False
|
||||
|
||||
return self.MirrorCheckStrategy(known["strategy"])
|
||||
|
||||
return self.MirrorCheckStrategy(
|
||||
self.app.dr_cfg.apt_mirror_check_default_strategy)
|
||||
|
||||
async def apt_config_check_failure(self, output: io.StringIO) -> None:
|
||||
""" Pretend that the execution of the apt-get update command results in
|
||||
a failure. """
|
||||
url = self.app.base_model.mirror.get_mirror()
|
||||
release = lsb_release(dry_run=True)["codename"]
|
||||
host = url.split("/")[2]
|
||||
|
||||
output.write(f"""\
|
||||
Ign:1 {url} {release} InRelease
|
||||
Ign:2 {url} {release}-updates InRelease
|
||||
Ign:3 {url} {release}-backports InRelease
|
||||
Ign:4 {url} {release}-security InRelease
|
||||
Ign:2 {url} {release} InRelease
|
||||
Ign:3 {url} {release}-updates InRelease
|
||||
Err:1 {url} kinetic InRelease
|
||||
Temporary failure resolving '{host}'
|
||||
Err:2 {url} kinetic-updates InRelease
|
||||
Temporary failure resolving '{host}'
|
||||
Err:3 {url} kinetic-backports InRelease
|
||||
Temporary failure resolving '{host}'
|
||||
Err:4 {url} kinetic-security InRelease
|
||||
Temporary failure resolving '{host}'
|
||||
Reading package lists...
|
||||
E: Failed to fetch {url}/dists/{release}/InRelease\
|
||||
Temporary failure resolving '{host}'
|
||||
E: Failed to fetch {url}/dists/{release}-updates/InRelease\
|
||||
Temporary failure resolving '{host}'
|
||||
E: Failed to fetch {url}/dists/{release}-backports/InRelease\
|
||||
Temporary failure resolving '{host}'
|
||||
E: Failed to fetch {url}/dists/{release}-security/InRelease\
|
||||
Temporary failure resolving '{host}'
|
||||
E: Some index files failed to download. They have been ignored,
|
||||
or old ones used instead.
|
||||
""")
|
||||
raise AptConfigCheckError
|
||||
|
||||
async def apt_config_check_success(self, output: io.StringIO) -> None:
|
||||
""" Pretend that the execution of the apt-get update command results in
|
||||
a success. """
|
||||
url = self.app.base_model.mirror.get_mirror()
|
||||
release = lsb_release(dry_run=True)["codename"]
|
||||
|
||||
output.write(f"""\
|
||||
Get:1 {url} {release} InRelease [267 kB]
|
||||
Get:2 {url} {release}-updates InRelease [109 kB]
|
||||
Get:3 {url} {release}-backports InRelease [99.9 kB]
|
||||
Get:4 {url} {release}-security InRelease [109 kB]
|
||||
Fetched 585 kB in 1s (1057 kB/s)
|
||||
Reading package lists...
|
||||
""")
|
||||
|
||||
async def run_apt_config_check(self, output: io.StringIO) -> None:
|
||||
""" Dry-run implementation of the Apt config check.
|
||||
The strategy used is based on the URL of the primary mirror. The
|
||||
apt-get command can either run on the host or be faked entirely. """
|
||||
assert self.configured_tree is not None
|
||||
|
||||
failure = self.apt_config_check_failure
|
||||
success = self.apt_config_check_success
|
||||
|
||||
strategies = {
|
||||
self.MirrorCheckStrategy.RUN_ON_HOST: super().run_apt_config_check,
|
||||
self.MirrorCheckStrategy.FAILURE: failure,
|
||||
self.MirrorCheckStrategy.SUCCESS: success,
|
||||
self.MirrorCheckStrategy.RANDOM: random.choice([failure, success]),
|
||||
}
|
||||
mirror_url = self.app.base_model.mirror.get_mirror()
|
||||
|
||||
strategy = strategies[self.get_mirror_check_strategy(mirror_url)]
|
||||
|
||||
await strategy(output)
|
||||
|
||||
|
||||
def get_apt_configurer(app, source: str):
|
||||
if app.opts.dry_run:
|
||||
|
|
|
|||
|
|
@ -189,7 +189,8 @@ class MirrorController(SubiquityController):
|
|||
log.debug(data)
|
||||
self.model.disabled_components = set(data)
|
||||
|
||||
async def check_mirror_start_POST(self, cancel_ongoing: bool = False) -> None:
|
||||
async def check_mirror_start_POST(
|
||||
self, cancel_ongoing: bool = False) -> None:
|
||||
if self.mirror_check is not None and not self.mirror_check.task.done():
|
||||
if cancel_ongoing:
|
||||
await self.check_mirror_abort_POST()
|
||||
|
|
@ -206,6 +207,8 @@ class MirrorController(SubiquityController):
|
|||
return None
|
||||
if self.mirror_check.task.done():
|
||||
if self.mirror_check.task.exception():
|
||||
log.warning("Mirror check failed: %s",
|
||||
self.mirror_check.task.exception())
|
||||
status = MirrorCheckStatus.FAILED
|
||||
else:
|
||||
status = MirrorCheckStatus.OK
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@
|
|||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
from typing import List, TypedDict
|
||||
import yaml
|
||||
|
||||
import attr
|
||||
|
|
@ -28,6 +29,15 @@ class DryRunController:
|
|||
1/0
|
||||
|
||||
|
||||
class KnownMirror(TypedDict, total=False):
|
||||
""" Dictionary type hints for a known mirror. Either url or pattern should
|
||||
be specified. """
|
||||
url: str
|
||||
pattern: str
|
||||
|
||||
strategy: str
|
||||
|
||||
|
||||
@attr.s(auto_attribs=True)
|
||||
class DRConfig:
|
||||
""" Configuration for dry-run-only executions.
|
||||
|
|
@ -43,6 +53,18 @@ class DRConfig:
|
|||
# ua-contrats.
|
||||
pro_ua_contracts_url: str = "https://contracts.staging.canonical.com"
|
||||
|
||||
apt_mirror_check_default_strategy: str = "run-on-host"
|
||||
apt_mirrors_known: List[KnownMirror] = [
|
||||
{"pattern": r"https?://archive\.ubuntu\.com/ubuntu/?",
|
||||
"strategy": "success"},
|
||||
{"pattern": r"https?://[a-z]{2,}\.archive\.ubuntu\.com/ubuntu/?",
|
||||
"strategy": "success"},
|
||||
{"pattern": r"/success/?$", "strategy": "success"},
|
||||
{"pattern": r"/rand(om)?/?$", "strategy": "random"},
|
||||
{"pattern": r"/host/?$", "strategy": "run-on-host"},
|
||||
{"pattern": r"/fail(ed)?/?$", "strategy": "failure"},
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def load(cls, stream):
|
||||
data = yaml.safe_load(stream)
|
||||
|
|
|
|||
|
|
@ -22,9 +22,11 @@ from subiquitycore.tests.mocks import make_app
|
|||
from subiquitycore.utils import astart_command
|
||||
from subiquity.server.apt import (
|
||||
AptConfigurer,
|
||||
DryRunAptConfigurer,
|
||||
AptConfigCheckError,
|
||||
OverlayMountpoint,
|
||||
)
|
||||
from subiquity.server.dryrun import DRConfig
|
||||
from subiquity.models.mirror import MirrorModel
|
||||
from subiquity.models.proxy import ProxyModel
|
||||
|
||||
|
|
@ -126,3 +128,67 @@ class TestAptConfigurer(SubiTestCase):
|
|||
with patch(self.astart_sym, side_effect=astart_failure):
|
||||
with self.assertRaises(AptConfigCheckError):
|
||||
await self.configurer.run_apt_config_check(output)
|
||||
|
||||
|
||||
class TestDRAptConfigurer(SubiTestCase):
|
||||
def setUp(self):
|
||||
self.model = Mock()
|
||||
self.model.mirror = MirrorModel()
|
||||
self.app = make_app(self.model)
|
||||
self.app.dr_cfg = DRConfig()
|
||||
self.app.dr_cfg.apt_mirror_check_default_strategy = "failure"
|
||||
self.app.dr_cfg.apt_mirrors_known = [
|
||||
{"url": "http://success", "strategy": "success"},
|
||||
{"url": "http://failure", "strategy": "failure"},
|
||||
{"url": "http://run-on-host", "strategy": "run-on-host"},
|
||||
{"pattern": "/random$", "strategy": "random"},
|
||||
]
|
||||
self.configurer = DryRunAptConfigurer(self.app, AsyncMock(), '')
|
||||
self.configurer.configured_tree = OverlayMountpoint(
|
||||
upperdir="upperdir-install-tree",
|
||||
lowers=["lowers1-install-tree"],
|
||||
mountpoint="mountpoint-install-tree",
|
||||
)
|
||||
|
||||
def test_get_mirror_check_strategy(self):
|
||||
Strategy = DryRunAptConfigurer.MirrorCheckStrategy
|
||||
self.assertEqual(
|
||||
Strategy.SUCCESS,
|
||||
self.configurer.get_mirror_check_strategy("http://success"))
|
||||
self.assertEqual(
|
||||
Strategy.FAILURE,
|
||||
self.configurer.get_mirror_check_strategy("http://failure"))
|
||||
self.assertEqual(Strategy.RUN_ON_HOST,
|
||||
self.configurer.get_mirror_check_strategy(
|
||||
"http://run-on-host"))
|
||||
self.assertEqual(Strategy.RANDOM,
|
||||
self.configurer.get_mirror_check_strategy(
|
||||
"http://mirror/random"))
|
||||
self.assertEqual(
|
||||
Strategy.FAILURE,
|
||||
self.configurer.get_mirror_check_strategy("http://default"))
|
||||
|
||||
async def test_run_apt_config_check_success(self):
|
||||
output = io.StringIO()
|
||||
self.app.dr_cfg.apt_mirror_check_default_strategy = "success"
|
||||
self.model.mirror.set_mirror("http://default")
|
||||
await self.configurer.run_apt_config_check(output)
|
||||
|
||||
async def test_run_apt_config_check_failed(self):
|
||||
output = io.StringIO()
|
||||
self.app.dr_cfg.apt_mirror_check_default_strategy = "failure"
|
||||
self.model.mirror.set_mirror("http://default")
|
||||
with self.assertRaises(AptConfigCheckError):
|
||||
await self.configurer.run_apt_config_check(output)
|
||||
|
||||
async def test_run_apt_config_check_random(self):
|
||||
output = io.StringIO()
|
||||
self.app.dr_cfg.apt_mirror_check_default_strategy = "random"
|
||||
self.model.mirror.set_mirror("http://default")
|
||||
with patch("subiquity.server.apt.random.choice",
|
||||
return_value=self.configurer.apt_config_check_success):
|
||||
await self.configurer.run_apt_config_check(output)
|
||||
with patch("subiquity.server.apt.random.choice",
|
||||
return_value=self.configurer.apt_config_check_failure):
|
||||
with self.assertRaises(AptConfigCheckError):
|
||||
await self.configurer.run_apt_config_check(output)
|
||||
|
|
|
|||
Loading…
Reference in New Issue