StealthOS/subiquity
StealthOS
/
subiquity
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1564 from ogayot/fallback-offline 

Optionally fallback to offline install if no usable mirror provided
This commit is contained in:
Olivier Gayot 2023-02-16 18:45:05 +01:00 committed by GitHub
parent 27e85717df 6c3ae3c6dd
commit 9ed734b48b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 219 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
autoinstall-schema.json
View File

@ -386,6 +386,14 @@
"pin-priority"
]
}
},
"fallback": {
"type": "string",
"enum": [
"abort",
"continue-anyway",
"offline-install"
]
}
}
},

13
documentation/autoinstall-reference.md
View File

@ -201,6 +201,7 @@ Apt configuration, used both during the install and once booted into the target
This section historically used the same format as curtin, [which is documented here](https://curtin.readthedocs.io/en/latest/topics/apt_source.html). Nonetheless, some key differences with the format supported by curtin have been introduced:
* Subiquity supports an alternative format for the `primary` section, allowing to configure a list of candidate primary mirrors. During installation, subiquity will automatically test the specified mirrors and select the first one that seems usable. This new behavior is only activated when the `primary` section is wrapped in the `mirror-selection` section.
* The `fallback` key controls what subiquity should do if no primary mirror is usable.
* The `geoip` key controls whether a geoip lookup is done to determine the correct country mirror.
The default is:
@ -214,6 +215,7 @@ The default is:
uri: "http://archive.ubuntu.com/ubuntu"
- arches: [s390x, arm64, armhf, powerpc, ppc64el, riscv64]
uri: "http://ports.ubuntu.com/ubuntu-ports"
fallback: abort
geoip: true
#### mirror-selection
@ -229,6 +231,17 @@ In the new format, the `primary` section expects a list of mirrors, which can be
* `uri`: the URI of the mirror to use, e.g., "http://fr.archive.ubuntu.com/ubuntu"
* `arches`: an optional list of architectures supported by the mirror. By default, this list contains the current CPU architecture.
#### fallback
**type:** string (enumeration)
**default:** abort
Controls what subiquity should do if no primary mirror is usable.
Supported values are:
* `abort` -> abort the installation
* `offline-install` -> revert to an offline installation
* `continue-anyway` -> attempt to install the system anyway (not recommended, the installation will certainly fail)
#### geoip
**type:** boolean
**default:**: `true`

8
documentation/autoinstall-schema.md
View File

@ -408,6 +408,14 @@ The [JSON schema](https://json-schema.org/) for autoinstall data is as follows:
"pin-priority"
]
}
},
"fallback": {
"type": "string",
"enum": [
"abort",
"continue-anyway",
"offline-install"
]
}
}
},

8
subiquity/common/apidef.py
View File

@ -45,8 +45,10 @@ from subiquity.common.types import (
KeyboardSetup,
IdentityData,
NetworkStatus,
MirrorSelectionFallback,
MirrorGet,
MirrorPost,
MirrorPostResponse,
MirrorCheckResponse,
ModifyPartitionV2,
ReformatDisk,
@ -353,7 +355,9 @@ class API:
class mirror:
def GET() -> MirrorGet: ...
def POST(data: Payload[Optional[MirrorPost]]) -> None: ...
def POST(data: Payload[Optional[MirrorPost]]) \
-> MirrorPostResponse: ...
class disable_components:
def GET() -> List[str]: ...
@ -369,6 +373,8 @@ class API:
class abort:
def POST() -> None: ...
fallback = simple_endpoint(MirrorSelectionFallback)
class ubuntu_pro:
def GET() -> UbuntuProResponse: ...
def POST(data: Payload[UbuntuProInfo]) -> None: ...

11
subiquity/common/types.py
View File

@ -754,6 +754,11 @@ class MirrorPost:
staged: Optional[str] = None
class MirrorPostResponse(enum.Enum):
OK = "ok"
NO_USABLE_MIRROR = "no-usable-mirror"
@attr.s(auto_attribs=True)
class MirrorGet:
elected: Optional[str]
@ -761,6 +766,12 @@ class MirrorGet:
staged: Optional[str]
class MirrorSelectionFallback(enum.Enum):
ABORT = 'abort'
CONTINUE_ANYWAY = 'continue-anyway'
OFFLINE_INSTALL = 'offline-install'
@attr.s(auto_attribs=True)
class ADConnectionInfo:
admin_name: str = ""

66
subiquity/models/mirror.py
View File

@ -75,11 +75,16 @@ import abc
import copy
import contextlib
import logging
from typing import Any, Dict, Iterator, List, Optional, Sequence, Set, Union
from typing import (
Any, Callable, Dict, Iterator, List,
Optional, Sequence, Set, Union,
)
from urllib import parse
import attr
from subiquity.common.types import MirrorSelectionFallback
from curtin.commands.apt_config import (
get_arch_mirrorconfig,
get_mirror,
@ -130,6 +135,11 @@ class BasePrimaryEntry(abc.ABC):
def serialize_for_ai(self) -> Any:
""" Serialize the entry for autoinstall. """
@abc.abstractmethod
def supports_arch(self, arch: str) -> bool:
""" Tells whether the mirror claims to support the architecture
specified. """
@attr.s(auto_attribs=True)
class PrimaryEntry(BasePrimaryEntry):
@ -159,8 +169,6 @@ class PrimaryEntry(BasePrimaryEntry):
return [{"uri": self.uri, "arches": ["default"]}]
def supports_arch(self, arch: str) -> bool:
""" Tells whether the mirror claims to support the architecture
specified. """
if self.arches is None:
return True
return arch in self.arches
@ -207,6 +215,11 @@ class LegacyPrimaryEntry(BasePrimaryEntry):
def serialize_for_ai(self) -> List[Any]:
return self.config
def supports_arch(self, arch: str) -> bool:
# Curtin will always find a mirror ; albeit with the ["default"]
# architectures.
return True
def countrify_uri(uri: str, cc: str) -> str:
""" Return a URL where the host is prefixed with a country code. """
@ -215,6 +228,18 @@ def countrify_uri(uri: str, cc: str) -> str:
return parse.urlunparse(new)
CandidateFilter = Callable[[BasePrimaryEntry], bool]
def filter_candidates(candidates: List[BasePrimaryEntry],
*, filters: Sequence[CandidateFilter]) \
-> Iterator[BasePrimaryEntry]:
candidates_iter = iter(candidates)
for filt in filters:
candidates_iter = filter(filt, candidates_iter)
return candidates_iter
class MirrorModel(object):
def __init__(self):
@ -232,6 +257,9 @@ class MirrorModel(object):
self.default_mirror = \
LegacyPrimaryEntry.new_from_default(parent=self).uri
# What to do if automatic mirror-selection fails.
self.fallback = MirrorSelectionFallback.ABORT
def _default_primary_entries(self) -> List[PrimaryEntry]:
return [
PrimaryEntry(parent=self, country_mirror=True),
@ -271,12 +299,15 @@ class MirrorModel(object):
entry = PrimaryEntry.from_config(section, parent=self)
primary_candidates.append(entry)
self.primary_candidates = primary_candidates
if "fallback" in data:
self.fallback = MirrorSelectionFallback(data.pop("fallback"))
merge_config(self.config, data)
def _get_apt_config_common(self) -> Dict[str, Any]:
assert "disable_components" not in self.config
assert "primary" not in self.config
assert "fallback" not in self.config
config = copy.deepcopy(self.config)
config["disable_components"] = sorted(self.disabled_components)
@ -312,8 +343,12 @@ class MirrorModel(object):
# install. It will be placed in etc/apt/sources.list of the target
# system.
with contextlib.suppress(StopIteration):
candidate = next(filter(lambda c: c.uri is not None,
self.compatible_primary_candidates()))
filters = [
lambda c: c.uri is not None,
lambda c: c.supports_arch(self.architecture),
]
candidate = next(filter_candidates(self.primary_candidates,
filters=filters))
return self._get_apt_config_using_candidate(candidate)
# Our last resort is to include no primary section. Curtin will use
# its own internal values.
@ -357,20 +392,20 @@ class MirrorModel(object):
return next(self.country_mirror_candidates(), None) is not None
def country_mirror_candidates(self) -> Iterator[BasePrimaryEntry]:
for candidate in self.primary_candidates:
def filt(candidate):
if self.legacy_primary and candidate.mirror_is_default():
yield candidate
return True
elif not self.legacy_primary and candidate.country_mirror:
yield candidate
return True
return False
return filter_candidates(self.primary_candidates, filters=[filt])
def compatible_primary_candidates(self) -> Iterator[BasePrimaryEntry]:
for candidate in self.primary_candidates:
if self.legacy_primary:
yield candidate
elif candidate.arches is None:
yield candidate
elif self.architecture in candidate.arches:
yield candidate
def filt(candidate):
return candidate.supports_arch(self.architecture)
return filter_candidates(self.primary_candidates, filters=[filt])
def render(self):
return {}
@ -388,5 +423,6 @@ class MirrorModel(object):
else:
primary = [c.serialize_for_ai() for c in self.primary_candidates]
config["mirror-selection"] = {"primary": primary}
config["fallback"] = self.fallback.value
return config

10
subiquity/models/tests/test_mirror.py
View File

@ -21,6 +21,7 @@ from subiquity.models.mirror import (
countrify_uri,
LEGACY_DEFAULT_PRIMARY_SECTION,
MirrorModel,
MirrorSelectionFallback,
LegacyPrimaryEntry,
PrimaryEntry,
)
@ -177,7 +178,10 @@ class TestMirrorModel(unittest.TestCase):
def test_from_autoinstall_no_primary(self):
# autoinstall loads to the config directly
model = MirrorModel()
data = {'disable_components': ['non-free']}
data = {
'disable_components': ['non-free'],
'fallback': 'offline-install',
}
model.load_autoinstall_data(data)
self.assertFalse(model.legacy_primary)
model.primary_candidates[0].stage()
@ -236,6 +240,7 @@ class TestMirrorModel(unittest.TestCase):
]
self.model.disabled_components = set(["non-free"])
self.model.legacy_primary = False
self.model.fallback = MirrorSelectionFallback.OFFLINE_INSTALL
self.model.primary_candidates = [
PrimaryEntry(uri=None, arches=None,
country_mirror=True, parent=self.model),
@ -246,17 +251,20 @@ class TestMirrorModel(unittest.TestCase):
]
cfg = self.model.make_autoinstall()
self.assertEqual(cfg["disable_components"], ["non-free"])
self.assertEqual(cfg["fallback"], "offline-install")
self.assertEqual(cfg["mirror-selection"]["primary"], expected_primary)
def test_make_autoinstall_legacy_primary(self):
primary = [{"arches": "amd64", "uri": "http://mirror"}]
self.model.disabled_components = set(["non-free"])
self.model.fallback = MirrorSelectionFallback.ABORT
self.model.legacy_primary = True
self.model.primary_candidates = \
[LegacyPrimaryEntry(primary, parent=self.model)]
self.model.primary_candidates[0].elect()
cfg = self.model.make_autoinstall()
self.assertEqual(cfg["disable_components"], ["non-free"])
self.assertEqual(cfg["fallback"], "abort")
self.assertEqual(cfg["primary"], primary)
def test_create_primary_candidate(self):

87
subiquity/server/controllers/mirror.py
View File

@ -29,7 +29,10 @@ from subiquity.common.types import (
MirrorCheckStatus,
MirrorGet,
MirrorPost,
MirrorPostResponse,
MirrorSelectionFallback,
)
from subiquity.models.mirror import filter_candidates
from subiquity.server.apt import get_apt_configurer, AptConfigCheckError
from subiquity.server.controller import SubiquityController
from subiquity.server.types import InstallerChannels
@ -121,7 +124,11 @@ class MirrorController(SubiquityController):
"pin-priority",
],
}
}
},
"fallback": {
"type": "string",
"enum": [fb.value for fb in MirrorSelectionFallback],
},
}
}
model_name = "mirror"
@ -219,9 +226,54 @@ class MirrorController(SubiquityController):
candidate.elect()
async def apply_fallback(self):
fallback = self.model.fallback
if fallback == MirrorSelectionFallback.ABORT:
log.error("aborting the install since no primary mirror is"
" usable")
# TODO there is no guarantee that raising this exception will
# actually abort the install. If this is raised from a request
# handler, for instance, it will just return a HTTP 500 error. For
# now, this is acceptable since we do not call apply_fallback from
# request handlers.
raise RuntimeError("aborting install since no mirror is usable")
elif fallback == MirrorSelectionFallback.OFFLINE_INSTALL:
log.warning("reverting to an offline install since no primary"
" mirror is usable")
self.app.base_model.network.force_offline = True
elif fallback == MirrorSelectionFallback.CONTINUE_ANYWAY:
log.warning("continuing the install despite no usable mirror")
# Pick a candidate that is supposedly compatible and that has a
# URI. If it does not work, well that's too bad.
filters = [
lambda c: c.uri is not None,
lambda c: c.supports_arch(self.model.architecture),
]
try:
candidate = next(filter_candidates(
self.model.primary_candidates, filters=filters))
except StopIteration:
candidate = next(filter_candidates(
self.model.get_default_primary_candidates(),
filters=filters))
log.warning("deciding to elect primary mirror %s",
candidate.serialize_for_ai())
candidate.elect()
else:
raise RuntimeError(f"invalid fallback value: {fallback}")
async def run_mirror_selection_or_fallback(self, context):
""" Perform the mirror selection and apply the configured fallback
method if no mirror is usable. """
try:
await self.find_and_elect_candidate_mirror(context=context)
except NoUsableMirrorError:
await self.apply_fallback()
@with_context()
async def apply_autoinstall_config(self, context):
await self.find_and_elect_candidate_mirror(context)
await self.run_mirror_selection_or_fallback(context)
def on_geoip(self):
if self.geoip_enabled:
@ -266,7 +318,7 @@ class MirrorController(SubiquityController):
# marked configured using a POST to mark_configured ; which is not
# recommended. Clients should do a POST request to /mirror with
# null as the body instead.
await self.find_and_elect_candidate_mirror(self.context)
await self.run_mirror_selection_or_fallback(self.context)
await self.apt_configurer.apply_apt_config(self.context, final=True)
async def run_mirror_testing(self, output: io.StringIO) -> None:
@ -291,15 +343,23 @@ class MirrorController(SubiquityController):
candidates = [c.uri for c in compatibles if c.uri is not None]
return MirrorGet(elected=elected, candidates=candidates, staged=staged)
async def POST(self, data: Optional[MirrorPost]) -> None:
async def POST(self, data: Optional[MirrorPost]) -> MirrorPostResponse:
log.debug(data)
if data is None:
# TODO If we want the ability to fallback to an offline install, we
# probably need to catch NoUsableMirrorError and inform the client
# somehow.
await self.find_and_elect_candidate_mirror(self.context)
await self.configured()
return
# If this call fails with NoUsableMirrorError, we do not
# automatically apply the fallback method. Instead, we let the
# client know that they need to adjust something. Disabling the
# network would be one way to do it. The client can also consider
# this a fatal error and give up on the install.
try:
await self.find_and_elect_candidate_mirror(self.context)
except NoUsableMirrorError:
log.warning("found no usable mirror, expecting the client to"
" give up or to adjust the settings and retry")
return MirrorPostResponse.NO_USABLE_MIRROR
else:
await self.configured()
return MirrorPostResponse.OK
if data.candidates is not None:
if not data.candidates:
@ -329,6 +389,7 @@ class MirrorController(SubiquityController):
ensure_elected_in_candidates()
await self.configured()
return MirrorPostResponse.OK
async def disable_components_GET(self) -> List[str]:
return sorted(self.model.disabled_components)
@ -373,3 +434,9 @@ class MirrorController(SubiquityController):
raise MirrorCheckNotStartedError
self.mirror_check.task.cancel()
self.mirror_check = None
async def fallback_GET(self) -> MirrorSelectionFallback:
return self.model.fallback
async def fallback_POST(self, data: MirrorSelectionFallback):
self.model.fallback = data

33
subiquity/server/controllers/tests/test_mirror.py
View File

@ -20,6 +20,7 @@ import unittest
from unittest import mock
from subiquitycore.tests.mocks import make_app
from subiquity.common.types import MirrorSelectionFallback
from subiquity.models.mirror import MirrorModel
from subiquity.server.apt import AptConfigCheckError
from subiquity.server.controllers.mirror import MirrorController
@ -59,6 +60,7 @@ class TestMirrorController(unittest.IsolatedAsyncioTestCase):
self.assertIn("disable_components", config.keys())
self.assertIn("mirror-selection", config.keys())
self.assertIn("geoip", config.keys())
self.assertIn("fallback", config.keys())
self.assertNotIn("primary", config.keys())
def test_make_autoinstall_legacy(self):
@ -71,6 +73,7 @@ class TestMirrorController(unittest.IsolatedAsyncioTestCase):
self.assertIn("disable_components", config.keys())
self.assertIn("primary", config.keys())
self.assertIn("geoip", config.keys())
self.assertIn("fallback", config.keys())
self.assertNotIn("mirror-selection", config.keys())
async def test_run_mirror_testing(self):
@ -209,3 +212,33 @@ class TestMirrorController(unittest.IsolatedAsyncioTestCase):
await self.controller.find_and_elect_candidate_mirror(
self.controller.app.context)
self.assertIsNone(self.controller.model.primary_elected)
async def test_apply_fallback(self):
model = self.controller.model = MirrorModel()
app = self.controller.app
model.fallback = MirrorSelectionFallback.ABORT
with self.assertRaises(RuntimeError):
await self.controller.apply_fallback()
model.fallback = MirrorSelectionFallback.OFFLINE_INSTALL
app.base_model.network.force_offline = False
await self.controller.apply_fallback()
self.assertTrue(app.base_model.network.force_offline)
model.fallback = MirrorSelectionFallback.CONTINUE_ANYWAY
app.base_model.network.force_offline = False
await self.controller.apply_fallback()
self.assertFalse(app.base_model.network.force_offline)
async def test_run_mirror_selection_or_fallback(self):
controller = self.controller
with mock.patch.object(controller, "apply_fallback") as mock_fallback:
with mock.patch.object(controller,
"find_and_elect_candidate_mirror",
side_effect=[None, NoUsableMirrorError]):
await controller.run_mirror_selection_or_fallback(context=None)
mock_fallback.assert_not_called()
await controller.run_mirror_selection_or_fallback(context=None)
mock_fallback.assert_called_once()

3
subiquitycore/models/network.py
View File

@ -417,10 +417,11 @@ class NetworkModel(object):
self.devices_by_name = {} # Maps interface names to NetworkDev
self._has_network = False
self.project = project
self.force_offline = False
@property
def has_network(self):
return self._has_network
return self._has_network and not self.force_offline
@has_network.setter
def has_network(self, val):