diff --git a/autoinstall-schema.json b/autoinstall-schema.json
index cb6c4976..9fa31681 100644
--- a/autoinstall-schema.json
+++ b/autoinstall-schema.json
@@ -313,6 +313,39 @@
"primary": {
"type": "array"
},
+ "mirror-selection": {
+ "type": "object",
+ "properties": {
+ "primary": {
+ "type": "array",
+ "items": {
+ "anyOf": [
+ {
+ "type": "string",
+ "const": "country-mirror"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "uri": {
+ "type": "string"
+ },
+ "arches": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ },
+ "required": [
+ "uri"
+ ]
+ }
+ ]
+ }
+ }
+ }
+ },
"geoip": {
"type": "boolean"
},
diff --git a/documentation/autoinstall-reference.md b/documentation/autoinstall-reference.md
index 015b8065..2e7e4120 100644
--- a/documentation/autoinstall-reference.md
+++ b/documentation/autoinstall-reference.md
@@ -198,29 +198,53 @@ The proxy to configure both during installation and for apt and for snapd in the
Apt configuration, used both during the install and once booted into the target system.
-This uses the same format as curtin which is documented at https://curtin.readthedocs.io/en/latest/topics/apt_source.html, with one extension: the `geoip` key controls whether a geoip lookup is done.
+This section historically used the same format as curtin, [which is documented here](https://curtin.readthedocs.io/en/latest/topics/apt_source.html). Nonetheless, some key differences with the format supported by curtin have been introduced:
+
+ * Subiquity supports an alternative format for the `primary` section, allowing to configure a list of candidate primary mirrors. During installation, subiquity will automatically test the specified mirrors and select the first one that seems usable. This new behavior is only activated when the `primary` section is wrapped in the `mirror-selection` section.
+ * The `geoip` key controls whether a geoip lookup is done to determine the correct country mirror.
The default is:
apt:
preserve_sources_list: false
- primary:
- - arches: [i386, amd64]
- uri: "http://archive.ubuntu.com/ubuntu"
- - arches: [default]
- uri: "http://ports.ubuntu.com/ubuntu-ports"
+ mirror-selection:
+ primary:
+ - country-mirror
+ - arches: [i386, amd64]
+ uri: "http://archive.ubuntu.com/ubuntu"
+ - arches: [s390x, arm64, armhf, powerpc, ppc64el, riscv64]
+ uri: "http://ports.ubuntu.com/ubuntu-ports"
geoip: true
-If geoip is true and the mirror to be used is the default, a request is made to `https://geoip.ubuntu.com/lookup` and the mirror uri to be used changed to be `http://CC.archive.ubuntu.com/ubuntu` where `CC` is the country code returned by the lookup (or similar for ports). If this section is not interactive, the request is timed out after 10 seconds.
+#### mirror-selection
+if the `primary` section is contained within the `mirror-selection` section, the automatic mirror selection is enabled. This is the default in new installations.
-Any supplied config is merged with the default rather than replacing it.
+#### primary (when placed inside the `mirror-selection` section):
+**type:** custom, see below
-If you just want to set a mirror, use a config like this:
+In the new format, the `primary` section expects a list of mirrors, which can be expressed in two different ways:
+
+ * the special value `country-mirror`
+ * a mapping with the following keys:
+ * `uri`: the URI of the mirror to use, e.g., "http://fr.archive.ubuntu.com/ubuntu"
+ * `arches`: an optional list of architectures supported by the mirror. By default, this list contains the current CPU architecture.
+
+#### geoip
+**type:** boolean
+**default:**: `true`
+
+If geoip is true and one of the candidate primary mirrors has the special value `country-mirror`, a request is made to `https://geoip.ubuntu.com/lookup`. Subiquity then sets the mirror uri to `http://CC.archive.ubuntu.com/ubuntu` (or similar for ports) where `CC` is the country code returned by the lookup. If this section is not interactive, the request is timed out after 10 seconds.
+
+If the legacy behavior (i.e., without mirror-selection) is in use, the geoip request is made if the mirror to be used is the default, and its uri ends up getting replaced by the proper country mirror uri.
+
+If you just want to specify a mirror, you can use a configuration like this:
apt:
- primary:
- - arches: [default]
- uri: YOUR_MIRROR_GOES_HERE
+ mirror-selection:
+ primary:
+ - uri: YOUR_MIRROR_GOES_HERE
+ - country-mirror
+ - uri: http://archive.ubuntu.com/ubuntu
To add a ppa:
diff --git a/documentation/autoinstall-schema.md b/documentation/autoinstall-schema.md
index 6ba66742..0820b569 100644
--- a/documentation/autoinstall-schema.md
+++ b/documentation/autoinstall-schema.md
@@ -335,6 +335,39 @@ The [JSON schema](https://json-schema.org/) for autoinstall data is as follows:
"primary": {
"type": "array"
},
+ "mirror-selection": {
+ "type": "object",
+ "properties": {
+ "primary": {
+ "type": "array",
+ "items": {
+ "anyOf": [
+ {
+ "type": "string",
+ "const": "country-mirror"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "uri": {
+ "type": "string"
+ },
+ "arches": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ },
+ "required": [
+ "uri"
+ ]
+ }
+ ]
+ }
+ }
+ }
+ },
"geoip": {
"type": "boolean"
},
diff --git a/examples/autoinstall-apt-legacy.yaml b/examples/autoinstall-apt-legacy.yaml
new file mode 100644
index 00000000..9a71379b
--- /dev/null
+++ b/examples/autoinstall-apt-legacy.yaml
@@ -0,0 +1,69 @@
+version: 1
+early-commands:
+ - echo a
+ - sleep 1
+ - echo a
+locale: en_GB.UTF-8
+refresh-installer:
+ update: yes
+ channel: edge
+network:
+ version: 2
+ ethernets:
+ all-eth:
+ match:
+ name: "en*"
+ dhcp6: yes
+debconf-selections: eek
+apt:
+ primary:
+ - arches: [default]
+ uri: "http://mymirror.local/repository/Apt/ubuntu/"
+ disable_components:
+ - non-free
+ - restricted
+ preferences:
+ - {package: "python3-*", pin: "origin *ubuntu.com*", pin-priority: 200}
+ - {package: "python-*", pin: "origin *ubuntu.com*", pin-priority: -1}
+packages:
+ - package1
+ - package2
+late-commands:
+ - echo late
+ - sleep 1
+ - echo late
+error-commands:
+ - echo OH NOES
+ - sleep 5
+ - echo OH WELL
+keyboard:
+ layout: gb
+identity:
+ realname: ''
+ username: ubuntu
+ password: '$6$wdAcoXrU039hKYPd$508Qvbe7ObUnxoj15DRCkzC3qO7edjH0VV7BPNRDYK4QR8ofJaEEF2heacn0QgD.f8pO8SNp83XNdWG6tocBM1'
+ hostname: ubuntu
+snaps:
+ - name: etcd
+ channel: 3.2/stable
+updates: all
+timezone: Pacific/Guam
+ubuntu-pro:
+ # Token that passes the basic format checking but is invalid (i.e. contains more than 16 bytes of random data)
+ token: C1NWcZTHLteJXGVMM6YhvHDpGrhyy7
+storage:
+ config:
+ - {type: disk, ptable: gpt, path: /dev/vdb, wipe: superblock, preserve: false, grub_device: true, id: disk-1}
+ - {type: disk, ptable: gpt, path: /dev/vdc, wipe: superblock, preserve: false, grub_device: true, id: disk-2}
+ - {type: partition, device: disk-1, size: 1M, wipe: superblock, flag: bios_grub, number: 1, preserve: false, id: partition-grub-1}
+ - {type: partition, device: disk-2, size: 1M, wipe: superblock, flag: bios_grub, number: 1, preserve: false, id: partition-grub-2}
+ - {type: partition, device: disk-1, size: 1G, wipe: superblock, number: 2, preserve: false, id: partition-boot-1}
+ - {type: partition, device: disk-2, size: 1G, wipe: superblock, number: 2, preserve: false, id: partition-boot-2}
+ - {type: partition, device: disk-1, size: 17%, wipe: superblock, number: 3, preserve: false, id: partition-system-1}
+ - {type: partition, device: disk-2, size: 17%, wipe: superblock, number: 3, preserve: false, id: partition-system-2}
+ - {type: raid, name: md0, raidlevel: raid1, devices: [partition-boot-1, partition-boot-2], preserve: false, id: raid-boot}
+ - {type: raid, name: md1, raidlevel: raid1, devices: [partition-system-1, partition-system-2], preserve: false, id: raid-system}
+ - {type: format, fstype: ext4, volume: raid-boot, preserve: false, id: format-boot}
+ - {type: format, fstype: ext4, volume: raid-system, preserve: false, id: format-system}
+ - {type: mount, device: format-system, path: /, id: mount-system}
+ - {type: mount, device: format-boot, path: /boot, id: mount-boot, options: 'errors=remount-ro'}
diff --git a/examples/autoinstall.yaml b/examples/autoinstall.yaml
index 9a71379b..5fa2055b 100644
--- a/examples/autoinstall.yaml
+++ b/examples/autoinstall.yaml
@@ -16,9 +16,11 @@ network:
dhcp6: yes
debconf-selections: eek
apt:
- primary:
- - arches: [default]
- uri: "http://mymirror.local/repository/Apt/ubuntu/"
+ mirror-selection:
+ primary:
+ - uri: http://mymirror.local/repository/Apt/ubuntu/
+ - country-mirror
+ - uri: http://archive.ubuntu.com/ubuntu
disable_components:
- non-free
- restricted
diff --git a/subiquity/client/controllers/mirror.py b/subiquity/client/controllers/mirror.py
index 8fb78ae8..e0616e5f 100644
--- a/subiquity/client/controllers/mirror.py
+++ b/subiquity/client/controllers/mirror.py
@@ -16,7 +16,11 @@
import asyncio
import logging
-from subiquity.common.types import MirrorCheckStatus
+from subiquity.common.types import (
+ MirrorCheckStatus,
+ MirrorGet,
+ MirrorPost,
+ )
from subiquity.client.controller import SubiquityTuiController
from subiquity.ui.views.mirror import MirrorView
@@ -28,13 +32,24 @@ class MirrorController(SubiquityTuiController):
endpoint_name = 'mirror'
async def make_ui(self):
- mirror = await self.endpoint.GET()
+ mirror_response: MirrorGet = await self.endpoint.GET()
+ # We could do all sort of things with the list of candidate mirrors in
+ # the UI ; like suggesting the next mirror automatically if the first
+ # candidate fails. For now, we keep things simple.
+ if mirror_response.elected is not None:
+ url = mirror_response.elected
+ elif mirror_response.staged:
+ url = mirror_response.staged
+ else:
+ # Just in case there is no candidate at all.
+ # In practise, it should seldom happen.
+ url = next(iter(mirror_response.candidates), "")
has_network = await self.app.client.network.has_network.GET()
if has_network:
check = await self.endpoint.check_mirror.progress.GET()
else:
check = None
- return MirrorView(self, mirror, check=check, has_network=has_network)
+ return MirrorView(self, url, check=check, has_network=has_network)
async def run_answers(self):
async def wait_mirror_check() -> None:
@@ -59,4 +74,5 @@ class MirrorController(SubiquityTuiController):
def done(self, mirror):
log.debug("MirrorController.done next_screen mirror=%s", mirror)
- self.app.next_screen(self.endpoint.POST(mirror))
+ self.app.next_screen(self.endpoint.POST(
+ MirrorPost(elected=mirror)))
diff --git a/subiquity/common/apidef.py b/subiquity/common/apidef.py
index ab12aad5..5b69b0bf 100644
--- a/subiquity/common/apidef.py
+++ b/subiquity/common/apidef.py
@@ -43,6 +43,8 @@ from subiquity.common.types import (
KeyboardSetup,
IdentityData,
NetworkStatus,
+ MirrorGet,
+ MirrorPost,
MirrorCheckResponse,
ModifyPartitionV2,
ReformatDisk,
@@ -348,11 +350,8 @@ class API:
def POST(mode: ShutdownMode, immediate: bool = False): ...
class mirror:
- def GET() -> str: ...
- def POST(data: Payload[str]): ...
-
- class candidate:
- def POST(url: Payload[str]) -> None: ...
+ def GET() -> MirrorGet: ...
+ def POST(data: Payload[Optional[MirrorPost]]) -> None: ...
class disable_components:
def GET() -> List[str]: ...
diff --git a/subiquity/common/types.py b/subiquity/common/types.py
index 29b86fa5..1d366fb3 100644
--- a/subiquity/common/types.py
+++ b/subiquity/common/types.py
@@ -747,6 +747,20 @@ class MirrorCheckResponse:
output: str
+@attr.s(auto_attribs=True)
+class MirrorPost:
+ elected: Optional[str] = None
+ candidates: Optional[List[str]] = None
+ staged: Optional[str] = None
+
+
+@attr.s(auto_attribs=True)
+class MirrorGet:
+ elected: Optional[str]
+ candidates: List[str]
+ staged: Optional[str]
+
+
@attr.s(auto_attribs=True)
class ADConnectionInfo:
admin_name: str = ""
diff --git a/subiquity/models/mirror.py b/subiquity/models/mirror.py
index 95d262c5..bcfb2062 100644
--- a/subiquity/models/mirror.py
+++ b/subiquity/models/mirror.py
@@ -12,15 +12,78 @@
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see .
+""" This model mainly manages the mirror selection but also covers all the
+settings that can be found under the 'apt' autoinstall section.
+Some settings are handled by Subiquity but others are directly forwarded to
+curtin.
+There are a few notions worth explaining related to mirror selection:
+
+primary
+-------
+ * a "primary mirror" (or a "primary archive") is what curtin historically
+ considers as the main repository where it can download Debian packages.
+ Surprisingly, there is no notion of "secondary mirror". Instead there is the
+ "security archive" where we download the packages from the -security pocket.
+
+candidates
+----------
+ * a given install can have multiple primary candidate mirrors organized in a
+ list. Providing multiple candidates increases the likelihood of having one
+ tested successfully.
+ When the process of mirror selection is run automatically, the candidates will
+ be tested one after another until one passes the test. The one passing is then
+ marked "elected".
+
+staged
+------
+ * a primary mirror candidate can be "staged" or "staged for testing". The
+ staged mirror is the one that will be used if subiquity decides to trigger a
+ test of the apt configuration (a.k.a., mirror testing).
+
+elected
+-------
+ * if a primary mirror candidate is marked "elected", then it is used when
+ subiquity requests the final apt configuration. This means it will be used
+ as the primary mirror during the install (only if we are online), and will
+ end up in etc/apt/sources.list in the target system.
+
+primary section
+---------------
+ * the "primary section" contains the different candidates for mirror
+ selection. Today we support two different formats for this section, and the
+ position of the primary section is what determines which format is used.
+ * the legacy format, inherited from curtin, where the primary section is a
+ direct child of the 'apt' section. In this format, the whole section denotes
+ a single primary candidate so it cannot be used to specify multiple
+ candidates.
+ * the more modern format where the primary section is a child of the
+ 'mirror-selection' key (which itself is a child of 'apt'). In this format,
+ the section is split into multiple "entries", each denoting a primary
+ candidate.
+
+primary entry
+-------------
+ * represents a fragment of the 'primary' autoinstall section. Each entry
+ can be used as a primary candidate.
+ * in the legacy format, the primary entry corresponds to the whole primary
+ section.
+ * in the new format, multiple primary entries make the primary section.
+"""
+
+import abc
import copy
+import contextlib
import logging
-from typing import Any, List, Set
+from typing import Any, Dict, Iterator, List, Optional, Sequence, Set, Union
from urllib import parse
+import attr
+
from curtin.commands.apt_config import (
get_arch_mirrorconfig,
get_mirror,
+ PORTS_ARCHES,
PRIMARY_ARCHES,
)
from curtin.config import merge_config
@@ -32,12 +95,119 @@ except ImportError:
log = logging.getLogger('subiquity.models.mirror')
+DEFAULT_SUPPORTED_ARCHES_URI = "http://archive.ubuntu.com/ubuntu"
+DEFAULT_PORTS_ARCHES_URI = "http://ports.ubuntu.com/ubuntu-ports"
+
+LEGACY_DEFAULT_PRIMARY_SECTION = [
+ {
+ "arches": PRIMARY_ARCHES,
+ "uri": DEFAULT_SUPPORTED_ARCHES_URI,
+ }, {
+ "arches": ["default"],
+ "uri": DEFAULT_PORTS_ARCHES_URI,
+ },
+]
DEFAULT = {
"preserve_sources_list": False,
}
+@attr.s(auto_attribs=True)
+class BasePrimaryEntry(abc.ABC):
+ """ Base class to represent an entry from the 'primary' autoinstall
+ section. A BasePrimaryEntry is expected to have a URI and therefore can be
+ used as a primary candidate. """
+ parent: "MirrorModel" = attr.ib(kw_only=True)
+
+ def stage(self) -> None:
+ self.parent.primary_staged = self
+
+ def elect(self) -> None:
+ self.parent.primary_elected = self
+
+ @abc.abstractmethod
+ def serialize_for_ai(self) -> Any:
+ """ Serialize the entry for autoinstall. """
+
+
+@attr.s(auto_attribs=True)
+class PrimaryEntry(BasePrimaryEntry):
+ """ Represents a single primary mirror candidate; which can be converted
+ to/from an entry of the 'apt->mirror-selection->primary' autoinstall
+ section. """
+ # Having uri set to None is only valid for a country mirror.
+ uri: Optional[str] = None
+ # When arches is None, it is assumed that the mirror is compatible with the
+ # current CPU architecture.
+ arches: Optional[List[str]] = None
+ country_mirror: bool = attr.ib(kw_only=True, default=False)
+
+ @classmethod
+ def from_config(cls, config: Any, parent: "MirrorModel") -> "PrimaryEntry":
+ if config == "country-mirror":
+ return cls(parent=parent, country_mirror=True)
+ if config.get("uri", None) is None:
+ raise ValueError("uri is mandatory")
+ return cls(**config, parent=parent)
+
+ @property
+ def config(self) -> List[Dict[str, Any]]:
+ assert self.uri is not None
+ # Do not bother passing specific arches to curtin, we are passing a
+ # single URI anyway.
+ return [{"uri": self.uri, "arches": ["default"]}]
+
+ def supports_arch(self, arch: str) -> bool:
+ """ Tells whether the mirror claims to support the architecture
+ specified. """
+ if self.arches is None:
+ return True
+ return arch in self.arches
+
+ def serialize_for_ai(self) -> Union[str, Dict[str, Any]]:
+ if self.country_mirror:
+ return "country-mirror"
+ ret: Dict[str, Any] = {"uri": self.uri}
+ if self.arches is not None:
+ ret["arches"] = self.arches
+ return ret
+
+
+class LegacyPrimaryEntry(BasePrimaryEntry):
+ """ Represents a single primary mirror candidate; which can be converted
+ to/from the whole 'apt->primary' autoinstall section (legacy format).
+ The format is defined by curtin, so we make use of curtin to access
+ the elements. """
+ def __init__(self, config: List[Any], *, parent: "MirrorModel") -> None:
+ self.config = config
+ super().__init__(parent=parent)
+
+ @property
+ def uri(self) -> str:
+ config = copy.deepcopy(self.parent.config)
+ config["primary"] = self.config
+ return get_mirror(config, "primary", self.parent.architecture)
+
+ @uri.setter
+ def uri(self, uri: str) -> None:
+ config = get_arch_mirrorconfig(
+ {"primary": self.config},
+ "primary", self.parent.architecture)
+ config["uri"] = uri
+
+ def mirror_is_default(self) -> bool:
+ return self.uri == self.parent.default_mirror
+
+ @classmethod
+ def new_from_default(cls, parent: "MirrorModel") -> "LegacyPrimaryEntry":
+ return cls(copy.deepcopy(LEGACY_DEFAULT_PRIMARY_SECTION),
+ parent=parent)
+
+ def serialize_for_ai(self) -> List[Any]:
+ return self.config
+
+
def countrify_uri(uri: str, cc: str) -> str:
""" Return a URL where the host is prefixed with a country code. """
parsed = parse.urlparse(uri)
@@ -49,51 +219,117 @@ class MirrorModel(object):
def __init__(self):
self.config = copy.deepcopy(DEFAULT)
+ self.legacy_primary = False
self.disabled_components: Set[str] = set()
- self.primary: List[Any] = [
- {
- "arches": PRIMARY_ARCHES,
- "uri": "http://archive.ubuntu.com/ubuntu",
- }, {
- "arches": ["default"],
- "uri": "http://ports.ubuntu.com/ubuntu-ports",
- },
- ]
+ self.primary_elected: Optional[BasePrimaryEntry] = None
+ self.primary_candidates: List[BasePrimaryEntry] = \
+ self._default_primary_entries()
+
+ self.primary_staged: Optional[BasePrimaryEntry] = None
self.architecture = get_architecture()
- self.default_mirror = self.get_mirror()
+ # Only useful for legacy primary sections
+ self.default_mirror = \
+ LegacyPrimaryEntry.new_from_default(parent=self).uri
+
+ def _default_primary_entries(self) -> List[PrimaryEntry]:
+ return [
+ PrimaryEntry(parent=self, country_mirror=True),
+ PrimaryEntry(uri=DEFAULT_SUPPORTED_ARCHES_URI,
+ arches=PRIMARY_ARCHES, parent=self),
+ PrimaryEntry(uri=DEFAULT_PORTS_ARCHES_URI,
+ arches=PORTS_ARCHES, parent=self),
+ ]
+
+ def get_default_primary_candidates(
+ self, legacy: Optional[bool] = None) -> Sequence[BasePrimaryEntry]:
+ want_legacy = legacy if legacy is not None else self.legacy_primary
+ if want_legacy:
+ return [LegacyPrimaryEntry.new_from_default(parent=self)]
+ else:
+ return self._default_primary_entries()
def load_autoinstall_data(self, data):
if "disable_components" in data:
self.disabled_components = set(data.pop("disable_components"))
+
+ if "primary" in data and "mirror-selection" in data:
+ raise ValueError("apt->primary and apt->mirror-selection are"
+ " mutually exclusive.")
+ self.legacy_primary = "primary" in data
+
+ primary_candidates = self.get_default_primary_candidates()
if "primary" in data:
- self.primary = data.pop("primary")
+ # Legacy sections only support a single candidate
+ primary_candidates = \
+ [LegacyPrimaryEntry(data.pop("primary"), parent=self)]
+ if "mirror-selection" in data:
+ mirror_selection = data.pop("mirror-selection")
+ if "primary" in mirror_selection:
+ primary_candidates = []
+ for section in mirror_selection["primary"]:
+ entry = PrimaryEntry.from_config(section, parent=self)
+ primary_candidates.append(entry)
+ self.primary_candidates = primary_candidates
+
merge_config(self.config, data)
- def get_apt_config(self):
+ def _get_apt_config_common(self) -> Dict[str, Any]:
+ assert "disable_components" not in self.config
+ assert "primary" not in self.config
+
config = copy.deepcopy(self.config)
- config["primary"] = copy.deepcopy(self.primary)
config["disable_components"] = sorted(self.disabled_components)
return config
- def mirror_is_default(self):
- return self.get_mirror() == self.default_mirror
+ def _get_apt_config_using_candidate(
+ self, candidate: BasePrimaryEntry) -> Dict[str, Any]:
+ config = self._get_apt_config_common()
+ config["primary"] = candidate.config
+ return config
+
+ def get_apt_config_staged(self) -> Dict[str, Any]:
+ assert self.primary_staged is not None
+ return self._get_apt_config_using_candidate(self.primary_staged)
+
+ def get_apt_config_elected(self) -> Dict[str, Any]:
+ assert self.primary_elected is not None
+ return self._get_apt_config_using_candidate(self.primary_elected)
+
+ def get_apt_config(
+ self, final: bool, has_network: bool) -> Dict[str, Any]:
+ if not final:
+ return self.get_apt_config_staged()
+ if has_network:
+ return self.get_apt_config_elected()
+ # We want the final configuration but have no network. In this scenario
+ # it is possible that we do not have an elected primary mirror.
+ if self.primary_elected is not None:
+ return self.get_apt_config_elected()
+ # Look for the first compatible candidate with a URI.
+ # There is no guarantee that it will be a working mirror since we have
+ # not tested it. But it is fine because it will not be used during the
+ # install. It will be placed in etc/apt/sources.list of the target
+ # system.
+ with contextlib.suppress(StopIteration):
+ candidate = next(filter(lambda c: c.uri is not None,
+ self.compatible_primary_candidates()))
+ return self._get_apt_config_using_candidate(candidate)
+ # Our last resort is to include no primary section. Curtin will use
+ # its own internal values.
+ return self._get_apt_config_common()
def set_country(self, cc):
- if not self.mirror_is_default():
- return
- uri = self.get_mirror()
- self.set_mirror(countrify_uri(uri, cc=cc))
-
- def get_mirror(self):
- config = copy.deepcopy(self.config)
- config["primary"] = self.primary
- return get_mirror(config, "primary", self.architecture)
-
- def set_mirror(self, mirror):
- config = get_arch_mirrorconfig(
- {"primary": self.primary}, "primary", self.architecture)
- config["uri"] = mirror
+ """ Set the URI of country-mirror candidates. """
+ for candidate in self.country_mirror_candidates():
+ if self.legacy_primary:
+ candidate.uri = countrify_uri(candidate.uri, cc=cc)
+ else:
+ if self.architecture in PRIMARY_ARCHES:
+ uri = DEFAULT_SUPPORTED_ARCHES_URI
+ else:
+ uri = DEFAULT_PORTS_ARCHES_URI
+ candidate.uri = countrify_uri(uri, cc=cc)
def disable_components(self, comps, add: bool) -> None:
""" Add (or remove) a component (e.g., multiverse) from the list of
@@ -104,8 +340,53 @@ class MirrorModel(object):
else:
self.disabled_components -= comps
+ def create_primary_candidate(
+ self, uri: Optional[str],
+ country_mirror: bool = False) -> BasePrimaryEntry:
+
+ if self.legacy_primary:
+ entry = LegacyPrimaryEntry.new_from_default(parent=self)
+ entry.uri = uri
+ return entry
+
+ return PrimaryEntry(uri=uri, country_mirror=country_mirror,
+ parent=self)
+
+ def wants_geoip(self) -> bool:
+ """ Tell whether geoip results would be useful. """
+ return next(self.country_mirror_candidates(), None) is not None
+
+ def country_mirror_candidates(self) -> Iterator[BasePrimaryEntry]:
+ for candidate in self.primary_candidates:
+ if self.legacy_primary and candidate.mirror_is_default():
+ yield candidate
+ elif not self.legacy_primary and candidate.country_mirror:
+ yield candidate
+
+ def compatible_primary_candidates(self) -> Iterator[BasePrimaryEntry]:
+ for candidate in self.primary_candidates:
+ if self.legacy_primary:
+ yield candidate
+ elif candidate.arches is None:
+ yield candidate
+ elif self.architecture in candidate.arches:
+ yield candidate
+
def render(self):
return {}
def make_autoinstall(self):
- return self.get_apt_config()
+ config = self._get_apt_config_common()
+ if self.legacy_primary:
+ # Only one candidate is supported
+ if self.primary_elected is not None:
+ to_serialize = self.primary_elected
+ else:
+ # In an offline autoinstall, there is no elected mirror.
+ to_serialize = self.primary_candidates[0]
+ config["primary"] = to_serialize.serialize_for_ai()
+ else:
+ primary = [c.serialize_for_ai() for c in self.primary_candidates]
+ config["mirror-selection"] = {"primary": primary}
+
+ return config
diff --git a/subiquity/models/proxy.py b/subiquity/models/proxy.py
index ac9f4b89..caebe3fd 100644
--- a/subiquity/models/proxy.py
+++ b/subiquity/models/proxy.py
@@ -32,7 +32,7 @@ class ProxyModel(object):
def proxy_systemd_dropin(self):
return dropin_template.format(proxy=self.proxy)
- def get_apt_config(self):
+ def get_apt_config(self, final: bool, has_network: bool):
if self.proxy:
return {
'http_proxy': self.proxy,
diff --git a/subiquity/models/tests/test_mirror.py b/subiquity/models/tests/test_mirror.py
index d6265f15..6776ba72 100644
--- a/subiquity/models/tests/test_mirror.py
+++ b/subiquity/models/tests/test_mirror.py
@@ -13,11 +13,16 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see .
+import copy
import unittest
+from unittest import mock
from subiquity.models.mirror import (
countrify_uri,
+ LEGACY_DEFAULT_PRIMARY_SECTION,
MirrorModel,
+ LegacyPrimaryEntry,
+ PrimaryEntry,
)
@@ -49,61 +54,229 @@ class TestCountrifyUrl(unittest.TestCase):
"http://us.ports.ubuntu.com/ubuntu-ports")
-class TestMirrorModel(unittest.TestCase):
+class TestPrimaryEntry(unittest.TestCase):
+ def test_initializer(self):
+ model = MirrorModel()
+
+ entry = PrimaryEntry(parent=model)
+ self.assertEqual(entry.parent, model)
+ self.assertIsNone(entry.uri, None)
+ self.assertIsNone(entry.arches, None)
+
+ entry = PrimaryEntry("http://mirror", ["amd64"], parent=model)
+ self.assertEqual(entry.parent, model)
+ self.assertEqual(entry.uri, "http://mirror")
+ self.assertEqual(entry.arches, ["amd64"])
+
+ entry = PrimaryEntry(uri="http://mirror", arches=[], parent=model)
+ self.assertEqual(entry.parent, model)
+ self.assertEqual(entry.uri, "http://mirror")
+ self.assertEqual(entry.arches, [])
+
+ def test_from_config(self):
+ model = MirrorModel()
+
+ entry = PrimaryEntry.from_config("country-mirror", parent=model)
+ self.assertEqual(entry, PrimaryEntry(parent=model,
+ country_mirror=True))
+
+ with self.assertRaises(ValueError):
+ entry = PrimaryEntry.from_config({}, parent=model)
+
+ entry = PrimaryEntry.from_config(
+ {"uri": "http://mirror"}, parent=model)
+ self.assertEqual(entry, PrimaryEntry(
+ uri="http://mirror", parent=model))
+
+ entry = PrimaryEntry.from_config(
+ {"uri": "http://mirror", "arches": ["amd64"]}, parent=model)
+ self.assertEqual(entry, PrimaryEntry(
+ uri="http://mirror", arches=["amd64"], parent=model))
+
+
+class TestLegacyPrimaryEntry(unittest.TestCase):
def setUp(self):
self.model = MirrorModel()
+ def test_initializer(self):
+ primary = LegacyPrimaryEntry([], parent=self.model)
+ self.assertEqual(primary.config, [])
+ self.assertEqual(primary.parent, self.model)
+
+ def test_new_from_default(self):
+ primary = LegacyPrimaryEntry.new_from_default(parent=self.model)
+ self.assertEqual(primary.config, LEGACY_DEFAULT_PRIMARY_SECTION)
+
+ def test_get_uri(self):
+ self.model.architecture = "amd64"
+ primary = LegacyPrimaryEntry(
+ [{"uri": "http://myurl", "arches": "amd64"}],
+ parent=self.model)
+ self.assertEqual(primary.uri, "http://myurl")
+
+ def test_set_uri(self):
+ primary = LegacyPrimaryEntry.new_from_default(parent=self.model)
+ primary.uri = "http://mymirror.invalid/"
+ self.assertEqual(primary.uri, "http://mymirror.invalid/")
+
+
+class TestMirrorModel(unittest.TestCase):
+ def setUp(self):
+ self.model = MirrorModel()
+ self.candidate = self.model.primary_candidates[1]
+ self.candidate.stage()
+
+ self.model_legacy = MirrorModel()
+ self.model_legacy.legacy_primary = True
+ self.model_legacy.primary_candidates = [
+ LegacyPrimaryEntry(copy.deepcopy(
+ LEGACY_DEFAULT_PRIMARY_SECTION), parent=self.model_legacy),
+ ]
+ self.candidate_legacy = self.model_legacy.primary_candidates[0]
+ self.model_legacy.primary_staged = self.candidate_legacy
+
+ def test_initializer(self):
+ model = MirrorModel()
+ self.assertFalse(model.legacy_primary)
+ self.assertIsNone(model.primary_staged)
+
def test_set_country(self):
- self.model.set_country("CC")
- self.assertIn(
- self.model.get_mirror(),
- [
- "http://CC.archive.ubuntu.com/ubuntu",
- "http://CC.ports.ubuntu.com/ubuntu-ports",
- ])
+ def do_test(model):
+ country_mirror_candidates = list(model.country_mirror_candidates())
+ self.assertEqual(len(country_mirror_candidates), 1)
+ model.set_country("CC")
+ for country_mirror_candidate in country_mirror_candidates:
+ self.assertIn(
+ country_mirror_candidate.uri,
+ [
+ "http://CC.archive.ubuntu.com/ubuntu",
+ "http://CC.ports.ubuntu.com/ubuntu-ports",
+ ])
- def test_set_mirror(self):
- self.model.set_mirror("http://mymirror.invalid/")
- self.assertEqual(self.model.get_mirror(), "http://mymirror.invalid/")
+ do_test(self.model)
+ do_test(self.model_legacy)
- def test_set_country_after_set_mirror(self):
- self.model.set_mirror("http://mymirror.invalid/")
- self.model.set_country("CC")
- self.assertEqual(self.model.get_mirror(), "http://mymirror.invalid/")
+ def test_set_country_after_set_uri_legacy(self):
+ for candidate in self.model_legacy.primary_candidates:
+ candidate.uri = "http://mymirror.invalid/"
+ self.model_legacy.set_country("CC")
+ for candidate in self.model_legacy.primary_candidates:
+ self.assertEqual(candidate.uri, "http://mymirror.invalid/")
def test_default_disable_components(self):
- config = self.model.get_apt_config()
- self.assertEqual([], config['disable_components'])
+ def do_test(model, candidate):
+ config = model.get_apt_config_staged()
+ self.assertEqual([], config['disable_components'])
- def test_from_autoinstall(self):
+ # The candidate[0] is a country-mirror, skip it.
+ candidate = self.model.primary_candidates[1]
+ candidate.stage()
+ do_test(self.model, candidate)
+ do_test(self.model_legacy, self.candidate_legacy)
+
+ def test_from_autoinstall_no_primary(self):
# autoinstall loads to the config directly
+ model = MirrorModel()
data = {'disable_components': ['non-free']}
- self.model.load_autoinstall_data(data)
- config = self.model.get_apt_config()
- self.assertEqual(['non-free'], config['disable_components'])
+ model.load_autoinstall_data(data)
+ self.assertFalse(model.legacy_primary)
+ model.primary_candidates[0].stage()
+ self.assertEqual(set(['non-free']), model.disabled_components)
+ self.assertEqual(model.primary_candidates,
+ model._default_primary_entries())
+
+ def test_from_autoinstall_modern(self):
+ data = {
+ "mirror-selection": {
+ "primary": [
+ "country-mirror",
+ {
+ "uri": "http://mirror",
+ },
+ ],
+ }
+ }
+ model = MirrorModel()
+ model.load_autoinstall_data(data)
+ self.assertEqual(model.primary_candidates, [
+ PrimaryEntry(parent=model, country_mirror=True),
+ PrimaryEntry(uri="http://mirror", parent=model),
+ ])
def test_disable_add(self):
- expected = ['things', 'stuff']
- self.model.disable_components(expected.copy(), add=True)
- actual = self.model.get_apt_config()['disable_components']
- actual.sort()
- expected.sort()
- self.assertEqual(expected, actual)
+ def do_test(model, candidate):
+ expected = ['things', 'stuff']
+ model.disable_components(expected.copy(), add=True)
+ actual = model.get_apt_config_staged()['disable_components']
+ actual.sort()
+ expected.sort()
+ self.assertEqual(expected, actual)
+
+ # The candidate[0] is a country-mirror, skip it.
+ candidate = self.model.primary_candidates[1]
+ candidate.stage()
+ do_test(self.model, candidate)
+ do_test(self.model_legacy, self.candidate_legacy)
def test_disable_remove(self):
self.model.disabled_components = set(['a', 'b', 'things'])
to_remove = ['things', 'stuff']
expected = ['a', 'b']
self.model.disable_components(to_remove, add=False)
- actual = self.model.get_apt_config()['disable_components']
+ actual = self.model.get_apt_config_staged()['disable_components']
actual.sort()
expected.sort()
self.assertEqual(expected, actual)
- def test_make_autoinstall(self):
+ def test_make_autoinstall_primary(self):
+ expected_primary = [
+ "country-mirror",
+ {"uri": "http://mirror.local/ubuntu"},
+ {"uri": "http://amd64.mirror.local/ubuntu", "arches": ["amd64"]},
+ ]
+ self.model.disabled_components = set(["non-free"])
+ self.model.legacy_primary = False
+ self.model.primary_candidates = [
+ PrimaryEntry(uri=None, arches=None,
+ country_mirror=True, parent=self.model),
+ PrimaryEntry(uri="http://mirror.local/ubuntu",
+ arches=None, parent=self.model),
+ PrimaryEntry(uri="http://amd64.mirror.local/ubuntu",
+ arches=["amd64"], parent=self.model),
+ ]
+ cfg = self.model.make_autoinstall()
+ self.assertEqual(cfg["disable_components"], ["non-free"])
+ self.assertEqual(cfg["mirror-selection"]["primary"], expected_primary)
+
+ def test_make_autoinstall_legacy_primary(self):
primary = [{"arches": "amd64", "uri": "http://mirror"}]
self.model.disabled_components = set(["non-free"])
- self.model.primary = primary
+ self.model.legacy_primary = True
+ self.model.primary_candidates = \
+ [LegacyPrimaryEntry(primary, parent=self.model)]
+ self.model.primary_candidates[0].elect()
cfg = self.model.make_autoinstall()
self.assertEqual(cfg["disable_components"], ["non-free"])
self.assertEqual(cfg["primary"], primary)
+
+ def test_create_primary_candidate(self):
+ self.model.legacy_primary = False
+ candidate = self.model.create_primary_candidate(
+ "http://mymirror.valid")
+ self.assertEqual(candidate.uri, "http://mymirror.valid")
+ self.model.legacy_primary = True
+ candidate = self.model.create_primary_candidate(
+ "http://mymirror.valid")
+ self.assertEqual(candidate.uri, "http://mymirror.valid")
+
+ def test_wants_geoip(self):
+ country_mirror_candidates = mock.patch.object(
+ self.model, "country_mirror_candidates", return_value=iter([]))
+ with country_mirror_candidates:
+ self.assertFalse(self.model.wants_geoip())
+
+ country_mirror_candidates = mock.patch.object(
+ self.model, "country_mirror_candidates",
+ return_value=iter([PrimaryEntry(parent=self.model)]))
+ with country_mirror_candidates:
+ self.assertTrue(self.model.wants_geoip())
diff --git a/subiquity/models/tests/test_subiquity.py b/subiquity/models/tests/test_subiquity.py
index f45beb69..fb4b8a9d 100644
--- a/subiquity/models/tests/test_subiquity.py
+++ b/subiquity/models/tests/test_subiquity.py
@@ -206,7 +206,7 @@ class TestSubiquityModel(unittest.IsolatedAsyncioTestCase):
def test_mirror(self):
model = self.make_model()
mirror_val = 'http://my-mirror'
- model.mirror.set_mirror(mirror_val)
+ model.mirror.create_primary_candidate(mirror_val).elect()
config = model.render()
self.assertNotIn('apt', config)
diff --git a/subiquity/server/apt.py b/subiquity/server/apt.py
index 402fb410..5ff278b4 100644
--- a/subiquity/server/apt.py
+++ b/subiquity/server/apt.py
@@ -112,18 +112,20 @@ class AptConfigurer:
self.install_tree: Optional[OverlayMountpoint] = None
self.install_mount = None
- def apt_config(self):
+ def apt_config(self, final: bool):
cfg = {}
- merge_config(cfg, self.app.base_model.mirror.get_apt_config())
- merge_config(cfg, self.app.base_model.proxy.get_apt_config())
+ has_network = self.app.base_model.network.has_network
+ for model in self.app.base_model.mirror, self.app.base_model.proxy:
+ merge_config(cfg, model.get_apt_config(
+ final=final, has_network=has_network))
return {'apt': cfg}
- async def apply_apt_config(self, context):
+ async def apply_apt_config(self, context, final: bool):
self.configured_tree = await self.mounter.setup_overlay([self.source])
config_location = os.path.join(
self.app.root, 'var/log/installer/subiquity-curtin-apt.conf')
- generate_config_yaml(config_location, self.apt_config())
+ generate_config_yaml(config_location, self.apt_config(final))
self.app.note_data_for_apport("CurtinAptConfig", config_location)
await run_curtin_command(
@@ -320,7 +322,7 @@ class DryRunAptConfigurer(AptConfigurer):
async def apt_config_check_failure(self, output: io.StringIO) -> None:
""" Pretend that the execution of the apt-get update command results in
a failure. """
- url = self.app.base_model.mirror.get_mirror()
+ url = self.app.base_model.mirror.primary_staged.uri
release = lsb_release(dry_run=True)["codename"]
host = url.split("/")[2]
@@ -356,7 +358,7 @@ E: Some index files failed to download. They have been ignored,
async def apt_config_check_success(self, output: io.StringIO) -> None:
""" Pretend that the execution of the apt-get update command results in
a success. """
- url = self.app.base_model.mirror.get_mirror()
+ url = self.app.base_model.mirror.primary_staged.uri
release = lsb_release(dry_run=True)["codename"]
output.write(f"""\
@@ -383,7 +385,7 @@ Reading package lists...
self.MirrorCheckStrategy.SUCCESS: success,
self.MirrorCheckStrategy.RANDOM: random.choice([failure, success]),
}
- mirror_url = self.app.base_model.mirror.get_mirror()
+ mirror_url = self.app.base_model.mirror.primary_staged.uri
strategy = strategies[self.get_mirror_check_strategy(mirror_url)]
diff --git a/subiquity/server/controllers/mirror.py b/subiquity/server/controllers/mirror.py
index 9e2d4119..8af48603 100644
--- a/subiquity/server/controllers/mirror.py
+++ b/subiquity/server/controllers/mirror.py
@@ -27,6 +27,8 @@ from subiquity.common.apidef import API
from subiquity.common.types import (
MirrorCheckResponse,
MirrorCheckStatus,
+ MirrorGet,
+ MirrorPost,
)
from subiquity.server.apt import get_apt_configurer, AptConfigCheckError
from subiquity.server.controller import SubiquityController
@@ -35,6 +37,11 @@ from subiquity.server.types import InstallerChannels
log = logging.getLogger('subiquity.server.controllers.mirror')
+class NoUsableMirrorError(Exception):
+ """ Exception to be raised when none of the candidate mirrors passed the
+ test. """
+
+
class MirrorCheckNotStartedError(Exception):
""" Exception to be raised when trying to cancel a mirror
check that was not started. """
@@ -56,7 +63,33 @@ class MirrorController(SubiquityController):
'type': 'object',
'properties': {
'preserve_sources_list': {'type': 'boolean'},
- 'primary': {'type': 'array'},
+ 'primary': {'type': 'array'}, # Legacy format defined by curtin.
+ 'mirror-selection': {
+ 'type': 'object',
+ 'properties': {
+ 'primary': {
+ 'type': 'array',
+ 'items': {
+ 'anyOf': [
+ {
+ 'type': 'string',
+ 'const': 'country-mirror',
+ }, {
+ 'type': 'object',
+ 'properties': {
+ 'uri': {'type': 'string'},
+ 'arches': {
+ 'type': 'array',
+ 'items': {'type': 'string'},
+ },
+ },
+ 'required': ['uri'],
+ },
+ ],
+ },
+ },
+ },
+ },
'geoip': {'type': 'boolean'},
'sources': {'type': 'object'},
'disable_components': {
@@ -96,12 +129,20 @@ class MirrorController(SubiquityController):
def __init__(self, app):
super().__init__(app)
self.geoip_enabled = True
- self.app.hub.subscribe(InstallerChannels.GEOIP, self.on_geoip)
- self.app.hub.subscribe(
- (InstallerChannels.CONFIGURED, 'source'), self.on_source)
self.cc_event = asyncio.Event()
self.configured_event = asyncio.Event()
self.source_configured_event = asyncio.Event()
+ self.network_configured_event = asyncio.Event()
+ self.proxy_configured_event = asyncio.Event()
+ self.app.hub.subscribe(InstallerChannels.GEOIP, self.on_geoip)
+ self.app.hub.subscribe(
+ (InstallerChannels.CONFIGURED, 'source'), self.on_source)
+ self.app.hub.subscribe(
+ (InstallerChannels.CONFIGURED, 'network'),
+ self.network_configured_event.set)
+ self.app.hub.subscribe(
+ (InstallerChannels.CONFIGURED, 'proxy'),
+ self.proxy_configured_event.set)
self._apt_config_key = None
self._apply_apt_config_task = SingleInstanceTask(
self._promote_mirror)
@@ -113,7 +154,7 @@ class MirrorController(SubiquityController):
return
geoip = data.pop('geoip', True)
self.model.load_autoinstall_data(data)
- self.geoip_enabled = geoip and self.model.mirror_is_default()
+ self.geoip_enabled = geoip and self.model.wants_geoip()
async def try_mirror_checking_once(self) -> None:
""" Try mirror checking and log result. """
@@ -130,8 +171,12 @@ class MirrorController(SubiquityController):
for line in output.getvalue().splitlines():
log.debug("%s", line)
- @with_context()
- async def apply_autoinstall_config(self, context):
+ async def find_and_elect_candidate_mirror(self, context):
+ # Ensure we block until the proxy and network models have been
+ # configured. This is particularly important in partially-automated
+ # installs.
+ await self.network_configured_event.wait()
+ await self.proxy_configured_event.wait()
if self.geoip_enabled:
try:
with context.child('waiting'):
@@ -143,13 +188,40 @@ class MirrorController(SubiquityController):
log.debug("Skipping mirror check since network is not available.")
return
- try:
- await self.try_mirror_checking_once()
- except AptConfigCheckError:
- log.debug("Retrying in 10 seconds...")
+ # Try each mirror one after another.
+ compatibles = self.model.compatible_primary_candidates()
+ for idx, candidate in enumerate(compatibles):
+ log.debug("Iterating over %s", candidate.serialize_for_ai())
+ if idx != 0:
+ # Sleep before testing the next candidate..
+ log.debug("Will check next candiate mirror after 10 seconds.")
+ await asyncio.sleep(10)
+ if candidate.uri is None:
+ log.debug("Skipping unresolved country mirror")
+ continue
+ candidate.stage()
+ try:
+ await self.try_mirror_checking_once()
+ except AptConfigCheckError:
+ log.debug("Retrying in 10 seconds...")
+ else:
+ break
await asyncio.sleep(10)
- # If the test fails a second time, consider it fatal.
- await self.try_mirror_checking_once()
+ # If the test fails a second time, give up on this mirror.
+ try:
+ await self.try_mirror_checking_once()
+ except AptConfigCheckError:
+ log.debug("Mirror is not usable.")
+ else:
+ break
+ else:
+ raise NoUsableMirrorError
+
+ candidate.elect()
+
+ @with_context()
+ async def apply_autoinstall_config(self, context):
+ await self.find_and_elect_candidate_mirror(context)
def on_geoip(self):
if self.geoip_enabled:
@@ -166,10 +238,15 @@ class MirrorController(SubiquityController):
self.source_configured_event.set()
def serialize(self):
- return self.model.get_mirror()
+ # TODO what to do with the candidates?
+ if self.model.primary_elected is not None:
+ return self.model.primary_elected.uri
+ return None
def deserialize(self, data):
- self.model.set_mirror(data)
+ # TODO what to do with the candidates?
+ if data is not None:
+ self.model.create_primary_candidate(data).elect()
def make_autoinstall(self):
config = self.model.make_autoinstall()
@@ -184,28 +261,74 @@ class MirrorController(SubiquityController):
async def _promote_mirror(self):
await asyncio.gather(self.source_configured_event.wait(),
self.configured_event.wait())
- await self.apt_configurer.apply_apt_config(self.context)
+ if self.model.primary_elected is None:
+ # NOTE: In practice, this should only happen if the mirror was
+ # marked configured using a POST to mark_configured ; which is not
+ # recommended. Clients should do a POST request to /mirror with
+ # null as the body instead.
+ await self.find_and_elect_candidate_mirror(self.context)
+ await self.apt_configurer.apply_apt_config(self.context, final=True)
async def run_mirror_testing(self, output: io.StringIO) -> None:
await self.source_configured_event.wait()
- await self.apt_configurer.apply_apt_config(self.context)
+ await self.apt_configurer.apply_apt_config(self.context, final=False)
await self.apt_configurer.run_apt_config_check(output)
async def wait_config(self):
await self._apply_apt_config_task.wait()
return self.apt_configurer
- async def GET(self) -> str:
- return self.model.get_mirror()
+ async def GET(self) -> MirrorGet:
+ elected: Optional[str] = None
+ staged: Optional[str] = None
+ if self.model.primary_elected is not None:
+ elected = self.model.primary_elected.uri
+ if self.model.primary_staged is not None:
+ staged = self.model.primary_staged.uri
- async def POST(self, data: str):
+ compatibles = self.model.compatible_primary_candidates()
+ # Skip the country-mirrors if they have not been resolved yet.
+ candidates = [c.uri for c in compatibles if c.uri is not None]
+ return MirrorGet(elected=elected, candidates=candidates, staged=staged)
+
+ async def POST(self, data: Optional[MirrorPost]) -> None:
log.debug(data)
- self.model.set_mirror(data)
- await self.configured()
+ if data is None:
+ # TODO If we want the ability to fallback to an offline install, we
+ # probably need to catch NoUsableMirrorError and inform the client
+ # somehow.
+ await self.find_and_elect_candidate_mirror(self.context)
+ await self.configured()
+ return
- async def candidate_POST(self, url: str) -> None:
- log.debug(url)
- self.model.set_mirror(url)
+ if data.candidates is not None:
+ if not data.candidates:
+ raise ValueError("cannot specify an empty list of candidates")
+ uris = data.candidates
+ self.model.primary_candidates = \
+ [self.model.create_primary_candidate(uri) for uri in uris]
+
+ if data.staged is not None:
+ self.model.create_primary_candidate(data.staged).stage()
+
+ if data.elected is not None:
+ self.model.create_primary_candidate(data.elected).elect()
+
+ # NOTE we could also do this unconditionally when generating the
+ # autoinstall configuration. But doing it here gives the user the
+ # ability to use a mirror for one install without it ending up in
+ # the autoinstall config. Is it worth it though?
+ def ensure_elected_in_candidates():
+ if any(map(lambda c: c.uri == data.elected,
+ self.model.primary_candidates)):
+ return
+ self.model.primary_candidates.insert(
+ 0, self.model.primary_elected)
+
+ if data.candidates is None:
+ ensure_elected_in_candidates()
+
+ await self.configured()
async def disable_components_GET(self) -> List[str]:
return sorted(self.model.disabled_components)
@@ -223,7 +346,7 @@ class MirrorController(SubiquityController):
assert False
output = io.StringIO()
self.mirror_check = MirrorCheck(
- uri=self.model.get_mirror(),
+ uri=self.model.primary_staged.uri,
task=asyncio.create_task(self.run_mirror_testing(output)),
output=output)
diff --git a/subiquity/server/controllers/tests/test_mirror.py b/subiquity/server/controllers/tests/test_mirror.py
index 61cab827..1fff2d4f 100644
--- a/subiquity/server/controllers/tests/test_mirror.py
+++ b/subiquity/server/controllers/tests/test_mirror.py
@@ -13,6 +13,7 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see .
+import contextlib
import io
import jsonschema
import unittest
@@ -23,6 +24,7 @@ from subiquity.models.mirror import MirrorModel
from subiquity.server.apt import AptConfigCheckError
from subiquity.server.controllers.mirror import MirrorController
from subiquity.server.controllers.mirror import log as MirrorLogger
+from subiquity.server.controllers.mirror import NoUsableMirrorError
class TestMirrorSchema(unittest.TestCase):
@@ -52,10 +54,24 @@ class TestMirrorController(unittest.IsolatedAsyncioTestCase):
def test_make_autoinstall(self):
self.controller.model = MirrorModel()
+ self.controller.model.primary_candidates[0].elect()
+ config = self.controller.make_autoinstall()
+ self.assertIn("disable_components", config.keys())
+ self.assertIn("mirror-selection", config.keys())
+ self.assertIn("geoip", config.keys())
+ self.assertNotIn("primary", config.keys())
+
+ def test_make_autoinstall_legacy(self):
+ self.controller.model = MirrorModel()
+ self.controller.model.legacy_primary = True
+ self.controller.model.primary_candidates = \
+ self.controller.model.get_default_primary_candidates()
+ self.controller.model.primary_candidates[0].elect()
config = self.controller.make_autoinstall()
self.assertIn("disable_components", config.keys())
self.assertIn("primary", config.keys())
self.assertIn("geoip", config.keys())
+ self.assertNotIn("mirror-selection", config.keys())
async def test_run_mirror_testing(self):
def fake_mirror_check_success(output):
@@ -105,3 +121,91 @@ class TestMirrorController(unittest.IsolatedAsyncioTestCase):
[record.msg for record in debug.records])
self.assertIn("APT output follows",
[record.msg for record in debug.records])
+
+ @mock.patch("subiquity.server.controllers.mirror.asyncio.sleep")
+ async def test_find_and_elect_candidate_mirror(self, mock_sleep):
+ self.controller.app.context.child = contextlib.nullcontext
+ self.controller.app.base_model.network.has_network = True
+ self.controller.model = MirrorModel()
+ self.controller.network_configured_event.set()
+ self.controller.proxy_configured_event.set()
+ self.controller.cc_event.set()
+
+ # Test with no candidate
+ self.controller.model.primary_elected = None
+ self.controller.model.primary_candidates = []
+ with self.assertRaises(NoUsableMirrorError):
+ await self.controller.find_and_elect_candidate_mirror(
+ self.controller.app.context)
+ self.assertIsNone(self.controller.model.primary_elected)
+
+ # Test one succeeding candidate
+ self.controller.model.primary_elected = None
+ self.controller.model.primary_candidates = \
+ [self.controller.model.create_primary_candidate("http://mirror")]
+ with mock.patch.object(self.controller, "try_mirror_checking_once"):
+ await self.controller.find_and_elect_candidate_mirror(
+ self.controller.app.context)
+ self.assertEqual(self.controller.model.primary_elected.uri,
+ "http://mirror")
+
+ # Test one succeeding candidate, on second try
+ self.controller.model.primary_elected = None
+ self.controller.model.primary_candidates = \
+ [self.controller.model.create_primary_candidate("http://mirror")]
+ with mock.patch.object(self.controller, "try_mirror_checking_once",
+ side_effect=(AptConfigCheckError, None)):
+ await self.controller.find_and_elect_candidate_mirror(
+ self.controller.app.context)
+ self.assertEqual(self.controller.model.primary_elected.uri,
+ "http://mirror")
+
+ # Test with a single candidate, failing twice
+ self.controller.model.primary_elected = None
+ self.controller.model.primary_candidates = \
+ [self.controller.model.create_primary_candidate("http://mirror")]
+ with mock.patch.object(self.controller, "try_mirror_checking_once",
+ side_effect=AptConfigCheckError):
+ with self.assertRaises(NoUsableMirrorError):
+ await self.controller.find_and_elect_candidate_mirror(
+ self.controller.app.context)
+ self.assertIsNone(self.controller.model.primary_elected)
+
+ # Test with one candidate failing twice, then one succeeding
+ self.controller.model.primary_elected = None
+ self.controller.model.primary_candidates = [
+ self.controller.model.create_primary_candidate("http://failed"),
+ self.controller.model.create_primary_candidate("http://success"),
+ ]
+ with mock.patch.object(
+ self.controller, "try_mirror_checking_once",
+ side_effect=(AptConfigCheckError, AptConfigCheckError, None)):
+ await self.controller.find_and_elect_candidate_mirror(
+ self.controller.app.context)
+ self.assertEqual(self.controller.model.primary_elected.uri,
+ "http://success")
+
+ # Test with an unresolved country mirror
+ self.controller.model.primary_elected = None
+ self.controller.model.primary_candidates = [
+ self.controller.model.create_primary_candidate(
+ uri=None, country_mirror=True),
+ self.controller.model.create_primary_candidate("http://success"),
+ ]
+ with mock.patch.object(self.controller, "try_mirror_checking_once"):
+ await self.controller.find_and_elect_candidate_mirror(
+ self.controller.app.context)
+ self.assertEqual(self.controller.model.primary_elected.uri,
+ "http://success")
+
+ async def test_find_and_elect_candidate_mirror_no_network(self):
+ self.controller.app.context.child = contextlib.nullcontext
+ self.controller.app.base_model.network.has_network = False
+ self.controller.model = MirrorModel()
+ self.controller.network_configured_event.set()
+ self.controller.proxy_configured_event.set()
+ self.controller.cc_event.set()
+
+ await self.controller.find_and_elect_candidate_mirror(
+ self.controller.app.context)
+ self.assertIsNone(self.controller.model.primary_elected)
diff --git a/subiquity/server/tests/test_apt.py b/subiquity/server/tests/test_apt.py
index e84dc2b2..3e6423ec 100644
--- a/subiquity/server/tests/test_apt.py
+++ b/subiquity/server/tests/test_apt.py
@@ -59,6 +59,7 @@ class TestAptConfigurer(SubiTestCase):
def setUp(self):
self.model = Mock()
self.model.mirror = MirrorModel()
+ self.model.mirror.create_primary_candidate("http://mymirror").elect()
self.model.proxy = ProxyModel()
self.model.locale.selected_language = "en_US.UTF-8"
self.app = make_app(self.model)
@@ -67,7 +68,7 @@ class TestAptConfigurer(SubiTestCase):
self.astart_sym = "subiquity.server.apt.astart_command"
def test_apt_config_noproxy(self):
- config = self.configurer.apt_config()
+ config = self.configurer.apt_config(final=True)
self.assertNotIn("http_proxy", config["apt"])
self.assertNotIn("https_proxy", config["apt"])
@@ -75,7 +76,7 @@ class TestAptConfigurer(SubiTestCase):
proxy = 'http://apt-cacher-ng:3142'
self.model.proxy.proxy = proxy
- config = self.configurer.apt_config()
+ config = self.configurer.apt_config(final=True)
self.assertEqual(proxy, config["apt"]["http_proxy"])
self.assertEqual(proxy, config["apt"]["https_proxy"])
@@ -135,6 +136,8 @@ class TestDRAptConfigurer(SubiTestCase):
def setUp(self):
self.model = Mock()
self.model.mirror = MirrorModel()
+ self.candidate = self.model.mirror.primary_candidates[0]
+ self.candidate.stage()
self.app = make_app(self.model)
self.app.dr_cfg = DRConfig()
self.app.dr_cfg.apt_mirror_check_default_strategy = "failure"
@@ -172,20 +175,20 @@ class TestDRAptConfigurer(SubiTestCase):
async def test_run_apt_config_check_success(self):
output = io.StringIO()
self.app.dr_cfg.apt_mirror_check_default_strategy = "success"
- self.model.mirror.set_mirror("http://default")
+ self.candidate.uri = "http://default"
await self.configurer.run_apt_config_check(output)
async def test_run_apt_config_check_failed(self):
output = io.StringIO()
self.app.dr_cfg.apt_mirror_check_default_strategy = "failure"
- self.model.mirror.set_mirror("http://default")
+ self.candidate.uri = "http://default"
with self.assertRaises(AptConfigCheckError):
await self.configurer.run_apt_config_check(output)
async def test_run_apt_config_check_random(self):
output = io.StringIO()
self.app.dr_cfg.apt_mirror_check_default_strategy = "random"
- self.model.mirror.set_mirror("http://default")
+ self.candidate.uri = "http://default"
with patch("subiquity.server.apt.random.choice",
return_value=self.configurer.apt_config_check_success):
await self.configurer.run_apt_config_check(output)
diff --git a/subiquity/tests/api/test_api.py b/subiquity/tests/api/test_api.py
index 661fb019..3811d5cc 100644
--- a/subiquity/tests/api/test_api.py
+++ b/subiquity/tests/api/test_api.py
@@ -319,7 +319,8 @@ class TestFlow(TestAPI):
source_id='ubuntu-server', search_drivers=True)
await inst.post('/network')
await inst.post('/proxy', '')
- await inst.post('/mirror', 'http://us.archive.ubuntu.com/ubuntu')
+ await inst.post('/mirror',
+ {'elected': 'http://us.archive.ubuntu.com/ubuntu'})
resp = await inst.get('/storage/v2/guided?wait=true')
[reformat] = resp['possible']
diff --git a/subiquity/ui/views/mirror.py b/subiquity/ui/views/mirror.py
index 119fd177..2ed503d6 100644
--- a/subiquity/ui/views/mirror.py
+++ b/subiquity/ui/views/mirror.py
@@ -46,6 +46,7 @@ from subiquitycore.ui.utils import button_pile, rewrap
from subiquitycore.view import BaseView
from subiquity.common.types import (
+ MirrorPost,
MirrorCheckResponse,
MirrorCheckStatus,
)
@@ -173,7 +174,8 @@ class MirrorView(BaseView):
async_helpers.run_bg_task(self._check_url(url))
async def _check_url(self, url, cancel_ongoing=False):
- await self.controller.endpoint.candidate.POST(url)
+ await self.controller.endpoint.POST(
+ MirrorPost(staged=url))
await self.controller.endpoint.check_mirror.start.POST(True)
state = await self.controller.endpoint.check_mirror.progress.GET()
self.update_status(state)