add setting policy for post-install updates via autoinstall (#920)

* Add element updates (non-UI)

This can be controlled with autoinstall
updates: security or all

Also an API for controlling this:
curl --silent --unix-socket .subiquity/socket a/updates ->
"security"
curl -d '"all"' --unix-socket .subiquity/socket a/updates

* Automated tests - log grep for default/none/all states

* Enforce possible values on Updates controller

Route all the various get/set thru 2 common functions.
Validate incoming data.

autoinstall-schema.json

@@ -323,6 +323,13 @@
                 "additionalProperties": false
             }
         },
+        "updates": {
+            "type": "string",
+            "enum": [
+                "security",
+                "all"
+            ]
+        },
         "late-commands": {
             "type": "array",
             "items": {

examples/autoinstall-user-data.yaml

@@ -13,6 +13,7 @@ late-commands:
   - echo a
 keyboard:
   layout: gb
+updates: security
 user-data:
   users:
     - username: ubuntu

examples/autoinstall.yaml

@@ -37,6 +37,7 @@ identity:
   username: ubuntu
   password: '$6$wdAcoXrU039hKYPd$508Qvbe7ObUnxoj15DRCkzC3qO7edjH0VV7BPNRDYK4QR8ofJaEEF2heacn0QgD.f8pO8SNp83XNdWG6tocBM1'
   hostname: ubuntu
+updates: all
 storage:
   config:
   - {type: disk, ptable: gpt, path: /dev/vdb, wipe: superblock, preserve: false, grub_device: true, id: disk-1}

scripts/runtests.sh

@@ -36,6 +36,7 @@ for answers in examples/answers*.yaml; do
     # The --foreground is important to avoid subiquity getting SIGTTOU-ed.
     timeout --foreground 60 sh -c "LANG=C.UTF-8 python3 -m subiquity.cmd.tui --answers $answers --dry-run --snaps-from-examples --machine-config $config" < $tty
     validate
+    grep -q 'finish: subiquity/Install/install/run_unattended_upgrades: SUCCESS: downloading and installing security updates' .subiquity/subiquity-server-debug.log
 done
 
 clean
@@ -52,11 +53,14 @@ grep -q 'finish: subiquity/Install/install/postinstall/install_package1: SUCCESS
 grep -q 'finish: subiquity/Install/install/postinstall/install_package2: SUCCESS: installing package2' \
      .subiquity/subiquity-server-debug.log
 grep -q 'switching subiquity to edge' .subiquity/subiquity-server-debug.log
+grep -q 'finish: subiquity/Install/install/run_unattended_upgrades: SUCCESS: downloading and installing all updates' \
+    .subiquity/subiquity-server-debug.log
 
 clean
 timeout --foreground 60 sh -c "LANG=C.UTF-8 python3 -m subiquity.cmd.tui --autoinstall examples/autoinstall-user-data.yaml \
                                --dry-run --machine-config examples/simple.json --kernel-cmdline 'autoinstall'"
 validate
+grep -q 'finish: subiquity/Install/install/run_unattended_upgrades: SUCCESS: downloading and installing security updates' .subiquity/subiquity-server-debug.log
 
 python3 -m subiquity.cmd.schema > "$testschema"
 diff -u "autoinstall-schema.json" "$testschema"

subiquity/common/apidef.py

@@ -47,11 +47,12 @@ from subiquity.common.types import (
 @api
 class API:
     """The API offered by the subiquity installer process."""
-    locale = simple_endpoint(str)
-    proxy = simple_endpoint(str)
-    mirror = simple_endpoint(str)
     identity = simple_endpoint(IdentityData)
+    locale = simple_endpoint(str)
+    mirror = simple_endpoint(str)
+    proxy = simple_endpoint(str)
     ssh = simple_endpoint(SSHData)
+    updates = simple_endpoint(str)
 
     class meta:
         class status:

subiquity/models/subiquity.py

@@ -29,14 +29,15 @@ from subiquitycore.file_util import write_file
 from subiquitycore.utils import run_command
 
 from .filesystem import FilesystemModel
+from .identity import IdentityModel
 from .keyboard import KeyboardModel
 from .locale import LocaleModel
-from .proxy import ProxyModel
 from .mirror import MirrorModel
 from .network import NetworkModel
+from .proxy import ProxyModel
 from .snaplist import SnapListModel
 from .ssh import SSHModel
-from .identity import IdentityModel
+from .updates import UpdatesModel
 
 
 log = logging.getLogger('subiquity.models.subiquity')
@@ -117,6 +118,7 @@ class SubiquityModel:
         self.proxy = ProxyModel()
         self.snaplist = SnapListModel()
         self.ssh = SSHModel()
+        self.updates = UpdatesModel()
         self.userdata = {}
 
         self.confirmation = asyncio.Event()

subiquity/models/updates.py

@@ -0,0 +1,27 @@
+# Copyright 2021 Canonical, Ltd.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+
+log = logging.getLogger('subiquity.models.updates')
+
+
+class UpdatesModel(object):
+    """ Model representing updates selection"""
+
+    updates = 'security'
+
+    def __repr__(self):
+        return "<Updates: {}>".format(self.updates)

subiquity/server/controllers/__init__.py

@@ -29,6 +29,7 @@ from .refresh import RefreshController
 from .reporting import ReportingController
 from .snaplist import SnapListController
 from .ssh import SSHController
+from .updates import UpdatesController
 from .userdata import UserdataController
 from .zdev import ZdevController
 
@@ -51,6 +52,7 @@ __all__ = [
     'ReportingController',
     'SnapListController',
     'SSHController',
+    'UpdatesController',
     'UserdataController',
     'ZdevController',
 ]

subiquity/server/controllers/install.py

@@ -14,13 +14,13 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import asyncio
+import contextlib
 import datetime
 import logging
 import os
 import re
 import shutil
 import sys
-import tempfile
 
 from curtin.commands.install import (
     ERROR_TARFILE,
@@ -135,7 +135,7 @@ class InstallController(SubiquityController):
 
     def _write_config(self, path, config):
         with open(path, 'w') as conf:
-            datestr = '# Autogenerated by SUbiquity: {} UTC\n'.format(
+            datestr = '# Autogenerated by Subiquity: {} UTC\n'.format(
                 str(datetime.datetime.utcnow()))
             conf.write(datestr)
             conf.write(yaml.dump(config))
@@ -234,7 +234,9 @@ class InstallController(SubiquityController):
 
             if self.model.network.has_network:
                 self.app.update_state(ApplicationState.UU_RUNNING)
-                await self.run_unattended_upgrades(context=context)
+                policy = self.model.updates.updates
+                await self.run_unattended_upgrades(context=context,
+                                                   policy=policy)
 
             self.app.update_state(ApplicationState.DONE)
         except Exception:
@@ -307,31 +309,31 @@ class InstallController(SubiquityController):
         for cmd in cmds:
             await arun_command(self.logged_command(cmd), check=True)
 
-    @with_context(description="downloading and installing security updates")
-    async def run_unattended_upgrades(self, context):
-        target_tmp = os.path.join(self.model.target, "tmp")
-        os.makedirs(target_tmp, exist_ok=True)
-        apt_conf = tempfile.NamedTemporaryFile(
-            dir=target_tmp, delete=False, mode='w')
-        apt_conf.write(uu_apt_conf)
-        apt_conf.close()
-        env = os.environ.copy()
-        env["APT_CONFIG"] = apt_conf.name[len(self.model.target):]
-        self.unattended_upgrades_ctx = context
+    @with_context(description="downloading and installing {policy} updates")
+    async def run_unattended_upgrades(self, context, policy):
         if self.app.opts.dry_run:
-            self.unattended_upgrades_proc = await astart_command(
-                self.logged_command(
-                    ["sleep", str(5/self.app.scale_factor)]), env=env)
+            command = ["sleep", str(5/self.app.scale_factor)]
+            aptdir = os.path.join(self.model.target, "tmp")
         else:
+            command = [sys.executable, "-m", "curtin", "in-target",
+                       "-t", "/target", "--", "unattended-upgrades", "-v"]
+            aptdir = os.path.join(self.model.target, "etc/apt/apt.conf.d")
+        os.makedirs(aptdir, exist_ok=True)
+        apt_conf_contents = uu_apt_conf
+        if policy == 'all':
+            apt_conf_contents += uu_apt_conf_update_all
+        else:
+            apt_conf_contents += uu_apt_conf_update_security
+        fname = 'zzzz-temp-installer-unattended-upgrade'
+        with external_temp_file(aptdir, fname) as apt_conf:
+            apt_conf.write(apt_conf_contents)
+            apt_conf.close()
+            self.unattended_upgrades_ctx = context
             self.unattended_upgrades_proc = await astart_command(
-                self.logged_command([
-                    sys.executable, "-m", "curtin", "in-target", "-t",
-                    "/target", "--", "unattended-upgrades", "-v",
-                    ]), env=env)
-        await self.unattended_upgrades_proc.communicate()
-        self.unattended_upgrades_proc = None
-        self.unattended_upgrades_ctx = None
-        os.remove(apt_conf.name)
+                self.logged_command(command))
+            await self.unattended_upgrades_proc.communicate()
+            self.unattended_upgrades_proc = None
+            self.unattended_upgrades_ctx = None
 
     async def stop_unattended_upgrades(self):
         with self.unattended_upgrades_ctx.parent.child(
@@ -349,9 +351,43 @@ class InstallController(SubiquityController):
                     ]), check=True)
 
 
-uu_apt_conf = """\
+@contextlib.contextmanager
+def external_temp_file(dir, fname):
+    path = os.path.join(dir, fname)
+    fid = open(path, 'wb')
+    try:
+        yield fid
+    finally:
+        fid.close()
+        os.remove(path)
+
+
+uu_apt_conf = b"""\
 # Config for the unattended-upgrades run to avoid failing on battery power or
 # a metered connection.
 Unattended-Upgrade::OnlyOnACPower "false";
 Unattended-Upgrade::Skip-Updates-On-Metered-Connections "true";
 """
+
+uu_apt_conf_update_security = b"""\
+# A copy of the current default unattended-upgrades config to grab
+# security.
+Unattended-Upgrade::Allowed-Origins {
+        "${distro_id}:${distro_codename}";
+        "${distro_id}:${distro_codename}-security";
+        "${distro_id}ESMApps:${distro_codename}-apps-security";
+        "${distro_id}ESM:${distro_codename}-infra-security";
+};
+"""
+
+uu_apt_conf_update_all = b"""\
+# A modified version of the unattended-upgrades default Allowed-Origins
+# to include updates in the permitted origins.
+Unattended-Upgrade::Allowed-Origins {
+        "${distro_id}:${distro_codename}";
+        "${distro_id}:${distro_codename}-updates";
+        "${distro_id}:${distro_codename}-security";
+        "${distro_id}ESMApps:${distro_codename}-apps-security";
+        "${distro_id}ESM:${distro_codename}-infra-security";
+};
+"""

subiquity/server/controllers/updates.py

@@ -0,0 +1,56 @@
+# Copyright 2018 Canonical, Ltd.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+
+from subiquity.common.apidef import API
+from subiquity.server.controller import SubiquityController
+
+
+log = logging.getLogger('subiquity.server.controllers.updates')
+
+
+class UpdatesController(SubiquityController):
+
+    endpoint = API.updates
+
+    possible = ['security', 'all']
+
+    autoinstall_key = model_name = "updates"
+    autoinstall_schema = {
+        'type': 'string',
+        'enum': possible,
+    }
+    autoinstall_default = 'security'
+
+    def load_autoinstall_data(self, data):
+        self.deserialize(data)
+
+    def make_autoinstall(self):
+        return self.serialize()
+
+    def serialize(self):
+        return self.model.updates
+
+    def deserialize(self, data):
+        if data not in self.possible:
+            raise ValueError
+        self.model.updates = data
+
+    async def GET(self) -> str:
+        return self.serialize()
+
+    async def POST(self, data: str):
+        self.deserialize(data)

subiquity/server/server.py

@@ -188,6 +188,7 @@ class SubiquityServer(Application):
         "SSH",
         "SnapList",
         "Install",
+        "Updates",
         "Late",
         "Reboot",
         ]