From a531ade6c3602036ac72dcdb973e24d421e72837 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 14 Jul 2022 21:48:47 -0600 Subject: [PATCH 1/3] file_util: generate_timestamped_header function to any subiquity written files --- subiquitycore/file_util.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/subiquitycore/file_util.py b/subiquitycore/file_util.py index 0c5fe9e4..23524d93 100644 --- a/subiquitycore/file_util.py +++ b/subiquitycore/file_util.py @@ -69,10 +69,14 @@ def write_file(filename, content, **kwargs): tf.write(content) +def generate_timestamped_header() -> str: + now = datetime.datetime.utcnow() + return f'# Autogenerated by Subiquity: {now} UTC\n' + + def generate_config_yaml(filename, content, **kwargs): with open_perms(filename, **kwargs) as tf: - now = datetime.datetime.utcnow() - tf.write(f'# Autogenerated by Subiquity: {now} UTC\n') + tf.write(generate_timestamped_header()) tf.write(yaml.dump(content)) From ddaf48b3348b3be7092bb1d25c08dedb06f97179 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 14 Jul 2022 21:35:40 -0600 Subject: [PATCH 2/3] models: subiquitycore.network.rendered_config_paths to list generated files --- subiquitycore/models/network.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/subiquitycore/models/network.py b/subiquitycore/models/network.py index 7f084a90..4764a1ba 100644 --- a/subiquitycore/models/network.py +++ b/subiquitycore/models/network.py @@ -523,6 +523,13 @@ class NetworkModel(object): return config + def rendered_config_paths(self): + """Return a list of file paths rendered by this model.""" + return [ + '/' + write_file['path'] + for write_file in self.render().get('write_files').values() + ] + def render(self): return { 'write_files': { From 17408993f6bb021f5d0825bb8d64d895fab3bcd3 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 12 Jul 2022 04:04:53 +0000 Subject: [PATCH 3/3] cloud_init_files: render a clean script to /etc/cloud/clean.d To support golden image creation from the live installer, provide cloud-init with /etc/cloud/clean.d/99-installer script. This script will only be invoked when sudo cloud-init clean is run, which is typically only used when trying to create customized golden images. --- subiquity/models/subiquity.py | 33 ++++++++++++++- subiquity/models/tests/test_subiquity.py | 52 +++++++++++++++++++++++- 2 files changed, 83 insertions(+), 2 deletions(-) diff --git a/subiquity/models/subiquity.py b/subiquity/models/subiquity.py index 275d18b1..584e8812 100644 --- a/subiquity/models/subiquity.py +++ b/subiquity/models/subiquity.py @@ -16,6 +16,7 @@ import asyncio from collections import OrderedDict import functools +import json import logging import os from typing import Set @@ -25,7 +26,10 @@ import yaml from curtin.commands.install import CONFIG_BUILTIN from curtin.config import merge_config -from subiquitycore.file_util import write_file +from subiquitycore.file_util import ( + generate_timestamped_header, + write_file, +) from subiquity.common.resources import get_users_and_groups from subiquity.server.types import InstallerChannels @@ -93,6 +97,20 @@ ff02::1 ip6-allnodes ff02::2 ip6-allrouters """ +CLOUDINIT_CLEAN_FILE_TMPL = """\ +#!/usr/bin/env python3 +# Remove live-installer config artifacts when running: sudo cloud-init clean +{header} + +import os + +for cfg_file in {cfg_files}: + try: + os.remove(cfg_file) + except FileNotFoundError: + pass +""" + class ModelNames: def __init__(self, default_names, **per_variant_names): @@ -308,12 +326,25 @@ class SubiquityModel: ('etc/cloud/cloud.cfg.d/99-installer.cfg', config, 0o600), ('etc/cloud/ds-identify.cfg', 'policy: enabled\n', 0o644), ] + # Add cloud-init clean hooks to support golden-image creation. + cfg_files = ["/" + path for (path, _content, _cmode) in files] + cfg_files.extend(self.network.rendered_config_paths()) + if self.identity.hostname is not None: hostname = self.identity.hostname.strip() files.extend([ ('etc/hostname', hostname + "\n", 0o644), ('etc/hosts', HOSTS_CONTENT.format(hostname=hostname), 0o644), ]) + + files.append(( + 'etc/cloud/clean.d/99-installer', + CLOUDINIT_CLEAN_FILE_TMPL.format( + header=generate_timestamped_header(), + cfg_files=json.dumps(sorted(cfg_files)) + ), + 0o755 + )) return files def configure_cloud_init(self): diff --git a/subiquity/models/tests/test_subiquity.py b/subiquity/models/tests/test_subiquity.py index 065e243e..d5221375 100644 --- a/subiquity/models/tests/test_subiquity.py +++ b/subiquity/models/tests/test_subiquity.py @@ -13,15 +13,23 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . + import fnmatch +import json import unittest from unittest import mock +import re import yaml from subiquitycore.pubsub import MessageHub from subiquity.common.types import IdentityData -from subiquity.models.subiquity import ModelNames, SubiquityModel +from subiquity.models.subiquity import ( + CLOUDINIT_CLEAN_FILE_TMPL, + HOSTS_CONTENT, + ModelNames, + SubiquityModel, +) from subiquity.server.server import ( INSTALL_MODEL_NAMES, POSTINSTALL_MODEL_NAMES, @@ -227,3 +235,45 @@ class TestSubiquityModel(unittest.IsolatedAsyncioTestCase): cloud_init_config = model._cloud_init_config() self.assertEqual(len(cloud_init_config['users']), 1) self.assertEqual(cloud_init_config['users'][0]['name'], 'user2') + + @mock.patch('subiquitycore.file_util.datetime.datetime') + def test_cloud_init_files_emits_datasource_config_and_clean_script( + self, datetime + ): + datetime.utcnow.return_value = "2004-03-05 ..." + main_user = IdentityData( + username='mainuser', + crypted_password='sample_pass', + hostname='somehost') + + model = self.make_model() + model.identity.add_user(main_user) + model.userdata = {} + expected_files = { + 'etc/cloud/cloud.cfg.d/subiquity-disable-cloudinit-networking.cfg': + 'network: {config: disabled}\n', + 'etc/cloud/cloud.cfg.d/99-installer.cfg': re.compile('datasource:\n None:\n metadata:\n instance-id: .*\n userdata_raw: "#cloud-config\\\\ngrowpart:\\\\n mode: \\\'off\\\'\\\\npreserve_hostname: true\\\\n\\\\\n'), # noqa + 'etc/hostname': 'somehost\n', + 'etc/cloud/ds-identify.cfg': 'policy: enabled\n', + 'etc/hosts': HOSTS_CONTENT.format(hostname='somehost'), + } + + # Avoid removing /etc/hosts and /etc/hostname in cloud-init clean + cfg_files = [ + "/" + key for key in expected_files.keys() if "host" not in key + ] + cfg_files.extend( # Obtained from NetworkModel.render + ["/etc/netplan/00-installer-config.yaml"] + ) + + expected_files['etc/cloud/clean.d/99-installer'] = ( + CLOUDINIT_CLEAN_FILE_TMPL.format( + header="# Autogenerated by Subiquity: 2004-03-05 ... UTC\n", + cfg_files=json.dumps(sorted(cfg_files)) + ) + ) + for (cpath, content, perms) in model._cloud_init_files(): + if isinstance(expected_files[cpath], re.Pattern): + self.assertIsNotNone(expected_files[cpath].match(content)) + else: + self.assertEqual(expected_files[cpath], content)