StealthOS/subiquity
StealthOS
/
subiquity
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

kvm-test: cleanups 

* better command execution
* various build arguments imply --build, so one can do just '-q' where
  one would have to previously '--build -q'
This commit is contained in:
Dan Bungert 2021-12-01 16:48:12 -07:00
parent 784a46845f
commit 06ac3f92b5
1 changed files with 84 additions and 151 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

233
scripts/kvm-test.py
View File

@ -2,7 +2,7 @@
'''kvm-test - boot a kvm with a test iso, possibly building that test iso first '''kvm-test - boot a kvm with a test iso, possibly building that test iso first
kvm-test --build -q --install -o --boot kvm-test -q --install -o --boot
slimy build, install, overwrite existing image if it exists, slimy build, install, overwrite existing image if it exists,
and boot the result after install and boot the result after install
@ -16,19 +16,21 @@ import crypt
import os import os
import random import random
import socket import socket
import subprocess
import sys import sys
import tempfile import tempfile
import yaml import yaml
cfg = ''' cfg = '''
default_mem: '8G'
iso: iso:
basedir: /srv/iso basedir: /srv/iso
release: release:
edge: jammy/subiquity-edge/jammy-live-server-subiquity-edge-amd64.iso edge: jammy/subiquity-edge/jammy-live-server-subiquity-edge-amd64.iso
canary: jammy/jammy-desktop-canary-amd64.iso canary: jammy/jammy-desktop-canary-amd64.iso
jammy: jammy/jammy-live-server-amd64.iso jammy: jammy/jammy-live-server-amd64.iso
desktop: impish/jammy-desktop-amd64.iso desktop: jammy/jammy-desktop-amd64.iso
impish: impish/ubuntu-21.10-live-server-amd64.iso impish: impish/ubuntu-21.10-live-server-amd64.iso
hirsute: hirsute/ubuntu-21.04-live-server-amd64.iso hirsute: hirsute/ubuntu-21.04-live-server-amd64.iso
groovy: groovy/ubuntu-20.10-live-server-amd64.iso groovy: groovy/ubuntu-20.10-live-server-amd64.iso
@ -37,9 +39,6 @@ iso:
default: edge default: edge
''' '''
sys_memory = '8G'
def salted_crypt(plaintext_password): def salted_crypt(plaintext_password):
# match subiquity documentation # match subiquity documentation
salt = '$6$exDY1mhS4KUYCE/2' salt = '$6$exDY1mhS4KUYCE/2'
@ -51,6 +50,7 @@ class Context:
self.config = self.load_config() self.config = self.load_config()
self.args = args self.args = args
self.release = args.release self.release = args.release
self.default_mem = self.config('default_mem', '8G')
if not self.release: if not self.release:
self.release = self.config["iso"]["default"] self.release = self.config["iso"]["default"]
iso = self.config["iso"] iso = self.config["iso"]
@ -60,10 +60,8 @@ class Context:
except KeyError: except KeyError:
pass pass
self.curdir = os.getcwd() self.curdir = os.getcwd()
# self.iso = f'{self.curdir}/{self.release}-test.iso'
self.iso = f'/tmp/kvm-test/{self.release}-test.iso' self.iso = f'/tmp/kvm-test/{self.release}-test.iso'
self.hostname = f'{self.release}-test' self.hostname = f'{self.release}-test'
# self.target = f'{self.curdir}/{self.hostname}.img'
self.target = f'/tmp/kvm-test/{self.hostname}.img' self.target = f'/tmp/kvm-test/{self.hostname}.img'
self.password = salted_crypt('ubuntu') self.password = salted_crypt('ubuntu')
self.cloudconfig = f'''\ self.cloudconfig = f'''\
@ -80,9 +78,6 @@ autoinstall:
password: "{self.password}" password: "{self.password}"
username: ubuntu username: ubuntu
''' '''
# refresh-installer:
# update: yes
# channel: candidate
def merge(self, a, b): def merge(self, a, b):
'''Take a pair of dictionaries, and provide the merged result. '''Take a pair of dictionaries, and provide the merged result.
@ -137,8 +132,8 @@ parser.add_argument('--basesnap', default=None, action='store',
parser.add_argument('--snap', default=None, action='store', parser.add_argument('--snap', default=None, action='store',
help='inject this snap into the ISO') help='inject this snap into the ISO')
parser.add_argument('-B', '--bios', action='store_true', default=False, parser.add_argument('-B', '--bios', action='store_true', default=False,
help='boot in BIOS mode') help='boot in BIOS mode (default mode is UEFI)')
parser.add_argument('-c', '--channel', default=False, action='store', parser.add_argument('-c', '--channel', action='store',
help='build iso with snap from channel') help='build iso with snap from channel')
parser.add_argument('-d', '--disksize', default='12G', action='store', parser.add_argument('-d', '--disksize', default='12G', action='store',
help='size of disk to create (12G default)') help='size of disk to create (12G default)')
@ -157,14 +152,15 @@ parser.add_argument('-s', '--serial', default=False, action='store_true',
help='attach to serial console') help='attach to serial console')
parser.add_argument('-S', '--sound', default=False, action='store_true', parser.add_argument('-S', '--sound', default=False, action='store_true',
help='enable sound') help='enable sound')
parser.add_argument('-t', '--this', action='store', parser.add_argument('--iso', action='store', help='use this iso')
help='use this iso')
parser.add_argument('-u', '--update', action='store', parser.add_argument('-u', '--update', action='store',
help='subiquity-channel argument') help='subiquity-channel argument')
parser.add_argument('-m', '--memory', action='store',
help='memory for VM')
parser.add_argument('--save', action='store_true', parser.add_argument('--save', action='store_true',
help='preserve built snap') help='preserve built snap')
parser.add_argument('--reuse', action='store_true', parser.add_argument('--reuse', action='store_true',
help='reuse previously saved snap') help='reuse previously saved snap. Implies --save')
parser.add_argument('--build', default=False, action='store_true', parser.add_argument('--build', default=False, action='store_true',
help='build test iso') help='build test iso')
parser.add_argument('--install', default=False, action='store_true', parser.add_argument('--install', default=False, action='store_true',
@ -174,44 +170,31 @@ parser.add_argument('--boot', default=False, action='store_true',
help='boot test image') help='boot test image')
def waitstatus_to_exitcode(waitstatus): def parse_args():
'''If the process exited normally (if WIFEXITED(status) is true), return ctx = Context(parser.parse_args())
the process exit status (return WEXITSTATUS(status)): result greater if ctx.args.quick or ctx.args.basesnap or ctx.args.snap \
than or equal to 0. or ctx.args.channel or ctx.args.reuse:
ctx.args.build = True
If the process was terminated by a signal (if WIFSIGNALED(status) is if ctx.args.reuse:
true), return -signum where signum is the number of the signal that ctx.args.save = True
caused the process to terminate (return -WTERMSIG(status)): result less return ctx
than 0.
Otherwise, raise a ValueError.'''
# This function is for python 3.9 compat
if 'waitstatus_to_exitcode' in dir(os):
return os.waitstatus_to_exitcode(waitstatus)
if os.WIFEXITED(waitstatus):
return os.WEXITSTATUS(waitstatus)
if os.WIFSIGNALED(waitstatus):
return -os.WTERMSIG(waitstatus)
raise ValueError
class SubProcessFailure(Exception): def run(cmd):
pass if isinstance(cmd, str):
cmd_str = cmd
cmd_array = cmd.split(' ')
def run(cmds): else:
for cmd in [line.strip() for line in cmds.splitlines()]: cmd_str = ' '.join(cmd)
if len(cmd) < 1: cmd_array = cmd
continue
# semi-simulate "bash -x" # semi-simulate "bash -x"
print(f'+ {cmd}') print(f'+ {cmd_str}')
# FIXME subprocess check subprocess.run(cmd_array, check=True)
ec = waitstatus_to_exitcode(os.system(cmd))
if ec != 0:
raise SubProcessFailure(f'command [{cmd}] returned [{ec}]') def assert_exists(path):
if not os.path.exists(path):
raise Exception('Expected file {path} not found')
@contextlib.contextmanager @contextlib.contextmanager
@ -236,16 +219,17 @@ def mounter(src, dest):
run(f'sudo umount {dest}') run(f'sudo umount {dest}')
def livefs_edit(ctx, *args):
run(f'sudo PYTHONPATH=$LIVEFS_EDITOR python3 -m livefs_edit \
{ctx.baseiso} {ctx.iso} {*args}')
def build(ctx): def build(ctx):
run('sudo -v') os.remove(ctx.iso)
run(f'rm -f {ctx.iso}')
project = os.path.basename(os.getcwd()) project = os.path.basename(os.getcwd())
snap_manager = noop if ctx.args.save else delete_later snap_manager = noop if ctx.args.save else delete_later
# FIXME generalize to if project == 'subiquity':
# PYTHONPATH=$LIVEFS_EDITOR python3 -m livefs_edit $OLD_ISO $NEW_ISO
# --inject-snap $SUBIQUITY_SNAP_PATH
if project.startswith('subiquity'):
if ctx.args.quick: if ctx.args.quick:
run(f'sudo ./scripts/quick-test-this-branch.sh {ctx.baseiso} \ run(f'sudo ./scripts/quick-test-this-branch.sh {ctx.baseiso} \
{ctx.iso}') {ctx.iso}')
@ -259,38 +243,27 @@ def build(ctx):
run(f'sudo ./scripts/inject-subiquity-snap.sh {ctx.baseiso} \ run(f'sudo ./scripts/inject-subiquity-snap.sh {ctx.baseiso} \
{ctx.args.snap} {ctx.iso}') {ctx.args.snap} {ctx.iso}')
elif ctx.args.channel: elif ctx.args.channel:
run(f'sudo PYTHONPATH=$LIVEFS_EDITOR python3 -m livefs_edit \ livefs_edit(ctx, '--add-snap-from-store', 'core20', 'stable',
{ctx.baseiso} {ctx.iso} \ '--add-snap-from-store', 'subiquity',
--add-snap-from-store subiquity {ctx.args.channel}') ctx.args.channel)
else: else:
with snap_manager('subiquity_test.snap') as snap: with snap_manager('subiquity_test.snap') as snap:
if not ctx.args.reuse: if not ctx.args.reuse:
run(f''' run('snapcraft clean --use-lxd')
snapcraft clean --use-lxd run(f'snapcraft snap --use-lxd --output {snap}')
snapcraft snap --use-lxd --output {snap} assert_exists(snap)
''') livefs_edit(ctx, '--add-snap-from-store', 'core20', 'stable',
run(f''' '--inject-snap', snap)
test -f {snap}
sudo PYTHONPATH=$LIVEFS_EDITOR python3 -m livefs_edit \
{ctx.baseiso} {ctx.iso} \
--add-snap-from-store core20 stable \
--inject-snap {snap}
''')
# sudo ./scripts/inject-subiquity-snap.sh {ctx.baseiso} \
# {snap} {ctx.iso}
elif project == 'ubuntu-desktop-installer': elif project == 'ubuntu-desktop-installer':
with snap_manager('udi_test.snap') as snap: with snap_manager('udi_test.snap') as snap:
run(f''' run('snapcraft clean --use-lxd')
snapcraft clean --use-lxd run(f'snapcraft snap --use-lxd --output {snap}')
snapcraft snap --use-lxd --output {snap} assert_exists(snap)
test -f {snap} run(f'sudo ./scripts/inject-snap {ctx.baseiso} {ctx.iso} {snap}')
sudo ./scripts/inject-snap {ctx.baseiso} \
{ctx.iso} {snap}
''')
else: else:
raise Exception(f'do not know how to build {project}') raise Exception(f'do not know how to build {project}')
run(f'test -f {ctx.iso}') assert_exists(ctx.iso)
def write(dest, data): def write(dest, data):
@ -315,15 +288,12 @@ def drive(path, format='qcow2'):
kwargs = [] kwargs = []
serial = None serial = None
cparam = 'writethrough' cparam = 'writethrough'
# if cache == False: cparam = 'none' kwargs.append(f'file={path}')
# serial doesn't work.. kwargs.append(f'format={format}')
# serial = str(int(random.random() * 100000000)).zfill(8) kwargs.append(f'cache={cparam}')
kwargs += [f'file={path}'] kwargs.append('if=virtio')
kwargs += [f'format={format}']
kwargs += [f'cache={cparam}']
kwargs += ['if=virtio']
if serial: if serial:
kwargs += [f'serial={serial}'] kwargs.append(f'serial={serial}')
return '-drive ' + ','.join(kwargs) return '-drive ' + ','.join(kwargs)
@ -346,15 +316,15 @@ class PortFinder:
def nets(ctx): def nets(ctx):
ports = PortFinder() ports = PortFinder()
ret = []
if ctx.args.nets > 0: if ctx.args.nets > 0:
ret = []
for _ in range(ctx.args.nets): for _ in range(ctx.args.nets):
port = ports.get() port = ports.get()
ret += ['-nic', ret.extend(('-nic',
'user,model=virtio-net-pci,' + 'user,model=virtio-net-pci,' +
f'hostfwd=tcp::{port}-:22'] # ,restrict=on'] f'hostfwd=tcp::{port}-:22'))
else: else:
ret += ['-nic', 'none'] ret = ['-nic', 'none']
return ret return ret
@ -362,12 +332,12 @@ def bios(ctx):
ret = [] ret = []
# https://help.ubuntu.com/community/UEFI # https://help.ubuntu.com/community/UEFI
if not ctx.args.bios: if not ctx.args.bios:
ret += ['-bios', '/usr/share/qemu/OVMF.fd'] ret = ['-bios', '/usr/share/qemu/OVMF.fd']
return ret return ret
def memory(ctx): def memory(ctx):
return ['-m', str(sys_memory)] return ['-m', ctx.args.memory or ctx.default_mem]
def kvm_common(ctx): def kvm_common(ctx):
@ -381,35 +351,6 @@ def kvm_common(ctx):
return ret return ret
def grub_get_extra_args(mntdir):
# The inject-snap process of livefs-edit adds a new layer squash
# that must be in place, or the injected snap isn't there.
# We don't want to hardcode the current value, because that layer
# isn't there if a base iso is being used.
# Parse the args out of grub.cfg and include them with ours.
cfgpath = f'{mntdir}/boot/grub/grub.cfg'
ret = []
try:
with open(cfgpath, 'r') as fp:
for line in fp.readlines():
chunks = line.strip().split('\t')
# ['linux', '/casper/vmlinuz ---']
# layerfs-path=a.b.c.d.e.squashfs
if not chunks or not chunks[0] == 'linux':
continue
subchunks = chunks[1].split(' ')
if not subchunks or not subchunks[0] == '/casper/vmlinuz':
continue
for sc in subchunks[1:]:
if sc != '---':
ret.append(sc)
break
# breakpoint()
except FileNotFoundError:
pass
return ret
def get_initrd(mntdir): def get_initrd(mntdir):
for initrd in ('initrd', 'initrd.lz', 'initrd.lz4'): for initrd in ('initrd', 'initrd.lz', 'initrd.lz4'):
path = f'{mntdir}/casper/{initrd}' path = f'{mntdir}/casper/{initrd}'
@ -422,8 +363,8 @@ def install(ctx):
if os.path.exists(ctx.target): if os.path.exists(ctx.target):
if ctx.args.overwrite: if ctx.args.overwrite:
os.remove(ctx.target) os.remove(ctx.target)
else:
run('sudo -v') raise Exception('refusing to overwrite existing image')
with tempfile.TemporaryDirectory() as tempdir: with tempfile.TemporaryDirectory() as tempdir:
mntdir = f'{tempdir}/mnt' mntdir = f'{tempdir}/mnt'
@ -439,40 +380,36 @@ def install(ctx):
else: else:
iso = ctx.iso iso = ctx.iso
kvm += ['-cdrom', iso] kvm.extend(('-cdrom', iso))
if ctx.args.serial: if ctx.args.serial:
kvm += ['-nographic'] kvm.append('-nographic')
appends += ['console=ttyS0'] appends.append('console=ttyS0')
if ctx.args.autoinstall or ctx.args.autoinstall_file: if ctx.args.autoinstall or ctx.args.autoinstall_file:
if ctx.args.autoinstall_file: if ctx.args.autoinstall_file:
ctx.cloudconfig = ctx.args.autoinstall_file.read() ctx.cloudconfig = ctx.args.autoinstall_file.read()
kvm += [drive(create_seed(ctx.cloudconfig, tempdir), 'raw')] kvm.append(drive(create_seed(ctx.cloudconfig, tempdir), 'raw'))
appends += ['autoinstall'] appends.append('autoinstall')
if ctx.args.update: if ctx.args.update:
appends += ['subiquity-channel=candidate'] appends.append('subiquity-channel=' + ctx.args.update)
kvm += [drive(ctx.target)] kvm.append(drive(ctx.target))
if not os.path.exists(ctx.target) or ctx.args.overwrite: if not os.path.exists(ctx.target) or ctx.args.overwrite:
run(f'qemu-img create -f qcow2 {ctx.target} {ctx.args.disksize}') run(f'qemu-img create -f qcow2 {ctx.target} {ctx.args.disksize}')
# drive2 = f'{ctx.curdir}/drive2.img'
# appends += ['subiquity-channel=edge']
with mounter(iso, mntdir):
if len(appends) > 0: if len(appends) > 0:
appends += grub_get_extra_args(mntdir) with mounter(iso, mntdir):
# if we're passing kernel args, we need to manually specify # if we're passing kernel args, we need to manually specify
# kernel / initrd # kernel / initrd
kvm += ['-kernel', f'{mntdir}/casper/vmlinuz'] kvm.extend(('-kernel', f'{mntdir}/casper/vmlinuz'))
kvm += ['-initrd', get_initrd(mntdir)] kvm.extend(('-initrd', get_initrd(mntdir)))
toappend = ' '.join(appends) toappend = ' '.join(appends)
kvm += ['-append', f'"{toappend}"'] kvm.extend(('-append', f'"{toappend}"'))
run(kvm)
run(' '.join(kvm)) else:
run(kvm)
def boot(ctx): def boot(ctx):
@ -481,8 +418,8 @@ def boot(ctx):
target = ctx.args.img target = ctx.args.img
kvm = kvm_common(ctx) kvm = kvm_common(ctx)
kvm += [drive(target)] kvm.append(target)
run(' '.join(kvm)) run(kvm)
def help(ctx): def help(ctx):
@ -490,12 +427,8 @@ def help(ctx):
sys.exit(1) sys.exit(1)
def cloud(ctx):
print(ctx.cloudconfig)
try: try:
ctx = Context(parser.parse_args()) ctx = parse_args()
except TypeError: except TypeError:
help() help()