From 2b06bdc1dafe8cb6c78eb60954e874714d2dcc87 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Mon, 6 Jul 2015 10:55:49 -0500 Subject: [PATCH] Call curtin to complete install - Use updated probert *Info class for accessing object data - Add --dry-run parameter to prevent invoking curtin when testing - Add new make target to run ui-view by default with --dry-run - Use a template to write out curtin config file for installation - Use virtio disks and multiple nics to test probing. Signed-off-by: Ryan Harper --- Makefile | 8 ++- bin/curtin_wrap.sh | 15 ++-- bin/subiquity | 3 + installer/geninstaller | 23 +++---- .../resources/user-data/installer-user-data | 1 + installer/runinstaller | 10 +-- subiquity/controllers/filesystem.py | 27 +++++++- subiquity/controllers/installpath.py | 3 +- subiquity/curtin.py | 68 +++++++++++++++++++ subiquity/models/filesystem.py | 34 ++++++---- subiquity/views/filesystem.py | 27 ++++++-- 11 files changed, 171 insertions(+), 48 deletions(-) create mode 100644 subiquity/curtin.py diff --git a/Makefile b/Makefile index 0d019a69..17fe171b 100644 --- a/Makefile +++ b/Makefile @@ -13,12 +13,16 @@ INSTALLIMG=ubuntu-server-${STREAM}-${RELEASE}-${ARCH}-installer.img INSTALLER_RESOURCES += $(shell find installer/resources -type f) .PHONY: run clean +all: dryrun + +dryrun: + $(MAKE) ui-view DRYRUN="--dry-run" ui-view: - (PYTHONPATH=$(PYTHONPATH) bin/$(PYTHONSRC)) + (PYTHONPATH=$(shell pwd) bin/$(PYTHONSRC) $(DRYRUN)) ui-view-serial: - (TERM=att4424 PYTHONPATH=$(PYTHONPATH) bin/$(PYTHONSRC) --serial) + (TERM=att4424 PYTHONPATH=$(PYTHONPATH) bin/$(PYTHONSRC) $(DRYRUN) --serial) lint: echo "Running flake8 lint tests..." diff --git a/bin/curtin_wrap.sh b/bin/curtin_wrap.sh index 7ef4fc4f..bbed42ef 100755 --- a/bin/curtin_wrap.sh +++ b/bin/curtin_wrap.sh @@ -1,14 +1,21 @@ -#!/bin/bash +#!/bin/bash -x -CMD="-v --showtrace install cp:///" +CMD="-v --showtrace install -c /tmp/subiquity-config.yaml cp:///" CURTIN="/usr/local/curtin/bin/curtin" OUTPUT="/tmp/.curtin_wrap_ran" if [ -e $CURTIN ]; then $CURTIN $CMD | tee -a $OUTPUT + RV=$? else + RV=1 echo "$CURTIN not found" > $OUTPUT echo $CURTIN $CMD | tee -a $OUTPUT fi -echo "Shutting down after running $CURTIN $CMD" -telinit 0 +if [ $RV == 0 ]; then + echo "Shutting down after running $CURTIN $CMD" + #telinit 0 +else + echo "Error running curting, exiting..." + exit 1; +fi diff --git a/bin/subiquity b/bin/subiquity index fd38eda3..e3e06732 100755 --- a/bin/subiquity +++ b/bin/subiquity @@ -27,6 +27,9 @@ def parse_options(argv): parser = argparse.ArgumentParser( description='SUbiquity - Ubiquity for Servers', prog='subiquity') + parser.add_argument('--dry-run', action='store_true', + dest='dry_run', + help='menu-only, do not call installer function') parser.add_argument('--serial', action='store_true', dest='run_on_serial', help='Run the installer over serial console.') diff --git a/installer/geninstaller b/installer/geninstaller index 0c130d2f..4e8827ac 100755 --- a/installer/geninstaller +++ b/installer/geninstaller @@ -30,13 +30,14 @@ simplestreams syslinux-common grub2-common" SRC_DEPS=( - "bzr" "lp:curtin" - "git" "https://github.com/CanonicalLtd/probert.git" + "bzr" "lp:~wesley-wiedenmeier/curtin/custom-partitioning-layout" "curtin" + "git" "https://github.com/CanonicalLtd/probert.git" "probert" ) CACHEDIR="" cleanup_noexit() { [ -n "${CACHEDIR}" ] && { + sync sudo umount ${CACHEDIR}/mnt/{dev,proc,sys} || exit sudo umount ${CACHEDIR}/mnt || exit sudo kpartx -v -d ${CACHEDIR}/installer.img || exit @@ -146,19 +147,14 @@ install_src() { while [ $# -gt 0 ]; do local proto=${1}; shift local url=${1}; shift + local localdir=${1}; shift [ -z "${proto}" -o -z "${url}" ] && { log "ERROR installing source with args: $@" return 1; } - local target="" - case $url in - lp:*) - target="$dldir/${url#lp:*}";; - *) - target="$dldir/`basename $url`";; - esac + local target="$dldir/$localdir" case "$proto" in git) cmd="git clone $url $target";; @@ -304,8 +300,11 @@ generate_seed() { ( instcmd="curtin install cp:///" cd $dldir/curtin - PYTHONPATH=$PYTHONPATH:`pwd` ./bin/curtin pack -- $instcmd > $curtin_cmd - ) + PYTHONPATH=`pwd` ./bin/curtin pack -- $instcmd > $curtin_cmd + ) || { + log "ERROR: failed to pack curtin installer"; + return 1; + } # inject user-data/meta-data into seed log "Writing seed meta-data" @@ -340,7 +339,7 @@ generate_seed() { } log "Writing seed user-data (probert)" local probert_tar=$dldir/probert.tar - (cd ${dldir}/probert.git && + (cd ${dldir}/probert && tar -cpf $probert_tar bin probert) || { log "ERROR: Failed to package probert installer"; return 1; diff --git a/installer/resources/user-data/installer-user-data b/installer/resources/user-data/installer-user-data index 9b51773a..677b9302 100644 --- a/installer/resources/user-data/installer-user-data +++ b/installer/resources/user-data/installer-user-data @@ -11,6 +11,7 @@ output: {all: '| tee -a /var/log/cloud-init-output.log'} packages: - python-urwid - python3-urwid + - python3-parted - python3-pyudev - python3-netifaces runcmd: diff --git a/installer/runinstaller b/installer/runinstaller index 3d55e9eb..fdc56014 100755 --- a/installer/runinstaller +++ b/installer/runinstaller @@ -37,11 +37,13 @@ if [ -z $QEMU ]; then } fi -[ ! -f ${TARGET} ] && { - qemu-img create -f raw ${TARGET} 10G || exit 1 -} +# always recreate the target image +qemu-img create -f raw ${TARGET} 10G + # TODO, curses should work, but my xmonad setup blocks using it -sudo qemu-system-$ARCH -smp 2 -m $MEM -enable-kvm -hda $INSTALLER -hdb $TARGET \ +sudo qemu-system-$ARCH -smp 2 -m $MEM -enable-kvm \ + -drive cache=unsafe,if=virtio,file=$INSTALLER,serial=QM_INSTALL_01 \ + -drive cache=unsafe,if=virtio,file=$TARGET,serial=QM_TARGET_01 \ -net nic,model=e1000 \ -net nic,model=virtio \ -net nic,model=i82559er \ diff --git a/subiquity/controllers/filesystem.py b/subiquity/controllers/filesystem.py index 41552760..9dcccfd1 100644 --- a/subiquity/controllers/filesystem.py +++ b/subiquity/controllers/filesystem.py @@ -16,10 +16,13 @@ from subiquity.controllers.policy import ControllerPolicy from subiquity.views.filesystem import FilesystemView from subiquity.models.filesystem import FilesystemModel +from subiquity.curtin import curtin_write_storage_template import logging +import subprocess -log = logging.getLogger('subiquity.filesystem') + +log = logging.getLogger('subiquity.filesystemController') class FilesystemController(ControllerPolicy): @@ -36,10 +39,30 @@ class FilesystemController(ControllerPolicy): self.ui.set_body(FilesystemView(model, self.finish)) return - def finish(self, disk=None): + def finish(self, disk=None, disk_model=None, disk_serial=None): if disk is None: return self.ui.prev_controller() log.info("Filesystem Interface choosen: {}".format(disk)) + log.info("params: disk={} model={} serial={}".format( + disk, disk_model, disk_serial)) + log.debug( + "FilesystemController: dry_run: {}".format(self.ui.opts.dry_run)) + + log.info("Rendering curtin config from user choices") + curtin_write_storage_template(disk, disk_model, disk_serial) + if self.ui.opts.dry_run: + log.debug("filesystem: this is a dry-run") + print("\033c") + print("**** DRY_RUN ****") + print('NOT calling: ' + 'subprocess.check_call("/usr/local/bin/curtin_wrap.sh")') + print("**** DRY_RUN ****") + else: + log.debug("filesystem: this is the *real* thing") + print("\033c") + print("**** Calling curtin installer ****") + subprocess.check_call("/usr/local/bin/curtin_wrap.sh") + return self.ui.exit() __controller_class__ = FilesystemController diff --git a/subiquity/controllers/installpath.py b/subiquity/controllers/installpath.py index d78680dd..c98c9cd6 100644 --- a/subiquity/controllers/installpath.py +++ b/subiquity/controllers/installpath.py @@ -17,7 +17,6 @@ from subiquity.controllers.policy import ControllerPolicy from subiquity.views.installpath import InstallpathView from subiquity.models.installpath import InstallpathModel import logging -import subprocess log = logging.getLogger('subiquity.installpath') @@ -42,9 +41,9 @@ class InstallpathController(ControllerPolicy): return def finish(self, install_selection=None): + log.debug("installpath cb selection: {}".format(install_selection)) if install_selection is None: return self.ui.prev_controller() - # subprocess.check_call("/usr/local/bin/curtin_wrap.sh") return self.ui.next_controller() __controller_class__ = InstallpathController diff --git a/subiquity/curtin.py b/subiquity/curtin.py new file mode 100644 index 00000000..6f9df1a9 --- /dev/null +++ b/subiquity/curtin.py @@ -0,0 +1,68 @@ +import jinja2 +import os +import datetime + +CURTIN_STORAGE_CONFIG_FILE = '/tmp/subiquity-config.yaml' +CURTIN_STORAGE_CONFIG_TEMPLATE = """ +# Autogenerated by SUbiquity: {{DATE}} UTC +partitioning_commands: + builtin: curtin block-meta custom + +storage: + - id: {{TARGET_DISK_NAME}} + type: disk + ptable: msdos + model: {{TARGET_DISK_MODEL}} + serial: {{TARGET_DISK_SERIAL}} + - id: {{TARGET_DISK_NAME}}1 + type: partition + offset: 512MB + size: 8GB + device: {{TARGET_DISK_NAME}} + flag: boot + - id: {{TARGET_DISK_NAME}}2 + type: partition + offset: 8512MB + size: 1GB + device: {{TARGET_DISK_NAME}} + - id: {{TARGET_DISK_NAME}}1_root + type: format + fstype: ext4 + volume: {{TARGET_DISK_NAME}}1 + - id: {{TARGET_DISK_NAME}}2_home + type: format + fstype: ext4 + volume: {{TARGET_DISK_NAME}}2 + - id: {{TARGET_DISK_NAME}}1_mount + type: mount + path: / + device: {{TARGET_DISK_NAME}}1_root + - id: {{TARGET_DISK_NAME}}2_mount + type: mount + path: /home + device: {{TARGET_DISK_NAME}}2_home +""" + + +def curtin_write_storage_template(disk_name, disk_model, disk_serial): + ''' write out the storage yaml template for curtin + params: + disk_name: kernel name of disk (/dev/sda) + disk_model: disk model name + disk_serial: serial of disk from probert storage output + ''' + template = jinja2.Template(CURTIN_STORAGE_CONFIG_TEMPLATE, + undefined=jinja2.StrictUndefined) + + ctxt = { + 'DATE': str(datetime.datetime.utcnow()), + 'TARGET_DISK_NAME': os.path.basename(disk_name), + 'TARGET_DISK_MODEL': disk_model, + 'TARGET_DISK_SERIAL': disk_serial, + } + curtin_config = template.render(ctxt) + with open(CURTIN_STORAGE_CONFIG_FILE, 'w') as conf: + conf.write(curtin_config) + conf.close() + + return CURTIN_STORAGE_CONFIG_FILE diff --git a/subiquity/models/filesystem.py b/subiquity/models/filesystem.py index ac2f72fb..a6c05a72 100644 --- a/subiquity/models/filesystem.py +++ b/subiquity/models/filesystem.py @@ -22,8 +22,11 @@ configuration. from subiquity import models import argparse -import math from probert import prober +from probert.storage import StorageInfo +import logging + +log = logging.getLogger('subiquity.filesystemModel') class FilesystemModel(models.Model): @@ -41,29 +44,30 @@ class FilesystemModel(models.Model): self.options = argparse.Namespace(probe_storage=True, probe_network=False) self.prober = prober.Prober(self.options) + self.probe_storage() def probe_storage(self): + self.disks = {} self.prober.probe() self.storage = self.prober.get_results().get('storage') + log.info('storage probe data:\n{}'.format(self.storage)) - def get_available_disks(self): - return [disk for disk in self.storage.keys() - if self.storage[disk]['DEVTYPE'] == 'disk' and - self.storage[disk]['MAJOR'] == '8'] + # TODO: replace this with Storage.get_device_by_match() + # which takes a lambda fn for matching + VALID_MAJORS = ['8', '253'] + for disk in self.storage.keys(): + if self.storage[disk]['DEVTYPE'] == 'disk' and \ + self.storage[disk]['MAJOR'] in VALID_MAJORS: + log.info('disk={} storage={}'.format(disk, self.storage)) + self.disks[disk] = StorageInfo({disk: self.storage[disk]}) def get_partitions(self): return [part for part in self.storage.keys() if self.storage[part]['DEVTYPE'] == 'partition' and self.storage[part]['MAJOR'] == '8'] - def _humanize_size(self, size): - size = abs(size) - if size == 0: - return "0B" - units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'] - p = math.floor(math.log(size, 2) / 10) - return "%.3f %s" % (size / math.pow(1024, p), units[int(p)]) + def get_available_disks(self): + return self.disks.keys() - def get_disk_size(self, disk): - return self._humanize_size( - int(self.storage[disk]['attrs']['size']) * 512) + def get_disk_info(self, disk): + return self.disks[disk] diff --git a/subiquity/views/filesystem.py b/subiquity/views/filesystem.py index 58228ea2..50bff359 100644 --- a/subiquity/views/filesystem.py +++ b/subiquity/views/filesystem.py @@ -14,6 +14,7 @@ # along with this program. If not, see . import logging +import math from urwid import (WidgetWrap, ListBox, Pile, BoxAdapter, Text, Columns) from subiquity.ui.lists import SimpleList from subiquity.ui.buttons import confirm_btn, cancel_btn @@ -23,6 +24,15 @@ from subiquity.ui.utils import Padding, Color log = logging.getLogger('subiquity.filesystemView') +def _humanize_size(size): + size = abs(size) + if size == 0: + return "0B" + units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'] + p = math.floor(math.log(size, 2) / 10) + return "%.3f %s" % (size / math.pow(1024, p), units[int(p)]) + + class FilesystemView(WidgetWrap): def __init__(self, model, cb): self.model = model @@ -59,18 +69,16 @@ class FilesystemView(WidgetWrap): return Pile(buttons) def _build_model_inputs(self): - self.model.probe_storage() - disks = self.model.get_available_disks() col_1 = [] col_2 = [] - for disk in disks: + for dname in self.model.get_available_disks(): + disk = self.model.get_disk_info(dname) col_1.append( - Color.button_primary(confirm_btn(label=disk, + Color.button_primary(confirm_btn(label=disk.name, on_press=self.confirm), focus_map='button_primary focus')) - disk_sz = self.model.get_disk_size(disk) - + disk_sz = str(_humanize_size(disk.size)) col_2.append(Text(disk_sz)) col_1 = BoxAdapter(SimpleList(col_1), @@ -89,7 +97,12 @@ class FilesystemView(WidgetWrap): return Pile(opts) def confirm(self, button): - return self.cb(button.label) + log.info("Filesystem View confirm() getting disk info") + disk = self.model.get_disk_info(button.label) + log.info("Filesystem View callback({}, {}, {})".format(disk.name, + disk.model, + disk.serial)) + return self.cb(disk.name, disk.model, disk.serial) def cancel(self, button): return self.cb(None)