Call curtin to complete install

- Use updated probert *Info class for accessing object data
- Add --dry-run parameter to prevent invoking curtin when testing
- Add new make target to run ui-view by default with --dry-run
- Use a template to write out curtin config file for installation
- Use virtio disks and multiple nics to test probing.

Signed-off-by: Ryan Harper <ryan.harper@canonical.com>

Makefile

@@ -13,12 +13,16 @@ INSTALLIMG=ubuntu-server-${STREAM}-${RELEASE}-${ARCH}-installer.img
 INSTALLER_RESOURCES += $(shell find installer/resources -type f)
 .PHONY: run clean
 
+all: dryrun
+
+dryrun:
+	$(MAKE) ui-view DRYRUN="--dry-run"
 
 ui-view:
-	(PYTHONPATH=$(PYTHONPATH) bin/$(PYTHONSRC))
+	(PYTHONPATH=$(shell pwd) bin/$(PYTHONSRC) $(DRYRUN))
 
 ui-view-serial:
-	(TERM=att4424 PYTHONPATH=$(PYTHONPATH) bin/$(PYTHONSRC) --serial)
+	(TERM=att4424 PYTHONPATH=$(PYTHONPATH) bin/$(PYTHONSRC) $(DRYRUN) --serial)
 
 lint:
 	echo "Running flake8 lint tests..."

bin/curtin_wrap.sh

@@ -1,14 +1,21 @@
-#!/bin/bash
+#!/bin/bash -x
 
-CMD="-v --showtrace install cp:///"
+CMD="-v --showtrace install -c /tmp/subiquity-config.yaml cp:///"
 CURTIN="/usr/local/curtin/bin/curtin"
 OUTPUT="/tmp/.curtin_wrap_ran"
 
 if [ -e $CURTIN ]; then 
    $CURTIN $CMD | tee -a $OUTPUT
+   RV=$?
 else
+   RV=1
    echo "$CURTIN not found" > $OUTPUT
    echo $CURTIN $CMD | tee -a $OUTPUT
 fi
-echo "Shutting down after running $CURTIN $CMD"
+if [ $RV == 0 ]; then
-telinit 0
+    echo "Shutting down after running $CURTIN $CMD"
+    #telinit 0
+else
+    echo "Error running curting, exiting..."
+    exit 1;
+fi

bin/subiquity

@@ -27,6 +27,9 @@ def parse_options(argv):
     parser = argparse.ArgumentParser(
         description='SUbiquity - Ubiquity for Servers',
         prog='subiquity')
+    parser.add_argument('--dry-run', action='store_true',
+                        dest='dry_run',
+                        help='menu-only, do not call installer function')
     parser.add_argument('--serial', action='store_true',
                         dest='run_on_serial',
                         help='Run the installer over serial console.')

installer/geninstaller

@@ -30,13 +30,14 @@ simplestreams
 syslinux-common
 grub2-common"
 SRC_DEPS=(
-  "bzr" "lp:curtin"
+  "bzr" "lp:~wesley-wiedenmeier/curtin/custom-partitioning-layout" "curtin"
-  "git" "https://github.com/CanonicalLtd/probert.git"
+  "git" "https://github.com/CanonicalLtd/probert.git" "probert"
 )
 CACHEDIR=""
 
 cleanup_noexit() {
     [ -n "${CACHEDIR}" ] && {
+        sync
         sudo umount ${CACHEDIR}/mnt/{dev,proc,sys} || exit
         sudo umount ${CACHEDIR}/mnt  || exit
         sudo kpartx -v -d ${CACHEDIR}/installer.img || exit
@@ -146,19 +147,14 @@ install_src() {
     while [ $# -gt 0 ]; do
         local proto=${1}; shift
         local url=${1}; shift
+        local localdir=${1}; shift
 
         [ -z "${proto}" -o -z "${url}" ] && {
             log "ERROR installing source with args: $@"
             return 1;
         }
 
-        local target=""
+        local target="$dldir/$localdir"
-        case $url in
-            lp:*)
-                target="$dldir/${url#lp:*}";;
-            *)
-                target="$dldir/`basename $url`";;
-        esac
         case "$proto" in
             git)
                 cmd="git clone $url $target";;
@@ -304,8 +300,11 @@ generate_seed() {
     (
         instcmd="curtin install cp:///"
         cd $dldir/curtin
-        PYTHONPATH=$PYTHONPATH:`pwd` ./bin/curtin pack -- $instcmd > $curtin_cmd
+        PYTHONPATH=`pwd` ./bin/curtin pack -- $instcmd > $curtin_cmd
-    )
+    ) || {
+            log "ERROR: failed to pack curtin installer";
+            return 1;
+    }
 
     # inject user-data/meta-data into seed
     log "Writing seed meta-data"
@@ -340,7 +339,7 @@ generate_seed() {
     }
     log "Writing seed user-data (probert)"
     local probert_tar=$dldir/probert.tar
-    (cd ${dldir}/probert.git &&
+    (cd ${dldir}/probert &&
      tar -cpf $probert_tar bin probert) || {
         log "ERROR: Failed to package probert installer";
         return 1;

installer/resources/user-data/installer-user-data

@@ -11,6 +11,7 @@ output: {all: '| tee -a /var/log/cloud-init-output.log'}
 packages:
  - python-urwid
  - python3-urwid
+ - python3-parted
  - python3-pyudev
  - python3-netifaces
 runcmd:

installer/runinstaller

@@ -37,11 +37,13 @@ if [ -z $QEMU ]; then
     }
 fi
 
-[ ! -f ${TARGET} ] && {
+# always recreate the target image
-    qemu-img create -f raw ${TARGET} 10G || exit 1
+qemu-img create -f raw ${TARGET} 10G
-}
+
 # TODO, curses should work, but my xmonad setup blocks using it
-sudo qemu-system-$ARCH -smp 2 -m $MEM -enable-kvm -hda $INSTALLER -hdb $TARGET \
+sudo qemu-system-$ARCH -smp 2 -m $MEM -enable-kvm \
+                       -drive cache=unsafe,if=virtio,file=$INSTALLER,serial=QM_INSTALL_01 \
+                       -drive cache=unsafe,if=virtio,file=$TARGET,serial=QM_TARGET_01 \
                        -net nic,model=e1000 \
                        -net nic,model=virtio \
                        -net nic,model=i82559er \

subiquity/controllers/filesystem.py

@@ -16,10 +16,13 @@
 from subiquity.controllers.policy import ControllerPolicy
 from subiquity.views.filesystem import FilesystemView
 from subiquity.models.filesystem import FilesystemModel
+from subiquity.curtin import curtin_write_storage_template
 
 import logging
+import subprocess
 
-log = logging.getLogger('subiquity.filesystem')
+
+log = logging.getLogger('subiquity.filesystemController')
 
 
 class FilesystemController(ControllerPolicy):
@@ -36,10 +39,30 @@ class FilesystemController(ControllerPolicy):
         self.ui.set_body(FilesystemView(model, self.finish))
         return
 
-    def finish(self, disk=None):
+    def finish(self, disk=None, disk_model=None, disk_serial=None):
         if disk is None:
             return self.ui.prev_controller()
         log.info("Filesystem Interface choosen: {}".format(disk))
+        log.info("params: disk={} model={} serial={}".format(
+                 disk, disk_model, disk_serial))
+        log.debug(
+            "FilesystemController: dry_run: {}".format(self.ui.opts.dry_run))
+
+        log.info("Rendering curtin config from user choices")
+        curtin_write_storage_template(disk, disk_model, disk_serial)
+        if self.ui.opts.dry_run:
+            log.debug("filesystem: this is a dry-run")
+            print("\033c")
+            print("**** DRY_RUN ****")
+            print('NOT calling: '
+                  'subprocess.check_call("/usr/local/bin/curtin_wrap.sh")')
+            print("**** DRY_RUN ****")
+        else:
+            log.debug("filesystem: this is the *real* thing")
+            print("\033c")
+            print("**** Calling curtin installer ****")
+            subprocess.check_call("/usr/local/bin/curtin_wrap.sh")
+
         return self.ui.exit()
 
 __controller_class__ = FilesystemController

subiquity/controllers/installpath.py

@@ -17,7 +17,6 @@ from subiquity.controllers.policy import ControllerPolicy
 from subiquity.views.installpath import InstallpathView
 from subiquity.models.installpath import InstallpathModel
 import logging
-import subprocess
 
 log = logging.getLogger('subiquity.installpath')
 
@@ -42,9 +41,9 @@ class InstallpathController(ControllerPolicy):
         return
 
     def finish(self, install_selection=None):
+        log.debug("installpath cb selection: {}".format(install_selection))
         if install_selection is None:
             return self.ui.prev_controller()
-        # subprocess.check_call("/usr/local/bin/curtin_wrap.sh")
         return self.ui.next_controller()
 
 __controller_class__ = InstallpathController

subiquity/curtin.py

@@ -0,0 +1,68 @@
+import jinja2
+import os
+import datetime
+
+CURTIN_STORAGE_CONFIG_FILE = '/tmp/subiquity-config.yaml'
+CURTIN_STORAGE_CONFIG_TEMPLATE = """
+# Autogenerated by SUbiquity: {{DATE}} UTC
+partitioning_commands:
+ builtin: curtin block-meta custom
+
+storage:
+  - id: {{TARGET_DISK_NAME}}
+    type: disk
+    ptable: msdos
+    model: {{TARGET_DISK_MODEL}}
+    serial: {{TARGET_DISK_SERIAL}}
+  - id: {{TARGET_DISK_NAME}}1
+    type: partition
+    offset: 512MB
+    size: 8GB
+    device: {{TARGET_DISK_NAME}}
+    flag: boot
+  - id: {{TARGET_DISK_NAME}}2
+    type: partition
+    offset: 8512MB
+    size: 1GB
+    device: {{TARGET_DISK_NAME}}
+  - id: {{TARGET_DISK_NAME}}1_root
+    type: format
+    fstype: ext4
+    volume: {{TARGET_DISK_NAME}}1
+  - id: {{TARGET_DISK_NAME}}2_home
+    type: format
+    fstype: ext4
+    volume: {{TARGET_DISK_NAME}}2
+  - id: {{TARGET_DISK_NAME}}1_mount
+    type: mount
+    path: /
+    device: {{TARGET_DISK_NAME}}1_root
+  - id: {{TARGET_DISK_NAME}}2_mount
+    type: mount
+    path: /home
+    device: {{TARGET_DISK_NAME}}2_home
+"""
+
+
+def curtin_write_storage_template(disk_name, disk_model, disk_serial):
+    ''' write out the storage yaml template for curtin
+        params:
+            disk_name: kernel name of disk (/dev/sda)
+            disk_model: disk model name
+            disk_serial:  serial of disk from probert storage  output
+    '''
+    template = jinja2.Template(CURTIN_STORAGE_CONFIG_TEMPLATE,
+                               undefined=jinja2.StrictUndefined)
+
+    ctxt = {
+        'DATE': str(datetime.datetime.utcnow()),
+        'TARGET_DISK_NAME': os.path.basename(disk_name),
+        'TARGET_DISK_MODEL': disk_model,
+        'TARGET_DISK_SERIAL': disk_serial,
+    }
+    curtin_config = template.render(ctxt)
+    with open(CURTIN_STORAGE_CONFIG_FILE, 'w') as conf:
+        conf.write(curtin_config)
+        conf.close()
+
+    return CURTIN_STORAGE_CONFIG_FILE

subiquity/models/filesystem.py

@@ -22,8 +22,11 @@ configuration.
 
 from subiquity import models
 import argparse
-import math
 from probert import prober
+from probert.storage import StorageInfo
+import logging
+
+log = logging.getLogger('subiquity.filesystemModel')
 
 
 class FilesystemModel(models.Model):
@@ -41,29 +44,30 @@ class FilesystemModel(models.Model):
         self.options = argparse.Namespace(probe_storage=True,
                                           probe_network=False)
         self.prober = prober.Prober(self.options)
+        self.probe_storage()
 
     def probe_storage(self):
+        self.disks = {}
         self.prober.probe()
         self.storage = self.prober.get_results().get('storage')
+        log.info('storage probe data:\n{}'.format(self.storage))
 
-    def get_available_disks(self):
+        # TODO: replace this with Storage.get_device_by_match()
-        return [disk for disk in self.storage.keys()
+        # which takes a lambda fn for matching
-                if self.storage[disk]['DEVTYPE'] == 'disk' and
+        VALID_MAJORS = ['8', '253']
-                self.storage[disk]['MAJOR'] == '8']
+        for disk in self.storage.keys():
+            if self.storage[disk]['DEVTYPE'] == 'disk' and \
+               self.storage[disk]['MAJOR'] in VALID_MAJORS:
+                log.info('disk={} storage={}'.format(disk, self.storage))
+                self.disks[disk] = StorageInfo({disk: self.storage[disk]})
 
     def get_partitions(self):
         return [part for part in self.storage.keys()
                 if self.storage[part]['DEVTYPE'] == 'partition' and
                 self.storage[part]['MAJOR'] == '8']
 
-    def _humanize_size(self, size):
+    def get_available_disks(self):
-        size = abs(size)
+        return self.disks.keys()
-        if size == 0:
-            return "0B"
-        units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB']
-        p = math.floor(math.log(size, 2) / 10)
-        return "%.3f %s" % (size / math.pow(1024, p), units[int(p)])
 
-    def get_disk_size(self, disk):
+    def get_disk_info(self, disk):
-        return self._humanize_size(
+        return self.disks[disk]
-            int(self.storage[disk]['attrs']['size']) * 512)

subiquity/views/filesystem.py

@@ -14,6 +14,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import logging
+import math
 from urwid import (WidgetWrap, ListBox, Pile, BoxAdapter, Text, Columns)
 from subiquity.ui.lists import SimpleList
 from subiquity.ui.buttons import confirm_btn, cancel_btn
@@ -23,6 +24,15 @@ from subiquity.ui.utils import Padding, Color
 log = logging.getLogger('subiquity.filesystemView')
 
 
+def _humanize_size(size):
+    size = abs(size)
+    if size == 0:
+        return "0B"
+    units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB']
+    p = math.floor(math.log(size, 2) / 10)
+    return "%.3f %s" % (size / math.pow(1024, p), units[int(p)])
+
+
 class FilesystemView(WidgetWrap):
     def __init__(self, model, cb):
         self.model = model
@@ -59,18 +69,16 @@ class FilesystemView(WidgetWrap):
         return Pile(buttons)
 
     def _build_model_inputs(self):
-        self.model.probe_storage()
-        disks = self.model.get_available_disks()
         col_1 = []
         col_2 = []
 
-        for disk in disks:
+        for dname in self.model.get_available_disks():
+            disk = self.model.get_disk_info(dname)
             col_1.append(
-                Color.button_primary(confirm_btn(label=disk,
+                Color.button_primary(confirm_btn(label=disk.name,
                                                  on_press=self.confirm),
                                      focus_map='button_primary focus'))
-            disk_sz = self.model.get_disk_size(disk)
+            disk_sz = str(_humanize_size(disk.size))
-
             col_2.append(Text(disk_sz))
 
         col_1 = BoxAdapter(SimpleList(col_1),
@@ -89,7 +97,12 @@ class FilesystemView(WidgetWrap):
         return Pile(opts)
 
     def confirm(self, button):
-        return self.cb(button.label)
+        log.info("Filesystem View confirm() getting disk info")
+        disk = self.model.get_disk_info(button.label)
+        log.info("Filesystem View callback({}, {}, {})".format(disk.name,
+                                                               disk.model,
+                                                               disk.serial))
+        return self.cb(disk.name, disk.model, disk.serial)
 
     def cancel(self, button):
         return self.cb(None)