Add probert to subiquity

Update subiquity welcome controller to call probert to extract
host information and present it to the user briefly before
invokin the curtin installer.

Add the install_src to git clone probert.  Pack this as a tarball
in the user-data like subiquity.

Update user-data package installs for new probert deps.

Modify boot parameters to handle systemd persistent nic naming;
we may not always have eth0 as a nic.  Subsequently update
user-data to use bootcmd to automatically attempt to bring up
nics during the boot before the installer.

Finally fix commandline parsing/output in geninstaller.
Implement version parameter and better caching of previous
maas ephemeral images.

Fix issue with removing cache after unpacking rootfs which is
now owned by root (add sudo to rm of cache).

Signed-off-by: Ryan Harper <ryan.harper@canonical.com>

installer/geninstaller

@@ -31,6 +31,7 @@ syslinux-common
 grub2-common"
 SRC_DEPS=(
   "bzr" "lp:curtin"
+  "git" "https://github.com/CanonicalLtd/probert.git"
 )
 CACHEDIR=""
 
@@ -101,6 +102,8 @@ usage: $PROG [PARAMS] [ARGS]
  -b, --bootloader=TYPE          For TYPE in [syslinux, grub2, uboot]
  -h, --help                     This output.
  -r, --release=RELEASE          For RELEASE in  [trusty, utopic, vivid, wily]
+ -s, --stream=STREAM            For STREAM in [daily, released]
+ -V, --version=VERSION          VERSION=YYYYMMDD , 20150623
  -v, --verbose
 
 
@@ -194,7 +197,7 @@ acquire_image() {
     local version=${4};
 
     # run a query unless they specify all params
-    if [ $# -lt 5 ]; then
+    if [ $# -le 4 -a -z "${version}" ]; then
         log "Querying simplestreams for latest image: $label $release $arch"
         case "$label" in
             daily)
@@ -226,12 +229,17 @@ acquire_image() {
     #   and re-assemble roottar and rootfs which was based on ephimg
 
     [ -r "$ephimg" ] && CACHE_SUM=( `sha256sum $ephimg 2>/dev/null` )
+    if [ -z "${sha256}" -a -f "${ephimg}.sha256" ]; then
+        log "Using sha256sum from cached file";
+        sha256=`cat ${ephimg}.sha256 | cut -d' ' -f1`
+    fi
     if [ "${sha256}" != "${CACHE_SUM[0]}" ]; then
         log "WARNING: sha256 csum mismatch"
         log "WARNING: expected: [$sha256]"
         log "            found: [${CACHE_SUM[0]}]"
         # didn't match so nuke the cache
-        rm -fr "${cachedir}" || {
+        log "WARNING: removing old cache"
+        sudo rm -fr "${cachedir}" || {
             log "ERROR: failed to remove stale cachedir: $cachedir";
             return 1;
         }
@@ -245,6 +253,8 @@ acquire_image() {
             log "ERROR: failed to download: ${item_url}";
             return 1;
         }
+        (cd `dirname ${ephimg}` &&
+         sha256sum `basename ${ephimg}` > ${ephimg}.sha256)
     else
         log "  Using cached $label $release $arch $item_name:"
         log "  $ephimg"
@@ -328,7 +338,19 @@ generate_seed() {
         log "Failed to subiquity into $seed";
         return 1;
     }
-
+    log "Writing seed user-data (probert)"
+    local probert_tar=$dldir/probert.tar
+    (cd ${dldir}/probert.git &&
+     tar -cpf $probert_tar bin probert) || {
+        log "ERROR: Failed to package probert installer";
+        return 1;
+    }
+    userdata_write_file "/tmp/probert.tar" \
+                        "root:root" "0644" "b64" \
+                        "$probert_tar" >> $seed/user-data || {
+        log "Failed to subiquity into $seed";
+        return 1;
+    }
     return 0
 }
 
@@ -437,8 +459,8 @@ parse_args() {
     # args:
     [ $# -lt 1 ] && { usage; exit 0; }
 
-    OPTS_LONG="arch:bootloader:,download:help,release:,stream:verbose"
+    OPTS_LONG="arch:,bootloader:,download:,help,release:,stream:,verbose,version:"
-    OPTS="a:b:d:h,r:s:v"
+    OPTS="a:b:d:hr:s:vV:"
     ARGS=`getopt --name "$PROG" --long $OPTS_LONG --options $OPTS -- "$@"`
     if [ $? -ne 0 ]; then
         echo "$PROG: usage error (use -h for help)" >&2
@@ -452,6 +474,7 @@ parse_args() {
     RELEASE="wily"
     STREAM="daily"
     VERBOSE="no"
+    VERSION=""
     while [ $# -gt 0 ]; do
         case "$1" in
             -a | --arch)        ARCH="$2"; shift;;
@@ -460,6 +483,7 @@ parse_args() {
             -h | --help)        usage; exit 0;;
             -r | --release)     RELEASE="$2"; shift;;
             -s | --stream)      STREAM="$2"; shift;;
+            -V | --version)     VERSION="$2"; shift;;
             -v | --verbose)     VERBOSE="yes";;
             --)                 shift; break;; # end of options
         esac
@@ -488,7 +512,7 @@ main() {
         return 1;
     }
 
-    acquire_image ${DLDIR} "$STREAM" "$RELEASE" "$ARCH" || {
+    acquire_image ${DLDIR} "$STREAM" "$RELEASE" "$ARCH" "$VERSION" || {
         return 1;
     }
     CACHEDIR=${_RETVAL}

installer/resources/menu/extlinux-menu.conf

@@ -9,9 +9,9 @@ label text-installer
    menu label ^Ubuntu Server Installer (text)
    menu default
    linux /vmlinuz
-   append initrd=/initrd.img ip=::::myhostname:BOOTIF ro root=LABEL=cloudimg-rootfs overlayroot=tmpfs BOOTIF_DEFAULT=eth0 console=ttyS0 console=tty0 splash
+   append initrd=/initrd.img ip=dhcp ro root=LABEL=cloudimg-rootfs overlayroot=tmpfs console=ttyS0 console=tty0 splash
 
 label serial-installer
    menu label ^Ubuntu Server Installer (serial)
    linux /vmlinuz
-   append initrd=/initrd.img ip=::::myhostname:BOOTIF ro root=LABEL=cloudimg-rootfs overlayroot=tmpfs BOOTIF_DEFAULT=eth0 console=tty0 console=ttyS0 splash
+   append initrd=/initrd.img ip=dhcp ro root=LABEL=cloudimg-rootfs overlayroot=tmpfs console=tty0 console=ttyS0 splash

installer/resources/user-data/installer-user-data

@@ -1,13 +1,21 @@
 #cloud-config
 #http_proxy: http://my-proxy:3129/
+bootcmd:
+    - rm -f /etc/network/interfaces
+    - echo -e "auto lo\niface lo inet loopback\n" > /etc/network/interfaces
+    - /bin/ls -1 /sys/class/net | grep -v ^lo$ | xargs -i bash -c 'D={}; echo -e "auto $D\niface $D inet dhcp"' | tee /etc/network/interfaces
+    - /bin/ls -1 /sys/class/net | grep -v ^lo$ | xargs -i ifup {}
 password: passw0rd
 chpasswd: { expire: False }
 output: {all: '| tee -a /var/log/cloud-init-output.log'}
 packages:
  - python-urwid
  - python3-urwid
+ - python3-pyudev
+ - python3-netifaces
 runcmd:
     - tar -C /usr/local -xf /tmp/subiquity.tar
+    - tar -C /usr/local -xf /tmp/probert.tar
     - (cd /usr/local && bin/curtin-archive extract --no-execute)
     - /tmp/installer.sh
 write_files:

installer/runinstaller

@@ -41,7 +41,11 @@ fi
     qemu-img create -f raw ${TARGET} 10G || exit 1
 }
 # TODO, curses should work, but my xmonad setup blocks using it
-sudo qemu-system-$ARCH -m $MEM -enable-kvm -hda $INSTALLER -hdb $TARGET \
+sudo qemu-system-$ARCH -smp 2 -m $MEM -enable-kvm -hda $INSTALLER -hdb $TARGET \
+                       -net nic,model=e1000 \
+                       -net nic,model=virtio \
+                       -net nic,model=i82559er \
+                       -net user \
                        -monitor telnet:127.0.0.1:2446,server,nowait -serial stdio
 #sudo qemu-system-$ARCH -m $MEM -enable-kvm -hda $INSTALLER -hdb $TARGET \
 #                       -monitor telnet:127.0.0.1:2446,server,nowait -curses

subiquity/controllers/welcome.py

@@ -44,3 +44,5 @@ class WelcomeController(ControllerPolicy):
         return self.ui.next_controller()
 
 __controller_class__ = WelcomeController
+
+

subiquity/models/filesystem.py

@@ -21,20 +21,42 @@ configuration.
 """
 
 from subiquity import models
+import argparse
+import math
+import time
+from probert import prober
 
 
 class FilesystemModel(models.Model):
     """ Model representing storage options
     """
 
-    available_disks = ['/dev/sda',
-                       '/dev/sdb',
-                       '/dev/sdc',
-                       '/dev/sdd',
-                       '/dev/sde']
-
     additional_options = ['Connecti iSCSI network disk',
                           'Connect Ceph network disk',
                           'Create volume group (LVM2)',
                           'Create software RAID (MD)',
                           'Setup hierarchichal storage (bcache)']
+    def __init__(self):
+        self.storage= {}
+        self.options = argparse.Namespace(probe_storage=True, probe_network=False)
+        self.prober = prober.Prober(self.options)
+
+    def probe_storage(self):
+        self.prober.probe()
+        self.storage = self.prober.get_results().get('storage'}
+
+    def get_available_disks(self):
+        return [disk for disk in self.storage.keys()
+                if self.storage[disk]['DEVTYPE'] == 'disk' and
+                    self.storage[disk]['MAJOR'] == '8']
+
+    def _humanize_size(self, size):
+        size = abs(size)
+        if (size==0):
+            return "0B"
+        units = ['B','KiB','MiB','GiB','TiB','PiB','EiB','ZiB','YiB']
+        p = math.floor(math.log(size, 2)/10)
+        return "%.3f %s" % (size/math.pow(1024,p),units[int(p)])
+
+    def get_disk_size(self, disk):
+        return self._humanize_size(int(self.storage[disk]['attrs']['size']))

subiquity/models/network.py

@@ -20,16 +20,29 @@ Provides network device listings and extended network information
 """
 
 from subiquity import models
+import argparse
+import math
+import time
+from probert import prober
 
 
 class NetworkModel(models.Model):
     """ Model representing network interfaces
     """
 
-    interfaces = ['em1',
-                  'em2',
-                  'bond0']
-
     additional_options = ['Set default route',
                           'Bond interfaces',
                           'Install network driver']
+
+    def __init__(self):
+        self.hwdata = {}
+        self.options = argparse.Namespace(probe_storage=False, probe_network=True)
+        self.prober = prober.Prober(self.options)
+
+    def probe_network(self):
+        self.prober.probe()
+        self.hwdata = self.prober.get_results()
+
+    def get_interfaces(self):
+        return [n for n in self.hwdata['network'].keys() 
+                if self.hwdata['network'][n]['type'] == 'eth']

subiquity/views/filesystem.py

@@ -46,11 +46,14 @@ class FilesystemView(WidgetWrap):
 
     def _build_model_inputs(self):
         sl = []
-        for iface in self.model.available_disks:
+        self.model.probe_storage()
-            sl.append(Color.button_primary(confirm_btn(label=iface,
+        for disk in self.model.get_available_disks():
+
+            sl.append(Color.button_primary(confirm_btn(label=disk,
                                                        on_press=self.confirm),
                                            focus_map='button_primary focus'))
-            sl.append(Padding.push_10(Text("2 tb free")))
+            disk_sz = self.model.get_disk_size(disk)
+            sl.append(Padding.push_10(Text(disk_sz)))
 
         return BoxAdapter(SimpleList(sl),
                           height=len(sl))

subiquity/views/network.py

@@ -46,7 +46,8 @@ class NetworkView(WidgetWrap):
 
     def _build_model_inputs(self):
         sl = []
-        for iface in self.model.interfaces:
+        self.model.probe_network()
+        for iface in self.model.get_interfaces():
             sl.append(Color.button_primary(confirm_btn(label=iface,
                                                        on_press=self.confirm),
                                            focus_map='button_primary focus'))