diff --git a/Makefile b/Makefile index 03e35662..08256e88 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,25 @@ # # Makefile for subiquity # +STREAM=daily +RELEASE=wily +ARCH=amd64 +INSTALLIMG=ubuntu-server-${STREAM}-${RELEASE}-${ARCH}-installer.img +.PHONY: installer run clean ui-view: PYTHONPATH=$(shell pwd):$(PYTHONPATH) bin/subiquity + +installer: + [ -e "installer/$(INSTALLIMG)" ] || \ + (cd installer && ./geninstaller -r $(RELEASE) -a $(ARCH) -s $(STREAM)) + +run: installer + (cd installer && INSTALLER=$(INSTALLIMG) ./runinstaller) + +clean: + rm -f installer/target.img + rm -f installer/installer.img + rm -f installer/geninstaller.log + find installer -type f -name *-installer.img | xargs -i rm {} + diff --git a/README.md b/README.md index 69d9aad5..23b231b3 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,12 @@ # subiquity Ubuntu Server Installer + +# building installer +make installer + +# running installer +make run + +# overrides +make RELEASE=[wily, vivid, trusty] ARCH=[amd64, i386, armf, arm64, ppc64el] +make RELEAse=wily run diff --git a/installer/README.md b/installer/README.md new file mode 100644 index 00000000..871fae79 --- /dev/null +++ b/installer/README.md @@ -0,0 +1,42 @@ +Getting Started +--------------- + +Install package dependencies: + + PKGS=" + bzr + extlinux + gdisk + kpartx + parted + qemu-system-x86 + qemu-utils + syslinux-common + " + apt-get install $PKGS + +Generate the install image + + ./geninstaller.sh + + +Run the installer + + # generate target device + qemu-img create -f raw target.img 10G + + # run installer + sudo qemu-system-x86_64 -m 1024 -enable-kvm \ + -hda installer.img -hdb test.img \ + -serial telnet:127.0.0.1:2445,server,nowait \ + -monitor stdio + + # login and shutdown, ubuntu/passw0rd + + +Boot the installed image + + sudo qemu-system-x86_64 -m 1024 -enable-kvm \ + -hda test.img \ + -serial telnet:127.0.0.1:2445,server,nowait + diff --git a/installer/geninstaller b/installer/geninstaller new file mode 100755 index 00000000..69739eef --- /dev/null +++ b/installer/geninstaller @@ -0,0 +1,483 @@ +#!/bin/bash +# Copyright 2015 Canonical, Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +PROG=`basename $0` +TOPDIR=`pwd` +USQUERY=${TOPDIR}/usquery +LOGFILE="geninstaller.log" +PREFIX="${PROG}" +RESOURCES=`pwd`/resources +PKG_DEPS=" +qemu-utils +kpartx +parted +extlinux +syslinux-common +grub2-common" +SRC_DEPS=( + "bzr" "lp:curtin" +) +CACHEDIR="" + +cleanup_noexit() { + [ -n "${CACHEDIR}" ] && { + sudo umount ${CACHEDIR}/mnt/{dev,proc,sys} || exit + sudo umount ${CACHEDIR}/mnt || exit + sudo kpartx -v -d ${CACHEDIR}/installer.img || exit + [ -e "/dev/mapper/`basename $DEV`" ] && { + sudo dmsetup remove $DEV || exit + } + if sudo losetup -a | grep -q $LOOPDEV; then + sudo losetup -d $LOOPDEV || exit + fi + } +} + +cleanup() { + cleanup_noexit + exit +} + +trap cleanup EXIT HUP INT TERM + +log() { + echo "`date +%s`: $@" | tee -a ${LOGFILE} +} + +write_metadata() { + cat </dev/null; then + to_install="$to_install $p"; + fi + done + + [ -n "$to_install" ] && { + log "Installing dependencies: $to_install"; + sudo apt-get install -q -y $to_install || { + log "Failed to install one or more dependencies in $to_install"; + return 1; + } + } + return 0 +} + +install_src() { + local dldir=${1}; shift; + + log "Acquiring src packages..." + while [ $# -gt 0 ]; do + local proto=${1}; shift + local url=${1}; shift + + [ -z "${proto}" -o -z "${url}" ] && { + log "ERROR installing source with args: $@" + return 1; + } + + local target="" + case $url in + lp:*) + target="$dldir/${url#lp:*}";; + *) + target="$dldir/`basename $url`";; + esac + case "$proto" in + git) + cmd="git clone $url $target";; + bzr) + cmd="bzr branch $url $target";; + *) + log "ERROR: unsupported src protocol: $proto $url";; + esac + + if [ ! -d "${target}" ]; then + log "Acquiring src @ $url with $proto" + $cmd || { + log "ERROR: failed to fetch src: $proto $url into $target"; + return 1; + } + else + log " Using cached src for repo $url @ $target" + fi + done +} + +acquire_image() { + _RETVAL="" + [ $# -lt 1 ] && { + log "ERROR: not enough arguments passed to $FUNCNAME"; + return 1; + } + + # what to get from maas + local item_name="root-image.gz" + local dldir=${1}; shift; + + local label=${1:-"daily"} + local release=${2-"wily"}; + local arch=${3-"amd64"}; + local version=${4}; + + # run a query unless they specify all params + if [ $# -lt 5 ]; then + log "Querying simplestreams for latest image: $label $release $arch" + case "$label" in + daily) + ssresult=( `${USQUERY} --output-format="%(version_name)s %(item_url)s %(sha256)s" --max=1 maas-daily release=$release arch=$arch item_name=$item_name` ) + local version=${ssresult[0]} + local item_url=${ssresult[1]} + local sha256=${ssresult[2]} + ;; + release) + ssresult=( `${USQUERY} --output-format="%(version_name)s %(item_url)s %(sha256)s" --max=1 maas-release release=$release arch=$arch item_name=$item_name` ) + local version=${ssresult[0]} + local item_url=${ssresult[1]} + local sha256=${ssresult[2]} + ;; + *) + log "ERROR: simplestream label must be one of: [daily, release]" + return 1; + ;; + esac + fi + local cachedir=$dldir/maas/${label}/${release}/${arch}/${version} + local ephimg=${cachedir}/${item_name} + local roottar=${cachedir}/root.tar.gz + local rootfs=${cachedir}/rootfs + + # cache policy: + # $ephimg must be a file and it must checksum to $sha256 value otherwise + # we will nuke the cachedir and reacquire $item_name @ $item_url + # and re-assemble roottar and rootfs which was based on ephimg + + [ -r "$ephimg" ] && CACHE_SUM=( `sha256sum $ephimg 2>/dev/null` ) + if [ "${sha256}" != "${CACHE_SUM[0]}" ]; then + log "WARNING: sha256 csum mismatch" + log "WARNING: expected: [$sha256]" + log " found: [${CACHE_SUM[0]}]" + # didn't match so nuke the cache + rm -fr "${cachedir}" || { + log "ERROR: failed to remove stale cachedir: $cachedir"; + return 1; + } + fi + + # download the image if it's not in the cache + log "Downloading installer root image @ $item_url" + if [ ! -r "${ephimg}" ]; then + mkdir -p $cachedir && + wget --progress=bar -c "${item_url}" -O "${ephimg}" || { + log "ERROR: failed to download: ${item_url}"; + return 1; + } + else + log " Using cached $label $release $arch $item_name:" + log " $ephimg" + fi + + # convert to root.tar.gz + log "Converting mass ephemeral image to roottar" + if [ ! -r ${roottar} ]; then + $dldir/curtin/tools/maas2roottar $ephimg $roottar + [ "$?" != "0" ] && { + log "ERROR: Failed to convert ephemeral to roottar"; + return 1; + } + else + log " Using cached $label $release $arch root.tar.gz" + log " $roottar" + fi + + # unpack rootfs tar + log "Unpacking roottar: $label $release $arch"; + if [ ! -e ${rootfs}/vmlinuz ]; then + mkdir -p ${rootfs} && + sudo tar -C $rootfs -xz --numeric-owner --xattrs -f $roottar + else + log " Using cached $label $release $arch rootfs:" + log " $rootfs" + fi + + _RETVAL=${cachedir} +} + +generate_seed() { + _RETVAL="" + [ $# -lt 2 ] && { + log "ERROR: not enough arguments passed to $FUNCNAME"; + return 1; + } + + local dldir=${1}; + local cachedir=${2}; + local seed=$cachedir/seed/nocloud-net + local installer_user_data=${TOPDIR}/resources/user-data/installer-user-data + + # create curtin payload + log "Generating curtin payload file" + local curtin_cmd=$dldir/curtin-cmd + ( + instcmd="curtin install cp:///" + cd $dldir/curtin + PYTHONPATH=$PYTHONPATH:`pwd` ./bin/curtin pack -- $instcmd > $curtin_cmd + ) + + # inject user-data/meta-data into seed + log "Writing seed meta-data" + mkdir -p ${seed} && write_metadata > $seed/meta-data || { + log "Failed to write meta-data into $seed"; + return 1; + } + log "Writing seed user-data" + # remove the old seed; copy in the base template and + # append the curtin-cmd file + rm -f ${seed}/user-data && + cp $installer_user_data $seed/user-data && + userdata_write_file "/usr/local/bin/curtin" \ + "root:root" "0755" \ + "$curtin_cmd" >> $seed/user-data || { + log "Failed to write user-data into $seed"; + return 1; + } + + return 0 +} + +generate_img() { + _RETVAL="" + [ $# -lt 2 ] && { + log "ERROR: not enough arguments passed to $FUNCNAME"; + return 1; + } + + local dldir=${1}; + local cachedir=${2}; + local extlinux_conf=${TOPDIR}/resources/menu/extlinux-menu.conf + local gptmbr=$(dpkg -L syslinux-common | grep \/gptmbr.bin | grep -v efi) + local installimg=${cachedir}/installer.img + local mnt=${cachedir}/mnt + local rootfs=${cachedir}/rootfs + local seed=$cachedir/seed/nocloud-net + local splash=${TOPDIR}/resources/images/splash.png + local syslinux_path=$(dpkg -L syslinux-common | grep \/vesamenu.c32 | + grep -v efi | xargs -i dirname {}) + + # prep image + log "Generating Installer image file" + qemu-img create -f raw $installimg 2G || { + log "Failed to create empty file: $installimg" + return 1; + } + + log "Partitioning Installer image" + parted -s $installimg mklabel gpt && + parted -s $installimg mkpart primary 2048s 100% || { + log "Failed to partition image: $installimg" + return 1; + } + + [ -z "$gptmbr" ] && { + log "ERROR: failed to find GPT MBR record on host"; + return 1; + } + log "Embedding bootloader" + dd bs=440 conv=notrunc count=1 if=$gptmbr of=$installimg || { + log "Failed to embed bootloader into $installimg"; + return 1; + } + + log "Syncing rootfs into install image" + local kpartx_ret=$(sudo kpartx -va $installimg) + [ -z "$kpartx_ret" ] && { + log "Failed to map image partitions into LVM" + return 1; + } + + local looppart=`echo ${kpartx_ret} | fmt -w 1 | grep ^loop` + DEV="/dev/mapper/${looppart}" + + # frob the bits for gpt boot? + # http://www.funtoo.org/Extlinux + # /dev/loopX is what we want + LOOPDEV="`echo ${kpartx_ret} | fmt -w 1 | grep ^/dev`" + log "Marking partitions bootable on $installimg ($LOOPDEV)" + sudo sgdisk $LOOPDEV --attributes=1:set:2 && + sudo sgdisk $LOOPDEV --attributes=1:show || { + log "ERROR: failed to set bootable partition in $installimg"; + return 1 + } + + log "Creating and syncing filesystem for install image rootfs" + sudo mkfs.ext3 -L cloudimg-rootfs $DEV && + mkdir -p ${mnt} && + sudo mount $DEV ${mnt} && + sudo rsync -a ${rootfs}/ ${mnt}/ || { + log "ERROR: failed to sync rootfs into install image"; + return 1 + } + + + log "Installing bootloader configuration" + set +x + sudo mount none -t proc ${mnt}/proc && + sudo mount none -t sysfs ${mnt}/sys && + sudo mount -o bind /dev ${mnt}/dev && + sudo mkdir -p ${mnt}/boot/extlinux && + sudo extlinux --install ${mnt}/boot/extlinux && + sudo cp -av ${syslinux_path}/*menu* ${mnt}/boot/extlinux && + sudo cp -av ${splash} ${mnt}/boot/extlinux && + cat ${extlinux_conf} | sudo tee ${mnt}/boot/extlinux/extlinux.conf && + sudo mkdir -p ${mnt}/var/lib/cloud/seed && + sudo cp -a ${seed} ${mnt}/var/lib/cloud/seed && sync || { + log "Failed to install bootloader and configuration"; + return 1; + } + set -x + + + _RETVAL="$installimg"; + return 0; +} + +parse_args() { + # -b,--bootloader [syslinux, grub2] + # -h,--help + # -r,--release [trusty, utopic, vivid, wily] + # -v,--verbose + + # args: + [ $# -lt 1 ] && { usage; exit 0; } + + OPTS_LONG="arch:bootloader:,download:help,release:,stream:verbose" + OPTS="a:b:d:h,r:s:v" + ARGS=`getopt --name "$PROG" --long $OPTS_LONG --options $OPTS -- "$@"` + if [ $? -ne 0 ]; then + echo "$PROG: usage error (use -h for help)" >&2 + exit 2 + fi + eval set -- $ARGS + + ARCH="amd64" + BOOTLOADER="syslinux" + DLDIR=~/download + RELEASE="wily" + STREAM="daily" + VERBOSE="no" + while [ $# -gt 0 ]; do + case "$1" in + -a | --arch) ARCH="$2"; shift;; + -b | --bootloader) BOOTLOADER="$2"; shift;; + -d | --download) DLDIR="$2"; shift;; + -h | --help) usage; exit 0;; + -r | --release) RELEASE="$2"; shift;; + -s | --stream) STREAM="$2"; shift;; + -v | --verbose) VERBOSE="yes";; + --) shift; break;; # end of options + esac + shift + done + ARGS="$@" + + [ "${VERBOSE}" == "yes" ] && set -x + + return 0 +} + +main() { + log "INFO: Starting $PROG with params: $@" + + parse_args "$@" + + # get prereqs installed first + install_deps "$PKG_DEPS" || { return 1; } + + [ -z "$OUTPUT" ] && { + OUTPUT="ubuntu-server-${STREAM}-${RELEASE}-${ARCH}-installer.img" + } + + install_src ${DLDIR} ${SRC_DEPS[@]} || { + return 1; + } + + acquire_image ${DLDIR} "$STREAM" "$RELEASE" "$ARCH" || { + return 1; + } + CACHEDIR=${_RETVAL} + log "CACHEDIR=$CACHEDIR" + + generate_seed ${DLDIR} $CACHEDIR || { + return 1; + } + + generate_img ${DLDIR} $CACHEDIR || { + return 1; + } + INSTALLIMG=${_RETVAL} + cleanup_noexit && + mv $INSTALLIMG ${OUTPUT} && + ln -fs ${OUTPUT} installer.img || { + log "ERROR: failed to move $INSTALLIMG to $OUTPUT"; + return 1; + } + log "Installer image complete: $OUTPUT" + + return 0 +} + +main $@ +exit $? diff --git a/installer/resources/images/splash.png b/installer/resources/images/splash.png new file mode 100644 index 00000000..1b842d4c Binary files /dev/null and b/installer/resources/images/splash.png differ diff --git a/installer/resources/menu/extlinux-menu.conf b/installer/resources/menu/extlinux-menu.conf new file mode 100644 index 00000000..9f05b851 --- /dev/null +++ b/installer/resources/menu/extlinux-menu.conf @@ -0,0 +1,12 @@ +SERIAL 0 115200 +ui vesamenu.c32 +prompt 0 +menu background splash.png +menu title Boot Menu +timeout 50 + +label installer + menu label ^Ubuntu Server Installer + menu default + linux /vmlinuz + append initrd=/initrd.img ip=::::myhostname:BOOTIF ro root=LABEL=cloudimg-rootfs overlayroot=tmpfs BOOTIF_DEFAULT=eth0 console=tty0 console=ttyS0 splash diff --git a/installer/resources/user-data/installer-user-data b/installer/resources/user-data/installer-user-data new file mode 100644 index 00000000..a1188f0a --- /dev/null +++ b/installer/resources/user-data/installer-user-data @@ -0,0 +1,46 @@ +#cloud-config +#http_proxy: http://my-proxy:3129/ +password: passw0rd +chpasswd: { expire: False } +output: {all: '| tee -a /var/log/cloud-init-output.log'} +packages: + - python-urwid + - python3-urwid +runcmd: + - cp /usr/share/doc/python-urwid/examples/input_test.py /tmp/installer.py + - chmod +x /tmp/installer.py + - systemctl enable subiquity.service + - systemctl stop serial-getty@ttyS0.service + - /tmp/installer.sh +write_files: +- content: | + #!/bin/bash + + #chvt 2 + systemctl start subiquity + # restart getty service after exiting "installer" + # systemctl stop serial-getty@ttyS0.service + path: /tmp/installer.sh + owner: root:root + permissions: '0755' +- content: | + [Unit] + Description=Ubuntu Servier Installer Service + After=getty@tty2.service + + [Service] + Type=oneshot + ExecStart=/tmp/installer.py + StandardInput=tty-force + StandardOutput=tty + StandardError=tty + TTYPath=/dev/console + TTYReset=yes + TTYVHangup=yes + TTYVTDisallocate=yes + + [Install] + WantedBy=default.target + path: /lib/systemd/system/subiquity.service + owner: root:root + permissions: '0755' diff --git a/installer/runinstaller b/installer/runinstaller new file mode 100755 index 00000000..3462fd48 --- /dev/null +++ b/installer/runinstaller @@ -0,0 +1,32 @@ +#!/bin/bash +# Copyright 2015 Canonical, Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +ARCH=${ARCH-"x86_64"} +MEM=${MEM-"1024"} +INSTALLER=${INSTALLER-"installer.img"} +TARGET=${TARGET-"target.img"} +SPORT=2445 + +[ ! -f ${TARGET} ] && { + qemu-img create -f raw ${TARGET} 10G || exit 1 +} +# TODO, curses should work, but my xmonad setup blocks using it +sudo qemu-system-$ARCH -m $MEM -enable-kvm -hda $INSTALLER -hdb $TARGET \ + -monitor telnet:127.0.0.1:2446,server,nowait -serial stdio +#sudo qemu-system-$ARCH -m $MEM -enable-kvm -hda $INSTALLER -hdb $TARGET \ +# -monitor telnet:127.0.0.1:2446,server,nowait -curses + +exit $? diff --git a/installer/usquery b/installer/usquery new file mode 100755 index 00000000..12c47d2b --- /dev/null +++ b/installer/usquery @@ -0,0 +1,97 @@ +#!/bin/bash +# Copyright 2015 Canonical, Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# +# https://github.com/smoser/talk-simplestreams/blob/master/bin/u-stool + +declare -A sdata +CIU_COM="http://cloud-images.ubuntu.com" +CIU_COM_R="$CIU_COM/releases" +MU_COM="http://maas.ubuntu.com/images/ephemeral-v2/" +CST="https://swift.canonistack.canonical.com/v1/AUTH_a48765cc0e864be980ee21ae26aaaed4" +sdata=( + [uc-release]="$CIU_COM_R/streams/v1/index.sjson" + [uc-aws]="$CIU_COM_R/streams/v1/com.ubuntu.cloud:released:aws.sjson" + [uc-azure]="$CIU_COM_R/streams/v1/com.ubuntu.cloud:released:azure.sjson" + [uc-dl]="$CIU_COM_R/streams/v1/com.ubuntu.cloud:released:download.sjson" + [uc-daily]="$CIU_COM/daily/streams/v1/index.sjson" + [maas-release]="$MU_COM/releases/streams/v1/index.sjson" + [maas-daily]="$MU_COM/daily/streams/v1/index.sjson" + [cirros]="http://download.cirros-cloud.net/streams/v1/index.json" + [cstack]="$CST/simplestreams/data/streams/v1/index.json" + [luc-release]="./luc-release/streams/v1/index.json" + [luc-aws]="./luc-release/streams/v1/com.ubuntu.cloud:released:aws.json" +) + +SPROG="sstream-query" +case "$0" in + *smirror) SPROG="sstream-mirror";; + *squery) SPROG="sstream-query";; + *) + echo "Expect to be called usmirror or usquery, not ${0##*/}"; + exit 1;; +esac + +error() { echo "$@" 1>&2; } +fail() { [ $# -eq 0 ] || error "$@"; exit 1; } +Usage() { + cat <&2; exit 1; } + +keyopt="" +case "$url" in + *.json) :;; + *) keyopt="--keyring=/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg";; +esac + +cmd=( "$SPROG" ${keyopt:+"${keyopt}"} "${opts[@]}" "${args[@]}" ) + +$dry && echo "${cmd[@]}" && exit + +"${cmd[@]}"