#!/bin/bash # Copyright 2015 Canonical, Ltd. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as # published by the Free Software Foundation, either version 3 of the # License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . PROG=`basename $0` PREFIX="${PROG}" LOGFILE="geninstaller.log" TOPDIR=${TOPDIR-/usr/share/subiquity/installer} USQUERY=${TOPDIR}/usquery RESOURCES=${TOPDIR}/resources PKG_DEPS=" qemu-utils kpartx parted gdisk extlinux simplestreams syslinux-common grub2-common shim shim-signed grub-efi-amd64-signed probert " # "URL" "GPGKEY" # ppa:foo/bar "" PPAS=( "https://raharper:Q9F9bRlSxg70BGv8m6dc@private-ppa.launchpad.net/subiquity/subiquity-dev/ubuntu" "3D2F6C3B" ) SRC_DEPS=( "bzr" "lp:curtin" "curtin" ) INSTALLER_DEPS=( "petname" "python3-urwid" "python3-pyudev" "python3-netifaces" "python-urwid" "python3-tornado" "probert" ) CACHEDIR="" GRUB_MODS="configfile fat part_gpt part_msdos cat echo test search search_label search_fs_uuid boot chain linux reboot halt normal efi_gop efi_uga font gfxterm gfxterm_menu gfxterm_background gfxmenu serial" cleanup_noexit() { [ -n "${CACHEDIR}" ] && { sync sudo umount -l ${CACHEDIR}/mnt/{dev,proc,sys} sudo umount -l ${CACHEDIR}/mnt sudo umount -l ${CACHEDIR}/lower sudo umount -l ${CACHEDIR}/upper sudo umount -l ${CACHEDIR}/efimnt sudo kpartx -d ${CACHEDIR}/installer.img &>/dev/null || exit for DEV in $EFI_DEV $ROOTFS_DEV $OVERLAY_DEV; do [ -e "/dev/mapper/`basename $DEV`" ] && { sudo dmsetup remove $DEV &>/dev/null || exit } done # it's ok to fail here, it means kpartx did it for us sudo losetup -d $LOOPDEV &>/dev/null | : } } cleanup() { cleanup_noexit &>/dev/null exit } trap cleanup EXIT HUP INT TERM log() { echo "`date +%s`: $@" | tee -a ${LOGFILE} } write_metadata() { cat </dev/null; then to_install="$to_install $p"; fi done [ -n "$to_install" ] && { log "Installing dependencies: $to_install"; sudo apt-get install -q -y $to_install || { log "Failed to install one or more dependencies in $to_install"; return 1; } } return 0 } install_src() { local dldir=${1}; shift; mkdir -p ${dldir} || { log "ERROR: failed to mkdir $dldir"; return 1; } log "Acquiring src packages..." while [ $# -gt 0 ]; do local proto=${1}; shift local url=${1}; shift local localdir=${1}; shift [ -z "${proto}" -o -z "${url}" ] && { log "ERROR installing source with args: $@" return 1; } local target="$dldir/$localdir" case "$proto" in git) cmd="git clone $url $target";; bzr) cmd="bzr branch $url $target";; *) log "ERROR: unsupported src protocol: $proto $url";; esac if [ ! -d "${target}" ]; then log "Acquiring src @ $url with $proto" $cmd || { log "ERROR: failed to fetch src: $proto $url into $target"; return 1; } else log " Using cached src for repo $url @ $target" fi done } acquire_image() { _RETVAL="" [ $# -lt 1 ] && { log "ERROR: not enough arguments passed to $FUNCNAME"; return 1; } # what to get from maas local item_name="root-image.gz" local dldir=${1}; shift; local label=${1:-"daily"} local release=${2-"wily"}; local arch=${3-"amd64"}; local version=${4}; # run a query unless they specify all params if [ $# -le 4 -a -z "${version}" ]; then log "Querying simplestreams for latest image: $label $release $arch" case "$label" in daily) ssresult=( `${USQUERY} --output-format="%(version_name)s %(item_url)s %(sha256)s" --max=1 maas-daily release=$release arch=$arch item_name=$item_name` ) local version=${ssresult[0]} local item_url=${ssresult[1]} local sha256=${ssresult[2]} ;; release) ssresult=( `${USQUERY} --output-format="%(version_name)s %(item_url)s %(sha256)s" --max=1 maas-release release=$release arch=$arch item_name=$item_name` ) local version=${ssresult[0]} local item_url=${ssresult[1]} local sha256=${ssresult[2]} ;; *) log "ERROR: simplestream label must be one of: [daily, release]" return 1; ;; esac fi local cachedir=$dldir/maas/${label}/${release}/${arch}/${version} local ephimg=${cachedir}/${item_name} local roottar=${cachedir}/root.tar.gz local rootfs=${cachedir}/rootfs # cache policy: # $ephimg must be a file and it must checksum to $sha256 value otherwise # we will nuke the cachedir and reacquire $item_name @ $item_url # and re-assemble roottar and rootfs which was based on ephimg [ -r "$ephimg" ] && CACHE_SUM=( `sha256sum $ephimg 2>/dev/null` ) if [ -z "${sha256}" -a -f "${ephimg}.sha256" ]; then log "Using sha256sum from cached file"; sha256=`cat ${ephimg}.sha256 | cut -d' ' -f1` fi if [ "${sha256}" != "${CACHE_SUM[0]}" ]; then log "WARNING: sha256 csum mismatch" log "WARNING: expected: [$sha256]" log " found: [${CACHE_SUM[0]}]" # didn't match so nuke the cache log "WARNING: removing old cache" sudo rm -fr "${cachedir}" || { log "ERROR: failed to remove stale cachedir: $cachedir"; return 1; } fi # download the image if it's not in the cache log "Downloading installer root image @ $item_url" if [ ! -r "${ephimg}" ]; then mkdir -p $cachedir && wget --progress=bar -c "${item_url}" -O "${ephimg}" || { log "ERROR: failed to download: ${item_url}"; return 1; } (cd `dirname ${ephimg}` && sha256sum `basename ${ephimg}` > ${ephimg}.sha256) else log " Using cached $label $release $arch $item_name:" log " $ephimg" fi # convert to root.tar.gz log "Converting mass ephemeral image to roottar" if [ ! -r ${roottar} ]; then $dldir/curtin/tools/maas2roottar $ephimg $roottar [ "$?" != "0" ] && { log "ERROR: Failed to convert ephemeral to roottar"; return 1; } else log " Using cached $label $release $arch root.tar.gz" log " $roottar" fi # unpack rootfs tar log "Unpacking roottar: $label $release $arch"; if [ ! -e ${rootfs}/vmlinuz ]; then mkdir -p ${rootfs} && sudo tar -C $rootfs -xz --numeric-owner --xattrs -f $roottar else log " Using cached $label $release $arch rootfs:" log " $rootfs" fi _RETVAL=${cachedir} } generate_seed() { _RETVAL="" [ $# -lt 3 ] && { log "ERROR: not enough arguments passed to $FUNCNAME"; return 1; } local dldir=${1}; local cachedir=${2}; local seed=$cachedir/seed/nocloud-net local installer_user_data=${TOPDIR}/resources/user-data/installer-user-data # create curtin payload log "Generating curtin payload file" local curtin_cmd=$dldir/curtin-cmd ( instcmd="curtin install cp:///" cd $dldir/curtin PYTHONPATH=`pwd` ./bin/curtin pack -- $instcmd > $curtin_cmd ) || { log "ERROR: failed to pack curtin installer"; return 1; } # inject user-data/meta-data into seed log "Writing seed meta-data" mkdir -p ${seed} && write_metadata > $seed/meta-data || { log "Failed to write meta-data into $seed"; return 1; } log "Writing seed user-data (curtin)" # remove the old seed; copy in the base template and # append the curtin-cmd file rm -f ${seed}/user-data && cp $installer_user_data $seed/user-data && if [ "${OFFLINE}" == "no" ]; then log "Enabling cloud-init package installation" local packages="" for pkg in ${INSTALLER_DEPS[@]}; do packages="$packages - $pkg\n" done sed -i "s/#packages/packages:\n$packages/" ${seed}/user-data fi userdata_write_file "/usr/local/bin/curtin-archive" \ "root:root" "0755" "none" \ "$curtin_cmd" >> $seed/user-data || { log "Failed to embed curtin into $seed"; return 1; } log "Writing seed user-data (subiquity)" local subiquity_tar=$dldir/subiquity.tar local tar_cmd="tar -cpf $subiquity_tar subiquity bin" if [[ ${TOPDIR} = /usr/share/subiquity* ]]; then log "Using installed subiquity paths" tar_cmd="$tar_cmd subiquity-tui" fi log "subiquity_tar cmd: cd ${TOPDIR}/.. && ${tar_cmd}" (cd ${TOPDIR}/.. && $tarcmd) || { log "ERROR: Failed to package subiquity installer"; return 1; } userdata_write_file "/tmp/subiquity.tar" \ "root:root" "0644" "b64" \ "$subiquity_tar" >> $seed/user-data || { log "Failed to subiquity into $seed"; return 1; } return 0 } generate_img() { _RETVAL="" [ $# -lt 4 ] && { log "ERROR: not enough arguments passed to $FUNCNAME"; return 1; } local dldir=${1}; local cachedir=${2}; local bootloader=${3}; local offline=${4}; local extlinux_conf=${TOPDIR}/resources/menu/extlinux-menu.conf local overlay_path=${TOPDIR}/resources/overlay local grub_efi_core=${TOPDIR}/resources/grub/bootx64.efi # FIXME, ARCH local grub_conf=${TOPDIR}/resources/grub/grub.cfg local embed_conf=${TOPDIR}/resources/grub/embed_efi.cfg local efi_grub_conf=${TOPDIR}/resources/grub/efi_grub.cfg local gptmbr=$(dpkg -L syslinux-common | grep \/gptmbr.bin | grep -v efi) local installimg=${cachedir}/installer.img local mnt=${cachedir}/mnt local rootfs=${cachedir}/rootfs local efimnt=${cachedir}/efimnt local lower=${cachedir}/lower local upper=${cachedir}/upper local work=${cachedir}/upper/overlay-work local seed=$cachedir/seed/nocloud-net local splash=${TOPDIR}/resources/images/splash.png local syslinux_path=$(dpkg -L syslinux-common | grep \/vesamenu.c32 | grep -v efi | xargs -i dirname {}) # prep image log "Generating Installer image file" qemu-img create -f raw $installimg 2G 2>&1 >> ${LOGFILE} || { log "Failed to create empty file: $installimg" return 1; } log "Partitioning Installer image (UEFI)" (parted -s $installimg mklabel msdos && parted -s $installimg mkpart primary fat32 0% 200M && parted -s $installimg mkpart primary ext3 200M 1700M && parted -s $installimg mkpart primary ext3 1700M 2147M && parted -s $installimg set 1 boot on) 2>&1 >> ${LOGFILE} || { log "Failed to partition image: $installimg" return 1; } if [ "${bootloader}" != "grub2" ]; then log "Bootloader ${bootloader} not supported, cannot install" usage; return 1; fi log "Syncing rootfs into install image" local kpartx_ret=$(sudo kpartx -va $installimg) [ -z "$kpartx_ret" ] && { log "Failed to map image partitions into LVM" return 1; } sudo udevadm settle local loopparts=( `echo ${kpartx_ret} | fmt -w 1 | grep ^loop` ) EFI_DEV="/dev/mapper/${loopparts[0]}" ROOTFS_DEV="/dev/mapper/${loopparts[1]}" OVERLAY_DEV="/dev/mapper/${loopparts[2]}" LOOPDEV="`echo ${kpartx_ret} | fmt -w 1 | grep ^/dev | head -n1`" log "Building Grub (EFI and BIOS) boot partition" (sudo mkfs.vfat -F32 -n GRUB2EFI ${EFI_DEV} 2>/dev/null && mkdir -p ${efimnt} && sudo mount $EFI_DEV ${efimnt} && sudo mkdir -p ${efimnt}/EFI/BOOT && sudo mkdir -p ${efimnt}/boot/grub && sudo mkdir -p ${efimnt}/grub/fonts && sudo mkdir -p ${efimnt}/grub/x86_64-efi && sudo cp -a /usr/lib/shim/MokManager.efi.signed ${efimnt}/EFI/BOOT/MokManager.efi && sudo cp -a /usr/lib/shim/shim.efi.signed ${efimnt}/EFI/BOOT/shimx64.efi && sudo cp -a /usr/lib/grub/x86_64-efi-signed/grubx64.efi.signed ${efimnt}/EFI/BOOT/grubx64.efi && sudo cp -a /usr/share/grub/unicode.pf2 ${efimnt}/grub/fonts/ && sudo cp -a /usr/share/grub/unicode.pf2 ${efimnt}/boot/grub && cat ${efi_grub_conf} | sudo tee ${efimnt}/EFI/BOOT/grub.cfg && cat ${embed_conf} | sudo tee ${efimnt}/grub/embed_efi.cfg && cat ${grub_conf} | sudo tee ${efimnt}/grub/grub.cfg && sudo rsync -a /usr/lib/grub/x86_64-efi/ ${efimnt}/grub/x86_64-efi/ && sudo grub-mkimage -O x86_64-efi -p /grub -c ${efimnt}/grub/embed_efi.cfg -o ${efimnt}/EFI/BOOT/bootx64.efi ${GRUB_MODS} && sudo grub-install --force --removable --no-floppy \ --boot-directory=${efimnt}/boot $LOOPDEV &>/dev/null && sudo cp -v ${splash} ${efimnt}/boot/grub) 2>&1 >> ${LOGFILE} || { log "ERROR: failed to create multiboot partition" return 1 } log "Creating and syncing filesystem (original cloudimg rootfs)" (sudo mkfs.ext3 -q -L cloudimg-rootfs $ROOTFS_DEV && mkdir -p ${lower} && sudo mount $ROOTFS_DEV ${lower} && sudo rsync -a ${rootfs}/ ${lower}/) 2>&1 >> ${LOGFILE} || { log "ERROR: failed to sync rootfs into install image"; return 1 } log "Creating and syncing filesystem (installer overlay)" # mount -t overlay overlay -olowerdir=/lower,upperdir=/upper,\ # workdir=/work /merged OVERLAY_VERSION=( `modinfo -V overlayfs` ) # returns: kmod version XX if [ ${OVERLAY_VERSION[2]} -le 15 ]; then # trusty 3.13 overlayfs version 15 or older doesn't use workdir FS=overlayfs OPTS="-olowerdir=$lower,upperdir=$upper/overlay" else # newer kernels than trusty 3.13 have workdir FS=overlay OPTS="-olowerdir=$lower,upperdir=$upper/overlay,workdir=$work" fi # load the right overlay module if ! lsmod | grep -q ${FS}; then sudo modprobe -v $FS; fi (sudo mkfs.ext3 -q -L overlay-rootfs $OVERLAY_DEV sudo mkdir -p ${work} ${upper} ${mnt} && sudo mount $OVERLAY_DEV ${upper} && sudo mkdir -p ${upper}/overlay ${work} && sudo mount -t $FS $FS $OPTS ${mnt}) 2>&1 >> ${LOGFILE} || { log "ERROR: failed to overlay mount installer"; return 1 } log "Installing bootloader configuration" (sudo mkdir -p ${mnt}/proc sudo mkdir -p ${mnt}/sys sudo mkdir -p ${mnt}/dev sudo mount none -t proc ${mnt}/proc && sudo mount none -t sysfs ${mnt}/sys && sudo mount -o bind /dev ${mnt}/dev) 2>&1 >> ${LOGFILE} || { log "ERROR: failed to prepare target mounts for sync"; return 1; } if [ "${offline}" == "yes" ]; then log "Setting up for offline use" local resolvconf=${mnt}/etc/resolv.conf local packages="" for installer_package in "${INSTALLER_DEPS[@]}"; do packages="$packages $installer_package" done sudo mv ${resolvconf} ${resolvconf}.old && sudo cp /etc/resolv.conf ${resolvconf} && log "Installing ppas in rootfs" # export existing install_ppa and run it in the chroot install_ppas_cmds="$(install_ppas ${PPAS[@]})" sudo chroot ${mnt} /bin/bash -c "${install_ppas_cmds}" || { log "Failed to add installer ppas to chroot"; return 1; } log "Installing on rootfs: $packages" sudo chroot ${mnt} apt-get update && sudo chroot ${mnt} apt-get -y install $packages || { log "Failed to install packages on rootfs"; return 1; } log "Removing ppas in rootfs" remove_ppas_cmds="$(remove_ppas ${PPAS[@]})" sudo chroot ${mnt} /bin/bash -c "${remove_ppas_cmds}" || { log "Failed to remove installer ppas from chroot"; return 1; } sudo rm ${resolvconf} sudo mv ${resolvconf}.old ${resolvconf} fi if [ "${bootloader}" == "syslinux" ]; then sudo mkdir -p ${mnt}/boot/extlinux && sudo extlinux --install ${mnt}/boot/extlinux && sudo cp -av ${syslinux_path}/*.c32 ${mnt}/boot/extlinux && sudo cp -av ${splash} ${mnt}/boot/extlinux && cat ${extlinux_conf} | sudo tee ${mnt}/boot/extlinux/extlinux.conf else log "Installing grub2" cat ${grub_conf} | sudo tee ${efimnt}/boot/grub/grub.cfg 2>&1 >> ${LOGFILE} && true fi # syncing overlay log "Injecting installer configuration/scripts" sudo rsync -a ${overlay_path}/ ${mnt}/ || { log "Failed to sync local installer configuration/scripts"; return 1; } log "Installing cloud seed" sudo mkdir -p ${mnt}/var/lib/cloud/seed && sudo cp -a ${seed} ${mnt}/var/lib/cloud/seed && sync || { log "Failed to install bootloader and configuration"; return 1; } _RETVAL="$installimg"; return 0; } parse_args() { # -b,--bootloader [syslinux, grub2] # -h,--help # -r,--release [trusty, utopic, vivid, wily] # -v,--verbose # args: [ $# -lt 1 ] && { usage; exit 0; } OPTS_LONG="arch:,bootloader:,download:,help,offline,release:,stream:,verbose,version:" OPTS="a:b:d:hor:s:vV:" ARGS=`getopt --name "$PROG" --long $OPTS_LONG --options $OPTS -- "$@"` if [ $? -ne 0 ]; then echo "$PROG: usage error (use -h for help)" >&2 exit 2 fi eval set -- $ARGS ARCH="amd64" BOOTLOADER="syslinux" DLDIR=~/download OFFLINE="no" RELEASE="wily" STREAM="daily" VERBOSE="no" VERSION="" while [ $# -gt 0 ]; do case "$1" in -a | --arch) ARCH="$2"; shift;; -b | --bootloader) BOOTLOADER="$2"; shift;; -d | --download) DLDIR="$2"; shift;; -h | --help) usage; exit 0;; -o | --offline) OFFLINE="yes";; -r | --release) RELEASE="$2"; shift;; -s | --stream) STREAM="$2"; shift;; -V | --version) VERSION="$2"; shift;; -v | --verbose) VERBOSE="yes";; --) shift; break;; # end of options esac shift done ARGS="$@" [ "${VERBOSE}" == "yes" ] && set -x return 0 } main() { log "INFO: Starting $PROG with params: $@" parse_args "$@" # get prereqs installed first install_deps "$PKG_DEPS" || { return 1; } [ -z "$OUTPUT" ] && { OUTPUT="ubuntu-server-${STREAM}-${RELEASE}-${ARCH}-installer.img" } install_src ${DLDIR} ${SRC_DEPS[@]} || { return 1; } acquire_image ${DLDIR} "$STREAM" "$RELEASE" "$ARCH" "$VERSION" || { return 1; } CACHEDIR=${_RETVAL} log "CACHEDIR=$CACHEDIR" generate_seed ${DLDIR} $CACHEDIR $OFFLINE || { return 1; } generate_img ${DLDIR} $CACHEDIR $BOOTLOADER $OFFLINE || { return 1; } INSTALLIMG=${_RETVAL} log "Cleaning up ..." cleanup_noexit && mv $INSTALLIMG ${OUTPUT} && ln -fs ${OUTPUT} installer.img || { log "ERROR: failed to move $INSTALLIMG to $OUTPUT"; return 1; } log "Installer image complete: $OUTPUT" return 0 } main $@ exit $?