Drop python3-parted use

Using parted required root privileges since it opened the underlying device.
Instead create our own Disk model and use sysfs interface for extracting
size information.
This also clears the way for providing device data via probert input.  This
means we can feed subiquity a probert dump and have it present that to
the installer UI even if we're running on a different system.

Signed-off-by: Ryan Harper <ryan.harper@canonical.com>

Makefile

@@ -27,7 +27,7 @@ $(NAME)_$(VERSION).orig.tar.gz: clean
 tarball: $(NAME)_$(VERSION).orig.tar.gz
 
 install_deps:
-	sudo apt-get install python3-urwid python3-pyudev python3-netifaces python3-nose python3-flake8 python3-parted python3-yaml git bzr ubuntu-cloudimage-keyring python3-jinja2 python3-coverage
+	sudo apt-get install python3-urwid python3-pyudev python3-netifaces python3-nose python3-flake8 python3-yaml git bzr ubuntu-cloudimage-keyring python3-jinja2 python3-coverage
 
 dryrun:
 	$(MAKE) ui-view DRYRUN="--dry-run"

installer/geninstaller

@@ -35,7 +35,6 @@ SRC_DEPS=(
 )
 INSTALLER_DEPS=(
   "petname"
-  "python3-parted"
   "python3-urwid"
   "python3-pyudev"
   "python3-netifaces"

subiquity/controllers/filesystem.py

@@ -106,6 +106,7 @@ class FilesystemController(ControllerPolicy):
                 # adjust downward the partition size to accommodate
                 # the bios/grub partition
                 spec['bytes'] -= BIOS_GRUB_SIZE_BYTES
+                spec['partnum'] = 2
 
             if spec["fstype"] in ["swap"]:
                 current_disk.add_partition(partnum=spec["partnum"],

subiquity/models/actions.py

@@ -17,12 +17,13 @@ import yaml
 
 
 class DiskAction():
-    def __init__(self, action_id, model, serial, ptable='gpt'):
+    def __init__(self, action_id, model, serial, ptable='gpt', wipe=None):
         self._action_id = action_id
         self.parent = None
         self._ptable = ptable
         self._model = model
         self._serial = serial
+        self._wipe = wipe
 
     def get_parent(self):
         return self.parent
@@ -32,42 +33,55 @@ class DiskAction():
         return str(self._action_id)
 
     def get(self):
-        return {
+        action = {
             'id': self.action_id,
             'model': self._model,
             'ptable': self._ptable,
             'serial': self._serial,
             'type': 'disk',
         }
+        if self._wipe:
+            action.update({'wipe': self._wipe})
+        return action
 
     def dump(self):
         return yaml.dump(self.get(), default_flow_style=False)
 
 
 class PartitionAction(DiskAction):
-    def __init__(self, parent, partnumber, size, flags=None):
+    def __init__(self, parent, partnum, offset, size, flags=None):
         self.parent = parent
-        self.partnumber = int(partnumber)
+        self.partnum = int(partnum)
+        self._offset = int(offset)
         self._size = int(size)
         self.flags = flags
         self._action_id = "{}{}_part".format(self.parent.action_id,
-                                             self.partnumber)
+                                             self.partnum)
 
         ''' rename action_id for readability '''
         if self.flags in ['bios_grub']:
             self._action_id = 'bios_boot_partition'
 
+    @property
+    def path(self):
+        return "{}{}".format(self.parent.action_id, self.partnum)
+
     @property
     def size(self):
         return self._size
 
+    @property
+    def offset(self):
+        return self._offset
+
     def get(self):
         return {
             'device': self.parent.action_id,
             'flag': self.flags,
             'id': self.action_id,
-            'number': self.partnumber,
+            'number': self.partnum,
             'size': '{}B'.format(self.size),
+            'offset': '{}B'.format(self.offset),
             'type': 'partition',
         }
 

subiquity/models/blockdev.py

@@ -13,10 +13,10 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from collections import OrderedDict
 from itertools import count
 import logging
 import os
-import parted
 import re
 import yaml
 
@@ -57,45 +57,82 @@ class Bcachedev():
         return self._path
 
     def getSize(self, unit='MB'):
-        if type(self._backing) == parted.disk.Disk:
+        pass
-            return self._backing.device.getSize(unit=unit)
+
-        else:
+
-            return self._backing.getSize(unit=unit)
+class Disk():
+    def __init__(self, devpath, serial, model, parttype, size=0):
+        self._devpath = devpath
+        self._serial = serial
+        self._parttype = parttype
+        self._model = model
+        self._size = self._get_size(devpath, size)
+        self._partitions = OrderedDict()
+
+    def _get_size(self, devpath, size):
+        if size:
+            return size
+        sysblock = os.path.join('/sys/block', os.path.basename(devpath))
+        nr_blocks_f = os.path.join(sysblock, 'size')
+        block_sz_f = os.path.join(sysblock, 'queue', 'logical_block_size')
+        with open(nr_blocks_f, 'r') as r:
+            nr_blocks = int(r.read())
+        with open(block_sz_f, 'r') as r:
+            block_sz = int(r.read())
+
+        return nr_blocks * block_sz
+
+    @property
+    def devpath(self):
+        return self._devpath
+
+    @property
+    def serial(self):
+        return self._serial
+
+    @property
+    def model(self):
+        return self._model
+
+    @property
+    def parttype(self):
+        return self._parttype
+
+    @property
+    def size(self):
+        return self._size
+
+    @property
+    def partitions(self):
+        return self._partitions
+
+    def reset(self):
+        self._partitions = OrderedDict()
+        pass
 
 
 class Blockdev():
-    def __init__(self, devpath, serial, parttype='gpt'):
+    def __init__(self, devpath, serial, model, parttype='gpt', size=0):
-        self.serial = serial
+        self.disk = Disk(devpath, serial, model, parttype, size)
-        self.devpath = devpath
+        self._filesystems = {}
-        self._parttype = parttype
-        self.device = parted.getDevice(self.devpath)
-        self.disk = parted.freshDisk(self.device, self.parttype)
         self._mounts = {}
         self.bcache = []
         self.lvm = []
+        self.baseaction = DiskAction(os.path.basename(self.disk.devpath),
+                                     self.disk.model, self.disk.serial,
+                                     self.disk.parttype)
 
     def reset(self):
         ''' Wipe out any actions queued for this disk '''
-        self.disk = parted.freshDisk(self.device, self.parttype)
+        self.disk.reset()
+        self._filesystems = {}
         self._mounts = {}
         self.bcache = []
         self.lvm = []
 
-    def _get_largest_free_region(self):
+    @property
-        """Finds largest free region on the disk"""
+    def devpath(self):
-        # There are better ways to do it, but let's be straightforward
+        return self.disk.devpath
-        max_size = -1
-        region = None
-
-        alignment = self.device.optimumAlignment
-
-        for r in self.disk.getFreeSpaceRegions():
-            # Heuristic: Ignore alignment gaps
-            if r.length > max_size and r.length > alignment.grainSize:
-                region = r
-                max_size = r.length
-
-        return region
 
     @property
     def mounts(self):
@@ -103,7 +140,7 @@ class Blockdev():
 
     @property
     def parttype(self):
-        return self._parttype
+        return self.disk.parttype
 
     @parttype.setter  # NOQA
     def parttype(self, value):
@@ -111,12 +148,16 @@ class Blockdev():
 
     @property
     def size(self):
-        return self.disk.device.getLength(unit='B')
+        return self.disk.size
 
     @property
     def partitions(self):
         return self.disk.partitions
 
+    @property
+    def filesystems(self):
+        return self._filesystems
+
     @property
     def available(self):
         ''' return True if has free space or partitions not
@@ -128,27 +169,27 @@ class Blockdev():
     @property
     def usedspace(self, unit='b'):
         ''' return amount of used space'''
-        return sum([part.geometry.getSize(unit=unit) for part in
+        space = 0
-                    self.disk.partitions])
+        for (num, action) in self.disk.partitions.items():
+            space += int(action.offset)
+            space += int(action.size)
+
+        log.debug('{} usedspace: {}'.format(self.disk.devpath, space))
+        return space
 
     @property
     def freespace(self, unit='B'):
         ''' return amount of free space '''
-        geo = self._get_largest_free_region()
+        used = self.usedspace
-        if geo:
+        size = self.size
-            return geo.getLength(unit=unit)
+        log.debug('{} freespace: {} - {} = {}'.format(self.disk.devpath,
-        return 0
+                                                      size, used,
-
+                                                      size - used))
-    @property
+        return size - used
-    def freepartition(self, unit='b'):
-        ''' return amount of partitionable space'''
-        return sum([part.geometry.getSize(unit=unit) for part in
-                    self.disk.getFreeSpacePartitions()])
 
     @property
     def lastpartnumber(self):
-        return self.disk.lastPartitionNumber if \
+        return len(self.disk.partitions)
-            self.disk.lastPartitionNumber > 0 else 0
 
     def delete_partition(self, partnum=None, sector=None, mountpoint=None):
         # find part and then call deletePartition()
@@ -161,77 +202,53 @@ class Blockdev():
                   ' partnum:%s size:%s fstype:%s mountpoint:%s flag=%s' % (
                       partnum, size, fstype, mountpoint, flag))
 
-        if size > self.freepartition:
+        if size > self.freespace:
             raise Exception('Not enough space')
 
         if fstype in ["swap"]:
             fstype = "linux-swap(v1)"
 
-        geometry = self._get_largest_free_region()
+        if len(self.disk.partitions) == 0:
-        if not geometry:
+            offset = 1 << 20  # 1K offset/aligned
-            raise Exception('No free sectors available')
+            size += offset
-        log.debug('largest free region:\n{}'.format(geometry))
-
-        # convert size into a geometry based on existing partitions
-        try:
-            start = self.disk.partitions[-1].geometry.end + 1
-        except IndexError:
-            start = 0
-        length = parted.sizeToSectors(size, 'B', self.device.sectorSize)
-        log.debug('requested start: {} length: {}'.format(start, length))
-        req_geo = parted.Geometry(self.device, start=start, length=length)
-
-        # find common area
-        parttype = parted.PARTITION_NORMAL
-        alignment = self.device.optimalAlignedConstraint
-        geometry = geometry.intersect(req_geo)
-        # update geometry with alignment
-        constraint = parted.Constraint(maxGeom=geometry).intersect(alignment)
-        data = {
-            'start': constraint.startAlign.alignUp(geometry, geometry.start),
-            'end': constraint.endAlign.alignDown(geometry, geometry.end),
-        }
-        geometry = parted.Geometry(device=self.device,
-                                   start=data['start'],
-                                   end=data['end'])
-        # create partition
-        if fstype not in [None, 'bcache cache', 'bcache store']:
-            fs = parted.FileSystem(type=fstype, geometry=geometry)
         else:
-            fs = None
+            offset = 0
-        partition = parted.Partition(disk=self.disk, type=parttype,
-                                     fs=fs, geometry=geometry)
 
-        # add flags
+        log.debug('requested start: {} length: {}'.format(offset, size))
-        flags = {
+        valid_flags = [
-            "boot": parted.PARTITION_BOOT,
+            "boot",
-            "lvm": parted.PARTITION_LVM,
+            "lvm",
-            "raid": parted.PARTITION_RAID,
+            "raid",
-            "bios_grub": parted.PARTITION_BIOS_GRUB
+            "bios_grub",
-        }
+        ]
-        if flag in flags:
+        if flag and flag not in valid_flags:
-            partition.setFlag(flags[flag])
+            raise Exception('Flag: {} is not valid.'.format(flag))
 
-        self.disk.addPartition(partition=partition, constraint=constraint)
+        # create partition and add
+        part_action = PartitionAction(self.baseaction, partnum,
+                                      offset, size, flag)
+        log.debug('PartitionAction:\n{}'.format(part_action))
 
-        # fetch the newly created partition
+        self.disk.partitions.update({partnum: part_action})
-        partpath = "{}{}".format(self.disk.device.path, partition.number)
-        newpart = self.disk.getPartitionByPath(partpath)
 
-        # create bcachedev if neded
+        # record filesystem formating
-        if fstype and fstype.startswith('bcache'):
+        if fstype:
-            mode = fstype.split()[-1]
+            partpath = "{}{}".format(self.disk.devpath, partnum)
-            self.bcache.append(Bcachedev(backing=newpart, mode=mode))
+            fs_action = FormatAction(part_action, fstype)
+            log.debug('Adding filesystem: {}:{}'.format(partpath, fs_action))
+            self.filesystems.update({partpath: fs_action})
 
         # associate partition devpath with mountpoint
         if mountpoint:
             self._mounts[partpath] = mountpoint
 
+        log.debug('Partition Added')
+
     def is_mounted(self):
         with open('/proc/mounts') as pm:
             mounts = pm.read()
 
-        regexp = '{}.*'.format(self.disk.device.path)
+        regexp = '{}.*'.format(self.disk.devpath)
         matches = re.findall(regexp, mounts)
         if len(matches) > 0:
             log.debug('Device is mounted: {}'.format(matches))
@@ -245,43 +262,31 @@ class Blockdev():
             return []
 
         actions = []
-        baseaction = DiskAction(os.path.basename(self.disk.device.path),
+        action = self.baseaction.get()
-                                self.device.model, self.serial, self.parttype)
+        for (num, part) in self.disk.partitions.items():
-        action = baseaction.get()
+            partpath = "{}{}".format(self.disk.devpath, part.partnum)
-        for part in self.disk.partitions:
+            actions.append(part)
-            fs_size = int(part.getSize(unit='B'))
+            if partpath in self.filesystems:
-            if part.fileSystem:
+                format_action = self.filesystems[partpath]
-                fs_type = part.fileSystem.type
-            else:
-                fs_type = None
-            flags = part.getFlagsAsString()
-
-            partition_action = PartitionAction(baseaction,
-                                               part.number,
-                                               fs_size, flags)
-            actions.append(partition_action)
-            if fs_type:
-                format_action = FormatAction(partition_action,
-                                             fs_type)
                 actions.append(format_action)
-                mountpoint = self._mounts.get(part.path)
+
-                if mountpoint:
+            if partpath in self._mounts:
-                    mount_action = MountAction(format_action, mountpoint)
+                mount_action = MountAction(format_action,
-                    actions.append(mount_action)
+                                           self._mounts[partpath])
+                actions.append(mount_action)
 
         return [action] + [a.get() for a in actions]
 
     def get_fs_table(self):
         ''' list(mountpoint, humansize, fstype, partition_path) '''
         fs_table = []
-        for part in self.disk.partitions:
+        for (num, part) in self.disk.partitions.items():
-            if part.fileSystem:
+            partpath = "{}{}".format(self.disk.devpath, part.partnum)
-                mntpoint = self._mounts.get(part.path, part.fileSystem.type)
+            if partpath in self.filesystems:
-                fs_size = part.getSize(unit='B')
+                fs = self.filesystems[partpath]
-                fs_type = part.fileSystem.type
+                mntpoint = self._mounts.get(partpath, fs.fstype)
-                devpath = part.path
                 fs_table.append(
-                    (mntpoint, fs_size, fs_type, devpath))
+                    (mntpoint, part.size, fs.fstype, partpath))
 
         return fs_table
 
@@ -297,11 +302,17 @@ if __name__ == '__main__':
         print("USED DISKS")
 
     devices = []
-    sda = Blockdev('/dev/sda', 'QM_TARGET_01', parttype='gpt')
+    #Blockdev(devpath, serial, model, parttype='gpt'):
-    sdb = Blockdev('/dev/sdb', 'dafunk')
+    GB = 1 << 30
+    sda = Blockdev('/dev/sda', 'QM_TARGET_01', 'QEMU SSD DISK',
+                   parttype='gpt', size=128 * GB)
+    sdb = Blockdev('/dev/sdb', 'dafunk', 'QEMU SPINNER', size=500 * GB)
 
+    print(sda.freespace)
     sda.add_partition(1, 8 * 1024 * 1024 * 1024, 'ext4', '/', 'bios_grub')
+    print(sda.freespace)
     sda.add_partition(2, 2 * 1024 * 1024 * 1024, 'ext4', '/home')
+    print(sda.freespace)
     sdb.add_partition(1, 50 * 1024 * 1024 * 1024, 'btrfs', '/opt')
 
     get_filesystems([sda, sdb])

subiquity/models/filesystem.py

@@ -134,13 +134,15 @@ class FilesystemModel(ModelPolicy):
 
     def get_disk(self, disk):
         if disk not in self.devices:
-            self.devices[disk] = Blockdev(disk, self.info[disk].serial)
+            self.devices[disk] = Blockdev(disk, self.info[disk].serial,
+                                          self.info[disk].model)
         return self.devices[disk]
 
     def get_partitions(self):
         partitions = []
         for dev in self.devices.values():
-            partnames = [part.path for part in dev.disk.partitions]
+            partnames = [part.path for (num, part) in
+                         dev.disk.partitions.items()]
             partitions += partnames
 
         sorted(partitions)