Merge pull request #30 from CanonicalLtd/drop_pyparted

Drop python3-parted use

Makefile

@@ -27,7 +27,7 @@ $(NAME)_$(VERSION).orig.tar.gz: clean
 tarball: $(NAME)_$(VERSION).orig.tar.gz
 
 install_deps:
-	sudo apt-get install python3-urwid python3-pyudev python3-netifaces python3-nose python3-flake8 python3-parted python3-yaml git bzr ubuntu-cloudimage-keyring python3-jinja2 python3-coverage
+	sudo apt-get install python3-urwid python3-pyudev python3-netifaces python3-nose python3-flake8 python3-yaml git bzr ubuntu-cloudimage-keyring python3-jinja2 python3-coverage
 
 dryrun:
 	$(MAKE) ui-view DRYRUN="--dry-run"

installer/geninstaller

@@ -35,7 +35,6 @@ SRC_DEPS=(
 )
 INSTALLER_DEPS=(
   "petname"
-  "python3-parted"
   "python3-urwid"
   "python3-pyudev"
   "python3-netifaces"

subiquity/controllers/filesystem.py

@@ -106,6 +106,7 @@ class FilesystemController(ControllerPolicy):
                 # adjust downward the partition size to accommodate
                 # the bios/grub partition
                 spec['bytes'] -= BIOS_GRUB_SIZE_BYTES
+                spec['partnum'] = 2
 
             if spec["fstype"] in ["swap"]:
                 current_disk.add_partition(partnum=spec["partnum"],

subiquity/models/actions.py

@@ -17,12 +17,13 @@ import yaml
 
 
 class DiskAction():
-    def __init__(self, action_id, model, serial, ptable='gpt'):
+    def __init__(self, action_id, model, serial, ptable='gpt', wipe=None):
         self._action_id = action_id
         self.parent = None
         self._ptable = ptable
         self._model = model
         self._serial = serial
+        self._wipe = wipe
 
     def get_parent(self):
         return self.parent
@@ -32,42 +33,55 @@ class DiskAction():
         return str(self._action_id)
 
     def get(self):
-        return {
+        action = {
             'id': self.action_id,
             'model': self._model,
             'ptable': self._ptable,
             'serial': self._serial,
             'type': 'disk',
         }
+        if self._wipe:
+            action.update({'wipe': self._wipe})
+        return action
 
     def dump(self):
         return yaml.dump(self.get(), default_flow_style=False)
 
 
 class PartitionAction(DiskAction):
-    def __init__(self, parent, partnumber, size, flags=None):
+    def __init__(self, parent, partnum, offset, size, flags=None):
         self.parent = parent
-        self.partnumber = int(partnumber)
+        self.partnum = int(partnum)
+        self._offset = int(offset)
         self._size = int(size)
         self.flags = flags
         self._action_id = "{}{}_part".format(self.parent.action_id,
-                                             self.partnumber)
+                                             self.partnum)
 
         ''' rename action_id for readability '''
         if self.flags in ['bios_grub']:
             self._action_id = 'bios_boot_partition'
 
+    @property
+    def path(self):
+        return "{}{}".format(self.parent.action_id, self.partnum)
+
     @property
     def size(self):
         return self._size
 
+    @property
+    def offset(self):
+        return self._offset
+
     def get(self):
         return {
             'device': self.parent.action_id,
             'flag': self.flags,
             'id': self.action_id,
-            'number': self.partnumber,
+            'number': self.partnum,
             'size': '{}B'.format(self.size),
+            'offset': '{}B'.format(self.offset),
             'type': 'partition',
         }
 

subiquity/models/blockdev.py

@@ -13,10 +13,10 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from collections import OrderedDict
 from itertools import count
 import logging
 import os
-import parted
 import re
 import yaml
 
@@ -57,45 +57,82 @@ class Bcachedev():
         return self._path
 
     def getSize(self, unit='MB'):
-        if type(self._backing) == parted.disk.Disk:
+        pass
-            return self._backing.device.getSize(unit=unit)
+
-        else:
+
-            return self._backing.getSize(unit=unit)
+class Disk():
+    def __init__(self, devpath, serial, model, parttype, size=0):
+        self._devpath = devpath
+        self._serial = serial
+        self._parttype = parttype
+        self._model = model
+        self._size = self._get_size(devpath, size)
+        self._partitions = OrderedDict()
+
+    def _get_size(self, devpath, size):
+        if size:
+            return size
+        sysblock = os.path.join('/sys/block', os.path.basename(devpath))
+        nr_blocks_f = os.path.join(sysblock, 'size')
+        block_sz_f = os.path.join(sysblock, 'queue', 'logical_block_size')
+        with open(nr_blocks_f, 'r') as r:
+            nr_blocks = int(r.read())
+        with open(block_sz_f, 'r') as r:
+            block_sz = int(r.read())
+
+        return nr_blocks * block_sz
+
+    @property
+    def devpath(self):
+        return self._devpath
+
+    @property
+    def serial(self):
+        return self._serial
+
+    @property
+    def model(self):
+        return self._model
+
+    @property
+    def parttype(self):
+        return self._parttype
+
+    @property
+    def size(self):
+        return self._size
+
+    @property
+    def partitions(self):
+        return self._partitions
+
+    def reset(self):
+        self._partitions = OrderedDict()
+        pass
 
 
 class Blockdev():
-    def __init__(self, devpath, serial, parttype='gpt'):
+    def __init__(self, devpath, serial, model, parttype='gpt', size=0):
-        self.serial = serial
+        self.disk = Disk(devpath, serial, model, parttype, size)
-        self.devpath = devpath
+        self._filesystems = {}
-        self._parttype = parttype
-        self.device = parted.getDevice(self.devpath)
-        self.disk = parted.freshDisk(self.device, self.parttype)
         self._mounts = {}
         self.bcache = []
         self.lvm = []
+        self.baseaction = DiskAction(os.path.basename(self.disk.devpath),
+                                     self.disk.model, self.disk.serial,
+                                     self.disk.parttype)
 
     def reset(self):
         ''' Wipe out any actions queued for this disk '''
-        self.disk = parted.freshDisk(self.device, self.parttype)
+        self.disk.reset()
+        self._filesystems = {}
         self._mounts = {}
         self.bcache = []
         self.lvm = []
 
-    def _get_largest_free_region(self):
+    @property
-        """Finds largest free region on the disk"""
+    def devpath(self):
-        # There are better ways to do it, but let's be straightforward
+        return self.disk.devpath
-        max_size = -1
-        region = None
-
-        alignment = self.device.optimumAlignment
-
-        for r in self.disk.getFreeSpaceRegions():
-            # Heuristic: Ignore alignment gaps
-            if r.length > max_size and r.length > alignment.grainSize:
-                region = r
-                max_size = r.length
-
-        return region
 
     @property
     def mounts(self):
@@ -103,7 +140,7 @@ class Blockdev():
 
     @property
     def parttype(self):
-        return self._parttype
+        return self.disk.parttype
 
     @parttype.setter  # NOQA
     def parttype(self, value):
@@ -111,12 +148,16 @@ class Blockdev():
 
     @property
     def size(self):
-        return self.disk.device.getLength(unit='B')
+        return self.disk.size
 
     @property
     def partitions(self):
         return self.disk.partitions
 
+    @property
+    def filesystems(self):
+        return self._filesystems
+
     @property
     def available(self):
         ''' return True if has free space or partitions not
@@ -128,27 +169,27 @@ class Blockdev():
     @property
     def usedspace(self, unit='b'):
         ''' return amount of used space'''
-        return sum([part.geometry.getSize(unit=unit) for part in
+        space = 0
-                    self.disk.partitions])
+        for (num, action) in self.disk.partitions.items():
+            space += int(action.offset)
+            space += int(action.size)
+
+        log.debug('{} usedspace: {}'.format(self.disk.devpath, space))
+        return space
 
     @property
     def freespace(self, unit='B'):
         ''' return amount of free space '''
-        geo = self._get_largest_free_region()
+        used = self.usedspace
-        if geo:
+        size = self.size
-            return geo.getLength(unit=unit)
+        log.debug('{} freespace: {} - {} = {}'.format(self.disk.devpath,
-        return 0
+                                                      size, used,
-
+                                                      size - used))
-    @property
+        return size - used
-    def freepartition(self, unit='b'):
-        ''' return amount of partitionable space'''
-        return sum([part.geometry.getSize(unit=unit) for part in
-                    self.disk.getFreeSpacePartitions()])
 
     @property
     def lastpartnumber(self):
-        return self.disk.lastPartitionNumber if \
+        return len(self.disk.partitions)
-            self.disk.lastPartitionNumber > 0 else 0
 
     def delete_partition(self, partnum=None, sector=None, mountpoint=None):
         # find part and then call deletePartition()
@@ -161,77 +202,53 @@ class Blockdev():
                   ' partnum:%s size:%s fstype:%s mountpoint:%s flag=%s' % (
                       partnum, size, fstype, mountpoint, flag))
 
-        if size > self.freepartition:
+        if size > self.freespace:
             raise Exception('Not enough space')
 
         if fstype in ["swap"]:
             fstype = "linux-swap(v1)"
 
-        geometry = self._get_largest_free_region()
+        if len(self.disk.partitions) == 0:
-        if not geometry:
+            offset = 1 << 20  # 1K offset/aligned
-            raise Exception('No free sectors available')
+            size += offset
-        log.debug('largest free region:\n{}'.format(geometry))
-
-        # convert size into a geometry based on existing partitions
-        try:
-            start = self.disk.partitions[-1].geometry.end + 1
-        except IndexError:
-            start = 0
-        length = parted.sizeToSectors(size, 'B', self.device.sectorSize)
-        log.debug('requested start: {} length: {}'.format(start, length))
-        req_geo = parted.Geometry(self.device, start=start, length=length)
-
-        # find common area
-        parttype = parted.PARTITION_NORMAL
-        alignment = self.device.optimalAlignedConstraint
-        geometry = geometry.intersect(req_geo)
-        # update geometry with alignment
-        constraint = parted.Constraint(maxGeom=geometry).intersect(alignment)
-        data = {
-            'start': constraint.startAlign.alignUp(geometry, geometry.start),
-            'end': constraint.endAlign.alignDown(geometry, geometry.end),
-        }
-        geometry = parted.Geometry(device=self.device,
-                                   start=data['start'],
-                                   end=data['end'])
-        # create partition
-        if fstype not in [None, 'bcache cache', 'bcache store']:
-            fs = parted.FileSystem(type=fstype, geometry=geometry)
         else:
-            fs = None
+            offset = 0
-        partition = parted.Partition(disk=self.disk, type=parttype,
-                                     fs=fs, geometry=geometry)
 
-        # add flags
+        log.debug('requested start: {} length: {}'.format(offset, size))
-        flags = {
+        valid_flags = [
-            "boot": parted.PARTITION_BOOT,
+            "boot",
-            "lvm": parted.PARTITION_LVM,
+            "lvm",
-            "raid": parted.PARTITION_RAID,
+            "raid",
-            "bios_grub": parted.PARTITION_BIOS_GRUB
+            "bios_grub",
-        }
+        ]
-        if flag in flags:
+        if flag and flag not in valid_flags:
-            partition.setFlag(flags[flag])
+            raise Exception('Flag: {} is not valid.'.format(flag))
 
-        self.disk.addPartition(partition=partition, constraint=constraint)
+        # create partition and add
+        part_action = PartitionAction(self.baseaction, partnum,
+                                      offset, size, flag)
+        log.debug('PartitionAction:\n{}'.format(part_action))
 
-        # fetch the newly created partition
+        self.disk.partitions.update({partnum: part_action})
-        partpath = "{}{}".format(self.disk.device.path, partition.number)
-        newpart = self.disk.getPartitionByPath(partpath)
 
-        # create bcachedev if neded
+        # record filesystem formating
-        if fstype and fstype.startswith('bcache'):
+        if fstype:
-            mode = fstype.split()[-1]
+            partpath = "{}{}".format(self.disk.devpath, partnum)
-            self.bcache.append(Bcachedev(backing=newpart, mode=mode))
+            fs_action = FormatAction(part_action, fstype)
+            log.debug('Adding filesystem: {}:{}'.format(partpath, fs_action))
+            self.filesystems.update({partpath: fs_action})
 
         # associate partition devpath with mountpoint
         if mountpoint:
             self._mounts[partpath] = mountpoint
 
+        log.debug('Partition Added')
+
     def is_mounted(self):
         with open('/proc/mounts') as pm:
             mounts = pm.read()
 
-        regexp = '{}.*'.format(self.disk.device.path)
+        regexp = '{}.*'.format(self.disk.devpath)
         matches = re.findall(regexp, mounts)
         if len(matches) > 0:
             log.debug('Device is mounted: {}'.format(matches))
@@ -245,28 +262,17 @@ class Blockdev():
             return []
 
         actions = []
-        baseaction = DiskAction(os.path.basename(self.disk.device.path),
+        action = self.baseaction.get()
-                                self.device.model, self.serial, self.parttype)
+        for (num, part) in self.disk.partitions.items():
-        action = baseaction.get()
+            partpath = "{}{}".format(self.disk.devpath, part.partnum)
-        for part in self.disk.partitions:
+            actions.append(part)
-            fs_size = int(part.getSize(unit='B'))
+            if partpath in self.filesystems:
-            if part.fileSystem:
+                format_action = self.filesystems[partpath]
-                fs_type = part.fileSystem.type
-            else:
-                fs_type = None
-            flags = part.getFlagsAsString()
-
-            partition_action = PartitionAction(baseaction,
-                                               part.number,
-                                               fs_size, flags)
-            actions.append(partition_action)
-            if fs_type:
-                format_action = FormatAction(partition_action,
-                                             fs_type)
                 actions.append(format_action)
-                mountpoint = self._mounts.get(part.path)
+
-                if mountpoint:
+            if partpath in self._mounts:
-                    mount_action = MountAction(format_action, mountpoint)
+                mount_action = MountAction(format_action,
+                                           self._mounts[partpath])
                 actions.append(mount_action)
 
         return [action] + [a.get() for a in actions]
@@ -274,14 +280,13 @@ class Blockdev():
     def get_fs_table(self):
         ''' list(mountpoint, humansize, fstype, partition_path) '''
         fs_table = []
-        for part in self.disk.partitions:
+        for (num, part) in self.disk.partitions.items():
-            if part.fileSystem:
+            partpath = "{}{}".format(self.disk.devpath, part.partnum)
-                mntpoint = self._mounts.get(part.path, part.fileSystem.type)
+            if partpath in self.filesystems:
-                fs_size = part.getSize(unit='B')
+                fs = self.filesystems[partpath]
-                fs_type = part.fileSystem.type
+                mntpoint = self._mounts.get(partpath, fs.fstype)
-                devpath = part.path
                 fs_table.append(
-                    (mntpoint, fs_size, fs_type, devpath))
+                    (mntpoint, part.size, fs.fstype, partpath))
 
         return fs_table
 
@@ -297,11 +302,17 @@ if __name__ == '__main__':
         print("USED DISKS")
 
     devices = []
-    sda = Blockdev('/dev/sda', 'QM_TARGET_01', parttype='gpt')
+    #Blockdev(devpath, serial, model, parttype='gpt'):
-    sdb = Blockdev('/dev/sdb', 'dafunk')
+    GB = 1 << 30
+    sda = Blockdev('/dev/sda', 'QM_TARGET_01', 'QEMU SSD DISK',
+                   parttype='gpt', size=128 * GB)
+    sdb = Blockdev('/dev/sdb', 'dafunk', 'QEMU SPINNER', size=500 * GB)
 
+    print(sda.freespace)
     sda.add_partition(1, 8 * 1024 * 1024 * 1024, 'ext4', '/', 'bios_grub')
+    print(sda.freespace)
     sda.add_partition(2, 2 * 1024 * 1024 * 1024, 'ext4', '/home')
+    print(sda.freespace)
     sdb.add_partition(1, 50 * 1024 * 1024 * 1024, 'btrfs', '/opt')
 
     get_filesystems([sda, sdb])

subiquity/models/filesystem.py

@@ -134,13 +134,15 @@ class FilesystemModel(ModelPolicy):
 
     def get_disk(self, disk):
         if disk not in self.devices:
-            self.devices[disk] = Blockdev(disk, self.info[disk].serial)
+            self.devices[disk] = Blockdev(disk, self.info[disk].serial,
+                                          self.info[disk].model)
         return self.devices[disk]
 
     def get_partitions(self):
         partitions = []
         for dev in self.devices.values():
-            partnames = [part.path for part in dev.disk.partitions]
+            partnames = [part.path for (num, part) in
+                         dev.disk.partitions.items()]
             partitions += partnames
 
         sorted(partitions)