First cut on raid ux and model integration

Introduce a new Raiddev class to hold raid virtual device.
Add RaidAction to emit correct storage config
Tested with raid level 0 only.
Not sure if we can allow partitioning of raid devices.  Documentation
says it can be done but curtin will need to 'make it correctly'.

Signed-off-by: Ryan Harper <ryan.harper@canonical.com>

subiquity/controllers/filesystem.py

@@ -17,11 +17,11 @@ import logging
 import os
 from subiquity.controller import ControllerPolicy
 from subiquity.models.actions import preserve_action
-from subiquity.models import (FilesystemModel, IscsiDiskModel, RaidModel,
+from subiquity.models import (FilesystemModel,
-                              CephDiskModel)
+                              RaidModel)
 from subiquity.ui.views import (DiskPartitionView, AddPartitionView,
                                 FilesystemView, DiskInfoView,
-                                RaidView, CephDiskView, IscsiDiskView)
+                                RaidView)
 import subiquity.utils as utils
 from subiquity.ui.dummy import DummyView
 from subiquity.curtin import (curtin_write_storage_actions,
@@ -38,8 +38,8 @@ class FilesystemController(ControllerPolicy):
     def __init__(self, common):
         super().__init__(common)
         self.model = FilesystemModel(self.prober, self.opts)
-        self.iscsi_model = IscsiDiskModel()
+        #self.iscsi_model = IscsiDiskModel()
-        self.ceph_model = CephDiskModel()
+        #self.ceph_model = CephDiskModel()
         self.raid_model = RaidModel()
 
     def filesystem(self, reset=False):
@@ -172,7 +172,7 @@ class FilesystemController(ControllerPolicy):
         self.ui.set_body(DummyView(self.signal))
 
     def create_raid(self, *args, **kwargs):
-        title = ("Disk and filesystem setup")
+        title = ("Create software RAID (\"MD\") disk")
         footer = ("ENTER on a disk will show detailed "
                   "information for that disk")
         excerpt = ("Use SPACE to select disks to form your RAID array, "
@@ -181,9 +181,14 @@ class FilesystemController(ControllerPolicy):
                    "the same size and speed.")
         self.ui.set_header(title, excerpt)
         self.ui.set_footer(footer)
-        self.ui.set_body(RaidView(self.raid_model,
+        self.ui.set_body(RaidView(self.model,
                                   self.signal))
 
+    def add_raid_dev(self, result):
+        log.debug('add_raid_dev: result={}'.format(result))
+        self.model.add_raid_device(result)
+        self.signal.emit_signal('filesystem:show')
+
     def setup_bcache(self, *args, **kwargs):
         self.ui.set_body(DummyView(self.signal))
 
@@ -216,4 +221,5 @@ class FilesystemController(ControllerPolicy):
         if self.opts.dry_run and self.opts.uefi:
             log.debug('forcing is_uefi True beacuse of options')
             return True
+
         return os.path.exists('/sys/firmware/efi')

subiquity/controllers/raid.py

@@ -0,0 +1,32 @@
+# Copyright 2015 Canonical, Ltd.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+from subiquity.models import RaidDiskModel
+from subiquity.controller import ControllerPolicy
+
+log = logging.getLogger("subiquity.controller.raid")
+
+
+class RaidDiskController(ControllerPolicy):
+    def __init__(self, common):
+        super().__init__(common)
+        self.model = RaidDiskModel()
+
+    def raid(self):
+        pass
+
+    def raid_handler(self):
+        pass

subiquity/models/actions.py

@@ -121,6 +121,26 @@ class DiskAction():
         return yaml.dump(self.get(), default_flow_style=False)
 
 
+class RaidAction(DiskAction):
+    def __init__(self, action_id, raidlevel, dev_ids, spare_ids):
+        self._action_id = action_id
+        self.parent = None
+        self._raidlevel = raidlevel
+        self._devices = dev_ids
+        self._spares = spare_ids
+
+    def get(self):
+        action = {
+            'id': self.action_id,
+            'name': self.action_id,
+            'raidlevel': self._raidlevel,
+            'devices': self._devices,
+            'spare_devices': self._spares,
+            'type': 'raid',
+        }
+        return action
+
+
 class PartitionAction(DiskAction):
     def __init__(self, parent, partnum, offset, size, flags=None):
         self.parent = parent

subiquity/models/blockdev.py

@@ -24,7 +24,8 @@ from .actions import (
     DiskAction,
     PartitionAction,
     FormatAction,
-    MountAction
+    MountAction,
+    RaidAction,
 )
 
 log = logging.getLogger("subiquity.filesystem.blockdev")
@@ -125,6 +126,7 @@ class Blockdev():
         self._mounts = {}
         self.bcache = []
         self.lvm = []
+        self.holder = {}
         self.baseaction = DiskAction(os.path.basename(self.disk.devpath),
                                      self.disk.model, self.disk.serial,
                                      self.disk.parttype)
@@ -136,6 +138,7 @@ class Blockdev():
         self._mounts = {}
         self.bcache = []
         self.lvm = []
+        self.holder = {}
 
     @property
     def devpath(self):
@@ -165,11 +168,17 @@ class Blockdev():
     def filesystems(self):
         return self._filesystems
 
+    @property
+    def percent_free(self):
+        ''' return the device free percentage of the whole device'''
+        percent = ( int((1.0 - (self.usedspace / self.size)) * 100))
+        return percent
+
     @property
     def available(self):
         ''' return True if has free space or partitions not
             assigned '''
-        if not self.is_mounted() and self.freespace > 0.0:
+        if not self.is_mounted() and self.percent_free > 0:
             return True
         return False
 
@@ -269,6 +278,9 @@ class Blockdev():
         log.debug('Partition Added')
         return new_size
 
+    def set_holder(self, devpath, holdtype):
+        self.holder[holdtype] = devpath
+
     def is_mounted(self):
         with open('/proc/mounts') as pm:
             mounts = pm.read()
@@ -317,7 +329,7 @@ class Blockdev():
 
     def sort_actions(self, actions):
         def type_index(t):
-            order = ['disk', 'partition', 'format', 'mount']
+            order = ['disk', 'partition', 'raid', 'format', 'mount']
             return order.index(t.get('type'))
 
         def path_count(p):
@@ -349,6 +361,19 @@ class Blockdev():
         return fs_table
 
 
+class Raiddev(Blockdev):
+    def __init__(self, devpath, serial, model, parttype, size,
+                 raid_level, raid_devices, spare_devices):
+        super().__init__(devpath, serial, model, parttype, size)
+        self._raid_devices = raid_devices
+        self._raid_level = raid_level
+        self._spare_devices = spare_devices
+        self.baseaction = RaidAction(os.path.basename(self.disk.devpath),
+                                     self._raid_devices,
+                                     self._raid_level,
+                                     self._spare_devices)
+
+
 if __name__ == '__main__':
     def get_filesystems(devices):
         print("FILE SYSTEM")

subiquity/models/filesystem.py

@@ -16,7 +16,7 @@
 import json
 import logging
 
-from .blockdev import Blockdev
+from .blockdev import Blockdev, Raiddev
 import math
 from subiquity.model import ModelPolicy
 
@@ -25,6 +25,10 @@ HUMAN_UNITS = ['B', 'K', 'M', 'G', 'T', 'P']
 log = logging.getLogger('subiquity.models.filesystem')
 
 
+class AttrDict(dict): 
+    __getattr__ = dict.__getitem__
+    __setattr__ = dict.__setitem__
+
 class FilesystemModel(ModelPolicy):
     """ Model representing storage options
     """
@@ -58,7 +62,10 @@ class FilesystemModel(ModelPolicy):
          'create_swap_entire_device'),
         ('Show disk information',
          'filesystem:show-disk-information',
-         'show_disk_information')
+         'show_disk_information'),
+        ('Add Raid Device',
+         'filesystem:add-raid-dev',
+         'add_raid_dev'),
     ]
 
     # TODO: Re-add once curtin supports this.
@@ -72,9 +79,9 @@ class FilesystemModel(ModelPolicy):
         # ('Create volume group (LVM2)',
         #  'filesystem:create-volume-group',
         #  'create_volume_group'),
-        # ('Create software RAID (MD)',
+        ('Create software RAID (MD)',
-        #  'filesystem:create-raid',
+         'filesystem:create-raid',
-        #  'create_raid'),
+         'create_raid'),
         # ('Setup hierarchichal storage (bcache)',
         #  'filesystem:setup-bcache',
         #  'setup_bcache')
@@ -90,11 +97,21 @@ class FilesystemModel(ModelPolicy):
         'leave unformatted'
     ]
 
+    # TODO: what is "linear" level?
+    raid_levels = [
+        "0",
+        "1",
+        "5",
+        "6",
+        "10",
+    ]
+
     def __init__(self, prober, opts):
         self.opts = opts
         self.prober = prober
         self.info = {}
         self.devices = {}
+        self.raid_devices = {}
         self.storage = {}
 
     def reset(self):
@@ -142,8 +159,119 @@ class FilesystemModel(ModelPolicy):
         return self.devices[disk]
 
     def get_disks(self):
-        return [self.get_disk(d) for d in sorted(self.info.keys())
+        possible_devices = list(set(list(self.devices.keys()) +
-                if len(d) > 0]
+                                    list(self.info.keys())))
+        possible_disks = [self.get_disk(d) for d in sorted(possible_devices)]
+        return [d for d in possible_disks if d.available]
+
+    def calculate_raid_size(self, raid_level, raid_devices, spare_devices):
+        '''
+            0: array size is the size of the smallest component partition times
+               the number of component partitions
+            1: array size is the size of the smallest component partition
+            5: array size is the size of the smallest component partition times
+               the number of component partitions munus 1
+            6: array size is the size of the smallest component partition times
+               the number of component partitions munus 2
+        '''
+        # https://raid.wiki.kernel.org/ \
+        #       index.php/RAID_superblock_formats#Total_Size_of_superblock
+        # Version-1 superblock format on-disk layout:
+        # Total size of superblock: 256 Bytes plus 2 bytes per device in the
+        # array
+        log.debug('calc_raid_size: level={} rd={} sd={}'.format(raid_level,
+                                                                raid_devices,
+                                                                spare_devices))
+        overhead_bytes = 256 + (2 * (len(raid_devices) + len(spare_devices)))
+        log.debug('calc_raid_size: overhead_bytes={}'.format(overhead_bytes))
+
+        # find the smallest device
+        min_dev_size = min([d.size for d in raid_devices])
+        log.debug('calc_raid_size: min_dev_size={}'.format(min_dev_size))
+
+        if raid_level == 0:
+            array_size = min_dev_size * len(raid_devices)
+        elif raid_level == 1:
+            array_size = min_dev_size
+        elif raid_level == 5:
+            array_size = min_dev_size * (len(raid_devices) - 1)
+        elif raid_level == 10:
+            array_size = min_dev_size * int((len(raid_devices) /
+                                             len(spare_devices)))
+        total_size = array_size - overhead_bytes
+        log.debug('calc_raid_size: array_size:{} - overhead:{} = {}'.format(
+                  array_size, overhead_bytes, total_size))
+        return total_size
+
+    def add_raid_device(self, raidspec):
+        # assume raidspec has already been valided in view/controller
+        log.debug('Attempting to create a raid device')
+        '''
+        raidspec = {
+            'devices': ['/dev/sdb     1.819T, HDS5C3020ALA632',
+                        '/dev/sdc     1.819T, 001-9YN164',
+                        '/dev/sdf     1.819T, 001-9YN164',
+                        '/dev/sdg     1.819T, 001-9YN164',
+                        '/dev/sdh     1.819T, HDS5C3020ALA632',
+                        '/dev/sdi     1.819T, 001-9YN164'],
+            'raid_level': '0',
+            'hot_spares': '0',
+            'chunk_size': '4K',
+        }
+        '''
+        raid_devices = []
+        spare_devices = []
+        all_devices = [r.split() for r in raidspec.get('devices', [])]
+        nr_spares = int(raidspec.get('hot_spares'))
+
+        # XXX: curtin requires a partition table on the base devices
+        # and then one partition of type raid
+        for (devpath, _, _) in all_devices:
+            disk = self.get_disk(devpath)
+            disk.add_partition(1, disk.freespace, None, None, flag='raid')
+            if len(raid_devices) + nr_spares < len(all_devices):
+                raid_devices.append(disk)
+            else:
+                spare_devices.append(disk)
+
+        # auto increment md number based in registered devices
+        raid_dev_name = '/dev/md{}'.format(len(self.raid_devices))
+        raid_serial = '{}_serial'.format(raid_dev_name)
+        raid_model = '{}_model'.format(raid_dev_name)
+        raid_parttype = 'gpt'
+        raid_level = int(raidspec.get('raid_level'))
+        raid_size = self.calculate_raid_size(raid_level, raid_devices,
+                                             spare_devices)
+        # create a Raiddev (pass in only the names)
+        raid_dev = Raiddev(raid_dev_name, raid_serial, raid_model,
+                           raid_parttype, raid_size,
+                           [d.devpath for d in raid_devices],
+                           raid_level,
+                           [d.devpath for d in spare_devices])
+
+        # add it to the model's info dict
+        raid_dev_info = {
+            'type': 'disk',
+            'name': raid_dev_name,
+            'size': raid_size,
+            'serial': raid_serial,
+            'vendor': 'Linux Software RAID',
+            'model': raid_model,
+            'is_virtual': True,
+            'raw': {},
+        }
+        self.info[raid_dev_name] = AttrDict(raid_dev_info)
+
+        # add it to the model's raid devices
+        self.raid_devices[raid_dev_name] = raid_dev
+        # add it to the model's devices
+        self.add_device(raid_dev_name, raid_dev)
+
+        log.debug('Successfully added raid_dev: {}'.format(raid_dev))
+
+    def add_device(self, devpath, device):
+        log.debug("adding device: {} = {}".format(devpath, device))
+        self.devices[devpath] = device
 
     def get_partitions(self):
         log.debug('probe_storage: get_partitions()')
@@ -158,16 +286,23 @@ class FilesystemModel(ModelPolicy):
                   partitions))
         return partitions
 
+    def installable(self):
+        ''' one or more disks has used space
+            and has "/" as a mount
+        '''
+        for disk in self.get_disks():
+            if disk.usedspace > 0 and "/" in disk.mounts:
+                return True
+
     def get_available_disks(self):
-        return [dev.disk.devpath for dev in self.get_disks()
+        return [dev.disk.devpath for dev in self.get_disks()]
-                if self.opts.dry_run is True or dev.mounted is False]
 
     def get_used_disks(self):
         return [dev.disk.devpath for dev in self.devices.values()
                 if dev.available is False]
 
     def get_disk_info(self, disk):
-        return self.info[disk]
+        return self.info.get(disk, {})
 
     def get_disk_action(self, disk):
         return self.devices[disk].get_actions()

subiquity/models/raid.py

@@ -14,7 +14,6 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import logging
-from collections import OrderedDict
 from subiquity.model import ModelPolicy
 
 

subiquity/ui/views/filesystem.py

@@ -380,7 +380,6 @@ class FilesystemView(ViewPolicy):
         self.signal = signal
         self.items = []
         self.model.probe_storage()  # probe before we complete
-        self.installable = True
         self.body = [
             Padding.center_79(Text("FILE SYSTEM")),
             Padding.center_79(self._build_partition_list()),
@@ -441,7 +440,7 @@ class FilesystemView(ViewPolicy):
         buttons = []
 
         # don't enable done botton if we can't install
-        if self.installable:
+        if self.model.installable:
             buttons.append(
                 Color.button(done_btn(on_press=self.done),
                              focus_map='button focus'))
@@ -467,7 +466,6 @@ class FilesystemView(ViewPolicy):
 
         avail_disks = self.model.get_available_disks()
         if len(avail_disks) == 0:
-            self.installable = False
             return Pile([Color.info_minor(Text("No available disks."))])
 
         for dname in avail_disks:

subiquity/ui/views/raid.py

@@ -13,10 +13,12 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from urwid import Text, Columns, Pile, ListBox
+from urwid import Text, Columns, Pile, ListBox, CheckBox
+from subiquity.models.filesystem import _humanize_size
 from subiquity.view import ViewPolicy
 from subiquity.ui.buttons import cancel_btn, done_btn
-from subiquity.ui.interactive import StringEditor, IntegerEditor, Selector
+from subiquity.ui.interactive import (StringEditor, IntegerEditor,
+                                      Selector)
 from subiquity.ui.utils import Color, Padding
 import logging
 
@@ -29,7 +31,8 @@ class RaidView(ViewPolicy):
         self.signal = signal
         self.raid_level = Selector(self.model.raid_levels)
         self.hot_spares = IntegerEditor(caption="")
-        self.chunk_size = StringEditor(caption="")
+        self.chunk_size = StringEditor(edit_text="4K", caption="")
+        self.selected_disks = []
         body = [
             Padding.center_50(self._build_disk_selection()),
             Padding.line_break(""),
@@ -40,12 +43,32 @@ class RaidView(ViewPolicy):
         super().__init__(ListBox(body))
 
     def _build_disk_selection(self):
+        log.debug('raid: _build_disk_selection')
         items = [
             Text("DISK SELECTION")
         ]
+        avail_disks = self.model.get_available_disks()
+        if len(avail_disks) == 0:
+            self.installable = False
+            return items.append(
+                [Color.info_minor(Text("No available disks."))])
+
+        for dname in avail_disks:
+            disk = self.model.get_disk_info(dname)
+            #device = self.model.get_disk(dname)
+            disk_sz = _humanize_size(disk.size)
+            disk_string = "{}     {},     {}".format(disk.name,
+                                                     disk_sz,
+                                                     disk.model)
+            log.debug('raid: disk_string={}'.format(disk_string))
+            self.selected_disks.append(CheckBox(disk_string))
+
+        items += self.selected_disks
+
         return Pile(items)
 
     def _build_raid_configuration(self):
+        log.debug('raid: _build_raid_config')
         items = [
             Text("RAID CONFIGURATION"),
             Columns(
@@ -80,6 +103,7 @@ class RaidView(ViewPolicy):
         return Pile(items)
 
     def _build_buttons(self):
+        log.debug('raid: _build_buttons')
         cancel = cancel_btn(on_press=self.cancel)
         done = done_btn(on_press=self.done)
 
@@ -90,7 +114,15 @@ class RaidView(ViewPolicy):
         return Pile(buttons)
 
     def done(self, result):
-        self.signal.emit_signal('filesystem:show')
+        result = {
+            'devices': [x.get_label() for x in self.selected_disks if x.state],
+            'raid_level': self.raid_level.value,
+            'hot_spares': self.hot_spares.value,
+            'chunk_size': self.chunk_size.value,
+        }
+        log.debug('raid_done: result = {}'.format(result))
+        self.signal.emit_signal('filesystem:add-raid-dev', result)
 
     def cancel(self, button):
+        log.debug('raid: button_cancel')
         self.signal.emit_signal("quit")