Merge pull request #1222 from mwhudson/optional-v2-storage

Optional v2 storage

subiquity/cmd/server.py

@@ -71,6 +71,8 @@ def make_server_args_parser():
         '--output-base', action='store', dest='output_base',
         default='.subiquity',
         help='in dryrun, control basedir of files')
+    parser.add_argument(
+        '--storage-version', action='store', type=int, default=1)
     return parser
 
 

subiquity/common/filesystem/gaps.py

@@ -32,6 +32,8 @@ class Gap:
     device: object
     offset: int
     size: int
+    in_extended: bool = False
+
     type: str = 'gap'
 
     @property
@@ -44,11 +46,7 @@ def parts_and_gaps(device):
     raise NotImplementedError(device)
 
 
-@parts_and_gaps.register(Disk)
+def find_disk_gaps_v1(device):
-@parts_and_gaps.register(Raid)
-def parts_and_gaps_disk(device):
-    if device._fs is not None:
-        return []
     r = []
     used = 0
     ad = device.alignment_data()
@@ -66,6 +64,76 @@ def parts_and_gaps_disk(device):
     return r
 
 
+def find_disk_gaps_v2(device, info=None):
+    result = []
+    extended_end = None
+
+    if info is None:
+        info = device.alignment_data()
+
+    def au(v):  # au == "align up"
+        r = v % info.part_align
+        if r:
+            return v + info.part_align - r
+        else:
+            return v
+
+    def ad(v):  # ad == "align down"
+        return v - v % info.part_align
+
+    def maybe_add_gap(start, end, in_extended):
+        if end - start >= info.min_gap_size:
+            result.append(Gap(device, start, end - start, in_extended))
+
+    prev_end = info.min_start_offset
+
+    parts = sorted(device._partitions, key=lambda p: p.offset)
+    extended_end = None
+
+    for part in parts + [None]:
+        if part is None:
+            gap_end = ad(device.size - info.min_end_offset)
+        else:
+            gap_end = ad(part.offset)
+
+        gap_start = au(prev_end)
+
+        if extended_end is not None:
+            gap_start = min(
+                extended_end, au(gap_start + info.ebr_space))
+
+        if extended_end is not None and gap_end >= extended_end:
+            maybe_add_gap(gap_start, ad(extended_end), True)
+            maybe_add_gap(au(extended_end), gap_end, False)
+            extended_end = None
+        else:
+            maybe_add_gap(gap_start, gap_end, extended_end is not None)
+
+        if part is None:
+            break
+
+        result.append(part)
+
+        if part.flag == "extended":
+            prev_end = part.offset
+            extended_end = part.offset + part.size
+        else:
+            prev_end = part.offset + part.size
+
+    return result
+
+
+@parts_and_gaps.register(Disk)
+@parts_and_gaps.register(Raid)
+def parts_and_gaps_disk(device):
+    if device._fs is not None:
+        return []
+    if device._m.storage_version == 1:
+        return find_disk_gaps_v1(device)
+    else:
+        return find_disk_gaps_v2(device)
+
+
 @parts_and_gaps.register(LVM_VolGroup)
 def _parts_and_gaps_vg(device):
     used = 0

subiquity/common/filesystem/tests/test_gaps.py

@@ -15,8 +15,13 @@
 
 import unittest
 
+from subiquity.models.filesystem import (
+    PartitionAlignmentData,
+    MiB,
+    )
 from subiquity.models.tests.test_filesystem import (
     make_model_and_disk,
+    make_partition,
     )
 
 from subiquity.common.filesystem import gaps
@@ -29,4 +34,134 @@ class TestGaps(unittest.TestCase):
         pg = gaps.parts_and_gaps(disk1)
         self.assertEqual(1, len(pg))
         self.assertTrue(isinstance(pg[0], gaps.Gap))
-        self.assertEqual(1024 * 1024, pg[0].offset)
+        self.assertEqual(MiB, pg[0].offset)
+
+
+class TestDiskGaps(unittest.TestCase):
+
+    def test_no_partition_gpt(self):
+        size = 1 << 30
+        m, d = make_model_and_disk(size=size, ptable='gpt')
+        self.assertEqual(
+            gaps.find_disk_gaps_v2(d),
+            [gaps.Gap(d, MiB, size - 2*MiB, False)])
+
+    def test_no_partition_dos(self):
+        size = 1 << 30
+        m, d = make_model_and_disk(size=size, ptable='dos')
+        self.assertEqual(
+            gaps.find_disk_gaps_v2(d),
+            [gaps.Gap(d, MiB, size - MiB, False)])
+
+    def test_all_partition(self):
+        info = PartitionAlignmentData(
+            part_align=10, min_gap_size=1, min_start_offset=0,
+            min_end_offset=0, primary_part_limit=10)
+        m, d = make_model_and_disk(size=100)
+        p = make_partition(m, d, offset=0, size=100)
+        self.assertEqual(
+            gaps.find_disk_gaps_v2(d, info),
+            [p])
+
+    def test_all_partition_with_min_offsets(self):
+        info = PartitionAlignmentData(
+            part_align=10, min_gap_size=1, min_start_offset=10,
+            min_end_offset=10, primary_part_limit=10)
+        m, d = make_model_and_disk(size=100)
+        p = make_partition(m, d, offset=10, size=80)
+        self.assertEqual(
+            gaps.find_disk_gaps_v2(d, info),
+            [p])
+
+    def test_half_partition(self):
+        info = PartitionAlignmentData(
+            part_align=10, min_gap_size=1, min_start_offset=0,
+            min_end_offset=0, primary_part_limit=10)
+        m, d = make_model_and_disk(size=100)
+        p = make_partition(m, d, offset=0, size=50)
+        self.assertEqual(
+            gaps.find_disk_gaps_v2(d, info),
+            [p, gaps.Gap(d, 50, 50)])
+
+    def test_gap_in_middle(self):
+        info = PartitionAlignmentData(
+            part_align=10, min_gap_size=1, min_start_offset=0,
+            min_end_offset=0, primary_part_limit=10)
+        m, d = make_model_and_disk(size=100)
+        p1 = make_partition(m, d, offset=0, size=20)
+        p2 = make_partition(m, d, offset=80, size=20)
+        self.assertEqual(
+            gaps.find_disk_gaps_v2(d, info),
+            [p1, gaps.Gap(d, 20, 60), p2])
+
+    def test_small_gap(self):
+        info = PartitionAlignmentData(
+            part_align=10, min_gap_size=20, min_start_offset=0,
+            min_end_offset=0, primary_part_limit=10)
+        m, d = make_model_and_disk(size=100)
+        p1 = make_partition(m, d, offset=0, size=40)
+        p2 = make_partition(m, d, offset=50, size=50)
+        self.assertEqual(
+            gaps.find_disk_gaps_v2(d, info),
+            [p1, p2])
+
+    def test_align_gap(self):
+        info = PartitionAlignmentData(
+            part_align=10, min_gap_size=1, min_start_offset=0,
+            min_end_offset=0, primary_part_limit=10)
+        m, d = make_model_and_disk(size=100)
+        p1 = make_partition(m, d, offset=0, size=17)
+        p2 = make_partition(m, d, offset=53, size=47)
+        self.assertEqual(
+            gaps.find_disk_gaps_v2(d, info),
+            [p1, gaps.Gap(d, 20, 30), p2])
+
+    def test_all_extended(self):
+        info = PartitionAlignmentData(
+            part_align=5, min_gap_size=1, min_start_offset=0, min_end_offset=0,
+            ebr_space=2, primary_part_limit=10)
+        m, d = make_model_and_disk(size=100, ptable='dos')
+        p = make_partition(m, d, offset=0, size=100, flag='extended')
+        self.assertEqual(
+            gaps.find_disk_gaps_v2(d, info),
+            [
+                p,
+                gaps.Gap(d, 5, 95, True),
+            ])
+
+    def test_half_extended(self):
+        info = PartitionAlignmentData(
+            part_align=5, min_gap_size=1, min_start_offset=0, min_end_offset=0,
+            ebr_space=2, primary_part_limit=10)
+        m, d = make_model_and_disk(size=100)
+        p = make_partition(m, d, offset=0, size=50, flag='extended')
+        self.assertEqual(
+            gaps.find_disk_gaps_v2(d, info),
+            [p, gaps.Gap(d, 5, 45, True), gaps.Gap(d, 50, 50, False)])
+
+    def test_half_extended_one_logical(self):
+        info = PartitionAlignmentData(
+            part_align=5, min_gap_size=1, min_start_offset=0, min_end_offset=0,
+            ebr_space=2, primary_part_limit=10)
+        m, d = make_model_and_disk(size=100)
+        p1 = make_partition(m, d, offset=0, size=50, flag='extended')
+        p2 = make_partition(m, d, offset=5, size=45, flag='logical')
+        self.assertEqual(
+            gaps.find_disk_gaps_v2(d, info),
+            [p1, p2, gaps.Gap(d, 50, 50, False)])
+
+    def test_half_extended_half_logical(self):
+        info = PartitionAlignmentData(
+            part_align=5, min_gap_size=1, min_start_offset=0, min_end_offset=0,
+            ebr_space=2, primary_part_limit=10)
+        m, d = make_model_and_disk(size=100, ptable='dos')
+        p1 = make_partition(m, d, offset=0, size=50, flag='extended')
+        p2 = make_partition(m, d, offset=5, size=25, flag='logical')
+        self.assertEqual(
+            gaps.find_disk_gaps_v2(d, info),
+            [
+                p1,
+                p2,
+                gaps.Gap(d, 35, 15, True),
+                gaps.Gap(d, 50, 50, False),
+            ])

subiquity/common/types.py

@@ -316,6 +316,7 @@ class StorageResponse:
     config: Optional[list] = None
     blockdev: Optional[dict] = None
     dasd: Optional[dict] = None
+    storage_version: int = 1
 
 
 @attr.s(auto_attribs=True)

subiquity/models/filesystem.py

@@ -998,6 +998,7 @@ class FilesystemModel(object):
         if bootloader is None:
             bootloader = self._probe_bootloader()
         self.bootloader = bootloader
+        self.storage_version = 1
         self._probe_data = None
         self.reset()
 
@@ -1019,6 +1020,7 @@ class FilesystemModel(object):
     def load_server_data(self, status):
         log.debug('load_server_data %s', status)
         self._all_ids = set()
+        self.storage_version = status.storage_version
         self._orig_config = status.orig_config
         self._probe_data = {
             'blockdev': status.blockdev,
@@ -1294,7 +1296,7 @@ class FilesystemModel(object):
     def render(self):
         config = {
             'storage': {
-                'version': 1,
+                'version': self.storage_version,
                 'config': self._render_actions(),
                 },
             }

subiquity/models/tests/test_filesystem.py

@@ -156,9 +156,9 @@ def make_disk(fs_model, **kw):
     return disk
 
 
-def make_model_and_disk(bootloader=None):
+def make_model_and_disk(bootloader=None, **kw):
     model = make_model(bootloader)
-    return model, make_disk(model)
+    return model, make_disk(model, **kw)
 
 
 def make_partition(model, device=None, *, preserve=False, size=None,

subiquity/server/controllers/filesystem.py

@@ -87,6 +87,7 @@ class FilesystemController(SubiquityController, FilesystemManipulator):
         if self.opts.dry_run and self.opts.bootloader:
             name = self.opts.bootloader.upper()
             self.model.bootloader = getattr(Bootloader, name)
+        self.model.storage_version = self.opts.storage_version
         self._monitor = None
         self._errors = {}
         self._probe_once_task = SingleInstanceTask(
@@ -234,7 +235,8 @@ class FilesystemController(SubiquityController, FilesystemManipulator):
             orig_config=self.model._orig_config,
             config=self.model._render_actions(include_all=True),
             blockdev=self.model._probe_data['blockdev'],
-            dasd=self.model._probe_data.get('dasd', {}))
+            dasd=self.model._probe_data.get('dasd', {}),
+            storage_version=self.model.storage_version)
 
     async def POST(self, config: list):
         log.debug(config)