2019-11-06 21:32:09 +00:00
|
|
|
#!/usr/bin/python3
|
|
|
|
|
2019-11-06 22:31:23 +00:00
|
|
|
# The fine details of how big a RAID device ends up as a function of the sizes
|
|
|
|
# of its components is somewhat hairier than one might think, with a certain
|
|
|
|
# fraction of each component device being given over to metadata storage. This
|
|
|
|
# script tests the estimates subiquity uses against reality by creating actual
|
|
|
|
# raid devices (backed by sparse files in a tmpfs) and comparing their sizes
|
|
|
|
# with the estimates. It must be run as root.
|
|
|
|
|
2019-11-06 21:32:09 +00:00
|
|
|
import os
|
2019-11-06 23:56:13 +00:00
|
|
|
import random
|
2019-11-06 21:32:09 +00:00
|
|
|
import subprocess
|
|
|
|
import sys
|
|
|
|
import tempfile
|
|
|
|
import uuid
|
|
|
|
|
|
|
|
|
|
|
|
from subiquity.models.filesystem import (
|
2019-11-06 23:56:13 +00:00
|
|
|
align_down,
|
2019-11-06 21:32:09 +00:00
|
|
|
dehumanize_size,
|
|
|
|
get_raid_size,
|
|
|
|
humanize_size,
|
|
|
|
raidlevels,
|
|
|
|
)
|
2019-11-07 00:25:58 +00:00
|
|
|
from subiquity.models.tests.test_filesystem import (
|
|
|
|
FakeDev,
|
|
|
|
)
|
2019-11-06 21:32:09 +00:00
|
|
|
|
|
|
|
|
|
|
|
tmpdir = tempfile.mkdtemp()
|
|
|
|
|
|
|
|
def run(cmd):
|
|
|
|
try:
|
|
|
|
subprocess.run(
|
|
|
|
cmd, check=True,
|
|
|
|
stdout=subprocess.PIPE, stdin=subprocess.DEVNULL,
|
|
|
|
stderr=subprocess.STDOUT)
|
|
|
|
except subprocess.CalledProcessError as e:
|
|
|
|
print(e.stdout)
|
|
|
|
raise
|
|
|
|
|
2019-11-06 22:04:03 +00:00
|
|
|
raids = []
|
|
|
|
loopdevs = []
|
2019-11-06 21:32:09 +00:00
|
|
|
|
2019-11-06 22:04:03 +00:00
|
|
|
def cleanraids():
|
2019-11-06 21:32:09 +00:00
|
|
|
for raid in raids:
|
2019-11-06 22:04:03 +00:00
|
|
|
run(['mdadm', '--verbose', '--stop', raid])
|
|
|
|
del raids[:]
|
|
|
|
|
|
|
|
def cleanloops():
|
2019-11-06 21:32:09 +00:00
|
|
|
for loopdev in loopdevs:
|
|
|
|
subprocess.run(
|
|
|
|
['losetup', '-d', loopdev])
|
2019-11-06 22:04:03 +00:00
|
|
|
del loopdevs[:]
|
|
|
|
|
|
|
|
def cleanup():
|
|
|
|
cleanraids()
|
|
|
|
cleanloops()
|
2019-11-06 21:32:09 +00:00
|
|
|
|
|
|
|
|
|
|
|
def create_devices_for_sizes(sizes):
|
|
|
|
devs = []
|
|
|
|
for size in sizes:
|
|
|
|
fd, name = tempfile.mkstemp(dir=tmpdir)
|
|
|
|
os.ftruncate(fd, size)
|
|
|
|
os.close(fd)
|
|
|
|
dev = subprocess.run(
|
|
|
|
['losetup', '-f', '--show', name],
|
|
|
|
stdout=subprocess.PIPE, encoding='ascii').stdout.strip()
|
|
|
|
devs.append(dev)
|
|
|
|
loopdevs.append(dev)
|
|
|
|
return devs
|
|
|
|
|
|
|
|
|
|
|
|
def create_raid(level, images):
|
2019-11-06 23:56:13 +00:00
|
|
|
name = '/dev/md/test-{}'.format(uuid.uuid4())
|
2019-11-06 21:32:09 +00:00
|
|
|
cmd = [
|
|
|
|
'mdadm',
|
|
|
|
'--verbose',
|
|
|
|
'--create',
|
|
|
|
'--metadata', 'default',
|
|
|
|
'--level', level,
|
2019-11-06 23:56:13 +00:00
|
|
|
'--run',
|
2019-11-06 21:32:09 +00:00
|
|
|
'-n', str(len(images)),
|
|
|
|
'--assume-clean',
|
|
|
|
name,
|
|
|
|
] + images
|
|
|
|
run(cmd)
|
|
|
|
raids.append(name)
|
|
|
|
return name
|
|
|
|
|
|
|
|
|
|
|
|
def get_real_raid_size(raid):
|
|
|
|
return int(subprocess.run(
|
|
|
|
['blockdev', '--getsize64', raid],
|
|
|
|
stdout=subprocess.PIPE, encoding='ascii').stdout.strip())
|
|
|
|
|
|
|
|
|
|
|
|
def verify_size_ok(level, sizes):
|
2019-11-06 22:04:03 +00:00
|
|
|
r = False
|
|
|
|
try:
|
|
|
|
devs = create_devices_for_sizes(sizes)
|
|
|
|
raid = create_raid(level, devs)
|
|
|
|
devs = [FakeDev(size) for size in sizes]
|
|
|
|
calc_size = get_raid_size(level, devs)
|
|
|
|
real_size = get_real_raid_size(raid)
|
|
|
|
if len(set(sizes)) == 1:
|
|
|
|
sz = '[{}]*{}'.format(humanize_size(sizes[0]), len(sizes))
|
|
|
|
else:
|
|
|
|
sz = str([humanize_size(s) for s in sizes])
|
|
|
|
print("level {} sizes {} -> calc_size {} real_size {}".format(
|
|
|
|
level, sz , calc_size, real_size), end=' ')
|
|
|
|
if calc_size > real_size:
|
|
|
|
print("BAAAAAAAAAAAD", real_size - calc_size)
|
2019-11-07 01:21:47 +00:00
|
|
|
if os.environ.get('DEBUG'):
|
|
|
|
print(raid)
|
|
|
|
input('waiting: ')
|
|
|
|
elif calc_size == real_size:
|
|
|
|
print("exactly right!")
|
|
|
|
r = True
|
2019-11-06 22:04:03 +00:00
|
|
|
else:
|
2019-11-07 20:47:34 +00:00
|
|
|
print("subiquity wasted space", real_size - calc_size)
|
2019-11-06 22:04:03 +00:00
|
|
|
r = True
|
|
|
|
finally:
|
|
|
|
cleanup()
|
2019-11-06 21:32:09 +00:00
|
|
|
return r
|
|
|
|
|
|
|
|
|
|
|
|
fails = 0
|
2019-11-06 22:04:03 +00:00
|
|
|
run(['mount', '-t', 'tmpfs', 'tmpfs', tmpdir])
|
|
|
|
try:
|
2019-11-07 20:11:27 +00:00
|
|
|
for size in '1G', '10G', '100G', '1T', '10T':
|
2019-11-06 22:04:03 +00:00
|
|
|
size = dehumanize_size(size)
|
|
|
|
for level in raidlevels:
|
|
|
|
for count in range(2, 10):
|
|
|
|
if count >= level.min_devices:
|
|
|
|
if not verify_size_ok(level.value, [size]*count):
|
|
|
|
fails += 1
|
2019-11-06 23:56:13 +00:00
|
|
|
if not verify_size_ok(level.value, [align_down(random.randrange(size, 10*size))]*count):
|
|
|
|
fails += 1
|
|
|
|
sizes = [align_down(random.randrange(size, 10*size)) for _ in range(count)]
|
|
|
|
if not verify_size_ok(level.value, sizes):
|
|
|
|
fails += 1
|
2019-11-06 22:04:03 +00:00
|
|
|
finally:
|
2019-11-06 22:31:23 +00:00
|
|
|
run(['umount', '-l', tmpdir])
|
2019-11-06 21:32:09 +00:00
|
|
|
|
|
|
|
if fails > 0:
|
|
|
|
print("{} fails".format(fails))
|
|
|
|
sys.exit(1)
|
|
|
|
else:
|
|
|
|
print("all ok!!")
|