StealthOS/subiquity
StealthOS
/
subiquity
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1126 from dbungert/apitest-less-flaky 

Make api tests less flaky
This commit is contained in:
Dan Bungert 2021-11-22 16:55:14 -07:00 committed by GitHub
parent 200ceb1d58 9ef4e929f7
commit dc87d8075e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 47 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
subiquity/tests/api/test_api.py
View File

@ -14,6 +14,8 @@ from urllib.parse import unquote
from subiquitycore.utils import astart_command from subiquitycore.utils import astart_command
default_timeout = 10
def find(items, key, value): def find(items, key, value):
for item in items: for item in items:
@ -25,11 +27,11 @@ def first(items, key, value):
return next(find(items, key, value)) return next(find(items, key, value))
def timeout(_timeout): def timeout(multiplier=1):
def wrapper(coro): def wrapper(coro):
@wraps(coro) @wraps(coro)
async def run(*args, **kwargs): async def run(*args, **kwargs):
with async_timeout.timeout(_timeout): with async_timeout.timeout(default_timeout * multiplier):
return await coro(*args, **kwargs) return await coro(*args, **kwargs)
return run return run
return wrapper return wrapper
@ -78,9 +80,15 @@ class Client:
return self.loads(content) return self.loads(content)
async def poll_startup(self): async def poll_startup(self):
for _ in range(20): for _ in range(default_timeout * 10):
try: try:
await self.get('/meta/status') resp = await self.get('/meta/status')
if resp["state"] in ('STARTING_UP', 'CLOUD_INIT_WAIT',
'EARLY_COMMANDS'):
await asyncio.sleep(.5)
continue
if resp["state"] == 'ERROR':
raise Exception('server in error state')
return return
except aiohttp.client_exceptions.ClientConnectorError: except aiohttp.client_exceptions.ClientConnectorError:
await asyncio.sleep(.5) await asyncio.sleep(.5)
@ -103,12 +111,13 @@ class Server(Client):
+ ' --machine-config ' + machine_config + ' --machine-config ' + machine_config
cmd = cmd.split(' ') cmd = cmd.split(' ')
self.proc = await astart_command(cmd, env=env) self.proc = await astart_command(cmd, env=env)
self.server_task = asyncio.create_task(self.proc.communicate())
async def close(self): async def close(self):
try: try:
await asyncio.wait_for(self.server_shutdown(), timeout=5.0) await asyncio.wait_for(self.server_shutdown(), timeout=5.0)
await asyncio.wait_for(self.server_task, timeout=5.0) await asyncio.wait_for(self.proc.communicate(), timeout=5.0)
except asyncio.exceptions.TimeoutError:
pass
finally: finally:
try: try:
self.proc.kill() self.proc.kill()
@ -121,7 +130,7 @@ class TestAPI(unittest.IsolatedAsyncioTestCase):
async def poll_for_socket_exist(socket_path): async def poll_for_socket_exist(socket_path):
for _ in range(100): for _ in range(default_timeout * 5):
# test level timeout will trigger first, this loop is just a fallback # test level timeout will trigger first, this loop is just a fallback
if os.path.exists(socket_path): if os.path.exists(socket_path):
return return
@ -154,13 +163,13 @@ async def connect_server(*args, **kwargs):
class TestBitlocker(TestAPI): class TestBitlocker(TestAPI):
@timeout(5) @timeout()
async def test_has_bitlocker(self): async def test_has_bitlocker(self):
async with start_server('examples/win10.json') as inst: async with start_server('examples/win10.json') as inst:
resp = await inst.get('/storage/has_bitlocker') resp = await inst.get('/storage/has_bitlocker')
self.assertEqual(1, len(resp)) self.assertEqual(1, len(resp))
@timeout(5) @timeout()
async def test_not_bitlocker(self): async def test_not_bitlocker(self):
async with start_server('examples/simple.json') as inst: async with start_server('examples/simple.json') as inst:
resp = await inst.get('/storage/has_bitlocker') resp = await inst.get('/storage/has_bitlocker')
@ -168,7 +177,7 @@ class TestBitlocker(TestAPI):
class TestFlow(TestAPI): class TestFlow(TestAPI):
@timeout(10) @timeout(2)
async def test_server_flow(self): async def test_server_flow(self):
async with start_server('examples/simple.json') as inst: async with start_server('examples/simple.json') as inst:
await inst.post('/locale', 'en_US.UTF-8') await inst.post('/locale', 'en_US.UTF-8')
@ -209,7 +218,7 @@ class TestFlow(TestAPI):
for state in 'RUNNING', 'POST_WAIT', 'POST_RUNNING', 'UU_RUNNING': for state in 'RUNNING', 'POST_WAIT', 'POST_RUNNING', 'UU_RUNNING':
await inst.get('/meta/status', cur=state) await inst.get('/meta/status', cur=state)
@timeout(5) @timeout()
async def test_v2_flow(self): async def test_v2_flow(self):
async with start_server('examples/win10.json') as inst: async with start_server('examples/win10.json') as inst:
disk_id = 'disk-sda' disk_id = 'disk-sda'
@ -263,7 +272,7 @@ class TestFlow(TestAPI):
class TestGuided(TestAPI): class TestGuided(TestAPI):
@timeout(5) @timeout()
async def test_guided_v2(self): async def test_guided_v2(self):
async with start_server('examples/simple.json') as inst: async with start_server('examples/simple.json') as inst:
choice = {'disk_id': 'disk-sda'} choice = {'disk_id': 'disk-sda'}
@ -273,7 +282,7 @@ class TestGuided(TestAPI):
class TestAdd(TestAPI): class TestAdd(TestAPI):
@timeout(5) @timeout()
async def test_v2_add_boot_partition(self): async def test_v2_add_boot_partition(self):
async with start_server('examples/simple.json') as inst: async with start_server('examples/simple.json') as inst:
disk_id = 'disk-sda' disk_id = 'disk-sda'
@ -297,7 +306,7 @@ class TestAdd(TestAPI):
manual_add = await inst.post('/storage/v2/add_partition', data) manual_add = await inst.post('/storage/v2/add_partition', data)
self.assertEqual(single_add, manual_add) self.assertEqual(single_add, manual_add)
@timeout(5) @timeout()
async def test_v2_deny_multiple_add_boot_partition(self): async def test_v2_deny_multiple_add_boot_partition(self):
async with start_server('examples/simple.json') as inst: async with start_server('examples/simple.json') as inst:
disk_id = 'disk-sda' disk_id = 'disk-sda'
@ -306,7 +315,7 @@ class TestAdd(TestAPI):
await inst.post('/storage/v2/add_boot_partition', await inst.post('/storage/v2/add_boot_partition',
disk_id=disk_id) disk_id=disk_id)
@timeout(5) @timeout()
async def test_v2_deny_multiple_add_boot_partition_BIOS(self): async def test_v2_deny_multiple_add_boot_partition_BIOS(self):
async with start_server('examples/simple.json', 'bios') as inst: async with start_server('examples/simple.json', 'bios') as inst:
disk_id = 'disk-sda' disk_id = 'disk-sda'
@ -315,7 +324,7 @@ class TestAdd(TestAPI):
await inst.post('/storage/v2/add_boot_partition', await inst.post('/storage/v2/add_boot_partition',
disk_id=disk_id) disk_id=disk_id)
@timeout(5) @timeout()
async def test_v2_free_for_partitions(self): async def test_v2_free_for_partitions(self):
async with start_server('examples/simple.json') as inst: async with start_server('examples/simple.json') as inst:
disk_id = 'disk-sda' disk_id = 'disk-sda'
@ -338,7 +347,7 @@ class TestAdd(TestAPI):
sda = first(resp['disks'], 'id', disk_id) sda = first(resp['disks'], 'id', disk_id)
self.assertEqual(expected_free, sda['free_for_partitions']) self.assertEqual(expected_free, sda['free_for_partitions'])
@timeout(5) @timeout()
async def test_add_format_required(self): async def test_add_format_required(self):
disk_id = 'disk-sda' disk_id = 'disk-sda'
async with start_server('examples/simple.json') as inst: async with start_server('examples/simple.json') as inst:
@ -352,7 +361,7 @@ class TestAdd(TestAPI):
msg=f'data {data}'): msg=f'data {data}'):
await inst.post('/storage/v2/add_partition', data) await inst.post('/storage/v2/add_partition', data)
@timeout(5) @timeout()
async def test_add_default_size_handling(self): async def test_add_default_size_handling(self):
async with start_server('examples/simple.json') as inst: async with start_server('examples/simple.json') as inst:
disk_id = 'disk-sda' disk_id = 'disk-sda'
@ -373,7 +382,7 @@ class TestAdd(TestAPI):
sda2 = first(sda['partitions'], 'number', 2) sda2 = first(sda['partitions'], 'number', 2)
self.assertEqual(expected_total, sda1['size'] + sda2['size']) self.assertEqual(expected_total, sda1['size'] + sda2['size'])
@timeout(5) @timeout()
async def test_v2_add_boot_BIOS(self): async def test_v2_add_boot_BIOS(self):
async with start_server('examples/simple.json', 'bios') as inst: async with start_server('examples/simple.json', 'bios') as inst:
disk_id = 'disk-sda' disk_id = 'disk-sda'
@ -384,7 +393,7 @@ class TestAdd(TestAPI):
self.assertTrue(sda['boot_device']) self.assertTrue(sda['boot_device'])
self.assertTrue(sda1['boot']) self.assertTrue(sda1['boot'])
@timeout(5) @timeout()
async def test_v2_blank_is_not_boot(self): async def test_v2_blank_is_not_boot(self):
async with start_server('examples/simple.json', 'bios') as inst: async with start_server('examples/simple.json', 'bios') as inst:
disk_id = 'disk-sda' disk_id = 'disk-sda'
@ -392,7 +401,7 @@ class TestAdd(TestAPI):
sda = first(resp['disks'], 'id', disk_id) sda = first(resp['disks'], 'id', disk_id)
self.assertFalse(sda['boot_device']) self.assertFalse(sda['boot_device'])
@timeout(5) @timeout()
async def test_v2_multi_disk_multi_boot(self): async def test_v2_multi_disk_multi_boot(self):
async with start_server('examples/many-nics-and-disks.json') as inst: async with start_server('examples/many-nics-and-disks.json') as inst:
resp = await inst.get('/storage/v2') resp = await inst.get('/storage/v2')
@ -406,7 +415,7 @@ class TestAdd(TestAPI):
class TestDelete(TestAPI): class TestDelete(TestAPI):
@timeout(5) @timeout()
async def test_v2_delete_without_reformat(self): async def test_v2_delete_without_reformat(self):
async with start_server('examples/win10.json') as inst: async with start_server('examples/win10.json') as inst:
disk_id = 'disk-sda' disk_id = 'disk-sda'
@ -417,7 +426,7 @@ class TestDelete(TestAPI):
with self.assertRaises(ClientResponseError): with self.assertRaises(ClientResponseError):
await inst.post('/storage/v2/delete_partition', data) await inst.post('/storage/v2/delete_partition', data)
@timeout(5) @timeout()
async def test_v2_delete_with_reformat(self): async def test_v2_delete_with_reformat(self):
async with start_server('examples/win10.json') as inst: async with start_server('examples/win10.json') as inst:
disk_id = 'disk-sda' disk_id = 'disk-sda'
@ -436,7 +445,7 @@ class TestDelete(TestAPI):
} }
await inst.post('/storage/v2/delete_partition', data) await inst.post('/storage/v2/delete_partition', data)
@timeout(5) @timeout()
async def test_delete_nonexistant(self): async def test_delete_nonexistant(self):
async with start_server('examples/win10.json') as inst: async with start_server('examples/win10.json') as inst:
disk_id = 'disk-sda' disk_id = 'disk-sda'
@ -450,7 +459,7 @@ class TestDelete(TestAPI):
class TestEdit(TestAPI): class TestEdit(TestAPI):
@timeout(5) @timeout()
async def test_edit_no_change_size(self): async def test_edit_no_change_size(self):
async with start_server('examples/win10.json') as inst: async with start_server('examples/win10.json') as inst:
disk_id = 'disk-sda' disk_id = 'disk-sda'
@ -468,7 +477,7 @@ class TestEdit(TestAPI):
with self.assertRaises(ClientResponseError): with self.assertRaises(ClientResponseError):
await inst.post('/storage/v2/edit_partition', data) await inst.post('/storage/v2/edit_partition', data)
@timeout(5) @timeout()
async def test_edit_no_change_grub(self): async def test_edit_no_change_grub(self):
async with start_server('examples/win10.json') as inst: async with start_server('examples/win10.json') as inst:
disk_id = 'disk-sda' disk_id = 'disk-sda'
@ -482,7 +491,7 @@ class TestEdit(TestAPI):
with self.assertRaises(ClientResponseError): with self.assertRaises(ClientResponseError):
await inst.post('/storage/v2/edit_partition', data) await inst.post('/storage/v2/edit_partition', data)
@timeout(5) @timeout()
async def test_edit_format(self): async def test_edit_format(self):
async with start_server('examples/win10.json') as inst: async with start_server('examples/win10.json') as inst:
disk_id = 'disk-sda' disk_id = 'disk-sda'
@ -499,7 +508,7 @@ class TestEdit(TestAPI):
sda3 = first(sda['partitions'], 'number', 3) sda3 = first(sda['partitions'], 'number', 3)
self.assertEqual('btrfs', sda3['format']) self.assertEqual('btrfs', sda3['format'])
@timeout(5) @timeout()
async def test_edit_mount(self): async def test_edit_mount(self):
async with start_server('examples/win10.json') as inst: async with start_server('examples/win10.json') as inst:
disk_id = 'disk-sda' disk_id = 'disk-sda'
@ -516,7 +525,7 @@ class TestEdit(TestAPI):
sda3 = first(sda['partitions'], 'number', 3) sda3 = first(sda['partitions'], 'number', 3)
self.assertEqual('/', sda3['mount']) self.assertEqual('/', sda3['mount'])
@timeout(5) @timeout()
async def test_edit_format_and_mount(self): async def test_edit_format_and_mount(self):
async with start_server('examples/win10.json') as inst: async with start_server('examples/win10.json') as inst:
disk_id = 'disk-sda' disk_id = 'disk-sda'
@ -535,7 +544,7 @@ class TestEdit(TestAPI):
self.assertEqual('btrfs', sda3['format']) self.assertEqual('btrfs', sda3['format'])
self.assertEqual('/', sda3['mount']) self.assertEqual('/', sda3['mount'])
@timeout(5) @timeout()
async def test_v2_reuse(self): async def test_v2_reuse(self):
async with start_server('examples/win10.json') as inst: async with start_server('examples/win10.json') as inst:
disk_id = 'disk-sda' disk_id = 'disk-sda'
@ -574,21 +583,21 @@ class TestEdit(TestAPI):
class TestPartitionTableTypes(TestAPI): class TestPartitionTableTypes(TestAPI):
@timeout(5) @timeout()
async def test_ptable_gpt(self): async def test_ptable_gpt(self):
async with start_server('examples/win10.json') as inst: async with start_server('examples/win10.json') as inst:
resp = await inst.get('/storage/v2') resp = await inst.get('/storage/v2')
sda = first(resp['disks'], 'id', 'disk-sda') sda = first(resp['disks'], 'id', 'disk-sda')
self.assertEqual('gpt', sda['ptable']) self.assertEqual('gpt', sda['ptable'])
@timeout(5) @timeout()
async def test_ptable_msdos(self): async def test_ptable_msdos(self):
async with start_server('examples/many-nics-and-disks.json') as inst: async with start_server('examples/many-nics-and-disks.json') as inst:
resp = await inst.get('/storage/v2') resp = await inst.get('/storage/v2')
sda = first(resp['disks'], 'id', 'disk-sda') sda = first(resp['disks'], 'id', 'disk-sda')
self.assertEqual('msdos', sda['ptable']) self.assertEqual('msdos', sda['ptable'])
@timeout(5) @timeout()
async def test_ptable_none(self): async def test_ptable_none(self):
async with start_server('examples/simple.json') as inst: async with start_server('examples/simple.json') as inst:
resp = await inst.get('/storage/v2') resp = await inst.get('/storage/v2')
@ -597,7 +606,7 @@ class TestPartitionTableTypes(TestAPI):
class TestTodos(TestAPI): # server indicators of required client actions class TestTodos(TestAPI): # server indicators of required client actions
@timeout(5) @timeout()
async def test_todos_simple(self): async def test_todos_simple(self):
async with start_server('examples/simple.json') as inst: async with start_server('examples/simple.json') as inst:
disk_id = 'disk-sda' disk_id = 'disk-sda'
@ -617,7 +626,7 @@ class TestTodos(TestAPI): # server indicators of required client actions
self.assertFalse(resp['need_root']) self.assertFalse(resp['need_root'])
self.assertFalse(resp['need_boot']) self.assertFalse(resp['need_boot'])
@timeout(5) @timeout()
async def test_todos_manual(self): async def test_todos_manual(self):
async with start_server('examples/simple.json') as inst: async with start_server('examples/simple.json') as inst:
disk_id = 'disk-sda' disk_id = 'disk-sda'
@ -642,7 +651,7 @@ class TestTodos(TestAPI): # server indicators of required client actions
self.assertFalse(resp['need_root']) self.assertFalse(resp['need_root'])
self.assertFalse(resp['need_boot']) self.assertFalse(resp['need_boot'])
@timeout(5) @timeout()
async def test_todos_guided(self): async def test_todos_guided(self):
async with start_server('examples/simple.json') as inst: async with start_server('examples/simple.json') as inst:
disk_id = 'disk-sda' disk_id = 'disk-sda'
@ -658,7 +667,7 @@ class TestTodos(TestAPI): # server indicators of required client actions
class TestInfo(TestAPI): class TestInfo(TestAPI):
@timeout(5) @timeout()
async def test_path(self): async def test_path(self):
async with start_server('examples/simple.json') as inst: async with start_server('examples/simple.json') as inst:
disk_id = 'disk-sda' disk_id = 'disk-sda'
@ -668,7 +677,7 @@ class TestInfo(TestAPI):
class TestRegression(TestAPI): class TestRegression(TestAPI):
@timeout(5) @timeout()
async def test_edit_not_trigger_boot_device(self): async def test_edit_not_trigger_boot_device(self):
async with start_server('examples/simple.json') as inst: async with start_server('examples/simple.json') as inst:
disk_id = 'disk-sda' disk_id = 'disk-sda'