From 50ce450f2ae48287a69dbd896d84ab78f4b3a753 Mon Sep 17 00:00:00 2001 From: Michael Hudson-Doyle Date: Fri, 2 Jul 2021 13:48:45 +1200 Subject: [PATCH 1/4] add a gitdeps target to Makefile that checks out probert and curtin This uses a silly python script to get the exact version that the snapcraft.yaml specifies. --- Makefile | 11 ++++++++--- scripts/checkout-part.py | 27 +++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) create mode 100755 scripts/checkout-part.py diff --git a/Makefile b/Makefile index adf0d5b1..c3984d00 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ # NAME=subiquity PYTHONSRC=$(NAME) -PYTHONPATH=$(shell pwd):$(shell pwd)/probert +PYTHONPATH=$(shell pwd):$(shell pwd)/probert:$(shell pwd)/curtin PROBERTDIR=./probert PROBERT_REPO=https://github.com/canonical/probert DRYRUN?=--dry-run --bootloader uefi --machine-config examples/simple.json @@ -25,7 +25,7 @@ install_deps: sudo apt-get install -y python3-urwid python3-pyudev python3-nose python3-flake8 \ python3-yaml python3-coverage python3-dev pkg-config libnl-genl-3-dev \ libnl-route-3-dev python3-attr python3-distutils-extra python3-requests \ - python3-requests-unixsocket python3-jsonschema python3-curtin python3-apport \ + python3-requests-unixsocket python3-jsonschema python3-apport \ python3-bson xorriso isolinux python3-aiohttp probert cloud-init ssh-import-id i18n: @@ -59,12 +59,17 @@ integration: check: unit integration +curtin: + ./scripts/checkout-part.py curtin + probert: @if [ ! -d "$(PROBERTDIR)" ]; then \ - git clone -q $(PROBERT_REPO) $(PROBERTDIR); \ + ./scripts/checkout-part.py probert && \ (cd probert && $(PYTHON) setup.py build_ext -i); \ fi +gitdeps: curtin probert + schema: probert @$(PYTHON) -m subiquity.cmd.schema > autoinstall-schema.json diff --git a/scripts/checkout-part.py b/scripts/checkout-part.py new file mode 100755 index 00000000..a257316b --- /dev/null +++ b/scripts/checkout-part.py @@ -0,0 +1,27 @@ +#!/usr/bin/python3 +import subprocess +import sys +import yaml + +part = sys.argv[1] + + +def r(*args): + print('running', args) + subprocess.run(args, check=True) + + +with open("snapcraft.yaml") as f: + config = yaml.safe_load(f)["parts"][part] + +r('git', 'clone', config['source'], part) + + +c = None +for k in "source-commit", "source-tag", "source-branch": + if k in config: + c = config[k] + break + +if c is not None: + r('git', '-c', 'advice.detachedHead=false', '-C', part, 'checkout', c) From f9410e4e865fd2088046d46ac24f36ed202fbd92 Mon Sep 17 00:00:00 2001 From: Michael Hudson-Doyle Date: Fri, 2 Jul 2021 13:49:40 +1200 Subject: [PATCH 2/4] install probert and curtin via Makefile for testing rather than pip or apt respectively --- requirements.txt | 1 - scripts/installdeps.sh | 7 +++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 9ad0a5e6..c0a567aa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,4 +13,3 @@ requests requests-unixsocket yarl==1.5.1 # see https://github.com/aio-libs/aiohttp/issues/4972 aiohttp --e git+https://github.com/canonical/probert@b697ab779e7e056301e779f4708a9f1ce51b0027#egg=probert diff --git a/scripts/installdeps.sh b/scripts/installdeps.sh index 11c5052f..af5308ab 100755 --- a/scripts/installdeps.sh +++ b/scripts/installdeps.sh @@ -11,6 +11,9 @@ ConditionPathExists=/dev/zfs EOF cp -r /etc/systemd/system/zfs-mount.service.d/ /etc/systemd/system/zfs-share.service.d/ systemctl daemon-reload -apt-get install -o APT::Get::Always-Include-Phased-Updates=true -y --no-install-recommends libnl-3-dev libnl-genl-3-dev libnl-route-3-dev libsystemd-dev python3-distutils-extra pkg-config python3.5 python3-pip git lsb-release python3-setuptools gcc python3-dev python3-wheel curtin pep8 python3-pyflakes python3-bson make +apt-get install -o APT::Get::Always-Include-Phased-Updates=true -y --no-install-recommends libnl-3-dev libnl-genl-3-dev libnl-route-3-dev libsystemd-dev python3-distutils-extra pkg-config python3.5 python3-pip git lsb-release python3-setuptools gcc python3-dev python3-wheel pep8 python3-pyflakes python3-bson make pip3 install -r requirements.txt -python3 setup.py build + +make gitdeps + +PYTHONPATH=.:./curtin:./probert python3 setup.py build From cf931f673f930b1e620948def8faf7fa6f45ac95 Mon Sep 17 00:00:00 2001 From: Michael Hudson-Doyle Date: Mon, 5 Jul 2021 09:22:01 +1200 Subject: [PATCH 3/4] update script to update exsting checkout --- Makefile | 16 +++++++--------- scripts/checkout-part.py | 27 --------------------------- scripts/update-part.py | 40 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 36 deletions(-) delete mode 100755 scripts/checkout-part.py create mode 100755 scripts/update-part.py diff --git a/Makefile b/Makefile index c3984d00..24a40f11 100644 --- a/Makefile +++ b/Makefile @@ -50,23 +50,21 @@ flake8: @echo 'tox -e flake8' is preferred to 'make flake8' $(PYTHON) -m flake8 $(CHECK_DIRS) --exclude gettext38.py,contextlib38.py -unit: +unit: gitdeps python3 -m unittest discover -integration: +integration: gitdeps echo "Running integration tests..." ./scripts/runtests.sh check: unit integration -curtin: - ./scripts/checkout-part.py curtin +curtin: snapcraft.yaml + ./scripts/update-part.py curtin -probert: - @if [ ! -d "$(PROBERTDIR)" ]; then \ - ./scripts/checkout-part.py probert && \ - (cd probert && $(PYTHON) setup.py build_ext -i); \ - fi +probert: snapcraft.yaml + ./scripts/update-part.py probert + (cd probert && $(PYTHON) setup.py build_ext -i); gitdeps: curtin probert diff --git a/scripts/checkout-part.py b/scripts/checkout-part.py deleted file mode 100755 index a257316b..00000000 --- a/scripts/checkout-part.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/python3 -import subprocess -import sys -import yaml - -part = sys.argv[1] - - -def r(*args): - print('running', args) - subprocess.run(args, check=True) - - -with open("snapcraft.yaml") as f: - config = yaml.safe_load(f)["parts"][part] - -r('git', 'clone', config['source'], part) - - -c = None -for k in "source-commit", "source-tag", "source-branch": - if k in config: - c = config[k] - break - -if c is not None: - r('git', '-c', 'advice.detachedHead=false', '-C', part, 'checkout', c) diff --git a/scripts/update-part.py b/scripts/update-part.py new file mode 100755 index 00000000..648f1134 --- /dev/null +++ b/scripts/update-part.py @@ -0,0 +1,40 @@ +#!/usr/bin/python3 +import os +import subprocess +import sys +import yaml + +part = sys.argv[1] + + +def r(*args, **kw): + print('running', args) + subprocess.run(args, check=True, **kw) + + +def o(*args, **kw): + return subprocess.run(args, check=True, stdout=subprocess.PIPE).stdout + + +with open("snapcraft.yaml") as f: + config = yaml.safe_load(f)["parts"][part] + + +c = "master" +for k in "source-commit", "source-tag", "source-branch": + if k in config: + c = config[k] + break + +if not os.path.isdir(part): + r('git', 'clone', config['source'], part) + +expected = o('git', '-C', part, 'rev-parse', c) +actual = o('git', '-C', part, 'rev-parse', 'HEAD') + +if expected != actual: + try: + r('git', '-c', 'advice.detachedHead=false', '-C', part, 'checkout', c) + except subprocess.CalledProcessError: + r('git', '-C', part, 'fetch', config['source']) + r('git', '-c', 'advice.detachedHead=false', '-C', part, 'checkout', c) From cddfd42799ddf96dcd21e5195cb832ee01a3e8fe Mon Sep 17 00:00:00 2001 From: Michael Hudson-Doyle Date: Mon, 5 Jul 2021 09:25:01 +1200 Subject: [PATCH 4/4] update .gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 197a1879..ac9cf40f 100644 --- a/.gitignore +++ b/.gitignore @@ -63,8 +63,9 @@ logs/* venv .pybuild -# probert as its pulled externally, ignore top level only. +# top level dependencies fetched from git, ignore top level only. /probert/ +/curtin/ # installer isos *.iso