Split out curtin configuration files

Break out curtin configuration data into multiple configuration files
This sets up our initial install (network +storage) and our secondary
run (postinstall user config).

- encrypt the users password and never log the original value

Signed-off-by: Ryan Harper <ryan.harper@canonical.com>

Conflicts:
	subiquity/controllers/filesystem.py

subiquity/controllers/filesystem.py

@@ -15,6 +15,7 @@
 
 import logging
 from subiquity.controller import ControllerPolicy
+from subiquity.models.actions import preserve_action
 from subiquity.models import (FilesystemModel, IscsiDiskModel, RaidModel,
                               CephDiskModel)
 from subiquity.ui.views import (DiskPartitionView, AddPartitionView,
@@ -23,7 +24,7 @@ from subiquity.ui.views import (DiskPartitionView, AddPartitionView,
 import subiquity.utils as utils
 from subiquity.ui.dummy import DummyView
 from subiquity.curtin import (curtin_write_storage_actions,
-                              curtin_write_postinst_config)
+                              curtin_write_preserved_actions)
 
 
 log = logging.getLogger("subiquity.controller.filesystem")
@@ -58,8 +59,11 @@ class FilesystemController(ControllerPolicy):
 
         log.info("Rendering curtin config from user choices")
         curtin_write_storage_actions(actions=actions)
-        log.info("Generating post-install config")
+
-        curtin_write_postinst_config()
+        log.info("Rendering preserved config for post install")
+        preserved_actions = [preserve_action(a) for a in actions]
+        curtin_write_preserved_actions(actions=preserved_actions)
+
         self.signal.emit_signal('installprogress:do-initial-install')
         self.signal.emit_signal('identity:show')
 

subiquity/controllers/installprogress.py

@@ -19,6 +19,9 @@ import subiquity.utils as utils
 from subiquity.models import InstallProgressModel
 from subiquity.ui.views import ProgressView, ProgressOutput
 from subiquity.controller import ControllerPolicy
+from subiquity.curtin import (CURTIN_CONFIGS,
+                              curtin_install_cmd,
+                              curtin_write_postinst_config)
 
 log = logging.getLogger("subiquity.controller.installprogress")
 
@@ -34,32 +37,50 @@ class InstallProgressController(ControllerPolicy):
         self.signal.emit_signal('refresh')
 
     @coroutine
-    def curtin_dispatch(self):
+    def curtin_dispatch(self, postconfig):
+        ''' one time curtin dispatch requires the use of
+            the preserved storage config which allows executing
+            in-target commands by remounting up the configured
+            storage.
+        '''
+        log.debug('curtin dispatch using target={}'.format(
+            self.model.target_root))
         write_fd = self.loop.watch_pipe(self.install_progress_status)
+
+        log.debug('writing out postinst config')
+        curtin_write_postinst_config(postconfig)
+        configs = [CURTIN_CONFIGS['preserved'], CURTIN_CONFIGS['postinstall']]
+        curtin_cmd = curtin_install_cmd(configs, self.mount.target_root)
+        log.debug('Curtin postinstall install cmd: {}'.format(curtin_cmd))
+
         if self.opts.dry_run:
             log.debug("Install Progress: Curtin dispatch dry-run")
             yield utils.run_command_async("cat /var/log/syslog",
                                           write_fd)
         else:
             try:
-                yield utils.run_command_async("/usr/local/bin/curtin_wrap.sh",
+                yield utils.run_command_async(" ".join(curtin_cmd),
                                               write_fd)
             except:
                 log.error("Problem with curtin dispatch run")
                 raise Exception("Problem with curtin dispatch run")
 
     @coroutine
-    def initial_install(self):
+    def initial_install(self, target_root):
+        log.debug('User specified {} to hold rootfs'.format(target_root))
+        self.model.target_root = target_root
         write_fd = self.loop.watch_pipe(self.install_progress_status)
+        configs = [CURTIN_CONFIGS['network'], CURTIN_CONFIGS['storage']]
+        curtin_cmd = curtin_install_cmd(configs)
+        log.debug('Curtin install cmd: {}'.format(curtin_cmd))
         if self.opts.dry_run:
             log.debug("Filesystem: this is a dry-run")
             yield utils.run_command_async("cat /var/log/syslog",
                                           write_fd)
         else:
             log.debug("filesystem: this is the *real* thing")
-            yield utils.run_command_async(
+            yield utils.run_command_async(" ".join(curtin_cmd),
-                "/usr/local/bin/curtin_wrap.sh",
+                                          write_fd)
-                write_fd)
 
     @coroutine
     def show_progress(self):
@@ -74,8 +95,8 @@ class InstallProgressController(ControllerPolicy):
         if self.opts.dry_run:
             banner = [
                 "**** DRY_RUN ****",
-                "NOT calling:"
+                ""
-                "subprocess.check_call(/usr/local/bin/curtin_wrap.sh)"
+                "",
                 "",
                 "",
                 "Press (Q) to Quit."

subiquity/curtin.py

@@ -13,13 +13,30 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import jinja2
-import os
 import datetime
+import jinja2
+import logging
+import os
+import subprocess
 import yaml
 
-CURTIN_STORAGE_CONFIG_FILE = '/tmp/subiquity-config.yaml'
+
-CURTIN_STORAGE_CONFIG_HEADER = """
+log = logging.getLogger("subiquity.curtin")
+
+TMPDIR = '/tmp'
+CURTIN = '/usr/local/curtin/bin/curtin'
+CONF_PREFIX = os.path.join(TMPDIR, 'subiquity-config-')
+CURTIN_NETWORK_CONFIG_FILE = CONF_PREFIX + 'network.yaml'
+CURTIN_STORAGE_CONFIG_FILE = CONF_PREFIX + 'storage.yaml'
+CURTIN_PRESERVED_CONFIG_FILE = CONF_PREFIX + 'storage-preserved.yaml'
+POST_INSTALL_CONFIG_FILE = CONF_PREFIX + 'postinst.yaml'
+CURTIN_CONFIGS = {
+    'network': CURTIN_NETWORK_CONFIG_FILE,
+    'storage': CURTIN_STORAGE_CONFIG_FILE,
+    'postinstall': POST_INSTALL_CONFIG_FILE,
+    'preserved': CURTIN_PRESERVED_CONFIG_FILE,
+}
+CURTIN_CONFIG_HEADER = """
 reporter:
  subiquity:
   path: /tmp/curtin_progress_subiquity
@@ -28,10 +45,22 @@ reporter:
 partitioning_commands:
  builtin: curtin block-meta custom
 
+"""
+CURTIN_CONFIG_REBOOT = """
+power_state:
+  message: s-Ubiquity install complete. Rebooting
+  mode: reboot
+"""
+CURTIN_STORAGE_CONFIG_HEADER = """
 storage:
   version: 1
   config:
 """
+CURTIN_NETWORK_CONFIG_HEADER = """
+network:
+  version: 1
+  config:
+"""
 CURTIN_STORAGE_CONFIG_TEMPLATE = """
 # Autogenerated by SUbiquity: {{DATE}} UTC
 reporter:
@@ -79,20 +108,38 @@ storage:
       device: {{TARGET_DISK_NAME}}2_home
 """
 
-POST_INSTALL_CONFIG_FILE = '/tmp/subiquity-postinst.yaml'
 # TODO, this should be moved to the in-target cloud-config seed so on first boot
 # of the target, it reconfigures datasource_list to none for subsequent boots
 #    12_ds_to_none: [curtin, in-target, --, sh, '-c', "echo 'datasource_list: [ None ]' > /etc/cloud/cloud.cfg.d/
 POST_INSTALL = '''
 late_commands:
     10_set_hostname: curtin in-target -- sh -c 'echo $(petname) > /etc/hostname'
-    11_postinst_seed: [curtin, in-target, --, sh, '-c',"/bin/echo -e '#cloud-config\\npassword: passw0rd\\nchpasswd: { expire: False }\\n' > /var/lib/cloud/seed/nocloud-net/user-data"]
+    11_postinst_seed: [curtin, in-target, --, sh, '-c',"/bin/echo -e '#cloud-config\\npassword: passw0rd\\nchpasswd: {{ expire: False }}\\nusers:\\n{users}' > /var/lib/cloud/seed/nocloud-net/user-data"]
     12_disable_subiquity: curtin in-target -- systemctl disable subiquity.service
     13_delete_subiquity: curtin in-target -- rm -f /lib/systemd/system/subiquity.service
-    14_remove_subiquity: curtin in-target -- sh -c 'for d in probert curtin subiquity; do rm -rf /usr/local/${d}; rm -rf /usr/local/bin/${d}*; done'
+    14_remove_subiquity: curtin in-target -- sh -c 'for d in probert curtin subiquity; do rm -rf /usr/local/${{d}}; rm -rf /usr/local/bin/${{d}}*; done'
 '''
 
 
+def curtin_userinfo_to_config(userinfo):
+    user_template = '  - name: {username}\\n' + \
+        '    gecos: {realname}\\n' + \
+        '    passwd: {password}\\n'
+    return user_template.format(**userinfo)
+
+
+def curtin_write_postinst_config(userinfo):
+    config = {
+        'users': curtin_userinfo_to_config(userinfo)
+    }
+    with open(POST_INSTALL_CONFIG_FILE, 'w') as conf:
+        datestr = '# Autogenerated by SUbiquity: {} UTC'.format(
+            str(datetime.datetime.utcnow()))
+        conf.write(datestr)
+        conf.write(POST_INSTALL.format(**config))
+        conf.close()
+
+
 def curtin_write_storage_actions(actions):
     curtin_config = yaml.dump(actions, default_flow_style=False)
     curtin_config = "    " + "\n    ".join(curtin_config.splitlines())
@@ -100,7 +147,19 @@ def curtin_write_storage_actions(actions):
         str(datetime.datetime.utcnow()))
     with open(CURTIN_STORAGE_CONFIG_FILE, 'w') as conf:
         conf.write(datestr)
-        conf.write(CURTIN_STORAGE_CONFIG_HEADER)
+        conf.write(CURTIN_CONFIG_HEADER + CURTIN_STORAGE_CONFIG_HEADER)
+        conf.write(curtin_config)
+        conf.close()
+
+
+def curtin_write_network_actions(actions):
+    curtin_config = yaml.dump(actions, default_flow_style=False)
+    curtin_config = "    " + "\n    ".join(curtin_config.splitlines())
+    datestr = '# Autogenerated by SUbiquity: {} UTC'.format(
+        str(datetime.datetime.utcnow()))
+    with open(CURTIN_NETWORK_CONFIG_FILE, 'w') as conf:
+        conf.write(datestr)
+        conf.write(CURTIN_CONFIG_HEADER + CURTIN_NETWORK_CONFIG_HEADER)
         conf.write(curtin_config)
         conf.close()
 
@@ -129,10 +188,42 @@ def curtin_write_storage_template(disk_name, disk_model, disk_serial):
     return CURTIN_STORAGE_CONFIG_FILE
 
 
-def curtin_write_postinst_config():
+def curtin_write_preserved_actions(actions):
-    with open(POST_INSTALL_CONFIG_FILE, 'w') as conf:
+    ''' caller must use models.actions.preserve_action on
-        datestr = '# Autogenerated by SUbiquity: {} UTC'.format(
+        all elements of the actions'''
-            str(datetime.datetime.utcnow()))
+    curtin_config = yaml.dump(actions, default_flow_style=False)
+    curtin_config = "    " + "\n    ".join(curtin_config.splitlines())
+    datestr = '# Autogenerated by SUbiquity: {} UTC'.format(
+        str(datetime.datetime.utcnow()))
+    with open(CURTIN_PRESERVED_CONFIG_FILE, 'w') as conf:
         conf.write(datestr)
-        conf.write(POST_INSTALL)
+        conf.write(CURTIN_CONFIG_HEADER + CURTIN_STORAGE_CONFIG_HEADER)
+        conf.write(curtin_config)
         conf.close()
+
+
+def curtin_install_cmd(configs):
+    '''
+    curtin -v --showtrace install -c $CONFIGS cp:///
+    '''
+    install_cmd = [CURTIN, '-v', '--showtrace']
+    if configs:
+        install_cmd += ['-c {}'.format(c) for c in configs]
+    install_cmd += ['install', 'cp:///']
+    log.info('curtin install command: {}'.format(" ".join(install_cmd)))
+
+    return install_cmd
+
+
+def curtin_reboot():
+    cmd = "/sbin/reboot"
+    log.info("powering off with %s", cmd)
+    fid = os.fork()
+    if fid == 0:
+        try:
+            subprocess.call([cmd])
+            os._exit(0)
+        except:
+            log.warn("%s returned non-zero" % cmd)
+            os._exit(1)
+    return

subiquity/models/actions.py

@@ -13,6 +13,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import copy
 import yaml
 
 
@@ -217,3 +218,16 @@ class MountAction(DiskAction):
             'path': self.path,
             'type': 'mount',
         }
+
+
+def preserve_action(action):
+    a = copy.deepcopy(action)
+    a.update({'preserve': True})
+    return a
+
+
+def release_action(action):
+    a = copy.deepcopy(action)
+    if 'preserve' in action:
+        del a['preserve']
+    return a

subiquity/models/blockdev.py

@@ -249,10 +249,10 @@ class Blockdev():
         log.debug('PartitionAction:\n{}'.format(part_action.get()))
 
         self.disk.partitions.update({partnum: part_action})
+        partpath = "{}{}".format(self.disk.devpath, partnum)
 
         # record filesystem formating
         if fstype:
-            partpath = "{}{}".format(self.disk.devpath, partnum)
             fs_action = FormatAction(part_action, fstype)
             log.debug('Adding filesystem on {}'.format(partpath))
             log.debug('FormatAction:\n{}'.format(fs_action.get()))

subiquity/models/identity.py

@@ -15,6 +15,7 @@
 
 import logging
 from subiquity.model import ModelPolicy
+from subiquity.utils import crypt_password
 
 
 log = logging.getLogger('subiquity.models.identity')
@@ -57,9 +58,8 @@ class IdentityModel(ModelPolicy):
             if x == selection:
                 return y
 
-    def encrypt_password(self):
+    def encrypt_password(self, passinput):
-        # TODO: implement
+        return crypt_password(passinput)
-        pass
 
     def __repr__(self):
         return "<Username: {}>".format(self.username)

subiquity/ui/views/identity.py

@@ -87,16 +87,17 @@ class IdentityView(ViewPolicy):
         return Pile(sl)
 
     def done(self, result):
-        log.debug("User input: {} {} {}".format(self.username.value,
+        cpassword = self.model.encrypt_password(self.password.value)
-                                                self.password.value,
+        log.debug("*crypted* User input: {} {} {}".format(
-                                                self.confirm_password.value))
+            self.username.value, cpassword, cpassword))
         result = {
             "username": self.username.value,
-            "password": self.password.value,
+            "password": cpassword,
-            "confirm_password": self.confirm_password.value
+            "confirm_password": cpassword,
         }
+
         log.debug("User input: {}".format(result))
-        self.signal.emit_signal('installprogress:curtin-dispatch')
+        self.signal.emit_signal('installprogress:curtin-dispatch', result)
         self.signal.emit_signal('installprogress:show')
 
     def cancel(self, button):

subiquity/utils.py

@@ -13,11 +13,13 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import os
+import crypt
-from subprocess import Popen, PIPE
-from subiquity.async import Async
 import errno
 import logging
+import os
+import random
+from subprocess import Popen, PIPE
+from subiquity.async import Async
 
 log = logging.getLogger("subiquity.utils")
 SYS_CLASS_NET = "/sys/class/net/"
@@ -87,3 +89,19 @@ def read_sys_net(devname, path, translate=None, enoent=None, keyerror=None):
         if e.errno == errno.ENOENT and enoent is not None:
             return enoent
         raise
+
+
+## FIXME: replace with passlib and update package deps
+def crypt_password(passwd, algo='SHA-512'):
+    # encryption algo - id pairs for crypt()
+    algos = {'SHA-512': '$6$', 'SHA-256': '$5$', 'MD5': '$1$', 'DES': ''}
+    if algo not in algos:
+        raise Exception('Invalid algo({}), must be one of: {}. '.format(
+            algo, ','.join(algos.keys())))
+
+    salt_set = ('abcdefghijklmnopqrstuvwxyz'
+                'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+                '0123456789./')
+    salt = 16 * ' '
+    salt = ''.join([random.choice(salt_set) for c in salt])
+    return crypt.crypt(passwd, algos[algo] + salt)