process keyboard data into api friendly format when building snap

This is a bit sideways from the real thing I'm working on which is
moving most of the logic of keyboard handling to the server side but
anyway. This also lets me check some assumptions while processing the
data rather than in the view code.

scripts/make-kbd-info.py

@@ -0,0 +1,59 @@
+#!/usr/bin/python3
+
+from collections import defaultdict
+import os
+import shutil
+import subprocess
+
+from subiquity.common.serialize import Serializer
+from subiquity.common.types import (
+    KeyboardLayout,
+    KeyboardVariant,
+    )
+
+tdir = os.path.join(os.environ.get('SNAPCRAFT_PART_INSTALL', '.'), 'kbds')
+if os.path.exists(tdir):
+    shutil.rmtree(tdir)
+os.mkdir(tdir)
+
+p = subprocess.Popen(
+    ['/usr/share/console-setup/kbdnames-maker',
+     '/usr/share/console-setup/KeyboardNames.pl'],
+    stdout=subprocess.PIPE, encoding='utf-8')
+
+
+lang_to_layouts = defaultdict(dict)
+
+
+for line in p.stdout:
+    lang, element, name, value = line.strip().split("*", 3)
+    if element == 'model':
+        continue
+    elif element == 'variant':
+        layout = lang_to_layouts[lang][name]
+        variant, value = value.split('*', 1)
+        if not layout.variants and variant != "":
+            raise Exception(
+                "subiquity assumes all keyboard layouts have the default "
+                "variant at index 0!")
+        layout.variants.append(KeyboardVariant(code=variant, name=value))
+    elif element == 'layout':
+        lang_to_layouts[lang][name] = KeyboardLayout(
+            code=name, name=value, variants=[])
+
+
+s = Serializer(compact=True)
+
+
+for lang, layouts in lang_to_layouts.items():
+    if 'us' not in layouts:
+        raise Exception("subiquity assumes there is always a us keyboard "
+                        "layout")
+    outpath = os.path.join(tdir, lang + '.jsonl')
+    with open(outpath, 'w') as out:
+        for layout in layouts.values():
+            if len(layout.variants) == 0:
+                raise Exception(
+                    "subiquity assumes all keyboard layouts have at least one "
+                    "variant!")
+            out.write(s.to_json(KeyboardLayout, layout) + "\n")

snapcraft.yaml

@@ -122,19 +122,20 @@ parts:
     stage:
       - users-and-groups
       - reserved-usernames
-  kbdnames:
+  keyboard-data:
     plugin: nil
     build-packages:
       - console-setup
       - locales
       - xkb-data-i18n
-    override-build: |
+      - python3-attr
-      /usr/share/console-setup/kbdnames-maker /usr/share/console-setup/KeyboardNames.pl > $SNAPCRAFT_PART_INSTALL/kbdnames.txt
+    override-build: PYTHONPATH=$SNAPCRAFT_PROJECT_DIR/ $SNAPCRAFT_PROJECT_DIR/scripts/make-kbd-info.py
     stage:
-      - kbdnames.txt
+      - kbds/
   font:
     plugin: dump
     source: ./
+    source-type: git
     organize:
       font/subiquity.psf: subiquity.psf
     stage:

subiquity/client/keyboard.py

@@ -13,10 +13,10 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from collections import defaultdict
 import os
 
-from subiquity.common.types import KeyboardSetting
+from subiquity.common.serialize import Serializer
+from subiquity.common.types import KeyboardLayout, KeyboardSetting
 
 
 # Non-latin keyboard layouts that are handled in a uniform way
@@ -107,14 +107,15 @@ def for_ui(setting):
 class KeyboardList:
 
     def __init__(self):
-        self._kbnames_file = os.path.join(
+        self._kbnames_dir = os.path.join(os.environ.get("SNAP", '.'), 'kbds')
-            os.environ.get("SNAP", '.'),
+        self.serializer = Serializer(compact=True)
-            'kbdnames.txt')
         self._clear()
 
+    def _file_for_lang(self, code):
+        return os.path.join(self._kbnames_dir, code + '.jsonl')
+
     def has_language(self, code):
-        self.load_language(code)
+        return os.path.exists(self._file_for_lang(code))
-        return bool(self.layouts)
 
     def load_language(self, code):
         if code == self.current_lang:
@@ -122,33 +123,13 @@ class KeyboardList:
 
         self._clear()
 
-        with open(self._kbnames_file, encoding='utf-8') as kbdnames:
+        with open(self._file_for_lang(code)) as kbdnames:
-            self._load_file(code, kbdnames)
+            self.layouts = [
+                self.serializer.from_json(KeyboardLayout, line)
+                for line in kbdnames
+                ]
         self.current_lang = code
 
     def _clear(self):
         self.current_lang = None
-        self.layouts = {}
+        self.layouts = []
-        self.variants = defaultdict(dict)
-
-    def _load_file(self, code, kbdnames):
-        for line in kbdnames:
-            line = line.rstrip('\n')
-            got_lang, element, name, value = line.split("*", 3)
-            if got_lang != code:
-                continue
-
-            if element == "layout":
-                self.layouts[name] = value
-            elif element == "variant":
-                variantname, variantdesc = value.split("*", 1)
-                self.variants[name][variantname] = variantdesc
-
-    def lookup(self, code):
-        if ':' in code:
-            layout_code, variant_code = code.split(":", 1)
-            layout = self.layouts.get(layout_code, '?')
-            variant = self.variants.get(layout_code, {}).get(variant_code, '?')
-            return (layout, variant)
-        else:
-            return self.layouts.get(code, '?'), None

subiquity/common/types.py

@@ -99,6 +99,19 @@ class KeyboardSetting:
     toggle: Optional[str] = None
 
 
+@attr.s(auto_attribs=True)
+class KeyboardVariant:
+    code: str
+    name: str
+
+
+@attr.s(auto_attribs=True)
+class KeyboardLayout:
+    code: str
+    name: str
+    variants: List[KeyboardVariant]
+
+
 @attr.s(auto_attribs=True)
 class ZdevInfo:
     id: str

subiquity/ui/views/keyboard.py

@@ -13,6 +13,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import locale
 import logging
 
 from urwid import (
@@ -44,7 +45,7 @@ from subiquitycore.ui.stretchy import (
 from subiquitycore.ui.utils import button_pile, Color, Padding, screen
 from subiquitycore.view import BaseView
 
-from subiquity.client.keyboard import latinizable, for_ui
+from subiquity.client.keyboard import for_ui, latinizable
 from subiquity.common.types import KeyboardSetting
 from subiquity.ui.views import pc105
 
@@ -127,21 +128,23 @@ another layout or run the automated detection again.
 """)
 
     def ok(self, sender):
-        self.keyboard_detector.keyboard_view.found_layout(self.step.result)
+        self.keyboard_detector.keyboard_view.found_layout(
+            self.layout, self.variant)
 
     def make_body(self):
-        kl = self.keyboard_detector.keyboard_view.keyboard_list
+        if ':' in self.step.result:
-        layout, variant = kl.lookup(self.step.result)
+            layout_code, variant_code = self.step.result.split(':')
-        var_desc = []
+        else:
+            layout_code, variant_code = self.step.result, ""
+        view = self.keyboard_detector.keyboard_view
+        self.layout, self.variant = view.lookup(layout_code, variant_code)
         layout_text = _("Layout")
         var_text = _("Variant")
         width = max(len(layout_text), len(var_text), 12)
-        if variant is not None:
-            var_desc = [Text("%*s: %s" % (width, var_text, variant))]
         return Pile([
                 Text(_(self.preamble)),
-                Text("%*s: %s" % (width, layout_text, layout)),
+                Text("%*s: %s" % (width, layout_text, self.layout.name)),
-            ] + var_desc + [
+                Text("%*s: %s" % (width, var_text, self.variant.name)),
                 Text(_(self.postamble)),
                 button_pile([ok_btn(label=_("OK"), on_press=self.ok)]),
                 ])
@@ -394,25 +397,16 @@ class KeyboardView(BaseView):
 
         self.form = KeyboardForm()
         opts = []
-        for layout, desc in self.keyboard_list.layouts.items():
+        for layout in self.keyboard_list.layouts:
-            opts.append(Option((desc, True, layout)))
+            opts.append(Option((layout.name, True, layout)))
-        opts.sort(key=lambda o: o.label.text)
+        opts.sort(key=lambda o: locale.strxfrm(o.label.text))
         connect_signal(self.form, 'submit', self.done)
         connect_signal(self.form, 'cancel', self.cancel)
         connect_signal(self.form.layout.widget, "select", self.select_layout)
         self.form.layout.widget.options = opts
         setting = for_ui(initial_setting)
-        try:
+        layout, variant = self.lookup(setting.layout, setting.variant)
-            self.form.layout.widget.value = setting.layout
+        self.set_values(layout, variant)
-        except AttributeError:
-            # Don't crash on pre-existing invalid config.
-            pass
-        self.select_layout(None, setting.layout)
-        try:
-            self.form.variant.widget.value = setting.variant
-        except AttributeError:
-            # Don't crash on pre-existing invalid config.
-            pass
 
         if self.controller.opts.run_on_serial:
             excerpt = _('Please select the layout of the keyboard directly '
@@ -437,32 +431,24 @@ class KeyboardView(BaseView):
             narrow_rows=True))
 
     def detect(self, sender):
-        detector = Detector(self)
+        Detector(self).start()
-        detector.start()
 
-    def found_layout(self, result):
+    def found_layout(self, layout, variant):
         self.remove_overlay()
-        log.debug("found_layout %s", result)
+        log.debug("found_layout %r %r", layout.code, variant.code)
-        if ':' in result:
+        self.set_values(layout, variant)
-            layout, variant = result.split(':')
-        else:
-            layout, variant = result, ""
-        self.form.layout.widget.value = layout
-        self.select_layout(None, layout)
-        self.form.variant.widget.value = variant
         self._w.base_widget.focus_position = 4
 
     def done(self, result):
-        layout = self.form.layout.widget.value
+        data = result.as_data()
-        variant = ''
+        layout = data['layout']
-        if self.form.variant.widget.value is not None:
+        variant = data.get('variant', layout.variants[0])
-            variant = self.form.variant.widget.value
+        setting = KeyboardSetting(layout=layout.code, variant=variant.code)
-        setting = KeyboardSetting(layout=layout, variant=variant)
+        other = latinizable(setting)
-        new_setting = latinizable(setting)
+        if setting != other:
-        if new_setting != setting:
+            self.show_stretchy_overlay(ToggleQuestion(self, other))
-            self.show_stretchy_overlay(ToggleQuestion(self, new_setting))
+        else:
-            return
+            self.really_done(setting)
-        self.really_done(setting)
 
     def really_done(self, setting):
         apply = False
@@ -479,18 +465,29 @@ class KeyboardView(BaseView):
         if sender is not None:
             log.debug("select_layout %s", layout)
         opts = []
-        default_i = -1
+        for variant in layout.variants:
-        layout_items = enumerate(self.keyboard_list.variants[layout].items())
+            opts.append(Option((variant.name, True, variant)))
-        for i, (variant, variant_desc) in layout_items:
+        # ./scripts/make-kbd-info.py checks that the default is always
-            if variant == "":
+        # at index 0
-                default_i = i
+        opts[1:] = sorted(opts[1:], key=lambda o: locale.strxfrm(o.label.text))
-            opts.append(Option((variant_desc, True, variant)))
-        opts.sort(key=lambda o: o.label.text)
-        if default_i < 0:
-            opts.insert(0, Option(("default", True, "")))
         self.form.variant.widget.options = opts
-        if default_i < 0:
+        self.form.variant.widget.index = 0
-            self.form.variant.widget.index = 0
-        else:
-            self.form.variant.widget.index = default_i
         self.form.variant.enabled = len(opts) > 1
+
+    def lookup(self, layout_code, variant_code):
+        for layout in self.keyboard_list.layouts:
+            if layout.code == layout_code:
+                break
+            if layout.code == "us":
+                default = layout
+        else:
+            layout = default
+        for variant in layout.variants:
+            if variant.code == variant_code:
+                return layout, variant
+        return layout, layout.variants[0]
+
+    def set_values(self, layout, variant):
+        self.form.layout.widget.value = layout
+        self.select_layout(None, layout)
+        self.form.variant.widget.value = variant