change languagelist format a bit, add some special cases

scripts/make-language-lists

@@ -1,30 +1,63 @@
 #!/usr/bin/python3
 
+import gettext
 import glob
 import gzip
 import os
+import sys
 
+DATA_PATH = '/usr/share/localechooser/languagelist.data.gz'
 
 translations = set()
 for fname in glob.glob('po/*.po'):
     translations.add(os.path.splitext(os.path.basename(fname))[0])
 
-print('found translations', translations)
-    
-console_langs = open('languagelist-console', 'w')
-ssh_langs = open('languagelist-ssh', 'w')
-data = gzip.open('/usr/share/localechooser/languagelist.data.gz', mode='rt')
 
-for line in data:
-    level, code, name, desc = line.strip().split(':')
-    if code not in translations:
-        continue
-    level = int(level)
-    new_line = "{}:{}\n".format(code, desc)
-    ssh_langs.write(new_line)
-    if level <= 2:
-        console_langs.write(new_line)
+langs = [
+    ('console', 'en_US', 'English'),
+    ('console', 'en_GB', 'English (UK)'),
+    ('ssh',     'kab',   'Tamaziɣt Taqbaylit'),
+]
 
-console_langs.close()
-ssh_langs.close()
-data.close()
+for level, code, name in langs:
+    if code in translations:
+        translations.remove(code)
+
+
+def kind_for_level(level):
+    if int(level) <= 2:
+        return "console"
+    else:
+        return "ssh"
+
+
+with gzip.open(DATA_PATH, mode='rt') as data:
+    for line in data:
+        level, code, name, desc = line.strip().split(':')
+        if code not in translations:
+            continue
+        langs.append((kind_for_level(level), code, desc))
+        translations.remove(code)
+
+with gzip.open(DATA_PATH, mode='rt') as data:
+    for line in data:
+        level, code, name, desc = line.strip().split(':')
+        code = code.split("_", 1)[0]
+        if code not in translations:
+            continue
+        level = int(level)
+        langs.append((kind_for_level(level), code, desc))
+        translations.remove(code)
+
+for code in translations:
+    if gettext.find('iso_639_3', languages=[code]):
+        native_lang = gettext.translation('iso_639_3', languages=[code])
+        native = native_lang.gettext(name).capitalize()
+        langs.append(('ssh', code, native))
+    else:
+        print('do not know native name for {}'.format(code))
+        sys.exit(1)
+
+with open('languagelist', 'w') as langlist:
+    for level, code, name in langs:
+        langlist.write("{}:{}:{}\n".format(level, code, name))