compose: Add X11 compose sequences

compile.py implements a parser for X11's Compose.pre files. A lot of
code is necessary to interpret character names but thanksfully, the name
of most characters is contained in the file.

The state machine is compiled into two char arrays which unfortunately
requires an expensive initialisation and allocation.

.gitattributes

@@ -0,0 +1 @@
+srcs/juloo.keyboard2/ComposeKeyData.java -diff

build.gradle

@@ -143,7 +143,7 @@ tasks.register('compileComposeSequences') {
   println "\nGenerating ${out}"
   exec {
     def sequences = new File(projectDir, "srcs/compose").listFiles().findAll {
-      it.name.endsWith(".txt")
+      !it.name.endsWith(".py")
     }
     workingDir = projectDir
     commandLine("python", "srcs/compose/compile.py", *sequences)

check_layout.output

@@ -65,7 +65,7 @@ Layout doesn't define some important keys, missing: f11_placeholder, f12_placeho
 1 warnings
 # latn_bone
 Layout includes some ASCII punctuation but not all, missing: $
-Layout redefines the bottom row but some important keys are missing, missing: cursor_left, cursor_right, loc end, loc home, loc page_down, loc page_up, loc switch_greekmath, loc voice_typing, switch_backward
+Layout redefines the bottom row but some important keys are missing, missing: compose, cursor_left, cursor_right, loc end, loc home, loc page_down, loc page_up, loc switch_greekmath, loc voice_typing, switch_backward
 2 warnings
 # latn_colemak
 Some keys contain whitespaces, unexpected:  ́
@@ -73,7 +73,7 @@ Some keys contain whitespaces, unexpected:  ́
 # latn_dvorak
 0 warnings
 # latn_neo2
-Layout redefines the bottom row but some important keys are missing, missing: loc end, loc home, loc page_down, loc page_up
+Layout redefines the bottom row but some important keys are missing, missing: compose, loc end, loc home, loc page_down, loc page_up
 1 warnings
 # latn_qwerty_br
 0 warnings

srcs/compose/compile.py

@@ -1,8 +1,99 @@
-import textwrap, sys
+import textwrap, sys, re, string
 
-def parse_sequences_file(fname):
+# Names not defined in Compose.pre
+xkb_char_extra_names = {
+        "space": " ",
+        "minus": "-",
+        "asterisk": "*",
+        "colon": ":",
+        "equal": "=",
+        "exclam": "!",
+        "grave": "`",
+        "parenleft": "(",
+        "parenright": ")",
+        "percent": "%",
+        "period": ".",
+        "plus": "+",
+        "question": "?",
+        "semicolon": ";",
+        "underscore": "_",
+        }
+
+dropped_sequences = 0
+
+# Parse XKB's Compose.pre files
+def parse_sequences_file_xkb(fname):
+    # Parse a line of the form:
+    #     <Multi_key> <minus> <space>		: "~"	asciitilde # TILDE
+    # Sequences not starting with <Multi_key> are ignored.
+    line_re = re.compile(r'^((?:\s*<[^>]+>)+)\s*:\s*"((?:[^"\\]+|\\.)+)"\s*(\S+)?\s*(?:#.+)?$')
+    char_re = re.compile(r'\s*<(?:U([a-fA-F0-9]{4,6})|([^>]+))>')
+    def parse_seq_line(line):
+        global dropped_sequences
+        prefix = "<Multi_key>"
+        if not line.startswith(prefix):
+            return None
+        m = re.match(line_re, line[len(prefix):])
+        if m == None:
+            return None
+        def_ = m.group(1)
+        try:
+            def_ = parse_seq_chars(def_)
+            result = parse_seq_result(m.group(2))
+        except Exception as e:
+            # print(str(e) + ". Sequence dropped: " + line.strip(), file=sys.stderr)
+            dropped_sequences += 1
+            return None
+        return def_, result
+    char_names = { **xkb_char_extra_names }
+    # Interpret character names of the form "U0000" or using [char_names].
+    def parse_seq_char(c):
+        uchar, named_char = c
+        if uchar != "":
+            return chr(int(uchar, 16))
+        # else is a named char
+        if len(named_char) == 1:
+            return named_char
+        if not named_char in char_names:
+            raise Exception("Unknown char: " + named_char)
+        return char_names[named_char]
+    # Interpret the left hand side of a sequence.
+    def parse_seq_chars(def_):
+        return list(map(parse_seq_char, re.findall(char_re, def_)))
+    # Interpret the result of a sequence, as outputed by [line_re].
+    def parse_seq_result(r):
+        if len(r) == 2 and r[0] == '\\':
+            return r[1]
+        # The state machine can't represent characters that do not fit in a
+        # 16-bit char. This breaks some sequences that output letters with
+        # combined diacritics or emojis.
+        if len(r) > 1 or ord(r[0]) > 65535:
+            raise Exception("Char out of range: " + r)
+        return r
+    # Populate [char_names] with the information present in the file.
     with open(fname, "r") as inp:
-        return [ (s[:-2], s[-2]) for s in inp if len(s) > 1 ]
+        for line in inp:
+            m = re.match(line_re, line)
+            if m == None or m.group(3) == None:
+                continue
+            try:
+                char_names[m.group(3)] = parse_seq_result(m.group(2))
+            except Exception:
+                pass
+    # Parse the sequences
+    with open(fname, "r") as inp:
+        seqs = []
+        for line in inp:
+            s = parse_seq_line(line)
+            if s != None:
+                seqs.append(s)
+        return seqs
+
+# Format of the sequences file is determined by its extension
+def parse_sequences_file(fname):
+    if fname.endswith(".pre"):
+        return parse_sequences_file_xkb(fname)
+    raise Exception(fname + ": Unsupported format")
 
 # Turn a list of sequences into a trie.
 def add_sequences_to_trie(seqs, trie):
@@ -26,7 +117,7 @@ def make_automata(tree_root):
         i = len(states)
         s = len(t.keys())
         # Add node header
-        states.append((0, s + 1))
+        states.append(("\0", s + 1))
         i += 1
         # Reserve space for the current node in both arrays
         for c in range(s):
@@ -47,27 +138,53 @@ def make_automata(tree_root):
     add_tree(tree_root)
     return states
 
+def batched(ar, n):
+    i = 0
+    while i + n < len(ar):
+        yield ar[i:i+n]
+        i += n
+    if i < len(ar):
+        yield ar[i:]
+
 # Print the state machine compiled by make_automata into java code that can be
 # used by [ComposeKeyData.java].
 def gen_java(machine):
-    def gen_array(array, indent):
+    chars_map = {
-        return textwrap.fill(", ".join(map(str, array)), subsequent_indent=indent)
+            # These characters cannot be used in unicode form as Java's parser
+            # unescape unicode sequences before parsing.
+            "\"": "\\\"",
+            "\\": "\\\\",
+            "\n": "\\n",
+            ord("\""): "\\\"",
+            ord("\\"): "\\\\",
+            ord("\n"): "\\n",
+            }
+    def char_repr(c):
+        if c in chars_map:
+            return chars_map[c]
+        if type(c) == int: # The edges array contains ints
+            return "\\u%04x" % c
+        if c in string.printable:
+            return c
+        return "\\u%04x" % ord(c)
+    def gen_array(array):
+        chars = list(map(char_repr, array))
+        return "\" +\n    \"".join(map(lambda b: "".join(b), batched(chars, 72)))
     print("""package juloo.keyboard2;
 
 /** This file is generated, see [srcs/compose/compile.py]. */
 
 public final class ComposeKeyData
 {
-  public static final char[] states = {
+  public static final char[] states =
-    %s
+    ("%s").toCharArray();
-  };
 
-  public static final short[] edges = {
+  public static final char[] edges =
-    %s
+    ("%s").toCharArray();
-  };
 }""" % (
-    gen_array(map(lambda s: repr(s[0]), machine), '    '),
+    # Break the edges array every few characters using string concatenation.
-    gen_array(map(lambda s: s[1], machine), '    '),
+    gen_array(map(lambda s: s[0], machine)),
+    gen_array(map(lambda s: s[1], machine)),
 ))
 
 total_sequences = 0
@@ -76,5 +193,6 @@ for fname in sys.argv[1:]:
     sequences = parse_sequences_file(fname)
     add_sequences_to_trie(sequences, trie)
     total_sequences += len(sequences)
-gen_java(make_automata(trie))
+automata = make_automata(trie)
-print("Compiled %d sequences" % total_sequences, file=sys.stderr)
+gen_java(automata)
+print("Compiled %d sequences into %d states. Dropped %d sequences." % (total_sequences, len(automata), dropped_sequences), file=sys.stderr)

srcs/compose/sequences.txt

@@ -1,4 +0,0 @@
-=e€
-`eè
-`aà
-`uù

srcs/juloo.keyboard2/ComposeKey.java

@@ -28,7 +28,7 @@ public final class ComposeKey
   static KeyValue apply(int state, char c)
   {
     char[] states = ComposeKeyData.states;
-    short[] edges = ComposeKeyData.edges;
+    char[] edges = ComposeKeyData.edges;
     int length = edges[state];
     int next = Arrays.binarySearch(states, state + 1, state + length, c);
     if (next < 0)
@@ -42,8 +42,8 @@ public final class ComposeKey
 
   /** The [states] array represents the different states and their transition.
       A state occupies one or several cells of the array:
-      - The first cell is the result of the conpose sequence if the state is of
+      - The first cell is the result of the compose sequence if the state is of
-        size 1, [0] otherwise.
+        size 1, unspecified otherwise.
       - The remaining cells are the transitions, sorted alphabetically.
 
       The [edges] array represents the transition state corresponding to each