Send down event for modifiers on time

This allows to use modifiers in combination with other inputs like a
mouse click, for example under termux-x11.

The key down event and notification about modifiers changing are sent
down to KeyEventHandler. A mutable state remember for which modifier
down events have been sent.

When pressing down a modifier with one finger and typing with the
other, it might appear that the modifier is released after the first
time an other key is pressed and then pressed and released for the
following keys.
This prevents unintentionally type two modified keys instead of one
when the second key is pressed while the other is not yet released.

srcs/juloo.keyboard2/Config.java

@@ -400,7 +400,8 @@ public final class Config
   public static interface IKeyEventHandler
   {
     public void key_down(KeyValue value, boolean is_swipe);
-    public void key_up(KeyValue value, Pointers.Modifiers flags);
+    public void key_up(KeyValue value, Pointers.Modifiers mods);
+    public void mods_changed(Pointers.Modifiers mods);
   }
 
   /** Config migrations. */

srcs/juloo.keyboard2/KeyEventHandler.java

@@ -6,17 +6,26 @@ import android.view.inputmethod.EditorInfo;
 import android.view.inputmethod.ExtractedText;
 import android.view.inputmethod.ExtractedTextRequest;
 import android.view.inputmethod.InputConnection;
+import java.util.Iterator;
 
 public final class KeyEventHandler implements Config.IKeyEventHandler
 {
   IReceiver _recv;
   Autocapitalisation _autocap;
+  /** State of the system modifiers. It is updated whether a modifier is down
+      or up and a corresponding key event is sent. */
+  Pointers.Modifiers _mods;
+  /** Consistent with [_mods]. This is a mutable state rather than computed
+      from [_mods] to ensure that the meta state is correct while up and down
+      events are sent for the modifier keys. */
+  int _meta_state;
 
   public KeyEventHandler(Looper looper, IReceiver recv)
   {
     _recv = recv;
     _autocap = new Autocapitalisation(looper,
         this.new Autocapitalisation_callback());
+    _mods = Pointers.Modifiers.EMPTY;
   }
 
   /** Editing just started. */
@@ -33,6 +42,7 @@ public final class KeyEventHandler implements Config.IKeyEventHandler
 
   /** A key is being pressed. There will not necessarily be a corresponding
       [key_up] event. */
+  @Override
   public void key_down(KeyValue key, boolean isSwipe)
   {
     if (key == null)
@@ -55,19 +65,44 @@ public final class KeyEventHandler implements Config.IKeyEventHandler
   }
 
   /** A key has been released. */
+  @Override
   public void key_up(KeyValue key, Pointers.Modifiers mods)
   {
     if (key == null)
       return;
+    Pointers.Modifiers old_mods = _mods;
+    update_meta_state(mods);
     switch (key.getKind())
     {
       case Char: send_text(String.valueOf(key.getChar())); break;
       case String: send_text(key.getString()); break;
       case Event: _recv.handle_event_key(key.getEvent()); break;
-      case Keyevent: send_key_down_up(key.getKeyevent(), mods); break;
+      case Keyevent: send_key_down_up(key.getKeyevent()); break;
       case Modifier: break;
-      case Editing: handle_editing_key(key.getEditing(), mods); break;
+      case Editing: handle_editing_key(key.getEditing()); break;
     }
+    update_meta_state(old_mods);
+  }
+
+  @Override
+  public void mods_changed(Pointers.Modifiers mods)
+  {
+    update_meta_state(mods);
+  }
+
+  /** Update [_mods] to be consistent with the [mods], sending key events if
+      needed. */
+  void update_meta_state(Pointers.Modifiers mods)
+  {
+    // Released modifiers
+    Iterator<KeyValue.Modifier> it = _mods.diff(mods);
+    while (it.hasNext())
+      sendMetaKeyForModifier(it.next(), false);
+    // Activated modifiers
+    it = mods.diff(_mods);
+    while (it.hasNext())
+      sendMetaKeyForModifier(it.next(), true);
+    _mods = mods;
   }
 
   // private void handleDelKey(int before, int after)
@@ -80,54 +115,58 @@ public final class KeyEventHandler implements Config.IKeyEventHandler
   //  getCurrentInputConnection().deleteSurroundingText(before, after);
   // }
 
-  int sendMetaKey(int eventCode, int metaFlags, int metaState, boolean down)
+  void sendMetaKey(int eventCode, int meta_flags, boolean down)
   {
-    int action;
+    if (down)
-    int updatedMetaState;
+    {
-    if (down) { action = KeyEvent.ACTION_DOWN; updatedMetaState = metaState | metaFlags; }
+      _meta_state = _meta_state | meta_flags;
-    else { action = KeyEvent.ACTION_UP; updatedMetaState = metaState & ~metaFlags; }
+      send_keyevent(KeyEvent.ACTION_DOWN, eventCode);
-    send_keyevent(action, eventCode, metaState);
+    }
-    return updatedMetaState;
+    else
+    {
+      send_keyevent(KeyEvent.ACTION_UP, eventCode);
+      _meta_state = _meta_state & ~meta_flags;
+    }
   }
 
-  int sendMetaKeyForModifier(KeyValue.Modifier mod, int metaState, boolean down)
+  void sendMetaKeyForModifier(KeyValue.Modifier mod, boolean down)
   {
     switch (mod)
     {
       case CTRL:
-        return sendMetaKey(KeyEvent.KEYCODE_CTRL_LEFT, KeyEvent.META_CTRL_LEFT_ON | KeyEvent.META_CTRL_ON, metaState, down);
+        sendMetaKey(KeyEvent.KEYCODE_CTRL_LEFT, KeyEvent.META_CTRL_LEFT_ON | KeyEvent.META_CTRL_ON, down);
+        break;
       case ALT:
-        return sendMetaKey(KeyEvent.KEYCODE_ALT_LEFT, KeyEvent.META_ALT_LEFT_ON | KeyEvent.META_ALT_ON, metaState, down);
+        sendMetaKey(KeyEvent.KEYCODE_ALT_LEFT, KeyEvent.META_ALT_LEFT_ON | KeyEvent.META_ALT_ON, down);
+        break;
       case SHIFT:
-        return sendMetaKey(KeyEvent.KEYCODE_SHIFT_LEFT, KeyEvent.META_SHIFT_LEFT_ON | KeyEvent.META_SHIFT_ON, metaState, down);
+        sendMetaKey(KeyEvent.KEYCODE_SHIFT_LEFT, KeyEvent.META_SHIFT_LEFT_ON | KeyEvent.META_SHIFT_ON, down);
+        break;
       case META:
-        return sendMetaKey(KeyEvent.KEYCODE_META_LEFT, KeyEvent.META_META_LEFT_ON | KeyEvent.META_META_ON, metaState, down);
+        sendMetaKey(KeyEvent.KEYCODE_META_LEFT, KeyEvent.META_META_LEFT_ON | KeyEvent.META_META_ON, down);
-      default: return metaState;
+        break;
+      default:
+        break;
     }
   }
 
   /*
    * Don't set KeyEvent.FLAG_SOFT_KEYBOARD.
    */
-  void send_key_down_up(int keyCode, Pointers.Modifiers mods)
+  void send_key_down_up(int keyCode)
   {
-    int metaState = 0;
+    send_keyevent(KeyEvent.ACTION_DOWN, keyCode);
-    for (int i = 0; i < mods.size(); i++)
+    send_keyevent(KeyEvent.ACTION_UP, keyCode);
-      metaState = sendMetaKeyForModifier(mods.get(i), metaState, true);
-    send_keyevent(KeyEvent.ACTION_DOWN, keyCode, metaState);
-    send_keyevent(KeyEvent.ACTION_UP, keyCode, metaState);
-    for (int i = mods.size() - 1; i >= 0; i--)
-      metaState = sendMetaKeyForModifier(mods.get(i), metaState, false);
   }
 
-  void send_keyevent(int eventAction, int eventCode, int meta)
+  void send_keyevent(int eventAction, int eventCode)
   {
     InputConnection conn = _recv.getCurrentInputConnection();
     if (conn == null)
       return;
-    conn.sendKeyEvent(new KeyEvent(1, 1, eventAction, eventCode, 0, meta));
+    conn.sendKeyEvent(new KeyEvent(1, 1, eventAction, eventCode, 0, _meta_state));
     if (eventAction == KeyEvent.ACTION_UP)
-      _autocap.event_sent(eventCode, meta);
+      _autocap.event_sent(eventCode, _meta_state);
   }
 
   void send_text(CharSequence text)
@@ -148,7 +187,7 @@ public final class KeyEventHandler implements Config.IKeyEventHandler
     conn.performContextMenuAction(id);
   }
 
-  void handle_editing_key(KeyValue.Editing ev, Pointers.Modifiers mods)
+  void handle_editing_key(KeyValue.Editing ev)
   {
     switch (ev)
     {
@@ -163,8 +202,8 @@ public final class KeyEventHandler implements Config.IKeyEventHandler
       case REPLACE: send_context_menu_action(android.R.id.replaceText); break;
       case ASSIST: send_context_menu_action(android.R.id.textAssist); break;
       case AUTOFILL: send_context_menu_action(android.R.id.autofill); break;
-      case CURSOR_LEFT: move_cursor(-1, mods); break;
+      case CURSOR_LEFT: move_cursor(-1); break;
-      case CURSOR_RIGHT: move_cursor(1, mods); break;
+      case CURSOR_RIGHT: move_cursor(1); break;
     }
   }
 
@@ -186,7 +225,7 @@ public final class KeyEventHandler implements Config.IKeyEventHandler
       Unlike arrow keys, the selection is not removed even if shift is not on.
       Falls back to sending arrow keys events if the editor do not support
       moving the cursor or a modifier other than shift is pressed. */
-  void move_cursor(int d, Pointers.Modifiers mods)
+  void move_cursor(int d)
   {
     InputConnection conn = _recv.getCurrentInputConnection();
     if (conn == null)
@@ -194,11 +233,11 @@ public final class KeyEventHandler implements Config.IKeyEventHandler
     ExtractedText et = get_cursor_pos(conn);
     // Fallback to sending key events
     if (et == null
-        || mods.has(KeyValue.Modifier.CTRL)
+        || _mods.has(KeyValue.Modifier.CTRL)
-        || mods.has(KeyValue.Modifier.ALT)
+        || _mods.has(KeyValue.Modifier.ALT)
-        || mods.has(KeyValue.Modifier.META))
+        || _mods.has(KeyValue.Modifier.META))
     {
-      move_cursor_fallback(d, mods);
+      move_cursor_fallback(d);
       return;
     }
     int sel_start = et.selectionStart;
@@ -214,7 +253,7 @@ public final class KeyEventHandler implements Config.IKeyEventHandler
     {
       sel_end += d;
       // Leave 'sel_start' where it is if shift is pressed
-      if (!mods.has(KeyValue.Modifier.SHIFT))
+      if (!_mods.has(KeyValue.Modifier.SHIFT))
         sel_start = sel_end;
     }
     conn.setSelection(sel_start, sel_end);
@@ -222,16 +261,16 @@ public final class KeyEventHandler implements Config.IKeyEventHandler
 
   /** Send arrow keys as a fallback for editors that do not support
       [getExtractedText] like Termux. */
-  void move_cursor_fallback(int d, Pointers.Modifiers mods)
+  void move_cursor_fallback(int d)
   {
     while (d < 0)
     {
-      send_key_down_up(KeyEvent.KEYCODE_DPAD_LEFT, mods);
+      send_key_down_up(KeyEvent.KEYCODE_DPAD_LEFT);
       d++;
     }
     while (d > 0)
     {
-      send_key_down_up(KeyEvent.KEYCODE_DPAD_RIGHT, mods);
+      send_key_down_up(KeyEvent.KEYCODE_DPAD_RIGHT);
       d--;
     }
   }

srcs/juloo.keyboard2/Keyboard2View.java

@@ -125,7 +125,6 @@ public class Keyboard2View extends View
         return; // Don't remove locked pointers
       _pointers.remove_fake_pointer(_shift_kv, _shift_key);
     }
-    invalidate();
   }
 
   public KeyValue modifyKey(KeyValue k, Pointers.Modifiers mods)
@@ -144,6 +143,7 @@ public class Keyboard2View extends View
 
   public void onPointerDown(KeyValue k, boolean isSwipe)
   {
+    updateFlags();
     _config.handler.key_down(k, isSwipe);
     invalidate();
     vibrate();
@@ -151,17 +151,22 @@ public class Keyboard2View extends View
 
   public void onPointerUp(KeyValue k, Pointers.Modifiers mods)
   {
+    // [key_up] must be called before [updateFlags]. The latter might disable
+    // flags.
     _config.handler.key_up(k, mods);
+    updateFlags();
     invalidate();
   }
 
   public void onPointerHold(KeyValue k, Pointers.Modifiers mods)
   {
     _config.handler.key_up(k, mods);
+    updateFlags();
   }
 
   public void onPointerFlagsChanged(boolean shouldVibrate)
   {
+    updateFlags();
     invalidate();
     if (shouldVibrate)
       vibrate();
@@ -170,6 +175,7 @@ public class Keyboard2View extends View
   private void updateFlags()
   {
     _mods = _pointers.getModifiers();
+    _config.handler.mods_changed(_mods);
   }
 
   @Override
@@ -287,7 +293,6 @@ public class Keyboard2View extends View
   @Override
   protected void onDraw(Canvas canvas)
   {
-    updateFlags();
     // Set keyboard background opacity
     getBackground().setAlpha(_config.keyboardOpacity);
     // Set keys opacity

srcs/juloo.keyboard2/Pointers.java

@@ -2,8 +2,10 @@ package juloo.keyboard2;
 
 import android.os.Handler;
 import android.os.Message;
-import java.util.Arrays;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
 
 /**
  * Manage pointers (fingers) on the screen and long presses.
@@ -95,6 +97,7 @@ public final class Pointers implements Handler.Callback
     if (locked)
       ptr.flags |= KeyValue.FLAG_LOCKED;
     _ptrs.add(ptr);
+    _handler.onPointerFlagsChanged(false);
   }
 
   public void remove_fake_pointer(KeyValue kv, KeyboardData.Key key)
@@ -102,6 +105,7 @@ public final class Pointers implements Handler.Callback
     Pointer ptr = getLatched(key, kv);
     if (ptr != null && (ptr.flags & KeyValue.FLAG_FAKE_PTR) != 0)
       removePtr(ptr);
+    _handler.onPointerFlagsChanged(false);
   }
 
   // Receiving events
@@ -483,6 +487,12 @@ public final class Pointers implements Handler.Callback
       return (Arrays.binarySearch(_mods, 0, _size, m) >= 0);
     }
 
+    /** Returns the activated modifiers that are not in [m2]. */
+    public Iterator<KeyValue.Modifier> diff(Modifiers m2)
+    {
+      return new ModifiersDiffIterator(this, m2);
+    }
+
     @Override
     public int hashCode() { return Arrays.hashCode(_mods); }
     @Override
@@ -514,6 +524,60 @@ public final class Pointers implements Handler.Callback
       }
       return new Modifiers(mods, size);
     }
+
+    /** Returns modifiers that are in [m1_] but not in [m2_]. */
+    static final class ModifiersDiffIterator
+        implements Iterator<KeyValue.Modifier>
+    {
+      Modifiers m1;
+      int i1 = 0;
+      Modifiers m2;
+      int i2 = 0;
+
+      public ModifiersDiffIterator(Modifiers m1_, Modifiers m2_)
+      {
+        m1 = m1_;
+        m2 = m2_;
+        advance();
+      }
+
+      public boolean hasNext()
+      {
+        return i1 < m1._size;
+      }
+
+      public KeyValue.Modifier next()
+      {
+        if (i1 >= m1._size)
+          throw new NoSuchElementException();
+        KeyValue.Modifier m = m1._mods[i1];
+        i1++;
+        advance();
+        return m;
+      }
+
+      /** Advance to the next element if [i1] is not a valid element. The end
+          is reached when [i1 = m1.size()].  */
+      void advance()
+      {
+        while (i1 < m1.size())
+        {
+          KeyValue.Modifier m = m1._mods[i1];
+          while (true)
+          {
+            if (i2 >= m2._size)
+              return;
+            int d = m.compareTo(m2._mods[i2]);
+            if (d < 0)
+              return;
+            i2++;
+            if (d == 0)
+              break;
+          }
+          i1++;
+        }
+      }
+    }
   }
 
   public interface IPointerEventHandler