Add support for notifying clients about pointer movements

This change adds support for the VMware Mouse Position
pseudo-encoding[1], which is used to notify VNC clients when X11 clients
call `XWarpPointer()`[2]. This function is called by SDL (and other
similar libraries)  when they detect that the server does not support
native relative motion, like some RFB clients.

With this, RFB clients can choose to adjust the local cursor position
under certain circumstances to match what the server has set. For
instance, if pointer lock has been enabled on the client's machine and
the cursor is not being drawn locally, the local position of the cursor
is irrelevant, so the RFB client can use what the server sends as the
canonical absolute position of the cursor. This ultimately enables the
possibility of games (especially FPS games) to behave how users expect
(if the clients implement the corresponding change).

Part of: #619

1: https://github.com/rfbproto/rfbproto/blob/master/rfbproto.rst#vmware-cursor-position-pseudo-encoding
2: https://tronche.com/gui/x/xlib/input/XWarpPointer.html
3: https://hg.libsdl.org/SDL/file/28e3b60e2131/src/events/SDL_mouse.c#l804

common/rfb/ConnParams.cxx

@@ -36,6 +36,7 @@ ConnParams::ConnParams()
     width(0), height(0), useCopyRect(false),
     supportsLocalCursor(false), supportsLocalXCursor(false),
     supportsLocalCursorWithAlpha(false),
+    supportsCursorPosition(false),
     supportsDesktopResize(false), supportsExtendedDesktopSize(false),
     supportsDesktopRename(false), supportsLastRect(false),
     supportsLEDState(false), supportsQEMUKeyEvent(false),
@@ -43,7 +44,7 @@ ConnParams::ConnParams()
     supportsSetDesktopSize(false), supportsFence(false),
     supportsContinuousUpdates(false),
     compressLevel(2), qualityLevel(-1), fineQualityLevel(-1),
-    subsampling(subsampleUndefined), name_(0), verStrPos(0),
+    subsampling(subsampleUndefined), name_(0), cursorPos_(0, 0), verStrPos(0),
     ledState_(ledUnknown), shandler(NULL)
 {
   memset(kasmPassed, 0, KASM_NUM_SETTINGS);
@@ -101,6 +102,11 @@ void ConnParams::setCursor(const Cursor& other)
   cursor_ = new Cursor(other);
 }
 
+void ConnParams::setCursorPos(const Point& pos)
+{
+    cursorPos_ = pos;
+}
+
 bool ConnParams::supportsEncoding(rdr::S32 encoding) const
 {
   return encodings_.count(encoding) != 0;
@@ -147,6 +153,9 @@ void ConnParams::setEncodings(int nEncodings, const rdr::S32* encodings)
     case pseudoEncodingExtendedDesktopSize:
       supportsExtendedDesktopSize = true;
       break;
+    case pseudoEncodingVMwareCursorPosition:
+      supportsCursorPosition = true;
+      break;
     case pseudoEncodingDesktopName:
       supportsDesktopRename = true;
       break;

common/rfb/ConnParams.h

@@ -84,6 +84,9 @@ namespace rfb {
     const Cursor& cursor() const { return *cursor_; }
     void setCursor(const Cursor& cursor);
 
+    const Point& cursorPos() const { return cursorPos_; }
+    void setCursorPos(const Point& pos);
+
     bool supportsEncoding(rdr::S32 encoding) const;
 
     void setEncodings(int nEncodings, const rdr::S32* encodings);
@@ -96,6 +99,7 @@ namespace rfb {
     bool supportsLocalCursor;
     bool supportsLocalXCursor;
     bool supportsLocalCursorWithAlpha;
+    bool supportsCursorPosition;
     bool supportsDesktopResize;
     bool supportsExtendedDesktopSize;
     bool supportsDesktopRename;
@@ -136,6 +140,7 @@ namespace rfb {
     PixelFormat pf_;
     char* name_;
     Cursor* cursor_;
+    Point cursorPos_;
     std::set<rdr::S32> encodings_;
     char verStr[13];
     int verStrPos;

common/rfb/SMsgWriter.cxx

@@ -39,6 +39,7 @@ SMsgWriter::SMsgWriter(ConnParams* cp_, rdr::OutStream* os_)
     needSetDesktopSize(false), needExtendedDesktopSize(false),
     needSetDesktopName(false), needSetCursor(false),
     needSetXCursor(false), needSetCursorWithAlpha(false),
+    needCursorPos(false),
     needLEDState(false), needQEMUKeyEvent(false)
 {
 }
@@ -204,6 +205,14 @@ bool SMsgWriter::writeSetCursorWithAlpha()
   return true;
 }
 
+void SMsgWriter::writeCursorPos()
+{
+  if (!cp->supportsEncoding(pseudoEncodingVMwareCursorPosition))
+    throw Exception("Client does not support cursor position");
+
+  needCursorPos = true;
+}
+
 bool SMsgWriter::writeLEDState()
 {
   if (!cp->supportsLEDState)
@@ -232,6 +241,8 @@ bool SMsgWriter::needFakeUpdate()
     return true;
   if (needSetCursor || needSetXCursor || needSetCursorWithAlpha)
     return true;
+  if (needCursorPos)
+    return true;
   if (needLEDState)
     return true;
   if (needQEMUKeyEvent)
@@ -284,6 +295,8 @@ void SMsgWriter::writeFramebufferUpdateStart(int nRects)
       nRects++;
     if (needSetCursorWithAlpha)
       nRects++;
+    if (needCursorPos)
+      nRects++;
     if (needLEDState)
       nRects++;
     if (needQEMUKeyEvent)
@@ -399,6 +412,18 @@ void SMsgWriter::writePseudoRects()
     needSetCursorWithAlpha = false;
   }
 
+  if (needCursorPos) {
+    const Point& cursorPos = cp->cursorPos();
+
+    if (cp->supportsEncoding(pseudoEncodingVMwareCursorPosition)) {
+      writeSetVMwareCursorPositionRect(cursorPos.x, cursorPos.y);
+    } else {
+      throw Exception("Client does not support cursor position");
+    }
+
+    needCursorPos = false;
+  }
+
   if (needSetDesktopName) {
     writeSetDesktopNameRect(cp->name());
     needSetDesktopName = false;
@@ -577,6 +602,20 @@ void SMsgWriter::writeSetCursorWithAlphaRect(int width, int height,
   }
 }
 
+void SMsgWriter::writeSetVMwareCursorPositionRect(int hotspotX, int hotspotY)
+{
+  if (!cp->supportsEncoding(pseudoEncodingVMwareCursorPosition))
+    throw Exception("Client does not support cursor position");
+  if (++nRectsInUpdate > nRectsInHeader && nRectsInHeader)
+    throw Exception("SMsgWriter::writeSetVMwareCursorRect: nRects out of sync");
+
+  os->writeS16(hotspotX);
+  os->writeS16(hotspotY);
+  os->writeU16(0);
+  os->writeU16(0);
+  os->writeU32(pseudoEncodingVMwareCursorPosition);
+}
+
 void SMsgWriter::writeLEDStateRect(rdr::U8 state)
 {
   if (!cp->supportsLEDState)

common/rfb/SMsgWriter.h

@@ -83,6 +83,9 @@ namespace rfb {
     bool writeSetXCursor();
     bool writeSetCursorWithAlpha();
 
+    // Notifies the client that the cursor pointer was moved by the server.
+    void writeCursorPos();
+
     // Same for LED state message
     bool writeLEDState();
 
@@ -138,6 +141,7 @@ namespace rfb {
     void writeSetCursorWithAlphaRect(int width, int height,
                                      int hotspotX, int hotspotY,
                                      const rdr::U8* data);
+    void writeSetVMwareCursorPositionRect(int hotspotX, int hotspotY);
     void writeLEDStateRect(rdr::U8 state);
     void writeQEMUKeyEventRect();
 
@@ -153,6 +157,7 @@ namespace rfb {
     bool needSetCursor;
     bool needSetXCursor;
     bool needSetCursorWithAlpha;
+    bool needCursorPos;
     bool needLEDState;
     bool needQEMUKeyEvent;
 

common/rfb/VNCSConnectionST.cxx

@@ -517,6 +517,15 @@ void VNCSConnectionST::renderedCursorChange()
   }
 }
 
+// cursorPositionChange() is called whenever the cursor has changed position by
+// the server.  If the client supports being informed about these changes then
+// it will arrange for the new cursor position to be sent to the client.
+
+void VNCSConnectionST::cursorPositionChange()
+{
+  setCursorPos();
+}
+
 // needRenderedCursor() returns true if this client needs the server-side
 // rendered cursor.  This may be because it does not support local cursor or
 // because the current cursor position has not been set by this client.
@@ -1482,6 +1491,21 @@ void VNCSConnectionST::setCursor()
   }
 }
 
+// setCursorPos() is called whenever the cursor has changed position by the
+// server.  If the client supports being informed about these changes then it
+// will arrange for the new cursor position to be sent to the client.
+
+void VNCSConnectionST::setCursorPos()
+{
+  if (state() != RFBSTATE_NORMAL)
+    return;
+
+  if (cp.supportsCursorPosition) {
+    cp.setCursorPos(server->cursorPos);
+    writer()->writeCursorPos();
+  }
+}
+
 void VNCSConnectionST::setDesktopName(const char *name)
 {
   cp.setName(name);

common/rfb/VNCSConnectionST.h

@@ -97,6 +97,11 @@ namespace rfb {
     // cursor.
     void renderedCursorChange();
 
+    // cursorPositionChange() is called whenever the cursor has changed position by
+    // the server.  If the client supports being informed about these changes then
+    // it will arrange for the new cursor position to be sent to the client.
+    void cursorPositionChange();
+
     // needRenderedCursor() returns true if this client needs the server-side
     // rendered cursor.  This may be because it does not support local cursor
     // or because the current cursor position has not been set by this client.
@@ -223,6 +228,7 @@ namespace rfb {
 
     void screenLayoutChange(rdr::U16 reason);
     void setCursor();
+    void setCursorPos();
     void setDesktopName(const char *name);
     void setLEDState(unsigned int state);
     void setSocketTimeouts();

common/rfb/VNCServer.h

@@ -69,8 +69,10 @@ namespace rfb {
     virtual void setCursor(int width, int height, const Point& hotspot,
                            const rdr::U8* cursorData) = 0;
 
-    // setCursorPos() tells the server the current position of the cursor.
-    virtual void setCursorPos(const Point& p) = 0;
+    // setCursorPos() tells the server the current position of the cursor, and
+    // whether the server initiated that change (e.g. through another X11
+    // client calling XWarpPointer()).
+    virtual void setCursorPos(const Point& p, bool warped) = 0;
 
     // setName() tells the server what desktop title to supply to clients
     virtual void setName(const char* name) = 0;

common/rfb/VNCServerST.cxx

@@ -565,14 +565,17 @@ void VNCServerST::setCursor(int width, int height, const Point& newHotspot,
   }
 }
 
-void VNCServerST::setCursorPos(const Point& pos)
+void VNCServerST::setCursorPos(const Point& pos, bool warped)
 {
   if (!cursorPos.equals(pos)) {
     cursorPos = pos;
     renderedCursorInvalid = true;
     std::list<VNCSConnectionST*>::iterator ci;
-    for (ci = clients.begin(); ci != clients.end(); ci++)
+    for (ci = clients.begin(); ci != clients.end(); ci++) {
       (*ci)->renderedCursorChange();
+      if (warped)
+        (*ci)->cursorPositionChange();
+    }
   }
 }
 

common/rfb/VNCServerST.h

@@ -101,7 +101,7 @@ namespace rfb {
     virtual void add_copied(const Region &dest, const Point &delta);
     virtual void setCursor(int width, int height, const Point& hotspot,
                            const rdr::U8* data);
-    virtual void setCursorPos(const Point& p);
+    virtual void setCursorPos(const Point& p, bool warped);
     virtual void setLEDState(unsigned state);
 
     virtual void bell();

common/rfb/encodings.h

@@ -85,6 +85,9 @@ namespace rfb {
   const int pseudoEncodingVideoOutTimeLevel1 = -1986;
   const int pseudoEncodingVideoOutTimeLevel100 = -1887;
 
+  // VMware-specific
+  const int pseudoEncodingVMwareCursorPosition = 0x574d5666;
+
   int encodingNum(const char* name);
   const char* encodingName(int num);
 }

unix/xserver/hw/vnc/XserverDesktop.cc

@@ -241,6 +241,15 @@ void XserverDesktop::setCursor(int width, int height, int hotX, int hotY,
   delete [] cursorData;
 }
 
+void XserverDesktop::setCursorPos(int x, int y, bool warped)
+{
+  try {
+    server->setCursorPos(Point(x, y), warped);
+  } catch (rdr::Exception& e) {
+    vlog.error("XserverDesktop::setCursorPos: %s",e.str());
+  }
+}
+
 void XserverDesktop::add_changed(const rfb::Region &region)
 {
   try {
@@ -358,7 +367,7 @@ void XserverDesktop::blockHandler(int* timeout)
     if (oldCursorPos.x != cursorX || oldCursorPos.y != cursorY) {
       oldCursorPos.x = cursorX;
       oldCursorPos.y = cursorY;
-      server->setCursorPos(oldCursorPos);
+      server->setCursorPos(oldCursorPos, false);
     }
 
     // Trigger timers and check when the next will expire

unix/xserver/hw/vnc/XserverDesktop.h

@@ -66,6 +66,7 @@ public:
   void setDesktopName(const char* name);
   void setCursor(int width, int height, int hotX, int hotY,
                  const unsigned char *rgbaData);
+  void setCursorPos(int x, int y, bool warped);
   void add_changed(const rfb::Region &region);
   void add_copied(const rfb::Region &dest, const rfb::Point &delta);
   void handleSocketEvent(int fd, bool read, bool write);

unix/xserver/hw/vnc/vncExtInit.cc

@@ -418,6 +418,11 @@ void vncSetCursor(int width, int height, int hotX, int hotY,
     desktop[scr]->setCursor(width, height, hotX, hotY, rgbaData);
 }
 
+void vncSetCursorPos(int scrIdx, int x, int y)
+{
+  desktop[scrIdx]->setCursorPos(x, y, true);
+}
+
 void vncPreScreenResize(int scrIdx)
 {
   // We need to prevent the RFB core from accessing the framebuffer

unix/xserver/hw/vnc/vncExtInit.h

@@ -86,6 +86,7 @@ void vncAddCopied(int scrIdx, const struct UpdateRect *extents,
 
 void vncSetCursor(int width, int height, int hotX, int hotY,
                   const unsigned char *rgbaData);
+void vncSetCursorPos(int scrIdx, int x, int y);
 
 void vncPreScreenResize(int scrIdx);
 void vncPostScreenResize(int scrIdx, int success, int width, int height);

unix/xserver/hw/vnc/vncHooks.c

@@ -65,6 +65,9 @@ typedef struct _vncHooksScreenRec {
   RestoreAreasProcPtr          RestoreAreas;
 #endif
   DisplayCursorProcPtr         DisplayCursor;
+#if XORG >= 119
+  CursorWarpedToProcPtr        CursorWarpedTo;
+#endif
   ScreenBlockHandlerProcPtr    BlockHandler;
 #ifdef RENDER
   CompositeProcPtr             Composite;
@@ -137,6 +140,12 @@ static RegionPtr vncHooksRestoreAreas(WindowPtr pWin, RegionPtr prgnExposed);
 #endif
 static Bool vncHooksDisplayCursor(DeviceIntPtr pDev,
                                   ScreenPtr pScreen, CursorPtr cursor);
+#if XORG >= 119
+static void vncHooksCursorWarpedTo(DeviceIntPtr pDev,
+                                   ScreenPtr pScreen_, ClientPtr pClient,
+                                   WindowPtr pWindow, SpritePtr pSprite,
+                                   int x, int y);
+#endif
 #if XORG <= 112
 static void vncHooksBlockHandler(int i, pointer blockData, pointer pTimeout,
                                  pointer pReadmask);
@@ -316,6 +325,9 @@ int vncHooksInit(int scrIdx)
   wrap(vncHooksScreen, pScreen, RestoreAreas, vncHooksRestoreAreas);
 #endif
   wrap(vncHooksScreen, pScreen, DisplayCursor, vncHooksDisplayCursor);
+#if XORG >= 119
+  wrap(vncHooksScreen, pScreen, CursorWarpedTo, vncHooksCursorWarpedTo);
+#endif
   wrap(vncHooksScreen, pScreen, BlockHandler, vncHooksBlockHandler);
 #ifdef RENDER
   ps = GetPictureScreenIfSet(pScreen);
@@ -725,6 +737,20 @@ out:
   return ret;
 }
 
+// CursorWarpedTo - notify that the cursor was warped
+
+#if XORG >= 119
+static void vncHooksCursorWarpedTo(DeviceIntPtr pDev,
+                                   ScreenPtr pScreen_, ClientPtr pClient,
+                                   WindowPtr pWindow, SpritePtr pSprite,
+                                   int x, int y)
+{
+  SCREEN_PROLOGUE(pScreen_, CursorWarpedTo);
+  vncSetCursorPos(pScreen->myNum, x, y);
+  SCREEN_EPILOGUE(CursorWarpedTo);
+}
+#endif
+
 // BlockHandler - ignore any changes during the block handler - it's likely
 // these are just drawing the cursor.
 

win/rfb_win32/SDisplay.cxx

@@ -385,7 +385,7 @@ SDisplay::processEvent(HANDLE event) {
         // Update the cursor position
         // NB: First translate from Screen coordinates to Desktop
         Point desktopPos = info.position.translate(screenRect.tl.negate());
-        server->setCursorPos(desktopPos);
+        server->setCursorPos(desktopPos, false);
 
         old_cursor = info;
       }