Add support for vmware cursor

common/rfb/ConnParams.cxx

@@ -37,6 +37,7 @@ ConnParams::ConnParams()
     width(0), height(0), useCopyRect(false),
     supportsLocalCursor(false), supportsLocalXCursor(false),
     supportsLocalCursorWithAlpha(false),
+    supportsVMWareCursor(false),
     supportsCursorPosition(false),
     supportsDesktopResize(false), supportsExtendedDesktopSize(false),
     supportsDesktopRename(false), supportsLastRect(false),
@@ -123,6 +124,7 @@ void ConnParams::setEncodings(int nEncodings, const rdr::S32* encodings)
   useCopyRect = false;
   supportsLocalCursor = false;
   supportsLocalCursorWithAlpha = false;
+  supportsVMWareCursor = false;
   supportsDesktopResize = false;
   supportsExtendedDesktopSize = false;
   supportsLocalXCursor = false;
@@ -153,6 +155,9 @@ void ConnParams::setEncodings(int nEncodings, const rdr::S32* encodings)
     case pseudoEncodingCursorWithAlpha:
       supportsLocalCursorWithAlpha = true;
       break;
+    case pseudoEncodingVMwareCursor:
+      supportsVMWareCursor = true;
+      break;
     case pseudoEncodingDesktopSize:
       supportsDesktopResize = true;
       break;

common/rfb/ConnParams.h

@@ -102,6 +102,7 @@ namespace rfb {
     bool supportsLocalCursor;
     bool supportsLocalXCursor;
     bool supportsLocalCursorWithAlpha;
+    bool supportsVMWareCursor;
     bool supportsCursorPosition;
     bool supportsDesktopResize;
     bool supportsExtendedDesktopSize;

common/rfb/SMsgWriter.cxx

@@ -43,6 +43,7 @@ SMsgWriter::SMsgWriter(ConnParams* cp_, rdr::OutStream* os_)
     needSetDesktopSize(false), needExtendedDesktopSize(false),
     needSetDesktopName(false), needSetCursor(false),
     needSetXCursor(false), needSetCursorWithAlpha(false),
+    needSetVMWareCursor(false),
     needCursorPos(false),
     needLEDState(false), needQEMUKeyEvent(false)
 {
@@ -321,6 +322,16 @@ bool SMsgWriter::writeSetCursorWithAlpha()
   return true;
 }
 
+bool SMsgWriter::writeSetVMwareCursor()
+{
+  if (!cp->supportsVMWareCursor)
+    return false;
+
+  needSetVMWareCursor = true;
+
+  return true;
+}
+
 void SMsgWriter::writeCursorPos()
 {
   if (!cp->supportsEncoding(pseudoEncodingVMwareCursorPosition))
@@ -355,7 +366,7 @@ bool SMsgWriter::needFakeUpdate()
 {
   if (needSetDesktopName)
     return true;
-  if (needSetCursor || needSetXCursor || needSetCursorWithAlpha)
+  if (needSetCursor || needSetXCursor || needSetCursorWithAlpha || needSetVMWareCursor)
     return true;
   if (needCursorPos)
     return true;
@@ -411,6 +422,8 @@ void SMsgWriter::writeFramebufferUpdateStart(int nRects)
       nRects++;
     if (needSetCursorWithAlpha)
       nRects++;
+    if (needSetVMWareCursor)
+      nRects++;
     if (needCursorPos)
       nRects++;
     if (needLEDState)
@@ -528,6 +541,15 @@ void SMsgWriter::writePseudoRects()
     needSetCursorWithAlpha = false;
   }
 
+  if (needSetVMWareCursor) {
+    const Cursor& cursor = cp->cursor();
+
+    writeSetVMwareCursorRect(cursor.width(), cursor.height(),
+                                cursor.hotspot().x, cursor.hotspot().y,
+                                cursor.getBuffer());
+    needSetVMWareCursor = false;
+  }
+
   if (needCursorPos) {
     const Point& cursorPos = cp->cursorPos();
 
@@ -718,6 +740,28 @@ void SMsgWriter::writeSetCursorWithAlphaRect(int width, int height,
   }
 }
 
+void SMsgWriter::writeSetVMwareCursorRect(int width, int height,
+                                          int hotspotX, int hotspotY,
+                                          const rdr::U8* data)
+{
+  if (!cp->supportsVMWareCursor)
+    throw Exception("Client does not support local cursors");
+  if (++nRectsInUpdate > nRectsInHeader && nRectsInHeader)
+    throw Exception("SMsgWriter::writeSetVMwareCursorRect: nRects out of sync");
+
+  os->writeS16(hotspotX);
+  os->writeS16(hotspotY);
+  os->writeU16(width);
+  os->writeU16(height);
+  os->writeU32(pseudoEncodingVMwareCursor);
+
+  os->writeU8(1); // Alpha cursor
+  os->pad(1);
+
+  // FIXME: Should alpha be premultiplied?
+  os->writeBytes(data, width*height*4);
+}
+
 void SMsgWriter::writeSetVMwareCursorPositionRect(int hotspotX, int hotspotY)
 {
   if (!cp->supportsEncoding(pseudoEncodingVMwareCursorPosition))

common/rfb/SMsgWriter.h

@@ -92,6 +92,7 @@ namespace rfb {
     bool writeSetCursor();
     bool writeSetXCursor();
     bool writeSetCursorWithAlpha();
+    bool writeSetVMwareCursor();
 
     // Notifies the client that the cursor pointer was moved by the server.
     void writeCursorPos();
@@ -151,6 +152,9 @@ namespace rfb {
     void writeSetCursorWithAlphaRect(int width, int height,
                                      int hotspotX, int hotspotY,
                                      const rdr::U8* data);
+    void writeSetVMwareCursorRect(int width, int height,
+                                  int hotspotX, int hotspotY,
+                                  const rdr::U8* data);
     void writeSetVMwareCursorPositionRect(int hotspotX, int hotspotY);
     void writeLEDStateRect(rdr::U8 state);
     void writeQEMUKeyEventRect();
@@ -167,6 +171,7 @@ namespace rfb {
     bool needSetCursor;
     bool needSetXCursor;
     bool needSetCursorWithAlpha;
+    bool needSetVMWareCursor;
     bool needCursorPos;
     bool needLEDState;
     bool needQEMUKeyEvent;

common/rfb/VNCSConnectionST.cxx

@@ -574,6 +574,7 @@ bool VNCSConnectionST::needRenderedCursor()
     return false;
 
   if (!cp.supportsLocalCursorWithAlpha &&
+      !cp.supportsVMWareCursor &&
       !cp.supportsLocalCursor && !cp.supportsLocalXCursor)
     return true;
   if (!server->cursorPos.equals(pointerEventPos) &&
@@ -1550,11 +1551,13 @@ void VNCSConnectionST::setCursor()
     clientHasCursor = true;
   }
 
-  if (!writer()->writeSetCursorWithAlpha()) {
+  if (!writer()->writeSetVMwareCursor()) {
-    if (!writer()->writeSetCursor()) {
+    if (!writer()->writeSetCursorWithAlpha()) {
-      if (!writer()->writeSetXCursor()) {
+      if (!writer()->writeSetCursor()) {
-        // No client support
+        if (!writer()->writeSetXCursor()) {
-        return;
+          // No client support
+          return;
+        }
       }
     }
   }

common/rfb/encodings.h

@@ -86,6 +86,7 @@ namespace rfb {
   const int pseudoEncodingVideoOutTimeLevel100 = -1887;
 
   // VMware-specific
+  const int pseudoEncodingVMwareCursor = 0x574d5664;
   const int pseudoEncodingVMwareCursorPosition = 0x574d5666;
 
   // UltraVNC-specific