libvncserver: Add API to add custom I/O entry points

Add API to make it possible to channel RFB input and output through
another layer, for example TLS. This is done by making it possible to
override the default read/write/peek functions.

Author: jadahl

include/rfb/rfb.h

@@ -402,6 +402,14 @@ typedef struct sraRegion* sraRegionPtr;
 typedef void (*ClientGoneHookPtr)(struct _rfbClientRec* cl);
 typedef void (*ClientFramebufferUpdateRequestHookPtr)(struct _rfbClientRec* cl, rfbFramebufferUpdateRequestMsg* furMsg);
 
+typedef int (*ClientReadFromSocket)(struct _rfbClientRec* cl,
+                                    char *buf, int len);
+typedef int (*ClientPeekAtSocket)(struct _rfbClientRec* cl,
+                                  char *buf, int len);
+typedef rfbBool (*ClientHasPendingOnSocket)(struct _rfbClientRec* cl);
+typedef int (*ClientWriteToSocket)(struct _rfbClientRec* cl,
+                                   const char *buf, int len);
+
 typedef struct _rfbFileTransferData {
   int fd;
   int compressionEnabled;
@@ -715,6 +723,11 @@ typedef struct _rfbClientRec {
     int destPort;
     /** ID on repeater in case of an UltraVNC mode 2 repeater connection */
     char *repeaterId;
+
+    ClientReadFromSocket readFromSocket;         /* Read data from socket */
+    ClientPeekAtSocket peekAtSocket;             /* Peek at data from socket */
+    ClientHasPendingOnSocket hasPendingOnSocket; /* Has pending data on socket */
+    ClientWriteToSocket writeToSocket;           /* Write data to socket */
 } rfbClientRec, *rfbClientPtr;
 
 /**
@@ -767,8 +780,12 @@ extern void rfbDisconnectUDPSock(rfbScreenInfoPtr rfbScreen);
 extern void rfbCloseClient(rfbClientPtr cl);
 extern int rfbReadExact(rfbClientPtr cl, char *buf, int len);
 extern int rfbReadExactTimeout(rfbClientPtr cl, char *buf, int len,int timeout);
+extern int rfbDefaultReadFromSocket(rfbClientPtr cl, char *buf, int len);
 extern int rfbPeekExactTimeout(rfbClientPtr cl, char *buf, int len,int timeout);
+extern int rfbDefaultPeekAtSocket(rfbClientPtr cl, char *buf, int len);
+extern rfbBool rfbDefaultHasPendingOnSocket(rfbClientPtr cl);
 extern int rfbWriteExact(rfbClientPtr cl, const char *buf, int len);
+extern int rfbDefaultWriteToSocket(rfbClientPtr cl, const char *buf, int len);
 extern int rfbCheckFds(rfbScreenInfoPtr rfbScreen,long usec);
 extern rfbSocket rfbConnect(rfbScreenInfoPtr rfbScreen, char* host, int port);
 extern rfbSocket rfbConnectToTcpAddr(char* host, int port);

src/libvncserver/rfbserver.c

@@ -376,6 +376,10 @@ rfbNewTCPOrUDPClient(rfbScreenInfoPtr rfbScreen,
 
     cl->screen = rfbScreen;
     cl->sock = sock;
+    cl->readFromSocket = rfbDefaultReadFromSocket;
+    cl->peekAtSocket = rfbDefaultPeekAtSocket;
+    cl->hasPendingOnSocket = rfbDefaultHasPendingOnSocket;
+    cl->writeToSocket = rfbDefaultWriteToSocket;
     cl->viewOnly = FALSE;
     /* setup pseudo scaling */
     cl->scaledScreen = rfbScreen;

src/libvncserver/sockets.c

@@ -101,6 +101,9 @@ int deny_severity=LOG_WARNING;
 int rfbMaxClientWait = 20000;   /* time (ms) after which we decide client has
                                    gone away - needed to stop us hanging */
 
+static rfbBool
+rfbHasPendingOnSocket(rfbClientPtr cl);
+
 static rfbBool
 rfbNewConnectionFromSock(rfbScreenInfoPtr rfbScreen, rfbSocket sock)
 {
@@ -364,16 +367,20 @@ rfbCheckFds(rfbScreenInfoPtr rfbScreen,long usec)
 	tv.tv_usec = usec;
 	nfds = select(rfbScreen->maxFd + 1, &fds, NULL, NULL /* &fds */, &tv);
 	if (nfds == 0) {
+            rfbBool hasPendingData = FALSE;
+
 	    /* timed out, check for async events */
             i = rfbGetClientIterator(rfbScreen);
             while((cl = rfbClientIteratorNext(i))) {
                 if (cl->onHold)
                     continue;
+                hasPendingData |= rfbHasPendingOnSocket(cl);
                 if (FD_ISSET(cl->sock, &(rfbScreen->allFds)))
                     rfbSendFileTransferChunk(cl);
             }
             rfbReleaseClientIterator(i);
-	    return result;
+            if (!hasPendingData)
+                return result;
 	}
 
 	if (nfds < 0) {
@@ -449,9 +456,11 @@ rfbCheckFds(rfbScreenInfoPtr rfbScreen,long usec)
 	    if (cl->onHold)
 		continue;
 
-            if (FD_ISSET(cl->sock, &(rfbScreen->allFds)))
+            if (rfbHasPendingOnSocket (cl) ||
+                FD_ISSET(cl->sock, &(rfbScreen->allFds)))
             {
-                if (FD_ISSET(cl->sock, &fds))
+                if (rfbHasPendingOnSocket (cl) ||
+                    FD_ISSET(cl->sock, &fds))
                 {
 #ifdef LIBVNCSERVER_WITH_WEBSOCKETS
                     do {
@@ -649,6 +658,30 @@ size_t fuzz_size;
 const uint8_t *fuzz_data;
 #endif
 
+int
+rfbDefaultReadFromSocket(rfbClientPtr cl, char *buf, int len)
+{
+    return read(cl->sock, buf, len);
+}
+
+static int
+rfbReadFromSocket(rfbClientPtr cl, char *buf, int len)
+{
+    return cl->readFromSocket(cl, buf, len);
+}
+
+rfbBool
+rfbDefaultHasPendingOnSocket(rfbClientPtr cl)
+{
+    return FALSE;
+}
+
+static rfbBool
+rfbHasPendingOnSocket(rfbClientPtr cl)
+{
+    return cl->hasPendingOnSocket(cl);
+}
+
 /*
  * ReadExact reads an exact number of bytes from a client.  Returns 1 if
  * those bytes have been read, 0 if the other end has closed, or -1 if an error
@@ -682,10 +715,10 @@ rfbReadExactTimeout(rfbClientPtr cl, char* buf, int len, int timeout)
         } else if (cl->sslctx) {
 	    n = rfbssl_read(cl, buf, len);
 	} else {
-            n = read(sock, buf, len);
+            n = rfbReadFromSocket(cl, buf, len);
         }
 #else
-        n = read(sock, buf, len);
+        n = rfbReadFromSocket(cl, buf, len);
 #endif
 
         if (n > 0) {
@@ -717,6 +750,10 @@ rfbReadExactTimeout(rfbClientPtr cl, char* buf, int len, int timeout)
 		    continue;
 	    }
 #endif
+
+            if (rfbHasPendingOnSocket(cl))
+                continue;
+
             FD_ZERO(&fds);
             FD_SET(sock, &fds);
             tv.tv_sec = timeout / 1000;
@@ -753,6 +790,18 @@ int rfbReadExact(rfbClientPtr cl,char* buf,int len)
     return(rfbReadExactTimeout(cl,buf,len,rfbMaxClientWait));
 }
 
+int
+rfbDefaultPeekAtSocket(rfbClientPtr cl, char *buf, int len)
+{
+    return recv(cl->sock, buf, len, MSG_PEEK);
+}
+
+int
+rfbPeekAtSocket(rfbClientPtr cl, char *buf, int len)
+{
+    return cl->peekAtSocket(cl, buf, len);
+}
+
 /*
  * PeekExact peeks at an exact number of bytes from a client.  Returns 1 if
  * those bytes have been read, 0 if the other end has closed, or -1 if an
@@ -785,7 +834,7 @@ rfbPeekExactTimeout(rfbClientPtr cl, char* buf, int len, int timeout)
 	    n = rfbssl_peek(cl, buf, len);
 	else
 #endif
-	    n = recv(sock, buf, len, MSG_PEEK);
+            n = rfbPeekAtSocket(cl, buf, len);
 
         if (n == len) {
 
@@ -841,6 +890,22 @@ rfbPeekExactTimeout(rfbClientPtr cl, char* buf, int len, int timeout)
     return 1;
 }
 
+int
+rfbDefaultWriteToSocket(rfbClientPtr cl,
+			const char *buf,
+			int len)
+{
+    return write(cl->sock, buf, len);
+}
+
+static int
+rfbWriteToSocket(rfbClientPtr cl,
+		 const char *buf,
+		 int len)
+{
+    return cl->writeToSocket(cl, buf, len);
+}
+
 /*
  * WriteExact writes an exact number of bytes to a client.  Returns 1 if
  * those bytes have been written, or -1 if an error occurred (errno is set to
@@ -905,7 +970,7 @@ rfbWriteExact(rfbClientPtr cl,
 	    n = rfbssl_write(cl, buf, len);
 	else
 #endif
-	    n = write(sock, buf, len);
+	    n = rfbWriteToSocket(cl, buf, len);
 
         if (n > 0) {