vncv: implement rfb 3.8 protocol (thanks Iruatã)

Gnome and bhyve's VNC servers implement the RFB 3.8
protocol, so we need to support it in the client in
order to connect.
This commit is contained in:
Ori Bernstein 2020-09-27 09:43:43 -07:00
parent 16766169e2
commit fb42cf0dda

View file

@ -9,14 +9,16 @@ enum
VerLen = 12 VerLen = 12
}; };
static char version[VerLen+1] = "RFB 003.003\n"; static char version33[VerLen+1] = "RFB 003.003\n";
static char version38[VerLen+1] = "RFB 003.008\n";
static int srvversion;
int int
vncsrvhandshake(Vnc *v) vncsrvhandshake(Vnc *v)
{ {
char msg[VerLen+1]; char msg[VerLen+1];
strecpy(msg, msg+sizeof msg, version); strecpy(msg, msg+sizeof msg, version33);
if(verbose) if(verbose)
fprint(2, "server version: %s\n", msg); fprint(2, "server version: %s\n", msg);
vncwrbytes(v, msg, VerLen); vncwrbytes(v, msg, VerLen);
@ -35,18 +37,52 @@ vnchandshake(Vnc *v)
msg[VerLen] = 0; msg[VerLen] = 0;
vncrdbytes(v, msg, VerLen); vncrdbytes(v, msg, VerLen);
if(strncmp(msg, "RFB ", 4) != 0){ if(strncmp(msg, "RFB 003.", 8) != 0 ||
strncmp(msg, "RFB 003.007\n", VerLen) == 0){
werrstr("bad rfb version \"%s\"", msg); werrstr("bad rfb version \"%s\"", msg);
return -1; return -1;
} }
if(strncmp(msg, "RFB 003.008\n", VerLen) == 0)
srvversion = 38;
else
srvversion = 33;
if(verbose) if(verbose)
fprint(2, "server version: %s\n", msg); fprint(2, "server version: %s\n", msg);
strcpy(msg, version); strcpy(msg, version38);
vncwrbytes(v, msg, VerLen); vncwrbytes(v, msg, VerLen);
vncflush(v); vncflush(v);
return 0; return 0;
} }
ulong
sectype38(Vnc *v)
{
ulong auth, type;
int i, ntypes;
ntypes = vncrdchar(v);
if(ntypes == 0){
werrstr("no security types from server");
return AFailed;
}
/* choose the "most secure" security type */
auth = AFailed;
for(i = 0; i < ntypes; i++){
type = vncrdchar(v);
if(verbose){
fprint(2, "auth type %s\n",
type == AFailed ? "Invalid" :
type == ANoAuth ? "None" :
type == AVncAuth ? "VNC" : "Unknown");
}
if(type > auth)
auth = type;
}
return auth;
}
int int
vncauth(Vnc *v, char *keypattern) vncauth(Vnc *v, char *keypattern)
{ {
@ -56,7 +92,9 @@ vncauth(Vnc *v, char *keypattern)
if(keypattern == nil) if(keypattern == nil)
keypattern = ""; keypattern = "";
auth = vncrdlong(v);
auth = srvversion == 38 ? sectype38(v) : vncrdlong(v);
switch(auth){ switch(auth){
default: default:
werrstr("unknown auth type 0x%lux", auth); werrstr("unknown auth type 0x%lux", auth);
@ -65,6 +103,7 @@ vncauth(Vnc *v, char *keypattern)
return -1; return -1;
case AFailed: case AFailed:
failed:
reason = vncrdstring(v); reason = vncrdstring(v);
werrstr("%s", reason); werrstr("%s", reason);
if(verbose) if(verbose)
@ -72,11 +111,20 @@ vncauth(Vnc *v, char *keypattern)
return -1; return -1;
case ANoAuth: case ANoAuth:
if(srvversion == 38){
vncwrchar(v, auth);
vncflush(v);
}
if(verbose) if(verbose)
fprint(2, "no auth needed\n"); fprint(2, "no auth needed\n");
break; break;
case AVncAuth: case AVncAuth:
if(srvversion == 38){
vncwrchar(v, auth);
vncflush(v);
}
vncrdbytes(v, chal, VncChalLen); vncrdbytes(v, chal, VncChalLen);
if(auth_respond(chal, VncChalLen, nil, 0, chal, VncChalLen, auth_getkey, if(auth_respond(chal, VncChalLen, nil, 0, chal, VncChalLen, auth_getkey,
"proto=vnc role=client server=%s %s", serveraddr, keypattern) != VncChalLen){ "proto=vnc role=client server=%s %s", serveraddr, keypattern) != VncChalLen){
@ -84,13 +132,20 @@ vncauth(Vnc *v, char *keypattern)
} }
vncwrbytes(v, chal, VncChalLen); vncwrbytes(v, chal, VncChalLen);
vncflush(v); vncflush(v);
break;
}
auth = vncrdlong(v); /* in version 3.8 the auth status is always sent, in 3.3 only in AVncAuth */
if(srvversion == 38 || auth == AVncAuth){
auth = vncrdlong(v); /* auth status */
switch(auth){ switch(auth){
default: default:
werrstr("unknown server response 0x%lux", auth); werrstr("unknown server response 0x%lux", auth);
return -1; return -1;
case VncAuthFailed: case VncAuthFailed:
if (srvversion == 38)
goto failed;
werrstr("server says authentication failed"); werrstr("server says authentication failed");
return -1; return -1;
case VncAuthTooMany: case VncAuthTooMany:
@ -99,7 +154,6 @@ vncauth(Vnc *v, char *keypattern)
case VncAuthOK: case VncAuthOK:
break; break;
} }
break;
} }
return 0; return 0;
} }