From 4fcace61801f418786c42487c6b06b693ee87666 Mon Sep 17 00:00:00 2001
From: Romain Vimont <rom1v@videolabs.io>
Date: Mon, 19 Sep 2022 17:17:01 +0200
Subject: [PATCH] vnc: fix possible buffer overflow

Upstream-Status: Inappropriate

RPI-Distro repo forks original vlc and applies patches
to enable raspiberry pi support.

Thanks to 0xMitsurugi [1] from Synacktiv [2] for the bug report and fix.

[1] https://twitter.com/0xMitsurugi
[2] https://www.synacktiv.com/

Fixes #27335

(cherry picked from commit 5eb783fd44ed6298db3e38f7765f21c42e4405f9)
---
 modules/access/vnc.c | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

--- a/modules/access/vnc.c
+++ b/modules/access/vnc.c
@@ -33,6 +33,7 @@
 #ifdef HAVE_CONFIG_H
 # include "config.h"
 #endif
+#include <assert.h>
 
 #include <vlc_common.h>
 #include <vlc_plugin.h>
@@ -115,7 +116,7 @@
     int i_cancel_state;
 
     rfbClient* p_client;
-    int i_framebuffersize;
+    size_t i_framebuffersize;
     block_t *p_block;
 
     float f_fps;
@@ -143,11 +144,16 @@
         p_sys->es = NULL;
     }
 
-    int i_width = p_client->width;
-    int i_height = p_client->height;
-    int i_depth = p_client->format.bitsPerPixel;
+    assert(!(p_client->width & ~0xffff)); // fits in 16 bits
+    uint16_t i_width = p_client->width;
 
-    switch( i_depth )
+    assert(!(p_client->height & ~0xffff)); // fits in 16 bits
+    uint16_t i_height = p_client->height;
+
+    uint8_t i_bits_per_pixel = p_client->format.bitsPerPixel;
+    assert((i_bits_per_pixel & 0x7) == 0); // multiple of 8
+
+    switch( i_bits_per_pixel )
     {
         case 8:
             i_chroma = VLC_CODEC_RGB8;
@@ -180,7 +186,10 @@
     }
 
     /* Set up framebuffer */
-    p_sys->i_framebuffersize = i_width * i_height * i_depth / 8;
+    if (mul_overflow(i_width, i_height * (i_bits_per_pixel / 8), &p_sys->i_framebuffersize)) {
+        msg_Err(p_demux, "VNC framebuffersize overflow");
+        return FALSE;
+    }
 
     /* Reuse unsent block */
     if ( p_sys->p_block )
@@ -211,7 +220,7 @@
     fmt.video.i_frame_rate_base = 1000;
     fmt.video.i_frame_rate = 1000 * p_sys->f_fps;
 
-    fmt.video.i_bits_per_pixel = i_depth;
+    fmt.video.i_bits_per_pixel = i_bits_per_pixel;
     fmt.video.i_rmask = p_client->format.redMax << p_client->format.redShift;
     fmt.video.i_gmask = p_client->format.greenMax << p_client->format.greenShift;
     fmt.video.i_bmask = p_client->format.blueMax << p_client->format.blueShift;