From 87b5bb5840e843f16b422dfb76bbaf63204a0977 Mon Sep 17 00:00:00 2001
From: Sam Lantinga <slouken@libsdl.org>
Date: Fri, 19 Jan 2024 08:31:56 -0800
Subject: [PATCH] Fixed potential overflow in software triangle rendering

(cherry picked from commit bd5d4d61edc95f1f003ec91597418fc416ff8315)
---
 src/render/software/SDL_triangle.c | 52 ++++++++++++++++++++++++------
 1 file changed, 42 insertions(+), 10 deletions(-)

diff --git a/src/render/software/SDL_triangle.c b/src/render/software/SDL_triangle.c
index abfb9c921..31588b999 100644
--- a/src/render/software/SDL_triangle.c
+++ b/src/render/software/SDL_triangle.c
@@ -22,6 +22,8 @@
 
 #if SDL_VIDEO_RENDER_SW && !defined(SDL_RENDER_DISABLED)
 
+#include <limits.h>
+
 #include "SDL_surface.h"
 #include "SDL_triangle.h"
 
@@ -223,7 +225,8 @@ int SDL_SW_FillTriangle(SDL_Surface *dst, SDL_Point *d0, SDL_Point *d1, SDL_Poin
     Uint8 *dst_ptr;
     int dst_pitch;
 
-    int area, is_clockwise;
+    Sint64 area;
+    int is_clockwise;
 
     int d2d1_y, d1d2_x, d0d2_y, d2d0_x, d1d0_y, d0d1_x;
     Sint64 w0_row, w1_row, w2_row;
@@ -309,7 +312,9 @@ int SDL_SW_FillTriangle(SDL_Surface *dst, SDL_Point *d0, SDL_Point *d1, SDL_Poin
     }
 
     is_clockwise = area > 0;
-    area = SDL_abs(area);
+    if (area < 0) {
+        area = -area;
+    }
 
     {
         int val;
@@ -464,7 +469,8 @@ int SDL_SW_BlitTriangle(
     int *src_ptr;
     int src_pitch;
 
-    int area, is_clockwise;
+    Sint64 area, tmp64;
+    int is_clockwise;
 
     int d2d1_y, d1d2_x, d0d2_y, d2d0_x, d1d0_y, d0d1_x;
     int s2s0_x, s2s1_x, s2s0_y, s2s1_y;
@@ -476,8 +482,11 @@ int SDL_SW_BlitTriangle(
 
     int has_modulation;
 
-    if (!src || !dst) {
-        return -1;
+    if (!src) {
+        return SDL_InvalidParamError("src");
+    }
+    if (!src) {
+        return SDL_InvalidParamError("dst");
     }
 
     area = cross_product(d0, d1, d2->x, d2->y);
@@ -579,7 +588,9 @@ int SDL_SW_BlitTriangle(
     src_pitch = src->pitch;
 
     is_clockwise = area > 0;
-    area = SDL_abs(area);
+    if (area < 0) {
+        area = -area;
+    }
 
     {
         int val;
@@ -628,8 +639,20 @@ int SDL_SW_BlitTriangle(
     bias_w2 = (is_top_left(d0, d1, is_clockwise) ? 0 : -1);
 
     /* precompute constant 's2->x * area' used in TRIANGLE_GET_TEXTCOORD */
-    s2_x_area.x = s2->x * area;
-    s2_x_area.y = s2->y * area;
+    tmp64 = s2->x * area;
+    if (tmp64 >= INT_MIN && tmp64 <= INT_MAX) {
+        s2_x_area.x = (int)tmp64;
+    } else {
+        ret = SDL_SetError("triangle area overflow");
+        goto end;
+    }
+    tmp64 = s2->y * area;
+    if (tmp64 >= INT_MIN && tmp64 <= INT_MAX) {
+        s2_x_area.y = (int)tmp64;
+    } else {
+        ret = SDL_SetError("triangle area overflow");
+        goto end;
+    }
 
     if (blend != SDL_BLENDMODE_NONE || src->format->format != dst->format->format || has_modulation || !is_uniform) {
         /* Use SDL_BlitTriangle_Slow */
@@ -675,9 +698,18 @@ int SDL_SW_BlitTriangle(
         tmp_info.dst = dst_ptr;
         tmp_info.dst_pitch = dst_pitch;
 
-        SDL_BlitTriangle_Slow(&tmp_info, s2_x_area, dstrect, area, bias_w0, bias_w1, bias_w2,
+#define CHECK_INT_RANGE(X) \
+    if ((X) < INT_MIN || (X) > INT_MAX) { \
+        ret = SDL_SetError("integer overflow (%s = %" SDL_PRIs64 ")", #X, X); \
+        goto end; \
+    }
+        CHECK_INT_RANGE(area);
+        CHECK_INT_RANGE(w0_row);
+        CHECK_INT_RANGE(w1_row);
+        CHECK_INT_RANGE(w2_row);
+        SDL_BlitTriangle_Slow(&tmp_info, s2_x_area, dstrect, (int)area, bias_w0, bias_w1, bias_w2,
                               d2d1_y, d1d2_x, d0d2_y, d2d0_x, d1d0_y, d0d1_x,
-                              s2s0_x, s2s1_x, s2s0_y, s2s1_y, w0_row, w1_row, w2_row,
+                              s2s0_x, s2s1_x, s2s0_y, s2s1_y, (int)w0_row, (int)w1_row, (int)w2_row,
                               c0, c1, c2, is_uniform);
 
         goto end;