ryujinx/SDL
1
0
Fork 0
mirror of https://github.com/Ryujinx/SDL.git synced 2025-01-03 18:35:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixed potential overflow in software triangle rendering

Browse source 
(cherry picked from commit bd5d4d61edc95f1f003ec91597418fc416ff8315)
This commit is contained in:
Sam Lantinga 2024-01-19 08:31:56 -08:00
parent 11b111e1f8
commit 87b5bb5840
1 changed files with 42 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

52
src/render/software/SDL_triangle.c
View file

@ -22,6 +22,8 @@
#if SDL_VIDEO_RENDER_SW && !defined(SDL_RENDER_DISABLED) #if SDL_VIDEO_RENDER_SW && !defined(SDL_RENDER_DISABLED)
#include <limits.h>
#include "SDL_surface.h" #include "SDL_surface.h"
#include "SDL_triangle.h" #include "SDL_triangle.h"
@ -223,7 +225,8 @@ int SDL_SW_FillTriangle(SDL_Surface *dst, SDL_Point *d0, SDL_Point *d1, SDL_Poin
Uint8 *dst_ptr; Uint8 *dst_ptr;
int dst_pitch; int dst_pitch;
int area, is_clockwise; Sint64 area;
int is_clockwise;
int d2d1_y, d1d2_x, d0d2_y, d2d0_x, d1d0_y, d0d1_x; int d2d1_y, d1d2_x, d0d2_y, d2d0_x, d1d0_y, d0d1_x;
Sint64 w0_row, w1_row, w2_row; Sint64 w0_row, w1_row, w2_row;
@ -309,7 +312,9 @@ int SDL_SW_FillTriangle(SDL_Surface *dst, SDL_Point *d0, SDL_Point *d1, SDL_Poin
} }
is_clockwise = area > 0; is_clockwise = area > 0;
area = SDL_abs(area); if (area < 0) {
area = -area;
}
{ {
int val; int val;
@ -464,7 +469,8 @@ int SDL_SW_BlitTriangle(
int *src_ptr; int *src_ptr;
int src_pitch; int src_pitch;
int area, is_clockwise; Sint64 area, tmp64;
int is_clockwise;
int d2d1_y, d1d2_x, d0d2_y, d2d0_x, d1d0_y, d0d1_x; int d2d1_y, d1d2_x, d0d2_y, d2d0_x, d1d0_y, d0d1_x;
int s2s0_x, s2s1_x, s2s0_y, s2s1_y; int s2s0_x, s2s1_x, s2s0_y, s2s1_y;
@ -476,8 +482,11 @@ int SDL_SW_BlitTriangle(
int has_modulation; int has_modulation;
if (!src || !dst) { if (!src) {
return -1; return SDL_InvalidParamError("src");
}
if (!src) {
return SDL_InvalidParamError("dst");
} }
area = cross_product(d0, d1, d2->x, d2->y); area = cross_product(d0, d1, d2->x, d2->y);
@ -579,7 +588,9 @@ int SDL_SW_BlitTriangle(
src_pitch = src->pitch; src_pitch = src->pitch;
is_clockwise = area > 0; is_clockwise = area > 0;
area = SDL_abs(area); if (area < 0) {
area = -area;
}
{ {
int val; int val;
@ -628,8 +639,20 @@ int SDL_SW_BlitTriangle(
bias_w2 = (is_top_left(d0, d1, is_clockwise) ? 0 : -1); bias_w2 = (is_top_left(d0, d1, is_clockwise) ? 0 : -1);
/* precompute constant 's2->x * area' used in TRIANGLE_GET_TEXTCOORD */ /* precompute constant 's2->x * area' used in TRIANGLE_GET_TEXTCOORD */
s2_x_area.x = s2->x * area; tmp64 = s2->x * area;
s2_x_area.y = s2->y * area; if (tmp64 >= INT_MIN && tmp64 <= INT_MAX) {
s2_x_area.x = (int)tmp64;
} else {
ret = SDL_SetError("triangle area overflow");
goto end;
}
tmp64 = s2->y * area;
if (tmp64 >= INT_MIN && tmp64 <= INT_MAX) {
s2_x_area.y = (int)tmp64;
} else {
ret = SDL_SetError("triangle area overflow");
goto end;
}
if (blend != SDL_BLENDMODE_NONE || src->format->format != dst->format->format || has_modulation || !is_uniform) { if (blend != SDL_BLENDMODE_NONE || src->format->format != dst->format->format || has_modulation || !is_uniform) {
/* Use SDL_BlitTriangle_Slow */ /* Use SDL_BlitTriangle_Slow */
@ -675,9 +698,18 @@ int SDL_SW_BlitTriangle(
tmp_info.dst = dst_ptr; tmp_info.dst = dst_ptr;
tmp_info.dst_pitch = dst_pitch; tmp_info.dst_pitch = dst_pitch;
SDL_BlitTriangle_Slow(&tmp_info, s2_x_area, dstrect, area, bias_w0, bias_w1, bias_w2, #define CHECK_INT_RANGE(X) \
if ((X) < INT_MIN || (X) > INT_MAX) { \
ret = SDL_SetError("integer overflow (%s = %" SDL_PRIs64 ")", #X, X); \
goto end; \
}
CHECK_INT_RANGE(area);
CHECK_INT_RANGE(w0_row);
CHECK_INT_RANGE(w1_row);
CHECK_INT_RANGE(w2_row);
SDL_BlitTriangle_Slow(&tmp_info, s2_x_area, dstrect, (int)area, bias_w0, bias_w1, bias_w2,
d2d1_y, d1d2_x, d0d2_y, d2d0_x, d1d0_y, d0d1_x, d2d1_y, d1d2_x, d0d2_y, d2d0_x, d1d0_y, d0d1_x,
s2s0_x, s2s1_x, s2s0_y, s2s1_y, w0_row, w1_row, w2_row, s2s0_x, s2s1_x, s2s0_y, s2s1_y, (int)w0_row, (int)w1_row, (int)w2_row,
c0, c1, c2, is_uniform); c0, c1, c2, is_uniform);
goto end; goto end;