From 7147c3c911f4ebf5bd1c21444423d8f78c6541ea Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Mon, 17 Jun 2024 19:26:08 +0200 Subject: [PATCH] avcodec/ratecontrol: Handle wanted bits overflow Fixes: 5.92611e+20 is outside the range of representable values of type 'unsigned long' Fixes: 68984/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_SNOW_fuzzer-5155755073273856 Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg Signed-off-by: Michael Niedermayer --- libavcodec/ratecontrol.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/libavcodec/ratecontrol.c b/libavcodec/ratecontrol.c index df27639ca7..86ec7a3443 100644 --- a/libavcodec/ratecontrol.c +++ b/libavcodec/ratecontrol.c @@ -936,6 +936,7 @@ float ff_rate_estimate_qscale(MpegEncContext *s, int dry_run) wanted_bits = rce->expected_bits; } else { const MPVPicture *dts_pic; + double wanted_bits_double; rce = &local_rce; /* FIXME add a dts field to AVFrame and ensure it is set and use it @@ -947,9 +948,14 @@ float ff_rate_estimate_qscale(MpegEncContext *s, int dry_run) dts_pic = s->last_pic.ptr; if (!dts_pic || dts_pic->f->pts == AV_NOPTS_VALUE) - wanted_bits = (uint64_t)(s->bit_rate * (double)picture_number / fps); + wanted_bits_double = s->bit_rate * (double)picture_number / fps; else - wanted_bits = (uint64_t)(s->bit_rate * (double)dts_pic->f->pts / fps); + wanted_bits_double = s->bit_rate * (double)dts_pic->f->pts / fps; + if (wanted_bits_double > INT64_MAX) { + av_log(s, AV_LOG_WARNING, "Bits exceed 64bit range\n"); + wanted_bits = INT64_MAX; + } else + wanted_bits = (int64_t)wanted_bits_double; } diff = s->total_bits - wanted_bits;