changeset 10542:5a298a61c6cc libavcodec

avoid integer overflow in dnxhd encoder, fixes #1557
author bcoudurier
date Fri, 20 Nov 2009 00:13:34 +0000
parents 5afea08b89d4
children 7ff7a34848bf
files dnxhdenc.c
diffstat 1 files changed, 3 insertions(+), 1 deletions(-) [+]
line wrap: on
line diff
--- a/dnxhdenc.c	Thu Nov 19 23:28:21 2009 +0000
+++ b/dnxhdenc.c	Fri Nov 20 00:13:34 2009 +0000
@@ -574,9 +574,11 @@
             last_higher = FFMAX(lambda, last_higher);
             if (last_lower != INT_MAX)
                 lambda = (lambda+last_lower)>>1;
+            else if ((int64_t)lambda + up_step > INT_MAX)
+                return -1;
             else
                 lambda += up_step;
-            up_step *= 5;
+            up_step = FFMIN((int64_t)up_step*5, INT_MAX);
             down_step = 1<<LAMBDA_FRAC_BITS;
         }
     }