changeset 36199:25e1fb36262e

mcdeint: avoid uninited data read Do not read padding or out-of-buffer values when computing the output value for a pixel close to the image buffer edge. This avoids non visible artifacts which affected the output checksum. Patch by Stefano Sabatini, stefasab gmail
author cehoyos
date Sat, 01 Jun 2013 23:07:27 +0000
parents 3e4454636ad9
children a86d2ecfc4ee
files libmpcodecs/vf_mcdeint.c
diffstat 1 files changed, 33 insertions(+), 11 deletions(-) [+]
line wrap: on
line diff
--- a/libmpcodecs/vf_mcdeint.c	Sat Jun 01 04:05:02 2013 +0000
+++ b/libmpcodecs/vf_mcdeint.c	Sat Jun 01 23:07:27 2013 +0000
@@ -115,27 +115,49 @@
         for(y=0; y<h; y++){
             if((y ^ p->parity) & 1){
                 for(x=0; x<w; x++){
-                    if((x-2)+(y-1)*w>=0 && (x+2)+(y+1)*w<w*h){ //FIXME either alloc larger images or optimize this
+                    if(y>0 && y<h-1){
+                        int is_edge= x<3 || x>w-4;
                         uint8_t *filp= &p->frame_dec->data[i][x + y*fils];
                         uint8_t *srcp= &src[i][x + y*srcs];
                         int diff0= filp[-fils] - srcp[-srcs];
                         int diff1= filp[+fils] - srcp[+srcs];
-                        int spatial_score= ABS(srcp[-srcs-1] - srcp[+srcs-1])
-                                          +ABS(srcp[-srcs  ] - srcp[+srcs  ])
-                                          +ABS(srcp[-srcs+1] - srcp[+srcs+1]) - 1;
                         int temp= filp[0];
 
+#define DELTA(j) av_clip(j, -x, w-1-x)
+
+#define GET_SCORE_EDGE(j)\
+   ABS(srcp[-srcs+DELTA(-1+(j))] - srcp[+srcs+DELTA(-1-(j))])+\
+   ABS(srcp[-srcs+DELTA(j)     ] - srcp[+srcs+DELTA(  -(j))])+\
+   ABS(srcp[-srcs+DELTA(1+(j)) ] - srcp[+srcs+DELTA( 1-(j))])
+
+#define GET_SCORE(j)\
+   ABS(srcp[-srcs-1+(j)] - srcp[+srcs-1-(j)])+\
+   ABS(srcp[-srcs  +(j)] - srcp[+srcs  -(j)])+\
+   ABS(srcp[-srcs+1+(j)] - srcp[+srcs+1-(j)])
+
+#define CHECK_EDGE(j)\
+    {   int score= GET_SCORE_EDGE(j);\
+        if(score < spatial_score){\
+            spatial_score= score;\
+            diff0= filp[-fils+DELTA(j)]    - srcp[-srcs+DELTA(j)];\
+            diff1= filp[+fils+DELTA(-(j))] - srcp[+srcs+DELTA(-(j))];\
+
 #define CHECK(j)\
-    {   int score= ABS(srcp[-srcs-1+j] - srcp[+srcs-1-j])\
-                 + ABS(srcp[-srcs  +j] - srcp[+srcs  -j])\
-                 + ABS(srcp[-srcs+1+j] - srcp[+srcs+1-j]);\
+    {   int score= GET_SCORE(j);\
         if(score < spatial_score){\
             spatial_score= score;\
-            diff0= filp[-fils+j] - srcp[-srcs+j];\
-            diff1= filp[+fils-j] - srcp[+srcs-j];
+            diff0= filp[-fils+(j)] - srcp[-srcs+(j)];\
+            diff1= filp[+fils-(j)] - srcp[+srcs-(j)];\
 
-                        CHECK(-1) CHECK(-2) }} }}
-                        CHECK( 1) CHECK( 2) }} }}
+                        if (is_edge) {
+                            int spatial_score= GET_SCORE_EDGE(0)-1;
+                            CHECK_EDGE(-1) CHECK_EDGE(-2) }} }}
+                            CHECK_EDGE( 1) CHECK_EDGE( 2) }} }}
+                        } else {
+                            int spatial_score= GET_SCORE(0)-1;
+                            CHECK(-1) CHECK(-2) }} }}
+                            CHECK( 1) CHECK( 2) }} }}
+                        }
 #if 0
                         if((diff0 ^ diff1) > 0){
                             int mindiff= ABS(diff0) > ABS(diff1) ? diff1 : diff0;