# HG changeset patch
# User cehoyos
# Date 1370128047 0
# Node ID 25e1fb36262e75fb3579c3a2984b1c871e6020c9
# Parent  3e4454636ad934bf238e6edb06ec7d1a8c56a244
mcdeint: avoid uninited data read

Do not read padding or out-of-buffer values when computing the output
value for a pixel close to the image buffer edge.

This avoids non visible artifacts which affected the output checksum.

Patch by Stefano Sabatini, stefasab gmail

diff -r 3e4454636ad9 -r 25e1fb36262e libmpcodecs/vf_mcdeint.c
--- a/libmpcodecs/vf_mcdeint.c	Sat Jun 01 04:05:02 2013 +0000
+++ b/libmpcodecs/vf_mcdeint.c	Sat Jun 01 23:07:27 2013 +0000
@@ -115,27 +115,49 @@
         for(y=0; y<h; y++){
             if((y ^ p->parity) & 1){
                 for(x=0; x<w; x++){
-                    if((x-2)+(y-1)*w>=0 && (x+2)+(y+1)*w<w*h){ //FIXME either alloc larger images or optimize this
+                    if(y>0 && y<h-1){
+                        int is_edge= x<3 || x>w-4;
                         uint8_t *filp= &p->frame_dec->data[i][x + y*fils];
                         uint8_t *srcp= &src[i][x + y*srcs];
                         int diff0= filp[-fils] - srcp[-srcs];
                         int diff1= filp[+fils] - srcp[+srcs];
-                        int spatial_score= ABS(srcp[-srcs-1] - srcp[+srcs-1])
-                                          +ABS(srcp[-srcs  ] - srcp[+srcs  ])
-                                          +ABS(srcp[-srcs+1] - srcp[+srcs+1]) - 1;
                         int temp= filp[0];
 
+#define DELTA(j) av_clip(j, -x, w-1-x)
+
+#define GET_SCORE_EDGE(j)\
+   ABS(srcp[-srcs+DELTA(-1+(j))] - srcp[+srcs+DELTA(-1-(j))])+\
+   ABS(srcp[-srcs+DELTA(j)     ] - srcp[+srcs+DELTA(  -(j))])+\
+   ABS(srcp[-srcs+DELTA(1+(j)) ] - srcp[+srcs+DELTA( 1-(j))])
+
+#define GET_SCORE(j)\
+   ABS(srcp[-srcs-1+(j)] - srcp[+srcs-1-(j)])+\
+   ABS(srcp[-srcs  +(j)] - srcp[+srcs  -(j)])+\
+   ABS(srcp[-srcs+1+(j)] - srcp[+srcs+1-(j)])
+
+#define CHECK_EDGE(j)\
+    {   int score= GET_SCORE_EDGE(j);\
+        if(score < spatial_score){\
+            spatial_score= score;\
+            diff0= filp[-fils+DELTA(j)]    - srcp[-srcs+DELTA(j)];\
+            diff1= filp[+fils+DELTA(-(j))] - srcp[+srcs+DELTA(-(j))];\
+
 #define CHECK(j)\
-    {   int score= ABS(srcp[-srcs-1+j] - srcp[+srcs-1-j])\
-                 + ABS(srcp[-srcs  +j] - srcp[+srcs  -j])\
-                 + ABS(srcp[-srcs+1+j] - srcp[+srcs+1-j]);\
+    {   int score= GET_SCORE(j);\
         if(score < spatial_score){\
             spatial_score= score;\
-            diff0= filp[-fils+j] - srcp[-srcs+j];\
-            diff1= filp[+fils-j] - srcp[+srcs-j];
+            diff0= filp[-fils+(j)] - srcp[-srcs+(j)];\
+            diff1= filp[+fils-(j)] - srcp[+srcs-(j)];\
 
-                        CHECK(-1) CHECK(-2) }} }}
-                        CHECK( 1) CHECK( 2) }} }}
+                        if (is_edge) {
+                            int spatial_score= GET_SCORE_EDGE(0)-1;
+                            CHECK_EDGE(-1) CHECK_EDGE(-2) }} }}
+                            CHECK_EDGE( 1) CHECK_EDGE( 2) }} }}
+                        } else {
+                            int spatial_score= GET_SCORE(0)-1;
+                            CHECK(-1) CHECK(-2) }} }}
+                            CHECK( 1) CHECK( 2) }} }}
+                        }
 #if 0
                         if((diff0 ^ diff1) > 0){
                             int mindiff= ABS(diff0) > ABS(diff1) ? diff1 : diff0;