changeset 9739:96e6cab7470a libavcodec

H264: Fix out of bounds reads in SSSE3 MC Reading above src[-2] isn't safe, so move loads and palignr ahead 3 pixels to load starting at the first pixel actually used. Fixes issue941.
author astrange
date Sat, 30 May 2009 22:19:14 +0000
parents d5929e456b07
children 30b00f4bdeb1
files x86/h264dsp_mmx.c
diffstat 1 files changed, 38 insertions(+), 38 deletions(-) [+]
line wrap: on
line diff
--- a/x86/h264dsp_mmx.c	Sat May 30 11:20:07 2009 +0000
+++ b/x86/h264dsp_mmx.c	Sat May 30 22:19:14 2009 +0000
@@ -1497,8 +1497,8 @@
         "movdqa %6, %%xmm14         \n\t"\
         "movdqa %7, %%xmm13         \n\t"\
         "1:                         \n\t"\
-        "lddqu    3(%0), %%xmm1     \n\t"\
-        "lddqu   -5(%0), %%xmm7     \n\t"\
+        "lddqu    6(%0), %%xmm1     \n\t"\
+        "lddqu   -2(%0), %%xmm7     \n\t"\
         "movdqa  %%xmm1, %%xmm0     \n\t"\
         "punpckhbw %%xmm15, %%xmm1  \n\t"\
         "punpcklbw %%xmm15, %%xmm0  \n\t"\
@@ -1509,20 +1509,20 @@
         "movdqa  %%xmm0, %%xmm8     \n\t"\
         "movdqa  %%xmm1, %%xmm4     \n\t"\
         "movdqa  %%xmm0, %%xmm9     \n\t"\
-        "movdqa  %%xmm1, %%xmm5     \n\t"\
-        "movdqa  %%xmm0, %%xmm10    \n\t"\
-        "palignr $6, %%xmm0, %%xmm5 \n\t"\
-        "palignr $6, %%xmm7, %%xmm10\n\t"\
-        "palignr $8, %%xmm0, %%xmm4 \n\t"\
-        "palignr $8, %%xmm7, %%xmm9 \n\t"\
-        "palignr $10,%%xmm0, %%xmm3 \n\t"\
-        "palignr $10,%%xmm7, %%xmm8 \n\t"\
-        "paddw   %%xmm1, %%xmm5     \n\t"\
-        "paddw   %%xmm0, %%xmm10    \n\t"\
-        "palignr $12,%%xmm0, %%xmm2 \n\t"\
-        "palignr $12,%%xmm7, %%xmm6 \n\t"\
-        "palignr $14,%%xmm0, %%xmm1 \n\t"\
-        "palignr $14,%%xmm7, %%xmm0 \n\t"\
+        "movdqa  %%xmm0, %%xmm12    \n\t"\
+        "movdqa  %%xmm1, %%xmm11    \n\t"\
+        "palignr $10,%%xmm0, %%xmm11\n\t"\
+        "palignr $10,%%xmm7, %%xmm12\n\t"\
+        "palignr $2, %%xmm0, %%xmm4 \n\t"\
+        "palignr $2, %%xmm7, %%xmm9 \n\t"\
+        "palignr $4, %%xmm0, %%xmm3 \n\t"\
+        "palignr $4, %%xmm7, %%xmm8 \n\t"\
+        "palignr $6, %%xmm0, %%xmm2 \n\t"\
+        "palignr $6, %%xmm7, %%xmm6 \n\t"\
+        "paddw   %%xmm0 ,%%xmm11    \n\t"\
+        "palignr $8, %%xmm0, %%xmm1 \n\t"\
+        "palignr $8, %%xmm7, %%xmm0 \n\t"\
+        "paddw   %%xmm12,%%xmm7     \n\t"\
         "paddw   %%xmm3, %%xmm2     \n\t"\
         "paddw   %%xmm8, %%xmm6     \n\t"\
         "paddw   %%xmm4, %%xmm1     \n\t"\
@@ -1531,13 +1531,13 @@
         "psllw   $2,     %%xmm6     \n\t"\
         "psubw   %%xmm1, %%xmm2     \n\t"\
         "psubw   %%xmm0, %%xmm6     \n\t"\
-        "paddw   %%xmm13,%%xmm5     \n\t"\
-        "paddw   %%xmm13,%%xmm10    \n\t"\
+        "paddw   %%xmm13,%%xmm11    \n\t"\
+        "paddw   %%xmm13,%%xmm7     \n\t"\
         "pmullw  %%xmm14,%%xmm2     \n\t"\
         "pmullw  %%xmm14,%%xmm6     \n\t"\
         "lddqu   (%2),   %%xmm3     \n\t"\
-        "paddw   %%xmm5, %%xmm2     \n\t"\
-        "paddw   %%xmm10,%%xmm6     \n\t"\
+        "paddw   %%xmm11,%%xmm2     \n\t"\
+        "paddw   %%xmm7, %%xmm6     \n\t"\
         "psraw   $5,     %%xmm2     \n\t"\
         "psraw   $5,     %%xmm6     \n\t"\
         "packuswb %%xmm2,%%xmm6     \n\t"\
@@ -1577,7 +1577,7 @@
     );\
     do{\
     __asm__ volatile(\
-        "lddqu   -5(%0), %%xmm1     \n\t"\
+        "lddqu   -2(%0), %%xmm1     \n\t"\
         "movdqa  %%xmm1, %%xmm0     \n\t"\
         "punpckhbw %%xmm7, %%xmm1   \n\t"\
         "punpcklbw %%xmm7, %%xmm0   \n\t"\
@@ -1585,20 +1585,20 @@
         "movdqa  %%xmm1, %%xmm3     \n\t"\
         "movdqa  %%xmm1, %%xmm4     \n\t"\
         "movdqa  %%xmm1, %%xmm5     \n\t"\
-        "palignr $6, %%xmm0, %%xmm5 \n\t"\
-        "palignr $8, %%xmm0, %%xmm4 \n\t"\
-        "palignr $10,%%xmm0, %%xmm3 \n\t"\
-        "paddw   %%xmm1, %%xmm5     \n\t"\
-        "palignr $12,%%xmm0, %%xmm2 \n\t"\
-        "palignr $14,%%xmm0, %%xmm1 \n\t"\
+        "palignr $2, %%xmm0, %%xmm4 \n\t"\
+        "palignr $4, %%xmm0, %%xmm3 \n\t"\
+        "palignr $6, %%xmm0, %%xmm2 \n\t"\
+        "palignr $8, %%xmm0, %%xmm1 \n\t"\
+        "palignr $10,%%xmm0, %%xmm5 \n\t"\
+        "paddw   %%xmm5, %%xmm0     \n\t"\
         "paddw   %%xmm3, %%xmm2     \n\t"\
         "paddw   %%xmm4, %%xmm1     \n\t"\
         "psllw   $2,     %%xmm2     \n\t"\
         "movq    (%2),   %%xmm3     \n\t"\
         "psubw   %%xmm1, %%xmm2     \n\t"\
-        "paddw   %5,     %%xmm5     \n\t"\
+        "paddw   %5,     %%xmm0     \n\t"\
         "pmullw  %%xmm6, %%xmm2     \n\t"\
-        "paddw   %%xmm5, %%xmm2     \n\t"\
+        "paddw   %%xmm0, %%xmm2     \n\t"\
         "psraw   $5,     %%xmm2     \n\t"\
         "packuswb %%xmm2, %%xmm2    \n\t"\
         "pavgb   %%xmm3, %%xmm2     \n\t"\
@@ -1621,7 +1621,7 @@
         "pxor %%xmm7, %%xmm7        \n\t"\
         "movdqa %5, %%xmm6          \n\t"\
         "1:                         \n\t"\
-        "lddqu   -5(%0), %%xmm1     \n\t"\
+        "lddqu   -2(%0), %%xmm1     \n\t"\
         "movdqa  %%xmm1, %%xmm0     \n\t"\
         "punpckhbw %%xmm7, %%xmm1   \n\t"\
         "punpcklbw %%xmm7, %%xmm0   \n\t"\
@@ -1629,19 +1629,19 @@
         "movdqa  %%xmm1, %%xmm3     \n\t"\
         "movdqa  %%xmm1, %%xmm4     \n\t"\
         "movdqa  %%xmm1, %%xmm5     \n\t"\
-        "palignr $6, %%xmm0, %%xmm5 \n\t"\
-        "palignr $8, %%xmm0, %%xmm4 \n\t"\
-        "palignr $10,%%xmm0, %%xmm3 \n\t"\
-        "paddw   %%xmm1, %%xmm5     \n\t"\
-        "palignr $12,%%xmm0, %%xmm2 \n\t"\
-        "palignr $14,%%xmm0, %%xmm1 \n\t"\
+        "palignr $2, %%xmm0, %%xmm4 \n\t"\
+        "palignr $4, %%xmm0, %%xmm3 \n\t"\
+        "palignr $6, %%xmm0, %%xmm2 \n\t"\
+        "palignr $8, %%xmm0, %%xmm1 \n\t"\
+        "palignr $10,%%xmm0, %%xmm5 \n\t"\
+        "paddw   %%xmm5, %%xmm0     \n\t"\
         "paddw   %%xmm3, %%xmm2     \n\t"\
         "paddw   %%xmm4, %%xmm1     \n\t"\
         "psllw   $2,     %%xmm2     \n\t"\
         "psubw   %%xmm1, %%xmm2     \n\t"\
-        "paddw   %6,     %%xmm5     \n\t"\
+        "paddw   %6,     %%xmm0     \n\t"\
         "pmullw  %%xmm6, %%xmm2     \n\t"\
-        "paddw   %%xmm5, %%xmm2     \n\t"\
+        "paddw   %%xmm0, %%xmm2     \n\t"\
         "psraw   $5,     %%xmm2     \n\t"\
         "packuswb %%xmm2, %%xmm2    \n\t"\
         OP(%%xmm2, (%1), %%xmm4, q)\