Mercurial > libavcodec.hg
annotate i386/h264dsp_mmx.c @ 3102:fcc2892eeab3 libavcodec
10l in 1.12
author | lorenm |
---|---|
date | Thu, 09 Feb 2006 02:43:23 +0000 |
parents | 966b3f321af0 |
children | 2d35fb3cb940 |
rev | line source |
---|---|
2754 | 1 /* |
2 * Copyright (c) 2004-2005 Michael Niedermayer, Loren Merritt | |
3 * | |
4 * This library is free software; you can redistribute it and/or | |
5 * modify it under the terms of the GNU Lesser General Public | |
6 * License as published by the Free Software Foundation; either | |
7 * version 2 of the License, or (at your option) any later version. | |
8 * | |
9 * This library is distributed in the hope that it will be useful, | |
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
12 * Lesser General Public License for more details. | |
13 * | |
14 * You should have received a copy of the GNU Lesser General Public | |
15 * License along with this library; if not, write to the Free Software | |
3036
0b546eab515d
Update licensing information: The FSF changed postal address.
diego
parents:
3029
diff
changeset
|
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
2754 | 17 */ |
18 | |
19 | |
20 /***********************************/ | |
21 /* IDCT */ | |
22 | |
23 /* in/out: mma=mma+mmb, mmb=mmb-mma */ | |
24 #define SUMSUB_BA( a, b ) \ | |
25 "paddw "#b", "#a" \n\t"\ | |
26 "paddw "#b", "#b" \n\t"\ | |
27 "psubw "#a", "#b" \n\t" | |
28 | |
29 #define SUMSUB_BADC( a, b, c, d ) \ | |
30 "paddw "#b", "#a" \n\t"\ | |
31 "paddw "#d", "#c" \n\t"\ | |
32 "paddw "#b", "#b" \n\t"\ | |
33 "paddw "#d", "#d" \n\t"\ | |
34 "psubw "#a", "#b" \n\t"\ | |
35 "psubw "#c", "#d" \n\t" | |
36 | |
37 #define SUMSUBD2_AB( a, b, t ) \ | |
38 "movq "#b", "#t" \n\t"\ | |
39 "psraw $1 , "#b" \n\t"\ | |
40 "paddw "#a", "#b" \n\t"\ | |
41 "psraw $1 , "#a" \n\t"\ | |
42 "psubw "#t", "#a" \n\t" | |
43 | |
44 #define IDCT4_1D( s02, s13, d02, d13, t ) \ | |
45 SUMSUB_BA ( s02, d02 )\ | |
46 SUMSUBD2_AB( s13, d13, t )\ | |
47 SUMSUB_BADC( d13, s02, s13, d02 ) | |
48 | |
49 #define SBUTTERFLY(a,b,t,n)\ | |
2979 | 50 "movq " #a ", " #t " \n\t" /* abcd */\ |
51 "punpckl" #n " " #b ", " #a " \n\t" /* aebf */\ | |
52 "punpckh" #n " " #b ", " #t " \n\t" /* cgdh */\ | |
2754 | 53 |
54 #define TRANSPOSE4(a,b,c,d,t)\ | |
55 SBUTTERFLY(a,b,t,wd) /* a=aebf t=cgdh */\ | |
56 SBUTTERFLY(c,d,b,wd) /* c=imjn b=kolp */\ | |
57 SBUTTERFLY(a,c,d,dq) /* a=aeim d=bfjn */\ | |
58 SBUTTERFLY(t,b,c,dq) /* t=cgko c=dhlp */ | |
59 | |
60 #define STORE_DIFF_4P( p, t, z ) \ | |
61 "psraw $6, "#p" \n\t"\ | |
62 "movd (%0), "#t" \n\t"\ | |
63 "punpcklbw "#z", "#t" \n\t"\ | |
64 "paddsw "#t", "#p" \n\t"\ | |
65 "packuswb "#z", "#p" \n\t"\ | |
66 "movd "#p", (%0) \n\t" | |
67 | |
68 void ff_h264_idct_add_mmx2(uint8_t *dst, int16_t *block, int stride) | |
69 { | |
70 /* Load dct coeffs */ | |
71 asm volatile( | |
72 "movq (%0), %%mm0 \n\t" | |
73 "movq 8(%0), %%mm1 \n\t" | |
74 "movq 16(%0), %%mm2 \n\t" | |
75 "movq 24(%0), %%mm3 \n\t" | |
76 :: "r"(block) ); | |
77 | |
78 asm volatile( | |
79 /* mm1=s02+s13 mm2=s02-s13 mm4=d02+d13 mm0=d02-d13 */ | |
80 IDCT4_1D( %%mm2, %%mm1, %%mm0, %%mm3, %%mm4 ) | |
81 | |
82 "movq %0, %%mm6 \n\t" | |
83 /* in: 1,4,0,2 out: 1,2,3,0 */ | |
84 TRANSPOSE4( %%mm3, %%mm1, %%mm0, %%mm2, %%mm4 ) | |
85 | |
86 "paddw %%mm6, %%mm3 \n\t" | |
87 | |
88 /* mm2=s02+s13 mm3=s02-s13 mm4=d02+d13 mm1=d02-d13 */ | |
89 IDCT4_1D( %%mm4, %%mm2, %%mm3, %%mm0, %%mm1 ) | |
90 | |
91 "pxor %%mm7, %%mm7 \n\t" | |
92 :: "m"(ff_pw_32)); | |
93 | |
94 asm volatile( | |
95 STORE_DIFF_4P( %%mm0, %%mm1, %%mm7) | |
96 "add %1, %0 \n\t" | |
97 STORE_DIFF_4P( %%mm2, %%mm1, %%mm7) | |
98 "add %1, %0 \n\t" | |
99 STORE_DIFF_4P( %%mm3, %%mm1, %%mm7) | |
100 "add %1, %0 \n\t" | |
101 STORE_DIFF_4P( %%mm4, %%mm1, %%mm7) | |
102 : "+r"(dst) | |
103 : "r" ((long)stride) | |
104 ); | |
105 } | |
106 | |
107 | |
108 /***********************************/ | |
109 /* deblocking */ | |
110 | |
111 // out: o = |x-y|>a | |
112 // clobbers: t | |
113 #define DIFF_GT_MMX(x,y,a,o,t)\ | |
114 "movq "#y", "#t" \n\t"\ | |
115 "movq "#x", "#o" \n\t"\ | |
116 "psubusb "#x", "#t" \n\t"\ | |
117 "psubusb "#y", "#o" \n\t"\ | |
118 "por "#t", "#o" \n\t"\ | |
119 "psubusb "#a", "#o" \n\t" | |
120 | |
121 // in: mm0=p1 mm1=p0 mm2=q0 mm3=q1 | |
122 // out: mm5=beta-1, mm7=mask | |
123 // clobbers: mm4,mm6 | |
124 #define H264_DEBLOCK_MASK(alpha1, beta1) \ | |
125 "pshufw $0, "#alpha1", %%mm4 \n\t"\ | |
126 "pshufw $0, "#beta1 ", %%mm5 \n\t"\ | |
127 "packuswb %%mm4, %%mm4 \n\t"\ | |
128 "packuswb %%mm5, %%mm5 \n\t"\ | |
129 DIFF_GT_MMX(%%mm1, %%mm2, %%mm4, %%mm7, %%mm6) /* |p0-q0| > alpha-1 */\ | |
130 DIFF_GT_MMX(%%mm0, %%mm1, %%mm5, %%mm4, %%mm6) /* |p1-p0| > beta-1 */\ | |
131 "por %%mm4, %%mm7 \n\t"\ | |
132 DIFF_GT_MMX(%%mm3, %%mm2, %%mm5, %%mm4, %%mm6) /* |q1-q0| > beta-1 */\ | |
133 "por %%mm4, %%mm7 \n\t"\ | |
134 "pxor %%mm6, %%mm6 \n\t"\ | |
135 "pcmpeqb %%mm6, %%mm7 \n\t" | |
136 | |
137 // in: mm0=p1 mm1=p0 mm2=q0 mm3=q1 mm7=(tc&mask) | |
138 // out: mm1=p0' mm2=q0' | |
139 // clobbers: mm0,3-6 | |
140 #define H264_DEBLOCK_P0_Q0(pb_01, pb_3f)\ | |
141 /* a = q0^p0^((p1-q1)>>2) */\ | |
142 "movq %%mm0, %%mm4 \n\t"\ | |
143 "psubb %%mm3, %%mm4 \n\t"\ | |
144 "psrlw $2, %%mm4 \n\t"\ | |
145 "pxor %%mm1, %%mm4 \n\t"\ | |
146 "pxor %%mm2, %%mm4 \n\t"\ | |
147 /* b = p0^(q1>>2) */\ | |
148 "psrlw $2, %%mm3 \n\t"\ | |
149 "pand "#pb_3f", %%mm3 \n\t"\ | |
150 "movq %%mm1, %%mm5 \n\t"\ | |
151 "pxor %%mm3, %%mm5 \n\t"\ | |
152 /* c = q0^(p1>>2) */\ | |
153 "psrlw $2, %%mm0 \n\t"\ | |
154 "pand "#pb_3f", %%mm0 \n\t"\ | |
155 "movq %%mm2, %%mm6 \n\t"\ | |
156 "pxor %%mm0, %%mm6 \n\t"\ | |
157 /* d = (c^b) & ~(b^a) & 1 */\ | |
158 "pxor %%mm5, %%mm6 \n\t"\ | |
159 "pxor %%mm4, %%mm5 \n\t"\ | |
160 "pandn %%mm6, %%mm5 \n\t"\ | |
161 "pand "#pb_01", %%mm5 \n\t"\ | |
162 /* delta = (avg(q0, p1>>2) + (d&a)) | |
163 * - (avg(p0, q1>>2) + (d&~a)) */\ | |
164 "pavgb %%mm2, %%mm0 \n\t"\ | |
2855
da95623de901
simplify (d&a) and (d&~a) calculation, hint by skal
michael
parents:
2754
diff
changeset
|
165 "pand %%mm5, %%mm4 \n\t"\ |
da95623de901
simplify (d&a) and (d&~a) calculation, hint by skal
michael
parents:
2754
diff
changeset
|
166 "paddusb %%mm4, %%mm0 \n\t"\ |
2754 | 167 "pavgb %%mm1, %%mm3 \n\t"\ |
2855
da95623de901
simplify (d&a) and (d&~a) calculation, hint by skal
michael
parents:
2754
diff
changeset
|
168 "pxor %%mm5, %%mm4 \n\t"\ |
2754 | 169 "paddusb %%mm4, %%mm3 \n\t"\ |
170 /* p0 += clip(delta, -tc0, tc0) | |
171 * q0 -= clip(delta, -tc0, tc0) */\ | |
172 "movq %%mm0, %%mm4 \n\t"\ | |
173 "psubusb %%mm3, %%mm0 \n\t"\ | |
174 "psubusb %%mm4, %%mm3 \n\t"\ | |
175 "pminub %%mm7, %%mm0 \n\t"\ | |
176 "pminub %%mm7, %%mm3 \n\t"\ | |
177 "paddusb %%mm0, %%mm1 \n\t"\ | |
178 "paddusb %%mm3, %%mm2 \n\t"\ | |
179 "psubusb %%mm3, %%mm1 \n\t"\ | |
180 "psubusb %%mm0, %%mm2 \n\t" | |
181 | |
182 // in: mm0=p1 mm1=p0 mm2=q0 mm3=q1 mm7=(tc&mask) %8=mm_bone | |
183 // out: (q1addr) = clip( (q2+((p0+q0+1)>>1))>>1, q1-tc0, q1+tc0 ) | |
184 // clobbers: q2, tmp, tc0 | |
185 #define H264_DEBLOCK_Q1(p1, q2, q2addr, q1addr, tc0, tmp)\ | |
186 "movq %%mm1, "#tmp" \n\t"\ | |
187 "pavgb %%mm2, "#tmp" \n\t"\ | |
188 "pavgb "#tmp", "#q2" \n\t" /* avg(p2,avg(p0,q0)) */\ | |
189 "pxor "q2addr", "#tmp" \n\t"\ | |
190 "pand %8, "#tmp" \n\t" /* (p2^avg(p0,q0))&1 */\ | |
191 "psubusb "#tmp", "#q2" \n\t" /* (p2+((p0+q0+1)>>1))>>1 */\ | |
192 "movq "#p1", "#tmp" \n\t"\ | |
193 "psubusb "#tc0", "#tmp" \n\t"\ | |
194 "paddusb "#p1", "#tc0" \n\t"\ | |
195 "pmaxub "#tmp", "#q2" \n\t"\ | |
196 "pminub "#tc0", "#q2" \n\t"\ | |
197 "movq "#q2", "q1addr" \n\t" | |
198 | |
199 static inline void h264_loop_filter_luma_mmx2(uint8_t *pix, int stride, int alpha1, int beta1, int8_t *tc0) | |
200 { | |
201 uint64_t tmp0; | |
202 uint64_t tc = (uint8_t)tc0[1]*0x01010000 | (uint8_t)tc0[0]*0x0101; | |
203 // with luma, tc0=0 doesn't mean no filtering, so we need a separate input mask | |
204 uint32_t mask[2] = { (tc0[0]>=0)*0xffffffff, (tc0[1]>=0)*0xffffffff }; | |
205 | |
206 asm volatile( | |
207 "movq (%1,%3), %%mm0 \n\t" //p1 | |
208 "movq (%1,%3,2), %%mm1 \n\t" //p0 | |
209 "movq (%2), %%mm2 \n\t" //q0 | |
210 "movq (%2,%3), %%mm3 \n\t" //q1 | |
211 H264_DEBLOCK_MASK(%6, %7) | |
212 "pand %5, %%mm7 \n\t" | |
213 "movq %%mm7, %0 \n\t" | |
214 | |
215 /* filter p1 */ | |
216 "movq (%1), %%mm3 \n\t" //p2 | |
217 DIFF_GT_MMX(%%mm1, %%mm3, %%mm5, %%mm6, %%mm4) // |p2-p0|>beta-1 | |
218 "pandn %%mm7, %%mm6 \n\t" | |
219 "pcmpeqb %%mm7, %%mm6 \n\t" | |
220 "pand %%mm7, %%mm6 \n\t" // mask & |p2-p0|<beta | |
221 "pshufw $80, %4, %%mm4 \n\t" | |
222 "pand %%mm7, %%mm4 \n\t" // mask & tc0 | |
223 "movq %8, %%mm7 \n\t" | |
224 "pand %%mm6, %%mm7 \n\t" // mask & |p2-p0|<beta & 1 | |
225 "pand %%mm4, %%mm6 \n\t" // mask & |p2-p0|<beta & tc0 | |
226 "paddb %%mm4, %%mm7 \n\t" // tc++ | |
227 H264_DEBLOCK_Q1(%%mm0, %%mm3, "(%1)", "(%1,%3)", %%mm6, %%mm4) | |
228 | |
229 /* filter q1 */ | |
230 "movq (%2,%3,2), %%mm4 \n\t" //q2 | |
231 DIFF_GT_MMX(%%mm2, %%mm4, %%mm5, %%mm6, %%mm3) // |q2-q0|>beta-1 | |
232 "pandn %0, %%mm6 \n\t" | |
233 "pcmpeqb %0, %%mm6 \n\t" | |
234 "pand %0, %%mm6 \n\t" | |
235 "pshufw $80, %4, %%mm5 \n\t" | |
236 "pand %%mm6, %%mm5 \n\t" | |
237 "pand %8, %%mm6 \n\t" | |
238 "paddb %%mm6, %%mm7 \n\t" | |
239 "movq (%2,%3), %%mm3 \n\t" | |
240 H264_DEBLOCK_Q1(%%mm3, %%mm4, "(%2,%3,2)", "(%2,%3)", %%mm5, %%mm6) | |
241 | |
242 /* filter p0, q0 */ | |
243 H264_DEBLOCK_P0_Q0(%8, %9) | |
244 "movq %%mm1, (%1,%3,2) \n\t" | |
245 "movq %%mm2, (%2) \n\t" | |
246 | |
247 : "=m"(tmp0) | |
248 : "r"(pix-3*stride), "r"(pix), "r"((long)stride), | |
249 "m"(tc), "m"(*(uint64_t*)mask), "m"(alpha1), "m"(beta1), | |
250 "m"(mm_bone), "m"(ff_pb_3F) | |
251 ); | |
252 } | |
253 | |
254 static void h264_v_loop_filter_luma_mmx2(uint8_t *pix, int stride, int alpha, int beta, int8_t *tc0) | |
255 { | |
256 if((tc0[0] & tc0[1]) >= 0) | |
257 h264_loop_filter_luma_mmx2(pix, stride, alpha-1, beta-1, tc0); | |
258 if((tc0[2] & tc0[3]) >= 0) | |
259 h264_loop_filter_luma_mmx2(pix+8, stride, alpha-1, beta-1, tc0+2); | |
260 } | |
261 static void h264_h_loop_filter_luma_mmx2(uint8_t *pix, int stride, int alpha, int beta, int8_t *tc0) | |
262 { | |
263 //FIXME: could cut some load/stores by merging transpose with filter | |
264 // also, it only needs to transpose 6x8 | |
265 uint8_t trans[8*8]; | |
266 int i; | |
267 for(i=0; i<2; i++, pix+=8*stride, tc0+=2) { | |
268 if((tc0[0] & tc0[1]) < 0) | |
269 continue; | |
270 transpose4x4(trans, pix-4, 8, stride); | |
271 transpose4x4(trans +4*8, pix, 8, stride); | |
272 transpose4x4(trans+4, pix-4+4*stride, 8, stride); | |
273 transpose4x4(trans+4+4*8, pix +4*stride, 8, stride); | |
274 h264_loop_filter_luma_mmx2(trans+4*8, 8, alpha-1, beta-1, tc0); | |
275 transpose4x4(pix-2, trans +2*8, stride, 8); | |
276 transpose4x4(pix-2+4*stride, trans+4+2*8, stride, 8); | |
277 } | |
278 } | |
279 | |
280 static inline void h264_loop_filter_chroma_mmx2(uint8_t *pix, int stride, int alpha1, int beta1, int8_t *tc0) | |
281 { | |
282 asm volatile( | |
283 "movq (%0), %%mm0 \n\t" //p1 | |
284 "movq (%0,%2), %%mm1 \n\t" //p0 | |
285 "movq (%1), %%mm2 \n\t" //q0 | |
286 "movq (%1,%2), %%mm3 \n\t" //q1 | |
287 H264_DEBLOCK_MASK(%4, %5) | |
288 "movd %3, %%mm6 \n\t" | |
289 "punpcklbw %%mm6, %%mm6 \n\t" | |
290 "pand %%mm6, %%mm7 \n\t" // mm7 = tc&mask | |
291 H264_DEBLOCK_P0_Q0(%6, %7) | |
292 "movq %%mm1, (%0,%2) \n\t" | |
293 "movq %%mm2, (%1) \n\t" | |
294 | |
295 :: "r"(pix-2*stride), "r"(pix), "r"((long)stride), | |
296 "r"(*(uint32_t*)tc0), | |
297 "m"(alpha1), "m"(beta1), "m"(mm_bone), "m"(ff_pb_3F) | |
298 ); | |
299 } | |
300 | |
301 static void h264_v_loop_filter_chroma_mmx2(uint8_t *pix, int stride, int alpha, int beta, int8_t *tc0) | |
302 { | |
303 h264_loop_filter_chroma_mmx2(pix, stride, alpha-1, beta-1, tc0); | |
304 } | |
305 | |
306 static void h264_h_loop_filter_chroma_mmx2(uint8_t *pix, int stride, int alpha, int beta, int8_t *tc0) | |
307 { | |
308 //FIXME: could cut some load/stores by merging transpose with filter | |
309 uint8_t trans[8*4]; | |
310 transpose4x4(trans, pix-2, 8, stride); | |
311 transpose4x4(trans+4, pix-2+4*stride, 8, stride); | |
312 h264_loop_filter_chroma_mmx2(trans+2*8, 8, alpha-1, beta-1, tc0); | |
313 transpose4x4(pix-2, trans, stride, 8); | |
314 transpose4x4(pix-2+4*stride, trans+4, stride, 8); | |
315 } | |
316 | |
317 // p0 = (p0 + q1 + 2*p1 + 2) >> 2 | |
318 #define H264_FILTER_CHROMA4(p0, p1, q1, one) \ | |
319 "movq "#p0", %%mm4 \n\t"\ | |
320 "pxor "#q1", %%mm4 \n\t"\ | |
321 "pand "#one", %%mm4 \n\t" /* mm4 = (p0^q1)&1 */\ | |
322 "pavgb "#q1", "#p0" \n\t"\ | |
323 "psubusb %%mm4, "#p0" \n\t"\ | |
324 "pavgb "#p1", "#p0" \n\t" /* dst = avg(p1, avg(p0,q1) - ((p0^q1)&1)) */\ | |
325 | |
326 static inline void h264_loop_filter_chroma_intra_mmx2(uint8_t *pix, int stride, int alpha1, int beta1) | |
327 { | |
328 asm volatile( | |
329 "movq (%0), %%mm0 \n\t" | |
330 "movq (%0,%2), %%mm1 \n\t" | |
331 "movq (%1), %%mm2 \n\t" | |
332 "movq (%1,%2), %%mm3 \n\t" | |
333 H264_DEBLOCK_MASK(%3, %4) | |
334 "movq %%mm1, %%mm5 \n\t" | |
335 "movq %%mm2, %%mm6 \n\t" | |
336 H264_FILTER_CHROMA4(%%mm1, %%mm0, %%mm3, %5) //p0' | |
337 H264_FILTER_CHROMA4(%%mm2, %%mm3, %%mm0, %5) //q0' | |
338 "psubb %%mm5, %%mm1 \n\t" | |
339 "psubb %%mm6, %%mm2 \n\t" | |
340 "pand %%mm7, %%mm1 \n\t" | |
341 "pand %%mm7, %%mm2 \n\t" | |
342 "paddb %%mm5, %%mm1 \n\t" | |
343 "paddb %%mm6, %%mm2 \n\t" | |
344 "movq %%mm1, (%0,%2) \n\t" | |
345 "movq %%mm2, (%1) \n\t" | |
346 :: "r"(pix-2*stride), "r"(pix), "r"((long)stride), | |
347 "m"(alpha1), "m"(beta1), "m"(mm_bone) | |
348 ); | |
349 } | |
350 | |
351 static void h264_v_loop_filter_chroma_intra_mmx2(uint8_t *pix, int stride, int alpha, int beta) | |
352 { | |
353 h264_loop_filter_chroma_intra_mmx2(pix, stride, alpha-1, beta-1); | |
354 } | |
355 | |
356 static void h264_h_loop_filter_chroma_intra_mmx2(uint8_t *pix, int stride, int alpha, int beta) | |
357 { | |
358 //FIXME: could cut some load/stores by merging transpose with filter | |
359 uint8_t trans[8*4]; | |
360 transpose4x4(trans, pix-2, 8, stride); | |
361 transpose4x4(trans+4, pix-2+4*stride, 8, stride); | |
362 h264_loop_filter_chroma_intra_mmx2(trans+2*8, 8, alpha-1, beta-1); | |
363 transpose4x4(pix-2, trans, stride, 8); | |
364 transpose4x4(pix-2+4*stride, trans+4, stride, 8); | |
365 } | |
366 | |
367 | |
368 /***********************************/ | |
369 /* motion compensation */ | |
370 | |
371 #define QPEL_H264V(A,B,C,D,E,F,OP)\ | |
2979 | 372 "movd (%0), "#F" \n\t"\ |
373 "movq "#C", %%mm6 \n\t"\ | |
374 "paddw "#D", %%mm6 \n\t"\ | |
375 "psllw $2, %%mm6 \n\t"\ | |
376 "psubw "#B", %%mm6 \n\t"\ | |
377 "psubw "#E", %%mm6 \n\t"\ | |
378 "pmullw %4, %%mm6 \n\t"\ | |
379 "add %2, %0 \n\t"\ | |
380 "punpcklbw %%mm7, "#F" \n\t"\ | |
381 "paddw %5, "#A" \n\t"\ | |
382 "paddw "#F", "#A" \n\t"\ | |
383 "paddw "#A", %%mm6 \n\t"\ | |
384 "psraw $5, %%mm6 \n\t"\ | |
385 "packuswb %%mm6, %%mm6 \n\t"\ | |
2754 | 386 OP(%%mm6, (%1), A, d)\ |
2979 | 387 "add %3, %1 \n\t" |
2754 | 388 |
389 #define QPEL_H264HV(A,B,C,D,E,F,OF)\ | |
2979 | 390 "movd (%0), "#F" \n\t"\ |
391 "movq "#C", %%mm6 \n\t"\ | |
392 "paddw "#D", %%mm6 \n\t"\ | |
393 "psllw $2, %%mm6 \n\t"\ | |
394 "psubw "#B", %%mm6 \n\t"\ | |
395 "psubw "#E", %%mm6 \n\t"\ | |
396 "pmullw %3, %%mm6 \n\t"\ | |
397 "add %2, %0 \n\t"\ | |
398 "punpcklbw %%mm7, "#F" \n\t"\ | |
399 "paddw "#F", "#A" \n\t"\ | |
400 "paddw "#A", %%mm6 \n\t"\ | |
401 "movq %%mm6, "#OF"(%1) \n\t" | |
2967 | 402 |
2754 | 403 #define QPEL_H264(OPNAME, OP, MMX)\ |
404 static void OPNAME ## h264_qpel4_h_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\ | |
405 int h=4;\ | |
406 \ | |
407 asm volatile(\ | |
2979 | 408 "pxor %%mm7, %%mm7 \n\t"\ |
409 "movq %5, %%mm4 \n\t"\ | |
410 "movq %6, %%mm5 \n\t"\ | |
411 "1: \n\t"\ | |
412 "movd -1(%0), %%mm1 \n\t"\ | |
413 "movd (%0), %%mm2 \n\t"\ | |
414 "movd 1(%0), %%mm3 \n\t"\ | |
415 "movd 2(%0), %%mm0 \n\t"\ | |
416 "punpcklbw %%mm7, %%mm1 \n\t"\ | |
417 "punpcklbw %%mm7, %%mm2 \n\t"\ | |
418 "punpcklbw %%mm7, %%mm3 \n\t"\ | |
419 "punpcklbw %%mm7, %%mm0 \n\t"\ | |
420 "paddw %%mm0, %%mm1 \n\t"\ | |
421 "paddw %%mm3, %%mm2 \n\t"\ | |
422 "movd -2(%0), %%mm0 \n\t"\ | |
423 "movd 3(%0), %%mm3 \n\t"\ | |
424 "punpcklbw %%mm7, %%mm0 \n\t"\ | |
425 "punpcklbw %%mm7, %%mm3 \n\t"\ | |
426 "paddw %%mm3, %%mm0 \n\t"\ | |
427 "psllw $2, %%mm2 \n\t"\ | |
428 "psubw %%mm1, %%mm2 \n\t"\ | |
429 "pmullw %%mm4, %%mm2 \n\t"\ | |
430 "paddw %%mm5, %%mm0 \n\t"\ | |
431 "paddw %%mm2, %%mm0 \n\t"\ | |
432 "psraw $5, %%mm0 \n\t"\ | |
433 "packuswb %%mm0, %%mm0 \n\t"\ | |
2754 | 434 OP(%%mm0, (%1),%%mm6, d)\ |
2979 | 435 "add %3, %0 \n\t"\ |
436 "add %4, %1 \n\t"\ | |
437 "decl %2 \n\t"\ | |
438 " jnz 1b \n\t"\ | |
2754 | 439 : "+a"(src), "+c"(dst), "+m"(h)\ |
440 : "d"((long)srcStride), "S"((long)dstStride), "m"(ff_pw_5), "m"(ff_pw_16)\ | |
441 : "memory"\ | |
442 );\ | |
443 }\ | |
444 static void OPNAME ## h264_qpel4_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\ | |
445 src -= 2*srcStride;\ | |
446 asm volatile(\ | |
2979 | 447 "pxor %%mm7, %%mm7 \n\t"\ |
448 "movd (%0), %%mm0 \n\t"\ | |
449 "add %2, %0 \n\t"\ | |
450 "movd (%0), %%mm1 \n\t"\ | |
451 "add %2, %0 \n\t"\ | |
452 "movd (%0), %%mm2 \n\t"\ | |
453 "add %2, %0 \n\t"\ | |
454 "movd (%0), %%mm3 \n\t"\ | |
455 "add %2, %0 \n\t"\ | |
456 "movd (%0), %%mm4 \n\t"\ | |
457 "add %2, %0 \n\t"\ | |
458 "punpcklbw %%mm7, %%mm0 \n\t"\ | |
459 "punpcklbw %%mm7, %%mm1 \n\t"\ | |
460 "punpcklbw %%mm7, %%mm2 \n\t"\ | |
461 "punpcklbw %%mm7, %%mm3 \n\t"\ | |
462 "punpcklbw %%mm7, %%mm4 \n\t"\ | |
2754 | 463 QPEL_H264V(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, OP)\ |
464 QPEL_H264V(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, OP)\ | |
465 QPEL_H264V(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, OP)\ | |
466 QPEL_H264V(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, OP)\ | |
467 \ | |
468 : "+a"(src), "+c"(dst)\ | |
469 : "S"((long)srcStride), "D"((long)dstStride), "m"(ff_pw_5), "m"(ff_pw_16)\ | |
470 : "memory"\ | |
471 );\ | |
472 }\ | |
473 static void OPNAME ## h264_qpel4_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride){\ | |
474 int h=4;\ | |
475 int w=3;\ | |
476 src -= 2*srcStride+2;\ | |
477 while(w--){\ | |
478 asm volatile(\ | |
2979 | 479 "pxor %%mm7, %%mm7 \n\t"\ |
480 "movd (%0), %%mm0 \n\t"\ | |
481 "add %2, %0 \n\t"\ | |
482 "movd (%0), %%mm1 \n\t"\ | |
483 "add %2, %0 \n\t"\ | |
484 "movd (%0), %%mm2 \n\t"\ | |
485 "add %2, %0 \n\t"\ | |
486 "movd (%0), %%mm3 \n\t"\ | |
487 "add %2, %0 \n\t"\ | |
488 "movd (%0), %%mm4 \n\t"\ | |
489 "add %2, %0 \n\t"\ | |
490 "punpcklbw %%mm7, %%mm0 \n\t"\ | |
491 "punpcklbw %%mm7, %%mm1 \n\t"\ | |
492 "punpcklbw %%mm7, %%mm2 \n\t"\ | |
493 "punpcklbw %%mm7, %%mm3 \n\t"\ | |
494 "punpcklbw %%mm7, %%mm4 \n\t"\ | |
2754 | 495 QPEL_H264HV(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, 0*8*3)\ |
496 QPEL_H264HV(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, 1*8*3)\ | |
497 QPEL_H264HV(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, 2*8*3)\ | |
498 QPEL_H264HV(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, 3*8*3)\ | |
499 \ | |
500 : "+a"(src)\ | |
501 : "c"(tmp), "S"((long)srcStride), "m"(ff_pw_5)\ | |
502 : "memory"\ | |
503 );\ | |
504 tmp += 4;\ | |
505 src += 4 - 9*srcStride;\ | |
506 }\ | |
507 tmp -= 3*4;\ | |
508 asm volatile(\ | |
2979 | 509 "movq %4, %%mm6 \n\t"\ |
510 "1: \n\t"\ | |
511 "movq (%0), %%mm0 \n\t"\ | |
512 "paddw 10(%0), %%mm0 \n\t"\ | |
513 "movq 2(%0), %%mm1 \n\t"\ | |
514 "paddw 8(%0), %%mm1 \n\t"\ | |
515 "movq 4(%0), %%mm2 \n\t"\ | |
516 "paddw 6(%0), %%mm2 \n\t"\ | |
517 "psubw %%mm1, %%mm0 \n\t"/*a-b (abccba)*/\ | |
518 "psraw $2, %%mm0 \n\t"/*(a-b)/4 */\ | |
519 "psubw %%mm1, %%mm0 \n\t"/*(a-b)/4-b */\ | |
520 "paddsw %%mm2, %%mm0 \n\t"\ | |
3001
b52d8ee430f6
fix some potential arithmetic overflows in pred_direct_motion() and
lorenm
parents:
2979
diff
changeset
|
521 "psraw $2, %%mm0 \n\t"/*((a-b)/4-b+c)/4 */\ |
2979 | 522 "paddw %%mm6, %%mm2 \n\t"\ |
3001
b52d8ee430f6
fix some potential arithmetic overflows in pred_direct_motion() and
lorenm
parents:
2979
diff
changeset
|
523 "paddw %%mm2, %%mm0 \n\t"/*(a-5*b+20*c)/16 +32 */\ |
2979 | 524 "psraw $6, %%mm0 \n\t"\ |
525 "packuswb %%mm0, %%mm0 \n\t"\ | |
2754 | 526 OP(%%mm0, (%1),%%mm7, d)\ |
2979 | 527 "add $24, %0 \n\t"\ |
528 "add %3, %1 \n\t"\ | |
529 "decl %2 \n\t"\ | |
530 " jnz 1b \n\t"\ | |
2754 | 531 : "+a"(tmp), "+c"(dst), "+m"(h)\ |
532 : "S"((long)dstStride), "m"(ff_pw_32)\ | |
533 : "memory"\ | |
534 );\ | |
535 }\ | |
536 \ | |
537 static void OPNAME ## h264_qpel8_h_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\ | |
538 int h=8;\ | |
539 asm volatile(\ | |
2979 | 540 "pxor %%mm7, %%mm7 \n\t"\ |
541 "movq %5, %%mm6 \n\t"\ | |
542 "1: \n\t"\ | |
543 "movq (%0), %%mm0 \n\t"\ | |
544 "movq 1(%0), %%mm2 \n\t"\ | |
545 "movq %%mm0, %%mm1 \n\t"\ | |
546 "movq %%mm2, %%mm3 \n\t"\ | |
547 "punpcklbw %%mm7, %%mm0 \n\t"\ | |
548 "punpckhbw %%mm7, %%mm1 \n\t"\ | |
549 "punpcklbw %%mm7, %%mm2 \n\t"\ | |
550 "punpckhbw %%mm7, %%mm3 \n\t"\ | |
551 "paddw %%mm2, %%mm0 \n\t"\ | |
552 "paddw %%mm3, %%mm1 \n\t"\ | |
553 "psllw $2, %%mm0 \n\t"\ | |
554 "psllw $2, %%mm1 \n\t"\ | |
555 "movq -1(%0), %%mm2 \n\t"\ | |
556 "movq 2(%0), %%mm4 \n\t"\ | |
557 "movq %%mm2, %%mm3 \n\t"\ | |
558 "movq %%mm4, %%mm5 \n\t"\ | |
559 "punpcklbw %%mm7, %%mm2 \n\t"\ | |
560 "punpckhbw %%mm7, %%mm3 \n\t"\ | |
561 "punpcklbw %%mm7, %%mm4 \n\t"\ | |
562 "punpckhbw %%mm7, %%mm5 \n\t"\ | |
563 "paddw %%mm4, %%mm2 \n\t"\ | |
564 "paddw %%mm3, %%mm5 \n\t"\ | |
565 "psubw %%mm2, %%mm0 \n\t"\ | |
566 "psubw %%mm5, %%mm1 \n\t"\ | |
567 "pmullw %%mm6, %%mm0 \n\t"\ | |
568 "pmullw %%mm6, %%mm1 \n\t"\ | |
569 "movd -2(%0), %%mm2 \n\t"\ | |
570 "movd 7(%0), %%mm5 \n\t"\ | |
571 "punpcklbw %%mm7, %%mm2 \n\t"\ | |
572 "punpcklbw %%mm7, %%mm5 \n\t"\ | |
573 "paddw %%mm3, %%mm2 \n\t"\ | |
574 "paddw %%mm5, %%mm4 \n\t"\ | |
575 "movq %6, %%mm5 \n\t"\ | |
576 "paddw %%mm5, %%mm2 \n\t"\ | |
577 "paddw %%mm5, %%mm4 \n\t"\ | |
578 "paddw %%mm2, %%mm0 \n\t"\ | |
579 "paddw %%mm4, %%mm1 \n\t"\ | |
580 "psraw $5, %%mm0 \n\t"\ | |
581 "psraw $5, %%mm1 \n\t"\ | |
582 "packuswb %%mm1, %%mm0 \n\t"\ | |
2754 | 583 OP(%%mm0, (%1),%%mm5, q)\ |
2979 | 584 "add %3, %0 \n\t"\ |
585 "add %4, %1 \n\t"\ | |
586 "decl %2 \n\t"\ | |
587 " jnz 1b \n\t"\ | |
2754 | 588 : "+a"(src), "+c"(dst), "+m"(h)\ |
589 : "d"((long)srcStride), "S"((long)dstStride), "m"(ff_pw_5), "m"(ff_pw_16)\ | |
590 : "memory"\ | |
591 );\ | |
592 }\ | |
593 \ | |
3094 | 594 static inline void OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride, int h){\ |
595 int w= 2;\ | |
2754 | 596 src -= 2*srcStride;\ |
597 \ | |
3094 | 598 while(w--){\ |
2754 | 599 asm volatile(\ |
2979 | 600 "pxor %%mm7, %%mm7 \n\t"\ |
601 "movd (%0), %%mm0 \n\t"\ | |
602 "add %2, %0 \n\t"\ | |
603 "movd (%0), %%mm1 \n\t"\ | |
604 "add %2, %0 \n\t"\ | |
605 "movd (%0), %%mm2 \n\t"\ | |
606 "add %2, %0 \n\t"\ | |
607 "movd (%0), %%mm3 \n\t"\ | |
608 "add %2, %0 \n\t"\ | |
609 "movd (%0), %%mm4 \n\t"\ | |
610 "add %2, %0 \n\t"\ | |
611 "punpcklbw %%mm7, %%mm0 \n\t"\ | |
612 "punpcklbw %%mm7, %%mm1 \n\t"\ | |
613 "punpcklbw %%mm7, %%mm2 \n\t"\ | |
614 "punpcklbw %%mm7, %%mm3 \n\t"\ | |
615 "punpcklbw %%mm7, %%mm4 \n\t"\ | |
2754 | 616 QPEL_H264V(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, OP)\ |
617 QPEL_H264V(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, OP)\ | |
618 QPEL_H264V(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, OP)\ | |
619 QPEL_H264V(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, OP)\ | |
620 QPEL_H264V(%%mm4, %%mm5, %%mm0, %%mm1, %%mm2, %%mm3, OP)\ | |
621 QPEL_H264V(%%mm5, %%mm0, %%mm1, %%mm2, %%mm3, %%mm4, OP)\ | |
622 QPEL_H264V(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, OP)\ | |
623 QPEL_H264V(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, OP)\ | |
624 \ | |
625 : "+a"(src), "+c"(dst)\ | |
626 : "S"((long)srcStride), "D"((long)dstStride), "m"(ff_pw_5), "m"(ff_pw_16)\ | |
627 : "memory"\ | |
628 );\ | |
3094 | 629 if(h==16){\ |
630 asm volatile(\ | |
631 QPEL_H264V(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, OP)\ | |
632 QPEL_H264V(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, OP)\ | |
633 QPEL_H264V(%%mm4, %%mm5, %%mm0, %%mm1, %%mm2, %%mm3, OP)\ | |
634 QPEL_H264V(%%mm5, %%mm0, %%mm1, %%mm2, %%mm3, %%mm4, OP)\ | |
635 QPEL_H264V(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, OP)\ | |
636 QPEL_H264V(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, OP)\ | |
637 QPEL_H264V(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, OP)\ | |
638 QPEL_H264V(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, OP)\ | |
639 \ | |
640 : "+a"(src), "+c"(dst)\ | |
641 : "S"((long)srcStride), "D"((long)dstStride), "m"(ff_pw_5), "m"(ff_pw_16)\ | |
642 : "memory"\ | |
643 );\ | |
644 }\ | |
645 src += 4-(h+5)*srcStride;\ | |
646 dst += 4-h*dstStride;\ | |
2754 | 647 }\ |
648 }\ | |
3093 | 649 static inline void OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride, int size){\ |
650 int h = size;\ | |
651 int w = (size+8)>>2;\ | |
2754 | 652 src -= 2*srcStride+2;\ |
653 while(w--){\ | |
654 asm volatile(\ | |
2979 | 655 "pxor %%mm7, %%mm7 \n\t"\ |
656 "movd (%0), %%mm0 \n\t"\ | |
657 "add %2, %0 \n\t"\ | |
658 "movd (%0), %%mm1 \n\t"\ | |
659 "add %2, %0 \n\t"\ | |
660 "movd (%0), %%mm2 \n\t"\ | |
661 "add %2, %0 \n\t"\ | |
662 "movd (%0), %%mm3 \n\t"\ | |
663 "add %2, %0 \n\t"\ | |
664 "movd (%0), %%mm4 \n\t"\ | |
665 "add %2, %0 \n\t"\ | |
666 "punpcklbw %%mm7, %%mm0 \n\t"\ | |
667 "punpcklbw %%mm7, %%mm1 \n\t"\ | |
668 "punpcklbw %%mm7, %%mm2 \n\t"\ | |
669 "punpcklbw %%mm7, %%mm3 \n\t"\ | |
670 "punpcklbw %%mm7, %%mm4 \n\t"\ | |
3093 | 671 QPEL_H264HV(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, 0*48)\ |
672 QPEL_H264HV(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, 1*48)\ | |
673 QPEL_H264HV(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, 2*48)\ | |
674 QPEL_H264HV(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, 3*48)\ | |
675 QPEL_H264HV(%%mm4, %%mm5, %%mm0, %%mm1, %%mm2, %%mm3, 4*48)\ | |
676 QPEL_H264HV(%%mm5, %%mm0, %%mm1, %%mm2, %%mm3, %%mm4, 5*48)\ | |
677 QPEL_H264HV(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, 6*48)\ | |
678 QPEL_H264HV(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, 7*48)\ | |
2754 | 679 : "+a"(src)\ |
680 : "c"(tmp), "S"((long)srcStride), "m"(ff_pw_5)\ | |
681 : "memory"\ | |
682 );\ | |
3093 | 683 if(size==16){\ |
684 asm volatile(\ | |
685 QPEL_H264HV(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, 8*48)\ | |
686 QPEL_H264HV(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, 9*48)\ | |
687 QPEL_H264HV(%%mm4, %%mm5, %%mm0, %%mm1, %%mm2, %%mm3, 10*48)\ | |
688 QPEL_H264HV(%%mm5, %%mm0, %%mm1, %%mm2, %%mm3, %%mm4, 11*48)\ | |
689 QPEL_H264HV(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, 12*48)\ | |
690 QPEL_H264HV(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, 13*48)\ | |
691 QPEL_H264HV(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, 14*48)\ | |
692 QPEL_H264HV(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, 15*48)\ | |
693 : "+a"(src)\ | |
694 : "c"(tmp), "S"((long)srcStride), "m"(ff_pw_5)\ | |
695 : "memory"\ | |
696 );\ | |
697 }\ | |
2754 | 698 tmp += 4;\ |
3093 | 699 src += 4 - (size+5)*srcStride;\ |
2754 | 700 }\ |
3093 | 701 tmp -= size+8;\ |
702 w = size>>4;\ | |
703 do{\ | |
704 h = size;\ | |
2754 | 705 asm volatile(\ |
2979 | 706 "movq %4, %%mm6 \n\t"\ |
707 "1: \n\t"\ | |
708 "movq (%0), %%mm0 \n\t"\ | |
709 "movq 8(%0), %%mm3 \n\t"\ | |
710 "movq 2(%0), %%mm1 \n\t"\ | |
711 "movq 10(%0), %%mm4 \n\t"\ | |
712 "paddw %%mm4, %%mm0 \n\t"\ | |
713 "paddw %%mm3, %%mm1 \n\t"\ | |
714 "paddw 18(%0), %%mm3 \n\t"\ | |
715 "paddw 16(%0), %%mm4 \n\t"\ | |
716 "movq 4(%0), %%mm2 \n\t"\ | |
717 "movq 12(%0), %%mm5 \n\t"\ | |
718 "paddw 6(%0), %%mm2 \n\t"\ | |
719 "paddw 14(%0), %%mm5 \n\t"\ | |
720 "psubw %%mm1, %%mm0 \n\t"\ | |
721 "psubw %%mm4, %%mm3 \n\t"\ | |
722 "psraw $2, %%mm0 \n\t"\ | |
723 "psraw $2, %%mm3 \n\t"\ | |
724 "psubw %%mm1, %%mm0 \n\t"\ | |
725 "psubw %%mm4, %%mm3 \n\t"\ | |
726 "paddsw %%mm2, %%mm0 \n\t"\ | |
727 "paddsw %%mm5, %%mm3 \n\t"\ | |
728 "psraw $2, %%mm0 \n\t"\ | |
729 "psraw $2, %%mm3 \n\t"\ | |
730 "paddw %%mm6, %%mm2 \n\t"\ | |
731 "paddw %%mm6, %%mm5 \n\t"\ | |
732 "paddw %%mm2, %%mm0 \n\t"\ | |
733 "paddw %%mm5, %%mm3 \n\t"\ | |
734 "psraw $6, %%mm0 \n\t"\ | |
735 "psraw $6, %%mm3 \n\t"\ | |
736 "packuswb %%mm3, %%mm0 \n\t"\ | |
2754 | 737 OP(%%mm0, (%1),%%mm7, q)\ |
3093 | 738 "add $48, %0 \n\t"\ |
2979 | 739 "add %3, %1 \n\t"\ |
740 "decl %2 \n\t"\ | |
741 " jnz 1b \n\t"\ | |
2754 | 742 : "+a"(tmp), "+c"(dst), "+m"(h)\ |
743 : "S"((long)dstStride), "m"(ff_pw_32)\ | |
744 : "memory"\ | |
745 );\ | |
3093 | 746 tmp += 8 - size*24;\ |
747 dst += 8 - size*dstStride;\ | |
748 }while(w--);\ | |
2754 | 749 }\ |
3094 | 750 \ |
751 static void OPNAME ## h264_qpel8_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\ | |
752 OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst , src , dstStride, srcStride, 8);\ | |
753 }\ | |
2754 | 754 static void OPNAME ## h264_qpel16_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\ |
3094 | 755 OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst , src , dstStride, srcStride, 16);\ |
756 OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride, 16);\ | |
2754 | 757 }\ |
758 \ | |
759 static void OPNAME ## h264_qpel16_h_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\ | |
760 OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst , src , dstStride, srcStride);\ | |
761 OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride);\ | |
762 src += 8*srcStride;\ | |
763 dst += 8*dstStride;\ | |
764 OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst , src , dstStride, srcStride);\ | |
765 OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride);\ | |
766 }\ | |
767 \ | |
3093 | 768 static void OPNAME ## h264_qpel8_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride){\ |
769 OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(dst , tmp , src , dstStride, tmpStride, srcStride, 8);\ | |
770 }\ | |
771 \ | |
2754 | 772 static void OPNAME ## h264_qpel16_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride){\ |
3093 | 773 OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(dst , tmp , src , dstStride, tmpStride, srcStride, 16);\ |
2754 | 774 }\ |
3095 | 775 \ |
776 static void OPNAME ## pixels4_l2_shift5_ ## MMX(uint8_t *dst, int16_t *src16, uint8_t *src8, int dstStride, int src8Stride, int h)\ | |
777 {\ | |
778 asm volatile(\ | |
779 "pxor %%mm7, %%mm7 \n\t"\ | |
780 "movq %5, %%mm6 \n\t"\ | |
781 "movq (%1), %%mm0 \n\t"\ | |
782 "movq 24(%1), %%mm1 \n\t"\ | |
783 "paddw %%mm6, %%mm0 \n\t"\ | |
784 "paddw %%mm6, %%mm1 \n\t"\ | |
3102 | 785 "psraw $5, %%mm0 \n\t"\ |
786 "psraw $5, %%mm1 \n\t"\ | |
3095 | 787 "pmaxsw %%mm7, %%mm0 \n\t"\ |
788 "pmaxsw %%mm7, %%mm1 \n\t"\ | |
789 "packuswb %%mm7, %%mm0 \n\t"\ | |
790 "packuswb %%mm7, %%mm1 \n\t"\ | |
791 PAVGB" (%0), %%mm0 \n\t"\ | |
792 PAVGB" (%0,%3), %%mm1 \n\t"\ | |
793 OP(%%mm0, (%2), %%mm4, d)\ | |
794 OP(%%mm1, (%2,%4), %%mm5, d)\ | |
795 "lea (%0,%3,2), %0 \n\t"\ | |
796 "lea (%2,%4,2), %2 \n\t"\ | |
797 "movq 48(%1), %%mm0 \n\t"\ | |
798 "movq 72(%1), %%mm1 \n\t"\ | |
799 "paddw %%mm6, %%mm0 \n\t"\ | |
800 "paddw %%mm6, %%mm1 \n\t"\ | |
3102 | 801 "psraw $5, %%mm0 \n\t"\ |
802 "psraw $5, %%mm1 \n\t"\ | |
3095 | 803 "pmaxsw %%mm7, %%mm0 \n\t"\ |
804 "pmaxsw %%mm7, %%mm1 \n\t"\ | |
805 "packuswb %%mm7, %%mm0 \n\t"\ | |
806 "packuswb %%mm7, %%mm1 \n\t"\ | |
807 PAVGB" (%0), %%mm0 \n\t"\ | |
808 PAVGB" (%0,%3), %%mm1 \n\t"\ | |
809 OP(%%mm0, (%2), %%mm4, d)\ | |
810 OP(%%mm1, (%2,%4), %%mm5, d)\ | |
811 :"+a"(src8), "+c"(src16), "+d"(dst)\ | |
3096 | 812 :"S"((long)src8Stride), "D"((long)dstStride), "m"(ff_pw_16)\ |
3095 | 813 :"memory");\ |
814 }\ | |
815 static void OPNAME ## pixels8_l2_shift5_ ## MMX(uint8_t *dst, int16_t *src16, uint8_t *src8, int dstStride, int src8Stride, int h)\ | |
816 {\ | |
817 asm volatile(\ | |
818 "pxor %%mm7, %%mm7 \n\t"\ | |
819 "movq %0, %%mm6 \n\t"\ | |
820 ::"m"(ff_pw_16)\ | |
821 );\ | |
822 while(h--){\ | |
823 asm volatile(\ | |
824 "movq (%1), %%mm0 \n\t"\ | |
825 "movq 8(%1), %%mm1 \n\t"\ | |
826 "paddw %%mm6, %%mm0 \n\t"\ | |
827 "paddw %%mm6, %%mm1 \n\t"\ | |
3102 | 828 "psraw $5, %%mm0 \n\t"\ |
829 "psraw $5, %%mm1 \n\t"\ | |
3095 | 830 "pmaxsw %%mm7, %%mm0 \n\t"\ |
831 "pmaxsw %%mm7, %%mm1 \n\t"\ | |
832 "packuswb %%mm1, %%mm0 \n\t"\ | |
833 PAVGB" (%0), %%mm0 \n\t"\ | |
834 OP(%%mm0, (%2), %%mm5, q)\ | |
835 ::"a"(src8), "c"(src16), "d"(dst)\ | |
836 :"memory");\ | |
837 src8 += src8Stride;\ | |
838 src16 += 24;\ | |
839 dst += dstStride;\ | |
840 }\ | |
841 }\ | |
842 static void OPNAME ## pixels16_l2_shift5_ ## MMX(uint8_t *dst, int16_t *src16, uint8_t *src8, int dstStride, int src8Stride, int h)\ | |
843 {\ | |
844 OPNAME ## pixels8_l2_shift5_ ## MMX(dst , src16 , src8 , dstStride, src8Stride, h);\ | |
845 OPNAME ## pixels8_l2_shift5_ ## MMX(dst+8, src16+8, src8+8, dstStride, src8Stride, h);\ | |
846 }\ | |
847 | |
2754 | 848 |
849 #define H264_MC(OPNAME, SIZE, MMX) \ | |
850 static void OPNAME ## h264_qpel ## SIZE ## _mc00_ ## MMX (uint8_t *dst, uint8_t *src, int stride){\ | |
851 OPNAME ## pixels ## SIZE ## _mmx(dst, src, stride, SIZE);\ | |
852 }\ | |
853 \ | |
854 static void OPNAME ## h264_qpel ## SIZE ## _mc10_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\ | |
855 uint64_t temp[SIZE*SIZE/8];\ | |
856 uint8_t * const half= (uint8_t*)temp;\ | |
857 put_h264_qpel ## SIZE ## _h_lowpass_ ## MMX(half, src, SIZE, stride);\ | |
858 OPNAME ## pixels ## SIZE ## _l2_ ## MMX(dst, src, half, stride, stride, SIZE);\ | |
859 }\ | |
860 \ | |
861 static void OPNAME ## h264_qpel ## SIZE ## _mc20_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\ | |
862 OPNAME ## h264_qpel ## SIZE ## _h_lowpass_ ## MMX(dst, src, stride, stride);\ | |
863 }\ | |
864 \ | |
865 static void OPNAME ## h264_qpel ## SIZE ## _mc30_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\ | |
866 uint64_t temp[SIZE*SIZE/8];\ | |
867 uint8_t * const half= (uint8_t*)temp;\ | |
868 put_h264_qpel ## SIZE ## _h_lowpass_ ## MMX(half, src, SIZE, stride);\ | |
869 OPNAME ## pixels ## SIZE ## _l2_ ## MMX(dst, src+1, half, stride, stride, SIZE);\ | |
870 }\ | |
871 \ | |
872 static void OPNAME ## h264_qpel ## SIZE ## _mc01_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\ | |
873 uint64_t temp[SIZE*SIZE/8];\ | |
874 uint8_t * const half= (uint8_t*)temp;\ | |
875 put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(half, src, SIZE, stride);\ | |
876 OPNAME ## pixels ## SIZE ## _l2_ ## MMX(dst, src, half, stride, stride, SIZE);\ | |
877 }\ | |
878 \ | |
879 static void OPNAME ## h264_qpel ## SIZE ## _mc02_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\ | |
880 OPNAME ## h264_qpel ## SIZE ## _v_lowpass_ ## MMX(dst, src, stride, stride);\ | |
881 }\ | |
882 \ | |
883 static void OPNAME ## h264_qpel ## SIZE ## _mc03_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\ | |
884 uint64_t temp[SIZE*SIZE/8];\ | |
885 uint8_t * const half= (uint8_t*)temp;\ | |
886 put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(half, src, SIZE, stride);\ | |
887 OPNAME ## pixels ## SIZE ## _l2_ ## MMX(dst, src+stride, half, stride, stride, SIZE);\ | |
888 }\ | |
889 \ | |
890 static void OPNAME ## h264_qpel ## SIZE ## _mc11_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\ | |
891 uint64_t temp[SIZE*SIZE/4];\ | |
892 uint8_t * const halfH= (uint8_t*)temp;\ | |
893 uint8_t * const halfV= ((uint8_t*)temp) + SIZE*SIZE;\ | |
894 put_h264_qpel ## SIZE ## _h_lowpass_ ## MMX(halfH, src, SIZE, stride);\ | |
895 put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(halfV, src, SIZE, stride);\ | |
896 OPNAME ## pixels ## SIZE ## _l2_ ## MMX(dst, halfH, halfV, stride, SIZE, SIZE);\ | |
897 }\ | |
898 \ | |
899 static void OPNAME ## h264_qpel ## SIZE ## _mc31_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\ | |
900 uint64_t temp[SIZE*SIZE/4];\ | |
901 uint8_t * const halfH= (uint8_t*)temp;\ | |
902 uint8_t * const halfV= ((uint8_t*)temp) + SIZE*SIZE;\ | |
903 put_h264_qpel ## SIZE ## _h_lowpass_ ## MMX(halfH, src, SIZE, stride);\ | |
904 put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(halfV, src+1, SIZE, stride);\ | |
905 OPNAME ## pixels ## SIZE ## _l2_ ## MMX(dst, halfH, halfV, stride, SIZE, SIZE);\ | |
906 }\ | |
907 \ | |
908 static void OPNAME ## h264_qpel ## SIZE ## _mc13_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\ | |
909 uint64_t temp[SIZE*SIZE/4];\ | |
910 uint8_t * const halfH= (uint8_t*)temp;\ | |
911 uint8_t * const halfV= ((uint8_t*)temp) + SIZE*SIZE;\ | |
912 put_h264_qpel ## SIZE ## _h_lowpass_ ## MMX(halfH, src + stride, SIZE, stride);\ | |
913 put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(halfV, src, SIZE, stride);\ | |
914 OPNAME ## pixels ## SIZE ## _l2_ ## MMX(dst, halfH, halfV, stride, SIZE, SIZE);\ | |
915 }\ | |
916 \ | |
917 static void OPNAME ## h264_qpel ## SIZE ## _mc33_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\ | |
918 uint64_t temp[SIZE*SIZE/4];\ | |
919 uint8_t * const halfH= (uint8_t*)temp;\ | |
920 uint8_t * const halfV= ((uint8_t*)temp) + SIZE*SIZE;\ | |
921 put_h264_qpel ## SIZE ## _h_lowpass_ ## MMX(halfH, src + stride, SIZE, stride);\ | |
922 put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(halfV, src+1, SIZE, stride);\ | |
923 OPNAME ## pixels ## SIZE ## _l2_ ## MMX(dst, halfH, halfV, stride, SIZE, SIZE);\ | |
924 }\ | |
925 \ | |
926 static void OPNAME ## h264_qpel ## SIZE ## _mc22_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\ | |
3093 | 927 uint64_t temp[SIZE*(SIZE<8?12:24)/4];\ |
2754 | 928 int16_t * const tmp= (int16_t*)temp;\ |
929 OPNAME ## h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(dst, tmp, src, stride, SIZE, stride);\ | |
930 }\ | |
931 \ | |
932 static void OPNAME ## h264_qpel ## SIZE ## _mc21_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\ | |
3093 | 933 uint64_t temp[SIZE*(SIZE<8?12:24)/4 + SIZE*SIZE/4];\ |
2754 | 934 uint8_t * const halfH= (uint8_t*)temp;\ |
935 uint8_t * const halfHV= ((uint8_t*)temp) + SIZE*SIZE;\ | |
936 int16_t * const tmp= ((int16_t*)temp) + SIZE*SIZE;\ | |
937 put_h264_qpel ## SIZE ## _h_lowpass_ ## MMX(halfH, src, SIZE, stride);\ | |
938 put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, tmp, src, SIZE, SIZE, stride);\ | |
939 OPNAME ## pixels ## SIZE ## _l2_ ## MMX(dst, halfH, halfHV, stride, SIZE, SIZE);\ | |
940 }\ | |
941 \ | |
942 static void OPNAME ## h264_qpel ## SIZE ## _mc23_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\ | |
3093 | 943 uint64_t temp[SIZE*(SIZE<8?12:24)/4 + SIZE*SIZE/4];\ |
2754 | 944 uint8_t * const halfH= (uint8_t*)temp;\ |
945 uint8_t * const halfHV= ((uint8_t*)temp) + SIZE*SIZE;\ | |
946 int16_t * const tmp= ((int16_t*)temp) + SIZE*SIZE;\ | |
947 put_h264_qpel ## SIZE ## _h_lowpass_ ## MMX(halfH, src + stride, SIZE, stride);\ | |
948 put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, tmp, src, SIZE, SIZE, stride);\ | |
949 OPNAME ## pixels ## SIZE ## _l2_ ## MMX(dst, halfH, halfHV, stride, SIZE, SIZE);\ | |
950 }\ | |
951 \ | |
952 static void OPNAME ## h264_qpel ## SIZE ## _mc12_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\ | |
3095 | 953 uint64_t temp[SIZE*(SIZE<8?12:24)/4 + SIZE*SIZE/8];\ |
954 int16_t * const halfV= ((int16_t*)temp) + SIZE*SIZE/2;\ | |
955 uint8_t * const halfHV= ((uint8_t*)temp);\ | |
956 put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, halfV, src, SIZE, SIZE, stride);\ | |
957 OPNAME ## pixels ## SIZE ## _l2_shift5_ ## MMX(dst, halfV+2, halfHV, stride, SIZE, SIZE);\ | |
2754 | 958 }\ |
959 \ | |
960 static void OPNAME ## h264_qpel ## SIZE ## _mc32_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\ | |
3095 | 961 uint64_t temp[SIZE*(SIZE<8?12:24)/4 + SIZE*SIZE/8];\ |
962 int16_t * const halfV= ((int16_t*)temp) + SIZE*SIZE/2;\ | |
963 uint8_t * const halfHV= ((uint8_t*)temp);\ | |
964 put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, halfV, src, SIZE, SIZE, stride);\ | |
965 OPNAME ## pixels ## SIZE ## _l2_shift5_ ## MMX(dst, halfV+3, halfHV, stride, SIZE, SIZE);\ | |
2754 | 966 }\ |
967 | |
968 | |
2979 | 969 #define PUT_OP(a,b,temp, size) "mov" #size " " #a ", " #b " \n\t" |
2754 | 970 #define AVG_3DNOW_OP(a,b,temp, size) \ |
2979 | 971 "mov" #size " " #b ", " #temp " \n\t"\ |
972 "pavgusb " #temp ", " #a " \n\t"\ | |
973 "mov" #size " " #a ", " #b " \n\t" | |
2754 | 974 #define AVG_MMX2_OP(a,b,temp, size) \ |
2979 | 975 "mov" #size " " #b ", " #temp " \n\t"\ |
976 "pavgb " #temp ", " #a " \n\t"\ | |
977 "mov" #size " " #a ", " #b " \n\t" | |
2754 | 978 |
3095 | 979 #define PAVGB "pavgusb" |
2754 | 980 QPEL_H264(put_, PUT_OP, 3dnow) |
981 QPEL_H264(avg_, AVG_3DNOW_OP, 3dnow) | |
3095 | 982 #undef PAVGB |
983 #define PAVGB "pavgb" | |
2754 | 984 QPEL_H264(put_, PUT_OP, mmx2) |
985 QPEL_H264(avg_, AVG_MMX2_OP, mmx2) | |
3095 | 986 #undef PAVGB |
2754 | 987 |
988 H264_MC(put_, 4, 3dnow) | |
989 H264_MC(put_, 8, 3dnow) | |
990 H264_MC(put_, 16,3dnow) | |
991 H264_MC(avg_, 4, 3dnow) | |
992 H264_MC(avg_, 8, 3dnow) | |
993 H264_MC(avg_, 16,3dnow) | |
994 H264_MC(put_, 4, mmx2) | |
995 H264_MC(put_, 8, mmx2) | |
996 H264_MC(put_, 16,mmx2) | |
997 H264_MC(avg_, 4, mmx2) | |
998 H264_MC(avg_, 8, mmx2) | |
999 H264_MC(avg_, 16,mmx2) | |
1000 | |
1001 | |
1002 #define H264_CHROMA_OP(S,D) | |
2922
d772011258ec
faster h264_chroma_mc8_mmx, added h264_chroma_mc4_mmx.
lorenm
parents:
2902
diff
changeset
|
1003 #define H264_CHROMA_OP4(S,D,T) |
2754 | 1004 #define H264_CHROMA_MC8_TMPL put_h264_chroma_mc8_mmx |
2922
d772011258ec
faster h264_chroma_mc8_mmx, added h264_chroma_mc4_mmx.
lorenm
parents:
2902
diff
changeset
|
1005 #define H264_CHROMA_MC4_TMPL put_h264_chroma_mc4_mmx |
d772011258ec
faster h264_chroma_mc8_mmx, added h264_chroma_mc4_mmx.
lorenm
parents:
2902
diff
changeset
|
1006 #define H264_CHROMA_MC8_MV0 put_pixels8_mmx |
2754 | 1007 #include "dsputil_h264_template_mmx.c" |
1008 #undef H264_CHROMA_OP | |
2922
d772011258ec
faster h264_chroma_mc8_mmx, added h264_chroma_mc4_mmx.
lorenm
parents:
2902
diff
changeset
|
1009 #undef H264_CHROMA_OP4 |
2754 | 1010 #undef H264_CHROMA_MC8_TMPL |
2922
d772011258ec
faster h264_chroma_mc8_mmx, added h264_chroma_mc4_mmx.
lorenm
parents:
2902
diff
changeset
|
1011 #undef H264_CHROMA_MC4_TMPL |
d772011258ec
faster h264_chroma_mc8_mmx, added h264_chroma_mc4_mmx.
lorenm
parents:
2902
diff
changeset
|
1012 #undef H264_CHROMA_MC8_MV0 |
2754 | 1013 |
1014 #define H264_CHROMA_OP(S,D) "pavgb " #S ", " #D " \n\t" | |
2922
d772011258ec
faster h264_chroma_mc8_mmx, added h264_chroma_mc4_mmx.
lorenm
parents:
2902
diff
changeset
|
1015 #define H264_CHROMA_OP4(S,D,T) "movd " #S ", " #T " \n\t"\ |
d772011258ec
faster h264_chroma_mc8_mmx, added h264_chroma_mc4_mmx.
lorenm
parents:
2902
diff
changeset
|
1016 "pavgb " #T ", " #D " \n\t" |
2754 | 1017 #define H264_CHROMA_MC8_TMPL avg_h264_chroma_mc8_mmx2 |
2922
d772011258ec
faster h264_chroma_mc8_mmx, added h264_chroma_mc4_mmx.
lorenm
parents:
2902
diff
changeset
|
1018 #define H264_CHROMA_MC4_TMPL avg_h264_chroma_mc4_mmx2 |
d772011258ec
faster h264_chroma_mc8_mmx, added h264_chroma_mc4_mmx.
lorenm
parents:
2902
diff
changeset
|
1019 #define H264_CHROMA_MC8_MV0 avg_pixels8_mmx2 |
2754 | 1020 #include "dsputil_h264_template_mmx.c" |
1021 #undef H264_CHROMA_OP | |
2922
d772011258ec
faster h264_chroma_mc8_mmx, added h264_chroma_mc4_mmx.
lorenm
parents:
2902
diff
changeset
|
1022 #undef H264_CHROMA_OP4 |
2754 | 1023 #undef H264_CHROMA_MC8_TMPL |
2922
d772011258ec
faster h264_chroma_mc8_mmx, added h264_chroma_mc4_mmx.
lorenm
parents:
2902
diff
changeset
|
1024 #undef H264_CHROMA_MC4_TMPL |
d772011258ec
faster h264_chroma_mc8_mmx, added h264_chroma_mc4_mmx.
lorenm
parents:
2902
diff
changeset
|
1025 #undef H264_CHROMA_MC8_MV0 |
2754 | 1026 |
1027 #define H264_CHROMA_OP(S,D) "pavgusb " #S ", " #D " \n\t" | |
2922
d772011258ec
faster h264_chroma_mc8_mmx, added h264_chroma_mc4_mmx.
lorenm
parents:
2902
diff
changeset
|
1028 #define H264_CHROMA_OP4(S,D,T) "movd " #S ", " #T " \n\t"\ |
d772011258ec
faster h264_chroma_mc8_mmx, added h264_chroma_mc4_mmx.
lorenm
parents:
2902
diff
changeset
|
1029 "pavgusb " #T ", " #D " \n\t" |
2754 | 1030 #define H264_CHROMA_MC8_TMPL avg_h264_chroma_mc8_3dnow |
2922
d772011258ec
faster h264_chroma_mc8_mmx, added h264_chroma_mc4_mmx.
lorenm
parents:
2902
diff
changeset
|
1031 #define H264_CHROMA_MC4_TMPL avg_h264_chroma_mc4_3dnow |
d772011258ec
faster h264_chroma_mc8_mmx, added h264_chroma_mc4_mmx.
lorenm
parents:
2902
diff
changeset
|
1032 #define H264_CHROMA_MC8_MV0 avg_pixels8_3dnow |
2754 | 1033 #include "dsputil_h264_template_mmx.c" |
1034 #undef H264_CHROMA_OP | |
2922
d772011258ec
faster h264_chroma_mc8_mmx, added h264_chroma_mc4_mmx.
lorenm
parents:
2902
diff
changeset
|
1035 #undef H264_CHROMA_OP4 |
2754 | 1036 #undef H264_CHROMA_MC8_TMPL |
2922
d772011258ec
faster h264_chroma_mc8_mmx, added h264_chroma_mc4_mmx.
lorenm
parents:
2902
diff
changeset
|
1037 #undef H264_CHROMA_MC4_TMPL |
d772011258ec
faster h264_chroma_mc8_mmx, added h264_chroma_mc4_mmx.
lorenm
parents:
2902
diff
changeset
|
1038 #undef H264_CHROMA_MC8_MV0 |
2754 | 1039 |
2902
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1040 /***********************************/ |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1041 /* weighted prediction */ |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1042 |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1043 static inline void ff_h264_weight_WxH_mmx2(uint8_t *dst, int stride, int log2_denom, int weight, int offset, int w, int h) |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1044 { |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1045 int x, y; |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1046 offset <<= log2_denom; |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1047 offset += (1 << log2_denom) >> 1; |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1048 asm volatile( |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1049 "movd %0, %%mm4 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1050 "movd %1, %%mm5 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1051 "movd %2, %%mm6 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1052 "pshufw $0, %%mm4, %%mm4 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1053 "pshufw $0, %%mm5, %%mm5 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1054 "pxor %%mm7, %%mm7 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1055 :: "g"(weight), "g"(offset), "g"(log2_denom) |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1056 ); |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1057 for(y=0; y<h; y+=2){ |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1058 for(x=0; x<w; x+=4){ |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1059 asm volatile( |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1060 "movd %0, %%mm0 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1061 "movd %1, %%mm1 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1062 "punpcklbw %%mm7, %%mm0 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1063 "punpcklbw %%mm7, %%mm1 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1064 "pmullw %%mm4, %%mm0 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1065 "pmullw %%mm4, %%mm1 \n\t" |
3001
b52d8ee430f6
fix some potential arithmetic overflows in pred_direct_motion() and
lorenm
parents:
2979
diff
changeset
|
1066 "paddsw %%mm5, %%mm0 \n\t" |
b52d8ee430f6
fix some potential arithmetic overflows in pred_direct_motion() and
lorenm
parents:
2979
diff
changeset
|
1067 "paddsw %%mm5, %%mm1 \n\t" |
2902
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1068 "psraw %%mm6, %%mm0 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1069 "psraw %%mm6, %%mm1 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1070 "packuswb %%mm7, %%mm0 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1071 "packuswb %%mm7, %%mm1 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1072 "movd %%mm0, %0 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1073 "movd %%mm1, %1 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1074 : "+m"(*(uint32_t*)(dst+x)), |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1075 "+m"(*(uint32_t*)(dst+x+stride)) |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1076 ); |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1077 } |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1078 dst += 2*stride; |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1079 } |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1080 } |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1081 |
3029 | 1082 static inline void ff_h264_biweight_WxH_mmx2(uint8_t *dst, uint8_t *src, int stride, int log2_denom, int weightd, int weights, int offset, int w, int h) |
2902
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1083 { |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1084 int x, y; |
3029 | 1085 offset = ((offset + 1) | 1) << log2_denom; |
2902
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1086 asm volatile( |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1087 "movd %0, %%mm3 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1088 "movd %1, %%mm4 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1089 "movd %2, %%mm5 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1090 "movd %3, %%mm6 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1091 "pshufw $0, %%mm3, %%mm3 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1092 "pshufw $0, %%mm4, %%mm4 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1093 "pshufw $0, %%mm5, %%mm5 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1094 "pxor %%mm7, %%mm7 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1095 :: "g"(weightd), "g"(weights), "g"(offset), "g"(log2_denom+1) |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1096 ); |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1097 for(y=0; y<h; y++){ |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1098 for(x=0; x<w; x+=4){ |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1099 asm volatile( |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1100 "movd %0, %%mm0 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1101 "movd %1, %%mm1 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1102 "punpcklbw %%mm7, %%mm0 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1103 "punpcklbw %%mm7, %%mm1 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1104 "pmullw %%mm3, %%mm0 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1105 "pmullw %%mm4, %%mm1 \n\t" |
3001
b52d8ee430f6
fix some potential arithmetic overflows in pred_direct_motion() and
lorenm
parents:
2979
diff
changeset
|
1106 "paddsw %%mm1, %%mm0 \n\t" |
b52d8ee430f6
fix some potential arithmetic overflows in pred_direct_motion() and
lorenm
parents:
2979
diff
changeset
|
1107 "paddsw %%mm5, %%mm0 \n\t" |
2902
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1108 "psraw %%mm6, %%mm0 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1109 "packuswb %%mm0, %%mm0 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1110 "movd %%mm0, %0 \n\t" |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1111 : "+m"(*(uint32_t*)(dst+x)) |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1112 : "m"(*(uint32_t*)(src+x)) |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1113 ); |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1114 } |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1115 src += stride; |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1116 dst += stride; |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1117 } |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1118 } |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1119 |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1120 #define H264_WEIGHT(W,H) \ |
3029 | 1121 static void ff_h264_biweight_ ## W ## x ## H ## _mmx2(uint8_t *dst, uint8_t *src, int stride, int log2_denom, int weightd, int weights, int offset){ \ |
1122 ff_h264_biweight_WxH_mmx2(dst, src, stride, log2_denom, weightd, weights, offset, W, H); \ | |
2902
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1123 } \ |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1124 static void ff_h264_weight_ ## W ## x ## H ## _mmx2(uint8_t *dst, int stride, int log2_denom, int weight, int offset){ \ |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1125 ff_h264_weight_WxH_mmx2(dst, stride, log2_denom, weight, offset, W, H); \ |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1126 } |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1127 |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1128 H264_WEIGHT(16,16) |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1129 H264_WEIGHT(16, 8) |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1130 H264_WEIGHT( 8,16) |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1131 H264_WEIGHT( 8, 8) |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1132 H264_WEIGHT( 8, 4) |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1133 H264_WEIGHT( 4, 8) |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1134 H264_WEIGHT( 4, 4) |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1135 H264_WEIGHT( 4, 2) |
3c79bc9f3aa9
h264 mmx weighted prediction. up to 3% overall speedup.
lorenm
parents:
2855
diff
changeset
|
1136 |