|
15631
|
1 /*
|
|
|
2 Copyright (C) 2003 Michael Niedermayer <michaelni@gmx.at>
|
|
|
3 Copyright (C) 2005 Nikolaj Poroshin <porosh3@psu.ru>
|
|
|
4
|
|
|
5 This program is free software; you can redistribute it and/or modify
|
|
|
6 it under the terms of the GNU General Public License as published by
|
|
|
7 the Free Software Foundation; either version 2 of the License, or
|
|
|
8 (at your option) any later version.
|
|
|
9
|
|
|
10 This program is distributed in the hope that it will be useful,
|
|
|
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
13 GNU General Public License for more details.
|
|
|
14
|
|
|
15 You should have received a copy of the GNU General Public License
|
|
|
16 along with this program; if not, write to the Free Software
|
|
17367
|
17 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
|
|
15631
|
18 */
|
|
|
19
|
|
|
20 /*
|
|
|
21 * This implementation is based on an algorithm described in
|
|
|
22 * "Aria Nosratinia Embedded Post-Processing for
|
|
|
23 * Enhancement of Compressed Images (1999)"
|
|
|
24 * (http://citeseer.nj.nec.com/nosratinia99embedded.html)
|
|
|
25 * Futher, with splitting (i)dct into hor/ver passes, one of them can be
|
|
|
26 * performed once per block, not pixel. This allows for much better speed.
|
|
|
27 */
|
|
|
28
|
|
|
29 /*
|
|
|
30 Heavily optimized version of SPP filter by Nikolaj
|
|
|
31 */
|
|
|
32
|
|
|
33 #include <stdio.h>
|
|
|
34 #include <stdlib.h>
|
|
|
35 #include <string.h>
|
|
|
36 #include <inttypes.h>
|
|
|
37 #include <math.h>
|
|
|
38
|
|
17012
|
39 #include "config.h"
|
|
15631
|
40
|
|
17012
|
41 #include "mp_msg.h"
|
|
|
42 #include "cpudetect.h"
|
|
15631
|
43
|
|
|
44 #ifdef USE_LIBAVCODEC_SO
|
|
|
45 #include <ffmpeg/avcodec.h>
|
|
|
46 #include <ffmpeg/dsputil.h>
|
|
|
47 #else
|
|
17012
|
48 #include "libavcodec/avcodec.h"
|
|
|
49 #include "libavcodec/dsputil.h"
|
|
15631
|
50 #endif
|
|
|
51
|
|
|
52 #ifdef HAVE_MALLOC_H
|
|
|
53 #include <malloc.h>
|
|
|
54 #endif
|
|
|
55
|
|
|
56 #include "img_format.h"
|
|
|
57 #include "mp_image.h"
|
|
|
58 #include "vf.h"
|
|
17012
|
59 #include "libvo/fastmemcpy.h"
|
|
15631
|
60
|
|
|
61 //===========================================================================//
|
|
|
62 #define BLOCKSZ 12
|
|
|
63
|
|
|
64 static const short custom_threshold[64]=
|
|
|
65 // values (296) can't be too high
|
|
|
66 // -it causes too big quant dependence
|
|
|
67 // or maybe overflow(check), which results in some flashing
|
|
|
68 { 71, 296, 295, 237, 71, 40, 38, 19,
|
|
|
69 245, 193, 185, 121, 102, 73, 53, 27,
|
|
|
70 158, 129, 141, 107, 97, 73, 50, 26,
|
|
|
71 102, 116, 109, 98, 82, 66, 45, 23,
|
|
|
72 71, 94, 95, 81, 70, 56, 38, 20,
|
|
|
73 56, 77, 74, 66, 56, 44, 30, 15,
|
|
|
74 38, 53, 50, 45, 38, 30, 21, 11,
|
|
|
75 20, 27, 26, 23, 20, 15, 11, 5
|
|
|
76 };
|
|
|
77
|
|
|
78 static const uint8_t __attribute__((aligned(32))) dither[8][8]={
|
|
|
79 { 0, 48, 12, 60, 3, 51, 15, 63, },
|
|
|
80 { 32, 16, 44, 28, 35, 19, 47, 31, },
|
|
|
81 { 8, 56, 4, 52, 11, 59, 7, 55, },
|
|
|
82 { 40, 24, 36, 20, 43, 27, 39, 23, },
|
|
|
83 { 2, 50, 14, 62, 1, 49, 13, 61, },
|
|
|
84 { 34, 18, 46, 30, 33, 17, 45, 29, },
|
|
|
85 { 10, 58, 6, 54, 9, 57, 5, 53, },
|
|
|
86 { 42, 26, 38, 22, 41, 25, 37, 21, },
|
|
|
87 };
|
|
|
88
|
|
|
89 struct vf_priv_s { //align 16 !
|
|
|
90 uint64_t threshold_mtx_noq[8*2];
|
|
|
91 uint64_t threshold_mtx[8*2];//used in both C & MMX (& later SSE2) versions
|
|
|
92
|
|
|
93 int log2_count;
|
|
|
94 int temp_stride;
|
|
|
95 int qp;
|
|
17225
|
96 int mpeg2;
|
|
15631
|
97 int prev_q;
|
|
|
98 uint8_t *src;
|
|
|
99 int16_t *temp;
|
|
17225
|
100 int bframes;
|
|
17133
|
101 char *non_b_qp;
|
|
15631
|
102 };
|
|
|
103
|
|
|
104
|
|
|
105 #ifndef HAVE_MMX
|
|
|
106
|
|
|
107 //This func reads from 1 slice, 1 and clears 0 & 1
|
|
|
108 static void store_slice_c(uint8_t *dst, int16_t *src, int dst_stride, int src_stride, int width, int height, int log2_scale)
|
|
|
109 {int y, x;
|
|
|
110 #define STORE(pos) \
|
|
|
111 temp= (src[x + pos] + (d[pos]>>log2_scale))>>(6-log2_scale); \
|
|
|
112 src[x + pos]=src[x + pos - 8*src_stride]=0; \
|
|
|
113 if(temp & 0x100) temp= ~(temp>>31); \
|
|
|
114 dst[x + pos]= temp;
|
|
|
115
|
|
|
116 for(y=0; y<height; y++){
|
|
|
117 const uint8_t *d= dither[y];
|
|
|
118 for(x=0; x<width; x+=8){
|
|
|
119 int temp;
|
|
|
120 STORE(0);
|
|
|
121 STORE(1);
|
|
|
122 STORE(2);
|
|
|
123 STORE(3);
|
|
|
124 STORE(4);
|
|
|
125 STORE(5);
|
|
|
126 STORE(6);
|
|
|
127 STORE(7);
|
|
|
128 }
|
|
|
129 src+=src_stride;
|
|
|
130 dst+=dst_stride;
|
|
|
131 }
|
|
|
132 }
|
|
|
133
|
|
|
134 //This func reads from 2 slices, 0 & 2 and clears 2-nd
|
|
|
135 static void store_slice2_c(uint8_t *dst, int16_t *src, int dst_stride, int src_stride, int width, int height, int log2_scale)
|
|
|
136 {int y, x;
|
|
|
137 #define STORE2(pos) \
|
|
|
138 temp= (src[x + pos] + src[x + pos + 16*src_stride] + (d[pos]>>log2_scale))>>(6-log2_scale); \
|
|
|
139 src[x + pos + 16*src_stride]=0; \
|
|
|
140 if(temp & 0x100) temp= ~(temp>>31); \
|
|
|
141 dst[x + pos]= temp;
|
|
|
142
|
|
|
143 for(y=0; y<height; y++){
|
|
|
144 const uint8_t *d= dither[y];
|
|
|
145 for(x=0; x<width; x+=8){
|
|
|
146 int temp;
|
|
|
147 STORE2(0);
|
|
|
148 STORE2(1);
|
|
|
149 STORE2(2);
|
|
|
150 STORE2(3);
|
|
|
151 STORE2(4);
|
|
|
152 STORE2(5);
|
|
|
153 STORE2(6);
|
|
|
154 STORE2(7);
|
|
|
155 }
|
|
|
156 src+=src_stride;
|
|
|
157 dst+=dst_stride;
|
|
|
158 }
|
|
|
159 }
|
|
|
160
|
|
|
161 static void mul_thrmat_c(struct vf_priv_s *p,int q)
|
|
|
162 {
|
|
|
163 int a;
|
|
|
164 for(a=0;a<64;a++)
|
|
|
165 ((short*)p->threshold_mtx)[a]=q * ((short*)p->threshold_mtx_noq)[a];//ints faster in C
|
|
|
166 }
|
|
|
167
|
|
|
168 static void column_fidct_c(int16_t* thr_adr, DCTELEM *data, DCTELEM *output, int cnt);
|
|
|
169 static void row_idct_c(DCTELEM* workspace,
|
|
|
170 int16_t* output_adr, int output_stride, int cnt);
|
|
|
171 static void row_fdct_c(DCTELEM *data, const uint8_t *pixels, int line_size, int cnt);
|
|
|
172
|
|
|
173 //this is rather ugly, but there is no need for function pointers
|
|
|
174 #define store_slice_s store_slice_c
|
|
|
175 #define store_slice2_s store_slice2_c
|
|
|
176 #define mul_thrmat_s mul_thrmat_c
|
|
|
177 #define column_fidct_s column_fidct_c
|
|
|
178 #define row_idct_s row_idct_c
|
|
|
179 #define row_fdct_s row_fdct_c
|
|
|
180
|
|
|
181 #else /* HAVE_MMX */
|
|
|
182
|
|
|
183 //This func reads from 1 slice, 1 and clears 0 & 1
|
|
15634
|
184 static void store_slice_mmx(uint8_t *dst, int16_t *src, long dst_stride, long src_stride, long width, long height, long log2_scale)
|
|
15631
|
185 {
|
|
|
186 const uint8_t *od=&dither[0][0];
|
|
15634
|
187 const uint8_t *end=&dither[height][0];
|
|
15631
|
188 width = (width+7)&~7;
|
|
|
189 dst_stride-=width;
|
|
|
190 //src_stride=(src_stride-width)*2;
|
|
|
191 asm volatile(
|
|
15634
|
192 "mov %5, %%"REG_d" \n\t"
|
|
|
193 "mov %6, %%"REG_S" \n\t"
|
|
|
194 "mov %7, %%"REG_D" \n\t"
|
|
|
195 "mov %1, %%"REG_a" \n\t"
|
|
|
196 "movd %%"REG_d", %%mm5 \n\t"
|
|
|
197 "xor $-1, %%"REG_d" \n\t"
|
|
|
198 "mov %%"REG_a", %%"REG_c" \n\t"
|
|
|
199 "add $7, %%"REG_d" \n\t"
|
|
|
200 "neg %%"REG_a" \n\t"
|
|
|
201 "sub %0, %%"REG_c" \n\t"
|
|
|
202 "add %%"REG_c", %%"REG_c" \n\t"
|
|
|
203 "movd %%"REG_d", %%mm2 \n\t"
|
|
|
204 "mov %%"REG_c", %1 \n\t"
|
|
|
205 "mov %2, %%"REG_d" \n\t"
|
|
|
206 "shl $4, %%"REG_a" \n\t"
|
|
15631
|
207
|
|
|
208 "2: \n\t"
|
|
15634
|
209 "movq (%%"REG_d"), %%mm3 \n\t"
|
|
15631
|
210 "movq %%mm3, %%mm4 \n\t"
|
|
|
211 "pxor %%mm7, %%mm7 \n\t"
|
|
|
212 "punpcklbw %%mm7, %%mm3 \n\t"
|
|
|
213 "punpckhbw %%mm7, %%mm4 \n\t"
|
|
15634
|
214 "mov %0, %%"REG_c" \n\t"
|
|
15631
|
215 "psraw %%mm5, %%mm3 \n\t"
|
|
|
216 "psraw %%mm5, %%mm4 \n\t"
|
|
|
217 "1: \n\t"
|
|
15634
|
218 "movq %%mm7, (%%"REG_S",%%"REG_a",) \n\t"
|
|
|
219 "movq (%%"REG_S"), %%mm0 \n\t"
|
|
|
220 "movq 8(%%"REG_S"), %%mm1 \n\t"
|
|
|
221
|
|
|
222 "movq %%mm7, 8(%%"REG_S",%%"REG_a",) \n\t"
|
|
15631
|
223 "paddw %%mm3, %%mm0 \n\t"
|
|
|
224 "paddw %%mm4, %%mm1 \n\t"
|
|
|
225
|
|
15634
|
226 "movq %%mm7, (%%"REG_S") \n\t"
|
|
15631
|
227 "psraw %%mm2, %%mm0 \n\t"
|
|
|
228 "psraw %%mm2, %%mm1 \n\t"
|
|
|
229
|
|
15634
|
230 "movq %%mm7, 8(%%"REG_S") \n\t"
|
|
15631
|
231 "packuswb %%mm1, %%mm0 \n\t"
|
|
15634
|
232 "add $16, %%"REG_S" \n\t"
|
|
|
233
|
|
|
234 "movq %%mm0, (%%"REG_D") \n\t"
|
|
|
235 "add $8, %%"REG_D" \n\t"
|
|
|
236 "sub $8, %%"REG_c" \n\t"
|
|
15631
|
237 "jg 1b \n\t"
|
|
15634
|
238 "add %1, %%"REG_S" \n\t"
|
|
|
239 "add $8, %%"REG_d" \n\t"
|
|
|
240 "add %3, %%"REG_D" \n\t"
|
|
|
241 "cmp %4, %%"REG_d" \n\t"
|
|
15631
|
242 "jl 2b \n\t"
|
|
|
243
|
|
|
244 :
|
|
15634
|
245 : "m" (width), "m" (src_stride), "g" (od), "m" (dst_stride), "g" (end),
|
|
15631
|
246 "m" (log2_scale), "m" (src), "m" (dst) //input
|
|
15634
|
247 : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S, "%"REG_D
|
|
15631
|
248 );
|
|
|
249 }
|
|
|
250
|
|
|
251 //This func reads from 2 slices, 0 & 2 and clears 2-nd
|
|
15634
|
252 static void store_slice2_mmx(uint8_t *dst, int16_t *src, long dst_stride, long src_stride, long width, long height, long log2_scale)
|
|
15631
|
253 {
|
|
|
254 const uint8_t *od=&dither[0][0];
|
|
15634
|
255 const uint8_t *end=&dither[height][0];
|
|
15631
|
256 width = (width+7)&~7;
|
|
|
257 dst_stride-=width;
|
|
|
258 //src_stride=(src_stride-width)*2;
|
|
|
259 asm volatile(
|
|
15634
|
260 "mov %5, %%"REG_d" \n\t"
|
|
|
261 "mov %6, %%"REG_S" \n\t"
|
|
|
262 "mov %7, %%"REG_D" \n\t"
|
|
|
263 "mov %1, %%"REG_a" \n\t"
|
|
|
264 "movd %%"REG_d", %%mm5 \n\t"
|
|
|
265 "xor $-1, %%"REG_d" \n\t"
|
|
|
266 "mov %%"REG_a", %%"REG_c" \n\t"
|
|
|
267 "add $7, %%"REG_d" \n\t"
|
|
|
268 "sub %0, %%"REG_c" \n\t"
|
|
|
269 "add %%"REG_c", %%"REG_c" \n\t"
|
|
|
270 "movd %%"REG_d", %%mm2 \n\t"
|
|
|
271 "mov %%"REG_c", %1 \n\t"
|
|
|
272 "mov %2, %%"REG_d" \n\t"
|
|
|
273 "shl $5, %%"REG_a" \n\t"
|
|
15631
|
274
|
|
|
275 "2: \n\t"
|
|
15634
|
276 "movq (%%"REG_d"), %%mm3 \n\t"
|
|
15631
|
277 "movq %%mm3, %%mm4 \n\t"
|
|
|
278 "pxor %%mm7, %%mm7 \n\t"
|
|
|
279 "punpcklbw %%mm7, %%mm3 \n\t"
|
|
|
280 "punpckhbw %%mm7, %%mm4 \n\t"
|
|
15634
|
281 "mov %0, %%"REG_c" \n\t"
|
|
15631
|
282 "psraw %%mm5, %%mm3 \n\t"
|
|
|
283 "psraw %%mm5, %%mm4 \n\t"
|
|
|
284 "1: \n\t"
|
|
15634
|
285 "movq (%%"REG_S"), %%mm0 \n\t"
|
|
|
286 "movq 8(%%"REG_S"), %%mm1 \n\t"
|
|
15631
|
287 "paddw %%mm3, %%mm0 \n\t"
|
|
|
288
|
|
15634
|
289 "paddw (%%"REG_S",%%"REG_a",), %%mm0 \n\t"
|
|
15631
|
290 "paddw %%mm4, %%mm1 \n\t"
|
|
15634
|
291 "movq 8(%%"REG_S",%%"REG_a",), %%mm6 \n\t"
|
|
|
292
|
|
|
293 "movq %%mm7, (%%"REG_S",%%"REG_a",) \n\t"
|
|
15631
|
294 "psraw %%mm2, %%mm0 \n\t"
|
|
|
295 "paddw %%mm6, %%mm1 \n\t"
|
|
|
296
|
|
15634
|
297 "movq %%mm7, 8(%%"REG_S",%%"REG_a",) \n\t"
|
|
15631
|
298 "psraw %%mm2, %%mm1 \n\t"
|
|
|
299 "packuswb %%mm1, %%mm0 \n\t"
|
|
|
300
|
|
15634
|
301 "movq %%mm0, (%%"REG_D") \n\t"
|
|
|
302 "add $16, %%"REG_S" \n\t"
|
|
|
303 "add $8, %%"REG_D" \n\t"
|
|
|
304 "sub $8, %%"REG_c" \n\t"
|
|
15631
|
305 "jg 1b \n\t"
|
|
15634
|
306 "add %1, %%"REG_S" \n\t"
|
|
|
307 "add $8, %%"REG_d" \n\t"
|
|
|
308 "add %3, %%"REG_D" \n\t"
|
|
|
309 "cmp %4, %%"REG_d" \n\t"
|
|
15631
|
310 "jl 2b \n\t"
|
|
|
311
|
|
|
312 :
|
|
15634
|
313 : "m" (width), "m" (src_stride), "g" (od), "m" (dst_stride), "g" (end),
|
|
15631
|
314 "m" (log2_scale), "m" (src), "m" (dst) //input
|
|
15634
|
315 : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_D, "%"REG_S
|
|
15631
|
316 );
|
|
|
317 }
|
|
|
318
|
|
|
319 static void mul_thrmat_mmx(struct vf_priv_s *p, int q)
|
|
|
320 {
|
|
15634
|
321 uint64_t *adr=&p->threshold_mtx_noq[0];
|
|
15631
|
322 asm volatile(
|
|
|
323 "movd %0, %%mm7 \n\t"
|
|
15634
|
324 "add $8*8*2, %%"REG_D" \n\t"
|
|
|
325 "movq 0*8(%%"REG_S"), %%mm0 \n\t"
|
|
15631
|
326 "punpcklwd %%mm7, %%mm7 \n\t"
|
|
15634
|
327 "movq 1*8(%%"REG_S"), %%mm1 \n\t"
|
|
15631
|
328 "punpckldq %%mm7, %%mm7 \n\t"
|
|
|
329 "pmullw %%mm7, %%mm0 \n\t"
|
|
|
330
|
|
15634
|
331 "movq 2*8(%%"REG_S"), %%mm2 \n\t"
|
|
15631
|
332 "pmullw %%mm7, %%mm1 \n\t"
|
|
|
333
|
|
15634
|
334 "movq 3*8(%%"REG_S"), %%mm3 \n\t"
|
|
15631
|
335 "pmullw %%mm7, %%mm2 \n\t"
|
|
|
336
|
|
15634
|
337 "movq %%mm0, 0*8(%%"REG_D") \n\t"
|
|
|
338 "movq 4*8(%%"REG_S"), %%mm4 \n\t"
|
|
15631
|
339 "pmullw %%mm7, %%mm3 \n\t"
|
|
|
340
|
|
15634
|
341 "movq %%mm1, 1*8(%%"REG_D") \n\t"
|
|
|
342 "movq 5*8(%%"REG_S"), %%mm5 \n\t"
|
|
15631
|
343 "pmullw %%mm7, %%mm4 \n\t"
|
|
|
344
|
|
15634
|
345 "movq %%mm2, 2*8(%%"REG_D") \n\t"
|
|
|
346 "movq 6*8(%%"REG_S"), %%mm6 \n\t"
|
|
15631
|
347 "pmullw %%mm7, %%mm5 \n\t"
|
|
|
348
|
|
15634
|
349 "movq %%mm3, 3*8(%%"REG_D") \n\t"
|
|
|
350 "movq 7*8+0*8(%%"REG_S"), %%mm0 \n\t"
|
|
15631
|
351 "pmullw %%mm7, %%mm6 \n\t"
|
|
|
352
|
|
15634
|
353 "movq %%mm4, 4*8(%%"REG_D") \n\t"
|
|
|
354 "movq 7*8+1*8(%%"REG_S"), %%mm1 \n\t"
|
|
15631
|
355 "pmullw %%mm7, %%mm0 \n\t"
|
|
|
356
|
|
15634
|
357 "movq %%mm5, 5*8(%%"REG_D") \n\t"
|
|
|
358 "movq 7*8+2*8(%%"REG_S"), %%mm2 \n\t"
|
|
15631
|
359 "pmullw %%mm7, %%mm1 \n\t"
|
|
|
360
|
|
15634
|
361 "movq %%mm6, 6*8(%%"REG_D") \n\t"
|
|
|
362 "movq 7*8+3*8(%%"REG_S"), %%mm3 \n\t"
|
|
15631
|
363 "pmullw %%mm7, %%mm2 \n\t"
|
|
|
364
|
|
15634
|
365 "movq %%mm0, 7*8+0*8(%%"REG_D") \n\t"
|
|
|
366 "movq 7*8+4*8(%%"REG_S"), %%mm4 \n\t"
|
|
15631
|
367 "pmullw %%mm7, %%mm3 \n\t"
|
|
|
368
|
|
15634
|
369 "movq %%mm1, 7*8+1*8(%%"REG_D") \n\t"
|
|
|
370 "movq 7*8+5*8(%%"REG_S"), %%mm5 \n\t"
|
|
15631
|
371 "pmullw %%mm7, %%mm4 \n\t"
|
|
|
372
|
|
15634
|
373 "movq %%mm2, 7*8+2*8(%%"REG_D") \n\t"
|
|
|
374 "movq 7*8+6*8(%%"REG_S"), %%mm6 \n\t"
|
|
15631
|
375 "pmullw %%mm7, %%mm5 \n\t"
|
|
|
376
|
|
15634
|
377 "movq %%mm3, 7*8+3*8(%%"REG_D") \n\t"
|
|
|
378 "movq 14*8+0*8(%%"REG_S"), %%mm0 \n\t"
|
|
15631
|
379 "pmullw %%mm7, %%mm6 \n\t"
|
|
|
380
|
|
15634
|
381 "movq %%mm4, 7*8+4*8(%%"REG_D") \n\t"
|
|
|
382 "movq 14*8+1*8(%%"REG_S"), %%mm1 \n\t"
|
|
15631
|
383 "pmullw %%mm7, %%mm0 \n\t"
|
|
|
384
|
|
15634
|
385 "movq %%mm5, 7*8+5*8(%%"REG_D") \n\t"
|
|
15631
|
386 "pmullw %%mm7, %%mm1 \n\t"
|
|
|
387
|
|
15634
|
388 "movq %%mm6, 7*8+6*8(%%"REG_D") \n\t"
|
|
|
389 "movq %%mm0, 14*8+0*8(%%"REG_D") \n\t"
|
|
|
390 "movq %%mm1, 14*8+1*8(%%"REG_D") \n\t"
|
|
15631
|
391
|
|
|
392 : "+g" (q), "+S" (adr), "+D" (adr)
|
|
|
393 :
|
|
|
394 );
|
|
|
395 }
|
|
|
396
|
|
|
397 static void column_fidct_mmx(int16_t* thr_adr, DCTELEM *data, DCTELEM *output, int cnt);
|
|
|
398 static void row_idct_mmx(DCTELEM* workspace,
|
|
|
399 int16_t* output_adr, int output_stride, int cnt);
|
|
|
400 static void row_fdct_mmx(DCTELEM *data, const uint8_t *pixels, int line_size, int cnt);
|
|
|
401
|
|
|
402 #define store_slice_s store_slice_mmx
|
|
|
403 #define store_slice2_s store_slice2_mmx
|
|
|
404 #define mul_thrmat_s mul_thrmat_mmx
|
|
|
405 #define column_fidct_s column_fidct_mmx
|
|
|
406 #define row_idct_s row_idct_mmx
|
|
|
407 #define row_fdct_s row_fdct_mmx
|
|
|
408 #endif // HAVE_MMX
|
|
|
409
|
|
|
410 static void filter(struct vf_priv_s *p, uint8_t *dst, uint8_t *src,
|
|
|
411 int dst_stride, int src_stride,
|
|
|
412 int width, int height,
|
|
|
413 uint8_t *qp_store, int qp_stride, int is_luma)
|
|
|
414 {
|
|
|
415 int x, x0, y, es, qy, t;
|
|
|
416 const int stride= is_luma ? p->temp_stride : (width+16);//((width+16+15)&(~15))
|
|
|
417 const int step=6-p->log2_count;
|
|
|
418 const int qps= 3 + is_luma;
|
|
15634
|
419 int32_t __attribute__((aligned(32))) block_align[4*8*BLOCKSZ+ 4*8*BLOCKSZ];
|
|
15631
|
420 DCTELEM *block= (DCTELEM *)block_align;
|
|
|
421 DCTELEM *block3=(DCTELEM *)(block_align+4*8*BLOCKSZ);
|
|
|
422
|
|
|
423 memset(block3, 0, 4*8*BLOCKSZ);
|
|
|
424
|
|
|
425 //p->src=src-src_stride*8-8;//!
|
|
|
426 if (!src || !dst) return; // HACK avoid crash for Y8 colourspace
|
|
|
427 for(y=0; y<height; y++){
|
|
|
428 int index= 8 + 8*stride + y*stride;
|
|
|
429 memcpy(p->src + index, src + y*src_stride, width);//this line can be avoided by using DR & user fr.buffers
|
|
|
430 for(x=0; x<8; x++){
|
|
|
431 p->src[index - x - 1]= p->src[index + x ];
|
|
|
432 p->src[index + width + x ]= p->src[index + width - x - 1];
|
|
|
433 }
|
|
|
434 }
|
|
|
435 for(y=0; y<8; y++){
|
|
|
436 memcpy(p->src + ( 7-y)*stride, p->src + ( y+8)*stride, stride);
|
|
|
437 memcpy(p->src + (height+8+y)*stride, p->src + (height-y+7)*stride, stride);
|
|
|
438 }
|
|
|
439 //FIXME (try edge emu)
|
|
|
440
|
|
|
441 for(y=8; y<24; y++)
|
|
|
442 memset(p->temp+ 8 +y*stride, 0,width*sizeof(int16_t));
|
|
|
443
|
|
|
444 for(y=step; y<height+8; y+=step){ //step= 1,2
|
|
|
445 qy=y-4;
|
|
|
446 if (qy>height-1) qy=height-1;
|
|
|
447 if (qy<0) qy=0;
|
|
|
448 qy=(qy>>qps)*qp_stride;
|
|
|
449 row_fdct_s(block, p->src + y*stride +2-(y&1), stride, 2);
|
|
|
450 for(x0=0; x0<width+8-8*(BLOCKSZ-1); x0+=8*(BLOCKSZ-1)){
|
|
|
451 row_fdct_s(block+8*8, p->src + y*stride+8+x0 +2-(y&1), stride, 2*(BLOCKSZ-1));
|
|
|
452 if(p->qp)
|
|
|
453 column_fidct_s((int16_t*)(&p->threshold_mtx[0]), block+0*8, block3+0*8, 8*(BLOCKSZ-1)); //yes, this is a HOTSPOT
|
|
|
454 else
|
|
|
455 for (x=0; x<8*(BLOCKSZ-1); x+=8) {
|
|
|
456 t=x+x0-2; //correct t=x+x0-2-(y&1), but its the same
|
|
|
457 if (t<0) t=0;//t always < width-2
|
|
|
458 t=qp_store[qy+(t>>qps)];
|
|
17225
|
459 if(p->mpeg2) t>>=1; //copy p->mpeg2,prev_q to locals?
|
|
15631
|
460 if (t!=p->prev_q) p->prev_q=t, mul_thrmat_s(p, t);
|
|
|
461 column_fidct_s((int16_t*)(&p->threshold_mtx[0]), block+x*8, block3+x*8, 8); //yes, this is a HOTSPOT
|
|
|
462 }
|
|
|
463 row_idct_s(block3+0*8, p->temp + (y&15)*stride+x0+2-(y&1), stride, 2*(BLOCKSZ-1));
|
|
|
464 memcpy(block, block+(BLOCKSZ-1)*64, 8*8*sizeof(DCTELEM)); //cycling
|
|
|
465 memcpy(block3, block3+(BLOCKSZ-1)*64, 6*8*sizeof(DCTELEM));
|
|
|
466 }
|
|
|
467 //
|
|
|
468 es=width+8-x0; // 8, ...
|
|
|
469 if (es>8)
|
|
|
470 row_fdct_s(block+8*8, p->src + y*stride+8+x0 +2-(y&1), stride, (es-4)>>2);
|
|
|
471 column_fidct_s((int16_t*)(&p->threshold_mtx[0]), block, block3, es-0);
|
|
|
472 row_idct_s(block3+0*8, p->temp + (y&15)*stride+x0+2-(y&1), stride, es>>2);
|
|
|
473 {const int y1=y-8+step;//l5-7 l4-6
|
|
|
474 if (!(y1&7) && y1) {
|
|
|
475 if (y1&8) store_slice_s(dst + (y1-8)*dst_stride, p->temp+ 8 +8*stride,
|
|
|
476 dst_stride, stride, width, 8, 5-p->log2_count);
|
|
|
477 else store_slice2_s(dst + (y1-8)*dst_stride, p->temp+ 8 +0*stride,
|
|
|
478 dst_stride, stride, width, 8, 5-p->log2_count);
|
|
|
479 } }
|
|
|
480 }
|
|
|
481
|
|
|
482 if (y&7) { // == height & 7
|
|
|
483 if (y&8) store_slice_s(dst + ((y-8)&~7)*dst_stride, p->temp+ 8 +8*stride,
|
|
|
484 dst_stride, stride, width, y&7, 5-p->log2_count);
|
|
|
485 else store_slice2_s(dst + ((y-8)&~7)*dst_stride, p->temp+ 8 +0*stride,
|
|
|
486 dst_stride, stride, width, y&7, 5-p->log2_count);
|
|
|
487 }
|
|
|
488 }
|
|
|
489
|
|
|
490 static int config(struct vf_instance_s* vf,
|
|
|
491 int width, int height, int d_width, int d_height,
|
|
|
492 unsigned int flags, unsigned int outfmt)
|
|
|
493 {
|
|
|
494 int h= (height+16+15)&(~15);
|
|
|
495
|
|
|
496 vf->priv->temp_stride= (width+16+15)&(~15);
|
|
|
497 vf->priv->temp= (int16_t*)av_mallocz(vf->priv->temp_stride*3*8*sizeof(int16_t));
|
|
|
498 //this can also be avoided, see above
|
|
|
499 vf->priv->src = (uint8_t*)av_malloc(vf->priv->temp_stride*h*sizeof(uint8_t));
|
|
|
500
|
|
|
501 return vf_next_config(vf,width,height,d_width,d_height,flags,outfmt);
|
|
|
502 }
|
|
|
503
|
|
|
504 static void get_image(struct vf_instance_s* vf, mp_image_t *mpi)
|
|
|
505 {
|
|
|
506 if(mpi->flags&MP_IMGFLAG_PRESERVE) return; // don't change
|
|
|
507 // ok, we can do pp in-place (or pp disabled):
|
|
|
508 vf->dmpi=vf_get_image(vf->next,mpi->imgfmt,
|
|
16018
|
509 mpi->type, mpi->flags, mpi->width, mpi->height);
|
|
15631
|
510 mpi->planes[0]=vf->dmpi->planes[0];
|
|
|
511 mpi->stride[0]=vf->dmpi->stride[0];
|
|
|
512 mpi->width=vf->dmpi->width;
|
|
|
513 if(mpi->flags&MP_IMGFLAG_PLANAR){
|
|
|
514 mpi->planes[1]=vf->dmpi->planes[1];
|
|
|
515 mpi->planes[2]=vf->dmpi->planes[2];
|
|
|
516 mpi->stride[1]=vf->dmpi->stride[1];
|
|
|
517 mpi->stride[2]=vf->dmpi->stride[2];
|
|
|
518 }
|
|
|
519 mpi->flags|=MP_IMGFLAG_DIRECT;
|
|
|
520 }
|
|
|
521
|
|
|
522 static int put_image(struct vf_instance_s* vf, mp_image_t *mpi)
|
|
|
523 {
|
|
|
524 mp_image_t *dmpi;
|
|
|
525 if(!(mpi->flags&MP_IMGFLAG_DIRECT)){
|
|
|
526 // no DR, so get a new image! hope we'll get DR buffer:
|
|
|
527 dmpi=vf_get_image(vf->next,mpi->imgfmt,
|
|
|
528 MP_IMGTYPE_TEMP,
|
|
|
529 MP_IMGFLAG_ACCEPT_STRIDE|MP_IMGFLAG_PREFER_ALIGNED_STRIDE,
|
|
16018
|
530 mpi->width,mpi->height);
|
|
15631
|
531 vf_clone_mpi_attributes(dmpi, mpi);
|
|
|
532 }else{
|
|
|
533 dmpi=vf->dmpi;
|
|
|
534 }
|
|
|
535
|
|
17225
|
536 vf->priv->mpeg2= mpi->qscale_type;
|
|
17133
|
537 if(mpi->pict_type != 3 && mpi->qscale && !vf->priv->qp){
|
|
|
538 if(!vf->priv->non_b_qp)
|
|
|
539 vf->priv->non_b_qp= malloc(mpi->qstride * mpi->h);
|
|
|
540 memcpy(vf->priv->non_b_qp, mpi->qscale, mpi->qstride * mpi->h);
|
|
|
541 }
|
|
15631
|
542 if(vf->priv->log2_count || !(mpi->flags&MP_IMGFLAG_DIRECT)){
|
|
17133
|
543 char *qp_tab= vf->priv->non_b_qp;
|
|
17225
|
544 if(vf->priv->bframes || !qp_tab)
|
|
17133
|
545 qp_tab= mpi->qscale;
|
|
|
546
|
|
17225
|
547 if(qp_tab || vf->priv->qp){
|
|
17133
|
548 filter(vf->priv, dmpi->planes[0], mpi->planes[0], dmpi->stride[0], mpi->stride[0],
|
|
|
549 mpi->w, mpi->h, qp_tab, mpi->qstride, 1);
|
|
|
550 filter(vf->priv, dmpi->planes[1], mpi->planes[1], dmpi->stride[1], mpi->stride[1],
|
|
|
551 mpi->w>>mpi->chroma_x_shift, mpi->h>>mpi->chroma_y_shift, qp_tab, mpi->qstride, 0);
|
|
|
552 filter(vf->priv, dmpi->planes[2], mpi->planes[2], dmpi->stride[2], mpi->stride[2],
|
|
|
553 mpi->w>>mpi->chroma_x_shift, mpi->h>>mpi->chroma_y_shift, qp_tab, mpi->qstride, 0);
|
|
15631
|
554 }else{
|
|
|
555 memcpy_pic(dmpi->planes[0], mpi->planes[0], mpi->w, mpi->h, dmpi->stride[0], mpi->stride[0]);
|
|
|
556 memcpy_pic(dmpi->planes[1], mpi->planes[1], mpi->w>>mpi->chroma_x_shift, mpi->h>>mpi->chroma_y_shift, dmpi->stride[1], mpi->stride[1]);
|
|
|
557 memcpy_pic(dmpi->planes[2], mpi->planes[2], mpi->w>>mpi->chroma_x_shift, mpi->h>>mpi->chroma_y_shift, dmpi->stride[2], mpi->stride[2]);
|
|
|
558 }
|
|
|
559 }
|
|
|
560
|
|
|
561 #ifdef HAVE_MMX
|
|
|
562 if(gCpuCaps.hasMMX) asm volatile ("emms\n\t");
|
|
|
563 #endif
|
|
|
564 #ifdef HAVE_MMX2
|
|
|
565 if(gCpuCaps.hasMMX2) asm volatile ("sfence\n\t");
|
|
|
566 #endif
|
|
|
567 return vf_next_put_image(vf,dmpi);
|
|
|
568 }
|
|
|
569
|
|
|
570 static void uninit(struct vf_instance_s* vf)
|
|
|
571 {
|
|
|
572 if(!vf->priv) return;
|
|
|
573
|
|
|
574 if(vf->priv->temp) av_free(vf->priv->temp);
|
|
|
575 vf->priv->temp= NULL;
|
|
|
576 if(vf->priv->src) av_free(vf->priv->src);
|
|
|
577 vf->priv->src= NULL;
|
|
|
578 //if(vf->priv->avctx) free(vf->priv->avctx);
|
|
|
579 //vf->priv->avctx= NULL;
|
|
17133
|
580 if(vf->priv->non_b_qp) free(vf->priv->non_b_qp);
|
|
|
581 vf->priv->non_b_qp= NULL;
|
|
15631
|
582
|
|
|
583 av_free(vf->priv);
|
|
|
584 vf->priv=NULL;
|
|
|
585 }
|
|
|
586
|
|
|
587 //===========================================================================//
|
|
|
588
|
|
|
589 static int query_format(struct vf_instance_s* vf, unsigned int fmt)
|
|
|
590 {
|
|
|
591 switch(fmt){
|
|
|
592 case IMGFMT_YVU9:
|
|
|
593 case IMGFMT_IF09:
|
|
|
594 case IMGFMT_YV12:
|
|
|
595 case IMGFMT_I420:
|
|
|
596 case IMGFMT_IYUV:
|
|
|
597 case IMGFMT_CLPL:
|
|
|
598 case IMGFMT_Y800:
|
|
|
599 case IMGFMT_Y8:
|
|
|
600 case IMGFMT_444P:
|
|
|
601 case IMGFMT_422P:
|
|
|
602 case IMGFMT_411P:
|
|
|
603 return vf_next_query_format(vf,fmt);
|
|
|
604 }
|
|
|
605 return 0;
|
|
|
606 }
|
|
|
607
|
|
|
608 /*
|
|
|
609 static unsigned int fmt_list[]={
|
|
|
610 IMGFMT_YVU9,
|
|
|
611 IMGFMT_IF09,
|
|
|
612 IMGFMT_YV12,
|
|
|
613 IMGFMT_I420,
|
|
|
614 IMGFMT_IYUV,
|
|
|
615 IMGFMT_CLPL,
|
|
|
616 IMGFMT_Y800,
|
|
|
617 IMGFMT_Y8,
|
|
|
618 IMGFMT_444P,
|
|
|
619 IMGFMT_422P,
|
|
|
620 IMGFMT_411P,
|
|
|
621 0
|
|
|
622 };
|
|
|
623 */
|
|
|
624
|
|
|
625 static int control(struct vf_instance_s* vf, int request, void* data)
|
|
|
626 {
|
|
|
627 switch(request){
|
|
|
628 case VFCTRL_QUERY_MAX_PP_LEVEL:
|
|
|
629 return 5;
|
|
|
630 case VFCTRL_SET_PP_LEVEL:
|
|
|
631 vf->priv->log2_count= *((unsigned int*)data);
|
|
|
632 if (vf->priv->log2_count < 4) vf->priv->log2_count=4;
|
|
|
633 return CONTROL_TRUE;
|
|
|
634 }
|
|
|
635 return vf_next_control(vf,request,data);
|
|
|
636 }
|
|
|
637
|
|
|
638 static int open(vf_instance_t *vf, char* args)
|
|
|
639 {
|
|
|
640 int i=0, bias;
|
|
|
641 int custom_threshold_m[64];
|
|
|
642 int log2c=-1;
|
|
|
643
|
|
|
644 vf->config=config;
|
|
|
645 vf->put_image=put_image;
|
|
|
646 vf->get_image=get_image;
|
|
|
647 vf->query_format=query_format;
|
|
|
648 vf->uninit=uninit;
|
|
|
649 vf->control= control;
|
|
|
650 vf->priv=av_mallocz(sizeof(struct vf_priv_s));//assumes align 16 !
|
|
|
651
|
|
|
652 avcodec_init();
|
|
|
653
|
|
|
654 //vf->priv->avctx= avcodec_alloc_context();
|
|
|
655 //dsputil_init(&vf->priv->dsp, vf->priv->avctx);
|
|
|
656
|
|
|
657 vf->priv->log2_count= 4;
|
|
17225
|
658 vf->priv->bframes = 0;
|
|
15631
|
659
|
|
17225
|
660 if (args) sscanf(args, "%d:%d:%d:%d", &log2c, &vf->priv->qp, &i, &vf->priv->bframes);
|
|
|
661
|
|
|
662 if( log2c >=4 && log2c <=5 )
|
|
15631
|
663 vf->priv->log2_count = log2c;
|
|
15651
|
664 else if( log2c >= 6 )
|
|
|
665 vf->priv->log2_count = 5;
|
|
15631
|
666
|
|
|
667 if(vf->priv->qp < 0)
|
|
|
668 vf->priv->qp = 0;
|
|
15651
|
669
|
|
|
670 if (i < -15) i = -15;
|
|
|
671 if (i > 32) i = 32;
|
|
15631
|
672
|
|
|
673 bias= (1<<4)+i; //regulable
|
|
|
674 vf->priv->prev_q=0;
|
|
|
675 //
|
|
|
676 for(i=0;i<64;i++) //FIXME: tune custom_threshold[] and remove this !
|
|
|
677 custom_threshold_m[i]=(int)(custom_threshold[i]*(bias/71.)+ 0.5);
|
|
|
678 for(i=0;i<8;i++){
|
|
|
679 vf->priv->threshold_mtx_noq[2*i]=(uint64_t)custom_threshold_m[i*8+2]
|
|
|
680 |(((uint64_t)custom_threshold_m[i*8+6])<<16)
|
|
|
681 |(((uint64_t)custom_threshold_m[i*8+0])<<32)
|
|
|
682 |(((uint64_t)custom_threshold_m[i*8+4])<<48);
|
|
|
683 vf->priv->threshold_mtx_noq[2*i+1]=(uint64_t)custom_threshold_m[i*8+5]
|
|
|
684 |(((uint64_t)custom_threshold_m[i*8+3])<<16)
|
|
|
685 |(((uint64_t)custom_threshold_m[i*8+1])<<32)
|
|
|
686 |(((uint64_t)custom_threshold_m[i*8+7])<<48);
|
|
|
687 }
|
|
|
688
|
|
|
689 if (vf->priv->qp) vf->priv->prev_q=vf->priv->qp, mul_thrmat_s(vf->priv, vf->priv->qp);
|
|
|
690
|
|
|
691 return 1;
|
|
|
692 }
|
|
|
693
|
|
|
694 vf_info_t vf_info_fspp = {
|
|
|
695 "fast simple postprocess",
|
|
|
696 "fspp",
|
|
|
697 "Michael Niedermayer, Nikolaj Poroshin",
|
|
|
698 "",
|
|
|
699 open,
|
|
|
700 NULL
|
|
|
701 };
|
|
|
702
|
|
|
703 //====================================================================
|
|
|
704 //Specific spp's dct, idct and threshold functions
|
|
|
705 //I'd prefer to have them in the separate file.
|
|
|
706
|
|
17012
|
707 #include "mangle.h"
|
|
15631
|
708 //#define MANGLE(a) #a
|
|
|
709
|
|
|
710 //typedef int16_t DCTELEM; //! only int16_t
|
|
|
711
|
|
|
712 #define DCTSIZE 8
|
|
|
713 #define DCTSIZE_S "8"
|
|
|
714
|
|
|
715 #define FIX(x,s) ((int) ((x) * (1<<s) + 0.5)&0xffff)
|
|
|
716 #define C64(x) ((uint64_t)((x)|(x)<<16))<<32 | (uint64_t)(x) | (uint64_t)(x)<<16
|
|
|
717 #define FIX64(x,s) C64(FIX(x,s))
|
|
|
718
|
|
|
719 #define MULTIPLY16H(x,k) (((x)*(k))>>16)
|
|
|
720 #define THRESHOLD(r,x,t) if(((unsigned)((x)+t))>t*2) r=(x);else r=0;
|
|
|
721 #define DESCALE(x,n) (((x) + (1 << ((n)-1))) >> n)
|
|
|
722
|
|
|
723 #ifdef HAVE_MMX
|
|
|
724
|
|
|
725 static uint64_t attribute_used __attribute__((aligned(8))) temps[4];//!!
|
|
|
726
|
|
|
727 static uint64_t attribute_used __attribute__((aligned(8))) MM_FIX_0_382683433=FIX64(0.382683433, 14);
|
|
|
728 static uint64_t attribute_used __attribute__((aligned(8))) MM_FIX_0_541196100=FIX64(0.541196100, 14);
|
|
|
729 static uint64_t attribute_used __attribute__((aligned(8))) MM_FIX_0_707106781=FIX64(0.707106781, 14);
|
|
|
730 static uint64_t attribute_used __attribute__((aligned(8))) MM_FIX_1_306562965=FIX64(1.306562965, 14);
|
|
|
731
|
|
|
732 static uint64_t attribute_used __attribute__((aligned(8))) MM_FIX_1_414213562_A=FIX64(1.414213562, 14);
|
|
|
733
|
|
|
734 static uint64_t attribute_used __attribute__((aligned(8))) MM_FIX_1_847759065=FIX64(1.847759065, 13);
|
|
|
735 static uint64_t attribute_used __attribute__((aligned(8))) MM_FIX_2_613125930=FIX64(-2.613125930, 13); //-
|
|
|
736 static uint64_t attribute_used __attribute__((aligned(8))) MM_FIX_1_414213562=FIX64(1.414213562, 13);
|
|
|
737 static uint64_t attribute_used __attribute__((aligned(8))) MM_FIX_1_082392200=FIX64(1.082392200, 13);
|
|
|
738 //for t3,t5,t7 == 0 shortcut
|
|
|
739 static uint64_t attribute_used __attribute__((aligned(8))) MM_FIX_0_847759065=FIX64(0.847759065, 14);
|
|
|
740 static uint64_t attribute_used __attribute__((aligned(8))) MM_FIX_0_566454497=FIX64(0.566454497, 14);
|
|
|
741 static uint64_t attribute_used __attribute__((aligned(8))) MM_FIX_0_198912367=FIX64(0.198912367, 14);
|
|
|
742
|
|
|
743 static uint64_t attribute_used __attribute__((aligned(8))) MM_DESCALE_RND=C64(4);
|
|
|
744 static uint64_t attribute_used __attribute__((aligned(8))) MM_2=C64(2);
|
|
|
745
|
|
|
746 #else /* !HAVE_MMX */
|
|
|
747
|
|
|
748 typedef int32_t int_simd16_t;
|
|
|
749 static int16_t FIX_0_382683433=FIX(0.382683433, 14);
|
|
|
750 static int16_t FIX_0_541196100=FIX(0.541196100, 14);
|
|
|
751 static int16_t FIX_0_707106781=FIX(0.707106781, 14);
|
|
|
752 static int16_t FIX_1_306562965=FIX(1.306562965, 14);
|
|
|
753 static int16_t FIX_1_414213562_A=FIX(1.414213562, 14);
|
|
|
754 static int16_t FIX_1_847759065=FIX(1.847759065, 13);
|
|
|
755 static int16_t FIX_2_613125930=FIX(-2.613125930, 13); //-
|
|
|
756 static int16_t FIX_1_414213562=FIX(1.414213562, 13);
|
|
|
757 static int16_t FIX_1_082392200=FIX(1.082392200, 13);
|
|
|
758
|
|
|
759 #endif
|
|
|
760
|
|
|
761 #ifndef HAVE_MMX
|
|
|
762
|
|
|
763 static void column_fidct_c(int16_t* thr_adr, DCTELEM *data, DCTELEM *output, int cnt)
|
|
|
764 {
|
|
|
765 int_simd16_t tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
|
|
|
766 int_simd16_t tmp10, tmp11, tmp12, tmp13;
|
|
|
767 int_simd16_t z1,z2,z3,z4,z5, z10, z11, z12, z13;
|
|
|
768 int_simd16_t d0, d1, d2, d3, d4, d5, d6, d7;
|
|
|
769
|
|
|
770 DCTELEM* dataptr;
|
|
|
771 DCTELEM* wsptr;
|
|
|
772 int16_t *threshold;
|
|
|
773 int ctr;
|
|
|
774
|
|
|
775 dataptr = data;
|
|
|
776 wsptr = output;
|
|
|
777
|
|
|
778 for (; cnt > 0; cnt-=2) { //start positions
|
|
|
779 threshold=(int16_t*)thr_adr;//threshold_mtx
|
|
|
780 for (ctr = DCTSIZE; ctr > 0; ctr--) {
|
|
|
781 // Process columns from input, add to output.
|
|
|
782 tmp0 = dataptr[DCTSIZE*0] + dataptr[DCTSIZE*7];
|
|
|
783 tmp7 = dataptr[DCTSIZE*0] - dataptr[DCTSIZE*7];
|
|
|
784
|
|
|
785 tmp1 = dataptr[DCTSIZE*1] + dataptr[DCTSIZE*6];
|
|
|
786 tmp6 = dataptr[DCTSIZE*1] - dataptr[DCTSIZE*6];
|
|
|
787
|
|
|
788 tmp2 = dataptr[DCTSIZE*2] + dataptr[DCTSIZE*5];
|
|
|
789 tmp5 = dataptr[DCTSIZE*2] - dataptr[DCTSIZE*5];
|
|
|
790
|
|
|
791 tmp3 = dataptr[DCTSIZE*3] + dataptr[DCTSIZE*4];
|
|
|
792 tmp4 = dataptr[DCTSIZE*3] - dataptr[DCTSIZE*4];
|
|
|
793
|
|
|
794 // Even part of FDCT
|
|
|
795
|
|
|
796 tmp10 = tmp0 + tmp3;
|
|
|
797 tmp13 = tmp0 - tmp3;
|
|
|
798 tmp11 = tmp1 + tmp2;
|
|
|
799 tmp12 = tmp1 - tmp2;
|
|
|
800
|
|
|
801 d0 = tmp10 + tmp11;
|
|
|
802 d4 = tmp10 - tmp11;
|
|
|
803
|
|
|
804 z1 = MULTIPLY16H((tmp12 + tmp13) <<2, FIX_0_707106781);
|
|
|
805 d2 = tmp13 + z1;
|
|
|
806 d6 = tmp13 - z1;
|
|
|
807
|
|
|
808 // Even part of IDCT
|
|
|
809
|
|
|
810 THRESHOLD(tmp0, d0, threshold[0*8]);
|
|
|
811 THRESHOLD(tmp1, d2, threshold[2*8]);
|
|
|
812 THRESHOLD(tmp2, d4, threshold[4*8]);
|
|
|
813 THRESHOLD(tmp3, d6, threshold[6*8]);
|
|
|
814 tmp0+=2;
|
|
|
815 tmp10 = (tmp0 + tmp2)>>2;
|
|
|
816 tmp11 = (tmp0 - tmp2)>>2;
|
|
|
817
|
|
|
818 tmp13 = (tmp1 + tmp3)>>2; //+2 ! (psnr decides)
|
|
|
819 tmp12 = MULTIPLY16H((tmp1 - tmp3), FIX_1_414213562_A) - tmp13; //<<2
|
|
|
820
|
|
|
821 tmp0 = tmp10 + tmp13; //->temps
|
|
|
822 tmp3 = tmp10 - tmp13; //->temps
|
|
|
823 tmp1 = tmp11 + tmp12; //->temps
|
|
|
824 tmp2 = tmp11 - tmp12; //->temps
|
|
|
825
|
|
|
826 // Odd part of FDCT
|
|
|
827
|
|
|
828 tmp10 = tmp4 + tmp5;
|
|
|
829 tmp11 = tmp5 + tmp6;
|
|
|
830 tmp12 = tmp6 + tmp7;
|
|
|
831
|
|
|
832 z5 = MULTIPLY16H((tmp10 - tmp12)<<2, FIX_0_382683433);
|
|
|
833 z2 = MULTIPLY16H(tmp10 <<2, FIX_0_541196100) + z5;
|
|
|
834 z4 = MULTIPLY16H(tmp12 <<2, FIX_1_306562965) + z5;
|
|
|
835 z3 = MULTIPLY16H(tmp11 <<2, FIX_0_707106781);
|
|
|
836
|
|
|
837 z11 = tmp7 + z3;
|
|
|
838 z13 = tmp7 - z3;
|
|
|
839
|
|
|
840 d5 = z13 + z2;
|
|
|
841 d3 = z13 - z2;
|
|
|
842 d1 = z11 + z4;
|
|
|
843 d7 = z11 - z4;
|
|
|
844
|
|
|
845 // Odd part of IDCT
|
|
|
846
|
|
|
847 THRESHOLD(tmp4, d1, threshold[1*8]);
|
|
|
848 THRESHOLD(tmp5, d3, threshold[3*8]);
|
|
|
849 THRESHOLD(tmp6, d5, threshold[5*8]);
|
|
|
850 THRESHOLD(tmp7, d7, threshold[7*8]);
|
|
|
851
|
|
|
852 //Simd version uses here a shortcut for the tmp5,tmp6,tmp7 == 0
|
|
|
853 z13 = tmp6 + tmp5;
|
|
|
854 z10 = (tmp6 - tmp5)<<1;
|
|
|
855 z11 = tmp4 + tmp7;
|
|
|
856 z12 = (tmp4 - tmp7)<<1;
|
|
|
857
|
|
|
858 tmp7 = (z11 + z13)>>2; //+2 !
|
|
|
859 tmp11 = MULTIPLY16H((z11 - z13)<<1, FIX_1_414213562);
|
|
|
860 z5 = MULTIPLY16H(z10 + z12, FIX_1_847759065);
|
|
|
861 tmp10 = MULTIPLY16H(z12, FIX_1_082392200) - z5;
|
|
|
862 tmp12 = MULTIPLY16H(z10, FIX_2_613125930) + z5; // - !!
|
|
|
863
|
|
|
864 tmp6 = tmp12 - tmp7;
|
|
|
865 tmp5 = tmp11 - tmp6;
|
|
|
866 tmp4 = tmp10 + tmp5;
|
|
|
867
|
|
|
868 wsptr[DCTSIZE*0]+= (tmp0 + tmp7);
|
|
|
869 wsptr[DCTSIZE*1]+= (tmp1 + tmp6);
|
|
|
870 wsptr[DCTSIZE*2]+= (tmp2 + tmp5);
|
|
|
871 wsptr[DCTSIZE*3]+= (tmp3 - tmp4);
|
|
|
872 wsptr[DCTSIZE*4]+= (tmp3 + tmp4);
|
|
|
873 wsptr[DCTSIZE*5]+= (tmp2 - tmp5);
|
|
|
874 wsptr[DCTSIZE*6]= (tmp1 - tmp6);
|
|
|
875 wsptr[DCTSIZE*7]= (tmp0 - tmp7);
|
|
|
876 //
|
|
|
877 dataptr++; //next column
|
|
|
878 wsptr++;
|
|
|
879 threshold++;
|
|
|
880 }
|
|
|
881 dataptr+=8; //skip each second start pos
|
|
|
882 wsptr +=8;
|
|
|
883 }
|
|
|
884 }
|
|
|
885
|
|
|
886 #else /* HAVE_MMX */
|
|
|
887
|
|
|
888 static void column_fidct_mmx(int16_t* thr_adr, DCTELEM *data, DCTELEM *output, int cnt)
|
|
|
889 {
|
|
|
890 asm volatile(
|
|
|
891 ".align 16 \n\t"
|
|
|
892 "1: \n\t"
|
|
15634
|
893 "movq "DCTSIZE_S"*0*2(%%"REG_S"), %%mm1 \n\t"
|
|
15631
|
894 //
|
|
15634
|
895 "movq "DCTSIZE_S"*3*2(%%"REG_S"), %%mm7 \n\t"
|
|
15631
|
896 "movq %%mm1, %%mm0 \n\t"
|
|
|
897
|
|
15634
|
898 "paddw "DCTSIZE_S"*7*2(%%"REG_S"), %%mm1 \n\t" //t0
|
|
15631
|
899 "movq %%mm7, %%mm3 \n\t"
|
|
|
900
|
|
15634
|
901 "paddw "DCTSIZE_S"*4*2(%%"REG_S"), %%mm7 \n\t" //t3
|
|
15631
|
902 "movq %%mm1, %%mm5 \n\t"
|
|
|
903
|
|
15634
|
904 "movq "DCTSIZE_S"*1*2(%%"REG_S"), %%mm6 \n\t"
|
|
15631
|
905 "psubw %%mm7, %%mm1 \n\t" //t13
|
|
|
906
|
|
15634
|
907 "movq "DCTSIZE_S"*2*2(%%"REG_S"), %%mm2 \n\t"
|
|
15631
|
908 "movq %%mm6, %%mm4 \n\t"
|
|
|
909
|
|
15634
|
910 "paddw "DCTSIZE_S"*6*2(%%"REG_S"), %%mm6 \n\t" //t1
|
|
15631
|
911 "paddw %%mm7, %%mm5 \n\t" //t10
|
|
|
912
|
|
15634
|
913 "paddw "DCTSIZE_S"*5*2(%%"REG_S"), %%mm2 \n\t" //t2
|
|
15631
|
914 "movq %%mm6, %%mm7 \n\t"
|
|
|
915
|
|
|
916 "paddw %%mm2, %%mm6 \n\t" //t11
|
|
|
917 "psubw %%mm2, %%mm7 \n\t" //t12
|
|
|
918
|
|
|
919 "movq %%mm5, %%mm2 \n\t"
|
|
|
920 "paddw %%mm6, %%mm5 \n\t" //d0
|
|
|
921 // i0 t13 t12 i3 i1 d0 - d4
|
|
|
922 "psubw %%mm6, %%mm2 \n\t" //d4
|
|
|
923 "paddw %%mm1, %%mm7 \n\t"
|
|
|
924
|
|
15634
|
925 "movq 4*16(%%"REG_d"), %%mm6 \n\t"
|
|
15631
|
926 "psllw $2, %%mm7 \n\t"
|
|
|
927
|
|
15634
|
928 "psubw 0*16(%%"REG_d"), %%mm5 \n\t"
|
|
15631
|
929 "psubw %%mm6, %%mm2 \n\t"
|
|
|
930
|
|
15634
|
931 "paddusw 0*16(%%"REG_d"), %%mm5 \n\t"
|
|
15631
|
932 "paddusw %%mm6, %%mm2 \n\t"
|
|
|
933
|
|
|
934 "pmulhw "MANGLE(MM_FIX_0_707106781)", %%mm7 \n\t"
|
|
|
935 //
|
|
15634
|
936 "paddw 0*16(%%"REG_d"), %%mm5 \n\t"
|
|
15631
|
937 "paddw %%mm6, %%mm2 \n\t"
|
|
|
938
|
|
15634
|
939 "psubusw 0*16(%%"REG_d"), %%mm5 \n\t"
|
|
15631
|
940 "psubusw %%mm6, %%mm2 \n\t"
|
|
|
941
|
|
|
942 //This func is totally compute-bound, operates at huge speed. So, DC shortcut
|
|
|
943 // at this place isn't worthwhile due to BTB miss penalty (checked on Pent. 3).
|
|
|
944 //However, typical numbers: nondc - 29%%, dc - 46%%, zero - 25%%. All <> 0 case is very rare.
|
|
|
945 "paddw "MANGLE(MM_2)", %%mm5 \n\t"
|
|
|
946 "movq %%mm2, %%mm6 \n\t"
|
|
|
947
|
|
|
948 "paddw %%mm5, %%mm2 \n\t"
|
|
|
949 "psubw %%mm6, %%mm5 \n\t"
|
|
|
950
|
|
|
951 "movq %%mm1, %%mm6 \n\t"
|
|
|
952 "paddw %%mm7, %%mm1 \n\t" //d2
|
|
|
953
|
|
15634
|
954 "psubw 2*16(%%"REG_d"), %%mm1 \n\t"
|
|
15631
|
955 "psubw %%mm7, %%mm6 \n\t" //d6
|
|
|
956
|
|
15634
|
957 "movq 6*16(%%"REG_d"), %%mm7 \n\t"
|
|
15631
|
958 "psraw $2, %%mm5 \n\t"
|
|
|
959
|
|
15634
|
960 "paddusw 2*16(%%"REG_d"), %%mm1 \n\t"
|
|
15631
|
961 "psubw %%mm7, %%mm6 \n\t"
|
|
|
962 // t7 d2 /t11 t4 t6 - d6 /t10
|
|
|
963
|
|
15634
|
964 "paddw 2*16(%%"REG_d"), %%mm1 \n\t"
|
|
15631
|
965 "paddusw %%mm7, %%mm6 \n\t"
|
|
|
966
|
|
15634
|
967 "psubusw 2*16(%%"REG_d"), %%mm1 \n\t"
|
|
15631
|
968 "paddw %%mm7, %%mm6 \n\t"
|
|
|
969
|
|
15634
|
970 "psubw "DCTSIZE_S"*4*2(%%"REG_S"), %%mm3 \n\t"
|
|
15631
|
971 "psubusw %%mm7, %%mm6 \n\t"
|
|
|
972
|
|
|
973 //movq [edi+"DCTSIZE_S"*2*2], mm1
|
|
|
974 //movq [edi+"DCTSIZE_S"*6*2], mm6
|
|
|
975 "movq %%mm1, %%mm7 \n\t"
|
|
|
976 "psraw $2, %%mm2 \n\t"
|
|
|
977
|
|
15634
|
978 "psubw "DCTSIZE_S"*6*2(%%"REG_S"), %%mm4 \n\t"
|
|
15631
|
979 "psubw %%mm6, %%mm1 \n\t"
|
|
|
980
|
|
15634
|
981 "psubw "DCTSIZE_S"*7*2(%%"REG_S"), %%mm0 \n\t"
|
|
15631
|
982 "paddw %%mm7, %%mm6 \n\t" //'t13
|
|
|
983
|
|
|
984 "psraw $2, %%mm6 \n\t" //paddw mm6, MM_2 !! ---
|
|
|
985 "movq %%mm2, %%mm7 \n\t"
|
|
|
986
|
|
|
987 "pmulhw "MANGLE(MM_FIX_1_414213562_A)", %%mm1 \n\t"
|
|
|
988 "paddw %%mm6, %%mm2 \n\t" //'t0
|
|
|
989
|
|
|
990 "movq %%mm2, "MANGLE(temps)"+0*8 \n\t" //!
|
|
|
991 "psubw %%mm6, %%mm7 \n\t" //'t3
|
|
|
992
|
|
15634
|
993 "movq "DCTSIZE_S"*2*2(%%"REG_S"), %%mm2 \n\t"
|
|
15631
|
994 "psubw %%mm6, %%mm1 \n\t" //'t12
|
|
|
995
|
|
15634
|
996 "psubw "DCTSIZE_S"*5*2(%%"REG_S"), %%mm2 \n\t" //t5
|
|
15631
|
997 "movq %%mm5, %%mm6 \n\t"
|
|
|
998
|
|
|
999 "movq %%mm7, "MANGLE(temps)"+3*8 \n\t"
|
|
|
1000 "paddw %%mm2, %%mm3 \n\t" //t10
|
|
|
1001
|
|
|
1002 "paddw %%mm4, %%mm2 \n\t" //t11
|
|
|
1003 "paddw %%mm0, %%mm4 \n\t" //t12
|
|
|
1004
|
|
|
1005 "movq %%mm3, %%mm7 \n\t"
|
|
|
1006 "psubw %%mm4, %%mm3 \n\t"
|
|
|
1007
|
|
|
1008 "psllw $2, %%mm3 \n\t"
|
|
|
1009 "psllw $2, %%mm7 \n\t" //opt for P6
|
|
|
1010
|
|
|
1011 "pmulhw "MANGLE(MM_FIX_0_382683433)", %%mm3 \n\t"
|
|
|
1012 "psllw $2, %%mm4 \n\t"
|
|
|
1013
|
|
|
1014 "pmulhw "MANGLE(MM_FIX_0_541196100)", %%mm7 \n\t"
|
|
|
1015 "psllw $2, %%mm2 \n\t"
|
|
|
1016
|
|
|
1017 "pmulhw "MANGLE(MM_FIX_1_306562965)", %%mm4 \n\t"
|
|
|
1018 "paddw %%mm1, %%mm5 \n\t" //'t1
|
|
|
1019
|
|
|
1020 "pmulhw "MANGLE(MM_FIX_0_707106781)", %%mm2 \n\t"
|
|
|
1021 "psubw %%mm1, %%mm6 \n\t" //'t2
|
|
|
1022 // t7 't12 't11 t4 t6 - 't13 't10 ---
|
|
|
1023
|
|
|
1024 "paddw %%mm3, %%mm7 \n\t" //z2
|
|
|
1025
|
|
|
1026 "movq %%mm5, "MANGLE(temps)"+1*8 \n\t"
|
|
|
1027 "paddw %%mm3, %%mm4 \n\t" //z4
|
|
|
1028
|
|
15634
|
1029 "movq 3*16(%%"REG_d"), %%mm3 \n\t"
|
|
15631
|
1030 "movq %%mm0, %%mm1 \n\t"
|
|
|
1031
|
|
|
1032 "movq %%mm6, "MANGLE(temps)"+2*8 \n\t"
|
|
|
1033 "psubw %%mm2, %%mm1 \n\t" //z13
|
|
|
1034
|
|
|
1035 //===
|
|
|
1036 "paddw %%mm2, %%mm0 \n\t" //z11
|
|
|
1037 "movq %%mm1, %%mm5 \n\t"
|
|
|
1038
|
|
15634
|
1039 "movq 5*16(%%"REG_d"), %%mm2 \n\t"
|
|
15631
|
1040 "psubw %%mm7, %%mm1 \n\t" //d3
|
|
|
1041
|
|
|
1042 "paddw %%mm7, %%mm5 \n\t" //d5
|
|
|
1043 "psubw %%mm3, %%mm1 \n\t"
|
|
|
1044
|
|
15634
|
1045 "movq 1*16(%%"REG_d"), %%mm7 \n\t"
|
|
15631
|
1046 "psubw %%mm2, %%mm5 \n\t"
|
|
|
1047
|
|
|
1048 "movq %%mm0, %%mm6 \n\t"
|
|
|
1049 "paddw %%mm4, %%mm0 \n\t" //d1
|
|
|
1050
|
|
|
1051 "paddusw %%mm3, %%mm1 \n\t"
|
|
|
1052 "psubw %%mm4, %%mm6 \n\t" //d7
|
|
|
1053
|
|
|
1054 // d1 d3 - - - d5 d7 -
|
|
15634
|
1055 "movq 7*16(%%"REG_d"), %%mm4 \n\t"
|
|
15631
|
1056 "psubw %%mm7, %%mm0 \n\t"
|
|
|
1057
|
|
|
1058 "psubw %%mm4, %%mm6 \n\t"
|
|
|
1059 "paddusw %%mm2, %%mm5 \n\t"
|
|
|
1060
|
|
|
1061 "paddusw %%mm4, %%mm6 \n\t"
|
|
|
1062 "paddw %%mm3, %%mm1 \n\t"
|
|
|
1063
|
|
|
1064 "paddw %%mm2, %%mm5 \n\t"
|
|
|
1065 "paddw %%mm4, %%mm6 \n\t"
|
|
|
1066
|
|
|
1067 "psubusw %%mm3, %%mm1 \n\t"
|
|
|
1068 "psubusw %%mm2, %%mm5 \n\t"
|
|
|
1069
|
|
|
1070 "psubusw %%mm4, %%mm6 \n\t"
|
|
|
1071 "movq %%mm1, %%mm4 \n\t"
|
|
|
1072
|
|
|
1073 "por %%mm5, %%mm4 \n\t"
|
|
|
1074 "paddusw %%mm7, %%mm0 \n\t"
|
|
|
1075
|
|
|
1076 "por %%mm6, %%mm4 \n\t"
|
|
|
1077 "paddw %%mm7, %%mm0 \n\t"
|
|
|
1078
|
|
|
1079 "packssdw %%mm4, %%mm4 \n\t"
|
|
|
1080 "psubusw %%mm7, %%mm0 \n\t"
|
|
|
1081
|
|
15634
|
1082 "movd %%mm4, %%"REG_a" \n\t"
|
|
|
1083 "or %%"REG_a", %%"REG_a" \n\t"
|
|
15631
|
1084 "jnz 2f \n\t"
|
|
|
1085 //movq [edi+"DCTSIZE_S"*3*2], mm1
|
|
|
1086 //movq [edi+"DCTSIZE_S"*5*2], mm5
|
|
|
1087 //movq [edi+"DCTSIZE_S"*1*2], mm0
|
|
|
1088 //movq [edi+"DCTSIZE_S"*7*2], mm6
|
|
|
1089 // t4 t5 - - - t6 t7 -
|
|
|
1090 //--- t4 (mm0) may be <>0; mm1, mm5, mm6 == 0
|
|
|
1091 //Typical numbers: nondc - 19%%, dc - 26%%, zero - 55%%. zero case alone isn't worthwhile
|
|
|
1092 "movq "MANGLE(temps)"+0*8, %%mm4 \n\t"
|
|
|
1093 "movq %%mm0, %%mm1 \n\t"
|
|
|
1094
|
|
|
1095 "pmulhw "MANGLE(MM_FIX_0_847759065)", %%mm0 \n\t" //tmp6
|
|
|
1096 "movq %%mm1, %%mm2 \n\t"
|
|
|
1097
|
|
15634
|
1098 "movq "DCTSIZE_S"*0*2(%%"REG_D"), %%mm5 \n\t"
|
|
15631
|
1099 "movq %%mm2, %%mm3 \n\t"
|
|
|
1100
|
|
|
1101 "pmulhw "MANGLE(MM_FIX_0_566454497)", %%mm1 \n\t" //tmp5
|
|
|
1102 "paddw %%mm4, %%mm5 \n\t"
|
|
|
1103
|
|
|
1104 "movq "MANGLE(temps)"+1*8, %%mm6 \n\t"
|
|
|
1105 //paddw mm3, MM_2
|
|
|
1106 "psraw $2, %%mm3 \n\t" //tmp7
|
|
|
1107
|
|
|
1108 "pmulhw "MANGLE(MM_FIX_0_198912367)", %%mm2 \n\t" //-tmp4
|
|
|
1109 "psubw %%mm3, %%mm4 \n\t"
|
|
|
1110
|
|
15634
|
1111 "movq "DCTSIZE_S"*1*2(%%"REG_D"), %%mm7 \n\t"
|
|
15631
|
1112 "paddw %%mm3, %%mm5 \n\t"
|
|
|
1113
|
|
15634
|
1114 "movq %%mm4, "DCTSIZE_S"*7*2(%%"REG_D") \n\t"
|
|
15631
|
1115 "paddw %%mm6, %%mm7 \n\t"
|
|
|
1116
|
|
|
1117 "movq "MANGLE(temps)"+2*8, %%mm3 \n\t"
|
|
|
1118 "psubw %%mm0, %%mm6 \n\t"
|
|
|
1119
|
|
15634
|
1120 "movq "DCTSIZE_S"*2*2(%%"REG_D"), %%mm4 \n\t"
|
|
15631
|
1121 "paddw %%mm0, %%mm7 \n\t"
|
|
|
1122
|
|
15634
|
1123 "movq %%mm5, "DCTSIZE_S"*0*2(%%"REG_D") \n\t"
|
|
15631
|
1124 "paddw %%mm3, %%mm4 \n\t"
|
|
|
1125
|
|
15634
|
1126 "movq %%mm6, "DCTSIZE_S"*6*2(%%"REG_D") \n\t"
|
|
15631
|
1127 "psubw %%mm1, %%mm3 \n\t"
|
|
|
1128
|
|
15634
|
1129 "movq "DCTSIZE_S"*5*2(%%"REG_D"), %%mm5 \n\t"
|
|
15631
|
1130 "paddw %%mm1, %%mm4 \n\t"
|
|
|
1131
|
|
15634
|
1132 "movq "DCTSIZE_S"*3*2(%%"REG_D"), %%mm6 \n\t"
|
|
15631
|
1133 "paddw %%mm3, %%mm5 \n\t"
|
|
|
1134
|
|
|
1135 "movq "MANGLE(temps)"+3*8, %%mm0 \n\t"
|
|
15634
|
1136 "add $8, %%"REG_S" \n\t"
|
|
|
1137
|
|
|
1138 "movq %%mm7, "DCTSIZE_S"*1*2(%%"REG_D") \n\t"
|
|
15631
|
1139 "paddw %%mm0, %%mm6 \n\t"
|
|
|
1140
|
|
15634
|
1141 "movq %%mm4, "DCTSIZE_S"*2*2(%%"REG_D") \n\t"
|
|
15631
|
1142 "psubw %%mm2, %%mm0 \n\t"
|
|
|
1143
|
|
15634
|
1144 "movq "DCTSIZE_S"*4*2(%%"REG_D"), %%mm7 \n\t"
|
|
15631
|
1145 "paddw %%mm2, %%mm6 \n\t"
|
|
|
1146
|
|
15634
|
1147 "movq %%mm5, "DCTSIZE_S"*5*2(%%"REG_D") \n\t"
|
|
15631
|
1148 "paddw %%mm0, %%mm7 \n\t"
|
|
|
1149
|
|
15634
|
1150 "movq %%mm6, "DCTSIZE_S"*3*2(%%"REG_D") \n\t"
|
|
|
1151
|
|
|
1152 "movq %%mm7, "DCTSIZE_S"*4*2(%%"REG_D") \n\t"
|
|
|
1153 "add $8, %%"REG_D" \n\t"
|
|
15631
|
1154 "jmp 4f \n\t"
|
|
|
1155
|
|
|
1156 "2: \n\t"
|
|
|
1157 //--- non DC2
|
|
|
1158 //psraw mm1, 2 w/o it -> offset. thr1, thr1, thr1 (actually thr1, thr1, thr1-1)
|
|
|
1159 //psraw mm5, 2
|
|
|
1160 //psraw mm0, 2
|
|
|
1161 //psraw mm6, 2
|
|
|
1162 "movq %%mm5, %%mm3 \n\t"
|
|
|
1163 "psubw %%mm1, %%mm5 \n\t"
|
|
|
1164
|
|
|
1165 "psllw $1, %%mm5 \n\t" //'z10
|
|
|
1166 "paddw %%mm1, %%mm3 \n\t" //'z13
|
|
|
1167
|
|
|
1168 "movq %%mm0, %%mm2 \n\t"
|
|
|
1169 "psubw %%mm6, %%mm0 \n\t"
|
|
|
1170
|
|
|
1171 "movq %%mm5, %%mm1 \n\t"
|
|
|
1172 "psllw $1, %%mm0 \n\t" //'z12
|
|
|
1173
|
|
|
1174 "pmulhw "MANGLE(MM_FIX_2_613125930)", %%mm1 \n\t" //-
|
|
|
1175 "paddw %%mm0, %%mm5 \n\t"
|
|
|
1176
|
|
|
1177 "pmulhw "MANGLE(MM_FIX_1_847759065)", %%mm5 \n\t" //'z5
|
|
|
1178 "paddw %%mm6, %%mm2 \n\t" //'z11
|
|
|
1179
|
|
|
1180 "pmulhw "MANGLE(MM_FIX_1_082392200)", %%mm0 \n\t"
|
|
|
1181 "movq %%mm2, %%mm7 \n\t"
|
|
|
1182
|
|
|
1183 //---
|
|
|
1184 "movq "MANGLE(temps)"+0*8, %%mm4 \n\t"
|
|
|
1185 "psubw %%mm3, %%mm2 \n\t"
|
|
|
1186
|
|
|
1187 "psllw $1, %%mm2 \n\t"
|
|
|
1188 "paddw %%mm3, %%mm7 \n\t" //'t7
|
|
|
1189
|
|
|
1190 "pmulhw "MANGLE(MM_FIX_1_414213562)", %%mm2 \n\t" //'t11
|
|
|
1191 "movq %%mm4, %%mm6 \n\t"
|
|
|
1192 //paddw mm7, MM_2
|
|
|
1193 "psraw $2, %%mm7 \n\t"
|
|
|
1194
|
|
15634
|
1195 "paddw "DCTSIZE_S"*0*2(%%"REG_D"), %%mm4 \n\t"
|
|
15631
|
1196 "psubw %%mm7, %%mm6 \n\t"
|
|
|
1197
|
|
|
1198 "movq "MANGLE(temps)"+1*8, %%mm3 \n\t"
|
|
|
1199 "paddw %%mm7, %%mm4 \n\t"
|
|
|
1200
|
|
15634
|
1201 "movq %%mm6, "DCTSIZE_S"*7*2(%%"REG_D") \n\t"
|
|
15631
|
1202 "paddw %%mm5, %%mm1 \n\t" //'t12
|
|
|
1203
|
|
15634
|
1204 "movq %%mm4, "DCTSIZE_S"*0*2(%%"REG_D") \n\t"
|
|
15631
|
1205 "psubw %%mm7, %%mm1 \n\t" //'t6
|
|
|
1206
|
|
|
1207 "movq "MANGLE(temps)"+2*8, %%mm7 \n\t"
|
|
|
1208 "psubw %%mm5, %%mm0 \n\t" //'t10
|
|
|
1209
|
|
|
1210 "movq "MANGLE(temps)"+3*8, %%mm6 \n\t"
|
|
|
1211 "movq %%mm3, %%mm5 \n\t"
|
|
|
1212
|
|
15634
|
1213 "paddw "DCTSIZE_S"*1*2(%%"REG_D"), %%mm3 \n\t"
|
|
15631
|
1214 "psubw %%mm1, %%mm5 \n\t"
|
|
|
1215
|
|
|
1216 "psubw %%mm1, %%mm2 \n\t" //'t5
|
|
|
1217 "paddw %%mm1, %%mm3 \n\t"
|
|
|
1218
|
|
15634
|
1219 "movq %%mm5, "DCTSIZE_S"*6*2(%%"REG_D") \n\t"
|
|
15631
|
1220 "movq %%mm7, %%mm4 \n\t"
|
|
|
1221
|
|
15634
|
1222 "paddw "DCTSIZE_S"*2*2(%%"REG_D"), %%mm7 \n\t"
|
|
15631
|
1223 "psubw %%mm2, %%mm4 \n\t"
|
|
|
1224
|
|
15634
|
1225 "paddw "DCTSIZE_S"*5*2(%%"REG_D"), %%mm4 \n\t"
|
|
15631
|
1226 "paddw %%mm2, %%mm7 \n\t"
|
|
|
1227
|
|
15634
|
1228 "movq %%mm3, "DCTSIZE_S"*1*2(%%"REG_D") \n\t"
|
|
15631
|
1229 "paddw %%mm2, %%mm0 \n\t" //'t4
|
|
|
1230
|
|
|
1231 // 't4 't6 't5 - - - - 't7
|
|
15634
|
1232 "movq %%mm7, "DCTSIZE_S"*2*2(%%"REG_D") \n\t"
|
|
15631
|
1233 "movq %%mm6, %%mm1 \n\t"
|
|
|
1234
|
|
15634
|
1235 "paddw "DCTSIZE_S"*4*2(%%"REG_D"), %%mm6 \n\t"
|
|
15631
|
1236 "psubw %%mm0, %%mm1 \n\t"
|
|
|
1237
|
|
15634
|
1238 "paddw "DCTSIZE_S"*3*2(%%"REG_D"), %%mm1 \n\t"
|
|
15631
|
1239 "paddw %%mm0, %%mm6 \n\t"
|
|
|
1240
|
|
15634
|
1241 "movq %%mm4, "DCTSIZE_S"*5*2(%%"REG_D") \n\t"
|
|
|
1242 "add $8, %%"REG_S" \n\t"
|
|
|
1243
|
|
|
1244 "movq %%mm6, "DCTSIZE_S"*4*2(%%"REG_D") \n\t"
|
|
|
1245
|
|
|
1246 "movq %%mm1, "DCTSIZE_S"*3*2(%%"REG_D") \n\t"
|
|
|
1247 "add $8, %%"REG_D" \n\t"
|
|
15631
|
1248
|
|
|
1249 "4: \n\t"
|
|
|
1250 //=part 2 (the same)===========================================================
|
|
15634
|
1251 "movq "DCTSIZE_S"*0*2(%%"REG_S"), %%mm1 \n\t"
|
|
15631
|
1252 //
|
|
15634
|
1253 "movq "DCTSIZE_S"*3*2(%%"REG_S"), %%mm7 \n\t"
|
|
15631
|
1254 "movq %%mm1, %%mm0 \n\t"
|
|
|
1255
|
|
15634
|
1256 "paddw "DCTSIZE_S"*7*2(%%"REG_S"), %%mm1 \n\t" //t0
|
|
15631
|
1257 "movq %%mm7, %%mm3 \n\t"
|
|
|
1258
|
|
15634
|
1259 "paddw "DCTSIZE_S"*4*2(%%"REG_S"), %%mm7 \n\t" //t3
|
|
15631
|
1260 "movq %%mm1, %%mm5 \n\t"
|
|
|
1261
|
|
15634
|
1262 "movq "DCTSIZE_S"*1*2(%%"REG_S"), %%mm6 \n\t"
|
|
15631
|
1263 "psubw %%mm7, %%mm1 \n\t" //t13
|
|
|
1264
|
|
15634
|
1265 "movq "DCTSIZE_S"*2*2(%%"REG_S"), %%mm2 \n\t"
|
|
15631
|
1266 "movq %%mm6, %%mm4 \n\t"
|
|
|
1267
|
|
15634
|
1268 "paddw "DCTSIZE_S"*6*2(%%"REG_S"), %%mm6 \n\t" //t1
|
|
15631
|
1269 "paddw %%mm7, %%mm5 \n\t" //t10
|
|
|
1270
|
|
15634
|
1271 "paddw "DCTSIZE_S"*5*2(%%"REG_S"), %%mm2 \n\t" //t2
|
|
15631
|
1272 "movq %%mm6, %%mm7 \n\t"
|
|
|
1273
|
|
|
1274 "paddw %%mm2, %%mm6 \n\t" //t11
|
|
|
1275 "psubw %%mm2, %%mm7 \n\t" //t12
|
|
|
1276
|
|
|
1277 "movq %%mm5, %%mm2 \n\t"
|
|
|
1278 "paddw %%mm6, %%mm5 \n\t" //d0
|
|
|
1279 // i0 t13 t12 i3 i1 d0 - d4
|
|
|
1280 "psubw %%mm6, %%mm2 \n\t" //d4
|
|
|
1281 "paddw %%mm1, %%mm7 \n\t"
|
|
|
1282
|
|
15634
|
1283 "movq 1*8+4*16(%%"REG_d"), %%mm6 \n\t"
|
|
15631
|
1284 "psllw $2, %%mm7 \n\t"
|
|
|
1285
|
|
15634
|
1286 "psubw 1*8+0*16(%%"REG_d"), %%mm5 \n\t"
|
|
15631
|
1287 "psubw %%mm6, %%mm2 \n\t"
|
|
|
1288
|
|
15634
|
1289 "paddusw 1*8+0*16(%%"REG_d"), %%mm5 \n\t"
|
|
15631
|
1290 "paddusw %%mm6, %%mm2 \n\t"
|
|
|
1291
|
|
|
1292 "pmulhw "MANGLE(MM_FIX_0_707106781)", %%mm7 \n\t"
|
|
|
1293 //
|
|
15634
|
1294 "paddw 1*8+0*16(%%"REG_d"), %%mm5 \n\t"
|
|
15631
|
1295 "paddw %%mm6, %%mm2 \n\t"
|
|
|
1296
|
|
15634
|
1297 "psubusw 1*8+0*16(%%"REG_d"), %%mm5 \n\t"
|
|
15631
|
1298 "psubusw %%mm6, %%mm2 \n\t"
|
|
|
1299
|
|
|
1300 //This func is totally compute-bound, operates at huge speed. So, DC shortcut
|
|
|
1301 // at this place isn't worthwhile due to BTB miss penalty (checked on Pent. 3).
|
|
|
1302 //However, typical numbers: nondc - 29%%, dc - 46%%, zero - 25%%. All <> 0 case is very rare.
|
|
|
1303 "paddw "MANGLE(MM_2)", %%mm5 \n\t"
|
|
|
1304 "movq %%mm2, %%mm6 \n\t"
|
|
|
1305
|
|
|
1306 "paddw %%mm5, %%mm2 \n\t"
|
|
|
1307 "psubw %%mm6, %%mm5 \n\t"
|
|
|
1308
|
|
|
1309 "movq %%mm1, %%mm6 \n\t"
|
|
|
1310 "paddw %%mm7, %%mm1 \n\t" //d2
|
|
|
1311
|
|
15634
|
1312 "psubw 1*8+2*16(%%"REG_d"), %%mm1 \n\t"
|
|
15631
|
1313 "psubw %%mm7, %%mm6 \n\t" //d6
|
|
|
1314
|
|
15634
|
1315 "movq 1*8+6*16(%%"REG_d"), %%mm7 \n\t"
|
|
15631
|
1316 "psraw $2, %%mm5 \n\t"
|
|
|
1317
|
|
15634
|
1318 "paddusw 1*8+2*16(%%"REG_d"), %%mm1 \n\t"
|
|
15631
|
1319 "psubw %%mm7, %%mm6 \n\t"
|
|
|
1320 // t7 d2 /t11 t4 t6 - d6 /t10
|
|
|
1321
|
|
15634
|
1322 "paddw 1*8+2*16(%%"REG_d"), %%mm1 \n\t"
|
|
15631
|
1323 "paddusw %%mm7, %%mm6 \n\t"
|
|
|
1324
|
|
15634
|
1325 "psubusw 1*8+2*16(%%"REG_d"), %%mm1 \n\t"
|
|
15631
|
1326 "paddw %%mm7, %%mm6 \n\t"
|
|
|
1327
|
|
15634
|
1328 "psubw "DCTSIZE_S"*4*2(%%"REG_S"), %%mm3 \n\t"
|
|
15631
|
1329 "psubusw %%mm7, %%mm6 \n\t"
|
|
|
1330
|
|
|
1331 //movq [edi+"DCTSIZE_S"*2*2], mm1
|
|
|
1332 //movq [edi+"DCTSIZE_S"*6*2], mm6
|
|
|
1333 "movq %%mm1, %%mm7 \n\t"
|
|
|
1334 "psraw $2, %%mm2 \n\t"
|
|
|
1335
|
|
15634
|
1336 "psubw "DCTSIZE_S"*6*2(%%"REG_S"), %%mm4 \n\t"
|
|
15631
|
1337 "psubw %%mm6, %%mm1 \n\t"
|
|
|
1338
|
|
15634
|
1339 "psubw "DCTSIZE_S"*7*2(%%"REG_S"), %%mm0 \n\t"
|
|
15631
|
1340 "paddw %%mm7, %%mm6 \n\t" //'t13
|
|
|
1341
|
|
|
1342 "psraw $2, %%mm6 \n\t" //paddw mm6, MM_2 !! ---
|
|
|
1343 "movq %%mm2, %%mm7 \n\t"
|
|
|
1344
|
|
|
1345 "pmulhw "MANGLE(MM_FIX_1_414213562_A)", %%mm1 \n\t"
|
|
|
1346 "paddw %%mm6, %%mm2 \n\t" //'t0
|
|
|
1347
|
|
|
1348 "movq %%mm2, "MANGLE(temps)"+0*8 \n\t" //!
|
|
|
1349 "psubw %%mm6, %%mm7 \n\t" //'t3
|
|
|
1350
|
|
15634
|
1351 "movq "DCTSIZE_S"*2*2(%%"REG_S"), %%mm2 \n\t"
|
|
15631
|
1352 "psubw %%mm6, %%mm1 \n\t" //'t12
|
|
|
1353
|
|
15634
|
1354 "psubw "DCTSIZE_S"*5*2(%%"REG_S"), %%mm2 \n\t" //t5
|
|
15631
|
1355 "movq %%mm5, %%mm6 \n\t"
|
|
|
1356
|
|
|
1357 "movq %%mm7, "MANGLE(temps)"+3*8 \n\t"
|
|
|
1358 "paddw %%mm2, %%mm3 \n\t" //t10
|
|
|
1359
|
|
|
1360 "paddw %%mm4, %%mm2 \n\t" //t11
|
|
|
1361 "paddw %%mm0, %%mm4 \n\t" //t12
|
|
|
1362
|
|
|
1363 "movq %%mm3, %%mm7 \n\t"
|
|
|
1364 "psubw %%mm4, %%mm3 \n\t"
|
|
|
1365
|
|
|
1366 "psllw $2, %%mm3 \n\t"
|
|
|
1367 "psllw $2, %%mm7 \n\t" //opt for P6
|
|
|
1368
|
|
|
1369 "pmulhw "MANGLE(MM_FIX_0_382683433)", %%mm3 \n\t"
|
|
|
1370 "psllw $2, %%mm4 \n\t"
|
|
|
1371
|
|
|
1372 "pmulhw "MANGLE(MM_FIX_0_541196100)", %%mm7 \n\t"
|
|
|
1373 "psllw $2, %%mm2 \n\t"
|
|
|
1374
|
|
|
1375 "pmulhw "MANGLE(MM_FIX_1_306562965)", %%mm4 \n\t"
|
|
|
1376 "paddw %%mm1, %%mm5 \n\t" //'t1
|
|
|
1377
|
|
|
1378 "pmulhw "MANGLE(MM_FIX_0_707106781)", %%mm2 \n\t"
|
|
|
1379 "psubw %%mm1, %%mm6 \n\t" //'t2
|
|
|
1380 // t7 't12 't11 t4 t6 - 't13 't10 ---
|
|
|
1381
|
|
|
1382 "paddw %%mm3, %%mm7 \n\t" //z2
|
|
|
1383
|
|
|
1384 "movq %%mm5, "MANGLE(temps)"+1*8 \n\t"
|
|
|
1385 "paddw %%mm3, %%mm4 \n\t" //z4
|
|
|
1386
|
|
15634
|
1387 "movq 1*8+3*16(%%"REG_d"), %%mm3 \n\t"
|
|
15631
|
1388 "movq %%mm0, %%mm1 \n\t"
|
|
|
1389
|
|
|
1390 "movq %%mm6, "MANGLE(temps)"+2*8 \n\t"
|
|
|
1391 "psubw %%mm2, %%mm1 \n\t" //z13
|
|
|
1392
|
|
|
1393 //===
|
|
|
1394 "paddw %%mm2, %%mm0 \n\t" //z11
|
|
|
1395 "movq %%mm1, %%mm5 \n\t"
|
|
|
1396
|
|
15634
|
1397 "movq 1*8+5*16(%%"REG_d"), %%mm2 \n\t"
|
|
15631
|
1398 "psubw %%mm7, %%mm1 \n\t" //d3
|
|
|
1399
|
|
|
1400 "paddw %%mm7, %%mm5 \n\t" //d5
|
|
|
1401 "psubw %%mm3, %%mm1 \n\t"
|
|
|
1402
|
|
15634
|
1403 "movq 1*8+1*16(%%"REG_d"), %%mm7 \n\t"
|
|
15631
|
1404 "psubw %%mm2, %%mm5 \n\t"
|
|
|
1405
|
|
|
1406 "movq %%mm0, %%mm6 \n\t"
|
|
|
1407 "paddw %%mm4, %%mm0 \n\t" //d1
|
|
|
1408
|
|
|
1409 "paddusw %%mm3, %%mm1 \n\t"
|
|
|
1410 "psubw %%mm4, %%mm6 \n\t" //d7
|
|
|
1411
|
|
|
1412 // d1 d3 - - - d5 d7 -
|
|
15634
|
1413 "movq 1*8+7*16(%%"REG_d"), %%mm4 \n\t"
|
|
15631
|
1414 "psubw %%mm7, %%mm0 \n\t"
|
|
|
1415
|
|
|
1416 "psubw %%mm4, %%mm6 \n\t"
|
|
|
1417 "paddusw %%mm2, %%mm5 \n\t"
|
|
|
1418
|
|
|
1419 "paddusw %%mm4, %%mm6 \n\t"
|
|
|
1420 "paddw %%mm3, %%mm1 \n\t"
|
|
|
1421
|
|
|
1422 "paddw %%mm2, %%mm5 \n\t"
|
|
|
1423 "paddw %%mm4, %%mm6 \n\t"
|
|
|
1424
|
|
|
1425 "psubusw %%mm3, %%mm1 \n\t"
|
|
|
1426 "psubusw %%mm2, %%mm5 \n\t"
|
|
|
1427
|
|
|
1428 "psubusw %%mm4, %%mm6 \n\t"
|
|
|
1429 "movq %%mm1, %%mm4 \n\t"
|
|
|
1430
|
|
|
1431 "por %%mm5, %%mm4 \n\t"
|
|
|
1432 "paddusw %%mm7, %%mm0 \n\t"
|
|
|
1433
|
|
|
1434 "por %%mm6, %%mm4 \n\t"
|
|
|
1435 "paddw %%mm7, %%mm0 \n\t"
|
|
|
1436
|
|
|
1437 "packssdw %%mm4, %%mm4 \n\t"
|
|
|
1438 "psubusw %%mm7, %%mm0 \n\t"
|
|
|
1439
|
|
15634
|
1440 "movd %%mm4, %%"REG_a" \n\t"
|
|
|
1441 "or %%"REG_a", %%"REG_a" \n\t"
|
|
15631
|
1442 "jnz 3f \n\t"
|
|
|
1443 //movq [edi+"DCTSIZE_S"*3*2], mm1
|
|
|
1444 //movq [edi+"DCTSIZE_S"*5*2], mm5
|
|
|
1445 //movq [edi+"DCTSIZE_S"*1*2], mm0
|
|
|
1446 //movq [edi+"DCTSIZE_S"*7*2], mm6
|
|
|
1447 // t4 t5 - - - t6 t7 -
|
|
|
1448 //--- t4 (mm0) may be <>0; mm1, mm5, mm6 == 0
|
|
|
1449 //Typical numbers: nondc - 19%%, dc - 26%%, zero - 55%%. zero case alone isn't worthwhile
|
|
|
1450 "movq "MANGLE(temps)"+0*8, %%mm4 \n\t"
|
|
|
1451 "movq %%mm0, %%mm1 \n\t"
|
|
|
1452
|
|
|
1453 "pmulhw "MANGLE(MM_FIX_0_847759065)", %%mm0 \n\t" //tmp6
|
|
|
1454 "movq %%mm1, %%mm2 \n\t"
|
|
|
1455
|
|
15634
|
1456 "movq "DCTSIZE_S"*0*2(%%"REG_D"), %%mm5 \n\t"
|
|
15631
|
1457 "movq %%mm2, %%mm3 \n\t"
|
|
|
1458
|
|
|
1459 "pmulhw "MANGLE(MM_FIX_0_566454497)", %%mm1 \n\t" //tmp5
|
|
|
1460 "paddw %%mm4, %%mm5 \n\t"
|
|
|
1461
|
|
|
1462 "movq "MANGLE(temps)"+1*8, %%mm6 \n\t"
|
|
|
1463 //paddw mm3, MM_2
|
|
|
1464 "psraw $2, %%mm3 \n\t" //tmp7
|
|
|
1465
|
|
|
1466 "pmulhw "MANGLE(MM_FIX_0_198912367)", %%mm2 \n\t" //-tmp4
|
|
|
1467 "psubw %%mm3, %%mm4 \n\t"
|
|
|
1468
|
|
15634
|
1469 "movq "DCTSIZE_S"*1*2(%%"REG_D"), %%mm7 \n\t"
|
|
15631
|
1470 "paddw %%mm3, %%mm5 \n\t"
|
|
|
1471
|
|
15634
|
1472 "movq %%mm4, "DCTSIZE_S"*7*2(%%"REG_D") \n\t"
|
|
15631
|
1473 "paddw %%mm6, %%mm7 \n\t"
|
|
|
1474
|
|
|
1475 "movq "MANGLE(temps)"+2*8, %%mm3 \n\t"
|
|
|
1476 "psubw %%mm0, %%mm6 \n\t"
|
|
|
1477
|
|
15634
|
1478 "movq "DCTSIZE_S"*2*2(%%"REG_D"), %%mm4 \n\t"
|
|
15631
|
1479 "paddw %%mm0, %%mm7 \n\t"
|
|
|
1480
|
|
15634
|
1481 "movq %%mm5, "DCTSIZE_S"*0*2(%%"REG_D") \n\t"
|
|
15631
|
1482 "paddw %%mm3, %%mm4 \n\t"
|
|
|
1483
|
|
15634
|
1484 "movq %%mm6, "DCTSIZE_S"*6*2(%%"REG_D") \n\t"
|
|
15631
|
1485 "psubw %%mm1, %%mm3 \n\t"
|
|
|
1486
|
|
15634
|
1487 "movq "DCTSIZE_S"*5*2(%%"REG_D"), %%mm5 \n\t"
|
|
15631
|
1488 "paddw %%mm1, %%mm4 \n\t"
|
|
|
1489
|
|
15634
|
1490 "movq "DCTSIZE_S"*3*2(%%"REG_D"), %%mm6 \n\t"
|
|
15631
|
1491 "paddw %%mm3, %%mm5 \n\t"
|
|
|
1492
|
|
|
1493 "movq "MANGLE(temps)"+3*8, %%mm0 \n\t"
|
|
15634
|
1494 "add $24, %%"REG_S" \n\t"
|
|
|
1495
|
|
|
1496 "movq %%mm7, "DCTSIZE_S"*1*2(%%"REG_D") \n\t"
|
|
15631
|
1497 "paddw %%mm0, %%mm6 \n\t"
|
|
|
1498
|
|
15634
|
1499 "movq %%mm4, "DCTSIZE_S"*2*2(%%"REG_D") \n\t"
|
|
15631
|
1500 "psubw %%mm2, %%mm0 \n\t"
|
|
|
1501
|
|
15634
|
1502 "movq "DCTSIZE_S"*4*2(%%"REG_D"), %%mm7 \n\t"
|
|
15631
|
1503 "paddw %%mm2, %%mm6 \n\t"
|
|
|
1504
|
|
15634
|
1505 "movq %%mm5, "DCTSIZE_S"*5*2(%%"REG_D") \n\t"
|
|
15631
|
1506 "paddw %%mm0, %%mm7 \n\t"
|
|
|
1507
|
|
15634
|
1508 "movq %%mm6, "DCTSIZE_S"*3*2(%%"REG_D") \n\t"
|
|
|
1509
|
|
|
1510 "movq %%mm7, "DCTSIZE_S"*4*2(%%"REG_D") \n\t"
|
|
|
1511 "add $24, %%"REG_D" \n\t"
|
|
|
1512 "sub $2, %%"REG_c" \n\t"
|
|
15631
|
1513 "jnz 1b \n\t"
|
|
|
1514 "jmp 5f \n\t"
|
|
|
1515
|
|
|
1516 "3: \n\t"
|
|
|
1517 //--- non DC2
|
|
|
1518 //psraw mm1, 2 w/o it -> offset. thr1, thr1, thr1 (actually thr1, thr1, thr1-1)
|
|
|
1519 //psraw mm5, 2
|
|
|
1520 //psraw mm0, 2
|
|
|
1521 //psraw mm6, 2
|
|
|
1522 "movq %%mm5, %%mm3 \n\t"
|
|
|
1523 "psubw %%mm1, %%mm5 \n\t"
|
|
|
1524
|
|
|
1525 "psllw $1, %%mm5 \n\t" //'z10
|
|
|
1526 "paddw %%mm1, %%mm3 \n\t" //'z13
|
|
|
1527
|
|
|
1528 "movq %%mm0, %%mm2 \n\t"
|
|
|
1529 "psubw %%mm6, %%mm0 \n\t"
|
|
|
1530
|
|
|
1531 "movq %%mm5, %%mm1 \n\t"
|
|
|
1532 "psllw $1, %%mm0 \n\t" //'z12
|
|
|
1533
|
|
|
1534 "pmulhw "MANGLE(MM_FIX_2_613125930)", %%mm1 \n\t" //-
|
|
|
1535 "paddw %%mm0, %%mm5 \n\t"
|
|
|
1536
|
|
|
1537 "pmulhw "MANGLE(MM_FIX_1_847759065)", %%mm5 \n\t" //'z5
|
|
|
1538 "paddw %%mm6, %%mm2 \n\t" //'z11
|
|
|
1539
|
|
|
1540 "pmulhw "MANGLE(MM_FIX_1_082392200)", %%mm0 \n\t"
|
|
|
1541 "movq %%mm2, %%mm7 \n\t"
|
|
|
1542
|
|
|
1543 //---
|
|
|
1544 "movq "MANGLE(temps)"+0*8, %%mm4 \n\t"
|
|
|
1545 "psubw %%mm3, %%mm2 \n\t"
|
|
|
1546
|
|
|
1547 "psllw $1, %%mm2 \n\t"
|
|
|
1548 "paddw %%mm3, %%mm7 \n\t" //'t7
|
|
|
1549
|
|
|
1550 "pmulhw "MANGLE(MM_FIX_1_414213562)", %%mm2 \n\t" //'t11
|
|
|
1551 "movq %%mm4, %%mm6 \n\t"
|
|
|
1552 //paddw mm7, MM_2
|
|
|
1553 "psraw $2, %%mm7 \n\t"
|
|
|
1554
|
|
15634
|
1555 "paddw "DCTSIZE_S"*0*2(%%"REG_D"), %%mm4 \n\t"
|
|
15631
|
1556 "psubw %%mm7, %%mm6 \n\t"
|
|
|
1557
|
|
|
1558 "movq "MANGLE(temps)"+1*8, %%mm3 \n\t"
|
|
|
1559 "paddw %%mm7, %%mm4 \n\t"
|
|
|
1560
|
|
15634
|
1561 "movq %%mm6, "DCTSIZE_S"*7*2(%%"REG_D") \n\t"
|
|
15631
|
1562 "paddw %%mm5, %%mm1 \n\t" //'t12
|
|
|
1563
|
|
15634
|
1564 "movq %%mm4, "DCTSIZE_S"*0*2(%%"REG_D") \n\t"
|
|
15631
|
1565 "psubw %%mm7, %%mm1 \n\t" //'t6
|
|
|
1566
|
|
|
1567 "movq "MANGLE(temps)"+2*8, %%mm7 \n\t"
|
|
|
1568 "psubw %%mm5, %%mm0 \n\t" //'t10
|
|
|
1569
|
|
|
1570 "movq "MANGLE(temps)"+3*8, %%mm6 \n\t"
|
|
|
1571 "movq %%mm3, %%mm5 \n\t"
|
|
|
1572
|
|
15634
|
1573 "paddw "DCTSIZE_S"*1*2(%%"REG_D"), %%mm3 \n\t"
|
|
15631
|
1574 "psubw %%mm1, %%mm5 \n\t"
|
|
|
1575
|
|
|
1576 "psubw %%mm1, %%mm2 \n\t" //'t5
|
|
|
1577 "paddw %%mm1, %%mm3 \n\t"
|
|
|
1578
|
|
15634
|
1579 "movq %%mm5, "DCTSIZE_S"*6*2(%%"REG_D") \n\t"
|
|
15631
|
1580 "movq %%mm7, %%mm4 \n\t"
|
|
|
1581
|
|
15634
|
1582 "paddw "DCTSIZE_S"*2*2(%%"REG_D"), %%mm7 \n\t"
|
|
15631
|
1583 "psubw %%mm2, %%mm4 \n\t"
|
|
|
1584
|
|
15634
|
1585 "paddw "DCTSIZE_S"*5*2(%%"REG_D"), %%mm4 \n\t"
|
|
15631
|
1586 "paddw %%mm2, %%mm7 \n\t"
|
|
|
1587
|
|
15634
|
1588 "movq %%mm3, "DCTSIZE_S"*1*2(%%"REG_D") \n\t"
|
|
15631
|
1589 "paddw %%mm2, %%mm0 \n\t" //'t4
|
|
|
1590
|
|
|
1591 // 't4 't6 't5 - - - - 't7
|
|
15634
|
1592 "movq %%mm7, "DCTSIZE_S"*2*2(%%"REG_D") \n\t"
|
|
15631
|
1593 "movq %%mm6, %%mm1 \n\t"
|
|
|
1594
|
|
15634
|
1595 "paddw "DCTSIZE_S"*4*2(%%"REG_D"), %%mm6 \n\t"
|
|
15631
|
1596 "psubw %%mm0, %%mm1 \n\t"
|
|
|
1597
|
|
15634
|
1598 "paddw "DCTSIZE_S"*3*2(%%"REG_D"), %%mm1 \n\t"
|
|
15631
|
1599 "paddw %%mm0, %%mm6 \n\t"
|
|
|
1600
|
|
15634
|
1601 "movq %%mm4, "DCTSIZE_S"*5*2(%%"REG_D") \n\t"
|
|
|
1602 "add $24, %%"REG_S" \n\t"
|
|
|
1603
|
|
|
1604 "movq %%mm6, "DCTSIZE_S"*4*2(%%"REG_D") \n\t"
|
|
|
1605
|
|
|
1606 "movq %%mm1, "DCTSIZE_S"*3*2(%%"REG_D") \n\t"
|
|
|
1607 "add $24, %%"REG_D" \n\t"
|
|
|
1608 "sub $2, %%"REG_c" \n\t"
|
|
15631
|
1609 "jnz 1b \n\t"
|
|
|
1610 "5: \n\t"
|
|
|
1611
|
|
15632
|
1612 : "+S"(data), "+D"(output), "+c"(cnt)// input regs
|
|
|
1613 : "d"(thr_adr)
|
|
15634
|
1614 : "%"REG_a
|
|
15631
|
1615 );
|
|
|
1616 }
|
|
|
1617
|
|
|
1618 #endif // HAVE_MMX
|
|
|
1619
|
|
|
1620 #ifndef HAVE_MMX
|
|
|
1621
|
|
|
1622 static void row_idct_c(DCTELEM* workspace,
|
|
|
1623 int16_t* output_adr, int output_stride, int cnt)
|
|
|
1624 {
|
|
|
1625 int_simd16_t tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
|
|
|
1626 int_simd16_t tmp10, tmp11, tmp12, tmp13;
|
|
|
1627 int_simd16_t z5, z10, z11, z12, z13;
|
|
|
1628 int16_t* outptr;
|
|
|
1629 DCTELEM* wsptr;
|
|
|
1630
|
|
|
1631 cnt*=4;
|
|
|
1632 wsptr = workspace;
|
|
|
1633 outptr = output_adr;
|
|
|
1634 for (; cnt > 0; cnt--) {
|
|
|
1635 // Even part
|
|
|
1636 //Simd version reads 4x4 block and transposes it
|
|
|
1637 tmp10 = ( wsptr[2] + wsptr[3]);
|
|
|
1638 tmp11 = ( wsptr[2] - wsptr[3]);
|
|
|
1639
|
|
|
1640 tmp13 = ( wsptr[0] + wsptr[1]);
|
|
|
1641 tmp12 = (MULTIPLY16H( wsptr[0] - wsptr[1], FIX_1_414213562_A)<<2) - tmp13;//this shift order to avoid overflow
|
|
|
1642
|
|
|
1643 tmp0 = tmp10 + tmp13; //->temps
|
|
|
1644 tmp3 = tmp10 - tmp13; //->temps
|
|
|
1645 tmp1 = tmp11 + tmp12;
|
|
|
1646 tmp2 = tmp11 - tmp12;
|
|
|
1647
|
|
|
1648 // Odd part
|
|
|
1649 //Also transpose, with previous:
|
|
|
1650 // ---- ---- ||||
|
|
|
1651 // ---- ---- idct ||||
|
|
|
1652 // ---- ---- ---> ||||
|
|
|
1653 // ---- ---- ||||
|
|
|
1654 z13 = wsptr[4] + wsptr[5];
|
|
|
1655 z10 = wsptr[4] - wsptr[5];
|
|
|
1656 z11 = wsptr[6] + wsptr[7];
|
|
|
1657 z12 = wsptr[6] - wsptr[7];
|
|
|
1658
|
|
|
1659 tmp7 = z11 + z13;
|
|
|
1660 tmp11 = MULTIPLY16H(z11 - z13, FIX_1_414213562);
|
|
|
1661
|
|
|
1662 z5 = MULTIPLY16H(z10 + z12, FIX_1_847759065);
|
|
|
1663 tmp10 = MULTIPLY16H(z12, FIX_1_082392200) - z5;
|
|
|
1664 tmp12 = MULTIPLY16H(z10, FIX_2_613125930) + z5; // - FIX_
|
|
|
1665
|
|
|
1666 tmp6 = (tmp12<<3) - tmp7;
|
|
|
1667 tmp5 = (tmp11<<3) - tmp6;
|
|
|
1668 tmp4 = (tmp10<<3) + tmp5;
|
|
|
1669
|
|
|
1670 // Final output stage: descale and write column
|
|
|
1671 outptr[0*output_stride]+= DESCALE(tmp0 + tmp7, 3);
|
|
|
1672 outptr[1*output_stride]+= DESCALE(tmp1 + tmp6, 3);
|
|
|
1673 outptr[2*output_stride]+= DESCALE(tmp2 + tmp5, 3);
|
|
|
1674 outptr[3*output_stride]+= DESCALE(tmp3 - tmp4, 3);
|
|
|
1675 outptr[4*output_stride]+= DESCALE(tmp3 + tmp4, 3);
|
|
|
1676 outptr[5*output_stride]+= DESCALE(tmp2 - tmp5, 3);
|
|
|
1677 outptr[6*output_stride]+= DESCALE(tmp1 - tmp6, 3); //no += ?
|
|
|
1678 outptr[7*output_stride]+= DESCALE(tmp0 - tmp7, 3); //no += ?
|
|
|
1679 outptr++;
|
|
|
1680
|
|
|
1681 wsptr += DCTSIZE; // advance pointer to next row
|
|
|
1682 }
|
|
|
1683 }
|
|
|
1684
|
|
|
1685 #else /* HAVE_MMX */
|
|
|
1686
|
|
|
1687 static void row_idct_mmx (DCTELEM* workspace,
|
|
|
1688 int16_t* output_adr, int output_stride, int cnt)
|
|
|
1689 {
|
|
|
1690 asm volatile(
|
|
15634
|
1691 "lea (%%"REG_a",%%"REG_a",2), %%"REG_d" \n\t"
|
|
15631
|
1692 "1: \n\t"
|
|
15634
|
1693 "movq "DCTSIZE_S"*0*2(%%"REG_S"), %%mm0 \n\t"
|
|
15631
|
1694 //
|
|
|
1695
|
|
15634
|
1696 "movq "DCTSIZE_S"*1*2(%%"REG_S"), %%mm1 \n\t"
|
|
15631
|
1697 "movq %%mm0, %%mm4 \n\t"
|
|
|
1698
|
|
15634
|
1699 "movq "DCTSIZE_S"*2*2(%%"REG_S"), %%mm2 \n\t"
|
|
15631
|
1700 "punpcklwd %%mm1, %%mm0 \n\t"
|
|
|
1701
|
|
15634
|
1702 "movq "DCTSIZE_S"*3*2(%%"REG_S"), %%mm3 \n\t"
|
|
15631
|
1703 "punpckhwd %%mm1, %%mm4 \n\t"
|
|
|
1704
|
|
|
1705 //transpose 4x4
|
|
|
1706 "movq %%mm2, %%mm7 \n\t"
|
|
|
1707 "punpcklwd %%mm3, %%mm2 \n\t"
|
|
|
1708
|
|
|
1709 "movq %%mm0, %%mm6 \n\t"
|
|
|
1710 "punpckldq %%mm2, %%mm0 \n\t" //0
|
|
|
1711
|
|
|
1712 "punpckhdq %%mm2, %%mm6 \n\t" //1
|
|
|
1713 "movq %%mm0, %%mm5 \n\t"
|
|
|
1714
|
|
|
1715 "punpckhwd %%mm3, %%mm7 \n\t"
|
|
|
1716 "psubw %%mm6, %%mm0 \n\t"
|
|
|
1717
|
|
|
1718 "pmulhw "MANGLE(MM_FIX_1_414213562_A)", %%mm0 \n\t"
|
|
|
1719 "movq %%mm4, %%mm2 \n\t"
|
|
|
1720
|
|
|
1721 "punpckldq %%mm7, %%mm4 \n\t" //2
|
|
|
1722 "paddw %%mm6, %%mm5 \n\t"
|
|
|
1723
|
|
|
1724 "punpckhdq %%mm7, %%mm2 \n\t" //3
|
|
|
1725 "movq %%mm4, %%mm1 \n\t"
|
|
|
1726
|
|
|
1727 "psllw $2, %%mm0 \n\t"
|
|
|
1728 "paddw %%mm2, %%mm4 \n\t" //t10
|
|
|
1729
|
|
15634
|
1730 "movq "DCTSIZE_S"*0*2+"DCTSIZE_S"(%%"REG_S"), %%mm3 \n\t"
|
|
15631
|
1731 "psubw %%mm2, %%mm1 \n\t" //t11
|
|
|
1732
|
|
15634
|
1733 "movq "DCTSIZE_S"*1*2+"DCTSIZE_S"(%%"REG_S"), %%mm2 \n\t"
|
|
15631
|
1734 "psubw %%mm5, %%mm0 \n\t"
|
|
|
1735
|
|
|
1736 "movq %%mm4, %%mm6 \n\t"
|
|
|
1737 "paddw %%mm5, %%mm4 \n\t" //t0
|
|
|
1738
|
|
|
1739 "psubw %%mm5, %%mm6 \n\t" //t3
|
|
|
1740 "movq %%mm1, %%mm7 \n\t"
|
|
|
1741
|
|
15634
|
1742 "movq "DCTSIZE_S"*2*2+"DCTSIZE_S"(%%"REG_S"), %%mm5 \n\t"
|
|
15631
|
1743 "paddw %%mm0, %%mm1 \n\t" //t1
|
|
|
1744
|
|
|
1745 "movq %%mm4, "MANGLE(temps)"+0*8 \n\t" //t0
|
|
|
1746 "movq %%mm3, %%mm4 \n\t"
|
|
|
1747
|
|
|
1748 "movq %%mm6, "MANGLE(temps)"+1*8 \n\t" //t3
|
|
|
1749 "punpcklwd %%mm2, %%mm3 \n\t"
|
|
|
1750
|
|
|
1751 //transpose 4x4
|
|
15634
|
1752 "movq "DCTSIZE_S"*3*2+"DCTSIZE_S"(%%"REG_S"), %%mm6 \n\t"
|
|
15631
|
1753 "punpckhwd %%mm2, %%mm4 \n\t"
|
|
|
1754
|
|
|
1755 "movq %%mm5, %%mm2 \n\t"
|
|
|
1756 "punpcklwd %%mm6, %%mm5 \n\t"
|
|
|
1757
|
|
|
1758 "psubw %%mm0, %%mm7 \n\t" //t2
|
|
|
1759 "punpckhwd %%mm6, %%mm2 \n\t"
|
|
|
1760
|
|
|
1761 "movq %%mm3, %%mm0 \n\t"
|
|
|
1762 "punpckldq %%mm5, %%mm3 \n\t" //4
|
|
|
1763
|
|
|
1764 "punpckhdq %%mm5, %%mm0 \n\t" //5
|
|
|
1765 "movq %%mm4, %%mm5 \n\t"
|
|
|
1766
|
|
|
1767 //
|
|
|
1768 "movq %%mm3, %%mm6 \n\t"
|
|
|
1769 "punpckldq %%mm2, %%mm4 \n\t" //6
|
|
|
1770
|
|
|
1771 "psubw %%mm0, %%mm3 \n\t" //z10
|
|
|
1772 "punpckhdq %%mm2, %%mm5 \n\t" //7
|
|
|
1773
|
|
|
1774 "paddw %%mm0, %%mm6 \n\t" //z13
|
|
|
1775 "movq %%mm4, %%mm2 \n\t"
|
|
|
1776
|
|
|
1777 "movq %%mm3, %%mm0 \n\t"
|
|
|
1778 "psubw %%mm5, %%mm4 \n\t" //z12
|
|
|
1779
|
|
|
1780 "pmulhw "MANGLE(MM_FIX_2_613125930)", %%mm0 \n\t" //-
|
|
|
1781 "paddw %%mm4, %%mm3 \n\t"
|
|
|
1782
|
|
|
1783 "pmulhw "MANGLE(MM_FIX_1_847759065)", %%mm3 \n\t" //z5
|
|
|
1784 "paddw %%mm5, %%mm2 \n\t" //z11 >
|
|
|
1785
|
|
|
1786 "pmulhw "MANGLE(MM_FIX_1_082392200)", %%mm4 \n\t"
|
|
|
1787 "movq %%mm2, %%mm5 \n\t"
|
|
|
1788
|
|
|
1789 "psubw %%mm6, %%mm2 \n\t"
|
|
|
1790 "paddw %%mm6, %%mm5 \n\t" //t7
|
|
|
1791
|
|
|
1792 "pmulhw "MANGLE(MM_FIX_1_414213562)", %%mm2 \n\t" //t11
|
|
|
1793 "paddw %%mm3, %%mm0 \n\t" //t12
|
|
|
1794
|
|
|
1795 "psllw $3, %%mm0 \n\t"
|
|
|
1796 "psubw %%mm3, %%mm4 \n\t" //t10
|
|
|
1797
|
|
|
1798 "movq "MANGLE(temps)"+0*8, %%mm6 \n\t"
|
|
|
1799 "movq %%mm1, %%mm3 \n\t"
|
|
|
1800
|
|
|
1801 "psllw $3, %%mm4 \n\t"
|
|
|
1802 "psubw %%mm5, %%mm0 \n\t" //t6
|
|
|
1803
|
|
|
1804 "psllw $3, %%mm2 \n\t"
|
|
|
1805 "paddw %%mm0, %%mm1 \n\t" //d1
|
|
|
1806
|
|
|
1807 "psubw %%mm0, %%mm2 \n\t" //t5
|
|
|
1808 "psubw %%mm0, %%mm3 \n\t" //d6
|
|
|
1809
|
|
|
1810 "paddw %%mm2, %%mm4 \n\t" //t4
|
|
|
1811 "movq %%mm7, %%mm0 \n\t"
|
|
|
1812
|
|
|
1813 "paddw %%mm2, %%mm7 \n\t" //d2
|
|
|
1814 "psubw %%mm2, %%mm0 \n\t" //d5
|
|
|
1815
|
|
|
1816 "movq "MANGLE(MM_DESCALE_RND)", %%mm2 \n\t" //4
|
|
|
1817 "psubw %%mm5, %%mm6 \n\t" //d7
|
|
|
1818
|
|
|
1819 "paddw "MANGLE(temps)"+0*8, %%mm5 \n\t" //d0
|
|
|
1820 "paddw %%mm2, %%mm1 \n\t"
|
|
|
1821
|
|
|
1822 "paddw %%mm2, %%mm5 \n\t"
|
|
|
1823 "psraw $3, %%mm1 \n\t"
|
|
|
1824
|
|
|
1825 "paddw %%mm2, %%mm7 \n\t"
|
|
|
1826 "psraw $3, %%mm5 \n\t"
|
|
|
1827
|
|
15634
|
1828 "paddw (%%"REG_D"), %%mm5 \n\t"
|
|
15631
|
1829 "psraw $3, %%mm7 \n\t"
|
|
|
1830
|
|
15634
|
1831 "paddw (%%"REG_D",%%"REG_a",), %%mm1 \n\t"
|
|
15631
|
1832 "paddw %%mm2, %%mm0 \n\t"
|
|
|
1833
|
|
15634
|
1834 "paddw (%%"REG_D",%%"REG_a",2), %%mm7 \n\t"
|
|
15631
|
1835 "paddw %%mm2, %%mm3 \n\t"
|
|
|
1836
|
|
15634
|
1837 "movq %%mm5, (%%"REG_D") \n\t"
|
|
15631
|
1838 "paddw %%mm2, %%mm6 \n\t"
|
|
|
1839
|
|
15634
|
1840 "movq %%mm1, (%%"REG_D",%%"REG_a",) \n\t"
|
|
15631
|
1841 "psraw $3, %%mm0 \n\t"
|
|
|
1842
|
|
15634
|
1843 "movq %%mm7, (%%"REG_D",%%"REG_a",2) \n\t"
|
|
|
1844 "add %%"REG_d", %%"REG_D" \n\t" //3*ls
|
|
15631
|
1845
|
|
|
1846 "movq "MANGLE(temps)"+1*8, %%mm5 \n\t" //t3
|
|
|
1847 "psraw $3, %%mm3 \n\t"
|
|
|
1848
|
|
15634
|
1849 "paddw (%%"REG_D",%%"REG_a",2), %%mm0 \n\t"
|
|
15631
|
1850 "psubw %%mm4, %%mm5 \n\t" //d3
|
|
|
1851
|
|
15634
|
1852 "paddw (%%"REG_D",%%"REG_d",), %%mm3 \n\t"
|
|
15631
|
1853 "psraw $3, %%mm6 \n\t"
|
|
|
1854
|
|
|
1855 "paddw "MANGLE(temps)"+1*8, %%mm4 \n\t" //d4
|
|
|
1856 "paddw %%mm2, %%mm5 \n\t"
|
|
|
1857
|
|
15634
|
1858 "paddw (%%"REG_D",%%"REG_a",4), %%mm6 \n\t"
|
|
15631
|
1859 "paddw %%mm2, %%mm4 \n\t"
|
|
|
1860
|
|
15634
|
1861 "movq %%mm0, (%%"REG_D",%%"REG_a",2) \n\t"
|
|
15631
|
1862 "psraw $3, %%mm5 \n\t"
|
|
|
1863
|
|
15634
|
1864 "paddw (%%"REG_D"), %%mm5 \n\t"
|
|
15631
|
1865 "psraw $3, %%mm4 \n\t"
|
|
|
1866
|
|
15634
|
1867 "paddw (%%"REG_D",%%"REG_a",), %%mm4 \n\t"
|
|
|
1868 "add $"DCTSIZE_S"*2*4, %%"REG_S" \n\t" //4 rows
|
|
|
1869
|
|
|
1870 "movq %%mm3, (%%"REG_D",%%"REG_d",) \n\t"
|
|
|
1871 "movq %%mm6, (%%"REG_D",%%"REG_a",4) \n\t"
|
|
|
1872 "movq %%mm5, (%%"REG_D") \n\t"
|
|
|
1873 "movq %%mm4, (%%"REG_D",%%"REG_a",) \n\t"
|
|
|
1874
|
|
|
1875 "sub %%"REG_d", %%"REG_D" \n\t"
|
|
|
1876 "add $8, %%"REG_D" \n\t"
|
|
|
1877 "dec %%"REG_c" \n\t"
|
|
15631
|
1878 "jnz 1b \n\t"
|
|
|
1879
|
|
|
1880 : "+S"(workspace), "+D"(output_adr), "+c"(cnt) //input regs
|
|
|
1881 : "a"(output_stride*sizeof(short))
|
|
15634
|
1882 : "%"REG_d
|
|
15631
|
1883 );
|
|
|
1884 }
|
|
|
1885
|
|
|
1886 #endif // HAVE_MMX
|
|
|
1887
|
|
|
1888 #ifndef HAVE_MMX
|
|
|
1889
|
|
|
1890 static void row_fdct_c(DCTELEM *data, const uint8_t *pixels, int line_size, int cnt)
|
|
|
1891 {
|
|
|
1892 int_simd16_t tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
|
|
|
1893 int_simd16_t tmp10, tmp11, tmp12, tmp13;
|
|
|
1894 int_simd16_t z1, z2, z3, z4, z5, z11, z13;
|
|
|
1895 DCTELEM *dataptr;
|
|
|
1896
|
|
|
1897 cnt*=4;
|
|
|
1898 // Pass 1: process rows.
|
|
|
1899
|
|
|
1900 dataptr = data;
|
|
|
1901 for (; cnt > 0; cnt--) {
|
|
|
1902 tmp0 = pixels[line_size*0] + pixels[line_size*7];
|
|
|
1903 tmp7 = pixels[line_size*0] - pixels[line_size*7];
|
|
|
1904 tmp1 = pixels[line_size*1] + pixels[line_size*6];
|
|
|
1905 tmp6 = pixels[line_size*1] - pixels[line_size*6];
|
|
|
1906 tmp2 = pixels[line_size*2] + pixels[line_size*5];
|
|
|
1907 tmp5 = pixels[line_size*2] - pixels[line_size*5];
|
|
|
1908 tmp3 = pixels[line_size*3] + pixels[line_size*4];
|
|
|
1909 tmp4 = pixels[line_size*3] - pixels[line_size*4];
|
|
|
1910
|
|
|
1911 // Even part
|
|
|
1912
|
|
|
1913 tmp10 = tmp0 + tmp3;
|
|
|
1914 tmp13 = tmp0 - tmp3;
|
|
|
1915 tmp11 = tmp1 + tmp2;
|
|
|
1916 tmp12 = tmp1 - tmp2;
|
|
|
1917 //Even columns are written first, this leads to different order of columns
|
|
|
1918 //in column_fidct(), but they are processed independently, so all ok.
|
|
|
1919 //Later in the row_idct() columns readed at the same order.
|
|
|
1920 dataptr[2] = tmp10 + tmp11;
|
|
|
1921 dataptr[3] = tmp10 - tmp11;
|
|
|
1922
|
|
|
1923 z1 = MULTIPLY16H((tmp12 + tmp13)<<2, FIX_0_707106781);
|
|
|
1924 dataptr[0] = tmp13 + z1;
|
|
|
1925 dataptr[1] = tmp13 - z1;
|
|
|
1926
|
|
|
1927 // Odd part
|
|
|
1928
|
|
|
1929 tmp10 = (tmp4 + tmp5) <<2;
|
|
|
1930 tmp11 = (tmp5 + tmp6) <<2;
|
|
|
1931 tmp12 = (tmp6 + tmp7) <<2;
|
|
|
1932
|
|
|
1933 z5 = MULTIPLY16H(tmp10 - tmp12, FIX_0_382683433);
|
|
|
1934 z2 = MULTIPLY16H(tmp10, FIX_0_541196100) + z5;
|
|
|
1935 z4 = MULTIPLY16H(tmp12, FIX_1_306562965) + z5;
|
|
|
1936 z3 = MULTIPLY16H(tmp11, FIX_0_707106781);
|
|
|
1937
|
|
|
1938 z11 = tmp7 + z3;
|
|
|
1939 z13 = tmp7 - z3;
|
|
|
1940
|
|
|
1941 dataptr[4] = z13 + z2;
|
|
|
1942 dataptr[5] = z13 - z2;
|
|
|
1943 dataptr[6] = z11 + z4;
|
|
|
1944 dataptr[7] = z11 - z4;
|
|
|
1945
|
|
|
1946 pixels++; // advance pointer to next column
|
|
|
1947 dataptr += DCTSIZE;
|
|
|
1948 }
|
|
|
1949 }
|
|
|
1950
|
|
|
1951 #else /* HAVE_MMX */
|
|
|
1952
|
|
|
1953 static void row_fdct_mmx(DCTELEM *data, const uint8_t *pixels, int line_size, int cnt)
|
|
|
1954 {
|
|
|
1955 asm volatile(
|
|
15634
|
1956 "lea (%%"REG_a",%%"REG_a",2), %%"REG_d" \n\t"
|
|
15631
|
1957 "6: \n\t"
|
|
15634
|
1958 "movd (%%"REG_S"), %%mm0 \n\t"
|
|
15631
|
1959 "pxor %%mm7, %%mm7 \n\t"
|
|
|
1960
|
|
15634
|
1961 "movd (%%"REG_S",%%"REG_a",), %%mm1 \n\t"
|
|
15631
|
1962 "punpcklbw %%mm7, %%mm0 \n\t"
|
|
|
1963
|
|
15634
|
1964 "movd (%%"REG_S",%%"REG_a",2), %%mm2 \n\t"
|
|
15631
|
1965 "punpcklbw %%mm7, %%mm1 \n\t"
|
|
|
1966
|
|
|
1967 "punpcklbw %%mm7, %%mm2 \n\t"
|
|
15634
|
1968 "add %%"REG_d", %%"REG_S" \n\t"
|
|
15631
|
1969
|
|
|
1970 "movq %%mm0, %%mm5 \n\t"
|
|
|
1971 //
|
|
|
1972
|
|
15634
|
1973 "movd (%%"REG_S",%%"REG_a",4), %%mm3 \n\t" //7 ;prefetch!
|
|
15631
|
1974 "movq %%mm1, %%mm6 \n\t"
|
|
|
1975
|
|
15634
|
1976 "movd (%%"REG_S",%%"REG_d",), %%mm4 \n\t" //6
|
|
15631
|
1977 "punpcklbw %%mm7, %%mm3 \n\t"
|
|
|
1978
|
|
|
1979 "psubw %%mm3, %%mm5 \n\t"
|
|
|
1980 "punpcklbw %%mm7, %%mm4 \n\t"
|
|
|
1981
|
|
|
1982 "paddw %%mm3, %%mm0 \n\t"
|
|
|
1983 "psubw %%mm4, %%mm6 \n\t"
|
|
|
1984
|
|
15634
|
1985 "movd (%%"REG_S",%%"REG_a",2), %%mm3 \n\t" //5
|
|
15631
|
1986 "paddw %%mm4, %%mm1 \n\t"
|
|
|
1987
|
|
|
1988 "movq %%mm5, "MANGLE(temps)"+0*8 \n\t" //t7
|
|
|
1989 "punpcklbw %%mm7, %%mm3 \n\t"
|
|
|
1990
|
|
|
1991 "movq %%mm6, "MANGLE(temps)"+1*8 \n\t" //t6
|
|
|
1992 "movq %%mm2, %%mm4 \n\t"
|
|
|
1993
|
|
15634
|
1994 "movd (%%"REG_S"), %%mm5 \n\t" //3
|
|
15631
|
1995 "paddw %%mm3, %%mm2 \n\t"
|
|
|
1996
|
|
15634
|
1997 "movd (%%"REG_S",%%"REG_a",), %%mm6 \n\t" //4
|
|
15631
|
1998 "punpcklbw %%mm7, %%mm5 \n\t"
|
|
|
1999
|
|
|
2000 "psubw %%mm3, %%mm4 \n\t"
|
|
|
2001 "punpcklbw %%mm7, %%mm6 \n\t"
|
|
|
2002
|
|
|
2003 "movq %%mm5, %%mm3 \n\t"
|
|
|
2004 "paddw %%mm6, %%mm5 \n\t" //t3
|
|
|
2005
|
|
|
2006 "psubw %%mm6, %%mm3 \n\t" //t4 ; t0 t1 t2 t4 t5 t3 - -
|
|
|
2007 "movq %%mm0, %%mm6 \n\t"
|
|
|
2008
|
|
|
2009 "movq %%mm1, %%mm7 \n\t"
|
|
|
2010 "psubw %%mm5, %%mm0 \n\t" //t13
|
|
|
2011
|
|
|
2012 "psubw %%mm2, %%mm1 \n\t"
|
|
|
2013 "paddw %%mm2, %%mm7 \n\t" //t11
|
|
|
2014
|
|
|
2015 "paddw %%mm0, %%mm1 \n\t"
|
|
|
2016 "movq %%mm7, %%mm2 \n\t"
|
|
|
2017
|
|
|
2018 "psllw $2, %%mm1 \n\t"
|
|
|
2019 "paddw %%mm5, %%mm6 \n\t" //t10
|
|
|
2020
|
|
|
2021 "pmulhw "MANGLE(MM_FIX_0_707106781)", %%mm1 \n\t"
|
|
|
2022 "paddw %%mm6, %%mm7 \n\t" //d2
|
|
|
2023
|
|
|
2024 "psubw %%mm2, %%mm6 \n\t" //d3
|
|
|
2025 "movq %%mm0, %%mm5 \n\t"
|
|
|
2026
|
|
|
2027 //transpose 4x4
|
|
|
2028 "movq %%mm7, %%mm2 \n\t"
|
|
|
2029 "punpcklwd %%mm6, %%mm7 \n\t"
|
|
|
2030
|
|
|
2031 "paddw %%mm1, %%mm0 \n\t" //d0
|
|
|
2032 "punpckhwd %%mm6, %%mm2 \n\t"
|
|
|
2033
|
|
|
2034 "psubw %%mm1, %%mm5 \n\t" //d1
|
|
|
2035 "movq %%mm0, %%mm6 \n\t"
|
|
|
2036
|
|
|
2037 "movq "MANGLE(temps)"+1*8, %%mm1 \n\t"
|
|
|
2038 "punpcklwd %%mm5, %%mm0 \n\t"
|
|
|
2039
|
|
|
2040 "punpckhwd %%mm5, %%mm6 \n\t"
|
|
|
2041 "movq %%mm0, %%mm5 \n\t"
|
|
|
2042
|
|
|
2043 "punpckldq %%mm7, %%mm0 \n\t" //0
|
|
|
2044 "paddw %%mm4, %%mm3 \n\t"
|
|
|
2045
|
|
|
2046 "punpckhdq %%mm7, %%mm5 \n\t" //1
|
|
|
2047 "movq %%mm6, %%mm7 \n\t"
|
|
|
2048
|
|
15634
|
2049 "movq %%mm0, "DCTSIZE_S"*0*2(%%"REG_D") \n\t"
|
|
15631
|
2050 "punpckldq %%mm2, %%mm6 \n\t" //2
|
|
|
2051
|
|
15634
|
2052 "movq %%mm5, "DCTSIZE_S"*1*2(%%"REG_D") \n\t"
|
|
15631
|
2053 "punpckhdq %%mm2, %%mm7 \n\t" //3
|
|
|
2054
|
|
15634
|
2055 "movq %%mm6, "DCTSIZE_S"*2*2(%%"REG_D") \n\t"
|
|
15631
|
2056 "paddw %%mm1, %%mm4 \n\t"
|
|
|
2057
|
|
15634
|
2058 "movq %%mm7, "DCTSIZE_S"*3*2(%%"REG_D") \n\t"
|
|
15631
|
2059 "psllw $2, %%mm3 \n\t" //t10
|
|
|
2060
|
|
|
2061 "movq "MANGLE(temps)"+0*8, %%mm2 \n\t"
|
|
|
2062 "psllw $2, %%mm4 \n\t" //t11
|
|
|
2063
|
|
|
2064 "pmulhw "MANGLE(MM_FIX_0_707106781)", %%mm4 \n\t" //z3
|
|
|
2065 "paddw %%mm2, %%mm1 \n\t"
|
|
|
2066
|
|
|
2067 "psllw $2, %%mm1 \n\t" //t12
|
|
|
2068 "movq %%mm3, %%mm0 \n\t"
|
|
|
2069
|
|
|
2070 "pmulhw "MANGLE(MM_FIX_0_541196100)", %%mm0 \n\t"
|
|
|
2071 "psubw %%mm1, %%mm3 \n\t"
|
|
|
2072
|
|
|
2073 "pmulhw "MANGLE(MM_FIX_0_382683433)", %%mm3 \n\t" //z5
|
|
|
2074 "movq %%mm2, %%mm5 \n\t"
|
|
|
2075
|
|
|
2076 "pmulhw "MANGLE(MM_FIX_1_306562965)", %%mm1 \n\t"
|
|
|
2077 "psubw %%mm4, %%mm2 \n\t" //z13
|
|
|
2078
|
|
|
2079 "paddw %%mm4, %%mm5 \n\t" //z11
|
|
|
2080 "movq %%mm2, %%mm6 \n\t"
|
|
|
2081
|
|
|
2082 "paddw %%mm3, %%mm0 \n\t" //z2
|
|
|
2083 "movq %%mm5, %%mm7 \n\t"
|
|
|
2084
|
|
|
2085 "paddw %%mm0, %%mm2 \n\t" //d4
|
|
|
2086 "psubw %%mm0, %%mm6 \n\t" //d5
|
|
|
2087
|
|
|
2088 "movq %%mm2, %%mm4 \n\t"
|
|
|
2089 "paddw %%mm3, %%mm1 \n\t" //z4
|
|
|
2090
|
|
|
2091 //transpose 4x4
|
|
|
2092 "punpcklwd %%mm6, %%mm2 \n\t"
|
|
|
2093 "paddw %%mm1, %%mm5 \n\t" //d6
|
|
|
2094
|
|
|
2095 "punpckhwd %%mm6, %%mm4 \n\t"
|
|
|
2096 "psubw %%mm1, %%mm7 \n\t" //d7
|
|
|
2097
|
|
|
2098 "movq %%mm5, %%mm6 \n\t"
|
|
|
2099 "punpcklwd %%mm7, %%mm5 \n\t"
|
|
|
2100
|
|
|
2101 "punpckhwd %%mm7, %%mm6 \n\t"
|
|
|
2102 "movq %%mm2, %%mm7 \n\t"
|
|
|
2103
|
|
|
2104 "punpckldq %%mm5, %%mm2 \n\t" //4
|
|
15634
|
2105 "sub %%"REG_d", %%"REG_S" \n\t"
|
|
15631
|
2106
|
|
|
2107 "punpckhdq %%mm5, %%mm7 \n\t" //5
|
|
|
2108 "movq %%mm4, %%mm5 \n\t"
|
|
|
2109
|
|
15634
|
2110 "movq %%mm2, "DCTSIZE_S"*0*2+"DCTSIZE_S"(%%"REG_D") \n\t"
|
|
15631
|
2111 "punpckldq %%mm6, %%mm4 \n\t" //6
|
|
|
2112
|
|
15634
|
2113 "movq %%mm7, "DCTSIZE_S"*1*2+"DCTSIZE_S"(%%"REG_D") \n\t"
|
|
15631
|
2114 "punpckhdq %%mm6, %%mm5 \n\t" //7
|
|
|
2115
|
|
15634
|
2116 "movq %%mm4, "DCTSIZE_S"*2*2+"DCTSIZE_S"(%%"REG_D") \n\t"
|
|
|
2117 "add $4, %%"REG_S" \n\t"
|
|
|
2118
|
|
|
2119 "movq %%mm5, "DCTSIZE_S"*3*2+"DCTSIZE_S"(%%"REG_D") \n\t"
|
|
|
2120 "add $"DCTSIZE_S"*2*4, %%"REG_D" \n\t" //4 rows
|
|
|
2121 "dec %%"REG_c" \n\t"
|
|
15631
|
2122 "jnz 6b \n\t"
|
|
|
2123
|
|
15632
|
2124 : "+S"(pixels), "+D"(data), "+c"(cnt) //input regs
|
|
|
2125 : "a"(line_size)
|
|
15634
|
2126 : "%"REG_d);
|
|
15631
|
2127 }
|
|
|
2128
|
|
|
2129 #endif // HAVE_MMX
|
