|
8430
|
1 ;******************************************************************************
|
|
|
2 ;* FFT transform with SSE/3DNow optimizations
|
|
|
3 ;* Copyright (c) 2008 Loren Merritt
|
|
|
4 ;*
|
|
|
5 ;* This file is part of FFmpeg.
|
|
|
6 ;*
|
|
|
7 ;* FFmpeg is free software; you can redistribute it and/or
|
|
|
8 ;* modify it under the terms of the GNU Lesser General Public
|
|
|
9 ;* License as published by the Free Software Foundation; either
|
|
|
10 ;* version 2.1 of the License, or (at your option) any later version.
|
|
|
11 ;*
|
|
|
12 ;* FFmpeg is distributed in the hope that it will be useful,
|
|
|
13 ;* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
14 ;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
15 ;* Lesser General Public License for more details.
|
|
|
16 ;*
|
|
|
17 ;* You should have received a copy of the GNU Lesser General Public
|
|
|
18 ;* License along with FFmpeg; if not, write to the Free Software
|
|
|
19 ;* 51, Inc., Foundation Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
|
20 ;******************************************************************************
|
|
|
21
|
|
|
22 ; These functions are not individually interchangeable with the C versions.
|
|
|
23 ; While C takes arrays of FFTComplex, SSE/3DNow leave intermediate results
|
|
|
24 ; in blocks as conventient to the vector size.
|
|
|
25 ; i.e. {4x real, 4x imaginary, 4x real, ...} (or 2x respectively)
|
|
|
26
|
|
|
27 %include "x86inc.asm"
|
|
|
28
|
|
|
29 SECTION_RODATA
|
|
|
30
|
|
|
31 %define M_SQRT1_2 0.70710678118654752440
|
|
|
32 ps_root2: times 4 dd M_SQRT1_2
|
|
|
33 ps_root2mppm: dd -M_SQRT1_2, M_SQRT1_2, M_SQRT1_2, -M_SQRT1_2
|
|
|
34 ps_m1p1: dd 1<<31, 0
|
|
|
35
|
|
|
36 %assign i 16
|
|
|
37 %rep 13
|
|
|
38 cextern ff_cos_ %+ i
|
|
|
39 %assign i i<<1
|
|
|
40 %endrep
|
|
|
41
|
|
|
42 %ifdef ARCH_X86_64
|
|
|
43 %define pointer dq
|
|
|
44 %else
|
|
|
45 %define pointer dd
|
|
|
46 %endif
|
|
|
47
|
|
|
48 %macro IF0 1+
|
|
|
49 %endmacro
|
|
|
50 %macro IF1 1+
|
|
|
51 %1
|
|
|
52 %endmacro
|
|
|
53
|
|
|
54 section .text align=16
|
|
|
55
|
|
|
56 %macro T2_3DN 4 ; z0, z1, mem0, mem1
|
|
|
57 mova %1, %3
|
|
|
58 mova %2, %1
|
|
|
59 pfadd %1, %4
|
|
|
60 pfsub %2, %4
|
|
|
61 %endmacro
|
|
|
62
|
|
|
63 %macro T4_3DN 6 ; z0, z1, z2, z3, tmp0, tmp1
|
|
|
64 mova %5, %3
|
|
|
65 pfsub %3, %4
|
|
|
66 pfadd %5, %4 ; {t6,t5}
|
|
|
67 pxor %3, [ps_m1p1 GLOBAL] ; {t8,t7}
|
|
|
68 mova %6, %1
|
|
|
69 pswapd %3, %3
|
|
|
70 pfadd %1, %5 ; {r0,i0}
|
|
|
71 pfsub %6, %5 ; {r2,i2}
|
|
|
72 mova %4, %2
|
|
|
73 pfadd %2, %3 ; {r1,i1}
|
|
|
74 pfsub %4, %3 ; {r3,i3}
|
|
|
75 SWAP %3, %6
|
|
|
76 %endmacro
|
|
|
77
|
|
|
78 ; in: %1={r0,i0,r1,i1} %2={r2,i2,r3,i3}
|
|
|
79 ; out: %1={r0,r1,r2,r3} %2={i0,i1,i2,i3}
|
|
|
80 %macro T4_SSE 3
|
|
|
81 mova %3, %1
|
|
|
82 shufps %1, %2, 0x64 ; {r0,i0,r3,i2}
|
|
|
83 shufps %3, %2, 0xce ; {r1,i1,r2,i3}
|
|
|
84 mova %2, %1
|
|
|
85 addps %1, %3 ; {t1,t2,t6,t5}
|
|
|
86 subps %2, %3 ; {t3,t4,t8,t7}
|
|
|
87 mova %3, %1
|
|
|
88 shufps %1, %2, 0x44 ; {t1,t2,t3,t4}
|
|
|
89 shufps %3, %2, 0xbe ; {t6,t5,t7,t8}
|
|
|
90 mova %2, %1
|
|
|
91 addps %1, %3 ; {r0,i0,r1,i1}
|
|
|
92 subps %2, %3 ; {r2,i2,r3,i3}
|
|
|
93 mova %3, %1
|
|
|
94 shufps %1, %2, 0x88 ; {r0,r1,r2,r3}
|
|
|
95 shufps %3, %2, 0xdd ; {i0,i1,i2,i3}
|
|
|
96 SWAP %2, %3
|
|
|
97 %endmacro
|
|
|
98
|
|
|
99 %macro T8_SSE 6 ; r0,i0,r1,i1,t0,t1
|
|
|
100 mova %5, %3
|
|
|
101 shufps %3, %4, 0x44 ; {r4,i4,r6,i6}
|
|
|
102 shufps %5, %4, 0xee ; {r5,i5,r7,i7}
|
|
|
103 mova %6, %3
|
|
|
104 subps %3, %5 ; {r5,i5,r7,i7}
|
|
|
105 addps %6, %5 ; {t1,t2,t3,t4}
|
|
|
106 mova %5, %3
|
|
|
107 shufps %5, %5, 0xb1 ; {i5,r5,i7,r7}
|
|
|
108 mulps %3, [ps_root2mppm GLOBAL] ; {-r5,i5,r7,-i7}
|
|
|
109 mulps %5, [ps_root2 GLOBAL]
|
|
|
110 addps %3, %5 ; {t8,t7,ta,t9}
|
|
|
111 mova %5, %6
|
|
|
112 shufps %6, %3, 0x36 ; {t3,t2,t9,t8}
|
|
|
113 shufps %5, %3, 0x9c ; {t1,t4,t7,ta}
|
|
|
114 mova %3, %6
|
|
|
115 addps %6, %5 ; {t1,t2,t9,ta}
|
|
|
116 subps %3, %5 ; {t6,t5,tc,tb}
|
|
|
117 mova %5, %6
|
|
|
118 shufps %6, %3, 0xd8 ; {t1,t9,t5,tb}
|
|
|
119 shufps %5, %3, 0x8d ; {t2,ta,t6,tc}
|
|
|
120 mova %3, %1
|
|
|
121 mova %4, %2
|
|
|
122 addps %1, %6 ; {r0,r1,r2,r3}
|
|
|
123 addps %2, %5 ; {i0,i1,i2,i3}
|
|
|
124 subps %3, %6 ; {r4,r5,r6,r7}
|
|
|
125 subps %4, %5 ; {i4,i5,i6,i7}
|
|
|
126 %endmacro
|
|
|
127
|
|
|
128 ; scheduled for cpu-bound sizes
|
|
|
129 %macro PASS_SMALL 3 ; (to load m4-m7), wre, wim
|
|
|
130 IF%1 mova m4, Z(4)
|
|
|
131 IF%1 mova m5, Z(5)
|
|
|
132 mova m0, %2 ; wre
|
|
|
133 mova m2, m4
|
|
|
134 mova m1, %3 ; wim
|
|
|
135 mova m3, m5
|
|
|
136 mulps m2, m0 ; r2*wre
|
|
|
137 IF%1 mova m6, Z(6)
|
|
|
138 mulps m3, m1 ; i2*wim
|
|
|
139 IF%1 mova m7, Z(7)
|
|
|
140 mulps m4, m1 ; r2*wim
|
|
|
141 mulps m5, m0 ; i2*wre
|
|
|
142 addps m2, m3 ; r2*wre + i2*wim
|
|
|
143 mova m3, m1
|
|
|
144 mulps m1, m6 ; r3*wim
|
|
|
145 subps m5, m4 ; i2*wre - r2*wim
|
|
|
146 mova m4, m0
|
|
|
147 mulps m3, m7 ; i3*wim
|
|
|
148 mulps m4, m6 ; r3*wre
|
|
|
149 mulps m0, m7 ; i3*wre
|
|
|
150 subps m4, m3 ; r3*wre - i3*wim
|
|
|
151 mova m3, Z(0)
|
|
|
152 addps m0, m1 ; i3*wre + r3*wim
|
|
|
153 mova m1, m4
|
|
|
154 addps m4, m2 ; t5
|
|
|
155 subps m1, m2 ; t3
|
|
|
156 subps m3, m4 ; r2
|
|
|
157 addps m4, Z(0) ; r0
|
|
|
158 mova m6, Z(2)
|
|
|
159 mova Z(4), m3
|
|
|
160 mova Z(0), m4
|
|
|
161 mova m3, m5
|
|
|
162 subps m5, m0 ; t4
|
|
|
163 mova m4, m6
|
|
|
164 subps m6, m5 ; r3
|
|
|
165 addps m5, m4 ; r1
|
|
|
166 mova Z(6), m6
|
|
|
167 mova Z(2), m5
|
|
|
168 mova m2, Z(3)
|
|
|
169 addps m3, m0 ; t6
|
|
|
170 subps m2, m1 ; i3
|
|
|
171 mova m7, Z(1)
|
|
|
172 addps m1, Z(3) ; i1
|
|
|
173 mova Z(7), m2
|
|
|
174 mova Z(3), m1
|
|
|
175 mova m4, m7
|
|
|
176 subps m7, m3 ; i2
|
|
|
177 addps m3, m4 ; i0
|
|
|
178 mova Z(5), m7
|
|
|
179 mova Z(1), m3
|
|
|
180 %endmacro
|
|
|
181
|
|
|
182 ; scheduled to avoid store->load aliasing
|
|
|
183 %macro PASS_BIG 1 ; (!interleave)
|
|
|
184 mova m4, Z(4) ; r2
|
|
|
185 mova m5, Z(5) ; i2
|
|
|
186 mova m2, m4
|
|
|
187 mova m0, [wq] ; wre
|
|
|
188 mova m3, m5
|
|
|
189 mova m1, [wq+o1q] ; wim
|
|
|
190 mulps m2, m0 ; r2*wre
|
|
|
191 mova m6, Z(6) ; r3
|
|
|
192 mulps m3, m1 ; i2*wim
|
|
|
193 mova m7, Z(7) ; i3
|
|
|
194 mulps m4, m1 ; r2*wim
|
|
|
195 mulps m5, m0 ; i2*wre
|
|
|
196 addps m2, m3 ; r2*wre + i2*wim
|
|
|
197 mova m3, m1
|
|
|
198 mulps m1, m6 ; r3*wim
|
|
|
199 subps m5, m4 ; i2*wre - r2*wim
|
|
|
200 mova m4, m0
|
|
|
201 mulps m3, m7 ; i3*wim
|
|
|
202 mulps m4, m6 ; r3*wre
|
|
|
203 mulps m0, m7 ; i3*wre
|
|
|
204 subps m4, m3 ; r3*wre - i3*wim
|
|
|
205 mova m3, Z(0)
|
|
|
206 addps m0, m1 ; i3*wre + r3*wim
|
|
|
207 mova m1, m4
|
|
|
208 addps m4, m2 ; t5
|
|
|
209 subps m1, m2 ; t3
|
|
|
210 subps m3, m4 ; r2
|
|
|
211 addps m4, Z(0) ; r0
|
|
|
212 mova m6, Z(2)
|
|
|
213 mova Z(4), m3
|
|
|
214 mova Z(0), m4
|
|
|
215 mova m3, m5
|
|
|
216 subps m5, m0 ; t4
|
|
|
217 mova m4, m6
|
|
|
218 subps m6, m5 ; r3
|
|
|
219 addps m5, m4 ; r1
|
|
|
220 IF%1 mova Z(6), m6
|
|
|
221 IF%1 mova Z(2), m5
|
|
|
222 mova m2, Z(3)
|
|
|
223 addps m3, m0 ; t6
|
|
|
224 subps m2, m1 ; i3
|
|
|
225 mova m7, Z(1)
|
|
|
226 addps m1, Z(3) ; i1
|
|
|
227 IF%1 mova Z(7), m2
|
|
|
228 IF%1 mova Z(3), m1
|
|
|
229 mova m4, m7
|
|
|
230 subps m7, m3 ; i2
|
|
|
231 addps m3, m4 ; i0
|
|
|
232 IF%1 mova Z(5), m7
|
|
|
233 IF%1 mova Z(1), m3
|
|
|
234 %if %1==0
|
|
|
235 mova m4, m5 ; r1
|
|
|
236 mova m0, m6 ; r3
|
|
|
237 unpcklps m5, m1
|
|
|
238 unpckhps m4, m1
|
|
|
239 unpcklps m6, m2
|
|
|
240 unpckhps m0, m2
|
|
|
241 mova m1, Z(0)
|
|
|
242 mova m2, Z(4)
|
|
|
243 mova Z(2), m5
|
|
|
244 mova Z(3), m4
|
|
|
245 mova Z(6), m6
|
|
|
246 mova Z(7), m0
|
|
|
247 mova m5, m1 ; r0
|
|
|
248 mova m4, m2 ; r2
|
|
|
249 unpcklps m1, m3
|
|
|
250 unpckhps m5, m3
|
|
|
251 unpcklps m2, m7
|
|
|
252 unpckhps m4, m7
|
|
|
253 mova Z(0), m1
|
|
|
254 mova Z(1), m5
|
|
|
255 mova Z(4), m2
|
|
|
256 mova Z(5), m4
|
|
|
257 %endif
|
|
|
258 %endmacro
|
|
|
259
|
|
|
260 %macro PUNPCK 3
|
|
|
261 mova %3, %1
|
|
|
262 punpckldq %1, %2
|
|
|
263 punpckhdq %3, %2
|
|
|
264 %endmacro
|
|
|
265
|
|
|
266 INIT_XMM
|
|
|
267
|
|
|
268 %define Z(x) [r0+mmsize*x]
|
|
|
269
|
|
|
270 align 16
|
|
|
271 fft4_sse:
|
|
|
272 mova m0, Z(0)
|
|
|
273 mova m1, Z(1)
|
|
|
274 T4_SSE m0, m1, m2
|
|
|
275 mova Z(0), m0
|
|
|
276 mova Z(1), m1
|
|
|
277 ret
|
|
|
278
|
|
|
279 align 16
|
|
|
280 fft8_sse:
|
|
|
281 mova m0, Z(0)
|
|
|
282 mova m1, Z(1)
|
|
|
283 T4_SSE m0, m1, m2
|
|
|
284 mova m2, Z(2)
|
|
|
285 mova m3, Z(3)
|
|
|
286 T8_SSE m0, m1, m2, m3, m4, m5
|
|
|
287 mova Z(0), m0
|
|
|
288 mova Z(1), m1
|
|
|
289 mova Z(2), m2
|
|
|
290 mova Z(3), m3
|
|
|
291 ret
|
|
|
292
|
|
|
293 align 16
|
|
|
294 fft16_sse:
|
|
|
295 mova m0, Z(0)
|
|
|
296 mova m1, Z(1)
|
|
|
297 T4_SSE m0, m1, m2
|
|
|
298 mova m2, Z(2)
|
|
|
299 mova m3, Z(3)
|
|
|
300 T8_SSE m0, m1, m2, m3, m4, m5
|
|
|
301 mova m4, Z(4)
|
|
|
302 mova m5, Z(5)
|
|
|
303 mova Z(0), m0
|
|
|
304 mova Z(1), m1
|
|
|
305 mova Z(2), m2
|
|
|
306 mova Z(3), m3
|
|
|
307 T4_SSE m4, m5, m6
|
|
|
308 mova m6, Z(6)
|
|
|
309 mova m7, Z(7)
|
|
|
310 T4_SSE m6, m7, m0
|
|
|
311 PASS_SMALL 0, [ff_cos_16 GLOBAL], [ff_cos_16+16 GLOBAL]
|
|
|
312 ret
|
|
|
313
|
|
|
314
|
|
|
315 INIT_MMX
|
|
|
316
|
|
|
317 %macro FFT48_3DN 1
|
|
|
318 align 16
|
|
|
319 fft4%1:
|
|
|
320 T2_3DN m0, m1, Z(0), Z(1)
|
|
|
321 mova m2, Z(2)
|
|
|
322 mova m3, Z(3)
|
|
|
323 T4_3DN m0, m1, m2, m3, m4, m5
|
|
|
324 PUNPCK m0, m1, m4
|
|
|
325 PUNPCK m2, m3, m5
|
|
|
326 mova Z(0), m0
|
|
|
327 mova Z(1), m4
|
|
|
328 mova Z(2), m2
|
|
|
329 mova Z(3), m5
|
|
|
330 ret
|
|
|
331
|
|
|
332 align 16
|
|
|
333 fft8%1:
|
|
|
334 T2_3DN m0, m1, Z(0), Z(1)
|
|
|
335 mova m2, Z(2)
|
|
|
336 mova m3, Z(3)
|
|
|
337 T4_3DN m0, m1, m2, m3, m4, m5
|
|
|
338 mova Z(0), m0
|
|
|
339 mova Z(2), m2
|
|
|
340 T2_3DN m4, m5, Z(4), Z(5)
|
|
|
341 T2_3DN m6, m7, Z(6), Z(7)
|
|
|
342 pswapd m0, m5
|
|
|
343 pswapd m2, m7
|
|
|
344 pxor m0, [ps_m1p1 GLOBAL]
|
|
|
345 pxor m2, [ps_m1p1 GLOBAL]
|
|
|
346 pfsub m5, m0
|
|
|
347 pfadd m7, m2
|
|
|
348 pfmul m5, [ps_root2 GLOBAL]
|
|
|
349 pfmul m7, [ps_root2 GLOBAL]
|
|
|
350 T4_3DN m1, m3, m5, m7, m0, m2
|
|
|
351 mova Z(5), m5
|
|
|
352 mova Z(7), m7
|
|
|
353 mova m0, Z(0)
|
|
|
354 mova m2, Z(2)
|
|
|
355 T4_3DN m0, m2, m4, m6, m5, m7
|
|
|
356 PUNPCK m0, m1, m5
|
|
|
357 PUNPCK m2, m3, m7
|
|
|
358 mova Z(0), m0
|
|
|
359 mova Z(1), m5
|
|
|
360 mova Z(2), m2
|
|
|
361 mova Z(3), m7
|
|
|
362 PUNPCK m4, Z(5), m5
|
|
|
363 PUNPCK m6, Z(7), m7
|
|
|
364 mova Z(4), m4
|
|
|
365 mova Z(5), m5
|
|
|
366 mova Z(6), m6
|
|
|
367 mova Z(7), m7
|
|
|
368 ret
|
|
|
369 %endmacro
|
|
|
370
|
|
|
371 FFT48_3DN _3dn2
|
|
|
372
|
|
|
373 %macro pswapd 2
|
|
|
374 %ifidn %1, %2
|
|
|
375 movd [r0+12], %1
|
|
|
376 punpckhdq %1, [r0+8]
|
|
|
377 %else
|
|
|
378 movq %1, %2
|
|
|
379 psrlq %1, 32
|
|
|
380 punpckldq %1, %2
|
|
|
381 %endif
|
|
|
382 %endmacro
|
|
|
383
|
|
|
384 FFT48_3DN _3dn
|
|
|
385
|
|
|
386
|
|
|
387 %define Z(x) [zq + o1q*(x&6)*((x/6)^1) + o3q*(x/6) + mmsize*(x&1)]
|
|
|
388
|
|
|
389 %macro DECL_PASS 2+ ; name, payload
|
|
|
390 align 16
|
|
|
391 %1:
|
|
|
392 DEFINE_ARGS z, w, n, o1, o3
|
|
|
393 lea o3q, [nq*3]
|
|
|
394 lea o1q, [nq*8]
|
|
|
395 shl o3q, 4
|
|
|
396 .loop:
|
|
|
397 %2
|
|
|
398 add zq, mmsize*2
|
|
|
399 add wq, mmsize
|
|
|
400 sub nd, mmsize/8
|
|
|
401 jg .loop
|
|
|
402 rep ret
|
|
|
403 %endmacro
|
|
|
404
|
|
|
405 INIT_XMM
|
|
|
406 DECL_PASS pass_sse, PASS_BIG 1
|
|
|
407 DECL_PASS pass_interleave_sse, PASS_BIG 0
|
|
|
408
|
|
|
409 INIT_MMX
|
|
|
410 %define mulps pfmul
|
|
|
411 %define addps pfadd
|
|
|
412 %define subps pfsub
|
|
|
413 %define unpcklps punpckldq
|
|
|
414 %define unpckhps punpckhdq
|
|
|
415 DECL_PASS pass_3dn, PASS_SMALL 1, [wq], [wq+o1q]
|
|
|
416 DECL_PASS pass_interleave_3dn, PASS_BIG 0
|
|
|
417 %define pass_3dn2 pass_3dn
|
|
|
418 %define pass_interleave_3dn2 pass_interleave_3dn
|
|
|
419
|
|
|
420
|
|
|
421 %macro DECL_FFT 2-3 ; nbits, cpu, suffix
|
|
|
422 %xdefine list_of_fft fft4%2, fft8%2
|
|
|
423 %if %1==5
|
|
|
424 %xdefine list_of_fft list_of_fft, fft16%2
|
|
|
425 %endif
|
|
|
426
|
|
|
427 %assign n 1<<%1
|
|
|
428 %rep 17-%1
|
|
|
429 %assign n2 n/2
|
|
|
430 %assign n4 n/4
|
|
|
431 %xdefine list_of_fft list_of_fft, fft %+ n %+ %3%2
|
|
|
432
|
|
|
433 align 16
|
|
|
434 fft %+ n %+ %3%2:
|
|
|
435 call fft %+ n2 %+ %2
|
|
|
436 add r0, n*4 - (n&(-2<<%1))
|
|
|
437 call fft %+ n4 %+ %2
|
|
|
438 add r0, n*2 - (n2&(-2<<%1))
|
|
|
439 call fft %+ n4 %+ %2
|
|
|
440 sub r0, n*6 + (n2&(-2<<%1))
|
|
|
441 lea r1, [ff_cos_ %+ n GLOBAL]
|
|
|
442 mov r2d, n4/2
|
|
|
443 jmp pass%3%2
|
|
|
444
|
|
|
445 %assign n n*2
|
|
|
446 %endrep
|
|
|
447 %undef n
|
|
|
448
|
|
|
449 align 8
|
|
|
450 dispatch_tab%3%2: pointer list_of_fft
|
|
|
451
|
|
|
452 ; On x86_32, this function does the register saving and restoring for all of fft.
|
|
|
453 ; The others pass args in registers and don't spill anything.
|
|
|
454 cglobal fft_dispatch%3%2, 2,5,0, z, nbits
|
|
|
455 lea r2, [dispatch_tab%3%2 GLOBAL]
|
|
|
456 mov r2, [r2 + (nbitsq-2)*gprsize]
|
|
|
457 call r2
|
|
|
458 RET
|
|
|
459 %endmacro ; DECL_FFT
|
|
|
460
|
|
|
461 DECL_FFT 5, _sse
|
|
|
462 DECL_FFT 5, _sse, _interleave
|
|
|
463 DECL_FFT 4, _3dn
|
|
|
464 DECL_FFT 4, _3dn, _interleave
|
|
|
465 DECL_FFT 4, _3dn2
|
|
|
466 DECL_FFT 4, _3dn2, _interleave
|
|
|
467
|
