# HG changeset patch # User lorenm # Date 1155945229 0 # Node ID a3d97c60ea0719d59d6952aab87124ff468900c4 # Parent 19c437d6aba5d1c2dd57ee5f675b42cb1355f3ae ff_fft_calc_3dn/3dn2/sse: convert intrinsics to inline asm. 2.5% faster fft, 0.5% faster vorbis. diff -r 19c437d6aba5 -r a3d97c60ea07 fft.c --- a/fft.c Fri Aug 18 13:38:24 2006 +0000 +++ b/fft.c Fri Aug 18 23:53:49 2006 +0000 @@ -58,7 +58,7 @@ s->exptab1 = NULL; /* compute constant table for HAVE_SSE version */ -#if (defined(HAVE_MMX) && (defined(HAVE_BUILTIN_VECTOR) || defined(HAVE_MM3DNOW))) \ +#if defined(HAVE_MMX) \ || (defined(HAVE_ALTIVEC) && !defined(ALTIVEC_USE_REFERENCE_C_CODE)) { int has_vectors = mm_support(); @@ -67,15 +67,12 @@ #if defined(HAVE_MMX) if (has_vectors & MM_3DNOWEXT) s->imdct_calc = ff_imdct_calc_3dn2; -#ifdef HAVE_MM3DNOW if (has_vectors & MM_3DNOWEXT) /* 3DNowEx for Athlon(XP) */ s->fft_calc = ff_fft_calc_3dn2; else if (has_vectors & MM_3DNOW) /* 3DNow! for K6-2/3 */ s->fft_calc = ff_fft_calc_3dn; -#endif -#ifdef HAVE_BUILTIN_VECTOR if (has_vectors & MM_SSE2) /* SSE for P4/K8 */ s->fft_calc = ff_fft_calc_sse; @@ -83,7 +80,6 @@ s->fft_calc == ff_fft_calc_c) /* SSE for P3 */ s->fft_calc = ff_fft_calc_sse; -#endif #else /* HAVE_MMX */ if (has_vectors & MM_ALTIVEC) s->fft_calc = ff_fft_calc_altivec; diff -r 19c437d6aba5 -r a3d97c60ea07 i386/fft_3dn.c --- a/i386/fft_3dn.c Fri Aug 18 13:38:24 2006 +0000 +++ b/i386/fft_3dn.c Fri Aug 18 23:53:49 2006 +0000 @@ -1,6 +1,6 @@ /* * FFT/MDCT transform with 3DNow! optimizations - * Copyright (c) 2006 Zuxy MENG Jie. + * Copyright (c) 2006 Zuxy MENG Jie, Loren Merritt * Based on fft_sse.c copyright (c) 2002 Fabrice Bellard. * * This library is free software; you can redistribute it and/or @@ -18,11 +18,6 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ #include "../dsputil.h" -#include - -#ifdef HAVE_MM3DNOW - -#include static const int p1m1[2] __attribute__((aligned(8))) = { 0, 1 << 31 }; @@ -33,104 +28,96 @@ void ff_fft_calc_3dn(FFTContext *s, FFTComplex *z) { int ln = s->nbits; - int j, np, np2; - int nblocks, nloops; - register FFTComplex *p, *q; - FFTComplex *cptr, *cptr1; - int k; + long i, j; + long nblocks, nloops; + FFTComplex *p, *cptr; - np = 1 << ln; - /* FEMMS not a must here but recommended by AMD */ - _m_femms(); - - { - __m64 *r, a0, a1, b0, b1, tmp, c; - - r = (__m64 *)&z[0]; - if (s->inverse) - c = *(__m64 *)m1p1; - else - c = *(__m64 *)p1m1; + asm volatile( + /* FEMMS is not a must here but recommended by AMD */ + "femms \n\t" + "movq %0, %%mm7 \n\t" + ::"m"(*(s->inverse ? m1p1 : p1m1)) + ); - j = (np >> 2); - do { - /* do the pass 0 butterfly */ - a0 = _m_pfadd(r[0], r[1]); - a1 = _m_pfsub(r[0], r[1]); - - /* do the pass 0 butterfly */ - b0 = _m_pfadd(r[2], r[3]); - b1 = _m_pfsub(r[2], r[3]); - - /* multiply third by -i */ - tmp = _m_punpckhdq(b1, b1); - b1 = _m_punpckldq(b1, b1); - b1 = _m_punpckldq(tmp, b1); - b1 = _m_pxor(b1, c); - - /* do the pass 1 butterfly */ - r[0] = _m_pfadd(a0, b0); - r[1] = _m_pfadd(a1, b1); - r[2] = _m_pfsub(a0, b0); - r[3] = _m_pfsub(a1, b1); - r += 4; - } while (--j != 0); - } + i = 8 << ln; + asm volatile( + "1: \n\t" + "sub $32, %0 \n\t" + "movq (%0,%1), %%mm0 \n\t" + "movq 16(%0,%1), %%mm1 \n\t" + "movq 8(%0,%1), %%mm2 \n\t" + "movq 24(%0,%1), %%mm3 \n\t" + "movq %%mm0, %%mm4 \n\t" + "movq %%mm1, %%mm5 \n\t" + "pfadd %%mm2, %%mm0 \n\t" + "pfadd %%mm3, %%mm1 \n\t" + "pfsub %%mm2, %%mm4 \n\t" + "pfsub %%mm3, %%mm5 \n\t" + "movq %%mm0, %%mm2 \n\t" + "punpckldq %%mm5, %%mm6 \n\t" + "punpckhdq %%mm6, %%mm5 \n\t" + "movq %%mm4, %%mm3 \n\t" + "pxor %%mm7, %%mm5 \n\t" + "pfadd %%mm1, %%mm0 \n\t" + "pfadd %%mm5, %%mm4 \n\t" + "pfsub %%mm1, %%mm2 \n\t" + "pfsub %%mm5, %%mm3 \n\t" + "movq %%mm0, (%0,%1) \n\t" + "movq %%mm4, 8(%0,%1) \n\t" + "movq %%mm2, 16(%0,%1) \n\t" + "movq %%mm3, 24(%0,%1) \n\t" + "jg 1b \n\t" + :"+r"(i) + :"r"(z) + ); /* pass 2 .. ln-1 */ - nblocks = np >> 3; + nblocks = 1 << (ln-3); nloops = 1 << 2; - np2 = np >> 1; - - cptr1 = s->exptab1; + cptr = s->exptab1; do { p = z; - q = z + nloops; j = nblocks; do { - cptr = cptr1; - k = nloops >> 1; - do { - __m64 a0, a1, b0, b1, c0, c1, t10, t11, t20, t21; - - a0 = *(__m64 *)&p[0]; - a1 = *(__m64 *)&p[1]; - b0 = *(__m64 *)&q[0]; - b1 = *(__m64 *)&q[1]; - - /* complex mul */ - c0 = *(__m64 *)&cptr[0]; - c1 = *(__m64 *)&cptr[1]; - /* cre*re cim*re */ - t10 = _m_pfmul(c0, _m_punpckldq(b0, b0)); - t11 = _m_pfmul(c1, _m_punpckldq(b1, b1)); - c0 = *(__m64 *)&cptr[2]; - c1 = *(__m64 *)&cptr[3]; - /* -cim*im cre*im */ - t20 = _m_pfmul(c0, _m_punpckhdq(b0, b0)); - t21 = _m_pfmul(c1, _m_punpckhdq(b1, b1)); - b0 = _m_pfadd(t10, t20); - b1 = _m_pfadd(t11, t21); - - /* butterfly */ - *(__m64 *)&p[0] = _m_pfadd(a0, b0); - *(__m64 *)&p[1] = _m_pfadd(a1, b1); - *(__m64 *)&q[0] = _m_pfsub(a0, b0); - *(__m64 *)&q[1] = _m_pfsub(a1, b1); - - p += 2; - q += 2; - cptr += 4; - } while (--k); - - p += nloops; - q += nloops; + i = nloops*8; + asm volatile( + "1: \n\t" + "sub $16, %0 \n\t" + "movq (%1,%0), %%mm0 \n\t" + "movq 8(%1,%0), %%mm1 \n\t" + "movq (%2,%0), %%mm2 \n\t" + "movq 8(%2,%0), %%mm3 \n\t" + "movq %%mm2, %%mm4 \n\t" + "movq %%mm3, %%mm5 \n\t" + "punpckldq %%mm2, %%mm2 \n\t" + "punpckldq %%mm3, %%mm3 \n\t" + "punpckhdq %%mm4, %%mm4 \n\t" + "punpckhdq %%mm5, %%mm5 \n\t" + "pfmul (%3,%0,2), %%mm2 \n\t" // cre*re cim*re + "pfmul 8(%3,%0,2), %%mm3 \n\t" + "pfmul 16(%3,%0,2), %%mm4 \n\t" // -cim*im cre*im + "pfmul 24(%3,%0,2), %%mm5 \n\t" + "pfadd %%mm2, %%mm4 \n\t" // cre*re-cim*im cim*re+cre*im + "pfadd %%mm3, %%mm5 \n\t" + "movq %%mm0, %%mm2 \n\t" + "movq %%mm1, %%mm3 \n\t" + "pfadd %%mm4, %%mm0 \n\t" + "pfadd %%mm5, %%mm1 \n\t" + "pfsub %%mm4, %%mm2 \n\t" + "pfsub %%mm5, %%mm3 \n\t" + "movq %%mm0, (%1,%0) \n\t" + "movq %%mm1, 8(%1,%0) \n\t" + "movq %%mm2, (%2,%0) \n\t" + "movq %%mm3, 8(%2,%0) \n\t" + "jg 1b \n\t" + :"+r"(i) + :"r"(p), "r"(p + nloops), "r"(cptr) + ); + p += nloops*2; } while (--j); - cptr1 += nloops * 2; - nblocks = nblocks >> 1; - nloops = nloops << 1; + cptr += nloops*2; + nblocks >>= 1; + nloops <<= 1; } while (nblocks != 0); - _m_femms(); + asm volatile("femms"); } - -#endif diff -r 19c437d6aba5 -r a3d97c60ea07 i386/fft_3dn2.c --- a/i386/fft_3dn2.c Fri Aug 18 13:38:24 2006 +0000 +++ b/i386/fft_3dn2.c Fri Aug 18 23:53:49 2006 +0000 @@ -18,11 +18,6 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ #include "../dsputil.h" -#include - -#ifdef HAVE_MM3DNOW - -#include static const int p1m1[2] __attribute__((aligned(8))) = { 0, 1 << 31 }; @@ -33,108 +28,97 @@ void ff_fft_calc_3dn2(FFTContext *s, FFTComplex *z) { int ln = s->nbits; - int j, np, np2; - int nblocks, nloops; - register FFTComplex *p, *q; - FFTComplex *cptr, *cptr1; - int k; + long i, j; + long nblocks, nloops; + FFTComplex *p, *cptr; - np = 1 << ln; - /* FEMMS is not a must here but recommended by AMD */ - _m_femms(); - - { - __m64 *r, a0, a1, b0, b1, c; - - r = (__m64 *)&z[0]; - if (s->inverse) - c = *(__m64 *)m1p1; - else - c = *(__m64 *)p1m1; + asm volatile( + /* FEMMS is not a must here but recommended by AMD */ + "femms \n\t" + "movq %0, %%mm7 \n\t" + ::"m"(*(s->inverse ? m1p1 : p1m1)) + ); - j = (np >> 2); - do { - /* do the pass 0 butterfly */ - a0 = _m_pfadd(r[0], r[1]); - a1 = _m_pfsub(r[0], r[1]); - - /* do the pass 0 butterfly */ - b0 = _m_pfadd(r[2], r[3]); - b1 = _m_pfsub(r[2], r[3]); - - /* multiply third by -i */ - b1 = _m_pswapd(b1); - b1 = _m_pxor(b1, c); - - r[0] = _m_pfadd(a0, b0); - r[1] = _m_pfadd(a1, b1); - r[2] = _m_pfsub(a0, b0); - r[3] = _m_pfsub(a1, b1); - r += 4; - } while (--j != 0); - } + i = 8 << ln; + asm volatile( + "1: \n\t" + "sub $32, %0 \n\t" + "movq (%0,%1), %%mm0 \n\t" + "movq 16(%0,%1), %%mm1 \n\t" + "movq 8(%0,%1), %%mm2 \n\t" + "movq 24(%0,%1), %%mm3 \n\t" + "movq %%mm0, %%mm4 \n\t" + "movq %%mm1, %%mm5 \n\t" + "pfadd %%mm2, %%mm0 \n\t" + "pfadd %%mm3, %%mm1 \n\t" + "pfsub %%mm2, %%mm4 \n\t" + "pfsub %%mm3, %%mm5 \n\t" + "movq %%mm0, %%mm2 \n\t" + "pswapd %%mm5, %%mm5 \n\t" + "movq %%mm4, %%mm3 \n\t" + "pxor %%mm7, %%mm5 \n\t" + "pfadd %%mm1, %%mm0 \n\t" + "pfadd %%mm5, %%mm4 \n\t" + "pfsub %%mm1, %%mm2 \n\t" + "pfsub %%mm5, %%mm3 \n\t" + "movq %%mm0, (%0,%1) \n\t" + "movq %%mm4, 8(%0,%1) \n\t" + "movq %%mm2, 16(%0,%1) \n\t" + "movq %%mm3, 24(%0,%1) \n\t" + "jg 1b \n\t" + :"+r"(i) + :"r"(z) + ); /* pass 2 .. ln-1 */ - nblocks = np >> 3; + nblocks = 1 << (ln-3); nloops = 1 << 2; - np2 = np >> 1; - - cptr1 = s->exptab1; + cptr = s->exptab1; do { p = z; - q = z + nloops; j = nblocks; do { - cptr = cptr1; - k = nloops >> 1; - do { - __m64 a0, a1, b0, b1, c0, c1, t10, t11, t20, t21; - - a0 = *(__m64 *)&p[0]; - a1 = *(__m64 *)&p[1]; - b0 = *(__m64 *)&q[0]; - b1 = *(__m64 *)&q[1]; - - /* complex mul */ - c0 = *(__m64 *)&cptr[0]; - c1 = *(__m64 *)&cptr[1]; - /* cre*re cim*im */ - t10 = _m_pfmul(c0, b0); - t11 = _m_pfmul(c1, b1); - /* no need to access cptr[2] & cptr[3] */ - c0 = _m_pswapd(c0); - c1 = _m_pswapd(c1); - /* cim*re cre*im */ - t20 = _m_pfmul(c0, b0); - t21 = _m_pfmul(c1, b1); - - /* cre*re-cim*im cim*re+cre*im */ - b0 = _m_pfpnacc(t10, t20); - b1 = _m_pfpnacc(t11, t21); - - /* butterfly */ - *(__m64 *)&p[0] = _m_pfadd(a0, b0); - *(__m64 *)&p[1] = _m_pfadd(a1, b1); - *(__m64 *)&q[0] = _m_pfsub(a0, b0); - *(__m64 *)&q[1] = _m_pfsub(a1, b1); - - p += 2; - q += 2; - cptr += 4; - } while (--k); - - p += nloops; - q += nloops; + i = nloops*8; + asm volatile( + "1: \n\t" + "sub $16, %0 \n\t" + "movq (%1,%0), %%mm0 \n\t" + "movq 8(%1,%0), %%mm1 \n\t" + "movq (%2,%0), %%mm2 \n\t" + "movq 8(%2,%0), %%mm3 \n\t" + "movq (%3,%0,2), %%mm4 \n\t" + "movq 8(%3,%0,2), %%mm5 \n\t" + "pswapd %%mm4, %%mm6 \n\t" // no need for cptr[2] & cptr[3] + "pswapd %%mm5, %%mm7 \n\t" + "pfmul %%mm2, %%mm4 \n\t" // cre*re cim*im + "pfmul %%mm3, %%mm5 \n\t" + "pfmul %%mm2, %%mm6 \n\t" // cim*re cre*im + "pfmul %%mm3, %%mm7 \n\t" + "pfpnacc %%mm6, %%mm4 \n\t" // cre*re-cim*im cim*re+cre*im + "pfpnacc %%mm7, %%mm5 \n\t" + "movq %%mm0, %%mm2 \n\t" + "movq %%mm1, %%mm3 \n\t" + "pfadd %%mm4, %%mm0 \n\t" + "pfadd %%mm5, %%mm1 \n\t" + "pfsub %%mm4, %%mm2 \n\t" + "pfsub %%mm5, %%mm3 \n\t" + "movq %%mm0, (%1,%0) \n\t" + "movq %%mm1, 8(%1,%0) \n\t" + "movq %%mm2, (%2,%0) \n\t" + "movq %%mm3, 8(%2,%0) \n\t" + "jg 1b \n\t" + :"+r"(i) + :"r"(p), "r"(p + nloops), "r"(cptr) + ); + p += nloops*2; } while (--j); - cptr1 += nloops * 2; - nblocks = nblocks >> 1; - nloops = nloops << 1; + cptr += nloops*2; + nblocks >>= 1; + nloops <<= 1; } while (nblocks != 0); - _m_femms(); + asm volatile("femms"); } -#endif - void ff_imdct_calc_3dn2(MDCTContext *s, FFTSample *output, const FFTSample *input, FFTSample *tmp) { @@ -221,3 +205,4 @@ } asm volatile("femms"); } + diff -r 19c437d6aba5 -r a3d97c60ea07 i386/fft_sse.c --- a/i386/fft_sse.c Fri Aug 18 13:38:24 2006 +0000 +++ b/i386/fft_sse.c Fri Aug 18 23:53:49 2006 +0000 @@ -17,11 +17,6 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ #include "../dsputil.h" -#include - -#ifdef HAVE_BUILTIN_VECTOR - -#include static const int p1p1p1m1[4] __attribute__((aligned(16))) = { 0, 0, 0, 1 << 31 }; @@ -45,96 +40,81 @@ void ff_fft_calc_sse(FFTContext *s, FFTComplex *z) { int ln = s->nbits; - int j, np, np2; - int nblocks, nloops; - register FFTComplex *p, *q; - FFTComplex *cptr, *cptr1; - int k; - - np = 1 << ln; + long i, j; + long nblocks, nloops; + FFTComplex *p, *cptr; - { - __m128 *r, a, b, a1, c1, c2; - - r = (__m128 *)&z[0]; - c1 = *(__m128 *)p1p1m1m1; - if (s->inverse) - c2 = *(__m128 *)p1p1m1p1; - else - c2 = *(__m128 *)p1p1p1m1; + asm volatile( + "movaps %0, %%xmm4 \n\t" + "movaps %1, %%xmm5 \n\t" + ::"m"(*p1p1m1m1), + "m"(*(s->inverse ? p1p1m1p1 : p1p1p1m1)) + ); - j = (np >> 2); - do { - a = r[0]; - b = _mm_shuffle_ps(a, a, _MM_SHUFFLE(1, 0, 3, 2)); - a = _mm_xor_ps(a, c1); - /* do the pass 0 butterfly */ - a = _mm_add_ps(a, b); - - a1 = r[1]; - b = _mm_shuffle_ps(a1, a1, _MM_SHUFFLE(1, 0, 3, 2)); - a1 = _mm_xor_ps(a1, c1); - /* do the pass 0 butterfly */ - b = _mm_add_ps(a1, b); - - /* multiply third by -i */ - /* by toggling the sign bit */ - b = _mm_shuffle_ps(b, b, _MM_SHUFFLE(2, 3, 1, 0)); - b = _mm_xor_ps(b, c2); - - /* do the pass 1 butterfly */ - r[0] = _mm_add_ps(a, b); - r[1] = _mm_sub_ps(a, b); - r += 2; - } while (--j != 0); - } + i = 8 << ln; + asm volatile( + "1: \n\t" + "sub $32, %0 \n\t" + /* do the pass 0 butterfly */ + "movaps (%0,%1), %%xmm0 \n\t" + "movaps %%xmm0, %%xmm1 \n\t" + "shufps $0x4E, %%xmm0, %%xmm0 \n\t" + "xorps %%xmm4, %%xmm1 \n\t" + "addps %%xmm1, %%xmm0 \n\t" + "movaps 16(%0,%1), %%xmm2 \n\t" + "movaps %%xmm2, %%xmm3 \n\t" + "shufps $0x4E, %%xmm2, %%xmm2 \n\t" + "xorps %%xmm4, %%xmm3 \n\t" + "addps %%xmm3, %%xmm2 \n\t" + /* multiply third by -i */ + /* by toggling the sign bit */ + "shufps $0xB4, %%xmm2, %%xmm2 \n\t" + "xorps %%xmm5, %%xmm2 \n\t" + /* do the pass 1 butterfly */ + "movaps %%xmm0, %%xmm1 \n\t" + "addps %%xmm2, %%xmm0 \n\t" + "subps %%xmm2, %%xmm1 \n\t" + "movaps %%xmm0, (%0,%1) \n\t" + "movaps %%xmm1, 16(%0,%1) \n\t" + "jg 1b \n\t" + :"+r"(i) + :"r"(z) + ); /* pass 2 .. ln-1 */ - nblocks = np >> 3; + nblocks = 1 << (ln-3); nloops = 1 << 2; - np2 = np >> 1; - - cptr1 = s->exptab1; + cptr = s->exptab1; do { p = z; - q = z + nloops; j = nblocks; do { - cptr = cptr1; - k = nloops >> 1; - do { - __m128 a, b, c, t1, t2; - - a = *(__m128 *)p; - b = *(__m128 *)q; - - /* complex mul */ - c = *(__m128 *)cptr; - /* cre*re cim*re */ - t1 = _mm_mul_ps(c, - _mm_shuffle_ps(b, b, _MM_SHUFFLE(2, 2, 0, 0))); - c = *(__m128 *)(cptr + 2); - /* -cim*im cre*im */ - t2 = _mm_mul_ps(c, - _mm_shuffle_ps(b, b, _MM_SHUFFLE(3, 3, 1, 1))); - b = _mm_add_ps(t1, t2); - - /* butterfly */ - *(__m128 *)p = _mm_add_ps(a, b); - *(__m128 *)q = _mm_sub_ps(a, b); - - p += 2; - q += 2; - cptr += 4; - } while (--k); - - p += nloops; - q += nloops; + i = nloops*8; + asm volatile( + "1: \n\t" + "sub $16, %0 \n\t" + "movaps (%2,%0), %%xmm1 \n\t" + "movaps (%1,%0), %%xmm0 \n\t" + "movaps %%xmm1, %%xmm2 \n\t" + "shufps $0xA0, %%xmm1, %%xmm1 \n\t" + "shufps $0xF5, %%xmm2, %%xmm2 \n\t" + "mulps (%3,%0,2), %%xmm1 \n\t" // cre*re cim*re + "mulps 16(%3,%0,2), %%xmm2 \n\t" // -cim*im cre*im + "addps %%xmm2, %%xmm1 \n\t" + "movaps %%xmm0, %%xmm3 \n\t" + "addps %%xmm1, %%xmm0 \n\t" + "subps %%xmm1, %%xmm3 \n\t" + "movaps %%xmm0, (%1,%0) \n\t" + "movaps %%xmm3, (%2,%0) \n\t" + "jg 1b \n\t" + :"+r"(i) + :"r"(p), "r"(p + nloops), "r"(cptr) + ); + p += nloops*2; } while (--j); - cptr1 += nloops * 2; - nblocks = nblocks >> 1; - nloops = nloops << 1; + cptr += nloops*2; + nblocks >>= 1; + nloops <<= 1; } while (nblocks != 0); } -#endif