477
|
1 #ifndef __MPLAYER_MEMCPY
|
|
2 #define __MPLAYER_MEMCPY
|
|
3
|
358
|
4 /*
|
|
5 This part of code was taken by from Linux-2.4.3 and slightly modified
|
581
|
6 for MMX2, SSE instruction set. I have done it since linux uses page aligned
|
358
|
7 blocks but mplayer uses weakly ordered data and original sources can not
|
581
|
8 speedup them. Only using PREFETCHNTA and MOVNTQ together have effect!
|
|
9
|
|
10 From IA-32 Intel Architecture Software Developer's Manual Volume 1,
|
|
11 Order Number 245470:
|
|
12 "10.4.6. Cacheability Control, Prefetch, and Memory Ordering Instructions"
|
|
13
|
|
14 Data referenced by a program can be temporal (data will be used again) or
|
|
15 non-temporal (data will be referenced once and not reused in the immediate
|
|
16 future). To make efficient use of the processor's caches, it is generally
|
|
17 desirable to cache temporal data and not cache non-temporal data. Overloading
|
|
18 the processor's caches with non-temporal data is sometimes referred to as
|
|
19 "polluting the caches".
|
|
20 The non-temporal data is written to memory with Write-Combining semantics.
|
|
21
|
|
22 The PREFETCHh instructions permits a program to load data into the processor
|
|
23 at a suggested cache level, so that it is closer to the processors load and
|
|
24 store unit when it is needed. If the data is already present in a level of
|
|
25 the cache hierarchy that is closer to the processor, the PREFETCHh instruction
|
|
26 will not result in any data movement.
|
|
27 But we should you PREFETCHNTA: Non-temporal data fetch data into location
|
|
28 close to the processor, minimizing cache pollution.
|
|
29
|
|
30 The MOVNTQ (store quadword using non-temporal hint) instruction stores
|
|
31 packed integer data from an MMX register to memory, using a non-temporal hint.
|
|
32 The MOVNTPS (store packed single-precision floating-point values using
|
|
33 non-temporal hint) instruction stores packed floating-point data from an
|
|
34 XMM register to memory, using a non-temporal hint.
|
|
35
|
|
36 The SFENCE (Store Fence) instruction controls write ordering by creating a
|
|
37 fence for memory store operations. This instruction guarantees that the results
|
|
38 of every store instruction that precedes the store fence in program order is
|
|
39 globally visible before any store instruction that follows the fence. The
|
|
40 SFENCE instruction provides an efficient way of ensuring ordering between
|
|
41 procedures that produce weakly-ordered data and procedures that consume that
|
|
42 data.
|
|
43
|
358
|
44 If you have questions please contact with me: Nick Kurshev: nickols_k@mail.ru.
|
|
45 */
|
477
|
46
|
|
47 // 3dnow memcpy support from kernel 2.4.2
|
|
48 // by Pontscho/fresh!mindworkz
|
|
49
|
|
50 #if defined( HAVE_MMX2 ) || defined( HAVE_3DNOW )
|
|
51
|
581
|
52 #undef HAVE_K6_2PLUS
|
|
53 #if !defined( HAVE_MMX2) && defined( HAVE_3DNOW)
|
|
54 #define HAVE_K6_2PLUS
|
|
55 #endif
|
|
56
|
376
|
57 /* for small memory blocks (<256 bytes) this version is faster */
|
|
58 #define small_memcpy(to,from,n)\
|
|
59 {\
|
|
60 __asm__ __volatile__(\
|
|
61 "rep ; movsb\n"\
|
|
62 ::"D" (to), "S" (from),"c" (n)\
|
|
63 : "memory");\
|
|
64 }
|
358
|
65
|
370
|
66 inline static void * fast_memcpy(void * to, const void * from, unsigned len)
|
358
|
67 {
|
|
68 void *p;
|
|
69 int i;
|
|
70
|
567
|
71 #ifdef HAVE_SSE /* Only P3 (may be Cyrix3) */
|
|
72 // printf("fastmemcpy_pre(0x%X,0x%X,0x%X)\n",to,from,len);
|
|
73 // Align dest to 16-byte boundary:
|
581
|
74 if((unsigned long)to&15){
|
|
75 int len2=16-((unsigned long)to&15);
|
567
|
76 if(len>len2){
|
|
77 len-=len2;
|
|
78 __asm__ __volatile__(
|
|
79 "rep ; movsb\n"
|
|
80 :"=D" (to), "=S" (from)
|
|
81 : "D" (to), "S" (from),"c" (len2)
|
|
82 : "memory");
|
|
83 }
|
|
84 }
|
|
85 // printf("fastmemcpy(0x%X,0x%X,0x%X)\n",to,from,len);
|
|
86 #endif
|
|
87
|
358
|
88 if(len >= 0x200) /* 512-byte blocks */
|
|
89 {
|
|
90 p = to;
|
|
91 i = len >> 6; /* len/64 */
|
376
|
92 len&=63;
|
|
93
|
358
|
94 __asm__ __volatile__ (
|
581
|
95 #ifdef HAVE_K6_2PLUS
|
477
|
96 "prefetch (%0)\n"
|
|
97 "prefetch 64(%0)\n"
|
|
98 "prefetch 128(%0)\n"
|
|
99 "prefetch 192(%0)\n"
|
|
100 "prefetch 256(%0)\n"
|
581
|
101 #else /* K7, P3, CyrixIII */
|
409
|
102 "prefetchnta (%0)\n"
|
|
103 "prefetchnta 64(%0)\n"
|
|
104 "prefetchnta 128(%0)\n"
|
|
105 "prefetchnta 192(%0)\n"
|
|
106 "prefetchnta 256(%0)\n"
|
477
|
107 #endif
|
358
|
108 : : "r" (from) );
|
409
|
109 /*
|
|
110 This algorithm is top effective when the code consequently
|
|
111 reads and writes blocks which have size of cache line.
|
|
112 Size of cache line is processor-dependent.
|
|
113 It will, however, be a minimum of 32 bytes on any processors.
|
|
114 It would be better to have a number of instructions which
|
|
115 perform reading and writing to be multiple to a number of
|
|
116 processor's decoders, but it's not always possible.
|
|
117 */
|
581
|
118 #ifdef HAVE_SSE /* Only P3 (may be Cyrix3) */
|
|
119 if(((unsigned long)from) & 15)
|
|
120 /* if SRC is misaligned */
|
358
|
121 for(; i>0; i--)
|
|
122 {
|
|
123 __asm__ __volatile__ (
|
409
|
124 "prefetchnta 320(%0)\n"
|
|
125 "movups (%0), %%xmm0\n"
|
|
126 "movups 16(%0), %%xmm1\n"
|
|
127 "movntps %%xmm0, (%1)\n"
|
|
128 "movntps %%xmm1, 16(%1)\n"
|
|
129 "movups 32(%0), %%xmm0\n"
|
|
130 "movups 48(%0), %%xmm1\n"
|
|
131 "movntps %%xmm0, 32(%1)\n"
|
|
132 "movntps %%xmm1, 48(%1)\n"
|
581
|
133 :: "r" (from), "r" (to) : "memory");
|
|
134 from+=64;
|
|
135 to+=64;
|
|
136 }
|
|
137 else
|
|
138 /*
|
|
139 Only if SRC is aligned on 16-byte boundary.
|
|
140 It allows to use movaps instead of movups, which required data
|
|
141 to be aligned or a general-protection exception (#GP) is generated.
|
|
142 */
|
|
143 for(; i>0; i--)
|
|
144 {
|
|
145 __asm__ __volatile__ (
|
|
146 "prefetchnta 320(%0)\n"
|
|
147 "movaps (%0), %%xmm0\n"
|
|
148 "movaps 16(%0), %%xmm1\n"
|
|
149 "movntps %%xmm0, (%1)\n"
|
|
150 "movntps %%xmm1, 16(%1)\n"
|
|
151 "movaps 32(%0), %%xmm0\n"
|
|
152 "movaps 48(%0), %%xmm1\n"
|
|
153 "movntps %%xmm0, 32(%1)\n"
|
|
154 "movntps %%xmm1, 48(%1)\n"
|
|
155 :: "r" (from), "r" (to) : "memory");
|
|
156 from+=64;
|
|
157 to+=64;
|
|
158 }
|
|
159 #else
|
|
160 for(; i>0; i--)
|
|
161 {
|
|
162 __asm__ __volatile__ (
|
|
163 #ifdef HAVE_K6_2PLUS
|
|
164 "prefetch 320(%0)\n"
|
|
165 #else
|
|
166 "prefetchnta 320(%0)\n"
|
|
167 #endif
|
|
168 #ifdef HAVE_K6_2PLUS
|
477
|
169 "movq (%0), %%mm0\n"
|
|
170 "movq 8(%0), %%mm1\n"
|
|
171 "movq 16(%0), %%mm2\n"
|
|
172 "movq 24(%0), %%mm3\n"
|
|
173 "movq %%mm0, (%1)\n"
|
|
174 "movq %%mm1, 8(%1)\n"
|
|
175 "movq %%mm2, 16(%1)\n"
|
|
176 "movq %%mm3, 24(%1)\n"
|
|
177 "movq 32(%0), %%mm0\n"
|
|
178 "movq 40(%0), %%mm1\n"
|
|
179 "movq 48(%0), %%mm2\n"
|
|
180 "movq 56(%0), %%mm3\n"
|
|
181 "movq %%mm0, 32(%1)\n"
|
|
182 "movq %%mm1, 40(%1)\n"
|
|
183 "movq %%mm2, 48(%1)\n"
|
|
184 "movq %%mm3, 56(%1)\n"
|
581
|
185 #else /* K7 */
|
409
|
186 "movq (%0), %%mm0\n"
|
|
187 "movq 8(%0), %%mm1\n"
|
|
188 "movq 16(%0), %%mm2\n"
|
|
189 "movq 24(%0), %%mm3\n"
|
|
190 "movntq %%mm0, (%1)\n"
|
|
191 "movntq %%mm1, 8(%1)\n"
|
|
192 "movntq %%mm2, 16(%1)\n"
|
|
193 "movntq %%mm3, 24(%1)\n"
|
|
194 "movq 32(%0), %%mm0\n"
|
|
195 "movq 40(%0), %%mm1\n"
|
|
196 "movq 48(%0), %%mm2\n"
|
|
197 "movq 56(%0), %%mm3\n"
|
|
198 "movntq %%mm0, 32(%1)\n"
|
|
199 "movntq %%mm1, 40(%1)\n"
|
|
200 "movntq %%mm2, 48(%1)\n"
|
|
201 "movntq %%mm3, 56(%1)\n"
|
|
202 #endif
|
|
203 :: "r" (from), "r" (to) : "memory");
|
358
|
204 from+=64;
|
|
205 to+=64;
|
|
206 }
|
581
|
207 #endif /* Have SSE */
|
|
208 #ifdef HAVE_K6_2PLUS
|
|
209 /* On K6 femms is fatser of emms.
|
|
210 On K7 femms is directly mapped on emms. */
|
477
|
211 __asm__ __volatile__ ("femms":::"memory");
|
581
|
212 #else /* K7, P3, CyrixIII */
|
|
213 /* since movntq is weakly-ordered, a "sfence"
|
|
214 * is needed to become ordered again. */
|
|
215 __asm__ __volatile__ ("sfence":::"memory");
|
|
216 #ifndef HAVE_SSE
|
|
217 /* enables to use FPU */
|
409
|
218 __asm__ __volatile__ ("emms":::"memory");
|
581
|
219 #endif
|
477
|
220 #endif
|
358
|
221 }
|
|
222 /*
|
|
223 * Now do the tail of the block
|
|
224 */
|
376
|
225 small_memcpy(to, from, len);
|
358
|
226 return p;
|
|
227 }
|
376
|
228 #define memcpy(a,b,c) fast_memcpy(a,b,c)
|
513
|
229 #undef small_memcpy
|
|
230
|
358
|
231 #endif
|
|
232
|
478
|
233 #endif
|