1 /*
2 * Copyright (c) 2003, 2004, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 static void pd_conjoint_words(HeapWord* from, HeapWord* to, size_t count) {
26 #ifdef AMD64
27 (void)memmove(to, from, count * HeapWordSize);
28 #else
29 // Includes a zero-count check.
30 intx temp;
31 __asm__ volatile(" testl %6,%6 ;"
32 " jz 7f ;"
33 " cmpl %4,%5 ;"
34 " leal -4(%4,%6,4),%3;"
35 " jbe 1f ;"
36 " cmpl %7,%5 ;"
37 " jbe 4f ;"
38 "1: cmpl $32,%6 ;"
39 " ja 3f ;"
40 " subl %4,%1 ;"
41 "2: movl (%4),%3 ;"
42 " movl %7,(%5,%4,1) ;"
43 " addl $4,%0 ;"
44 " subl $1,%2 ;"
45 " jnz 2b ;"
46 " jmp 7f ;"
47 "3: rep; smovl ;"
48 " jmp 7f ;"
49 "4: cmpl $32,%2 ;"
50 " movl %7,%0 ;"
51 " leal -4(%5,%6,4),%1;"
52 " ja 6f ;"
53 " subl %4,%1 ;"
54 "5: movl (%4),%3 ;"
55 " movl %7,(%5,%4,1) ;"
56 " subl $4,%0 ;"
57 " subl $1,%2 ;"
58 " jnz 5b ;"
59 " jmp 7f ;"
60 "6: std ;"
61 " rep; smovl ;"
62 " cld ;"
63 "7: nop "
64 : "=S" (from), "=D" (to), "=c" (count), "=r" (temp)
65 : "0" (from), "1" (to), "2" (count), "3" (temp)
66 : "memory", "flags");
67 #endif // AMD64
68 }
69
70 static void pd_disjoint_words(HeapWord* from, HeapWord* to, size_t count) {
71 #ifdef AMD64
72 switch (count) {
73 case 8: to[7] = from[7];
74 case 7: to[6] = from[6];
75 case 6: to[5] = from[5];
76 case 5: to[4] = from[4];
77 case 4: to[3] = from[3];
78 case 3: to[2] = from[2];
79 case 2: to[1] = from[1];
80 case 1: to[0] = from[0];
81 case 0: break;
82 default:
83 (void)memcpy(to, from, count * HeapWordSize);
84 break;
85 }
86 #else
87 // Includes a zero-count check.
88 intx temp;
89 __asm__ volatile(" testl %6,%6 ;"
90 " jz 3f ;"
91 " cmpl $32,%6 ;"
92 " ja 2f ;"
93 " subl %4,%1 ;"
94 "1: movl (%4),%3 ;"
95 " movl %7,(%5,%4,1);"
96 " addl $4,%0 ;"
97 " subl $1,%2 ;"
98 " jnz 1b ;"
99 " jmp 3f ;"
100 "2: rep; smovl ;"
101 "3: nop "
102 : "=S" (from), "=D" (to), "=c" (count), "=r" (temp)
103 : "0" (from), "1" (to), "2" (count), "3" (temp)
104 : "memory", "cc");
105 #endif // AMD64
106 }
107
108 static void pd_disjoint_words_atomic(HeapWord* from, HeapWord* to, size_t count) {
109 #ifdef AMD64
110 switch (count) {
111 case 8: to[7] = from[7];
112 case 7: to[6] = from[6];
113 case 6: to[5] = from[5];
114 case 5: to[4] = from[4];
115 case 4: to[3] = from[3];
116 case 3: to[2] = from[2];
117 case 2: to[1] = from[1];
118 case 1: to[0] = from[0];
119 case 0: break;
120 default:
121 while (count-- > 0) {
122 *to++ = *from++;
123 }
124 break;
125 }
126 #else
127 // pd_disjoint_words is word-atomic in this implementation.
128 pd_disjoint_words(from, to, count);
129 #endif // AMD64
130 }
131
132 static void pd_aligned_conjoint_words(HeapWord* from, HeapWord* to, size_t count) {
133 pd_conjoint_words(from, to, count);
134 }
135
136 static void pd_aligned_disjoint_words(HeapWord* from, HeapWord* to, size_t count) {
137 pd_disjoint_words(from, to, count);
138 }
139
140 static void pd_conjoint_bytes(void* from, void* to, size_t count) {
141 #ifdef AMD64
142 (void)memmove(to, from, count);
143 #else
144 // Includes a zero-count check.
145 intx temp;
146 __asm__ volatile(" testl %6,%6 ;"
147 " jz 13f ;"
148 " cmpl %4,%5 ;"
149 " leal -1(%4,%6),%3 ;"
150 " jbe 1f ;"
151 " cmpl %7,%5 ;"
152 " jbe 8f ;"
153 "1: cmpl $3,%6 ;"
154 " jbe 6f ;"
155 " movl %6,%3 ;"
156 " movl $4,%2 ;"
157 " subl %4,%2 ;"
158 " andl $3,%2 ;"
159 " jz 2f ;"
160 " subl %6,%3 ;"
161 " rep; smovb ;"
162 "2: movl %7,%2 ;"
163 " shrl $2,%2 ;"
164 " jz 5f ;"
165 " cmpl $32,%2 ;"
166 " ja 4f ;"
167 " subl %4,%1 ;"
168 "3: movl (%4),%%edx ;"
169 " movl %%edx,(%5,%4,1);"
170 " addl $4,%0 ;"
171 " subl $1,%2 ;"
172 " jnz 3b ;"
173 " addl %4,%1 ;"
174 " jmp 5f ;"
175 "4: rep; smovl ;"
176 "5: movl %7,%2 ;"
177 " andl $3,%2 ;"
178 " jz 13f ;"
179 "6: xorl %7,%3 ;"
180 "7: movb (%4,%7,1),%%dl ;"
181 " movb %%dl,(%5,%7,1) ;"
182 " addl $1,%3 ;"
183 " subl $1,%2 ;"
184 " jnz 7b ;"
185 " jmp 13f ;"
186 "8: std ;"
187 " cmpl $12,%2 ;"
188 " ja 9f ;"
189 " movl %7,%0 ;"
190 " leal -1(%6,%5),%1 ;"
191 " jmp 11f ;"
192 "9: xchgl %3,%2 ;"
193 " movl %6,%0 ;"
194 " addl $1,%2 ;"
195 " leal -1(%7,%5),%1 ;"
196 " andl $3,%2 ;"
197 " jz 10f ;"
198 " subl %6,%3 ;"
199 " rep; smovb ;"
200 "10: movl %7,%2 ;"
201 " subl $3,%0 ;"
202 " shrl $2,%2 ;"
203 " subl $3,%1 ;"
204 " rep; smovl ;"
205 " andl $3,%3 ;"
206 " jz 12f ;"
207 " movl %7,%2 ;"
208 " addl $3,%0 ;"
209 " addl $3,%1 ;"
210 "11: rep; smovb ;"
211 "12: cld ;"
212 "13: nop ;"
213 : "=S" (from), "=D" (to), "=c" (count), "=r" (temp)
214 : "0" (from), "1" (to), "2" (count), "3" (temp)
215 : "memory", "flags", "%edx");
216 #endif // AMD64
217 }
218
219 static void pd_conjoint_bytes_atomic(void* from, void* to, size_t count) {
220 pd_conjoint_bytes(from, to, count);
221 }
222
223 static void pd_conjoint_jshorts_atomic(jshort* from, jshort* to, size_t count) {
224 _Copy_conjoint_jshorts_atomic(from, to, count);
225 }
226
227 static void pd_conjoint_jints_atomic(jint* from, jint* to, size_t count) {
228 #ifdef AMD64
229 _Copy_conjoint_jints_atomic(from, to, count);
230 #else
231 assert(HeapWordSize == BytesPerInt, "heapwords and jints must be the same size");
232 // pd_conjoint_words is word-atomic in this implementation.
233 pd_conjoint_words((HeapWord*)from, (HeapWord*)to, count);
234 #endif // AMD64
235 }
236
237 static void pd_conjoint_jlongs_atomic(jlong* from, jlong* to, size_t count) {
238 #ifdef AMD64
239 _Copy_conjoint_jlongs_atomic(from, to, count);
240 #else
241 // Guarantee use of fild/fistp or xmm regs via some asm code, because compilers won't.
242 if (from > to) {
243 while (count-- > 0) {
244 __asm__ volatile("fildll (%0); fistpll (%1)"
245 :
246 : "r" (from), "r" (to)
247 : "memory" );
248 ++from;
249 ++to;
250 }
251 } else {
252 while (count-- > 0) {
253 __asm__ volatile("fildll (%0,%2,8); fistpll (%1,%2,8)"
254 :
255 : "r" (from), "r" (to), "r" (count)
256 : "memory" );
257 }
258 }
259 #endif // AMD64
260 }
261
262 static void pd_conjoint_oops_atomic(oop* from, oop* to, size_t count) {
263 #ifdef AMD64
264 assert(BytesPerLong == BytesPerOop, "jlongs and oops must be the same size");
265 _Copy_conjoint_jlongs_atomic((jlong*)from, (jlong*)to, count);
266 #else
267 assert(HeapWordSize == BytesPerOop, "heapwords and oops must be the same size");
268 // pd_conjoint_words is word-atomic in this implementation.
269 pd_conjoint_words((HeapWord*)from, (HeapWord*)to, count);
270 #endif // AMD64
271 }
272
273 static void pd_arrayof_conjoint_bytes(HeapWord* from, HeapWord* to, size_t count) {
274 _Copy_arrayof_conjoint_bytes(from, to, count);
275 }
276
277 static void pd_arrayof_conjoint_jshorts(HeapWord* from, HeapWord* to, size_t count) {
278 _Copy_arrayof_conjoint_jshorts(from, to, count);
279 }
280
281 static void pd_arrayof_conjoint_jints(HeapWord* from, HeapWord* to, size_t count) {
282 #ifdef AMD64
283 _Copy_arrayof_conjoint_jints(from, to, count);
284 #else
285 pd_conjoint_jints_atomic((jint*)from, (jint*)to, count);
286 #endif // AMD64
287 }
288
289 static void pd_arrayof_conjoint_jlongs(HeapWord* from, HeapWord* to, size_t count) {
290 #ifdef AMD64
291 _Copy_arrayof_conjoint_jlongs(from, to, count);
292 #else
293 pd_conjoint_jlongs_atomic((jlong*)from, (jlong*)to, count);
294 #endif // AMD64
295 }
296
297 static void pd_arrayof_conjoint_oops(HeapWord* from, HeapWord* to, size_t count) {
298 #ifdef AMD64
299 assert(BytesPerLong == BytesPerOop, "jlongs and oops must be the same size");
300 _Copy_arrayof_conjoint_jlongs(from, to, count);
301 #else
302 pd_conjoint_oops_atomic((oop*)from, (oop*)to, count);
303 #endif // AMD64
304 }