13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #ifndef OS_CPU_LINUX_X86_VM_COPY_LINUX_X86_INLINE_HPP
26 #define OS_CPU_LINUX_X86_VM_COPY_LINUX_X86_INLINE_HPP
27
28 static void pd_conjoint_words(HeapWord* from, HeapWord* to, size_t count) {
29 #ifdef AMD64
30 (void)memmove(to, from, count * HeapWordSize);
31 #else
32 // Includes a zero-count check.
33 intx temp;
34 __asm__ volatile(" testl %6,%6 ;"
35 " jz 7f ;"
36 " cmpl %4,%5 ;"
37 " leal -4(%4,%6,4),%3;"
38 " jbe 1f ;"
39 " cmpl %7,%5 ;"
40 " jbe 4f ;"
41 "1: cmpl $32,%6 ;"
42 " ja 3f ;"
43 " subl %4,%1 ;"
44 "2: movl (%4),%3 ;"
45 " movl %7,(%5,%4,1) ;"
46 " addl $4,%0 ;"
47 " subl $1,%2 ;"
48 " jnz 2b ;"
49 " jmp 7f ;"
50 "3: rep; smovl ;"
51 " jmp 7f ;"
52 "4: cmpl $32,%2 ;"
53 " movl %7,%0 ;"
71 }
72
73 static void pd_disjoint_words(HeapWord* from, HeapWord* to, size_t count) {
74 #ifdef AMD64
75 switch (count) {
76 case 8: to[7] = from[7];
77 case 7: to[6] = from[6];
78 case 6: to[5] = from[5];
79 case 5: to[4] = from[4];
80 case 4: to[3] = from[3];
81 case 3: to[2] = from[2];
82 case 2: to[1] = from[1];
83 case 1: to[0] = from[0];
84 case 0: break;
85 default:
86 (void)memcpy(to, from, count * HeapWordSize);
87 break;
88 }
89 #else
90 // Includes a zero-count check.
91 intx temp;
92 __asm__ volatile(" testl %6,%6 ;"
93 " jz 3f ;"
94 " cmpl $32,%6 ;"
95 " ja 2f ;"
96 " subl %4,%1 ;"
97 "1: movl (%4),%3 ;"
98 " movl %7,(%5,%4,1);"
99 " addl $4,%0 ;"
100 " subl $1,%2 ;"
101 " jnz 1b ;"
102 " jmp 3f ;"
103 "2: rep; smovl ;"
104 "3: nop "
105 : "=S" (from), "=D" (to), "=c" (count), "=r" (temp)
106 : "0" (from), "1" (to), "2" (count), "3" (temp)
107 : "memory", "cc");
108 #endif // AMD64
109 }
110
111 static void pd_disjoint_words_atomic(HeapWord* from, HeapWord* to, size_t count) {
128 }
129 #else
130 // pd_disjoint_words is word-atomic in this implementation.
131 pd_disjoint_words(from, to, count);
132 #endif // AMD64
133 }
134
135 static void pd_aligned_conjoint_words(HeapWord* from, HeapWord* to, size_t count) {
136 pd_conjoint_words(from, to, count);
137 }
138
139 static void pd_aligned_disjoint_words(HeapWord* from, HeapWord* to, size_t count) {
140 pd_disjoint_words(from, to, count);
141 }
142
143 static void pd_conjoint_bytes(void* from, void* to, size_t count) {
144 #ifdef AMD64
145 (void)memmove(to, from, count);
146 #else
147 // Includes a zero-count check.
148 intx temp;
149 __asm__ volatile(" testl %6,%6 ;"
150 " jz 13f ;"
151 " cmpl %4,%5 ;"
152 " leal -1(%4,%6),%3 ;"
153 " jbe 1f ;"
154 " cmpl %7,%5 ;"
155 " jbe 8f ;"
156 "1: cmpl $3,%6 ;"
157 " jbe 6f ;"
158 " movl %6,%3 ;"
159 " movl $4,%2 ;"
160 " subl %4,%2 ;"
161 " andl $3,%2 ;"
162 " jz 2f ;"
163 " subl %6,%3 ;"
164 " rep; smovb ;"
165 "2: movl %7,%2 ;"
166 " shrl $2,%2 ;"
167 " jz 5f ;"
168 " cmpl $32,%2 ;"
|
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #ifndef OS_CPU_LINUX_X86_VM_COPY_LINUX_X86_INLINE_HPP
26 #define OS_CPU_LINUX_X86_VM_COPY_LINUX_X86_INLINE_HPP
27
28 static void pd_conjoint_words(HeapWord* from, HeapWord* to, size_t count) {
29 #ifdef AMD64
30 (void)memmove(to, from, count * HeapWordSize);
31 #else
32 // Includes a zero-count check.
33 intx temp = 0;
34 __asm__ volatile(" testl %6,%6 ;"
35 " jz 7f ;"
36 " cmpl %4,%5 ;"
37 " leal -4(%4,%6,4),%3;"
38 " jbe 1f ;"
39 " cmpl %7,%5 ;"
40 " jbe 4f ;"
41 "1: cmpl $32,%6 ;"
42 " ja 3f ;"
43 " subl %4,%1 ;"
44 "2: movl (%4),%3 ;"
45 " movl %7,(%5,%4,1) ;"
46 " addl $4,%0 ;"
47 " subl $1,%2 ;"
48 " jnz 2b ;"
49 " jmp 7f ;"
50 "3: rep; smovl ;"
51 " jmp 7f ;"
52 "4: cmpl $32,%2 ;"
53 " movl %7,%0 ;"
71 }
72
73 static void pd_disjoint_words(HeapWord* from, HeapWord* to, size_t count) {
74 #ifdef AMD64
75 switch (count) {
76 case 8: to[7] = from[7];
77 case 7: to[6] = from[6];
78 case 6: to[5] = from[5];
79 case 5: to[4] = from[4];
80 case 4: to[3] = from[3];
81 case 3: to[2] = from[2];
82 case 2: to[1] = from[1];
83 case 1: to[0] = from[0];
84 case 0: break;
85 default:
86 (void)memcpy(to, from, count * HeapWordSize);
87 break;
88 }
89 #else
90 // Includes a zero-count check.
91 intx temp = 0;
92 __asm__ volatile(" testl %6,%6 ;"
93 " jz 3f ;"
94 " cmpl $32,%6 ;"
95 " ja 2f ;"
96 " subl %4,%1 ;"
97 "1: movl (%4),%3 ;"
98 " movl %7,(%5,%4,1);"
99 " addl $4,%0 ;"
100 " subl $1,%2 ;"
101 " jnz 1b ;"
102 " jmp 3f ;"
103 "2: rep; smovl ;"
104 "3: nop "
105 : "=S" (from), "=D" (to), "=c" (count), "=r" (temp)
106 : "0" (from), "1" (to), "2" (count), "3" (temp)
107 : "memory", "cc");
108 #endif // AMD64
109 }
110
111 static void pd_disjoint_words_atomic(HeapWord* from, HeapWord* to, size_t count) {
128 }
129 #else
130 // pd_disjoint_words is word-atomic in this implementation.
131 pd_disjoint_words(from, to, count);
132 #endif // AMD64
133 }
134
135 static void pd_aligned_conjoint_words(HeapWord* from, HeapWord* to, size_t count) {
136 pd_conjoint_words(from, to, count);
137 }
138
139 static void pd_aligned_disjoint_words(HeapWord* from, HeapWord* to, size_t count) {
140 pd_disjoint_words(from, to, count);
141 }
142
143 static void pd_conjoint_bytes(void* from, void* to, size_t count) {
144 #ifdef AMD64
145 (void)memmove(to, from, count);
146 #else
147 // Includes a zero-count check.
148 intx temp = 0;
149 __asm__ volatile(" testl %6,%6 ;"
150 " jz 13f ;"
151 " cmpl %4,%5 ;"
152 " leal -1(%4,%6),%3 ;"
153 " jbe 1f ;"
154 " cmpl %7,%5 ;"
155 " jbe 8f ;"
156 "1: cmpl $3,%6 ;"
157 " jbe 6f ;"
158 " movl %6,%3 ;"
159 " movl $4,%2 ;"
160 " subl %4,%2 ;"
161 " andl $3,%2 ;"
162 " jz 2f ;"
163 " subl %6,%3 ;"
164 " rep; smovb ;"
165 "2: movl %7,%2 ;"
166 " shrl $2,%2 ;"
167 " jz 5f ;"
168 " cmpl $32,%2 ;"
|