1 /*
2 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #ifndef OS_CPU_LINUX_AARCH64_VM_COPY_LINUX_AARCH64_INLINE_HPP
27 #define OS_CPU_LINUX_AARCH64_VM_COPY_LINUX_AARCH64_INLINE_HPP
28
29 #define COPY_SMALL(from, to, count) \
30 { \
31 long tmp0, tmp1, tmp2, tmp3; \
32 long tmp4, tmp5, tmp6, tmp7; \
33 __asm volatile( \
34 " adr %[t0], 0f;" \
35 " add %[t0], %[t0], %[cnt], lsl #5;" \
36 " br %[t0];" \
37 " .align 5;" \
38 "0:" \
39 " b 1f;" \
40 " .align 5;" \
41 " ldr %[t0], [%[s], #0];" \
42 " str %[t0], [%[d], #0];" \
43 " b 1f;" \
44 " .align 5;" \
45 " ldp %[t0], %[t1], [%[s], #0];" \
46 " stp %[t0], %[t1], [%[d], #0];" \
47 " b 1f;" \
48 " .align 5;" \
49 " ldp %[t0], %[t1], [%[s], #0];" \
50 " ldr %[t2], [%[s], #16];" \
51 " stp %[t0], %[t1], [%[d], #0];" \
52 " str %[t2], [%[d], #16];" \
53 " b 1f;" \
54 " .align 5;" \
55 " ldp %[t0], %[t1], [%[s], #0];" \
56 " ldp %[t2], %[t3], [%[s], #16];" \
57 " stp %[t0], %[t1], [%[d], #0];" \
58 " stp %[t2], %[t3], [%[d], #16];" \
59 " b 1f;" \
60 " .align 5;" \
61 " ldp %[t0], %[t1], [%[s], #0];" \
62 " ldp %[t2], %[t3], [%[s], #16];" \
63 " ldr %[t4], [%[s], #32];" \
64 " stp %[t0], %[t1], [%[d], #0];" \
65 " stp %[t2], %[t3], [%[d], #16];" \
66 " str %[t4], [%[d], #32];" \
67 " b 1f;" \
68 " .align 5;" \
69 " ldp %[t0], %[t1], [%[s], #0];" \
70 " ldp %[t2], %[t3], [%[s], #16];" \
71 " ldp %[t4], %[t5], [%[s], #32];" \
72 "2:" \
73 " stp %[t0], %[t1], [%[d], #0];" \
74 " stp %[t2], %[t3], [%[d], #16];" \
75 " stp %[t4], %[t5], [%[d], #32];" \
76 " b 1f;" \
77 " .align 5;" \
78 " ldr %[t6], [%[s], #0];" \
79 " ldp %[t0], %[t1], [%[s], #8];" \
80 " ldp %[t2], %[t3], [%[s], #24];" \
81 " ldp %[t4], %[t5], [%[s], #40];" \
82 " str %[t6], [%[d]], #8;" \
83 " b 2b;" \
84 " .align 5;" \
85 " ldp %[t0], %[t1], [%[s], #0];" \
86 " ldp %[t2], %[t3], [%[s], #16];" \
87 " ldp %[t4], %[t5], [%[s], #32];" \
88 " ldp %[t6], %[t7], [%[s], #48];" \
89 " stp %[t0], %[t1], [%[d], #0];" \
90 " stp %[t2], %[t3], [%[d], #16];" \
91 " stp %[t4], %[t5], [%[d], #32];" \
92 " stp %[t6], %[t7], [%[d], #48];" \
93 "1:" \
94 \
95 : [s]"+r"(from), [d]"+r"(to), [cnt]"+r"(count), \
96 [t0]"=&r"(tmp0), [t1]"=&r"(tmp1), [t2]"=&r"(tmp2), [t3]"=&r"(tmp3), \
97 [t4]"=&r"(tmp4), [t5]"=&r"(tmp5), [t6]"=&r"(tmp6), [t7]"=&r"(tmp7) \
98 : \
99 : "memory", "cc"); \
100 }
101
102 static void pd_conjoint_words(HeapWord* from, HeapWord* to, size_t count) {
103 __asm volatile( "prfm pldl1strm, [%[s], #0];" :: [s]"r"(from) : "memory");
104 if (__builtin_expect(count <= 8, 1)) {
105 COPY_SMALL(from, to, count);
106 return;
107 }
108 _Copy_conjoint_words(from, to, count);
109 }
110
111 static void pd_disjoint_words(HeapWord* from, HeapWord* to, size_t count) {
112 if (__builtin_constant_p(count)) {
113 memcpy(to, from, count * sizeof(HeapWord));
114 return;
115 }
116 __asm volatile( "prfm pldl1strm, [%[s], #0];" :: [s]"r"(from) : "memory");
117 if (__builtin_expect(count <= 8, 1)) {
118 COPY_SMALL(from, to, count);
119 return;
120 }
121 _Copy_disjoint_words(from, to, count);
122 }
123
124 static void pd_disjoint_words_atomic(HeapWord* from, HeapWord* to, size_t count) {
125 __asm volatile( "prfm pldl1strm, [%[s], #0];" :: [s]"r"(from) : "memory");
126 if (__builtin_expect(count <= 8, 1)) {
127 COPY_SMALL(from, to, count);
128 return;
129 }
130 _Copy_disjoint_words(from, to, count);
131 }
132
133 static void pd_aligned_conjoint_words(HeapWord* from, HeapWord* to, size_t count) {
134 pd_conjoint_words(from, to, count);
135 }
136
137 static void pd_aligned_disjoint_words(HeapWord* from, HeapWord* to, size_t count) {
138 pd_disjoint_words(from, to, count);
139 }
140
141 static void pd_conjoint_bytes(void* from, void* to, size_t count) {
142 (void)memmove(to, from, count);
143 }
144
145 static void pd_conjoint_bytes_atomic(void* from, void* to, size_t count) {
146 pd_conjoint_bytes(from, to, count);
147 }
148
149 static void pd_conjoint_jshorts_atomic(jshort* from, jshort* to, size_t count) {
150 _Copy_conjoint_jshorts_atomic(from, to, count);
151 }
152
153 static void pd_conjoint_jints_atomic(jint* from, jint* to, size_t count) {
154 _Copy_conjoint_jints_atomic(from, to, count);
155 }
156
157 static void pd_conjoint_jlongs_atomic(jlong* from, jlong* to, size_t count) {
158 _Copy_conjoint_jlongs_atomic(from, to, count);
159 }
160
161 static void pd_conjoint_oops_atomic(oop* from, oop* to, size_t count) {
162 assert(!UseCompressedOops, "foo!");
163 assert(BytesPerLong == BytesPerOop, "jlongs and oops must be the same size");
164 _Copy_conjoint_jlongs_atomic((jlong*)from, (jlong*)to, count);
165 }
166
167 static void pd_arrayof_conjoint_bytes(HeapWord* from, HeapWord* to, size_t count) {
168 _Copy_arrayof_conjoint_bytes(from, to, count);
169 }
170
171 static void pd_arrayof_conjoint_jshorts(HeapWord* from, HeapWord* to, size_t count) {
172 _Copy_arrayof_conjoint_jshorts(from, to, count);
173 }
174
175 static void pd_arrayof_conjoint_jints(HeapWord* from, HeapWord* to, size_t count) {
176 _Copy_arrayof_conjoint_jints(from, to, count);
177 }
178
179 static void pd_arrayof_conjoint_jlongs(HeapWord* from, HeapWord* to, size_t count) {
180 _Copy_arrayof_conjoint_jlongs(from, to, count);
181 }
182
183 static void pd_arrayof_conjoint_oops(HeapWord* from, HeapWord* to, size_t count) {
184 assert(!UseCompressedOops, "foo!");
185 assert(BytesPerLong == BytesPerOop, "jlongs and oops must be the same size");
186 _Copy_arrayof_conjoint_jlongs(from, to, count);
187 }
188
189 #endif // OS_CPU_LINUX_AARCH64_VM_COPY_LINUX_AARCH64_INLINE_HPP