33 /*
34 * Atomic long operations on 32-bit ARM
35 * ARM v7 supports LDREXD/STREXD synchronization instructions so no problem.
36 * ARM < v7 does not have explicit 64 atomic load/store capability.
37 * However, gcc emits LDRD/STRD instructions on v5te and LDM/STM on v5t
38 * when loading/storing 64 bits.
39 * For non-MP machines (which is all we support for ARM < v7)
40 * under current Linux distros these instructions appear atomic.
41 * See section A3.5.3 of ARM Architecture Reference Manual for ARM v7.
42 * Also, for cmpxchg64, if ARM < v7 we check for cmpxchg64 support in the
43 * Linux kernel using _kuser_helper_version. See entry-armv.S in the Linux
44 * kernel source or kernel_user_helpers.txt in Linux Doc.
45 */
46
47 #ifndef AARCH64
48 template<>
49 template<typename T>
50 inline T Atomic::PlatformLoad<8>::operator()(T const volatile* src) const {
51 STATIC_ASSERT(8 == sizeof(T));
52 return PrimitiveConversions::cast<T>(
53 (*os::atomic_load_long_func)(reinterpret_cast<const volatile jlong*>(src)));
54 }
55
56 template<>
57 template<typename T>
58 inline void Atomic::PlatformStore<8>::operator()(T store_value,
59 T volatile* dest) const {
60 STATIC_ASSERT(8 == sizeof(T));
61 (*os::atomic_store_long_func)(
62 PrimitiveConversions::cast<jlong>(store_value), reinterpret_cast<volatile jlong*>(dest));
63 }
64 #endif
65
66 // As per atomic.hpp all read-modify-write operations have to provide two-way
67 // barriers semantics. For AARCH64 we are using load-acquire-with-reservation and
68 // store-release-with-reservation. While load-acquire combined with store-release
69 // do not generally form two-way barriers, their use with reservations does - the
70 // ARMv8 architecture manual Section F "Barrier Litmus Tests" indicates they
71 // provide sequentially consistent semantics. All we need to add is an explicit
72 // barrier in the failure path of the cmpxchg operations (as these don't execute
73 // the store) - arguably this may be overly cautious as there is a very low
74 // likelihood that the hardware would pull loads/stores into the region guarded
75 // by the reservation.
76 //
77 // For ARMv7 we add explicit barriers in the stubs.
78
79 template<size_t byte_size>
80 struct Atomic::PlatformAdd
81 : Atomic::AddAndFetch<Atomic::PlatformAdd<byte_size> >
82 {
86
87 template<>
88 template<typename I, typename D>
89 inline D Atomic::PlatformAdd<4>::add_and_fetch(I add_value, D volatile* dest) const {
90 STATIC_ASSERT(4 == sizeof(I));
91 STATIC_ASSERT(4 == sizeof(D));
92 #ifdef AARCH64
93 D val;
94 int tmp;
95 __asm__ volatile(
96 "1:\n\t"
97 " ldaxr %w[val], [%[dest]]\n\t"
98 " add %w[val], %w[val], %w[add_val]\n\t"
99 " stlxr %w[tmp], %w[val], [%[dest]]\n\t"
100 " cbnz %w[tmp], 1b\n\t"
101 : [val] "=&r" (val), [tmp] "=&r" (tmp)
102 : [add_val] "r" (add_value), [dest] "r" (dest)
103 : "memory");
104 return val;
105 #else
106 return add_using_helper<jint>(os::atomic_add_func, add_value, dest);
107 #endif
108 }
109
110 #ifdef AARCH64
111 template<>
112 template<typename I, typename D>
113 inline D Atomic::PlatformAdd<8>::add_and_fetch(I add_value, D volatile* dest) const {
114 STATIC_ASSERT(8 == sizeof(I));
115 STATIC_ASSERT(8 == sizeof(D));
116 D val;
117 int tmp;
118 __asm__ volatile(
119 "1:\n\t"
120 " ldaxr %[val], [%[dest]]\n\t"
121 " add %[val], %[val], %[add_val]\n\t"
122 " stlxr %w[tmp], %[val], [%[dest]]\n\t"
123 " cbnz %w[tmp], 1b\n\t"
124 : [val] "=&r" (val), [tmp] "=&r" (tmp)
125 : [add_val] "r" (add_value), [dest] "r" (dest)
126 : "memory");
129 #endif
130
131 template<>
132 template<typename T>
133 inline T Atomic::PlatformXchg<4>::operator()(T exchange_value,
134 T volatile* dest) const {
135 STATIC_ASSERT(4 == sizeof(T));
136 #ifdef AARCH64
137 T old_val;
138 int tmp;
139 __asm__ volatile(
140 "1:\n\t"
141 " ldaxr %w[old_val], [%[dest]]\n\t"
142 " stlxr %w[tmp], %w[new_val], [%[dest]]\n\t"
143 " cbnz %w[tmp], 1b\n\t"
144 : [old_val] "=&r" (old_val), [tmp] "=&r" (tmp)
145 : [new_val] "r" (exchange_value), [dest] "r" (dest)
146 : "memory");
147 return old_val;
148 #else
149 return xchg_using_helper<jint>(os::atomic_xchg_func, exchange_value, dest);
150 #endif
151 }
152
153 #ifdef AARCH64
154 template<>
155 template<typename T>
156 inline T Atomic::PlatformXchg<8>::operator()(T exchange_value,
157 T volatile* dest) const {
158 STATIC_ASSERT(8 == sizeof(T));
159 T old_val;
160 int tmp;
161 __asm__ volatile(
162 "1:\n\t"
163 " ldaxr %[old_val], [%[dest]]\n\t"
164 " stlxr %w[tmp], %[new_val], [%[dest]]\n\t"
165 " cbnz %w[tmp], 1b\n\t"
166 : [old_val] "=&r" (old_val), [tmp] "=&r" (tmp)
167 : [new_val] "r" (exchange_value), [dest] "r" (dest)
168 : "memory");
169 return old_val;
170 }
171 #endif // AARCH64
172
173 // The memory_order parameter is ignored - we always provide the strongest/most-conservative ordering
174
175 // No direct support for cmpxchg of bytes; emulate using int.
176 template<>
177 struct Atomic::PlatformCmpxchg<1> : Atomic::CmpxchgByteUsingInt {};
178
179 #ifndef AARCH64
180
181 inline jint reorder_cmpxchg_func(jint exchange_value,
182 jint volatile* dest,
183 jint compare_value) {
184 // Warning: Arguments are swapped to avoid moving them for kernel call
185 return (*os::atomic_cmpxchg_func)(compare_value, exchange_value, dest);
186 }
187
188 inline jlong reorder_cmpxchg_long_func(jlong exchange_value,
189 jlong volatile* dest,
190 jlong compare_value) {
191 assert(VM_Version::supports_cx8(), "Atomic compare and exchange jlong not supported on this architecture!");
192 // Warning: Arguments are swapped to avoid moving them for kernel call
193 return (*os::atomic_cmpxchg_long_func)(compare_value, exchange_value, dest);
194 }
195
196 #endif // !AARCH64
197
198 template<>
199 template<typename T>
200 inline T Atomic::PlatformCmpxchg<4>::operator()(T exchange_value,
201 T volatile* dest,
202 T compare_value,
203 cmpxchg_memory_order order) const {
204 STATIC_ASSERT(4 == sizeof(T));
205 #ifdef AARCH64
206 T rv;
207 int tmp;
208 __asm__ volatile(
209 "1:\n\t"
210 " ldaxr %w[rv], [%[dest]]\n\t"
211 " cmp %w[rv], %w[cv]\n\t"
212 " b.ne 2f\n\t"
213 " stlxr %w[tmp], %w[ev], [%[dest]]\n\t"
214 " cbnz %w[tmp], 1b\n\t"
215 " b 3f\n\t"
216 "2:\n\t"
217 " dmb sy\n\t"
218 "3:\n\t"
219 : [rv] "=&r" (rv), [tmp] "=&r" (tmp)
220 : [ev] "r" (exchange_value), [dest] "r" (dest), [cv] "r" (compare_value)
221 : "memory");
222 return rv;
223 #else
224 return cmpxchg_using_helper<jint>(reorder_cmpxchg_func, exchange_value, dest, compare_value);
225 #endif
226 }
227
228 template<>
229 template<typename T>
230 inline T Atomic::PlatformCmpxchg<8>::operator()(T exchange_value,
231 T volatile* dest,
232 T compare_value,
233 cmpxchg_memory_order order) const {
234 STATIC_ASSERT(8 == sizeof(T));
235 #ifdef AARCH64
236 T rv;
237 int tmp;
238 __asm__ volatile(
239 "1:\n\t"
240 " ldaxr %[rv], [%[dest]]\n\t"
241 " cmp %[rv], %[cv]\n\t"
242 " b.ne 2f\n\t"
243 " stlxr %w[tmp], %[ev], [%[dest]]\n\t"
244 " cbnz %w[tmp], 1b\n\t"
245 " b 3f\n\t"
246 "2:\n\t"
247 " dmb sy\n\t"
248 "3:\n\t"
249 : [rv] "=&r" (rv), [tmp] "=&r" (tmp)
250 : [ev] "r" (exchange_value), [dest] "r" (dest), [cv] "r" (compare_value)
251 : "memory");
252 return rv;
253 #else
254 return cmpxchg_using_helper<jlong>(reorder_cmpxchg_long_func, exchange_value, dest, compare_value);
255 #endif
256 }
257
258 #endif // OS_CPU_LINUX_ARM_VM_ATOMIC_LINUX_ARM_HPP
|
33 /*
34 * Atomic long operations on 32-bit ARM
35 * ARM v7 supports LDREXD/STREXD synchronization instructions so no problem.
36 * ARM < v7 does not have explicit 64 atomic load/store capability.
37 * However, gcc emits LDRD/STRD instructions on v5te and LDM/STM on v5t
38 * when loading/storing 64 bits.
39 * For non-MP machines (which is all we support for ARM < v7)
40 * under current Linux distros these instructions appear atomic.
41 * See section A3.5.3 of ARM Architecture Reference Manual for ARM v7.
42 * Also, for cmpxchg64, if ARM < v7 we check for cmpxchg64 support in the
43 * Linux kernel using _kuser_helper_version. See entry-armv.S in the Linux
44 * kernel source or kernel_user_helpers.txt in Linux Doc.
45 */
46
47 #ifndef AARCH64
48 template<>
49 template<typename T>
50 inline T Atomic::PlatformLoad<8>::operator()(T const volatile* src) const {
51 STATIC_ASSERT(8 == sizeof(T));
52 return PrimitiveConversions::cast<T>(
53 (*os::atomic_load_long_func)(reinterpret_cast<const volatile int64_t*>(src)));
54 }
55
56 template<>
57 template<typename T>
58 inline void Atomic::PlatformStore<8>::operator()(T store_value,
59 T volatile* dest) const {
60 STATIC_ASSERT(8 == sizeof(T));
61 (*os::atomic_store_long_func)(
62 PrimitiveConversions::cast<int64_t>(store_value), reinterpret_cast<volatile int64_t*>(dest));
63 }
64 #endif
65
66 // As per atomic.hpp all read-modify-write operations have to provide two-way
67 // barriers semantics. For AARCH64 we are using load-acquire-with-reservation and
68 // store-release-with-reservation. While load-acquire combined with store-release
69 // do not generally form two-way barriers, their use with reservations does - the
70 // ARMv8 architecture manual Section F "Barrier Litmus Tests" indicates they
71 // provide sequentially consistent semantics. All we need to add is an explicit
72 // barrier in the failure path of the cmpxchg operations (as these don't execute
73 // the store) - arguably this may be overly cautious as there is a very low
74 // likelihood that the hardware would pull loads/stores into the region guarded
75 // by the reservation.
76 //
77 // For ARMv7 we add explicit barriers in the stubs.
78
79 template<size_t byte_size>
80 struct Atomic::PlatformAdd
81 : Atomic::AddAndFetch<Atomic::PlatformAdd<byte_size> >
82 {
86
87 template<>
88 template<typename I, typename D>
89 inline D Atomic::PlatformAdd<4>::add_and_fetch(I add_value, D volatile* dest) const {
90 STATIC_ASSERT(4 == sizeof(I));
91 STATIC_ASSERT(4 == sizeof(D));
92 #ifdef AARCH64
93 D val;
94 int tmp;
95 __asm__ volatile(
96 "1:\n\t"
97 " ldaxr %w[val], [%[dest]]\n\t"
98 " add %w[val], %w[val], %w[add_val]\n\t"
99 " stlxr %w[tmp], %w[val], [%[dest]]\n\t"
100 " cbnz %w[tmp], 1b\n\t"
101 : [val] "=&r" (val), [tmp] "=&r" (tmp)
102 : [add_val] "r" (add_value), [dest] "r" (dest)
103 : "memory");
104 return val;
105 #else
106 return add_using_helper<int32_t>(os::atomic_add_func, add_value, dest);
107 #endif
108 }
109
110 #ifdef AARCH64
111 template<>
112 template<typename I, typename D>
113 inline D Atomic::PlatformAdd<8>::add_and_fetch(I add_value, D volatile* dest) const {
114 STATIC_ASSERT(8 == sizeof(I));
115 STATIC_ASSERT(8 == sizeof(D));
116 D val;
117 int tmp;
118 __asm__ volatile(
119 "1:\n\t"
120 " ldaxr %[val], [%[dest]]\n\t"
121 " add %[val], %[val], %[add_val]\n\t"
122 " stlxr %w[tmp], %[val], [%[dest]]\n\t"
123 " cbnz %w[tmp], 1b\n\t"
124 : [val] "=&r" (val), [tmp] "=&r" (tmp)
125 : [add_val] "r" (add_value), [dest] "r" (dest)
126 : "memory");
129 #endif
130
131 template<>
132 template<typename T>
133 inline T Atomic::PlatformXchg<4>::operator()(T exchange_value,
134 T volatile* dest) const {
135 STATIC_ASSERT(4 == sizeof(T));
136 #ifdef AARCH64
137 T old_val;
138 int tmp;
139 __asm__ volatile(
140 "1:\n\t"
141 " ldaxr %w[old_val], [%[dest]]\n\t"
142 " stlxr %w[tmp], %w[new_val], [%[dest]]\n\t"
143 " cbnz %w[tmp], 1b\n\t"
144 : [old_val] "=&r" (old_val), [tmp] "=&r" (tmp)
145 : [new_val] "r" (exchange_value), [dest] "r" (dest)
146 : "memory");
147 return old_val;
148 #else
149 return xchg_using_helper<int32_t>(os::atomic_xchg_func, exchange_value, dest);
150 #endif
151 }
152
153 #ifdef AARCH64
154 template<>
155 template<typename T>
156 inline T Atomic::PlatformXchg<8>::operator()(T exchange_value,
157 T volatile* dest) const {
158 STATIC_ASSERT(8 == sizeof(T));
159 T old_val;
160 int tmp;
161 __asm__ volatile(
162 "1:\n\t"
163 " ldaxr %[old_val], [%[dest]]\n\t"
164 " stlxr %w[tmp], %[new_val], [%[dest]]\n\t"
165 " cbnz %w[tmp], 1b\n\t"
166 : [old_val] "=&r" (old_val), [tmp] "=&r" (tmp)
167 : [new_val] "r" (exchange_value), [dest] "r" (dest)
168 : "memory");
169 return old_val;
170 }
171 #endif // AARCH64
172
173 // The memory_order parameter is ignored - we always provide the strongest/most-conservative ordering
174
175 // No direct support for cmpxchg of bytes; emulate using int.
176 template<>
177 struct Atomic::PlatformCmpxchg<1> : Atomic::CmpxchgByteUsingInt {};
178
179 #ifndef AARCH64
180
181 inline int32_t reorder_cmpxchg_func(int32_t exchange_value,
182 int32_t volatile* dest,
183 int32_t compare_value) {
184 // Warning: Arguments are swapped to avoid moving them for kernel call
185 return (*os::atomic_cmpxchg_func)(compare_value, exchange_value, dest);
186 }
187
188 inline int64_t reorder_cmpxchg_long_func(int64_t exchange_value,
189 int64_t volatile* dest,
190 int64_t compare_value) {
191 assert(VM_Version::supports_cx8(), "Atomic compare and exchange int64_t not supported on this architecture!");
192 // Warning: Arguments are swapped to avoid moving them for kernel call
193 return (*os::atomic_cmpxchg_long_func)(compare_value, exchange_value, dest);
194 }
195
196 #endif // !AARCH64
197
198 template<>
199 template<typename T>
200 inline T Atomic::PlatformCmpxchg<4>::operator()(T exchange_value,
201 T volatile* dest,
202 T compare_value,
203 cmpxchg_memory_order order) const {
204 STATIC_ASSERT(4 == sizeof(T));
205 #ifdef AARCH64
206 T rv;
207 int tmp;
208 __asm__ volatile(
209 "1:\n\t"
210 " ldaxr %w[rv], [%[dest]]\n\t"
211 " cmp %w[rv], %w[cv]\n\t"
212 " b.ne 2f\n\t"
213 " stlxr %w[tmp], %w[ev], [%[dest]]\n\t"
214 " cbnz %w[tmp], 1b\n\t"
215 " b 3f\n\t"
216 "2:\n\t"
217 " dmb sy\n\t"
218 "3:\n\t"
219 : [rv] "=&r" (rv), [tmp] "=&r" (tmp)
220 : [ev] "r" (exchange_value), [dest] "r" (dest), [cv] "r" (compare_value)
221 : "memory");
222 return rv;
223 #else
224 return cmpxchg_using_helper<int32_t>(reorder_cmpxchg_func, exchange_value, dest, compare_value);
225 #endif
226 }
227
228 template<>
229 template<typename T>
230 inline T Atomic::PlatformCmpxchg<8>::operator()(T exchange_value,
231 T volatile* dest,
232 T compare_value,
233 cmpxchg_memory_order order) const {
234 STATIC_ASSERT(8 == sizeof(T));
235 #ifdef AARCH64
236 T rv;
237 int tmp;
238 __asm__ volatile(
239 "1:\n\t"
240 " ldaxr %[rv], [%[dest]]\n\t"
241 " cmp %[rv], %[cv]\n\t"
242 " b.ne 2f\n\t"
243 " stlxr %w[tmp], %[ev], [%[dest]]\n\t"
244 " cbnz %w[tmp], 1b\n\t"
245 " b 3f\n\t"
246 "2:\n\t"
247 " dmb sy\n\t"
248 "3:\n\t"
249 : [rv] "=&r" (rv), [tmp] "=&r" (tmp)
250 : [ev] "r" (exchange_value), [dest] "r" (dest), [cv] "r" (compare_value)
251 : "memory");
252 return rv;
253 #else
254 return cmpxchg_using_helper<int64_t>(reorder_cmpxchg_long_func, exchange_value, dest, compare_value);
255 #endif
256 }
257
258 #endif // OS_CPU_LINUX_ARM_VM_ATOMIC_LINUX_ARM_HPP
|