27
28 #include "runtime/os.hpp"
29 #include "vm_version_arm.hpp"
30
31 // Implementation of class atomic
32
33 /*
34 * Atomic long operations on 32-bit ARM
35 * ARM v7 supports LDREXD/STREXD synchronization instructions so no problem.
36 * ARM < v7 does not have explicit 64 atomic load/store capability.
37 * However, gcc emits LDRD/STRD instructions on v5te and LDM/STM on v5t
38 * when loading/storing 64 bits.
39 * For non-MP machines (which is all we support for ARM < v7)
40 * under current Linux distros these instructions appear atomic.
41 * See section A3.5.3 of ARM Architecture Reference Manual for ARM v7.
42 * Also, for cmpxchg64, if ARM < v7 we check for cmpxchg64 support in the
43 * Linux kernel using _kuser_helper_version. See entry-armv.S in the Linux
44 * kernel source or kernel_user_helpers.txt in Linux Doc.
45 */
46
47 #ifndef AARCH64
48 template<>
49 template<typename T>
50 inline T Atomic::PlatformLoad<8>::operator()(T const volatile* src) const {
51 STATIC_ASSERT(8 == sizeof(T));
52 return PrimitiveConversions::cast<T>(
53 (*os::atomic_load_long_func)(reinterpret_cast<const volatile int64_t*>(src)));
54 }
55
56 template<>
57 template<typename T>
58 inline void Atomic::PlatformStore<8>::operator()(T store_value,
59 T volatile* dest) const {
60 STATIC_ASSERT(8 == sizeof(T));
61 (*os::atomic_store_long_func)(
62 PrimitiveConversions::cast<int64_t>(store_value), reinterpret_cast<volatile int64_t*>(dest));
63 }
64 #endif
65
66 // As per atomic.hpp all read-modify-write operations have to provide two-way
67 // barriers semantics. For AARCH64 we are using load-acquire-with-reservation and
68 // store-release-with-reservation. While load-acquire combined with store-release
69 // do not generally form two-way barriers, their use with reservations does - the
70 // ARMv8 architecture manual Section F "Barrier Litmus Tests" indicates they
71 // provide sequentially consistent semantics. All we need to add is an explicit
72 // barrier in the failure path of the cmpxchg operations (as these don't execute
73 // the store) - arguably this may be overly cautious as there is a very low
74 // likelihood that the hardware would pull loads/stores into the region guarded
75 // by the reservation.
76 //
77 // For ARMv7 we add explicit barriers in the stubs.
78
79 template<size_t byte_size>
80 struct Atomic::PlatformAdd
81 : Atomic::AddAndFetch<Atomic::PlatformAdd<byte_size> >
82 {
83 template<typename I, typename D>
84 D add_and_fetch(I add_value, D volatile* dest, atomic_memory_order order) const;
85 };
86
87 template<>
88 template<typename I, typename D>
89 inline D Atomic::PlatformAdd<4>::add_and_fetch(I add_value, D volatile* dest,
90 atomic_memory_order order) const {
91 STATIC_ASSERT(4 == sizeof(I));
92 STATIC_ASSERT(4 == sizeof(D));
93 #ifdef AARCH64
94 D val;
95 int tmp;
96 __asm__ volatile(
97 "1:\n\t"
98 " ldaxr %w[val], [%[dest]]\n\t"
99 " add %w[val], %w[val], %w[add_val]\n\t"
100 " stlxr %w[tmp], %w[val], [%[dest]]\n\t"
101 " cbnz %w[tmp], 1b\n\t"
102 : [val] "=&r" (val), [tmp] "=&r" (tmp)
103 : [add_val] "r" (add_value), [dest] "r" (dest)
104 : "memory");
105 return val;
106 #else
107 return add_using_helper<int32_t>(os::atomic_add_func, add_value, dest);
108 #endif
109 }
110
111 #ifdef AARCH64
112 template<>
113 template<typename I, typename D>
114 inline D Atomic::PlatformAdd<8>::add_and_fetch(I add_value, D volatile* dest,
115 atomic_memory_order order) const {
116 STATIC_ASSERT(8 == sizeof(I));
117 STATIC_ASSERT(8 == sizeof(D));
118 D val;
119 int tmp;
120 __asm__ volatile(
121 "1:\n\t"
122 " ldaxr %[val], [%[dest]]\n\t"
123 " add %[val], %[val], %[add_val]\n\t"
124 " stlxr %w[tmp], %[val], [%[dest]]\n\t"
125 " cbnz %w[tmp], 1b\n\t"
126 : [val] "=&r" (val), [tmp] "=&r" (tmp)
127 : [add_val] "r" (add_value), [dest] "r" (dest)
128 : "memory");
129 return val;
130 }
131 #endif
132
133 template<>
134 template<typename T>
135 inline T Atomic::PlatformXchg<4>::operator()(T exchange_value,
136 T volatile* dest,
137 atomic_memory_order order) const {
138 STATIC_ASSERT(4 == sizeof(T));
139 #ifdef AARCH64
140 T old_val;
141 int tmp;
142 __asm__ volatile(
143 "1:\n\t"
144 " ldaxr %w[old_val], [%[dest]]\n\t"
145 " stlxr %w[tmp], %w[new_val], [%[dest]]\n\t"
146 " cbnz %w[tmp], 1b\n\t"
147 : [old_val] "=&r" (old_val), [tmp] "=&r" (tmp)
148 : [new_val] "r" (exchange_value), [dest] "r" (dest)
149 : "memory");
150 return old_val;
151 #else
152 return xchg_using_helper<int32_t>(os::atomic_xchg_func, exchange_value, dest);
153 #endif
154 }
155
156 #ifdef AARCH64
157 template<>
158 template<typename T>
159 inline T Atomic::PlatformXchg<8>::operator()(T exchange_value,
160 T volatile* dest,
161 atomic_memory_order order) const {
162 STATIC_ASSERT(8 == sizeof(T));
163 T old_val;
164 int tmp;
165 __asm__ volatile(
166 "1:\n\t"
167 " ldaxr %[old_val], [%[dest]]\n\t"
168 " stlxr %w[tmp], %[new_val], [%[dest]]\n\t"
169 " cbnz %w[tmp], 1b\n\t"
170 : [old_val] "=&r" (old_val), [tmp] "=&r" (tmp)
171 : [new_val] "r" (exchange_value), [dest] "r" (dest)
172 : "memory");
173 return old_val;
174 }
175 #endif // AARCH64
176
177 // The memory_order parameter is ignored - we always provide the strongest/most-conservative ordering
178
179 // No direct support for cmpxchg of bytes; emulate using int.
180 template<>
181 struct Atomic::PlatformCmpxchg<1> : Atomic::CmpxchgByteUsingInt {};
182
183 #ifndef AARCH64
184
185 inline int32_t reorder_cmpxchg_func(int32_t exchange_value,
186 int32_t volatile* dest,
187 int32_t compare_value) {
188 // Warning: Arguments are swapped to avoid moving them for kernel call
189 return (*os::atomic_cmpxchg_func)(compare_value, exchange_value, dest);
190 }
191
192 inline int64_t reorder_cmpxchg_long_func(int64_t exchange_value,
193 int64_t volatile* dest,
194 int64_t compare_value) {
195 assert(VM_Version::supports_cx8(), "Atomic compare and exchange int64_t not supported on this architecture!");
196 // Warning: Arguments are swapped to avoid moving them for kernel call
197 return (*os::atomic_cmpxchg_long_func)(compare_value, exchange_value, dest);
198 }
199
200 #endif // !AARCH64
201
202 template<>
203 template<typename T>
204 inline T Atomic::PlatformCmpxchg<4>::operator()(T exchange_value,
205 T volatile* dest,
206 T compare_value,
207 atomic_memory_order order) const {
208 STATIC_ASSERT(4 == sizeof(T));
209 #ifdef AARCH64
210 T rv;
211 int tmp;
212 __asm__ volatile(
213 "1:\n\t"
214 " ldaxr %w[rv], [%[dest]]\n\t"
215 " cmp %w[rv], %w[cv]\n\t"
216 " b.ne 2f\n\t"
217 " stlxr %w[tmp], %w[ev], [%[dest]]\n\t"
218 " cbnz %w[tmp], 1b\n\t"
219 " b 3f\n\t"
220 "2:\n\t"
221 " dmb sy\n\t"
222 "3:\n\t"
223 : [rv] "=&r" (rv), [tmp] "=&r" (tmp)
224 : [ev] "r" (exchange_value), [dest] "r" (dest), [cv] "r" (compare_value)
225 : "memory");
226 return rv;
227 #else
228 return cmpxchg_using_helper<int32_t>(reorder_cmpxchg_func, exchange_value, dest, compare_value);
229 #endif
230 }
231
232 template<>
233 template<typename T>
234 inline T Atomic::PlatformCmpxchg<8>::operator()(T exchange_value,
235 T volatile* dest,
236 T compare_value,
237 atomic_memory_order order) const {
238 STATIC_ASSERT(8 == sizeof(T));
239 #ifdef AARCH64
240 T rv;
241 int tmp;
242 __asm__ volatile(
243 "1:\n\t"
244 " ldaxr %[rv], [%[dest]]\n\t"
245 " cmp %[rv], %[cv]\n\t"
246 " b.ne 2f\n\t"
247 " stlxr %w[tmp], %[ev], [%[dest]]\n\t"
248 " cbnz %w[tmp], 1b\n\t"
249 " b 3f\n\t"
250 "2:\n\t"
251 " dmb sy\n\t"
252 "3:\n\t"
253 : [rv] "=&r" (rv), [tmp] "=&r" (tmp)
254 : [ev] "r" (exchange_value), [dest] "r" (dest), [cv] "r" (compare_value)
255 : "memory");
256 return rv;
257 #else
258 return cmpxchg_using_helper<int64_t>(reorder_cmpxchg_long_func, exchange_value, dest, compare_value);
259 #endif
260 }
261
262 #endif // OS_CPU_LINUX_ARM_VM_ATOMIC_LINUX_ARM_HPP
|
27
28 #include "runtime/os.hpp"
29 #include "vm_version_arm.hpp"
30
31 // Implementation of class atomic
32
33 /*
34 * Atomic long operations on 32-bit ARM
35 * ARM v7 supports LDREXD/STREXD synchronization instructions so no problem.
36 * ARM < v7 does not have explicit 64 atomic load/store capability.
37 * However, gcc emits LDRD/STRD instructions on v5te and LDM/STM on v5t
38 * when loading/storing 64 bits.
39 * For non-MP machines (which is all we support for ARM < v7)
40 * under current Linux distros these instructions appear atomic.
41 * See section A3.5.3 of ARM Architecture Reference Manual for ARM v7.
42 * Also, for cmpxchg64, if ARM < v7 we check for cmpxchg64 support in the
43 * Linux kernel using _kuser_helper_version. See entry-armv.S in the Linux
44 * kernel source or kernel_user_helpers.txt in Linux Doc.
45 */
46
47 template<>
48 template<typename T>
49 inline T Atomic::PlatformLoad<8>::operator()(T const volatile* src) const {
50 STATIC_ASSERT(8 == sizeof(T));
51 return PrimitiveConversions::cast<T>(
52 (*os::atomic_load_long_func)(reinterpret_cast<const volatile int64_t*>(src)));
53 }
54
55 template<>
56 template<typename T>
57 inline void Atomic::PlatformStore<8>::operator()(T store_value,
58 T volatile* dest) const {
59 STATIC_ASSERT(8 == sizeof(T));
60 (*os::atomic_store_long_func)(
61 PrimitiveConversions::cast<int64_t>(store_value), reinterpret_cast<volatile int64_t*>(dest));
62 }
63
64 // As per atomic.hpp all read-modify-write operations have to provide two-way
65 // barriers semantics.
66 //
67 // For ARMv7 we add explicit barriers in the stubs.
68
69 template<size_t byte_size>
70 struct Atomic::PlatformAdd
71 : Atomic::AddAndFetch<Atomic::PlatformAdd<byte_size> >
72 {
73 template<typename I, typename D>
74 D add_and_fetch(I add_value, D volatile* dest, atomic_memory_order order) const;
75 };
76
77 template<>
78 template<typename I, typename D>
79 inline D Atomic::PlatformAdd<4>::add_and_fetch(I add_value, D volatile* dest,
80 atomic_memory_order order) const {
81 STATIC_ASSERT(4 == sizeof(I));
82 STATIC_ASSERT(4 == sizeof(D));
83 return add_using_helper<int32_t>(os::atomic_add_func, add_value, dest);
84 }
85
86
87 template<>
88 template<typename T>
89 inline T Atomic::PlatformXchg<4>::operator()(T exchange_value,
90 T volatile* dest,
91 atomic_memory_order order) const {
92 STATIC_ASSERT(4 == sizeof(T));
93 return xchg_using_helper<int32_t>(os::atomic_xchg_func, exchange_value, dest);
94 }
95
96
97 // The memory_order parameter is ignored - we always provide the strongest/most-conservative ordering
98
99 // No direct support for cmpxchg of bytes; emulate using int.
100 template<>
101 struct Atomic::PlatformCmpxchg<1> : Atomic::CmpxchgByteUsingInt {};
102
103
104 inline int32_t reorder_cmpxchg_func(int32_t exchange_value,
105 int32_t volatile* dest,
106 int32_t compare_value) {
107 // Warning: Arguments are swapped to avoid moving them for kernel call
108 return (*os::atomic_cmpxchg_func)(compare_value, exchange_value, dest);
109 }
110
111 inline int64_t reorder_cmpxchg_long_func(int64_t exchange_value,
112 int64_t volatile* dest,
113 int64_t compare_value) {
114 assert(VM_Version::supports_cx8(), "Atomic compare and exchange int64_t not supported on this architecture!");
115 // Warning: Arguments are swapped to avoid moving them for kernel call
116 return (*os::atomic_cmpxchg_long_func)(compare_value, exchange_value, dest);
117 }
118
119
120 template<>
121 template<typename T>
122 inline T Atomic::PlatformCmpxchg<4>::operator()(T exchange_value,
123 T volatile* dest,
124 T compare_value,
125 atomic_memory_order order) const {
126 STATIC_ASSERT(4 == sizeof(T));
127 return cmpxchg_using_helper<int32_t>(reorder_cmpxchg_func, exchange_value, dest, compare_value);
128 }
129
130 template<>
131 template<typename T>
132 inline T Atomic::PlatformCmpxchg<8>::operator()(T exchange_value,
133 T volatile* dest,
134 T compare_value,
135 atomic_memory_order order) const {
136 STATIC_ASSERT(8 == sizeof(T));
137 return cmpxchg_using_helper<int64_t>(reorder_cmpxchg_long_func, exchange_value, dest, compare_value);
138 }
139
140 #endif // OS_CPU_LINUX_ARM_VM_ATOMIC_LINUX_ARM_HPP
|