1 /*
2 * Copyright (c) 1997, 2013, Oracle and/or its affiliates. All rights reserved.
3 * Copyright 2012, 2013 SAP AG. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #ifndef OS_CPU_LINUX_PPC_VM_ATOMIC_LINUX_PPC_INLINE_HPP
27 #define OS_CPU_LINUX_PPC_VM_ATOMIC_LINUX_PPC_INLINE_HPP
28
29 #include "orderAccess_linux_ppc.inline.hpp"
30 #include "runtime/atomic.hpp"
31 #include "runtime/os.hpp"
32 #include "vm_version_ppc.hpp"
33
34 #ifndef PPC64
35 #error "Atomic currently only implemented for PPC64"
36 #endif
37
38 // Implementation of class atomic
39
40 inline void Atomic::store (jbyte store_value, jbyte* dest) { *dest = store_value; }
41 inline void Atomic::store (jshort store_value, jshort* dest) { *dest = store_value; }
42 inline void Atomic::store (jint store_value, jint* dest) { *dest = store_value; }
43 inline void Atomic::store (jlong store_value, jlong* dest) { *dest = store_value; }
44 inline void Atomic::store_ptr(intptr_t store_value, intptr_t* dest) { *dest = store_value; }
45 inline void Atomic::store_ptr(void* store_value, void* dest) { *(void**)dest = store_value; }
46
47 inline void Atomic::store (jbyte store_value, volatile jbyte* dest) { *dest = store_value; }
48 inline void Atomic::store (jshort store_value, volatile jshort* dest) { *dest = store_value; }
49 inline void Atomic::store (jint store_value, volatile jint* dest) { *dest = store_value; }
50 inline void Atomic::store (jlong store_value, volatile jlong* dest) { *dest = store_value; }
51 inline void Atomic::store_ptr(intptr_t store_value, volatile intptr_t* dest) { *dest = store_value; }
52 inline void Atomic::store_ptr(void* store_value, volatile void* dest) { *(void* volatile *)dest = store_value; }
53
54 inline jlong Atomic::load(volatile jlong* src) { return *src; }
55
56 /*
57 machine barrier instructions:
58
59 - sync two-way memory barrier, aka fence
60 - lwsync orders Store|Store,
61 Load|Store,
62 Load|Load,
63 but not Store|Load
64 - eieio orders memory accesses for device memory (only)
65 - isync invalidates speculatively executed instructions
66 From the POWER ISA 2.06 documentation:
67 "[...] an isync instruction prevents the execution of
68 instructions following the isync until instructions
69 preceding the isync have completed, [...]"
70 From IBM's AIX assembler reference:
71 "The isync [...] instructions causes the processor to
72 refetch any instructions that might have been fetched
73 prior to the isync instruction. The instruction isync
74 causes the processor to wait for all previous instructions
75 to complete. Then any instructions already fetched are
76 discarded and instruction processing continues in the
77 environment established by the previous instructions."
78
79 semantic barrier instructions:
80 (as defined in orderAccess.hpp)
81
82 - release orders Store|Store, (maps to lwsync)
83 Load|Store
84 - acquire orders Load|Store, (maps to lwsync)
85 Load|Load
86 - fence orders Store|Store, (maps to sync)
87 Load|Store,
88 Load|Load,
89 Store|Load
90 */
91
92 #define strasm_sync "\n sync \n"
93 #define strasm_lwsync "\n lwsync \n"
94 #define strasm_isync "\n isync \n"
95 #define strasm_release strasm_lwsync
96 #define strasm_acquire strasm_lwsync
97 #define strasm_fence strasm_sync
98 #define strasm_nobarrier ""
99 #define strasm_nobarrier_clobber_memory ""
100
101 inline jint Atomic::add (jint add_value, volatile jint* dest) {
102
103 unsigned int result;
104
105 __asm__ __volatile__ (
106 strasm_lwsync
107 "1: lwarx %0, 0, %2 \n"
108 " add %0, %0, %1 \n"
109 " stwcx. %0, 0, %2 \n"
110 " bne- 1b \n"
111 strasm_isync
112 : /*%0*/"=&r" (result)
113 : /*%1*/"r" (add_value), /*%2*/"r" (dest)
114 : "cc", "memory" );
115
116 return (jint) result;
117 }
118
119
120 inline intptr_t Atomic::add_ptr(intptr_t add_value, volatile intptr_t* dest) {
121
122 long result;
123
124 __asm__ __volatile__ (
125 strasm_lwsync
126 "1: ldarx %0, 0, %2 \n"
127 " add %0, %0, %1 \n"
128 " stdcx. %0, 0, %2 \n"
129 " bne- 1b \n"
130 strasm_isync
131 : /*%0*/"=&r" (result)
132 : /*%1*/"r" (add_value), /*%2*/"r" (dest)
133 : "cc", "memory" );
134
135 return (intptr_t) result;
136 }
137
138 inline void* Atomic::add_ptr(intptr_t add_value, volatile void* dest) {
139 return (void*)add_ptr(add_value, (volatile intptr_t*)dest);
140 }
141
142
143 inline void Atomic::inc (volatile jint* dest) {
144
145 unsigned int temp;
146
147 __asm__ __volatile__ (
148 strasm_nobarrier
149 "1: lwarx %0, 0, %2 \n"
150 " addic %0, %0, 1 \n"
151 " stwcx. %0, 0, %2 \n"
152 " bne- 1b \n"
153 strasm_nobarrier
154 : /*%0*/"=&r" (temp), "=m" (*dest)
155 : /*%2*/"r" (dest), "m" (*dest)
156 : "cc" strasm_nobarrier_clobber_memory);
157
158 }
159
160 inline void Atomic::inc_ptr(volatile intptr_t* dest) {
161
162 long temp;
163
164 __asm__ __volatile__ (
165 strasm_nobarrier
166 "1: ldarx %0, 0, %2 \n"
167 " addic %0, %0, 1 \n"
168 " stdcx. %0, 0, %2 \n"
169 " bne- 1b \n"
170 strasm_nobarrier
171 : /*%0*/"=&r" (temp), "=m" (*dest)
172 : /*%2*/"r" (dest), "m" (*dest)
173 : "cc" strasm_nobarrier_clobber_memory);
174
175 }
176
177 inline void Atomic::inc_ptr(volatile void* dest) {
178 inc_ptr((volatile intptr_t*)dest);
179 }
180
181
182 inline void Atomic::dec (volatile jint* dest) {
183
184 unsigned int temp;
185
186 __asm__ __volatile__ (
187 strasm_nobarrier
188 "1: lwarx %0, 0, %2 \n"
189 " addic %0, %0, -1 \n"
190 " stwcx. %0, 0, %2 \n"
191 " bne- 1b \n"
192 strasm_nobarrier
193 : /*%0*/"=&r" (temp), "=m" (*dest)
194 : /*%2*/"r" (dest), "m" (*dest)
195 : "cc" strasm_nobarrier_clobber_memory);
196
197 }
198
199 inline void Atomic::dec_ptr(volatile intptr_t* dest) {
200
201 long temp;
202
203 __asm__ __volatile__ (
204 strasm_nobarrier
205 "1: ldarx %0, 0, %2 \n"
206 " addic %0, %0, -1 \n"
207 " stdcx. %0, 0, %2 \n"
208 " bne- 1b \n"
209 strasm_nobarrier
210 : /*%0*/"=&r" (temp), "=m" (*dest)
211 : /*%2*/"r" (dest), "m" (*dest)
212 : "cc" strasm_nobarrier_clobber_memory);
213
214 }
215
216 inline void Atomic::dec_ptr(volatile void* dest) {
217 dec_ptr((volatile intptr_t*)dest);
218 }
219
220 inline jint Atomic::xchg(jint exchange_value, volatile jint* dest) {
221
222 // Note that xchg_ptr doesn't necessarily do an acquire
223 // (see synchronizer.cpp).
224
225 unsigned int old_value;
226 const uint64_t zero = 0;
227
228 __asm__ __volatile__ (
229 /* lwsync */
230 strasm_lwsync
231 /* atomic loop */
232 "1: \n"
233 " lwarx %[old_value], %[dest], %[zero] \n"
234 " stwcx. %[exchange_value], %[dest], %[zero] \n"
235 " bne- 1b \n"
236 /* isync */
237 strasm_sync
238 /* exit */
239 "2: \n"
240 /* out */
241 : [old_value] "=&r" (old_value),
242 "=m" (*dest)
243 /* in */
244 : [dest] "b" (dest),
245 [zero] "r" (zero),
246 [exchange_value] "r" (exchange_value),
247 "m" (*dest)
248 /* clobber */
249 : "cc",
250 "memory"
251 );
252
253 return (jint) old_value;
254 }
255
256 inline intptr_t Atomic::xchg_ptr(intptr_t exchange_value, volatile intptr_t* dest) {
257
258 // Note that xchg_ptr doesn't necessarily do an acquire
259 // (see synchronizer.cpp).
260
261 long old_value;
262 const uint64_t zero = 0;
263
264 __asm__ __volatile__ (
265 /* lwsync */
266 strasm_lwsync
267 /* atomic loop */
268 "1: \n"
269 " ldarx %[old_value], %[dest], %[zero] \n"
270 " stdcx. %[exchange_value], %[dest], %[zero] \n"
271 " bne- 1b \n"
272 /* isync */
273 strasm_sync
274 /* exit */
275 "2: \n"
276 /* out */
277 : [old_value] "=&r" (old_value),
278 "=m" (*dest)
279 /* in */
280 : [dest] "b" (dest),
281 [zero] "r" (zero),
282 [exchange_value] "r" (exchange_value),
283 "m" (*dest)
284 /* clobber */
285 : "cc",
286 "memory"
287 );
288
289 return (intptr_t) old_value;
290 }
291
292 inline void* Atomic::xchg_ptr(void* exchange_value, volatile void* dest) {
293 return (void*)xchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest);
294 }
295
296 inline jint Atomic::cmpxchg(jint exchange_value, volatile jint* dest, jint compare_value) {
297
298 // Note that cmpxchg guarantees a two-way memory barrier across
299 // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire'
300 // (see atomic.hpp).
301
302 unsigned int old_value;
303 const uint64_t zero = 0;
304
305 __asm__ __volatile__ (
306 /* fence */
307 strasm_sync
308 /* simple guard */
309 " lwz %[old_value], 0(%[dest]) \n"
310 " cmpw %[compare_value], %[old_value] \n"
311 " bne- 2f \n"
312 /* atomic loop */
313 "1: \n"
314 " lwarx %[old_value], %[dest], %[zero] \n"
315 " cmpw %[compare_value], %[old_value] \n"
316 " bne- 2f \n"
317 " stwcx. %[exchange_value], %[dest], %[zero] \n"
318 " bne- 1b \n"
319 /* acquire */
320 strasm_sync
321 /* exit */
322 "2: \n"
323 /* out */
324 : [old_value] "=&r" (old_value),
325 "=m" (*dest)
326 /* in */
327 : [dest] "b" (dest),
328 [zero] "r" (zero),
329 [compare_value] "r" (compare_value),
330 [exchange_value] "r" (exchange_value),
331 "m" (*dest)
332 /* clobber */
333 : "cc",
334 "memory"
335 );
336
337 return (jint) old_value;
338 }
339
340 inline jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong* dest, jlong compare_value) {
341
342 // Note that cmpxchg guarantees a two-way memory barrier across
343 // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire'
344 // (see atomic.hpp).
345
346 long old_value;
347 const uint64_t zero = 0;
348
349 __asm__ __volatile__ (
350 /* fence */
351 strasm_sync
352 /* simple guard */
353 " ld %[old_value], 0(%[dest]) \n"
354 " cmpd %[compare_value], %[old_value] \n"
355 " bne- 2f \n"
356 /* atomic loop */
357 "1: \n"
358 " ldarx %[old_value], %[dest], %[zero] \n"
359 " cmpd %[compare_value], %[old_value] \n"
360 " bne- 2f \n"
361 " stdcx. %[exchange_value], %[dest], %[zero] \n"
362 " bne- 1b \n"
363 /* acquire */
364 strasm_sync
365 /* exit */
366 "2: \n"
367 /* out */
368 : [old_value] "=&r" (old_value),
369 "=m" (*dest)
370 /* in */
371 : [dest] "b" (dest),
372 [zero] "r" (zero),
373 [compare_value] "r" (compare_value),
374 [exchange_value] "r" (exchange_value),
375 "m" (*dest)
376 /* clobber */
377 : "cc",
378 "memory"
379 );
380
381 return (jlong) old_value;
382 }
383
384 inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
385 return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
386 }
387
388 inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
389 return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
390 }
391
392 #undef strasm_sync
393 #undef strasm_lwsync
394 #undef strasm_isync
395 #undef strasm_release
396 #undef strasm_acquire
397 #undef strasm_fence
398 #undef strasm_nobarrier
399 #undef strasm_nobarrier_clobber_memory
400
401 #endif // OS_CPU_LINUX_PPC_VM_ATOMIC_LINUX_PPC_INLINE_HPP