1 /*
2 * Copyright (c) 1997, 2014, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2012, 2014 SAP SE. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #ifndef OS_CPU_AIX_OJDKPPC_VM_ATOMIC_AIX_PPC_INLINE_HPP
27 #define OS_CPU_AIX_OJDKPPC_VM_ATOMIC_AIX_PPC_INLINE_HPP
28
29 #include "runtime/atomic.hpp"
30 #include "runtime/os.hpp"
31
32 #ifndef _LP64
33 #error "Atomic currently only impleneted for PPC64"
34 #endif
35
36 // Implementation of class atomic
37
38 inline void Atomic::store (jbyte store_value, jbyte* dest) { *dest = store_value; }
39 inline void Atomic::store (jshort store_value, jshort* dest) { *dest = store_value; }
40 inline void Atomic::store (jint store_value, jint* dest) { *dest = store_value; }
41 inline void Atomic::store (jlong store_value, jlong* dest) { *dest = store_value; }
42 inline void Atomic::store_ptr(intptr_t store_value, intptr_t* dest) { *dest = store_value; }
43 inline void Atomic::store_ptr(void* store_value, void* dest) { *(void**)dest = store_value; }
44
45 inline void Atomic::store (jbyte store_value, volatile jbyte* dest) { *dest = store_value; }
46 inline void Atomic::store (jshort store_value, volatile jshort* dest) { *dest = store_value; }
47 inline void Atomic::store (jint store_value, volatile jint* dest) { *dest = store_value; }
48 inline void Atomic::store (jlong store_value, volatile jlong* dest) { *dest = store_value; }
49 inline void Atomic::store_ptr(intptr_t store_value, volatile intptr_t* dest) { *dest = store_value; }
50 inline void Atomic::store_ptr(void* store_value, volatile void* dest) { *(void* volatile *)dest = store_value; }
51
52 inline jlong Atomic::load(volatile jlong* src) { return *src; }
53
54 //
55 // machine barrier instructions:
56 //
57 // - ppc_sync two-way memory barrier, aka fence
58 // - ppc_lwsync orders Store|Store,
59 // Load|Store,
60 // Load|Load,
61 // but not Store|Load
62 // - ppc_eieio orders memory accesses for device memory (only)
63 // - ppc_isync invalidates speculatively executed instructions
64 // From the POWER ISA 2.06 documentation:
65 // "[...] an isync instruction prevents the execution of
66 // instructions following the isync until instructions
67 // preceding the isync have completed, [...]"
68 // From IBM's AIX assembler reference:
69 // "The isync [...] instructions causes the processor to
70 // refetch any instructions that might have been fetched
71 // prior to the isync instruction. The instruction isync
72 // causes the processor to wait for all previous instructions
73 // to complete. Then any instructions already fetched are
74 // discarded and instruction processing continues in the
75 // environment established by the previous instructions."
76 //
77 // semantic barrier instructions:
78 // (as defined in orderAccess.hpp)
79 //
80 // - ppc_release orders Store|Store, (maps to ppc_lwsync)
81 // Load|Store
82 // - ppc_acquire orders Load|Store, (maps to ppc_lwsync)
83 // Load|Load
84 // - ppc_fence orders Store|Store, (maps to ppc_sync)
85 // Load|Store,
86 // Load|Load,
87 // Store|Load
88 //
89
90 #define strasm_sync "\n sync \n"
91 #define strasm_lwsync "\n lwsync \n"
92 #define strasm_isync "\n isync \n"
93 #define strasm_release strasm_lwsync
94 #define strasm_acquire strasm_lwsync
95 #define strasm_fence strasm_sync
96 #define strasm_nobarrier ""
97 #define strasm_nobarrier_clobber_memory ""
98
99 inline jint Atomic::add (jint add_value, volatile jint* dest) {
100
101 unsigned int result;
102
103 __asm__ __volatile__ (
104 strasm_lwsync
105 "1: lwarx %0, 0, %2 \n"
106 " add %0, %0, %1 \n"
107 " stwcx. %0, 0, %2 \n"
108 " bne- 1b \n"
109 strasm_isync
110 : /*%0*/"=&r" (result)
111 : /*%1*/"r" (add_value), /*%2*/"r" (dest)
112 : "cc", "memory" );
113
114 return (jint) result;
115 }
116
117
118 inline intptr_t Atomic::add_ptr(intptr_t add_value, volatile intptr_t* dest) {
119
120 long result;
121
122 __asm__ __volatile__ (
123 strasm_lwsync
124 "1: ldarx %0, 0, %2 \n"
125 " add %0, %0, %1 \n"
126 " stdcx. %0, 0, %2 \n"
127 " bne- 1b \n"
128 strasm_isync
129 : /*%0*/"=&r" (result)
130 : /*%1*/"r" (add_value), /*%2*/"r" (dest)
131 : "cc", "memory" );
132
133 return (intptr_t) result;
134 }
135
136 inline void* Atomic::add_ptr(intptr_t add_value, volatile void* dest) {
137 return (void*)add_ptr(add_value, (volatile intptr_t*)dest);
138 }
139
140
141 inline void Atomic::inc (volatile jint* dest) {
142
143 unsigned int temp;
144
145 __asm__ __volatile__ (
146 strasm_nobarrier
147 "1: lwarx %0, 0, %2 \n"
148 " addic %0, %0, 1 \n"
149 " stwcx. %0, 0, %2 \n"
150 " bne- 1b \n"
151 strasm_nobarrier
152 : /*%0*/"=&r" (temp), "=m" (*dest)
153 : /*%2*/"r" (dest), "m" (*dest)
154 : "cc" strasm_nobarrier_clobber_memory);
155
156 }
157
158 inline void Atomic::inc_ptr(volatile intptr_t* dest) {
159
160 long temp;
161
162 __asm__ __volatile__ (
163 strasm_nobarrier
164 "1: ldarx %0, 0, %2 \n"
165 " addic %0, %0, 1 \n"
166 " stdcx. %0, 0, %2 \n"
167 " bne- 1b \n"
168 strasm_nobarrier
169 : /*%0*/"=&r" (temp), "=m" (*dest)
170 : /*%2*/"r" (dest), "m" (*dest)
171 : "cc" strasm_nobarrier_clobber_memory);
172
173 }
174
175 inline void Atomic::inc_ptr(volatile void* dest) {
176 inc_ptr((volatile intptr_t*)dest);
177 }
178
179
180 inline void Atomic::dec (volatile jint* dest) {
181
182 unsigned int temp;
183
184 __asm__ __volatile__ (
185 strasm_nobarrier
186 "1: lwarx %0, 0, %2 \n"
187 " addic %0, %0, -1 \n"
188 " stwcx. %0, 0, %2 \n"
189 " bne- 1b \n"
190 strasm_nobarrier
191 : /*%0*/"=&r" (temp), "=m" (*dest)
192 : /*%2*/"r" (dest), "m" (*dest)
193 : "cc" strasm_nobarrier_clobber_memory);
194
195 }
196
197 inline void Atomic::dec_ptr(volatile intptr_t* dest) {
198
199 long temp;
200
201 __asm__ __volatile__ (
202 strasm_nobarrier
203 "1: ldarx %0, 0, %2 \n"
204 " addic %0, %0, -1 \n"
205 " stdcx. %0, 0, %2 \n"
206 " bne- 1b \n"
207 strasm_nobarrier
208 : /*%0*/"=&r" (temp), "=m" (*dest)
209 : /*%2*/"r" (dest), "m" (*dest)
210 : "cc" strasm_nobarrier_clobber_memory);
211
212 }
213
214 inline void Atomic::dec_ptr(volatile void* dest) {
215 dec_ptr((volatile intptr_t*)dest);
216 }
217
218 inline jint Atomic::xchg(jint exchange_value, volatile jint* dest) {
219
220 // Note that xchg_ptr doesn't necessarily do an acquire
221 // (see synchronizer.cpp).
222
223 unsigned int old_value;
224 const uint64_t zero = 0;
225
226 __asm__ __volatile__ (
227 /* lwsync */
228 strasm_lwsync
229 /* atomic loop */
230 "1: \n"
231 " lwarx %[old_value], %[dest], %[zero] \n"
232 " stwcx. %[exchange_value], %[dest], %[zero] \n"
233 " bne- 1b \n"
234 /* isync */
235 strasm_sync
236 /* exit */
237 "2: \n"
238 /* out */
239 : [old_value] "=&r" (old_value),
240 "=m" (*dest)
241 /* in */
242 : [dest] "b" (dest),
243 [zero] "r" (zero),
244 [exchange_value] "r" (exchange_value),
245 "m" (*dest)
246 /* clobber */
247 : "cc",
248 "memory"
249 );
250
251 return (jint) old_value;
252 }
253
254 inline intptr_t Atomic::xchg_ptr(intptr_t exchange_value, volatile intptr_t* dest) {
255
256 // Note that xchg_ptr doesn't necessarily do an acquire
257 // (see synchronizer.cpp).
258
259 long old_value;
260 const uint64_t zero = 0;
261
262 __asm__ __volatile__ (
263 /* lwsync */
264 strasm_lwsync
265 /* atomic loop */
266 "1: \n"
267 " ldarx %[old_value], %[dest], %[zero] \n"
268 " stdcx. %[exchange_value], %[dest], %[zero] \n"
269 " bne- 1b \n"
270 /* isync */
271 strasm_sync
272 /* exit */
273 "2: \n"
274 /* out */
275 : [old_value] "=&r" (old_value),
276 "=m" (*dest)
277 /* in */
278 : [dest] "b" (dest),
279 [zero] "r" (zero),
280 [exchange_value] "r" (exchange_value),
281 "m" (*dest)
282 /* clobber */
283 : "cc",
284 "memory"
285 );
286
287 return (intptr_t) old_value;
288 }
289
290 inline void* Atomic::xchg_ptr(void* exchange_value, volatile void* dest) {
291 return (void*)xchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest);
292 }
293
294 #define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
295 inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) {
296
297 // Note that cmpxchg guarantees a two-way memory barrier across
298 // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire'
299 // (see atomic.hpp).
300
301 // Using 32 bit internally.
302 volatile int *dest_base = (volatile int*)((uintptr_t)dest & ~3);
303
304 #ifdef VM_LITTLE_ENDIAN
305 const unsigned int shift_amount = ((uintptr_t)dest & 3) * 8;
306 #else
307 const unsigned int shift_amount = ((~(uintptr_t)dest) & 3) * 8;
308 #endif
309 const unsigned int masked_compare_val = ((unsigned int)(unsigned char)compare_value),
310 masked_exchange_val = ((unsigned int)(unsigned char)exchange_value),
311 xor_value = (masked_compare_val ^ masked_exchange_val) << shift_amount;
312
313 unsigned int old_value, value32;
314
315 __asm__ __volatile__ (
316 /* fence */
317 strasm_sync
318 /* simple guard */
319 " lbz %[old_value], 0(%[dest]) \n"
320 " cmpw %[masked_compare_val], %[old_value] \n"
321 " bne- 2f \n"
322 /* atomic loop */
323 "1: \n"
324 " lwarx %[value32], 0, %[dest_base] \n"
325 /* extract byte and compare */
326 " srd %[old_value], %[value32], %[shift_amount] \n"
327 " clrldi %[old_value], %[old_value], 56 \n"
328 " cmpw %[masked_compare_val], %[old_value] \n"
329 " bne- 2f \n"
330 /* replace byte and try to store */
331 " xor %[value32], %[xor_value], %[value32] \n"
332 " stwcx. %[value32], 0, %[dest_base] \n"
333 " bne- 1b \n"
334 /* acquire */
335 strasm_sync
336 /* exit */
337 "2: \n"
338 /* out */
339 : [old_value] "=&r" (old_value),
340 [value32] "=&r" (value32),
341 "=m" (*dest),
342 "=m" (*dest_base)
343 /* in */
344 : [dest] "b" (dest),
345 [dest_base] "b" (dest_base),
346 [shift_amount] "r" (shift_amount),
347 [masked_compare_val] "r" (masked_compare_val),
348 [xor_value] "r" (xor_value),
349 "m" (*dest),
350 "m" (*dest_base)
351 /* clobber */
352 : "cc",
353 "memory"
354 );
355
356 return (jbyte)(unsigned char)old_value;
357 }
358
359 inline jint Atomic::cmpxchg(jint exchange_value, volatile jint* dest, jint compare_value) {
360
361 // Note that cmpxchg guarantees a two-way memory barrier across
362 // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire'
363 // (see atomic.hpp).
364
365 unsigned int old_value;
366 const uint64_t zero = 0;
367
368 __asm__ __volatile__ (
369 /* fence */
370 strasm_sync
371 /* simple guard */
372 " lwz %[old_value], 0(%[dest]) \n"
373 " cmpw %[compare_value], %[old_value] \n"
374 " bne- 2f \n"
375 /* atomic loop */
376 "1: \n"
377 " lwarx %[old_value], %[dest], %[zero] \n"
378 " cmpw %[compare_value], %[old_value] \n"
379 " bne- 2f \n"
380 " stwcx. %[exchange_value], %[dest], %[zero] \n"
381 " bne- 1b \n"
382 /* acquire */
383 strasm_sync
384 /* exit */
385 "2: \n"
386 /* out */
387 : [old_value] "=&r" (old_value),
388 "=m" (*dest)
389 /* in */
390 : [dest] "b" (dest),
391 [zero] "r" (zero),
392 [compare_value] "r" (compare_value),
393 [exchange_value] "r" (exchange_value),
394 "m" (*dest)
395 /* clobber */
396 : "cc",
397 "memory"
398 );
399
400 return (jint) old_value;
401 }
402
403 inline jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong* dest, jlong compare_value) {
404
405 // Note that cmpxchg guarantees a two-way memory barrier across
406 // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire'
407 // (see atomic.hpp).
408
409 long old_value;
410 const uint64_t zero = 0;
411
412 __asm__ __volatile__ (
413 /* fence */
414 strasm_sync
415 /* simple guard */
416 " ld %[old_value], 0(%[dest]) \n"
417 " cmpd %[compare_value], %[old_value] \n"
418 " bne- 2f \n"
419 /* atomic loop */
420 "1: \n"
421 " ldarx %[old_value], %[dest], %[zero] \n"
422 " cmpd %[compare_value], %[old_value] \n"
423 " bne- 2f \n"
424 " stdcx. %[exchange_value], %[dest], %[zero] \n"
425 " bne- 1b \n"
426 /* acquire */
427 strasm_sync
428 /* exit */
429 "2: \n"
430 /* out */
431 : [old_value] "=&r" (old_value),
432 "=m" (*dest)
433 /* in */
434 : [dest] "b" (dest),
435 [zero] "r" (zero),
436 [compare_value] "r" (compare_value),
437 [exchange_value] "r" (exchange_value),
438 "m" (*dest)
439 /* clobber */
440 : "cc",
441 "memory"
442 );
443
444 return (jlong) old_value;
445 }
446
447 inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
448 return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
449 }
450
451 inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
452 return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
453 }
454
455 #undef strasm_sync
456 #undef strasm_lwsync
457 #undef strasm_isync
458 #undef strasm_release
459 #undef strasm_acquire
460 #undef strasm_fence
461 #undef strasm_nobarrier
462 #undef strasm_nobarrier_clobber_memory
463
464 #endif // OS_CPU_AIX_OJDKPPC_VM_ATOMIC_AIX_PPC_INLINE_HPP