1 /* 2 * Copyright (c) 1997, 2016, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2012, 2014 SAP SE. All rights reserved. 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This code is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 only, as 8 * published by the Free Software Foundation. 9 * 10 * This code is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * version 2 for more details (a copy is included in the LICENSE file that 14 * accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License version 17 * 2 along with this work; if not, write to the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 * or visit www.oracle.com if you need additional information or have any 22 * questions. 23 * 24 */ 25 26 #ifndef OS_CPU_AIX_OJDKPPC_VM_ATOMIC_AIX_PPC_HPP 27 #define OS_CPU_AIX_OJDKPPC_VM_ATOMIC_AIX_PPC_HPP 28 29 #ifndef _LP64 30 #error "Atomic currently only impleneted for PPC64" 31 #endif 32 33 // Implementation of class atomic 34 35 inline void Atomic::store (jbyte store_value, jbyte* dest) { *dest = store_value; } 36 inline void Atomic::store (jshort store_value, jshort* dest) { *dest = store_value; } 37 inline void Atomic::store (jint store_value, jint* dest) { *dest = store_value; } 38 inline void Atomic::store (jlong store_value, jlong* dest) { *dest = store_value; } 39 inline void Atomic::store_ptr(intptr_t store_value, intptr_t* dest) { *dest = store_value; } 40 inline void Atomic::store_ptr(void* store_value, void* dest) { *(void**)dest = store_value; } 41 42 inline void Atomic::store (jbyte store_value, volatile jbyte* dest) { *dest = store_value; } 43 inline void Atomic::store (jshort store_value, volatile jshort* dest) { *dest = store_value; } 44 inline void Atomic::store (jint store_value, volatile jint* dest) { *dest = store_value; } 45 inline void Atomic::store (jlong store_value, volatile jlong* dest) { *dest = store_value; } 46 inline void Atomic::store_ptr(intptr_t store_value, volatile intptr_t* dest) { *dest = store_value; } 47 inline void Atomic::store_ptr(void* store_value, volatile void* dest) { *(void* volatile *)dest = store_value; } 48 49 inline jlong Atomic::load(volatile jlong* src) { return *src; } 50 51 // 52 // machine barrier instructions: 53 // 54 // - ppc_sync two-way memory barrier, aka fence 55 // - ppc_lwsync orders Store|Store, 56 // Load|Store, 57 // Load|Load, 58 // but not Store|Load 59 // - ppc_eieio orders memory accesses for device memory (only) 60 // - ppc_isync invalidates speculatively executed instructions 61 // From the POWER ISA 2.06 documentation: 62 // "[...] an isync instruction prevents the execution of 63 // instructions following the isync until instructions 64 // preceding the isync have completed, [...]" 65 // From IBM's AIX assembler reference: 66 // "The isync [...] instructions causes the processor to 67 // refetch any instructions that might have been fetched 68 // prior to the isync instruction. The instruction isync 69 // causes the processor to wait for all previous instructions 70 // to complete. Then any instructions already fetched are 71 // discarded and instruction processing continues in the 72 // environment established by the previous instructions." 73 // 74 // semantic barrier instructions: 75 // (as defined in orderAccess.hpp) 76 // 77 // - ppc_release orders Store|Store, (maps to ppc_lwsync) 78 // Load|Store 79 // - ppc_acquire orders Load|Store, (maps to ppc_lwsync) 80 // Load|Load 81 // - ppc_fence orders Store|Store, (maps to ppc_sync) 82 // Load|Store, 83 // Load|Load, 84 // Store|Load 85 // 86 87 #define strasm_sync "\n sync \n" 88 #define strasm_lwsync "\n lwsync \n" 89 #define strasm_isync "\n isync \n" 90 #define strasm_release strasm_lwsync 91 #define strasm_acquire strasm_lwsync 92 #define strasm_fence strasm_sync 93 #define strasm_nobarrier "" 94 #define strasm_nobarrier_clobber_memory "" 95 96 inline jint Atomic::add (jint add_value, volatile jint* dest) { 97 98 unsigned int result; 99 100 __asm__ __volatile__ ( 101 strasm_lwsync 102 "1: lwarx %0, 0, %2 \n" 103 " add %0, %0, %1 \n" 104 " stwcx. %0, 0, %2 \n" 105 " bne- 1b \n" 106 strasm_isync 107 : /*%0*/"=&r" (result) 108 : /*%1*/"r" (add_value), /*%2*/"r" (dest) 109 : "cc", "memory" ); 110 111 return (jint) result; 112 } 113 114 115 inline intptr_t Atomic::add_ptr(intptr_t add_value, volatile intptr_t* dest) { 116 117 long result; 118 119 __asm__ __volatile__ ( 120 strasm_lwsync 121 "1: ldarx %0, 0, %2 \n" 122 " add %0, %0, %1 \n" 123 " stdcx. %0, 0, %2 \n" 124 " bne- 1b \n" 125 strasm_isync 126 : /*%0*/"=&r" (result) 127 : /*%1*/"r" (add_value), /*%2*/"r" (dest) 128 : "cc", "memory" ); 129 130 return (intptr_t) result; 131 } 132 133 inline void* Atomic::add_ptr(intptr_t add_value, volatile void* dest) { 134 return (void*)add_ptr(add_value, (volatile intptr_t*)dest); 135 } 136 137 138 inline void Atomic::inc (volatile jint* dest) { 139 140 unsigned int temp; 141 142 __asm__ __volatile__ ( 143 strasm_nobarrier 144 "1: lwarx %0, 0, %2 \n" 145 " addic %0, %0, 1 \n" 146 " stwcx. %0, 0, %2 \n" 147 " bne- 1b \n" 148 strasm_nobarrier 149 : /*%0*/"=&r" (temp), "=m" (*dest) 150 : /*%2*/"r" (dest), "m" (*dest) 151 : "cc" strasm_nobarrier_clobber_memory); 152 153 } 154 155 inline void Atomic::inc_ptr(volatile intptr_t* dest) { 156 157 long temp; 158 159 __asm__ __volatile__ ( 160 strasm_nobarrier 161 "1: ldarx %0, 0, %2 \n" 162 " addic %0, %0, 1 \n" 163 " stdcx. %0, 0, %2 \n" 164 " bne- 1b \n" 165 strasm_nobarrier 166 : /*%0*/"=&r" (temp), "=m" (*dest) 167 : /*%2*/"r" (dest), "m" (*dest) 168 : "cc" strasm_nobarrier_clobber_memory); 169 170 } 171 172 inline void Atomic::inc_ptr(volatile void* dest) { 173 inc_ptr((volatile intptr_t*)dest); 174 } 175 176 177 inline void Atomic::dec (volatile jint* dest) { 178 179 unsigned int temp; 180 181 __asm__ __volatile__ ( 182 strasm_nobarrier 183 "1: lwarx %0, 0, %2 \n" 184 " addic %0, %0, -1 \n" 185 " stwcx. %0, 0, %2 \n" 186 " bne- 1b \n" 187 strasm_nobarrier 188 : /*%0*/"=&r" (temp), "=m" (*dest) 189 : /*%2*/"r" (dest), "m" (*dest) 190 : "cc" strasm_nobarrier_clobber_memory); 191 192 } 193 194 inline void Atomic::dec_ptr(volatile intptr_t* dest) { 195 196 long temp; 197 198 __asm__ __volatile__ ( 199 strasm_nobarrier 200 "1: ldarx %0, 0, %2 \n" 201 " addic %0, %0, -1 \n" 202 " stdcx. %0, 0, %2 \n" 203 " bne- 1b \n" 204 strasm_nobarrier 205 : /*%0*/"=&r" (temp), "=m" (*dest) 206 : /*%2*/"r" (dest), "m" (*dest) 207 : "cc" strasm_nobarrier_clobber_memory); 208 209 } 210 211 inline void Atomic::dec_ptr(volatile void* dest) { 212 dec_ptr((volatile intptr_t*)dest); 213 } 214 215 inline jint Atomic::xchg(jint exchange_value, volatile jint* dest) { 216 217 // Note that xchg_ptr doesn't necessarily do an acquire 218 // (see synchronizer.cpp). 219 220 unsigned int old_value; 221 const uint64_t zero = 0; 222 223 __asm__ __volatile__ ( 224 /* lwsync */ 225 strasm_lwsync 226 /* atomic loop */ 227 "1: \n" 228 " lwarx %[old_value], %[dest], %[zero] \n" 229 " stwcx. %[exchange_value], %[dest], %[zero] \n" 230 " bne- 1b \n" 231 /* isync */ 232 strasm_sync 233 /* exit */ 234 "2: \n" 235 /* out */ 236 : [old_value] "=&r" (old_value), 237 "=m" (*dest) 238 /* in */ 239 : [dest] "b" (dest), 240 [zero] "r" (zero), 241 [exchange_value] "r" (exchange_value), 242 "m" (*dest) 243 /* clobber */ 244 : "cc", 245 "memory" 246 ); 247 248 return (jint) old_value; 249 } 250 251 inline intptr_t Atomic::xchg_ptr(intptr_t exchange_value, volatile intptr_t* dest) { 252 253 // Note that xchg_ptr doesn't necessarily do an acquire 254 // (see synchronizer.cpp). 255 256 long old_value; 257 const uint64_t zero = 0; 258 259 __asm__ __volatile__ ( 260 /* lwsync */ 261 strasm_lwsync 262 /* atomic loop */ 263 "1: \n" 264 " ldarx %[old_value], %[dest], %[zero] \n" 265 " stdcx. %[exchange_value], %[dest], %[zero] \n" 266 " bne- 1b \n" 267 /* isync */ 268 strasm_sync 269 /* exit */ 270 "2: \n" 271 /* out */ 272 : [old_value] "=&r" (old_value), 273 "=m" (*dest) 274 /* in */ 275 : [dest] "b" (dest), 276 [zero] "r" (zero), 277 [exchange_value] "r" (exchange_value), 278 "m" (*dest) 279 /* clobber */ 280 : "cc", 281 "memory" 282 ); 283 284 return (intptr_t) old_value; 285 } 286 287 inline void* Atomic::xchg_ptr(void* exchange_value, volatile void* dest) { 288 return (void*)xchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest); 289 } 290 291 inline void cmpxchg_pre_membar(cmpxchg_memory_order order) { 292 if (order == memory_order_release) { 293 __asm__ __volatile__ ( 294 /* release */ 295 strasm_lwsync 296 ); 297 } else if (order != memory_order_relaxed) { 298 __asm__ __volatile__ ( 299 /* fence */ 300 strasm_sync 301 ); 302 } 303 } 304 305 inline void cmpxchg_post_membar(cmpxchg_memory_order order) { 306 if (order == memory_order_release) { 307 // no post membar 308 } else if (order == memory_order_conservative) { 309 __asm__ __volatile__ ( 310 /* fence */ 311 strasm_sync 312 ); 313 } 314 } 315 316 #define VM_HAS_SPECIALIZED_CMPXCHG_BYTE 317 inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, cmpxchg_memory_order order) { 318 319 // Note that cmpxchg guarantees a two-way memory barrier across 320 // the cmpxchg, so it's really a a 'fence_cmpxchg_fence' if not 321 // specified otherwise (see atomic.hpp). 322 323 // Using 32 bit internally. 324 volatile int *dest_base = (volatile int*)((uintptr_t)dest & ~3); 325 326 #ifdef VM_LITTLE_ENDIAN 327 const unsigned int shift_amount = ((uintptr_t)dest & 3) * 8; 328 #else 329 const unsigned int shift_amount = ((~(uintptr_t)dest) & 3) * 8; 330 #endif 331 const unsigned int masked_compare_val = ((unsigned int)(unsigned char)compare_value), 332 masked_exchange_val = ((unsigned int)(unsigned char)exchange_value), 333 xor_value = (masked_compare_val ^ masked_exchange_val) << shift_amount; 334 335 unsigned int old_value, value32; 336 337 cmpxchg_pre_membar(order); 338 339 __asm__ __volatile__ ( 340 /* simple guard */ 341 " lbz %[old_value], 0(%[dest]) \n" 342 " cmpw %[masked_compare_val], %[old_value] \n" 343 " bne- 2f \n" 344 /* atomic loop */ 345 "1: \n" 346 " lwarx %[value32], 0, %[dest_base] \n" 347 /* extract byte and compare */ 348 " srd %[old_value], %[value32], %[shift_amount] \n" 349 " clrldi %[old_value], %[old_value], 56 \n" 350 " cmpw %[masked_compare_val], %[old_value] \n" 351 " bne- 2f \n" 352 /* replace byte and try to store */ 353 " xor %[value32], %[xor_value], %[value32] \n" 354 " stwcx. %[value32], 0, %[dest_base] \n" 355 " bne- 1b \n" 356 /* exit */ 357 "2: \n" 358 /* out */ 359 : [old_value] "=&r" (old_value), 360 [value32] "=&r" (value32), 361 "=m" (*dest), 362 "=m" (*dest_base) 363 /* in */ 364 : [dest] "b" (dest), 365 [dest_base] "b" (dest_base), 366 [shift_amount] "r" (shift_amount), 367 [masked_compare_val] "r" (masked_compare_val), 368 [xor_value] "r" (xor_value), 369 "m" (*dest), 370 "m" (*dest_base) 371 /* clobber */ 372 : "cc", 373 "memory" 374 ); 375 376 cmpxchg_post_membar(order); 377 378 return (jbyte)(unsigned char)old_value; 379 } 380 381 inline jint Atomic::cmpxchg(jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) { 382 383 // Note that cmpxchg guarantees a two-way memory barrier across 384 // the cmpxchg, so it's really a a 'fence_cmpxchg_fence' if not 385 // specified otherwise (see atomic.hpp). 386 387 unsigned int old_value; 388 const uint64_t zero = 0; 389 390 cmpxchg_pre_membar(order); 391 392 __asm__ __volatile__ ( 393 /* simple guard */ 394 " lwz %[old_value], 0(%[dest]) \n" 395 " cmpw %[compare_value], %[old_value] \n" 396 " bne- 2f \n" 397 /* atomic loop */ 398 "1: \n" 399 " lwarx %[old_value], %[dest], %[zero] \n" 400 " cmpw %[compare_value], %[old_value] \n" 401 " bne- 2f \n" 402 " stwcx. %[exchange_value], %[dest], %[zero] \n" 403 " bne- 1b \n" 404 /* exit */ 405 "2: \n" 406 /* out */ 407 : [old_value] "=&r" (old_value), 408 "=m" (*dest) 409 /* in */ 410 : [dest] "b" (dest), 411 [zero] "r" (zero), 412 [compare_value] "r" (compare_value), 413 [exchange_value] "r" (exchange_value), 414 "m" (*dest) 415 /* clobber */ 416 : "cc", 417 "memory" 418 ); 419 420 cmpxchg_post_membar(order); 421 422 return (jint) old_value; 423 } 424 425 inline jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) { 426 427 // Note that cmpxchg guarantees a two-way memory barrier across 428 // the cmpxchg, so it's really a a 'fence_cmpxchg_fence' if not 429 // specified otherwise (see atomic.hpp). 430 431 long old_value; 432 const uint64_t zero = 0; 433 434 cmpxchg_pre_membar(order); 435 436 __asm__ __volatile__ ( 437 /* simple guard */ 438 " ld %[old_value], 0(%[dest]) \n" 439 " cmpd %[compare_value], %[old_value] \n" 440 " bne- 2f \n" 441 /* atomic loop */ 442 "1: \n" 443 " ldarx %[old_value], %[dest], %[zero] \n" 444 " cmpd %[compare_value], %[old_value] \n" 445 " bne- 2f \n" 446 " stdcx. %[exchange_value], %[dest], %[zero] \n" 447 " bne- 1b \n" 448 /* exit */ 449 "2: \n" 450 /* out */ 451 : [old_value] "=&r" (old_value), 452 "=m" (*dest) 453 /* in */ 454 : [dest] "b" (dest), 455 [zero] "r" (zero), 456 [compare_value] "r" (compare_value), 457 [exchange_value] "r" (exchange_value), 458 "m" (*dest) 459 /* clobber */ 460 : "cc", 461 "memory" 462 ); 463 464 cmpxchg_post_membar(order); 465 466 return (jlong) old_value; 467 } 468 469 inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) { 470 return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order); 471 } 472 473 inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) { 474 return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order); 475 } 476 477 #undef strasm_sync 478 #undef strasm_lwsync 479 #undef strasm_isync 480 #undef strasm_release 481 #undef strasm_acquire 482 #undef strasm_fence 483 #undef strasm_nobarrier 484 #undef strasm_nobarrier_clobber_memory 485 486 #endif // OS_CPU_AIX_OJDKPPC_VM_ATOMIC_AIX_PPC_HPP