1 /* 2 * Copyright (c) 1997, 2014, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2012, 2014 SAP SE. All rights reserved. 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This code is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 only, as 8 * published by the Free Software Foundation. 9 * 10 * This code is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * version 2 for more details (a copy is included in the LICENSE file that 14 * accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License version 17 * 2 along with this work; if not, write to the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 * or visit www.oracle.com if you need additional information or have any 22 * questions. 23 * 24 */ 25 26 #ifndef OS_CPU_LINUX_PPC_VM_ATOMIC_LINUX_PPC_INLINE_HPP 27 #define OS_CPU_LINUX_PPC_VM_ATOMIC_LINUX_PPC_INLINE_HPP 28 29 #include "runtime/atomic.hpp" 30 #include "runtime/os.hpp" 31 32 #ifndef PPC64 33 #error "Atomic currently only implemented for PPC64" 34 #endif 35 36 // Implementation of class atomic 37 38 inline void Atomic::store (jbyte store_value, jbyte* dest) { *dest = store_value; } 39 inline void Atomic::store (jshort store_value, jshort* dest) { *dest = store_value; } 40 inline void Atomic::store (jint store_value, jint* dest) { *dest = store_value; } 41 inline void Atomic::store (jlong store_value, jlong* dest) { *dest = store_value; } 42 inline void Atomic::store_ptr(intptr_t store_value, intptr_t* dest) { *dest = store_value; } 43 inline void Atomic::store_ptr(void* store_value, void* dest) { *(void**)dest = store_value; } 44 45 inline void Atomic::store (jbyte store_value, volatile jbyte* dest) { *dest = store_value; } 46 inline void Atomic::store (jshort store_value, volatile jshort* dest) { *dest = store_value; } 47 inline void Atomic::store (jint store_value, volatile jint* dest) { *dest = store_value; } 48 inline void Atomic::store (jlong store_value, volatile jlong* dest) { *dest = store_value; } 49 inline void Atomic::store_ptr(intptr_t store_value, volatile intptr_t* dest) { *dest = store_value; } 50 inline void Atomic::store_ptr(void* store_value, volatile void* dest) { *(void* volatile *)dest = store_value; } 51 52 inline jlong Atomic::load(volatile jlong* src) { return *src; } 53 54 // 55 // machine barrier instructions: 56 // 57 // - sync two-way memory barrier, aka fence 58 // - lwsync orders Store|Store, 59 // Load|Store, 60 // Load|Load, 61 // but not Store|Load 62 // - eieio orders memory accesses for device memory (only) 63 // - isync invalidates speculatively executed instructions 64 // From the POWER ISA 2.06 documentation: 65 // "[...] an isync instruction prevents the execution of 66 // instructions following the isync until instructions 67 // preceding the isync have completed, [...]" 68 // From IBM's AIX assembler reference: 69 // "The isync [...] instructions causes the processor to 70 // refetch any instructions that might have been fetched 71 // prior to the isync instruction. The instruction isync 72 // causes the processor to wait for all previous instructions 73 // to complete. Then any instructions already fetched are 74 // discarded and instruction processing continues in the 75 // environment established by the previous instructions." 76 // 77 // semantic barrier instructions: 78 // (as defined in orderAccess.hpp) 79 // 80 // - release orders Store|Store, (maps to lwsync) 81 // Load|Store 82 // - acquire orders Load|Store, (maps to lwsync) 83 // Load|Load 84 // - fence orders Store|Store, (maps to sync) 85 // Load|Store, 86 // Load|Load, 87 // Store|Load 88 // 89 90 #define strasm_sync "\n sync \n" 91 #define strasm_lwsync "\n lwsync \n" 92 #define strasm_isync "\n isync \n" 93 #define strasm_release strasm_lwsync 94 #define strasm_acquire strasm_lwsync 95 #define strasm_fence strasm_sync 96 #define strasm_nobarrier "" 97 #define strasm_nobarrier_clobber_memory "" 98 99 inline jint Atomic::add (jint add_value, volatile jint* dest) { 100 101 unsigned int result; 102 103 __asm__ __volatile__ ( 104 strasm_lwsync 105 "1: lwarx %0, 0, %2 \n" 106 " add %0, %0, %1 \n" 107 " stwcx. %0, 0, %2 \n" 108 " bne- 1b \n" 109 strasm_isync 110 : /*%0*/"=&r" (result) 111 : /*%1*/"r" (add_value), /*%2*/"r" (dest) 112 : "cc", "memory" ); 113 114 return (jint) result; 115 } 116 117 118 inline intptr_t Atomic::add_ptr(intptr_t add_value, volatile intptr_t* dest) { 119 120 long result; 121 122 __asm__ __volatile__ ( 123 strasm_lwsync 124 "1: ldarx %0, 0, %2 \n" 125 " add %0, %0, %1 \n" 126 " stdcx. %0, 0, %2 \n" 127 " bne- 1b \n" 128 strasm_isync 129 : /*%0*/"=&r" (result) 130 : /*%1*/"r" (add_value), /*%2*/"r" (dest) 131 : "cc", "memory" ); 132 133 return (intptr_t) result; 134 } 135 136 inline void* Atomic::add_ptr(intptr_t add_value, volatile void* dest) { 137 return (void*)add_ptr(add_value, (volatile intptr_t*)dest); 138 } 139 140 141 inline void Atomic::inc (volatile jint* dest) { 142 143 unsigned int temp; 144 145 __asm__ __volatile__ ( 146 strasm_nobarrier 147 "1: lwarx %0, 0, %2 \n" 148 " addic %0, %0, 1 \n" 149 " stwcx. %0, 0, %2 \n" 150 " bne- 1b \n" 151 strasm_nobarrier 152 : /*%0*/"=&r" (temp), "=m" (*dest) 153 : /*%2*/"r" (dest), "m" (*dest) 154 : "cc" strasm_nobarrier_clobber_memory); 155 156 } 157 158 inline void Atomic::inc_ptr(volatile intptr_t* dest) { 159 160 long temp; 161 162 __asm__ __volatile__ ( 163 strasm_nobarrier 164 "1: ldarx %0, 0, %2 \n" 165 " addic %0, %0, 1 \n" 166 " stdcx. %0, 0, %2 \n" 167 " bne- 1b \n" 168 strasm_nobarrier 169 : /*%0*/"=&r" (temp), "=m" (*dest) 170 : /*%2*/"r" (dest), "m" (*dest) 171 : "cc" strasm_nobarrier_clobber_memory); 172 173 } 174 175 inline void Atomic::inc_ptr(volatile void* dest) { 176 inc_ptr((volatile intptr_t*)dest); 177 } 178 179 180 inline void Atomic::dec (volatile jint* dest) { 181 182 unsigned int temp; 183 184 __asm__ __volatile__ ( 185 strasm_nobarrier 186 "1: lwarx %0, 0, %2 \n" 187 " addic %0, %0, -1 \n" 188 " stwcx. %0, 0, %2 \n" 189 " bne- 1b \n" 190 strasm_nobarrier 191 : /*%0*/"=&r" (temp), "=m" (*dest) 192 : /*%2*/"r" (dest), "m" (*dest) 193 : "cc" strasm_nobarrier_clobber_memory); 194 195 } 196 197 inline void Atomic::dec_ptr(volatile intptr_t* dest) { 198 199 long temp; 200 201 __asm__ __volatile__ ( 202 strasm_nobarrier 203 "1: ldarx %0, 0, %2 \n" 204 " addic %0, %0, -1 \n" 205 " stdcx. %0, 0, %2 \n" 206 " bne- 1b \n" 207 strasm_nobarrier 208 : /*%0*/"=&r" (temp), "=m" (*dest) 209 : /*%2*/"r" (dest), "m" (*dest) 210 : "cc" strasm_nobarrier_clobber_memory); 211 212 } 213 214 inline void Atomic::dec_ptr(volatile void* dest) { 215 dec_ptr((volatile intptr_t*)dest); 216 } 217 218 inline jint Atomic::xchg(jint exchange_value, volatile jint* dest) { 219 220 // Note that xchg_ptr doesn't necessarily do an acquire 221 // (see synchronizer.cpp). 222 223 unsigned int old_value; 224 const uint64_t zero = 0; 225 226 __asm__ __volatile__ ( 227 /* lwsync */ 228 strasm_lwsync 229 /* atomic loop */ 230 "1: \n" 231 " lwarx %[old_value], %[dest], %[zero] \n" 232 " stwcx. %[exchange_value], %[dest], %[zero] \n" 233 " bne- 1b \n" 234 /* isync */ 235 strasm_sync 236 /* exit */ 237 "2: \n" 238 /* out */ 239 : [old_value] "=&r" (old_value), 240 "=m" (*dest) 241 /* in */ 242 : [dest] "b" (dest), 243 [zero] "r" (zero), 244 [exchange_value] "r" (exchange_value), 245 "m" (*dest) 246 /* clobber */ 247 : "cc", 248 "memory" 249 ); 250 251 return (jint) old_value; 252 } 253 254 inline intptr_t Atomic::xchg_ptr(intptr_t exchange_value, volatile intptr_t* dest) { 255 256 // Note that xchg_ptr doesn't necessarily do an acquire 257 // (see synchronizer.cpp). 258 259 long old_value; 260 const uint64_t zero = 0; 261 262 __asm__ __volatile__ ( 263 /* lwsync */ 264 strasm_lwsync 265 /* atomic loop */ 266 "1: \n" 267 " ldarx %[old_value], %[dest], %[zero] \n" 268 " stdcx. %[exchange_value], %[dest], %[zero] \n" 269 " bne- 1b \n" 270 /* isync */ 271 strasm_sync 272 /* exit */ 273 "2: \n" 274 /* out */ 275 : [old_value] "=&r" (old_value), 276 "=m" (*dest) 277 /* in */ 278 : [dest] "b" (dest), 279 [zero] "r" (zero), 280 [exchange_value] "r" (exchange_value), 281 "m" (*dest) 282 /* clobber */ 283 : "cc", 284 "memory" 285 ); 286 287 return (intptr_t) old_value; 288 } 289 290 inline void* Atomic::xchg_ptr(void* exchange_value, volatile void* dest) { 291 return (void*)xchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest); 292 } 293 294 inline void cmpxchg_pre_membar(memory_order order) { 295 if (order == memory_order_seq_cst) { 296 __asm__ __volatile__ ( 297 /* fence */ 298 strasm_sync 299 ); 300 } else if (order == memory_order_release || order == memory_order_acq_rel) { 301 __asm__ __volatile__ ( 302 /* release */ 303 strasm_lwsync 304 ); 305 } 306 } 307 308 inline void cmpxchg_post_membar(memory_order order) { 309 if (order == memory_order_seq_cst) { 310 __asm__ __volatile__ ( 311 /* fence */ 312 strasm_sync 313 ); 314 } else if (order == memory_order_acquire || order == memory_order_acq_rel) { 315 __asm__ __volatile__ ( 316 /* acquire */ 317 strasm_isync 318 ); 319 } 320 } 321 322 #define VM_HAS_SPECIALIZED_CMPXCHG_BYTE 323 inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, memory_order order) { 324 325 // Note that cmpxchg guarantees a two-way memory barrier across 326 // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire' 327 // (see atomic.hpp). 328 329 // Using 32 bit internally. 330 volatile int *dest_base = (volatile int*)((uintptr_t)dest & ~3); 331 332 #ifdef VM_LITTLE_ENDIAN 333 const unsigned int shift_amount = ((uintptr_t)dest & 3) * 8; 334 #else 335 const unsigned int shift_amount = ((~(uintptr_t)dest) & 3) * 8; 336 #endif 337 const unsigned int masked_compare_val = ((unsigned int)(unsigned char)compare_value), 338 masked_exchange_val = ((unsigned int)(unsigned char)exchange_value), 339 xor_value = (masked_compare_val ^ masked_exchange_val) << shift_amount; 340 341 unsigned int old_value, value32; 342 343 cmpxchg_pre_membar(order); 344 345 __asm__ __volatile__ ( 346 /* simple guard */ 347 " lbz %[old_value], 0(%[dest]) \n" 348 " cmpw %[masked_compare_val], %[old_value] \n" 349 " bne- 2f \n" 350 /* atomic loop */ 351 "1: \n" 352 " lwarx %[value32], 0, %[dest_base] \n" 353 /* extract byte and compare */ 354 " srd %[old_value], %[value32], %[shift_amount] \n" 355 " clrldi %[old_value], %[old_value], 56 \n" 356 " cmpw %[masked_compare_val], %[old_value] \n" 357 " bne- 2f \n" 358 /* replace byte and try to store */ 359 " xor %[value32], %[xor_value], %[value32] \n" 360 " stwcx. %[value32], 0, %[dest_base] \n" 361 " bne- 1b \n" 362 /* exit */ 363 "2: \n" 364 /* out */ 365 : [old_value] "=&r" (old_value), 366 [value32] "=&r" (value32), 367 "=m" (*dest), 368 "=m" (*dest_base) 369 /* in */ 370 : [dest] "b" (dest), 371 [dest_base] "b" (dest_base), 372 [shift_amount] "r" (shift_amount), 373 [masked_compare_val] "r" (masked_compare_val), 374 [xor_value] "r" (xor_value), 375 "m" (*dest), 376 "m" (*dest_base) 377 /* clobber */ 378 : "cc", 379 "memory" 380 ); 381 382 cmpxchg_post_membar(order); 383 384 return (jbyte)(unsigned char)old_value; 385 } 386 387 inline jint Atomic::cmpxchg(jint exchange_value, volatile jint* dest, jint compare_value, memory_order order) { 388 389 // Note that cmpxchg guarantees a two-way memory barrier across 390 // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire' 391 // (see atomic.hpp). 392 393 unsigned int old_value; 394 const uint64_t zero = 0; 395 396 cmpxchg_pre_membar(order); 397 398 __asm__ __volatile__ ( 399 /* simple guard */ 400 " lwz %[old_value], 0(%[dest]) \n" 401 " cmpw %[compare_value], %[old_value] \n" 402 " bne- 2f \n" 403 /* atomic loop */ 404 "1: \n" 405 " lwarx %[old_value], %[dest], %[zero] \n" 406 " cmpw %[compare_value], %[old_value] \n" 407 " bne- 2f \n" 408 " stwcx. %[exchange_value], %[dest], %[zero] \n" 409 " bne- 1b \n" 410 /* exit */ 411 "2: \n" 412 /* out */ 413 : [old_value] "=&r" (old_value), 414 "=m" (*dest) 415 /* in */ 416 : [dest] "b" (dest), 417 [zero] "r" (zero), 418 [compare_value] "r" (compare_value), 419 [exchange_value] "r" (exchange_value), 420 "m" (*dest) 421 /* clobber */ 422 : "cc", 423 "memory" 424 ); 425 426 cmpxchg_post_membar(order); 427 428 return (jint) old_value; 429 } 430 431 inline jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong* dest, jlong compare_value, memory_order order) { 432 433 // Note that cmpxchg guarantees a two-way memory barrier across 434 // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire' 435 // (see atomic.hpp). 436 437 long old_value; 438 const uint64_t zero = 0; 439 440 cmpxchg_pre_membar(order); 441 442 __asm__ __volatile__ ( 443 /* simple guard */ 444 " ld %[old_value], 0(%[dest]) \n" 445 " cmpd %[compare_value], %[old_value] \n" 446 " bne- 2f \n" 447 /* atomic loop */ 448 "1: \n" 449 " ldarx %[old_value], %[dest], %[zero] \n" 450 " cmpd %[compare_value], %[old_value] \n" 451 " bne- 2f \n" 452 " stdcx. %[exchange_value], %[dest], %[zero] \n" 453 " bne- 1b \n" 454 /* exit */ 455 "2: \n" 456 /* out */ 457 : [old_value] "=&r" (old_value), 458 "=m" (*dest) 459 /* in */ 460 : [dest] "b" (dest), 461 [zero] "r" (zero), 462 [compare_value] "r" (compare_value), 463 [exchange_value] "r" (exchange_value), 464 "m" (*dest) 465 /* clobber */ 466 : "cc", 467 "memory" 468 ); 469 470 cmpxchg_post_membar(order); 471 472 return (jlong) old_value; 473 } 474 475 inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, memory_order order) { 476 return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order); 477 } 478 479 inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, memory_order order) { 480 return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order); 481 } 482 483 #undef strasm_sync 484 #undef strasm_lwsync 485 #undef strasm_isync 486 #undef strasm_release 487 #undef strasm_acquire 488 #undef strasm_fence 489 #undef strasm_nobarrier 490 #undef strasm_nobarrier_clobber_memory 491 492 #endif // OS_CPU_LINUX_PPC_VM_ATOMIC_LINUX_PPC_INLINE_HPP