1 /* 2 * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2016 SAP SE. All rights reserved. 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This code is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 only, as 8 * published by the Free Software Foundation. 9 * 10 * This code is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * version 2 for more details (a copy is included in the LICENSE file that 14 * accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License version 17 * 2 along with this work; if not, write to the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 * or visit www.oracle.com if you need additional information or have any 22 * questions. 23 * 24 */ 25 26 #ifndef OS_CPU_LINUX_S390_VM_ATOMIC_LINUX_S390_INLINE_HPP 27 #define OS_CPU_LINUX_S390_VM_ATOMIC_LINUX_S390_INLINE_HPP 28 29 #include "runtime/atomic.hpp" 30 #include "runtime/os.hpp" 31 #include "vm_version_s390.hpp" 32 33 // Note that the compare-and-swap instructions on System z perform 34 // a serialization function before the storage operand is fetched 35 // and again after the operation is completed. 36 // 37 // Used constraint modifiers: 38 // = write-only access: Value on entry to inline-assembler code irrelevant. 39 // + read/write access: Value on entry is used; on exit value is changed. 40 // read-only access: Value on entry is used and never changed. 41 // & early-clobber access: Might be modified before all read-only operands 42 // have been used. 43 // a address register operand (not GR0). 44 // d general register operand (including GR0) 45 // Q memory operand w/o index register. 46 // 0..9 operand reference (by operand position). 47 // Used for operands that fill multiple roles. One example would be a 48 // write-only operand receiving its initial value from a read-only operand. 49 // Refer to cmpxchg(..) operand #0 and variable cmp_val for a real-life example. 50 // 51 52 // On System z, all store operations are atomic if the address where the data is stored into 53 // is an integer multiple of the data length. Furthermore, all stores are ordered: 54 // a store which occurs conceptually before another store becomes visible to other CPUs 55 // before the other store becomes visible. 56 inline void Atomic::store (jbyte store_value, jbyte* dest) { *dest = store_value; } 57 inline void Atomic::store (jshort store_value, jshort* dest) { *dest = store_value; } 58 inline void Atomic::store (jint store_value, jint* dest) { *dest = store_value; } 59 inline void Atomic::store (jlong store_value, jlong* dest) { *dest = store_value; } 60 inline void Atomic::store_ptr(intptr_t store_value, intptr_t* dest) { *dest = store_value; } 61 inline void Atomic::store_ptr(void* store_value, void* dest) { *(void**)dest = store_value; } 62 63 inline void Atomic::store (jbyte store_value, volatile jbyte* dest) { *dest = store_value; } 64 inline void Atomic::store (jshort store_value, volatile jshort* dest) { *dest = store_value; } 65 inline void Atomic::store (jint store_value, volatile jint* dest) { *dest = store_value; } 66 inline void Atomic::store (jlong store_value, volatile jlong* dest) { *dest = store_value; } 67 inline void Atomic::store_ptr(intptr_t store_value, volatile intptr_t* dest) { *dest = store_value; } 68 inline void Atomic::store_ptr(void* store_value, volatile void* dest) { *(void* volatile *)dest = store_value; } 69 70 71 //------------ 72 // Atomic::add 73 //------------ 74 // These methods force the value in memory to be augmented by the passed increment. 75 // Both, memory value and increment, are treated as 32bit signed binary integers. 76 // No overflow exceptions are recognized, and the condition code does not hold 77 // information about the value in memory. 78 // 79 // The value in memory is updated by using a compare-and-swap instruction. The 80 // instruction is retried as often as required. 81 // 82 // The return value of the method is the value that was successfully stored. At the 83 // time the caller receives back control, the value in memory may have changed already. 84 85 template<size_t byte_size> 86 struct Atomic::PlatformAdd 87 : Atomic::AddAndFetch<Atomic::PlatformAdd<byte_size> > 88 { 89 template<typename I, typename D> 90 D add_and_fetch(I add_value, D volatile* dest) const; 91 }; 92 93 template<> 94 template<typename I, typename D> 95 inline D Atomic::PlatformAdd<4>::add_and_fetch(I add_value, D volatile* dest) const { 96 STATIC_CAST(4 == sizeof(I)); 97 STATIC_CAST(4 == sizeof(D)); 98 99 D old, upd; 100 101 if (VM_Version::has_LoadAndALUAtomicV1()) { 102 __asm__ __volatile__ ( 103 " LGFR 0,%[inc] \n\t" // save increment 104 " LA 3,%[mem] \n\t" // force data address into ARG2 105 // " LAA %[upd],%[inc],%[mem] \n\t" // increment and get old value 106 // " LAA 2,0,0(3) \n\t" // actually coded instruction 107 " .byte 0xeb \n\t" // LAA main opcode 108 " .byte 0x20 \n\t" // R1,R3 109 " .byte 0x30 \n\t" // R2,disp1 110 " .byte 0x00 \n\t" // disp2,disp3 111 " .byte 0x00 \n\t" // disp4,disp5 112 " .byte 0xf8 \n\t" // LAA minor opcode 113 " AR 2,0 \n\t" // calc new value in register 114 " LR %[upd],2 \n\t" // move to result register 115 //---< outputs >--- 116 : [upd] "=&d" (upd) // write-only, updated counter value 117 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 118 //---< inputs >--- 119 : [inc] "a" (inc) // read-only. 120 //---< clobbered >--- 121 : "cc", "r0", "r2", "r3" 122 ); 123 } else { 124 __asm__ __volatile__ ( 125 " LLGF %[old],%[mem] \n\t" // get old value 126 "0: LA %[upd],0(%[inc],%[old]) \n\t" // calc result 127 " CS %[old],%[upd],%[mem] \n\t" // try to xchg res with mem 128 " JNE 0b \n\t" // no success? -> retry 129 //---< outputs >--- 130 : [old] "=&a" (old) // write-only, old counter value 131 , [upd] "=&d" (upd) // write-only, updated counter value 132 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 133 //---< inputs >--- 134 : [inc] "a" (inc) // read-only. 135 //---< clobbered >--- 136 : "cc" 137 ); 138 } 139 140 return upd; 141 } 142 143 144 template<> 145 template<typename I, typename D> 146 inline D Atomic::PlatformAdd<8>::add_and_fetch(I add_value, D volatile* dest) const { 147 STATIC_CAST(8 == sizeof(I)); 148 STATIC_CAST(8 == sizeof(D)); 149 150 D old, upd; 151 152 if (VM_Version::has_LoadAndALUAtomicV1()) { 153 __asm__ __volatile__ ( 154 " LGR 0,%[inc] \n\t" // save increment 155 " LA 3,%[mem] \n\t" // force data address into ARG2 156 // " LAAG %[upd],%[inc],%[mem] \n\t" // increment and get old value 157 // " LAAG 2,0,0(3) \n\t" // actually coded instruction 158 " .byte 0xeb \n\t" // LAA main opcode 159 " .byte 0x20 \n\t" // R1,R3 160 " .byte 0x30 \n\t" // R2,disp1 161 " .byte 0x00 \n\t" // disp2,disp3 162 " .byte 0x00 \n\t" // disp4,disp5 163 " .byte 0xe8 \n\t" // LAA minor opcode 164 " AGR 2,0 \n\t" // calc new value in register 165 " LGR %[upd],2 \n\t" // move to result register 166 //---< outputs >--- 167 : [upd] "=&d" (upd) // write-only, updated counter value 168 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 169 //---< inputs >--- 170 : [inc] "a" (inc) // read-only. 171 //---< clobbered >--- 172 : "cc", "r0", "r2", "r3" 173 ); 174 } else { 175 __asm__ __volatile__ ( 176 " LG %[old],%[mem] \n\t" // get old value 177 "0: LA %[upd],0(%[inc],%[old]) \n\t" // calc result 178 " CSG %[old],%[upd],%[mem] \n\t" // try to xchg res with mem 179 " JNE 0b \n\t" // no success? -> retry 180 //---< outputs >--- 181 : [old] "=&a" (old) // write-only, old counter value 182 , [upd] "=&d" (upd) // write-only, updated counter value 183 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 184 //---< inputs >--- 185 : [inc] "a" (inc) // read-only. 186 //---< clobbered >--- 187 : "cc" 188 ); 189 } 190 191 return upd; 192 } 193 194 195 //------------ 196 // Atomic::inc 197 //------------ 198 // These methods force the value in memory to be incremented (augmented by 1). 199 // Both, memory value and increment, are treated as 32bit signed binary integers. 200 // No overflow exceptions are recognized, and the condition code does not hold 201 // information about the value in memory. 202 // 203 // The value in memory is updated by using a compare-and-swap instruction. The 204 // instruction is retried as often as required. 205 206 inline void Atomic::inc(volatile jint* dest) { 207 unsigned int old, upd; 208 209 if (VM_Version::has_LoadAndALUAtomicV1()) { 210 // tty->print_cr("Atomic::inc called... dest @%p", dest); 211 __asm__ __volatile__ ( 212 " LGHI 2,1 \n\t" // load increment 213 " LA 3,%[mem] \n\t" // force data address into ARG2 214 // " LAA %[upd],%[inc],%[mem] \n\t" // increment and get old value 215 // " LAA 2,2,0(3) \n\t" // actually coded instruction 216 " .byte 0xeb \n\t" // LAA main opcode 217 " .byte 0x22 \n\t" // R1,R3 218 " .byte 0x30 \n\t" // R2,disp1 219 " .byte 0x00 \n\t" // disp2,disp3 220 " .byte 0x00 \n\t" // disp4,disp5 221 " .byte 0xf8 \n\t" // LAA minor opcode 222 " AGHI 2,1 \n\t" // calc new value in register 223 " LR %[upd],2 \n\t" // move to result register 224 //---< outputs >--- 225 : [upd] "=&d" (upd) // write-only, updated counter value 226 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 227 //---< inputs >--- 228 : 229 // : [inc] "a" (inc) // read-only. 230 //---< clobbered >--- 231 : "cc", "r2", "r3" 232 ); 233 } else { 234 __asm__ __volatile__ ( 235 " LLGF %[old],%[mem] \n\t" // get old value 236 "0: LA %[upd],1(,%[old]) \n\t" // calc result 237 " CS %[old],%[upd],%[mem] \n\t" // try to xchg res with mem 238 " JNE 0b \n\t" // no success? -> retry 239 //---< outputs >--- 240 : [old] "=&a" (old) // write-only, old counter value 241 , [upd] "=&d" (upd) // write-only, updated counter value 242 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 243 //---< inputs >--- 244 : 245 //---< clobbered >--- 246 : "cc" 247 ); 248 } 249 } 250 251 inline void Atomic::inc_ptr(volatile intptr_t* dest) { 252 unsigned long old, upd; 253 254 if (VM_Version::has_LoadAndALUAtomicV1()) { 255 __asm__ __volatile__ ( 256 " LGHI 2,1 \n\t" // load increment 257 " LA 3,%[mem] \n\t" // force data address into ARG2 258 // " LAAG %[upd],%[inc],%[mem] \n\t" // increment and get old value 259 // " LAAG 2,2,0(3) \n\t" // actually coded instruction 260 " .byte 0xeb \n\t" // LAA main opcode 261 " .byte 0x22 \n\t" // R1,R3 262 " .byte 0x30 \n\t" // R2,disp1 263 " .byte 0x00 \n\t" // disp2,disp3 264 " .byte 0x00 \n\t" // disp4,disp5 265 " .byte 0xe8 \n\t" // LAA minor opcode 266 " AGHI 2,1 \n\t" // calc new value in register 267 " LR %[upd],2 \n\t" // move to result register 268 //---< outputs >--- 269 : [upd] "=&d" (upd) // write-only, updated counter value 270 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 271 //---< inputs >--- 272 : 273 // : [inc] "a" (inc) // read-only. 274 //---< clobbered >--- 275 : "cc", "r2", "r3" 276 ); 277 } else { 278 __asm__ __volatile__ ( 279 " LG %[old],%[mem] \n\t" // get old value 280 "0: LA %[upd],1(,%[old]) \n\t" // calc result 281 " CSG %[old],%[upd],%[mem] \n\t" // try to xchg res with mem 282 " JNE 0b \n\t" // no success? -> retry 283 //---< outputs >--- 284 : [old] "=&a" (old) // write-only, old counter value 285 , [upd] "=&d" (upd) // write-only, updated counter value 286 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 287 //---< inputs >--- 288 : 289 //---< clobbered >--- 290 : "cc" 291 ); 292 } 293 } 294 295 inline void Atomic::inc_ptr(volatile void* dest) { 296 inc_ptr((volatile intptr_t*)dest); 297 } 298 299 //------------ 300 // Atomic::dec 301 //------------ 302 // These methods force the value in memory to be decremented (augmented by -1). 303 // Both, memory value and decrement, are treated as 32bit signed binary integers. 304 // No overflow exceptions are recognized, and the condition code does not hold 305 // information about the value in memory. 306 // 307 // The value in memory is updated by using a compare-and-swap instruction. The 308 // instruction is retried as often as required. 309 310 inline void Atomic::dec(volatile jint* dest) { 311 unsigned int old, upd; 312 313 if (VM_Version::has_LoadAndALUAtomicV1()) { 314 __asm__ __volatile__ ( 315 " LGHI 2,-1 \n\t" // load increment 316 " LA 3,%[mem] \n\t" // force data address into ARG2 317 // " LAA %[upd],%[inc],%[mem] \n\t" // increment and get old value 318 // " LAA 2,2,0(3) \n\t" // actually coded instruction 319 " .byte 0xeb \n\t" // LAA main opcode 320 " .byte 0x22 \n\t" // R1,R3 321 " .byte 0x30 \n\t" // R2,disp1 322 " .byte 0x00 \n\t" // disp2,disp3 323 " .byte 0x00 \n\t" // disp4,disp5 324 " .byte 0xf8 \n\t" // LAA minor opcode 325 " AGHI 2,-1 \n\t" // calc new value in register 326 " LR %[upd],2 \n\t" // move to result register 327 //---< outputs >--- 328 : [upd] "=&d" (upd) // write-only, updated counter value 329 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 330 //---< inputs >--- 331 : 332 // : [inc] "a" (inc) // read-only. 333 //---< clobbered >--- 334 : "cc", "r2", "r3" 335 ); 336 } else { 337 __asm__ __volatile__ ( 338 " LLGF %[old],%[mem] \n\t" // get old value 339 // LAY not supported by inline assembler 340 // "0: LAY %[upd],-1(,%[old]) \n\t" // calc result 341 "0: LR %[upd],%[old] \n\t" // calc result 342 " AHI %[upd],-1 \n\t" 343 " CS %[old],%[upd],%[mem] \n\t" // try to xchg res with mem 344 " JNE 0b \n\t" // no success? -> retry 345 //---< outputs >--- 346 : [old] "=&a" (old) // write-only, old counter value 347 , [upd] "=&d" (upd) // write-only, updated counter value 348 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 349 //---< inputs >--- 350 : 351 //---< clobbered >--- 352 : "cc" 353 ); 354 } 355 } 356 357 inline void Atomic::dec_ptr(volatile intptr_t* dest) { 358 unsigned long old, upd; 359 360 if (VM_Version::has_LoadAndALUAtomicV1()) { 361 __asm__ __volatile__ ( 362 " LGHI 2,-1 \n\t" // load increment 363 " LA 3,%[mem] \n\t" // force data address into ARG2 364 // " LAAG %[upd],%[inc],%[mem] \n\t" // increment and get old value 365 // " LAAG 2,2,0(3) \n\t" // actually coded instruction 366 " .byte 0xeb \n\t" // LAA main opcode 367 " .byte 0x22 \n\t" // R1,R3 368 " .byte 0x30 \n\t" // R2,disp1 369 " .byte 0x00 \n\t" // disp2,disp3 370 " .byte 0x00 \n\t" // disp4,disp5 371 " .byte 0xe8 \n\t" // LAA minor opcode 372 " AGHI 2,-1 \n\t" // calc new value in register 373 " LR %[upd],2 \n\t" // move to result register 374 //---< outputs >--- 375 : [upd] "=&d" (upd) // write-only, updated counter value 376 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 377 //---< inputs >--- 378 : 379 // : [inc] "a" (inc) // read-only. 380 //---< clobbered >--- 381 : "cc", "r2", "r3" 382 ); 383 } else { 384 __asm__ __volatile__ ( 385 " LG %[old],%[mem] \n\t" // get old value 386 // LAY not supported by inline assembler 387 // "0: LAY %[upd],-1(,%[old]) \n\t" // calc result 388 "0: LGR %[upd],%[old] \n\t" // calc result 389 " AGHI %[upd],-1 \n\t" 390 " CSG %[old],%[upd],%[mem] \n\t" // try to xchg res with mem 391 " JNE 0b \n\t" // no success? -> retry 392 //---< outputs >--- 393 : [old] "=&a" (old) // write-only, old counter value 394 , [upd] "=&d" (upd) // write-only, updated counter value 395 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 396 //---< inputs >--- 397 : 398 //---< clobbered >--- 399 : "cc" 400 ); 401 } 402 } 403 404 inline void Atomic::dec_ptr(volatile void* dest) { 405 dec_ptr((volatile intptr_t*)dest); 406 } 407 408 //------------- 409 // Atomic::xchg 410 //------------- 411 // These methods force the value in memory to be replaced by the new value passed 412 // in as argument. 413 // 414 // The value in memory is replaced by using a compare-and-swap instruction. The 415 // instruction is retried as often as required. This makes sure that the new 416 // value can be seen, at least for a very short period of time, by other CPUs. 417 // 418 // If we would use a normal "load(old value) store(new value)" sequence, 419 // the new value could be lost unnoticed, due to a store(new value) from 420 // another thread. 421 // 422 // The return value is the (unchanged) value from memory as it was when the 423 // replacement succeeded. 424 inline jint Atomic::xchg (jint xchg_val, volatile jint* dest) { 425 unsigned int old; 426 427 __asm__ __volatile__ ( 428 " LLGF %[old],%[mem] \n\t" // get old value 429 "0: CS %[old],%[upd],%[mem] \n\t" // try to xchg upd with mem 430 " JNE 0b \n\t" // no success? -> retry 431 //---< outputs >--- 432 : [old] "=&d" (old) // write-only, prev value irrelevant 433 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 434 //---< inputs >--- 435 : [upd] "d" (xchg_val) // read-only, value to be written to memory 436 //---< clobbered >--- 437 : "cc" 438 ); 439 440 return (jint)old; 441 } 442 443 inline intptr_t Atomic::xchg_ptr(intptr_t xchg_val, volatile intptr_t* dest) { 444 unsigned long old; 445 446 __asm__ __volatile__ ( 447 " LG %[old],%[mem] \n\t" // get old value 448 "0: CSG %[old],%[upd],%[mem] \n\t" // try to xchg upd with mem 449 " JNE 0b \n\t" // no success? -> retry 450 //---< outputs >--- 451 : [old] "=&d" (old) // write-only, init from memory 452 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 453 //---< inputs >--- 454 : [upd] "d" (xchg_val) // read-only, value to be written to memory 455 //---< clobbered >--- 456 : "cc" 457 ); 458 459 return (intptr_t)old; 460 } 461 462 inline void *Atomic::xchg_ptr(void *exchange_value, volatile void *dest) { 463 return (void*)xchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest); 464 } 465 466 //---------------- 467 // Atomic::cmpxchg 468 //---------------- 469 // These methods compare the value in memory with a given compare value. 470 // If both values compare equal, the value in memory is replaced with 471 // the exchange value. 472 // 473 // The value in memory is compared and replaced by using a compare-and-swap 474 // instruction. The instruction is NOT retried (one shot only). 475 // 476 // The return value is the (unchanged) value from memory as it was when the 477 // compare-and-swap instruction completed. A successful exchange operation 478 // is indicated by (return value == compare_value). If unsuccessful, a new 479 // exchange value can be calculated based on the return value which is the 480 // latest contents of the memory location. 481 // 482 // Inspecting the return value is the only way for the caller to determine 483 // if the compare-and-swap instruction was successful: 484 // - If return value and compare value compare equal, the compare-and-swap 485 // instruction was successful and the value in memory was replaced by the 486 // exchange value. 487 // - If return value and compare value compare unequal, the compare-and-swap 488 // instruction was not successful. The value in memory was left unchanged. 489 // 490 // The s390 processors always fence before and after the csg instructions. 491 // Thus we ignore the memory ordering argument. The docu says: "A serialization 492 // function is performed before the operand is fetched and again after the 493 // operation is completed." 494 495 // No direct support for cmpxchg of bytes; emulate using int. 496 template<> 497 struct Atomic::PlatformCmpxchg<1> : Atomic::CmpxchgByteUsingInt {}; 498 499 template<> 500 template<typename T> 501 inline T Atomic::PlatformCmpxchg<4>::operator()(T xchg_val, 502 T volatile* dest, 503 T cmp_val, 504 cmpxchg_memory_order unused) const { 505 STATIC_ASSERT(4 == sizeof(T)); 506 T old; 507 508 __asm__ __volatile__ ( 509 " CS %[old],%[upd],%[mem] \n\t" // Try to xchg upd with mem. 510 // outputs 511 : [old] "=&d" (old) // Write-only, prev value irrelevant. 512 , [mem] "+Q" (*dest) // Read/write, memory to be updated atomically. 513 // inputs 514 : [upd] "d" (xchg_val) 515 , "0" (cmp_val) // Read-only, initial value for [old] (operand #0). 516 // clobbered 517 : "cc" 518 ); 519 520 return old; 521 } 522 523 template<> 524 template<typename T> 525 inline T Atomic::PlatformCmpxchg<8>::operator()(T xchg_val, 526 T volatile* dest, 527 T cmp_val, 528 cmpxchg_memory_order unused) const { 529 STATIC_ASSERT(8 == sizeof(T)); 530 T old; 531 532 __asm__ __volatile__ ( 533 " CSG %[old],%[upd],%[mem] \n\t" // Try to xchg upd with mem. 534 // outputs 535 : [old] "=&d" (old) // Write-only, prev value irrelevant. 536 , [mem] "+Q" (*dest) // Read/write, memory to be updated atomically. 537 // inputs 538 : [upd] "d" (xchg_val) 539 , "0" (cmp_val) // Read-only, initial value for [old] (operand #0). 540 // clobbered 541 : "cc" 542 ); 543 544 return old; 545 } 546 547 inline jlong Atomic::load(const volatile jlong* src) { return *src; } 548 549 #endif // OS_CPU_LINUX_S390_VM_ATOMIC_LINUX_S390_INLINE_HPP