1 /* 2 * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2016 SAP SE. All rights reserved. 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This code is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 only, as 8 * published by the Free Software Foundation. 9 * 10 * This code is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * version 2 for more details (a copy is included in the LICENSE file that 14 * accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License version 17 * 2 along with this work; if not, write to the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 * or visit www.oracle.com if you need additional information or have any 22 * questions. 23 * 24 */ 25 26 #ifndef OS_CPU_LINUX_S390_VM_ATOMIC_LINUX_S390_INLINE_HPP 27 #define OS_CPU_LINUX_S390_VM_ATOMIC_LINUX_S390_INLINE_HPP 28 29 #include "runtime/atomic.hpp" 30 #include "runtime/os.hpp" 31 #include "vm_version_s390.hpp" 32 33 // Note that the compare-and-swap instructions on System z perform 34 // a serialization function before the storage operand is fetched 35 // and again after the operation is completed. 36 // 37 // Used constraint modifiers: 38 // = write-only access: Value on entry to inline-assembler code irrelevant. 39 // + read/write access: Value on entry is used; on exit value is changed. 40 // read-only access: Value on entry is used and never changed. 41 // & early-clobber access: Might be modified before all read-only operands 42 // have been used. 43 // a address register operand (not GR0). 44 // d general register operand (including GR0) 45 // Q memory operand w/o index register. 46 // 0..9 operand reference (by operand position). 47 // Used for operands that fill multiple roles. One example would be a 48 // write-only operand receiving its initial value from a read-only operand. 49 // Refer to cmpxchg(..) operand #0 and variable cmp_val for a real-life example. 50 // 51 52 // On System z, all store operations are atomic if the address where the data is stored into 53 // is an integer multiple of the data length. Furthermore, all stores are ordered: 54 // a store which occurs conceptually before another store becomes visible to other CPUs 55 // before the other store becomes visible. 56 inline void Atomic::store (jbyte store_value, jbyte* dest) { *dest = store_value; } 57 inline void Atomic::store (jshort store_value, jshort* dest) { *dest = store_value; } 58 inline void Atomic::store (jint store_value, jint* dest) { *dest = store_value; } 59 inline void Atomic::store (jlong store_value, jlong* dest) { *dest = store_value; } 60 inline void Atomic::store_ptr(intptr_t store_value, intptr_t* dest) { *dest = store_value; } 61 inline void Atomic::store_ptr(void* store_value, void* dest) { *(void**)dest = store_value; } 62 63 inline void Atomic::store (jbyte store_value, volatile jbyte* dest) { *dest = store_value; } 64 inline void Atomic::store (jshort store_value, volatile jshort* dest) { *dest = store_value; } 65 inline void Atomic::store (jint store_value, volatile jint* dest) { *dest = store_value; } 66 inline void Atomic::store (jlong store_value, volatile jlong* dest) { *dest = store_value; } 67 inline void Atomic::store_ptr(intptr_t store_value, volatile intptr_t* dest) { *dest = store_value; } 68 inline void Atomic::store_ptr(void* store_value, volatile void* dest) { *(void* volatile *)dest = store_value; } 69 70 71 //------------ 72 // Atomic::add 73 //------------ 74 // These methods force the value in memory to be augmented by the passed increment. 75 // Both, memory value and increment, are treated as 32bit signed binary integers. 76 // No overflow exceptions are recognized, and the condition code does not hold 77 // information about the value in memory. 78 // 79 // The value in memory is updated by using a compare-and-swap instruction. The 80 // instruction is retried as often as required. 81 // 82 // The return value of the method is the value that was successfully stored. At the 83 // time the caller receives back control, the value in memory may have changed already. 84 85 template<size_t byte_size> 86 struct Atomic::PlatformAdd 87 : Atomic::AddAndFetch<Atomic::PlatformAdd<byte_size> > 88 { 89 template<typename I, typename D> 90 D add_and_fetch(I add_value, D volatile* dest) const; 91 }; 92 93 template<> 94 template<typename I, typename D> 95 inline D Atomic::PlatformAdd<4>::add_and_fetch(I add_value, D volatile* dest) const { 96 STATIC_CAST(4 == sizeof(I)); 97 STATIC_CAST(4 == sizeof(D)); 98 99 D old, upd; 100 101 if (VM_Version::has_LoadAndALUAtomicV1()) { 102 __asm__ __volatile__ ( 103 " LGFR 0,%[inc] \n\t" // save increment 104 " LA 3,%[mem] \n\t" // force data address into ARG2 105 // " LAA %[upd],%[inc],%[mem] \n\t" // increment and get old value 106 // " LAA 2,0,0(3) \n\t" // actually coded instruction 107 " .byte 0xeb \n\t" // LAA main opcode 108 " .byte 0x20 \n\t" // R1,R3 109 " .byte 0x30 \n\t" // R2,disp1 110 " .byte 0x00 \n\t" // disp2,disp3 111 " .byte 0x00 \n\t" // disp4,disp5 112 " .byte 0xf8 \n\t" // LAA minor opcode 113 " AR 2,0 \n\t" // calc new value in register 114 " LR %[upd],2 \n\t" // move to result register 115 //---< outputs >--- 116 : [upd] "=&d" (upd) // write-only, updated counter value 117 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 118 //---< inputs >--- 119 : [inc] "a" (inc) // read-only. 120 //---< clobbered >--- 121 : "cc", "r0", "r2", "r3" 122 ); 123 } else { 124 __asm__ __volatile__ ( 125 " LLGF %[old],%[mem] \n\t" // get old value 126 "0: LA %[upd],0(%[inc],%[old]) \n\t" // calc result 127 " CS %[old],%[upd],%[mem] \n\t" // try to xchg res with mem 128 " JNE 0b \n\t" // no success? -> retry 129 //---< outputs >--- 130 : [old] "=&a" (old) // write-only, old counter value 131 , [upd] "=&d" (upd) // write-only, updated counter value 132 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 133 //---< inputs >--- 134 : [inc] "a" (inc) // read-only. 135 //---< clobbered >--- 136 : "cc" 137 ); 138 } 139 140 return upd; 141 } 142 143 144 template<> 145 template<typename I, typename D> 146 inline D Atomic::PlatformAdd<8>::add_and_fetch(I add_value, D volatile* dest) const { 147 STATIC_CAST(8 == sizeof(I)); 148 STATIC_CAST(8 == sizeof(D)); 149 150 D old, upd; 151 152 if (VM_Version::has_LoadAndALUAtomicV1()) { 153 __asm__ __volatile__ ( 154 " LGR 0,%[inc] \n\t" // save increment 155 " LA 3,%[mem] \n\t" // force data address into ARG2 156 // " LAAG %[upd],%[inc],%[mem] \n\t" // increment and get old value 157 // " LAAG 2,0,0(3) \n\t" // actually coded instruction 158 " .byte 0xeb \n\t" // LAA main opcode 159 " .byte 0x20 \n\t" // R1,R3 160 " .byte 0x30 \n\t" // R2,disp1 161 " .byte 0x00 \n\t" // disp2,disp3 162 " .byte 0x00 \n\t" // disp4,disp5 163 " .byte 0xe8 \n\t" // LAA minor opcode 164 " AGR 2,0 \n\t" // calc new value in register 165 " LGR %[upd],2 \n\t" // move to result register 166 //---< outputs >--- 167 : [upd] "=&d" (upd) // write-only, updated counter value 168 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 169 //---< inputs >--- 170 : [inc] "a" (inc) // read-only. 171 //---< clobbered >--- 172 : "cc", "r0", "r2", "r3" 173 ); 174 } else { 175 __asm__ __volatile__ ( 176 " LG %[old],%[mem] \n\t" // get old value 177 "0: LA %[upd],0(%[inc],%[old]) \n\t" // calc result 178 " CSG %[old],%[upd],%[mem] \n\t" // try to xchg res with mem 179 " JNE 0b \n\t" // no success? -> retry 180 //---< outputs >--- 181 : [old] "=&a" (old) // write-only, old counter value 182 , [upd] "=&d" (upd) // write-only, updated counter value 183 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 184 //---< inputs >--- 185 : [inc] "a" (inc) // read-only. 186 //---< clobbered >--- 187 : "cc" 188 ); 189 } 190 191 return upd; 192 } 193 194 template<> 195 struct Atomic::PlatformAdd<2>: Atomic::AddShortUsingInt {}; 196 197 //------------ 198 // Atomic::inc 199 //------------ 200 // These methods force the value in memory to be incremented (augmented by 1). 201 // Both, memory value and increment, are treated as 32bit signed binary integers. 202 // No overflow exceptions are recognized, and the condition code does not hold 203 // information about the value in memory. 204 // 205 // The value in memory is updated by using a compare-and-swap instruction. The 206 // instruction is retried as often as required. 207 208 inline void Atomic::inc(volatile jint* dest) { 209 unsigned int old, upd; 210 211 if (VM_Version::has_LoadAndALUAtomicV1()) { 212 // tty->print_cr("Atomic::inc called... dest @%p", dest); 213 __asm__ __volatile__ ( 214 " LGHI 2,1 \n\t" // load increment 215 " LA 3,%[mem] \n\t" // force data address into ARG2 216 // " LAA %[upd],%[inc],%[mem] \n\t" // increment and get old value 217 // " LAA 2,2,0(3) \n\t" // actually coded instruction 218 " .byte 0xeb \n\t" // LAA main opcode 219 " .byte 0x22 \n\t" // R1,R3 220 " .byte 0x30 \n\t" // R2,disp1 221 " .byte 0x00 \n\t" // disp2,disp3 222 " .byte 0x00 \n\t" // disp4,disp5 223 " .byte 0xf8 \n\t" // LAA minor opcode 224 " AGHI 2,1 \n\t" // calc new value in register 225 " LR %[upd],2 \n\t" // move to result register 226 //---< outputs >--- 227 : [upd] "=&d" (upd) // write-only, updated counter value 228 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 229 //---< inputs >--- 230 : 231 // : [inc] "a" (inc) // read-only. 232 //---< clobbered >--- 233 : "cc", "r2", "r3" 234 ); 235 } else { 236 __asm__ __volatile__ ( 237 " LLGF %[old],%[mem] \n\t" // get old value 238 "0: LA %[upd],1(,%[old]) \n\t" // calc result 239 " CS %[old],%[upd],%[mem] \n\t" // try to xchg res with mem 240 " JNE 0b \n\t" // no success? -> retry 241 //---< outputs >--- 242 : [old] "=&a" (old) // write-only, old counter value 243 , [upd] "=&d" (upd) // write-only, updated counter value 244 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 245 //---< inputs >--- 246 : 247 //---< clobbered >--- 248 : "cc" 249 ); 250 } 251 } 252 253 inline void Atomic::inc_ptr(volatile intptr_t* dest) { 254 unsigned long old, upd; 255 256 if (VM_Version::has_LoadAndALUAtomicV1()) { 257 __asm__ __volatile__ ( 258 " LGHI 2,1 \n\t" // load increment 259 " LA 3,%[mem] \n\t" // force data address into ARG2 260 // " LAAG %[upd],%[inc],%[mem] \n\t" // increment and get old value 261 // " LAAG 2,2,0(3) \n\t" // actually coded instruction 262 " .byte 0xeb \n\t" // LAA main opcode 263 " .byte 0x22 \n\t" // R1,R3 264 " .byte 0x30 \n\t" // R2,disp1 265 " .byte 0x00 \n\t" // disp2,disp3 266 " .byte 0x00 \n\t" // disp4,disp5 267 " .byte 0xe8 \n\t" // LAA minor opcode 268 " AGHI 2,1 \n\t" // calc new value in register 269 " LR %[upd],2 \n\t" // move to result register 270 //---< outputs >--- 271 : [upd] "=&d" (upd) // write-only, updated counter value 272 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 273 //---< inputs >--- 274 : 275 // : [inc] "a" (inc) // read-only. 276 //---< clobbered >--- 277 : "cc", "r2", "r3" 278 ); 279 } else { 280 __asm__ __volatile__ ( 281 " LG %[old],%[mem] \n\t" // get old value 282 "0: LA %[upd],1(,%[old]) \n\t" // calc result 283 " CSG %[old],%[upd],%[mem] \n\t" // try to xchg res with mem 284 " JNE 0b \n\t" // no success? -> retry 285 //---< outputs >--- 286 : [old] "=&a" (old) // write-only, old counter value 287 , [upd] "=&d" (upd) // write-only, updated counter value 288 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 289 //---< inputs >--- 290 : 291 //---< clobbered >--- 292 : "cc" 293 ); 294 } 295 } 296 297 inline void Atomic::inc_ptr(volatile void* dest) { 298 inc_ptr((volatile intptr_t*)dest); 299 } 300 301 //------------ 302 // Atomic::dec 303 //------------ 304 // These methods force the value in memory to be decremented (augmented by -1). 305 // Both, memory value and decrement, are treated as 32bit signed binary integers. 306 // No overflow exceptions are recognized, and the condition code does not hold 307 // information about the value in memory. 308 // 309 // The value in memory is updated by using a compare-and-swap instruction. The 310 // instruction is retried as often as required. 311 312 inline void Atomic::dec(volatile jint* dest) { 313 unsigned int old, upd; 314 315 if (VM_Version::has_LoadAndALUAtomicV1()) { 316 __asm__ __volatile__ ( 317 " LGHI 2,-1 \n\t" // load increment 318 " LA 3,%[mem] \n\t" // force data address into ARG2 319 // " LAA %[upd],%[inc],%[mem] \n\t" // increment and get old value 320 // " LAA 2,2,0(3) \n\t" // actually coded instruction 321 " .byte 0xeb \n\t" // LAA main opcode 322 " .byte 0x22 \n\t" // R1,R3 323 " .byte 0x30 \n\t" // R2,disp1 324 " .byte 0x00 \n\t" // disp2,disp3 325 " .byte 0x00 \n\t" // disp4,disp5 326 " .byte 0xf8 \n\t" // LAA minor opcode 327 " AGHI 2,-1 \n\t" // calc new value in register 328 " LR %[upd],2 \n\t" // move to result register 329 //---< outputs >--- 330 : [upd] "=&d" (upd) // write-only, updated counter value 331 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 332 //---< inputs >--- 333 : 334 // : [inc] "a" (inc) // read-only. 335 //---< clobbered >--- 336 : "cc", "r2", "r3" 337 ); 338 } else { 339 __asm__ __volatile__ ( 340 " LLGF %[old],%[mem] \n\t" // get old value 341 // LAY not supported by inline assembler 342 // "0: LAY %[upd],-1(,%[old]) \n\t" // calc result 343 "0: LR %[upd],%[old] \n\t" // calc result 344 " AHI %[upd],-1 \n\t" 345 " CS %[old],%[upd],%[mem] \n\t" // try to xchg res with mem 346 " JNE 0b \n\t" // no success? -> retry 347 //---< outputs >--- 348 : [old] "=&a" (old) // write-only, old counter value 349 , [upd] "=&d" (upd) // write-only, updated counter value 350 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 351 //---< inputs >--- 352 : 353 //---< clobbered >--- 354 : "cc" 355 ); 356 } 357 } 358 359 inline void Atomic::dec_ptr(volatile intptr_t* dest) { 360 unsigned long old, upd; 361 362 if (VM_Version::has_LoadAndALUAtomicV1()) { 363 __asm__ __volatile__ ( 364 " LGHI 2,-1 \n\t" // load increment 365 " LA 3,%[mem] \n\t" // force data address into ARG2 366 // " LAAG %[upd],%[inc],%[mem] \n\t" // increment and get old value 367 // " LAAG 2,2,0(3) \n\t" // actually coded instruction 368 " .byte 0xeb \n\t" // LAA main opcode 369 " .byte 0x22 \n\t" // R1,R3 370 " .byte 0x30 \n\t" // R2,disp1 371 " .byte 0x00 \n\t" // disp2,disp3 372 " .byte 0x00 \n\t" // disp4,disp5 373 " .byte 0xe8 \n\t" // LAA minor opcode 374 " AGHI 2,-1 \n\t" // calc new value in register 375 " LR %[upd],2 \n\t" // move to result register 376 //---< outputs >--- 377 : [upd] "=&d" (upd) // write-only, updated counter value 378 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 379 //---< inputs >--- 380 : 381 // : [inc] "a" (inc) // read-only. 382 //---< clobbered >--- 383 : "cc", "r2", "r3" 384 ); 385 } else { 386 __asm__ __volatile__ ( 387 " LG %[old],%[mem] \n\t" // get old value 388 // LAY not supported by inline assembler 389 // "0: LAY %[upd],-1(,%[old]) \n\t" // calc result 390 "0: LGR %[upd],%[old] \n\t" // calc result 391 " AGHI %[upd],-1 \n\t" 392 " CSG %[old],%[upd],%[mem] \n\t" // try to xchg res with mem 393 " JNE 0b \n\t" // no success? -> retry 394 //---< outputs >--- 395 : [old] "=&a" (old) // write-only, old counter value 396 , [upd] "=&d" (upd) // write-only, updated counter value 397 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 398 //---< inputs >--- 399 : 400 //---< clobbered >--- 401 : "cc" 402 ); 403 } 404 } 405 406 inline void Atomic::dec_ptr(volatile void* dest) { 407 dec_ptr((volatile intptr_t*)dest); 408 } 409 410 //------------- 411 // Atomic::xchg 412 //------------- 413 // These methods force the value in memory to be replaced by the new value passed 414 // in as argument. 415 // 416 // The value in memory is replaced by using a compare-and-swap instruction. The 417 // instruction is retried as often as required. This makes sure that the new 418 // value can be seen, at least for a very short period of time, by other CPUs. 419 // 420 // If we would use a normal "load(old value) store(new value)" sequence, 421 // the new value could be lost unnoticed, due to a store(new value) from 422 // another thread. 423 // 424 // The return value is the (unchanged) value from memory as it was when the 425 // replacement succeeded. 426 inline jint Atomic::xchg (jint xchg_val, volatile jint* dest) { 427 unsigned int old; 428 429 __asm__ __volatile__ ( 430 " LLGF %[old],%[mem] \n\t" // get old value 431 "0: CS %[old],%[upd],%[mem] \n\t" // try to xchg upd with mem 432 " JNE 0b \n\t" // no success? -> retry 433 //---< outputs >--- 434 : [old] "=&d" (old) // write-only, prev value irrelevant 435 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 436 //---< inputs >--- 437 : [upd] "d" (xchg_val) // read-only, value to be written to memory 438 //---< clobbered >--- 439 : "cc" 440 ); 441 442 return (jint)old; 443 } 444 445 inline intptr_t Atomic::xchg_ptr(intptr_t xchg_val, volatile intptr_t* dest) { 446 unsigned long old; 447 448 __asm__ __volatile__ ( 449 " LG %[old],%[mem] \n\t" // get old value 450 "0: CSG %[old],%[upd],%[mem] \n\t" // try to xchg upd with mem 451 " JNE 0b \n\t" // no success? -> retry 452 //---< outputs >--- 453 : [old] "=&d" (old) // write-only, init from memory 454 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 455 //---< inputs >--- 456 : [upd] "d" (xchg_val) // read-only, value to be written to memory 457 //---< clobbered >--- 458 : "cc" 459 ); 460 461 return (intptr_t)old; 462 } 463 464 inline void *Atomic::xchg_ptr(void *exchange_value, volatile void *dest) { 465 return (void*)xchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest); 466 } 467 468 //---------------- 469 // Atomic::cmpxchg 470 //---------------- 471 // These methods compare the value in memory with a given compare value. 472 // If both values compare equal, the value in memory is replaced with 473 // the exchange value. 474 // 475 // The value in memory is compared and replaced by using a compare-and-swap 476 // instruction. The instruction is NOT retried (one shot only). 477 // 478 // The return value is the (unchanged) value from memory as it was when the 479 // compare-and-swap instruction completed. A successful exchange operation 480 // is indicated by (return value == compare_value). If unsuccessful, a new 481 // exchange value can be calculated based on the return value which is the 482 // latest contents of the memory location. 483 // 484 // Inspecting the return value is the only way for the caller to determine 485 // if the compare-and-swap instruction was successful: 486 // - If return value and compare value compare equal, the compare-and-swap 487 // instruction was successful and the value in memory was replaced by the 488 // exchange value. 489 // - If return value and compare value compare unequal, the compare-and-swap 490 // instruction was not successful. The value in memory was left unchanged. 491 // 492 // The s390 processors always fence before and after the csg instructions. 493 // Thus we ignore the memory ordering argument. The docu says: "A serialization 494 // function is performed before the operand is fetched and again after the 495 // operation is completed." 496 497 // No direct support for cmpxchg of bytes; emulate using int. 498 template<> 499 struct Atomic::PlatformCmpxchg<1> : Atomic::CmpxchgByteUsingInt {}; 500 501 template<> 502 template<typename T> 503 inline T Atomic::PlatformCmpxchg<4>::operator()(T xchg_val, 504 T volatile* dest, 505 T cmp_val, 506 cmpxchg_memory_order unused) const { 507 STATIC_ASSERT(4 == sizeof(T)); 508 T old; 509 510 __asm__ __volatile__ ( 511 " CS %[old],%[upd],%[mem] \n\t" // Try to xchg upd with mem. 512 // outputs 513 : [old] "=&d" (old) // Write-only, prev value irrelevant. 514 , [mem] "+Q" (*dest) // Read/write, memory to be updated atomically. 515 // inputs 516 : [upd] "d" (xchg_val) 517 , "0" (cmp_val) // Read-only, initial value for [old] (operand #0). 518 // clobbered 519 : "cc" 520 ); 521 522 return old; 523 } 524 525 template<> 526 template<typename T> 527 inline T Atomic::PlatformCmpxchg<8>::operator()(T xchg_val, 528 T volatile* dest, 529 T cmp_val, 530 cmpxchg_memory_order unused) const { 531 STATIC_ASSERT(8 == sizeof(T)); 532 T old; 533 534 __asm__ __volatile__ ( 535 " CSG %[old],%[upd],%[mem] \n\t" // Try to xchg upd with mem. 536 // outputs 537 : [old] "=&d" (old) // Write-only, prev value irrelevant. 538 , [mem] "+Q" (*dest) // Read/write, memory to be updated atomically. 539 // inputs 540 : [upd] "d" (xchg_val) 541 , "0" (cmp_val) // Read-only, initial value for [old] (operand #0). 542 // clobbered 543 : "cc" 544 ); 545 546 return old; 547 } 548 549 inline jlong Atomic::load(const volatile jlong* src) { return *src; } 550 551 #endif // OS_CPU_LINUX_S390_VM_ATOMIC_LINUX_S390_INLINE_HPP