1 /* 2 * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2016 SAP SE. All rights reserved. 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This code is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 only, as 8 * published by the Free Software Foundation. 9 * 10 * This code is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * version 2 for more details (a copy is included in the LICENSE file that 14 * accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License version 17 * 2 along with this work; if not, write to the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 * or visit www.oracle.com if you need additional information or have any 22 * questions. 23 * 24 */ 25 26 #ifndef OS_CPU_LINUX_S390_VM_ATOMIC_LINUX_S390_INLINE_HPP 27 #define OS_CPU_LINUX_S390_VM_ATOMIC_LINUX_S390_INLINE_HPP 28 29 #include "runtime/atomic.hpp" 30 #include "runtime/os.hpp" 31 #include "vm_version_s390.hpp" 32 33 // Note that the compare-and-swap instructions on System z perform 34 // a serialization function before the storage operand is fetched 35 // and again after the operation is completed. 36 // 37 // Used constraint modifiers: 38 // = write-only access: Value on entry to inline-assembler code irrelevant. 39 // + read/write access: Value on entry is used; on exit value is changed. 40 // read-only access: Value on entry is used and never changed. 41 // & early-clobber access: Might be modified before all read-only operands 42 // have been used. 43 // a address register operand (not GR0). 44 // d general register operand (including GR0) 45 // Q memory operand w/o index register. 46 // 0..9 operand reference (by operand position). 47 // Used for operands that fill multiple roles. One example would be a 48 // write-only operand receiving its initial value from a read-only operand. 49 // Refer to cmpxchg(..) operand #0 and variable cmp_val for a real-life example. 50 // 51 52 // On System z, all store operations are atomic if the address where the data is stored into 53 // is an integer multiple of the data length. Furthermore, all stores are ordered: 54 // a store which occurs conceptually before another store becomes visible to other CPUs 55 // before the other store becomes visible. 56 inline void Atomic::store (jbyte store_value, jbyte* dest) { *dest = store_value; } 57 inline void Atomic::store (jshort store_value, jshort* dest) { *dest = store_value; } 58 inline void Atomic::store (jint store_value, jint* dest) { *dest = store_value; } 59 inline void Atomic::store (jlong store_value, jlong* dest) { *dest = store_value; } 60 inline void Atomic::store_ptr(intptr_t store_value, intptr_t* dest) { *dest = store_value; } 61 inline void Atomic::store_ptr(void* store_value, void* dest) { *(void**)dest = store_value; } 62 63 inline void Atomic::store (jbyte store_value, volatile jbyte* dest) { *dest = store_value; } 64 inline void Atomic::store (jshort store_value, volatile jshort* dest) { *dest = store_value; } 65 inline void Atomic::store (jint store_value, volatile jint* dest) { *dest = store_value; } 66 inline void Atomic::store (jlong store_value, volatile jlong* dest) { *dest = store_value; } 67 inline void Atomic::store_ptr(intptr_t store_value, volatile intptr_t* dest) { *dest = store_value; } 68 inline void Atomic::store_ptr(void* store_value, volatile void* dest) { *(void* volatile *)dest = store_value; } 69 70 71 //------------ 72 // Atomic::add 73 //------------ 74 // These methods force the value in memory to be augmented by the passed increment. 75 // Both, memory value and increment, are treated as 32bit signed binary integers. 76 // No overflow exceptions are recognized, and the condition code does not hold 77 // information about the value in memory. 78 // 79 // The value in memory is updated by using a compare-and-swap instruction. The 80 // instruction is retried as often as required. 81 // 82 // The return value of the method is the value that was successfully stored. At the 83 // time the caller receives back control, the value in memory may have changed already. 84 85 inline jint Atomic::add(jint inc, volatile jint*dest) { 86 unsigned int old, upd; 87 88 if (VM_Version::has_LoadAndALUAtomicV1()) { 89 __asm__ __volatile__ ( 90 " LGFR 0,%[inc] \n\t" // save increment 91 " LA 3,%[mem] \n\t" // force data address into ARG2 92 // " LAA %[upd],%[inc],%[mem] \n\t" // increment and get old value 93 // " LAA 2,0,0(3) \n\t" // actually coded instruction 94 " .byte 0xeb \n\t" // LAA main opcode 95 " .byte 0x20 \n\t" // R1,R3 96 " .byte 0x30 \n\t" // R2,disp1 97 " .byte 0x00 \n\t" // disp2,disp3 98 " .byte 0x00 \n\t" // disp4,disp5 99 " .byte 0xf8 \n\t" // LAA minor opcode 100 " AR 2,0 \n\t" // calc new value in register 101 " LR %[upd],2 \n\t" // move to result register 102 //---< outputs >--- 103 : [upd] "=&d" (upd) // write-only, updated counter value 104 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 105 //---< inputs >--- 106 : [inc] "a" (inc) // read-only. 107 //---< clobbered >--- 108 : "cc", "r0", "r2", "r3" 109 ); 110 } else { 111 __asm__ __volatile__ ( 112 " LLGF %[old],%[mem] \n\t" // get old value 113 "0: LA %[upd],0(%[inc],%[old]) \n\t" // calc result 114 " CS %[old],%[upd],%[mem] \n\t" // try to xchg res with mem 115 " JNE 0b \n\t" // no success? -> retry 116 //---< outputs >--- 117 : [old] "=&a" (old) // write-only, old counter value 118 , [upd] "=&d" (upd) // write-only, updated counter value 119 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 120 //---< inputs >--- 121 : [inc] "a" (inc) // read-only. 122 //---< clobbered >--- 123 : "cc" 124 ); 125 } 126 127 return (jint)upd; 128 } 129 130 131 inline intptr_t Atomic::add_ptr(intptr_t inc, volatile intptr_t* dest) { 132 unsigned long old, upd; 133 134 if (VM_Version::has_LoadAndALUAtomicV1()) { 135 __asm__ __volatile__ ( 136 " LGR 0,%[inc] \n\t" // save increment 137 " LA 3,%[mem] \n\t" // force data address into ARG2 138 // " LAAG %[upd],%[inc],%[mem] \n\t" // increment and get old value 139 // " LAAG 2,0,0(3) \n\t" // actually coded instruction 140 " .byte 0xeb \n\t" // LAA main opcode 141 " .byte 0x20 \n\t" // R1,R3 142 " .byte 0x30 \n\t" // R2,disp1 143 " .byte 0x00 \n\t" // disp2,disp3 144 " .byte 0x00 \n\t" // disp4,disp5 145 " .byte 0xe8 \n\t" // LAA minor opcode 146 " AGR 2,0 \n\t" // calc new value in register 147 " LGR %[upd],2 \n\t" // move to result register 148 //---< outputs >--- 149 : [upd] "=&d" (upd) // write-only, updated counter value 150 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 151 //---< inputs >--- 152 : [inc] "a" (inc) // read-only. 153 //---< clobbered >--- 154 : "cc", "r0", "r2", "r3" 155 ); 156 } else { 157 __asm__ __volatile__ ( 158 " LG %[old],%[mem] \n\t" // get old value 159 "0: LA %[upd],0(%[inc],%[old]) \n\t" // calc result 160 " CSG %[old],%[upd],%[mem] \n\t" // try to xchg res with mem 161 " JNE 0b \n\t" // no success? -> retry 162 //---< outputs >--- 163 : [old] "=&a" (old) // write-only, old counter value 164 , [upd] "=&d" (upd) // write-only, updated counter value 165 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 166 //---< inputs >--- 167 : [inc] "a" (inc) // read-only. 168 //---< clobbered >--- 169 : "cc" 170 ); 171 } 172 173 return (intptr_t)upd; 174 } 175 176 inline void* Atomic::add_ptr(intptr_t add_value, volatile void* dest) { 177 return (void*)add_ptr(add_value, (volatile intptr_t*)dest); 178 } 179 180 181 //------------ 182 // Atomic::inc 183 //------------ 184 // These methods force the value in memory to be incremented (augmented by 1). 185 // Both, memory value and increment, are treated as 32bit signed binary integers. 186 // No overflow exceptions are recognized, and the condition code does not hold 187 // information about the value in memory. 188 // 189 // The value in memory is updated by using a compare-and-swap instruction. The 190 // instruction is retried as often as required. 191 192 inline void Atomic::inc(volatile jint* dest) { 193 unsigned int old, upd; 194 195 if (VM_Version::has_LoadAndALUAtomicV1()) { 196 // tty->print_cr("Atomic::inc called... dest @%p", dest); 197 __asm__ __volatile__ ( 198 " LGHI 2,1 \n\t" // load increment 199 " LA 3,%[mem] \n\t" // force data address into ARG2 200 // " LAA %[upd],%[inc],%[mem] \n\t" // increment and get old value 201 // " LAA 2,2,0(3) \n\t" // actually coded instruction 202 " .byte 0xeb \n\t" // LAA main opcode 203 " .byte 0x22 \n\t" // R1,R3 204 " .byte 0x30 \n\t" // R2,disp1 205 " .byte 0x00 \n\t" // disp2,disp3 206 " .byte 0x00 \n\t" // disp4,disp5 207 " .byte 0xf8 \n\t" // LAA minor opcode 208 " AGHI 2,1 \n\t" // calc new value in register 209 " LR %[upd],2 \n\t" // move to result register 210 //---< outputs >--- 211 : [upd] "=&d" (upd) // write-only, updated counter value 212 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 213 //---< inputs >--- 214 : 215 // : [inc] "a" (inc) // read-only. 216 //---< clobbered >--- 217 : "cc", "r2", "r3" 218 ); 219 } else { 220 __asm__ __volatile__ ( 221 " LLGF %[old],%[mem] \n\t" // get old value 222 "0: LA %[upd],1(,%[old]) \n\t" // calc result 223 " CS %[old],%[upd],%[mem] \n\t" // try to xchg res with mem 224 " JNE 0b \n\t" // no success? -> retry 225 //---< outputs >--- 226 : [old] "=&a" (old) // write-only, old counter value 227 , [upd] "=&d" (upd) // write-only, updated counter value 228 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 229 //---< inputs >--- 230 : 231 //---< clobbered >--- 232 : "cc" 233 ); 234 } 235 } 236 237 inline void Atomic::inc_ptr(volatile intptr_t* dest) { 238 unsigned long old, upd; 239 240 if (VM_Version::has_LoadAndALUAtomicV1()) { 241 __asm__ __volatile__ ( 242 " LGHI 2,1 \n\t" // load increment 243 " LA 3,%[mem] \n\t" // force data address into ARG2 244 // " LAAG %[upd],%[inc],%[mem] \n\t" // increment and get old value 245 // " LAAG 2,2,0(3) \n\t" // actually coded instruction 246 " .byte 0xeb \n\t" // LAA main opcode 247 " .byte 0x22 \n\t" // R1,R3 248 " .byte 0x30 \n\t" // R2,disp1 249 " .byte 0x00 \n\t" // disp2,disp3 250 " .byte 0x00 \n\t" // disp4,disp5 251 " .byte 0xe8 \n\t" // LAA minor opcode 252 " AGHI 2,1 \n\t" // calc new value in register 253 " LR %[upd],2 \n\t" // move to result register 254 //---< outputs >--- 255 : [upd] "=&d" (upd) // write-only, updated counter value 256 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 257 //---< inputs >--- 258 : 259 // : [inc] "a" (inc) // read-only. 260 //---< clobbered >--- 261 : "cc", "r2", "r3" 262 ); 263 } else { 264 __asm__ __volatile__ ( 265 " LG %[old],%[mem] \n\t" // get old value 266 "0: LA %[upd],1(,%[old]) \n\t" // calc result 267 " CSG %[old],%[upd],%[mem] \n\t" // try to xchg res with mem 268 " JNE 0b \n\t" // no success? -> retry 269 //---< outputs >--- 270 : [old] "=&a" (old) // write-only, old counter value 271 , [upd] "=&d" (upd) // write-only, updated counter value 272 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 273 //---< inputs >--- 274 : 275 //---< clobbered >--- 276 : "cc" 277 ); 278 } 279 } 280 281 inline void Atomic::inc_ptr(volatile void* dest) { 282 inc_ptr((volatile intptr_t*)dest); 283 } 284 285 //------------ 286 // Atomic::dec 287 //------------ 288 // These methods force the value in memory to be decremented (augmented by -1). 289 // Both, memory value and decrement, are treated as 32bit signed binary integers. 290 // No overflow exceptions are recognized, and the condition code does not hold 291 // information about the value in memory. 292 // 293 // The value in memory is updated by using a compare-and-swap instruction. The 294 // instruction is retried as often as required. 295 296 inline void Atomic::dec(volatile jint* dest) { 297 unsigned int old, upd; 298 299 if (VM_Version::has_LoadAndALUAtomicV1()) { 300 __asm__ __volatile__ ( 301 " LGHI 2,-1 \n\t" // load increment 302 " LA 3,%[mem] \n\t" // force data address into ARG2 303 // " LAA %[upd],%[inc],%[mem] \n\t" // increment and get old value 304 // " LAA 2,2,0(3) \n\t" // actually coded instruction 305 " .byte 0xeb \n\t" // LAA main opcode 306 " .byte 0x22 \n\t" // R1,R3 307 " .byte 0x30 \n\t" // R2,disp1 308 " .byte 0x00 \n\t" // disp2,disp3 309 " .byte 0x00 \n\t" // disp4,disp5 310 " .byte 0xf8 \n\t" // LAA minor opcode 311 " AGHI 2,-1 \n\t" // calc new value in register 312 " LR %[upd],2 \n\t" // move to result register 313 //---< outputs >--- 314 : [upd] "=&d" (upd) // write-only, updated counter value 315 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 316 //---< inputs >--- 317 : 318 // : [inc] "a" (inc) // read-only. 319 //---< clobbered >--- 320 : "cc", "r2", "r3" 321 ); 322 } else { 323 __asm__ __volatile__ ( 324 " LLGF %[old],%[mem] \n\t" // get old value 325 // LAY not supported by inline assembler 326 // "0: LAY %[upd],-1(,%[old]) \n\t" // calc result 327 "0: LR %[upd],%[old] \n\t" // calc result 328 " AHI %[upd],-1 \n\t" 329 " CS %[old],%[upd],%[mem] \n\t" // try to xchg res with mem 330 " JNE 0b \n\t" // no success? -> retry 331 //---< outputs >--- 332 : [old] "=&a" (old) // write-only, old counter value 333 , [upd] "=&d" (upd) // write-only, updated counter value 334 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 335 //---< inputs >--- 336 : 337 //---< clobbered >--- 338 : "cc" 339 ); 340 } 341 } 342 343 inline void Atomic::dec_ptr(volatile intptr_t* dest) { 344 unsigned long old, upd; 345 346 if (VM_Version::has_LoadAndALUAtomicV1()) { 347 __asm__ __volatile__ ( 348 " LGHI 2,-1 \n\t" // load increment 349 " LA 3,%[mem] \n\t" // force data address into ARG2 350 // " LAAG %[upd],%[inc],%[mem] \n\t" // increment and get old value 351 // " LAAG 2,2,0(3) \n\t" // actually coded instruction 352 " .byte 0xeb \n\t" // LAA main opcode 353 " .byte 0x22 \n\t" // R1,R3 354 " .byte 0x30 \n\t" // R2,disp1 355 " .byte 0x00 \n\t" // disp2,disp3 356 " .byte 0x00 \n\t" // disp4,disp5 357 " .byte 0xe8 \n\t" // LAA minor opcode 358 " AGHI 2,-1 \n\t" // calc new value in register 359 " LR %[upd],2 \n\t" // move to result register 360 //---< outputs >--- 361 : [upd] "=&d" (upd) // write-only, updated counter value 362 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 363 //---< inputs >--- 364 : 365 // : [inc] "a" (inc) // read-only. 366 //---< clobbered >--- 367 : "cc", "r2", "r3" 368 ); 369 } else { 370 __asm__ __volatile__ ( 371 " LG %[old],%[mem] \n\t" // get old value 372 // LAY not supported by inline assembler 373 // "0: LAY %[upd],-1(,%[old]) \n\t" // calc result 374 "0: LGR %[upd],%[old] \n\t" // calc result 375 " AGHI %[upd],-1 \n\t" 376 " CSG %[old],%[upd],%[mem] \n\t" // try to xchg res with mem 377 " JNE 0b \n\t" // no success? -> retry 378 //---< outputs >--- 379 : [old] "=&a" (old) // write-only, old counter value 380 , [upd] "=&d" (upd) // write-only, updated counter value 381 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 382 //---< inputs >--- 383 : 384 //---< clobbered >--- 385 : "cc" 386 ); 387 } 388 } 389 390 inline void Atomic::dec_ptr(volatile void* dest) { 391 dec_ptr((volatile intptr_t*)dest); 392 } 393 394 //------------- 395 // Atomic::xchg 396 //------------- 397 // These methods force the value in memory to be replaced by the new value passed 398 // in as argument. 399 // 400 // The value in memory is replaced by using a compare-and-swap instruction. The 401 // instruction is retried as often as required. This makes sure that the new 402 // value can be seen, at least for a very short period of time, by other CPUs. 403 // 404 // If we would use a normal "load(old value) store(new value)" sequence, 405 // the new value could be lost unnoticed, due to a store(new value) from 406 // another thread. 407 // 408 // The return value is the (unchanged) value from memory as it was when the 409 // replacement succeeded. 410 inline jint Atomic::xchg (jint xchg_val, volatile jint* dest) { 411 unsigned int old; 412 413 __asm__ __volatile__ ( 414 " LLGF %[old],%[mem] \n\t" // get old value 415 "0: CS %[old],%[upd],%[mem] \n\t" // try to xchg upd with mem 416 " JNE 0b \n\t" // no success? -> retry 417 //---< outputs >--- 418 : [old] "=&d" (old) // write-only, prev value irrelevant 419 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 420 //---< inputs >--- 421 : [upd] "d" (xchg_val) // read-only, value to be written to memory 422 //---< clobbered >--- 423 : "cc" 424 ); 425 426 return (jint)old; 427 } 428 429 inline intptr_t Atomic::xchg_ptr(intptr_t xchg_val, volatile intptr_t* dest) { 430 unsigned long old; 431 432 __asm__ __volatile__ ( 433 " LG %[old],%[mem] \n\t" // get old value 434 "0: CSG %[old],%[upd],%[mem] \n\t" // try to xchg upd with mem 435 " JNE 0b \n\t" // no success? -> retry 436 //---< outputs >--- 437 : [old] "=&d" (old) // write-only, init from memory 438 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically 439 //---< inputs >--- 440 : [upd] "d" (xchg_val) // read-only, value to be written to memory 441 //---< clobbered >--- 442 : "cc" 443 ); 444 445 return (intptr_t)old; 446 } 447 448 inline void *Atomic::xchg_ptr(void *exchange_value, volatile void *dest) { 449 return (void*)xchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest); 450 } 451 452 //---------------- 453 // Atomic::cmpxchg 454 //---------------- 455 // These methods compare the value in memory with a given compare value. 456 // If both values compare equal, the value in memory is replaced with 457 // the exchange value. 458 // 459 // The value in memory is compared and replaced by using a compare-and-swap 460 // instruction. The instruction is NOT retried (one shot only). 461 // 462 // The return value is the (unchanged) value from memory as it was when the 463 // compare-and-swap instruction completed. A successful exchange operation 464 // is indicated by (return value == compare_value). If unsuccessful, a new 465 // exchange value can be calculated based on the return value which is the 466 // latest contents of the memory location. 467 // 468 // Inspecting the return value is the only way for the caller to determine 469 // if the compare-and-swap instruction was successful: 470 // - If return value and compare value compare equal, the compare-and-swap 471 // instruction was successful and the value in memory was replaced by the 472 // exchange value. 473 // - If return value and compare value compare unequal, the compare-and-swap 474 // instruction was not successful. The value in memory was left unchanged. 475 // 476 // The s390 processors always fence before and after the csg instructions. 477 // Thus we ignore the memory ordering argument. The docu says: "A serialization 478 // function is performed before the operand is fetched and again after the 479 // operation is completed." 480 481 jint Atomic::cmpxchg(jint xchg_val, volatile jint* dest, jint cmp_val, cmpxchg_memory_order unused) { 482 unsigned long old; 483 484 __asm__ __volatile__ ( 485 " CS %[old],%[upd],%[mem] \n\t" // Try to xchg upd with mem. 486 // outputs 487 : [old] "=&d" (old) // Write-only, prev value irrelevant. 488 , [mem] "+Q" (*dest) // Read/write, memory to be updated atomically. 489 // inputs 490 : [upd] "d" (xchg_val) 491 , "0" (cmp_val) // Read-only, initial value for [old] (operand #0). 492 // clobbered 493 : "cc" 494 ); 495 496 return (jint)old; 497 } 498 499 jlong Atomic::cmpxchg(jlong xchg_val, volatile jlong* dest, jlong cmp_val, cmpxchg_memory_order unused) { 500 unsigned long old; 501 502 __asm__ __volatile__ ( 503 " CSG %[old],%[upd],%[mem] \n\t" // Try to xchg upd with mem. 504 // outputs 505 : [old] "=&d" (old) // Write-only, prev value irrelevant. 506 , [mem] "+Q" (*dest) // Read/write, memory to be updated atomically. 507 // inputs 508 : [upd] "d" (xchg_val) 509 , "0" (cmp_val) // Read-only, initial value for [old] (operand #0). 510 // clobbered 511 : "cc" 512 ); 513 514 return (jlong)old; 515 } 516 517 void* Atomic::cmpxchg_ptr(void *xchg_val, volatile void* dest, void* cmp_val, cmpxchg_memory_order unused) { 518 return (void*)cmpxchg((jlong)xchg_val, (volatile jlong*)dest, (jlong)cmp_val, unused); 519 } 520 521 intptr_t Atomic::cmpxchg_ptr(intptr_t xchg_val, volatile intptr_t* dest, intptr_t cmp_val, cmpxchg_memory_order unused) { 522 return (intptr_t)cmpxchg((jlong)xchg_val, (volatile jlong*)dest, (jlong)cmp_val, unused); 523 } 524 525 inline jlong Atomic::load(const volatile jlong* src) { return *src; } 526 527 #endif // OS_CPU_LINUX_S390_VM_ATOMIC_LINUX_S390_INLINE_HPP