# HG changeset patch # User mdoerr # Date 1462292919 -7200 # Node ID 4fc4858d1a87bd5cec7db314ce8aeee6f75a94a0 # Parent fc5b64f701991426a1e35a75878be87aacf33ddd 8155949: Support relaxed semantics in cmpxchg Reviewed-by: Contributed-by: HORII@jp.ibm.com, mdoerr diff --git a/src/os_cpu/aix_ppc/vm/atomic_aix_ppc.inline.hpp b/src/os_cpu/aix_ppc/vm/atomic_aix_ppc.inline.hpp --- a/src/os_cpu/aix_ppc/vm/atomic_aix_ppc.inline.hpp +++ b/src/os_cpu/aix_ppc/vm/atomic_aix_ppc.inline.hpp @@ -291,8 +291,26 @@ return (void*)xchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest); } +inline void cmpxchg_pre_membar(cmpxchg_memory_order order) { + if (order != memory_order_relaxed) { + __asm__ __volatile__ ( + /* fence */ + strasm_sync + ); + } +} + +inline void cmpxchg_post_membar(cmpxchg_memory_order order) { + if (order != memory_order_relaxed) { + __asm__ __volatile__ ( + /* fence */ + strasm_sync + ); + } +} + #define VM_HAS_SPECIALIZED_CMPXCHG_BYTE -inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) { +inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, cmpxchg_memory_order order) { // Note that cmpxchg guarantees a two-way memory barrier across // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire' @@ -312,9 +330,9 @@ unsigned int old_value, value32; + cmpxchg_pre_membar(order); + __asm__ __volatile__ ( - /* fence */ - strasm_sync /* simple guard */ " lbz %[old_value], 0(%[dest]) \n" " cmpw %[masked_compare_val], %[old_value] \n" @@ -331,8 +349,6 @@ " xor %[value32], %[xor_value], %[value32] \n" " stwcx. %[value32], 0, %[dest_base] \n" " bne- 1b \n" - /* acquire */ - strasm_sync /* exit */ "2: \n" /* out */ @@ -353,10 +369,12 @@ "memory" ); + cmpxchg_post_membar(order); + return (jbyte)(unsigned char)old_value; } -inline jint Atomic::cmpxchg(jint exchange_value, volatile jint* dest, jint compare_value) { +inline jint Atomic::cmpxchg(jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) { // Note that cmpxchg guarantees a two-way memory barrier across // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire' @@ -365,9 +383,9 @@ unsigned int old_value; const uint64_t zero = 0; + cmpxchg_pre_membar(order); + __asm__ __volatile__ ( - /* fence */ - strasm_sync /* simple guard */ " lwz %[old_value], 0(%[dest]) \n" " cmpw %[compare_value], %[old_value] \n" @@ -379,8 +397,6 @@ " bne- 2f \n" " stwcx. %[exchange_value], %[dest], %[zero] \n" " bne- 1b \n" - /* acquire */ - strasm_sync /* exit */ "2: \n" /* out */ @@ -397,10 +413,12 @@ "memory" ); + cmpxchg_post_membar(order); + return (jint) old_value; } -inline jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong* dest, jlong compare_value) { +inline jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) { // Note that cmpxchg guarantees a two-way memory barrier across // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire' @@ -409,9 +427,9 @@ long old_value; const uint64_t zero = 0; + cmpxchg_pre_membar(order); + __asm__ __volatile__ ( - /* fence */ - strasm_sync /* simple guard */ " ld %[old_value], 0(%[dest]) \n" " cmpd %[compare_value], %[old_value] \n" @@ -423,8 +441,6 @@ " bne- 2f \n" " stdcx. %[exchange_value], %[dest], %[zero] \n" " bne- 1b \n" - /* acquire */ - strasm_sync /* exit */ "2: \n" /* out */ @@ -441,15 +457,17 @@ "memory" ); + cmpxchg_post_membar(order); + return (jlong) old_value; } -inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) { - return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value); +inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) { + return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order); } -inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) { - return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value); +inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) { + return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order); } #undef strasm_sync diff --git a/src/os_cpu/bsd_x86/vm/atomic_bsd_x86.inline.hpp b/src/os_cpu/bsd_x86/vm/atomic_bsd_x86.inline.hpp --- a/src/os_cpu/bsd_x86/vm/atomic_bsd_x86.inline.hpp +++ b/src/os_cpu/bsd_x86/vm/atomic_bsd_x86.inline.hpp @@ -89,7 +89,7 @@ } #define VM_HAS_SPECIALIZED_CMPXCHG_BYTE -inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) { +inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, cmpxchg_memory_order order) { int mp = os::is_MP(); __asm__ volatile (LOCK_IF_MP(%4) "cmpxchgb %1,(%3)" : "=a" (exchange_value) @@ -98,7 +98,7 @@ return exchange_value; } -inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) { +inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) { int mp = os::is_MP(); __asm__ volatile (LOCK_IF_MP(%4) "cmpxchgl %1,(%3)" : "=a" (exchange_value) @@ -149,7 +149,7 @@ return exchange_value; } -inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) { +inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) { bool mp = os::is_MP(); __asm__ __volatile__ (LOCK_IF_MP(%4) "cmpxchgq %1,(%3)" : "=a" (exchange_value) @@ -158,12 +158,12 @@ return exchange_value; } -inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) { - return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value); +inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) { + return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order); } -inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) { - return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value); +inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) { + return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order); } inline jlong Atomic::load(volatile jlong* src) { return *src; } @@ -197,16 +197,16 @@ void _Atomic_move_long(volatile jlong* src, volatile jlong* dst); } -inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) { +inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) { return _Atomic_cmpxchg_long(exchange_value, dest, compare_value, os::is_MP()); } -inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) { - return (intptr_t)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value); +inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) { + return (intptr_t)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value, order); } -inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) { - return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value); +inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) { + return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value, order); } inline jlong Atomic::load(volatile jlong* src) { diff --git a/src/os_cpu/bsd_zero/vm/atomic_bsd_zero.inline.hpp b/src/os_cpu/bsd_zero/vm/atomic_bsd_zero.inline.hpp --- a/src/os_cpu/bsd_zero/vm/atomic_bsd_zero.inline.hpp +++ b/src/os_cpu/bsd_zero/vm/atomic_bsd_zero.inline.hpp @@ -270,7 +270,7 @@ inline jint Atomic::cmpxchg(jint exchange_value, volatile jint* dest, - jint compare_value) { + jint compare_value, cmpxchg_memory_order order) { #ifdef ARM return arm_compare_and_swap(dest, compare_value, exchange_value); #else @@ -284,14 +284,14 @@ inline jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong* dest, - jlong compare_value) { + jlong compare_value, cmpxchg_memory_order order) { return __sync_val_compare_and_swap(dest, compare_value, exchange_value); } inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, - intptr_t compare_value) { + intptr_t compare_value, cmpxchg_memory_order order) { #ifdef ARM return arm_compare_and_swap(dest, compare_value, exchange_value); #else @@ -305,7 +305,7 @@ inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, - void* compare_value) { + void* compare_value, cmpxchg_memory_order order) { return (void *) cmpxchg_ptr((intptr_t) exchange_value, (volatile intptr_t*) dest, diff --git a/src/os_cpu/linux_aarch64/vm/atomic_linux_aarch64.inline.hpp b/src/os_cpu/linux_aarch64/vm/atomic_linux_aarch64.inline.hpp --- a/src/os_cpu/linux_aarch64/vm/atomic_linux_aarch64.inline.hpp +++ b/src/os_cpu/linux_aarch64/vm/atomic_linux_aarch64.inline.hpp @@ -88,7 +88,7 @@ } -inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) +inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) { return __sync_val_compare_and_swap(dest, compare_value, exchange_value); } @@ -123,21 +123,21 @@ return res; } -inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) +inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) { return __sync_val_compare_and_swap(dest, compare_value, exchange_value); } -inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) +inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) { return __sync_val_compare_and_swap(dest, compare_value, exchange_value); } -inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) +inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) { return (void *) cmpxchg_ptr((intptr_t) exchange_value, (volatile intptr_t*) dest, - (intptr_t) compare_value); + (intptr_t) compare_value, order); } inline jlong Atomic::load(volatile jlong* src) { return *src; } diff --git a/src/os_cpu/linux_ppc/vm/atomic_linux_ppc.inline.hpp b/src/os_cpu/linux_ppc/vm/atomic_linux_ppc.inline.hpp --- a/src/os_cpu/linux_ppc/vm/atomic_linux_ppc.inline.hpp +++ b/src/os_cpu/linux_ppc/vm/atomic_linux_ppc.inline.hpp @@ -291,8 +291,26 @@ return (void*)xchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest); } +inline void cmpxchg_pre_membar(cmpxchg_memory_order order) { + if (order != memory_order_relaxed) { + __asm__ __volatile__ ( + /* fence */ + strasm_sync + ); + } +} + +inline void cmpxchg_post_membar(cmpxchg_memory_order order) { + if (order != memory_order_relaxed) { + __asm__ __volatile__ ( + /* fence */ + strasm_sync + ); + } +} + #define VM_HAS_SPECIALIZED_CMPXCHG_BYTE -inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) { +inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, cmpxchg_memory_order order) { // Note that cmpxchg guarantees a two-way memory barrier across // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire' @@ -312,9 +330,9 @@ unsigned int old_value, value32; + cmpxchg_pre_membar(order); + __asm__ __volatile__ ( - /* fence */ - strasm_sync /* simple guard */ " lbz %[old_value], 0(%[dest]) \n" " cmpw %[masked_compare_val], %[old_value] \n" @@ -331,8 +349,6 @@ " xor %[value32], %[xor_value], %[value32] \n" " stwcx. %[value32], 0, %[dest_base] \n" " bne- 1b \n" - /* acquire */ - strasm_sync /* exit */ "2: \n" /* out */ @@ -353,10 +369,12 @@ "memory" ); + cmpxchg_post_membar(order); + return (jbyte)(unsigned char)old_value; } -inline jint Atomic::cmpxchg(jint exchange_value, volatile jint* dest, jint compare_value) { +inline jint Atomic::cmpxchg(jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) { // Note that cmpxchg guarantees a two-way memory barrier across // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire' @@ -365,9 +383,9 @@ unsigned int old_value; const uint64_t zero = 0; + cmpxchg_pre_membar(order); + __asm__ __volatile__ ( - /* fence */ - strasm_sync /* simple guard */ " lwz %[old_value], 0(%[dest]) \n" " cmpw %[compare_value], %[old_value] \n" @@ -379,8 +397,6 @@ " bne- 2f \n" " stwcx. %[exchange_value], %[dest], %[zero] \n" " bne- 1b \n" - /* acquire */ - strasm_sync /* exit */ "2: \n" /* out */ @@ -397,10 +413,12 @@ "memory" ); + cmpxchg_post_membar(order); + return (jint) old_value; } -inline jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong* dest, jlong compare_value) { +inline jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) { // Note that cmpxchg guarantees a two-way memory barrier across // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire' @@ -409,9 +427,9 @@ long old_value; const uint64_t zero = 0; + cmpxchg_pre_membar(order); + __asm__ __volatile__ ( - /* fence */ - strasm_sync /* simple guard */ " ld %[old_value], 0(%[dest]) \n" " cmpd %[compare_value], %[old_value] \n" @@ -423,8 +441,6 @@ " bne- 2f \n" " stdcx. %[exchange_value], %[dest], %[zero] \n" " bne- 1b \n" - /* acquire */ - strasm_sync /* exit */ "2: \n" /* out */ @@ -441,15 +457,17 @@ "memory" ); + cmpxchg_post_membar(order); + return (jlong) old_value; } -inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) { - return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value); +inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) { + return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order); } -inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) { - return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value); +inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) { + return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order); } #undef strasm_sync diff --git a/src/os_cpu/linux_sparc/vm/atomic_linux_sparc.inline.hpp b/src/os_cpu/linux_sparc/vm/atomic_linux_sparc.inline.hpp --- a/src/os_cpu/linux_sparc/vm/atomic_linux_sparc.inline.hpp +++ b/src/os_cpu/linux_sparc/vm/atomic_linux_sparc.inline.hpp @@ -148,7 +148,7 @@ } -inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) { +inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) { jint rv; __asm__ volatile( " cas [%2], %3, %0" @@ -158,7 +158,7 @@ return rv; } -inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) { +inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) { #ifdef _LP64 jlong rv; __asm__ volatile( @@ -190,7 +190,7 @@ #endif } -inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) { +inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) { intptr_t rv; #ifdef _LP64 __asm__ volatile( @@ -208,8 +208,8 @@ return rv; } -inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) { - return (void*)cmpxchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest, (intptr_t)compare_value); +inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) { + return (void*)cmpxchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest, (intptr_t)compare_value, order); } #endif // OS_CPU_LINUX_SPARC_VM_ATOMIC_LINUX_SPARC_INLINE_HPP diff --git a/src/os_cpu/linux_x86/vm/atomic_linux_x86.inline.hpp b/src/os_cpu/linux_x86/vm/atomic_linux_x86.inline.hpp --- a/src/os_cpu/linux_x86/vm/atomic_linux_x86.inline.hpp +++ b/src/os_cpu/linux_x86/vm/atomic_linux_x86.inline.hpp @@ -89,7 +89,7 @@ } #define VM_HAS_SPECIALIZED_CMPXCHG_BYTE -inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) { +inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, cmpxchg_memory_order order) { int mp = os::is_MP(); __asm__ volatile (LOCK_IF_MP(%4) "cmpxchgb %1,(%3)" : "=a" (exchange_value) @@ -98,7 +98,7 @@ return exchange_value; } -inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) { +inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) { int mp = os::is_MP(); __asm__ volatile (LOCK_IF_MP(%4) "cmpxchgl %1,(%3)" : "=a" (exchange_value) @@ -149,7 +149,7 @@ return exchange_value; } -inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) { +inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) { bool mp = os::is_MP(); __asm__ __volatile__ (LOCK_IF_MP(%4) "cmpxchgq %1,(%3)" : "=a" (exchange_value) @@ -158,12 +158,12 @@ return exchange_value; } -inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) { - return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value); +inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) { + return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order); } -inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) { - return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value); +inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) { + return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order); } inline jlong Atomic::load(volatile jlong* src) { return *src; } @@ -197,16 +197,16 @@ void _Atomic_move_long(volatile jlong* src, volatile jlong* dst); } -inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) { +inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) { return _Atomic_cmpxchg_long(exchange_value, dest, compare_value, os::is_MP()); } -inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) { - return (intptr_t)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value); +inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) { + return (intptr_t)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value, order); } -inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) { - return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value); +inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) { + return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value, order); } inline jlong Atomic::load(volatile jlong* src) { diff --git a/src/os_cpu/linux_zero/vm/atomic_linux_zero.inline.hpp b/src/os_cpu/linux_zero/vm/atomic_linux_zero.inline.hpp --- a/src/os_cpu/linux_zero/vm/atomic_linux_zero.inline.hpp +++ b/src/os_cpu/linux_zero/vm/atomic_linux_zero.inline.hpp @@ -264,7 +264,7 @@ inline jint Atomic::cmpxchg(jint exchange_value, volatile jint* dest, - jint compare_value) { + jint compare_value, cmpxchg_memory_order order) { #ifdef ARM return arm_compare_and_swap(dest, compare_value, exchange_value); #else @@ -278,14 +278,14 @@ inline jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong* dest, - jlong compare_value) { + jlong compare_value, cmpxchg_memory_order order) { return __sync_val_compare_and_swap(dest, compare_value, exchange_value); } inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, - intptr_t compare_value) { + intptr_t compare_value, cmpxchg_memory_order order) { #ifdef ARM return arm_compare_and_swap(dest, compare_value, exchange_value); #else @@ -299,11 +299,11 @@ inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, - void* compare_value) { + void* compare_value, cmpxchg_memory_order order) { return (void *) cmpxchg_ptr((intptr_t) exchange_value, (volatile intptr_t*) dest, - (intptr_t) compare_value); + (intptr_t) compare_value, order); } inline jlong Atomic::load(volatile jlong* src) { diff --git a/src/os_cpu/solaris_sparc/vm/atomic_solaris_sparc.inline.hpp b/src/os_cpu/solaris_sparc/vm/atomic_solaris_sparc.inline.hpp --- a/src/os_cpu/solaris_sparc/vm/atomic_solaris_sparc.inline.hpp +++ b/src/os_cpu/solaris_sparc/vm/atomic_solaris_sparc.inline.hpp @@ -177,7 +177,7 @@ } -inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) { +inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) { jint rv; __asm__ volatile( " cas [%2], %3, %0" @@ -187,7 +187,7 @@ return rv; } -inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) { +inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) { #ifdef _LP64 jlong rv; __asm__ volatile( @@ -219,7 +219,7 @@ #endif //_LP64 } -inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) { +inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) { intptr_t rv; #ifdef _LP64 __asm__ volatile( @@ -237,8 +237,8 @@ return rv; } -inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) { - return (void*)cmpxchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest, (intptr_t)compare_value); +inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) { + return (void*)cmpxchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest, (intptr_t)compare_value, order); } #else // _GNU_SOURCE @@ -296,11 +296,11 @@ } -inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) { +inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) { return _Atomic_cas32(exchange_value, dest, compare_value); } -inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) { +inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) { #ifdef _LP64 // Return 64 bit value in %o0 return _Atomic_cas64((intptr_t)exchange_value, (intptr_t *)dest, (intptr_t)compare_value); @@ -310,7 +310,7 @@ #endif // _LP64 } -inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) { +inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) { #ifdef _LP64 return _Atomic_cas64(exchange_value, dest, compare_value); #else // _LP64 @@ -318,8 +318,8 @@ #endif // _LP64 } -inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) { - return (void*)cmpxchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest, (intptr_t)compare_value); +inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) { + return (void*)cmpxchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest, (intptr_t)compare_value, order); } @@ -354,20 +354,20 @@ } -inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) { +inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) { return (*os::atomic_cmpxchg_func)(exchange_value, dest, compare_value); } -inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) { +inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) { return (*os::atomic_cmpxchg_long_func)(exchange_value, dest, compare_value); } -inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) { - return (intptr_t)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value); +inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) { + return (intptr_t)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value, order); } -inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) { - return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value); +inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) { + return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value, order); } #endif // _LP64 || COMPILER2 diff --git a/src/os_cpu/solaris_x86/vm/atomic_solaris_x86.inline.hpp b/src/os_cpu/solaris_x86/vm/atomic_solaris_x86.inline.hpp --- a/src/os_cpu/solaris_x86/vm/atomic_solaris_x86.inline.hpp +++ b/src/os_cpu/solaris_x86/vm/atomic_solaris_x86.inline.hpp @@ -85,15 +85,15 @@ } #define VM_HAS_SPECIALIZED_CMPXCHG_BYTE -inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) { +inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, cmpxchg_memory_order order) { return _Atomic_cmpxchg_byte(exchange_value, dest, compare_value IS_MP_ARG()); } -inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) { +inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) { return _Atomic_cmpxchg(exchange_value, dest, compare_value IS_MP_ARG()); } -inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) { +inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) { return _Atomic_cmpxchg_long(exchange_value, dest, compare_value IS_MP_ARG()); } @@ -120,12 +120,12 @@ return (void*)_Atomic_xchg_long((jlong)exchange_value, (volatile jlong*)dest); } -inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) { - return (intptr_t)_Atomic_cmpxchg_long((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value); +inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) { + return (intptr_t)_Atomic_cmpxchg_long((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order); } -inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) { - return (void*)_Atomic_cmpxchg_long((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value); +inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) { + return (void*)_Atomic_cmpxchg_long((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order); } inline jlong Atomic::load(volatile jlong* src) { return *src; } @@ -148,12 +148,12 @@ return (void*)xchg((jint)exchange_value, (volatile jint*)dest); } -inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) { - return (intptr_t)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value); +inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) { + return (intptr_t)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value, order); } -inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) { - return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value); +inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) { + return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value, order); } extern "C" void _Atomic_move_long(volatile jlong* src, volatile jlong* dst); diff --git a/src/os_cpu/windows_x86/vm/atomic_windows_x86.inline.hpp b/src/os_cpu/windows_x86/vm/atomic_windows_x86.inline.hpp --- a/src/os_cpu/windows_x86/vm/atomic_windows_x86.inline.hpp +++ b/src/os_cpu/windows_x86/vm/atomic_windows_x86.inline.hpp @@ -119,25 +119,25 @@ return (void *)(os::atomic_xchg_ptr_func)((intptr_t)exchange_value, (volatile intptr_t*)dest); } -inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) { +inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) { return (*os::atomic_cmpxchg_func)(exchange_value, dest, compare_value); } #define VM_HAS_SPECIALIZED_CMPXCHG_BYTE -inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) { +inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, cmpxchg_memory_order order) { return (*os::atomic_cmpxchg_byte_func)(exchange_value, dest, compare_value); } -inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) { +inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) { return (*os::atomic_cmpxchg_long_func)(exchange_value, dest, compare_value); } -inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) { - return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value); +inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) { + return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order); } -inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) { - return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value); +inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) { + return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order); } inline jlong Atomic::load(volatile jlong* src) { return *src; } @@ -218,7 +218,7 @@ } #define VM_HAS_SPECIALIZED_CMPXCHG_BYTE -inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) { +inline jbyte Atomic::cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, cmpxchg_memory_order order) { // alternative for InterlockedCompareExchange int mp = os::is_MP(); __asm { @@ -230,7 +230,7 @@ } } -inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value) { +inline jint Atomic::cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) { // alternative for InterlockedCompareExchange int mp = os::is_MP(); __asm { @@ -242,7 +242,7 @@ } } -inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value) { +inline jlong Atomic::cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) { int mp = os::is_MP(); jint ex_lo = (jint)exchange_value; jint ex_hi = *( ((jint*)&exchange_value) + 1 ); @@ -263,12 +263,12 @@ } } -inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) { - return (intptr_t)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value); +inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) { + return (intptr_t)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value, order); } -inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) { - return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value); +inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) { + return (void*)cmpxchg((jint)exchange_value, (volatile jint*)dest, (jint)compare_value, order); } inline jlong Atomic::load(volatile jlong* src) { diff --git a/src/share/vm/runtime/atomic.cpp b/src/share/vm/runtime/atomic.cpp --- a/src/share/vm/runtime/atomic.cpp +++ b/src/share/vm/runtime/atomic.cpp @@ -57,10 +57,11 @@ } unsigned Atomic::cmpxchg(unsigned int exchange_value, - volatile unsigned int* dest, unsigned int compare_value) { + volatile unsigned int* dest, unsigned int compare_value, + cmpxchg_memory_order order) { assert(sizeof(unsigned int) == sizeof(jint), "more work to do"); return (unsigned int)Atomic::cmpxchg((jint)exchange_value, (volatile jint*)dest, - (jint)compare_value); + (jint)compare_value, order); } jlong Atomic::add(jlong add_value, volatile jlong* dest) { diff --git a/src/share/vm/runtime/atomic.hpp b/src/share/vm/runtime/atomic.hpp --- a/src/share/vm/runtime/atomic.hpp +++ b/src/share/vm/runtime/atomic.hpp @@ -27,6 +27,12 @@ #include "memory/allocation.hpp" +typedef enum cmpxchg_cmpxchg_memory_order { + memory_order_relaxed, + // Use value which doesn't interfere with C++2011. We need to be more conservative. + memory_order_conservative = 8 +} cmpxchg_memory_order; + class Atomic : AllStatic { private: static jbyte cmpxchg_general(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value); @@ -107,13 +113,13 @@ // *dest with exchange_value if the comparison succeeded. Returns prior // value of *dest. cmpxchg*() provide: // compare-and-exchange - inline static jbyte cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value); - inline static jint cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value); + inline static jbyte cmpxchg (jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, cmpxchg_memory_order order = memory_order_conservative); + inline static jint cmpxchg (jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order = memory_order_conservative); // See comment above about using jlong atomics on 32-bit platforms - inline static jlong cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value); - static unsigned int cmpxchg (unsigned int exchange_value, volatile unsigned int* dest, unsigned int compare_value); - inline static intptr_t cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value); - inline static void* cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value); + inline static jlong cmpxchg (jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order = memory_order_conservative); + static unsigned int cmpxchg (unsigned int exchange_value, volatile unsigned int* dest, unsigned int compare_value, cmpxchg_memory_order order = memory_order_conservative); + inline static intptr_t cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order = memory_order_conservative); + inline static void* cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order = memory_order_conservative); }; // To use Atomic::inc(jshort* dest) and Atomic::dec(jshort* dest), the address must be specially