1 /* 2 * Copyright (c) 2008, 2017, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #ifndef OS_CPU_LINUX_ARM_VM_ATOMIC_LINUX_ARM_HPP 26 #define OS_CPU_LINUX_ARM_VM_ATOMIC_LINUX_ARM_HPP 27 28 #include "runtime/os.hpp" 29 #include "vm_version_arm.hpp" 30 31 // Implementation of class atomic 32 33 /* 34 * Atomic long operations on 32-bit ARM 35 * ARM v7 supports LDREXD/STREXD synchronization instructions so no problem. 36 * ARM < v7 does not have explicit 64 atomic load/store capability. 37 * However, gcc emits LDRD/STRD instructions on v5te and LDM/STM on v5t 38 * when loading/storing 64 bits. 39 * For non-MP machines (which is all we support for ARM < v7) 40 * under current Linux distros these instructions appear atomic. 41 * See section A3.5.3 of ARM Architecture Reference Manual for ARM v7. 42 * Also, for cmpxchg64, if ARM < v7 we check for cmpxchg64 support in the 43 * Linux kernel using _kuser_helper_version. See entry-armv.S in the Linux 44 * kernel source or kernel_user_helpers.txt in Linux Doc. 45 */ 46 47 template <> 48 inline void GeneralizedAtomic::specialized_store<int64_t>(int64_t value, volatile int64_t* dest) { 49 assert(((intx)dest & (sizeof(int64_t)-1)) == 0, "Atomic 64 bit store mis-aligned"); 50 #ifdef AARCH64 51 *dest = value; 52 #else 53 (*os::atomic_store_long_func)(value, dest); 54 #endif 55 } 56 57 template <> 58 inline int64_t GeneralizedAtomic::specialized_load<int64_t>(const volatile int64_t* src) { 59 assert(((intx)src & (sizeof(int64_t)-1)) == 0, "Atomic 64 bit load mis-aligned"); 60 #ifdef AARCH64 61 return *src; 62 #else 63 return (*os::atomic_load_long_func)(src); 64 #endif 65 } 66 67 // As per atomic.hpp all read-modify-write operations have to provide two-way 68 // barriers semantics. For AARCH64 we are using load-acquire-with-reservation and 69 // store-release-with-reservation. While load-acquire combined with store-release 70 // do not generally form two-way barriers, their use with reservations does - the 71 // ARMv8 architecture manual Section F "Barrier Litmus Tests" indicates they 72 // provide sequentially consistent semantics. All we need to add is an explicit 73 // barrier in the failure path of the cmpxchg operations (as these don't execute 74 // the store) - arguably this may be overly cautious as there is a very low 75 // likelihood that the hardware would pull loads/stores into the region guarded 76 // by the reservation. 77 // 78 // For ARMv7 we add explicit barriers in the stubs. 79 80 template <> 81 inline int32_t GeneralizedAtomic::specialized_add<int32_t>(int32_t add_value, volatile int32_t* dest) { 82 #ifdef AARCH64 83 int32_t val; 84 int tmp; 85 __asm__ volatile( 86 "1:\n\t" 87 " ldaxr %w[val], [%[dest]]\n\t" 88 " add %w[val], %w[val], %w[add_val]\n\t" 89 " stlxr %w[tmp], %w[val], [%[dest]]\n\t" 90 " cbnz %w[tmp], 1b\n\t" 91 : [val] "=&r" (val), [tmp] "=&r" (tmp) 92 : [add_val] "r" (add_value), [dest] "r" (dest) 93 : "memory"); 94 return val; 95 #else 96 return (*os::atomic_add_func)(add_value, dest); 97 #endif 98 } 99 100 template <> 101 inline int32_t GeneralizedAtomic::specialized_xchg<int32_t>(int32_t exchange_value, volatile int32_t* dest) { 102 #ifdef AARCH64 103 int32_t old_val; 104 int tmp; 105 __asm__ volatile( 106 "1:\n\t" 107 " ldaxr %w[old_val], [%[dest]]\n\t" 108 " stlxr %w[tmp], %w[new_val], [%[dest]]\n\t" 109 " cbnz %w[tmp], 1b\n\t" 110 : [old_val] "=&r" (old_val), [tmp] "=&r" (tmp) 111 : [new_val] "r" (exchange_value), [dest] "r" (dest) 112 : "memory"); 113 return old_val; 114 #else 115 return (*os::atomic_xchg_func)(exchange_value, dest); 116 #endif 117 } 118 119 // The memory_order parameter is ignored - we always provide the strongest/most-conservative ordering 120 121 template <> 122 inline int32_t GeneralizedAtomic::specialized_cmpxchg<int32_t>(int32_t exchange_value, volatile int32_t* dest, int32_t compare_value, cmpxchg_memory_order order) { 123 #ifdef AARCH64 124 int32_t rv; 125 int tmp; 126 __asm__ volatile( 127 "1:\n\t" 128 " ldaxr %w[rv], [%[dest]]\n\t" 129 " cmp %w[rv], %w[cv]\n\t" 130 " b.ne 2f\n\t" 131 " stlxr %w[tmp], %w[ev], [%[dest]]\n\t" 132 " cbnz %w[tmp], 1b\n\t" 133 " b 3f\n\t" 134 "2:\n\t" 135 " dmb sy\n\t" 136 "3:\n\t" 137 : [rv] "=&r" (rv), [tmp] "=&r" (tmp) 138 : [ev] "r" (exchange_value), [dest] "r" (dest), [cv] "r" (compare_value) 139 : "memory"); 140 return rv; 141 #else 142 // Warning: Arguments are swapped to avoid moving them for kernel call 143 return (*os::atomic_cmpxchg_func)(compare_value, exchange_value, dest); 144 #endif 145 } 146 147 template <> 148 inline int64_t GeneralizedAtomic::specialized_cmpxchg<int64_t>(int64_t exchange_value, volatile int64_t* dest, int64_t compare_value, cmpxchg_memory_order order) { 149 #ifdef AARCH64 150 int64_t rv; 151 int tmp; 152 __asm__ volatile( 153 "1:\n\t" 154 " ldaxr %[rv], [%[dest]]\n\t" 155 " cmp %[rv], %[cv]\n\t" 156 " b.ne 2f\n\t" 157 " stlxr %w[tmp], %[ev], [%[dest]]\n\t" 158 " cbnz %w[tmp], 1b\n\t" 159 " b 3f\n\t" 160 "2:\n\t" 161 " dmb sy\n\t" 162 "3:\n\t" 163 : [rv] "=&r" (rv), [tmp] "=&r" (tmp) 164 : [ev] "r" (exchange_value), [dest] "r" (dest), [cv] "r" (compare_value) 165 : "memory"); 166 return rv; 167 #else 168 assert(VM_Version::supports_cx8(), "64 bit atomic compare and exchange not supported on this architecture!"); 169 return (*os::atomic_cmpxchg_long_func)(compare_value, exchange_value, dest); 170 #endif 171 } 172 173 #ifdef AARCH64 174 template <> 175 inline int64_t GeneralizedAtomic::specialized_add<int64_t>(int64_t add_value, volatile int64_t* dest) { 176 int64_t val; 177 int tmp; 178 __asm__ volatile( 179 "1:\n\t" 180 " ldaxr %[val], [%[dest]]\n\t" 181 " add %[val], %[val], %[add_val]\n\t" 182 " stlxr %w[tmp], %[val], [%[dest]]\n\t" 183 " cbnz %w[tmp], 1b\n\t" 184 : [val] "=&r" (val), [tmp] "=&r" (tmp) 185 : [add_val] "r" (add_value), [dest] "r" (dest) 186 : "memory"); 187 return val; 188 } 189 190 template <> 191 inline int64_t GeneralizedAtomic::specialized_xchg<int64_t>(int64_t exchange_value, volatile int64_t* dest) { 192 int64_t old_val; 193 int tmp; 194 __asm__ volatile( 195 "1:\n\t" 196 " ldaxr %[old_val], [%[dest]]\n\t" 197 " stlxr %w[tmp], %[new_val], [%[dest]]\n\t" 198 " cbnz %w[tmp], 1b\n\t" 199 : [old_val] "=&r" (old_val), [tmp] "=&r" (tmp) 200 : [new_val] "r" (exchange_value), [dest] "r" (dest) 201 : "memory"); 202 return old_val; 203 } 204 205 #endif 206 207 #endif // OS_CPU_LINUX_ARM_VM_ATOMIC_LINUX_ARM_HPP