1 /* 2 * Copyright (c) 1999, 2018, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #ifndef SHARE_VM_RUNTIME_ATOMIC_HPP 26 #define SHARE_VM_RUNTIME_ATOMIC_HPP 27 28 #include "memory/allocation.hpp" 29 #include "metaprogramming/conditional.hpp" 30 #include "metaprogramming/enableIf.hpp" 31 #include "metaprogramming/isIntegral.hpp" 32 #include "metaprogramming/isPointer.hpp" 33 #include "metaprogramming/isSame.hpp" 34 #include "metaprogramming/primitiveConversions.hpp" 35 #include "metaprogramming/removeCV.hpp" 36 #include "metaprogramming/removePointer.hpp" 37 #include "utilities/align.hpp" 38 #include "utilities/macros.hpp" 39 40 enum atomic_memory_order { 41 // The modes that align with C++11 are intended to 42 // follow the same semantics. 43 memory_order_relaxed = 0, 44 memory_order_acquire = 2, 45 memory_order_release = 3, 46 memory_order_acq_rel = 4, 47 // Strong two-way memory barrier. 48 memory_order_conservative = 8 49 }; 50 51 class Atomic : AllStatic { 52 public: 53 // Atomic operations on int64 types are not available on all 32-bit 54 // platforms. If atomic ops on int64 are defined here they must only 55 // be used from code that verifies they are available at runtime and 56 // can provide an alternative action if not - see supports_cx8() for 57 // a means to test availability. 58 59 // The memory operations that are mentioned with each of the atomic 60 // function families come from src/share/vm/runtime/orderAccess.hpp, 61 // e.g., <fence> is described in that file and is implemented by the 62 // OrderAccess::fence() function. See that file for the gory details 63 // on the Memory Access Ordering Model. 64 65 // All of the atomic operations that imply a read-modify-write action 66 // guarantee a two-way memory barrier across that operation. Historically 67 // these semantics reflect the strength of atomic operations that are 68 // provided on SPARC/X86. We assume that strength is necessary unless 69 // we can prove that a weaker form is sufficiently safe. 70 71 // Atomically store to a location 72 // The type T must be either a pointer type convertible to or equal 73 // to D, an integral/enum type equal to D, or a type equal to D that 74 // is primitive convertible using PrimitiveConversions. 75 template<typename T, typename D> 76 inline static void store(T store_value, volatile D* dest); 77 78 // Atomically load from a location 79 // The type T must be either a pointer type, an integral/enum type, 80 // or a type that is primitive convertible using PrimitiveConversions. 81 template<typename T> 82 inline static T load(const volatile T* dest); 83 84 // Atomically add to a location. Returns updated value. add*() provide: 85 // <fence> add-value-to-dest <membar StoreLoad|StoreStore> 86 87 template<typename I, typename D> 88 inline static D add(I add_value, D volatile* dest, 89 atomic_memory_order order = memory_order_conservative); 90 91 template<typename I, typename D> 92 inline static D sub(I sub_value, D volatile* dest, 93 atomic_memory_order order = memory_order_conservative); 94 95 // Atomically increment location. inc() provide: 96 // <fence> increment-dest <membar StoreLoad|StoreStore> 97 // The type D may be either a pointer type, or an integral 98 // type. If it is a pointer type, then the increment is 99 // scaled to the size of the type pointed to by the pointer. 100 template<typename D> 101 inline static void inc(D volatile* dest, 102 atomic_memory_order order = memory_order_conservative); 103 104 // Atomically decrement a location. dec() provide: 105 // <fence> decrement-dest <membar StoreLoad|StoreStore> 106 // The type D may be either a pointer type, or an integral 107 // type. If it is a pointer type, then the decrement is 108 // scaled to the size of the type pointed to by the pointer. 109 template<typename D> 110 inline static void dec(D volatile* dest, 111 atomic_memory_order order = memory_order_conservative); 112 113 // Performs atomic exchange of *dest with exchange_value. Returns old 114 // prior value of *dest. xchg*() provide: 115 // <fence> exchange-value-with-dest <membar StoreLoad|StoreStore> 116 // The type T must be either a pointer type convertible to or equal 117 // to D, an integral/enum type equal to D, or a type equal to D that 118 // is primitive convertible using PrimitiveConversions. 119 template<typename T, typename D> 120 inline static D xchg(T exchange_value, volatile D* dest, 121 atomic_memory_order order = memory_order_conservative); 122 123 // Performs atomic compare of *dest and compare_value, and exchanges 124 // *dest with exchange_value if the comparison succeeded. Returns prior 125 // value of *dest. cmpxchg*() provide: 126 // <fence> compare-and-exchange <membar StoreLoad|StoreStore> 127 128 template<typename T, typename D, typename U> 129 inline static D cmpxchg(T exchange_value, 130 D volatile* dest, 131 U compare_value, 132 atomic_memory_order order = memory_order_conservative); 133 134 // Performs atomic compare of *dest and NULL, and replaces *dest 135 // with exchange_value if the comparison succeeded. Returns true if 136 // the comparison succeeded and the exchange occurred. This is 137 // often used as part of lazy initialization, as a lock-free 138 // alternative to the Double-Checked Locking Pattern. 139 template<typename T, typename D> 140 inline static bool replace_if_null(T* value, D* volatile* dest, 141 atomic_memory_order order = memory_order_conservative); 142 143 private: 144 WINDOWS_ONLY(public:) // VS2017 warns (C2027) use of undefined type if IsPointerConvertible is declared private 145 // Test whether From is implicitly convertible to To. 146 // From and To must be pointer types. 147 // Note: Provides the limited subset of C++11 std::is_convertible 148 // that is needed here. 149 template<typename From, typename To> struct IsPointerConvertible; 150 151 protected: 152 // Dispatch handler for store. Provides type-based validity 153 // checking and limited conversions around calls to the platform- 154 // specific implementation layer provided by PlatformOp. 155 template<typename T, typename D, typename PlatformOp, typename Enable = void> 156 struct StoreImpl; 157 158 // Platform-specific implementation of store. Support for sizes 159 // of 1, 2, 4, and (if different) pointer size bytes are required. 160 // The class is a function object that must be default constructable, 161 // with these requirements: 162 // 163 // either: 164 // - dest is of type D*, an integral, enum or pointer type. 165 // - new_value are of type T, an integral, enum or pointer type D or 166 // pointer type convertible to D. 167 // or: 168 // - T and D are the same and are primitive convertible using PrimitiveConversions 169 // and either way: 170 // - platform_store is an object of type PlatformStore<sizeof(T)>. 171 // 172 // Then 173 // platform_store(new_value, dest) 174 // must be a valid expression. 175 // 176 // The default implementation is a volatile store. If a platform 177 // requires more for e.g. 64 bit stores, a specialization is required 178 template<size_t byte_size> struct PlatformStore; 179 180 // Dispatch handler for load. Provides type-based validity 181 // checking and limited conversions around calls to the platform- 182 // specific implementation layer provided by PlatformOp. 183 template<typename T, typename PlatformOp, typename Enable = void> 184 struct LoadImpl; 185 186 // Platform-specific implementation of load. Support for sizes of 187 // 1, 2, 4 bytes and (if different) pointer size bytes are required. 188 // The class is a function object that must be default 189 // constructable, with these requirements: 190 // 191 // - dest is of type T*, an integral, enum or pointer type, or 192 // T is convertible to a primitive type using PrimitiveConversions 193 // - platform_load is an object of type PlatformLoad<sizeof(T)>. 194 // 195 // Then 196 // platform_load(src) 197 // must be a valid expression, returning a result convertible to T. 198 // 199 // The default implementation is a volatile load. If a platform 200 // requires more for e.g. 64 bit loads, a specialization is required 201 template<size_t byte_size> struct PlatformLoad; 202 203 private: 204 // Dispatch handler for add. Provides type-based validity checking 205 // and limited conversions around calls to the platform-specific 206 // implementation layer provided by PlatformAdd. 207 template<typename I, typename D, typename Enable = void> 208 struct AddImpl; 209 210 // Platform-specific implementation of add. Support for sizes of 4 211 // bytes and (if different) pointer size bytes are required. The 212 // class is a function object that must be default constructable, 213 // with these requirements: 214 // 215 // - dest is of type D*, an integral or pointer type. 216 // - add_value is of type I, an integral type. 217 // - sizeof(I) == sizeof(D). 218 // - if D is an integral type, I == D. 219 // - platform_add is an object of type PlatformAdd<sizeof(D)>. 220 // 221 // Then 222 // platform_add(add_value, dest) 223 // must be a valid expression, returning a result convertible to D. 224 // 225 // No definition is provided; all platforms must explicitly define 226 // this class and any needed specializations. 227 template<size_t byte_size> struct PlatformAdd; 228 229 // Helper base classes for defining PlatformAdd. To use, define 230 // PlatformAdd or a specialization that derives from one of these, 231 // and include in the PlatformAdd definition the support function 232 // (described below) required by the base class. 233 // 234 // These classes implement the required function object protocol for 235 // PlatformAdd, using a support function template provided by the 236 // derived class. Let add_value (of type I) and dest (of type D) be 237 // the arguments the object is called with. If D is a pointer type 238 // P*, then let addend (of type I) be add_value * sizeof(P); 239 // otherwise, addend is add_value. 240 // 241 // FetchAndAdd requires the derived class to provide 242 // fetch_and_add(addend, dest) 243 // atomically adding addend to the value of dest, and returning the 244 // old value. 245 // 246 // AddAndFetch requires the derived class to provide 247 // add_and_fetch(addend, dest) 248 // atomically adding addend to the value of dest, and returning the 249 // new value. 250 // 251 // When D is a pointer type P*, both fetch_and_add and add_and_fetch 252 // treat it as if it were a uintptr_t; they do not perform any 253 // scaling of the addend, as that has already been done by the 254 // caller. 255 public: // Temporary, can't be private: C++03 11.4/2. Fixed by C++11. 256 template<typename Derived> struct FetchAndAdd; 257 template<typename Derived> struct AddAndFetch; 258 private: 259 260 // Support for platforms that implement some variants of add using a 261 // (typically out of line) non-template helper function. The 262 // generic arguments passed to PlatformAdd need to be translated to 263 // the appropriate type for the helper function, the helper function 264 // invoked on the translated arguments, and the result translated 265 // back. Type is the parameter / return type of the helper 266 // function. No scaling of add_value is performed when D is a pointer 267 // type, so this function can be used to implement the support function 268 // required by AddAndFetch. 269 template<typename Type, typename Fn, typename I, typename D> 270 static D add_using_helper(Fn fn, I add_value, D volatile* dest); 271 272 // Dispatch handler for cmpxchg. Provides type-based validity 273 // checking and limited conversions around calls to the 274 // platform-specific implementation layer provided by 275 // PlatformCmpxchg. 276 template<typename T, typename D, typename U, typename Enable = void> 277 struct CmpxchgImpl; 278 279 // Platform-specific implementation of cmpxchg. Support for sizes 280 // of 1, 4, and 8 are required. The class is a function object that 281 // must be default constructable, with these requirements: 282 // 283 // - dest is of type T*. 284 // - exchange_value and compare_value are of type T. 285 // - order is of type atomic_memory_order. 286 // - platform_cmpxchg is an object of type PlatformCmpxchg<sizeof(T)>. 287 // 288 // Then 289 // platform_cmpxchg(exchange_value, dest, compare_value, order) 290 // must be a valid expression, returning a result convertible to T. 291 // 292 // A default definition is provided, which declares a function template 293 // T operator()(T, T volatile*, T, atomic_memory_order) const 294 // 295 // For each required size, a platform must either provide an 296 // appropriate definition of that function, or must entirely 297 // specialize the class template for that size. 298 template<size_t byte_size> struct PlatformCmpxchg; 299 300 // Support for platforms that implement some variants of cmpxchg 301 // using a (typically out of line) non-template helper function. 302 // The generic arguments passed to PlatformCmpxchg need to be 303 // translated to the appropriate type for the helper function, the 304 // helper invoked on the translated arguments, and the result 305 // translated back. Type is the parameter / return type of the 306 // helper function. 307 template<typename Type, typename Fn, typename T> 308 static T cmpxchg_using_helper(Fn fn, 309 T exchange_value, 310 T volatile* dest, 311 T compare_value); 312 313 // Support platforms that do not provide Read-Modify-Write 314 // byte-level atomic access. To use, derive PlatformCmpxchg<1> from 315 // this class. 316 public: // Temporary, can't be private: C++03 11.4/2. Fixed by C++11. 317 struct CmpxchgByteUsingInt; 318 private: 319 320 // Dispatch handler for xchg. Provides type-based validity 321 // checking and limited conversions around calls to the 322 // platform-specific implementation layer provided by 323 // PlatformXchg. 324 template<typename T, typename D, typename Enable = void> 325 struct XchgImpl; 326 327 // Platform-specific implementation of xchg. Support for sizes 328 // of 4, and sizeof(intptr_t) are required. The class is a function 329 // object that must be default constructable, with these requirements: 330 // 331 // - dest is of type T*. 332 // - exchange_value is of type T. 333 // - platform_xchg is an object of type PlatformXchg<sizeof(T)>. 334 // 335 // Then 336 // platform_xchg(exchange_value, dest) 337 // must be a valid expression, returning a result convertible to T. 338 // 339 // A default definition is provided, which declares a function template 340 // T operator()(T, T volatile*, T, atomic_memory_order) const 341 // 342 // For each required size, a platform must either provide an 343 // appropriate definition of that function, or must entirely 344 // specialize the class template for that size. 345 template<size_t byte_size> struct PlatformXchg; 346 347 // Support for platforms that implement some variants of xchg 348 // using a (typically out of line) non-template helper function. 349 // The generic arguments passed to PlatformXchg need to be 350 // translated to the appropriate type for the helper function, the 351 // helper invoked on the translated arguments, and the result 352 // translated back. Type is the parameter / return type of the 353 // helper function. 354 template<typename Type, typename Fn, typename T> 355 static T xchg_using_helper(Fn fn, 356 T exchange_value, 357 T volatile* dest); 358 }; 359 360 template<typename From, typename To> 361 struct Atomic::IsPointerConvertible<From*, To*> : AllStatic { 362 // Determine whether From* is implicitly convertible to To*, using 363 // the "sizeof trick". 364 typedef char yes; 365 typedef char (&no)[2]; 366 367 static yes test(To*); 368 static no test(...); 369 static From* test_value; 370 371 static const bool value = (sizeof(yes) == sizeof(test(test_value))); 372 }; 373 374 // Handle load for pointer, integral and enum types. 375 template<typename T, typename PlatformOp> 376 struct Atomic::LoadImpl< 377 T, 378 PlatformOp, 379 typename EnableIf<IsIntegral<T>::value || IsRegisteredEnum<T>::value || IsPointer<T>::value>::type> 380 { 381 T operator()(T const volatile* dest) const { 382 // Forward to the platform handler for the size of T. 383 return PlatformOp()(dest); 384 } 385 }; 386 387 // Handle load for types that have a translator. 388 // 389 // All the involved types must be identical. 390 // 391 // This translates the original call into a call on the decayed 392 // arguments, and returns the recovered result of that translated 393 // call. 394 template<typename T, typename PlatformOp> 395 struct Atomic::LoadImpl< 396 T, 397 PlatformOp, 398 typename EnableIf<PrimitiveConversions::Translate<T>::value>::type> 399 { 400 T operator()(T const volatile* dest) const { 401 typedef PrimitiveConversions::Translate<T> Translator; 402 typedef typename Translator::Decayed Decayed; 403 STATIC_ASSERT(sizeof(T) == sizeof(Decayed)); 404 Decayed result = PlatformOp()(reinterpret_cast<Decayed const volatile*>(dest)); 405 return Translator::recover(result); 406 } 407 }; 408 409 // Default implementation of atomic load if a specific platform 410 // does not provide a specialization for a certain size class. 411 // For increased safety, the default implementation only allows 412 // load types that are pointer sized or smaller. If a platform still 413 // supports wide atomics, then it has to use specialization 414 // of Atomic::PlatformLoad for that wider size class. 415 template<size_t byte_size> 416 struct Atomic::PlatformLoad { 417 template<typename T> 418 T operator()(T const volatile* dest) const { 419 STATIC_ASSERT(sizeof(T) <= sizeof(void*)); // wide atomics need specialization 420 return *dest; 421 } 422 }; 423 424 // Handle store for integral and enum types. 425 // 426 // All the involved types must be identical. 427 template<typename T, typename PlatformOp> 428 struct Atomic::StoreImpl< 429 T, T, 430 PlatformOp, 431 typename EnableIf<IsIntegral<T>::value || IsRegisteredEnum<T>::value>::type> 432 { 433 void operator()(T new_value, T volatile* dest) const { 434 // Forward to the platform handler for the size of T. 435 PlatformOp()(new_value, dest); 436 } 437 }; 438 439 // Handle store for pointer types. 440 // 441 // The new_value must be implicitly convertible to the 442 // destination's type; it must be type-correct to store the 443 // new_value in the destination. 444 template<typename T, typename D, typename PlatformOp> 445 struct Atomic::StoreImpl< 446 T*, D*, 447 PlatformOp, 448 typename EnableIf<Atomic::IsPointerConvertible<T*, D*>::value>::type> 449 { 450 void operator()(T* new_value, D* volatile* dest) const { 451 // Allow derived to base conversion, and adding cv-qualifiers. 452 D* value = new_value; 453 PlatformOp()(value, dest); 454 } 455 }; 456 457 // Handle store for types that have a translator. 458 // 459 // All the involved types must be identical. 460 // 461 // This translates the original call into a call on the decayed 462 // arguments. 463 template<typename T, typename PlatformOp> 464 struct Atomic::StoreImpl< 465 T, T, 466 PlatformOp, 467 typename EnableIf<PrimitiveConversions::Translate<T>::value>::type> 468 { 469 void operator()(T new_value, T volatile* dest) const { 470 typedef PrimitiveConversions::Translate<T> Translator; 471 typedef typename Translator::Decayed Decayed; 472 STATIC_ASSERT(sizeof(T) == sizeof(Decayed)); 473 PlatformOp()(Translator::decay(new_value), 474 reinterpret_cast<Decayed volatile*>(dest)); 475 } 476 }; 477 478 // Default implementation of atomic store if a specific platform 479 // does not provide a specialization for a certain size class. 480 // For increased safety, the default implementation only allows 481 // storing types that are pointer sized or smaller. If a platform still 482 // supports wide atomics, then it has to use specialization 483 // of Atomic::PlatformStore for that wider size class. 484 template<size_t byte_size> 485 struct Atomic::PlatformStore { 486 template<typename T> 487 void operator()(T new_value, 488 T volatile* dest) const { 489 STATIC_ASSERT(sizeof(T) <= sizeof(void*)); // wide atomics need specialization 490 (void)const_cast<T&>(*dest = new_value); 491 } 492 }; 493 494 // Define FetchAndAdd and AddAndFetch helper classes before including 495 // platform file, which may use these as base classes, requiring they 496 // be complete. 497 498 template<typename Derived> 499 struct Atomic::FetchAndAdd { 500 template<typename I, typename D> 501 D operator()(I add_value, D volatile* dest, atomic_memory_order order) const; 502 }; 503 504 template<typename Derived> 505 struct Atomic::AddAndFetch { 506 template<typename I, typename D> 507 D operator()(I add_value, D volatile* dest, atomic_memory_order order) const; 508 }; 509 510 template<typename D> 511 inline void Atomic::inc(D volatile* dest, atomic_memory_order order) { 512 STATIC_ASSERT(IsPointer<D>::value || IsIntegral<D>::value); 513 typedef typename Conditional<IsPointer<D>::value, ptrdiff_t, D>::type I; 514 Atomic::add(I(1), dest, order); 515 } 516 517 template<typename D> 518 inline void Atomic::dec(D volatile* dest, atomic_memory_order order) { 519 STATIC_ASSERT(IsPointer<D>::value || IsIntegral<D>::value); 520 typedef typename Conditional<IsPointer<D>::value, ptrdiff_t, D>::type I; 521 // Assumes two's complement integer representation. 522 #pragma warning(suppress: 4146) 523 Atomic::add(I(-1), dest, order); 524 } 525 526 template<typename I, typename D> 527 inline D Atomic::sub(I sub_value, D volatile* dest, atomic_memory_order order) { 528 STATIC_ASSERT(IsPointer<D>::value || IsIntegral<D>::value); 529 STATIC_ASSERT(IsIntegral<I>::value); 530 // If D is a pointer type, use [u]intptr_t as the addend type, 531 // matching signedness of I. Otherwise, use D as the addend type. 532 typedef typename Conditional<IsSigned<I>::value, intptr_t, uintptr_t>::type PI; 533 typedef typename Conditional<IsPointer<D>::value, PI, D>::type AddendType; 534 // Only allow conversions that can't change the value. 535 STATIC_ASSERT(IsSigned<I>::value == IsSigned<AddendType>::value); 536 STATIC_ASSERT(sizeof(I) <= sizeof(AddendType)); 537 AddendType addend = sub_value; 538 // Assumes two's complement integer representation. 539 #pragma warning(suppress: 4146) // In case AddendType is not signed. 540 return Atomic::add(-addend, dest, order); 541 } 542 543 // Define the class before including platform file, which may specialize 544 // the operator definition. No generic definition of specializations 545 // of the operator template are provided, nor are there any generic 546 // specializations of the class. The platform file is responsible for 547 // providing those. 548 template<size_t byte_size> 549 struct Atomic::PlatformCmpxchg { 550 template<typename T> 551 T operator()(T exchange_value, 552 T volatile* dest, 553 T compare_value, 554 atomic_memory_order order) const; 555 }; 556 557 // Define the class before including platform file, which may use this 558 // as a base class, requiring it be complete. The definition is later 559 // in this file, near the other definitions related to cmpxchg. 560 struct Atomic::CmpxchgByteUsingInt { 561 template<typename T> 562 T operator()(T exchange_value, 563 T volatile* dest, 564 T compare_value, 565 atomic_memory_order order) const; 566 }; 567 568 // Define the class before including platform file, which may specialize 569 // the operator definition. No generic definition of specializations 570 // of the operator template are provided, nor are there any generic 571 // specializations of the class. The platform file is responsible for 572 // providing those. 573 template<size_t byte_size> 574 struct Atomic::PlatformXchg { 575 template<typename T> 576 T operator()(T exchange_value, 577 T volatile* dest, 578 atomic_memory_order order) const; 579 }; 580 581 // platform specific in-line definitions - must come before shared definitions 582 583 #include OS_CPU_HEADER(atomic) 584 585 // shared in-line definitions 586 587 // size_t casts... 588 #if (SIZE_MAX != UINTPTR_MAX) 589 #error size_t is not WORD_SIZE, interesting platform, but missing implementation here 590 #endif 591 592 template<typename T> 593 inline T Atomic::load(const volatile T* dest) { 594 return LoadImpl<T, PlatformLoad<sizeof(T)> >()(dest); 595 } 596 597 template<typename T, typename D> 598 inline void Atomic::store(T store_value, volatile D* dest) { 599 StoreImpl<T, D, PlatformStore<sizeof(D)> >()(store_value, dest); 600 } 601 602 template<typename I, typename D> 603 inline D Atomic::add(I add_value, D volatile* dest, 604 atomic_memory_order order) { 605 return AddImpl<I, D>()(add_value, dest, order); 606 } 607 608 template<typename I, typename D> 609 struct Atomic::AddImpl< 610 I, D, 611 typename EnableIf<IsIntegral<I>::value && 612 IsIntegral<D>::value && 613 (sizeof(I) <= sizeof(D)) && 614 (IsSigned<I>::value == IsSigned<D>::value)>::type> 615 { 616 D operator()(I add_value, D volatile* dest, atomic_memory_order order) const { 617 D addend = add_value; 618 return PlatformAdd<sizeof(D)>()(addend, dest, order); 619 } 620 }; 621 622 template<typename I, typename P> 623 struct Atomic::AddImpl< 624 I, P*, 625 typename EnableIf<IsIntegral<I>::value && (sizeof(I) <= sizeof(P*))>::type> 626 { 627 P* operator()(I add_value, P* volatile* dest, atomic_memory_order order) const { 628 STATIC_ASSERT(sizeof(intptr_t) == sizeof(P*)); 629 STATIC_ASSERT(sizeof(uintptr_t) == sizeof(P*)); 630 typedef typename Conditional<IsSigned<I>::value, 631 intptr_t, 632 uintptr_t>::type CI; 633 CI addend = add_value; 634 return PlatformAdd<sizeof(P*)>()(addend, dest, order); 635 } 636 }; 637 638 // Most platforms do not support atomic add on a 2-byte value. However, 639 // if the value occupies the most significant 16 bits of an aligned 32-bit 640 // word, then we can do this with an atomic add of (add_value << 16) 641 // to the 32-bit word. 642 // 643 // The least significant parts of this 32-bit word will never be affected, even 644 // in case of overflow/underflow. 645 // 646 // Use the ATOMIC_SHORT_PAIR macro (see macros.hpp) to get the desired alignment. 647 template<> 648 struct Atomic::AddImpl<short, short> { 649 short operator()(short add_value, short volatile* dest, atomic_memory_order order) const { 650 #ifdef VM_LITTLE_ENDIAN 651 assert((intx(dest) & 0x03) == 0x02, "wrong alignment"); 652 int new_value = Atomic::add(add_value << 16, (volatile int*)(dest-1), order); 653 #else 654 assert((intx(dest) & 0x03) == 0x00, "wrong alignment"); 655 int new_value = Atomic::add(add_value << 16, (volatile int*)(dest), order); 656 #endif 657 return (short)(new_value >> 16); // preserves sign 658 } 659 }; 660 661 template<typename Derived> 662 template<typename I, typename D> 663 inline D Atomic::FetchAndAdd<Derived>::operator()(I add_value, D volatile* dest, 664 atomic_memory_order order) const { 665 I addend = add_value; 666 // If D is a pointer type P*, scale by sizeof(P). 667 if (IsPointer<D>::value) { 668 addend *= sizeof(typename RemovePointer<D>::type); 669 } 670 D old = static_cast<const Derived*>(this)->fetch_and_add(addend, dest, order); 671 return old + add_value; 672 } 673 674 template<typename Derived> 675 template<typename I, typename D> 676 inline D Atomic::AddAndFetch<Derived>::operator()(I add_value, D volatile* dest, 677 atomic_memory_order order) const { 678 // If D is a pointer type P*, scale by sizeof(P). 679 if (IsPointer<D>::value) { 680 add_value *= sizeof(typename RemovePointer<D>::type); 681 } 682 return static_cast<const Derived*>(this)->add_and_fetch(add_value, dest, order); 683 } 684 685 template<typename Type, typename Fn, typename I, typename D> 686 inline D Atomic::add_using_helper(Fn fn, I add_value, D volatile* dest) { 687 return PrimitiveConversions::cast<D>( 688 fn(PrimitiveConversions::cast<Type>(add_value), 689 reinterpret_cast<Type volatile*>(dest))); 690 } 691 692 template<typename T, typename D, typename U> 693 inline D Atomic::cmpxchg(T exchange_value, 694 D volatile* dest, 695 U compare_value, 696 atomic_memory_order order) { 697 return CmpxchgImpl<T, D, U>()(exchange_value, dest, compare_value, order); 698 } 699 700 template<typename T, typename D> 701 inline bool Atomic::replace_if_null(T* value, D* volatile* dest, 702 atomic_memory_order order) { 703 // Presently using a trivial implementation in terms of cmpxchg. 704 // Consider adding platform support, to permit the use of compiler 705 // intrinsics like gcc's __sync_bool_compare_and_swap. 706 D* expected_null = NULL; 707 return expected_null == cmpxchg(value, dest, expected_null, order); 708 } 709 710 // Handle cmpxchg for integral and enum types. 711 // 712 // All the involved types must be identical. 713 template<typename T> 714 struct Atomic::CmpxchgImpl< 715 T, T, T, 716 typename EnableIf<IsIntegral<T>::value || IsRegisteredEnum<T>::value>::type> 717 { 718 T operator()(T exchange_value, T volatile* dest, T compare_value, 719 atomic_memory_order order) const { 720 // Forward to the platform handler for the size of T. 721 return PlatformCmpxchg<sizeof(T)>()(exchange_value, 722 dest, 723 compare_value, 724 order); 725 } 726 }; 727 728 // Handle cmpxchg for pointer types. 729 // 730 // The destination's type and the compare_value type must be the same, 731 // ignoring cv-qualifiers; we don't care about the cv-qualifiers of 732 // the compare_value. 733 // 734 // The exchange_value must be implicitly convertible to the 735 // destination's type; it must be type-correct to store the 736 // exchange_value in the destination. 737 template<typename T, typename D, typename U> 738 struct Atomic::CmpxchgImpl< 739 T*, D*, U*, 740 typename EnableIf<Atomic::IsPointerConvertible<T*, D*>::value && 741 IsSame<typename RemoveCV<D>::type, 742 typename RemoveCV<U>::type>::value>::type> 743 { 744 D* operator()(T* exchange_value, D* volatile* dest, U* compare_value, 745 atomic_memory_order order) const { 746 // Allow derived to base conversion, and adding cv-qualifiers. 747 D* new_value = exchange_value; 748 // Don't care what the CV qualifiers for compare_value are, 749 // but we need to match D* when calling platform support. 750 D* old_value = const_cast<D*>(compare_value); 751 return PlatformCmpxchg<sizeof(D*)>()(new_value, dest, old_value, order); 752 } 753 }; 754 755 // Handle cmpxchg for types that have a translator. 756 // 757 // All the involved types must be identical. 758 // 759 // This translates the original call into a call on the decayed 760 // arguments, and returns the recovered result of that translated 761 // call. 762 template<typename T> 763 struct Atomic::CmpxchgImpl< 764 T, T, T, 765 typename EnableIf<PrimitiveConversions::Translate<T>::value>::type> 766 { 767 T operator()(T exchange_value, T volatile* dest, T compare_value, 768 atomic_memory_order order) const { 769 typedef PrimitiveConversions::Translate<T> Translator; 770 typedef typename Translator::Decayed Decayed; 771 STATIC_ASSERT(sizeof(T) == sizeof(Decayed)); 772 return Translator::recover( 773 cmpxchg(Translator::decay(exchange_value), 774 reinterpret_cast<Decayed volatile*>(dest), 775 Translator::decay(compare_value), 776 order)); 777 } 778 }; 779 780 template<typename Type, typename Fn, typename T> 781 inline T Atomic::cmpxchg_using_helper(Fn fn, 782 T exchange_value, 783 T volatile* dest, 784 T compare_value) { 785 STATIC_ASSERT(sizeof(Type) == sizeof(T)); 786 return PrimitiveConversions::cast<T>( 787 fn(PrimitiveConversions::cast<Type>(exchange_value), 788 reinterpret_cast<Type volatile*>(dest), 789 PrimitiveConversions::cast<Type>(compare_value))); 790 } 791 792 template<typename T> 793 inline T Atomic::CmpxchgByteUsingInt::operator()(T exchange_value, 794 T volatile* dest, 795 T compare_value, 796 atomic_memory_order order) const { 797 STATIC_ASSERT(sizeof(T) == sizeof(uint8_t)); 798 uint8_t canon_exchange_value = exchange_value; 799 uint8_t canon_compare_value = compare_value; 800 volatile uint32_t* aligned_dest 801 = reinterpret_cast<volatile uint32_t*>(align_down(dest, sizeof(uint32_t))); 802 size_t offset = pointer_delta(dest, aligned_dest, 1); 803 uint32_t cur = *aligned_dest; 804 uint8_t* cur_as_bytes = reinterpret_cast<uint8_t*>(&cur); 805 806 // current value may not be what we are looking for, so force it 807 // to that value so the initial cmpxchg will fail if it is different 808 cur_as_bytes[offset] = canon_compare_value; 809 810 // always execute a real cmpxchg so that we get the required memory 811 // barriers even on initial failure 812 do { 813 // value to swap in matches current value ... 814 uint32_t new_value = cur; 815 // ... except for the one byte we want to update 816 reinterpret_cast<uint8_t*>(&new_value)[offset] = canon_exchange_value; 817 818 uint32_t res = cmpxchg(new_value, aligned_dest, cur, order); 819 if (res == cur) break; // success 820 821 // at least one byte in the int changed value, so update 822 // our view of the current int 823 cur = res; 824 // if our byte is still as cur we loop and try again 825 } while (cur_as_bytes[offset] == canon_compare_value); 826 827 return PrimitiveConversions::cast<T>(cur_as_bytes[offset]); 828 } 829 830 // Handle xchg for integral and enum types. 831 // 832 // All the involved types must be identical. 833 template<typename T> 834 struct Atomic::XchgImpl< 835 T, T, 836 typename EnableIf<IsIntegral<T>::value || IsRegisteredEnum<T>::value>::type> 837 { 838 T operator()(T exchange_value, T volatile* dest, atomic_memory_order order) const { 839 // Forward to the platform handler for the size of T. 840 return PlatformXchg<sizeof(T)>()(exchange_value, dest, order); 841 } 842 }; 843 844 // Handle xchg for pointer types. 845 // 846 // The exchange_value must be implicitly convertible to the 847 // destination's type; it must be type-correct to store the 848 // exchange_value in the destination. 849 template<typename T, typename D> 850 struct Atomic::XchgImpl< 851 T*, D*, 852 typename EnableIf<Atomic::IsPointerConvertible<T*, D*>::value>::type> 853 { 854 D* operator()(T* exchange_value, D* volatile* dest, atomic_memory_order order) const { 855 // Allow derived to base conversion, and adding cv-qualifiers. 856 D* new_value = exchange_value; 857 return PlatformXchg<sizeof(D*)>()(new_value, dest, order); 858 } 859 }; 860 861 // Handle xchg for types that have a translator. 862 // 863 // All the involved types must be identical. 864 // 865 // This translates the original call into a call on the decayed 866 // arguments, and returns the recovered result of that translated 867 // call. 868 template<typename T> 869 struct Atomic::XchgImpl< 870 T, T, 871 typename EnableIf<PrimitiveConversions::Translate<T>::value>::type> 872 { 873 T operator()(T exchange_value, T volatile* dest, atomic_memory_order order) const { 874 typedef PrimitiveConversions::Translate<T> Translator; 875 typedef typename Translator::Decayed Decayed; 876 STATIC_ASSERT(sizeof(T) == sizeof(Decayed)); 877 return Translator::recover( 878 xchg(Translator::decay(exchange_value), 879 reinterpret_cast<Decayed volatile*>(dest), 880 order)); 881 } 882 }; 883 884 template<typename Type, typename Fn, typename T> 885 inline T Atomic::xchg_using_helper(Fn fn, 886 T exchange_value, 887 T volatile* dest) { 888 STATIC_ASSERT(sizeof(Type) == sizeof(T)); 889 return PrimitiveConversions::cast<T>( 890 fn(PrimitiveConversions::cast<Type>(exchange_value), 891 reinterpret_cast<Type volatile*>(dest))); 892 } 893 894 template<typename T, typename D> 895 inline D Atomic::xchg(T exchange_value, volatile D* dest, atomic_memory_order order) { 896 return XchgImpl<T, D>()(exchange_value, dest, order); 897 } 898 899 #endif // SHARE_VM_RUNTIME_ATOMIC_HPP