1 /* 2 * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #ifndef SHARE_OOPS_ACCESSBACKEND_HPP 26 #define SHARE_OOPS_ACCESSBACKEND_HPP 27 28 #include "gc/shared/barrierSetConfig.hpp" 29 #include "memory/allocation.hpp" 30 #include "metaprogramming/conditional.hpp" 31 #include "metaprogramming/decay.hpp" 32 #include "metaprogramming/enableIf.hpp" 33 #include "metaprogramming/integralConstant.hpp" 34 #include "metaprogramming/isFloatingPoint.hpp" 35 #include "metaprogramming/isIntegral.hpp" 36 #include "metaprogramming/isPointer.hpp" 37 #include "metaprogramming/isSame.hpp" 38 #include "metaprogramming/isVolatile.hpp" 39 #include "oops/accessDecorators.hpp" 40 #include "oops/oopsHierarchy.hpp" 41 #include "utilities/debug.hpp" 42 #include "utilities/globalDefinitions.hpp" 43 44 45 // This metafunction returns either oop or narrowOop depending on whether 46 // an access needs to use compressed oops or not. 47 template <DecoratorSet decorators> 48 struct HeapOopType: AllStatic { 49 static const bool needs_oop_compress = HasDecorator<decorators, INTERNAL_CONVERT_COMPRESSED_OOP>::value && 50 HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value; 51 typedef typename Conditional<needs_oop_compress, narrowOop, oop>::type type; 52 }; 53 54 namespace AccessInternal { 55 enum BarrierType { 56 BARRIER_STORE, 57 BARRIER_STORE_AT, 58 BARRIER_LOAD, 59 BARRIER_LOAD_AT, 60 BARRIER_ATOMIC_CMPXCHG, 61 BARRIER_ATOMIC_CMPXCHG_AT, 62 BARRIER_ATOMIC_XCHG, 63 BARRIER_ATOMIC_XCHG_AT, 64 BARRIER_ARRAYCOPY, 65 BARRIER_CLONE, 66 BARRIER_RESOLVE, 67 BARRIER_EQUALS 68 }; 69 70 template <DecoratorSet decorators, typename T> 71 struct MustConvertCompressedOop: public IntegralConstant<bool, 72 HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value && 73 IsSame<typename HeapOopType<decorators>::type, narrowOop>::value && 74 IsSame<T, oop>::value> {}; 75 76 // This metafunction returns an appropriate oop type if the value is oop-like 77 // and otherwise returns the same type T. 78 template <DecoratorSet decorators, typename T> 79 struct EncodedType: AllStatic { 80 typedef typename Conditional< 81 HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value, 82 typename HeapOopType<decorators>::type, T>::type type; 83 }; 84 85 template <DecoratorSet decorators> 86 inline typename HeapOopType<decorators>::type* 87 oop_field_addr(oop base, ptrdiff_t byte_offset) { 88 return reinterpret_cast<typename HeapOopType<decorators>::type*>( 89 reinterpret_cast<intptr_t>((void*)base) + byte_offset); 90 } 91 92 // This metafunction returns whether it is possible for a type T to require 93 // locking to support wide atomics or not. 94 template <typename T> 95 #ifdef SUPPORTS_NATIVE_CX8 96 struct PossiblyLockedAccess: public IntegralConstant<bool, false> {}; 97 #else 98 struct PossiblyLockedAccess: public IntegralConstant<bool, (sizeof(T) > 4)> {}; 99 #endif 100 101 template <DecoratorSet decorators, typename T> 102 struct AccessFunctionTypes { 103 typedef T (*load_at_func_t)(oop base, ptrdiff_t offset); 104 typedef void (*store_at_func_t)(oop base, ptrdiff_t offset, T value); 105 typedef T (*atomic_cmpxchg_at_func_t)(T new_value, oop base, ptrdiff_t offset, T compare_value); 106 typedef T (*atomic_xchg_at_func_t)(T new_value, oop base, ptrdiff_t offset); 107 108 typedef T (*load_func_t)(void* addr); 109 typedef void (*store_func_t)(void* addr, T value); 110 typedef T (*atomic_cmpxchg_func_t)(T new_value, void* addr, T compare_value); 111 typedef T (*atomic_xchg_func_t)(T new_value, void* addr); 112 113 typedef bool (*arraycopy_func_t)(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length); 114 typedef void (*clone_func_t)(oop src, oop dst, size_t size); 115 typedef oop (*resolve_func_t)(oop obj); 116 typedef bool (*equals_func_t)(oop o1, oop o2); 117 }; 118 119 template <DecoratorSet decorators> 120 struct AccessFunctionTypes<decorators, void> { 121 typedef bool (*arraycopy_func_t)(arrayOop src_obj, arrayOop dst_obj, void* src, void* dst, size_t length); 122 }; 123 124 template <DecoratorSet decorators, typename T, BarrierType barrier> struct AccessFunction {}; 125 126 #define ACCESS_GENERATE_ACCESS_FUNCTION(bt, func) \ 127 template <DecoratorSet decorators, typename T> \ 128 struct AccessFunction<decorators, T, bt>: AllStatic{ \ 129 typedef typename AccessFunctionTypes<decorators, T>::func type; \ 130 } 131 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE, store_func_t); 132 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE_AT, store_at_func_t); 133 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD, load_func_t); 134 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD_AT, load_at_func_t); 135 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG, atomic_cmpxchg_func_t); 136 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG_AT, atomic_cmpxchg_at_func_t); 137 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG, atomic_xchg_func_t); 138 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG_AT, atomic_xchg_at_func_t); 139 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ARRAYCOPY, arraycopy_func_t); 140 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_CLONE, clone_func_t); 141 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_RESOLVE, resolve_func_t); 142 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_EQUALS, equals_func_t); 143 #undef ACCESS_GENERATE_ACCESS_FUNCTION 144 145 template <DecoratorSet decorators, typename T, BarrierType barrier_type> 146 typename AccessFunction<decorators, T, barrier_type>::type resolve_barrier(); 147 148 template <DecoratorSet decorators, typename T, BarrierType barrier_type> 149 typename AccessFunction<decorators, T, barrier_type>::type resolve_oop_barrier(); 150 151 class AccessLocker { 152 public: 153 AccessLocker(); 154 ~AccessLocker(); 155 }; 156 bool wide_atomic_needs_locking(); 157 158 void* field_addr(oop base, ptrdiff_t offset); 159 160 // Forward calls to Copy:: in the cpp file to reduce dependencies and allow 161 // faster build times, given how frequently included access is. 162 void arraycopy_arrayof_conjoint_oops(void* src, void* dst, size_t length); 163 void arraycopy_conjoint_oops(oop* src, oop* dst, size_t length); 164 void arraycopy_conjoint_oops(narrowOop* src, narrowOop* dst, size_t length); 165 166 void arraycopy_disjoint_words(void* src, void* dst, size_t length); 167 void arraycopy_disjoint_words_atomic(void* src, void* dst, size_t length); 168 169 template<typename T> 170 void arraycopy_conjoint(T* src, T* dst, size_t length); 171 template<typename T> 172 void arraycopy_arrayof_conjoint(T* src, T* dst, size_t length); 173 template<typename T> 174 void arraycopy_conjoint_atomic(T* src, T* dst, size_t length); 175 } 176 177 // This mask specifies what decorators are relevant for raw accesses. When passing 178 // accesses to the raw layer, irrelevant decorators are removed. 179 const DecoratorSet RAW_DECORATOR_MASK = INTERNAL_DECORATOR_MASK | MO_DECORATOR_MASK | 180 ARRAYCOPY_DECORATOR_MASK | OOP_DECORATOR_MASK; 181 182 // The RawAccessBarrier performs raw accesses with additional knowledge of 183 // memory ordering, so that OrderAccess/Atomic is called when necessary. 184 // It additionally handles compressed oops, and hence is not completely "raw" 185 // strictly speaking. 186 template <DecoratorSet decorators> 187 class RawAccessBarrier: public AllStatic { 188 protected: 189 static inline void* field_addr(oop base, ptrdiff_t byte_offset) { 190 return AccessInternal::field_addr(base, byte_offset); 191 } 192 193 protected: 194 // Only encode if INTERNAL_VALUE_IS_OOP 195 template <DecoratorSet idecorators, typename T> 196 static inline typename EnableIf< 197 AccessInternal::MustConvertCompressedOop<idecorators, T>::value, 198 typename HeapOopType<idecorators>::type>::type 199 encode_internal(T value); 200 201 template <DecoratorSet idecorators, typename T> 202 static inline typename EnableIf< 203 !AccessInternal::MustConvertCompressedOop<idecorators, T>::value, T>::type 204 encode_internal(T value) { 205 return value; 206 } 207 208 template <typename T> 209 static inline typename AccessInternal::EncodedType<decorators, T>::type 210 encode(T value) { 211 return encode_internal<decorators, T>(value); 212 } 213 214 // Only decode if INTERNAL_VALUE_IS_OOP 215 template <DecoratorSet idecorators, typename T> 216 static inline typename EnableIf< 217 AccessInternal::MustConvertCompressedOop<idecorators, T>::value, T>::type 218 decode_internal(typename HeapOopType<idecorators>::type value); 219 220 template <DecoratorSet idecorators, typename T> 221 static inline typename EnableIf< 222 !AccessInternal::MustConvertCompressedOop<idecorators, T>::value, T>::type 223 decode_internal(T value) { 224 return value; 225 } 226 227 template <typename T> 228 static inline T decode(typename AccessInternal::EncodedType<decorators, T>::type value) { 229 return decode_internal<decorators, T>(value); 230 } 231 232 protected: 233 template <DecoratorSet ds, typename T> 234 static typename EnableIf< 235 HasDecorator<ds, MO_SEQ_CST>::value, T>::type 236 load_internal(void* addr); 237 238 template <DecoratorSet ds, typename T> 239 static typename EnableIf< 240 HasDecorator<ds, MO_ACQUIRE>::value, T>::type 241 load_internal(void* addr); 242 243 template <DecoratorSet ds, typename T> 244 static typename EnableIf< 245 HasDecorator<ds, MO_RELAXED>::value, T>::type 246 load_internal(void* addr); 247 248 template <DecoratorSet ds, typename T> 249 static inline typename EnableIf< 250 HasDecorator<ds, MO_VOLATILE>::value, T>::type 251 load_internal(void* addr) { 252 return *reinterpret_cast<const volatile T*>(addr); 253 } 254 255 template <DecoratorSet ds, typename T> 256 static inline typename EnableIf< 257 HasDecorator<ds, MO_UNORDERED>::value, T>::type 258 load_internal(void* addr) { 259 return *reinterpret_cast<const T*>(addr); 260 } 261 262 template <DecoratorSet ds, typename T> 263 static typename EnableIf< 264 HasDecorator<ds, MO_SEQ_CST>::value>::type 265 store_internal(void* addr, T value); 266 267 template <DecoratorSet ds, typename T> 268 static typename EnableIf< 269 HasDecorator<ds, MO_RELEASE>::value>::type 270 store_internal(void* addr, T value); 271 272 template <DecoratorSet ds, typename T> 273 static typename EnableIf< 274 HasDecorator<ds, MO_RELAXED>::value>::type 275 store_internal(void* addr, T value); 276 277 template <DecoratorSet ds, typename T> 278 static inline typename EnableIf< 279 HasDecorator<ds, MO_VOLATILE>::value>::type 280 store_internal(void* addr, T value) { 281 (void)const_cast<T&>(*reinterpret_cast<volatile T*>(addr) = value); 282 } 283 284 template <DecoratorSet ds, typename T> 285 static inline typename EnableIf< 286 HasDecorator<ds, MO_UNORDERED>::value>::type 287 store_internal(void* addr, T value) { 288 *reinterpret_cast<T*>(addr) = value; 289 } 290 291 template <DecoratorSet ds, typename T> 292 static typename EnableIf< 293 HasDecorator<ds, MO_SEQ_CST>::value, T>::type 294 atomic_cmpxchg_internal(T new_value, void* addr, T compare_value); 295 296 template <DecoratorSet ds, typename T> 297 static typename EnableIf< 298 HasDecorator<ds, MO_RELAXED>::value, T>::type 299 atomic_cmpxchg_internal(T new_value, void* addr, T compare_value); 300 301 template <DecoratorSet ds, typename T> 302 static typename EnableIf< 303 HasDecorator<ds, MO_SEQ_CST>::value, T>::type 304 atomic_xchg_internal(T new_value, void* addr); 305 306 // The following *_locked mechanisms serve the purpose of handling atomic operations 307 // that are larger than a machine can handle, and then possibly opt for using 308 // a slower path using a mutex to perform the operation. 309 310 template <DecoratorSet ds, typename T> 311 static inline typename EnableIf< 312 !AccessInternal::PossiblyLockedAccess<T>::value, T>::type 313 atomic_cmpxchg_maybe_locked(T new_value, void* addr, T compare_value) { 314 return atomic_cmpxchg_internal<ds>(new_value, addr, compare_value); 315 } 316 317 template <DecoratorSet ds, typename T> 318 static typename EnableIf< 319 AccessInternal::PossiblyLockedAccess<T>::value, T>::type 320 atomic_cmpxchg_maybe_locked(T new_value, void* addr, T compare_value); 321 322 template <DecoratorSet ds, typename T> 323 static inline typename EnableIf< 324 !AccessInternal::PossiblyLockedAccess<T>::value, T>::type 325 atomic_xchg_maybe_locked(T new_value, void* addr) { 326 return atomic_xchg_internal<ds>(new_value, addr); 327 } 328 329 template <DecoratorSet ds, typename T> 330 static typename EnableIf< 331 AccessInternal::PossiblyLockedAccess<T>::value, T>::type 332 atomic_xchg_maybe_locked(T new_value, void* addr); 333 334 public: 335 template <typename T> 336 static inline void store(void* addr, T value) { 337 store_internal<decorators>(addr, value); 338 } 339 340 template <typename T> 341 static inline T load(void* addr) { 342 return load_internal<decorators, T>(addr); 343 } 344 345 template <typename T> 346 static inline T atomic_cmpxchg(T new_value, void* addr, T compare_value) { 347 return atomic_cmpxchg_maybe_locked<decorators>(new_value, addr, compare_value); 348 } 349 350 template <typename T> 351 static inline T atomic_xchg(T new_value, void* addr) { 352 return atomic_xchg_maybe_locked<decorators>(new_value, addr); 353 } 354 355 template <typename T> 356 static bool arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length); 357 358 template <typename T> 359 static void oop_store(void* addr, T value); 360 template <typename T> 361 static void oop_store_at(oop base, ptrdiff_t offset, T value); 362 363 template <typename T> 364 static T oop_load(void* addr); 365 template <typename T> 366 static T oop_load_at(oop base, ptrdiff_t offset); 367 368 template <typename T> 369 static T oop_atomic_cmpxchg(T new_value, void* addr, T compare_value); 370 template <typename T> 371 static T oop_atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value); 372 373 template <typename T> 374 static T oop_atomic_xchg(T new_value, void* addr); 375 template <typename T> 376 static T oop_atomic_xchg_at(T new_value, oop base, ptrdiff_t offset); 377 378 template <typename T> 379 static void store_at(oop base, ptrdiff_t offset, T value) { 380 store(field_addr(base, offset), value); 381 } 382 383 template <typename T> 384 static T load_at(oop base, ptrdiff_t offset) { 385 return load<T>(field_addr(base, offset)); 386 } 387 388 template <typename T> 389 static T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) { 390 return atomic_cmpxchg(new_value, field_addr(base, offset), compare_value); 391 } 392 393 template <typename T> 394 static T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) { 395 return atomic_xchg(new_value, field_addr(base, offset)); 396 } 397 398 template <typename T> 399 static bool oop_arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length); 400 401 static void clone(oop src, oop dst, size_t size); 402 403 static oop resolve(oop obj) { return obj; } 404 405 static bool equals(oop o1, oop o2) { return o1 == o2; } 406 }; 407 408 // Below is the implementation of the first 4 steps of the template pipeline: 409 // * Step 1: Set default decorators and decay types. This step gets rid of CV qualifiers 410 // and sets default decorators to sensible values. 411 // * Step 2: Reduce types. This step makes sure there is only a single T type and not 412 // multiple types. The P type of the address and T type of the value must 413 // match. 414 // * Step 3: Pre-runtime dispatch. This step checks whether a runtime call can be 415 // avoided, and in that case avoids it (calling raw accesses or 416 // primitive accesses in a build that does not require primitive GC barriers) 417 // * Step 4: Runtime-dispatch. This step performs a runtime dispatch to the corresponding 418 // BarrierSet::AccessBarrier accessor that attaches GC-required barriers 419 // to the access. 420 421 namespace AccessInternal { 422 template <typename T> 423 struct OopOrNarrowOopInternal: AllStatic { 424 typedef oop type; 425 }; 426 427 template <> 428 struct OopOrNarrowOopInternal<narrowOop>: AllStatic { 429 typedef narrowOop type; 430 }; 431 432 // This metafunction returns a canonicalized oop/narrowOop type for a passed 433 // in oop-like types passed in from oop_* overloads where the user has sworn 434 // that the passed in values should be oop-like (e.g. oop, oopDesc*, arrayOop, 435 // narrowOoop, instanceOopDesc*, and random other things). 436 // In the oop_* overloads, it must hold that if the passed in type T is not 437 // narrowOop, then it by contract has to be one of many oop-like types implicitly 438 // convertible to oop, and hence returns oop as the canonical oop type. 439 // If it turns out it was not, then the implicit conversion to oop will fail 440 // to compile, as desired. 441 template <typename T> 442 struct OopOrNarrowOop: AllStatic { 443 typedef typename OopOrNarrowOopInternal<typename Decay<T>::type>::type type; 444 }; 445 446 inline void* field_addr(oop base, ptrdiff_t byte_offset) { 447 return reinterpret_cast<void*>(reinterpret_cast<intptr_t>((void*)base) + byte_offset); 448 } 449 // Step 4: Runtime dispatch 450 // The RuntimeDispatch class is responsible for performing a runtime dispatch of the 451 // accessor. This is required when the access either depends on whether compressed oops 452 // is being used, or it depends on which GC implementation was chosen (e.g. requires GC 453 // barriers). The way it works is that a function pointer initially pointing to an 454 // accessor resolution function gets called for each access. Upon first invocation, 455 // it resolves which accessor to be used in future invocations and patches the 456 // function pointer to this new accessor. 457 458 template <DecoratorSet decorators, typename T, BarrierType type> 459 struct RuntimeDispatch: AllStatic {}; 460 461 template <DecoratorSet decorators, typename T> 462 struct RuntimeDispatch<decorators, T, BARRIER_STORE>: AllStatic { 463 typedef typename AccessFunction<decorators, T, BARRIER_STORE>::type func_t; 464 static func_t _store_func; 465 466 static void store_init(void* addr, T value); 467 468 static inline void store(void* addr, T value) { 469 _store_func(addr, value); 470 } 471 }; 472 473 template <DecoratorSet decorators, typename T> 474 struct RuntimeDispatch<decorators, T, BARRIER_STORE_AT>: AllStatic { 475 typedef typename AccessFunction<decorators, T, BARRIER_STORE_AT>::type func_t; 476 static func_t _store_at_func; 477 478 static void store_at_init(oop base, ptrdiff_t offset, T value); 479 480 static inline void store_at(oop base, ptrdiff_t offset, T value) { 481 _store_at_func(base, offset, value); 482 } 483 }; 484 485 template <DecoratorSet decorators, typename T> 486 struct RuntimeDispatch<decorators, T, BARRIER_LOAD>: AllStatic { 487 typedef typename AccessFunction<decorators, T, BARRIER_LOAD>::type func_t; 488 static func_t _load_func; 489 490 static T load_init(void* addr); 491 492 static inline T load(void* addr) { 493 return _load_func(addr); 494 } 495 }; 496 497 template <DecoratorSet decorators, typename T> 498 struct RuntimeDispatch<decorators, T, BARRIER_LOAD_AT>: AllStatic { 499 typedef typename AccessFunction<decorators, T, BARRIER_LOAD_AT>::type func_t; 500 static func_t _load_at_func; 501 502 static T load_at_init(oop base, ptrdiff_t offset); 503 504 static inline T load_at(oop base, ptrdiff_t offset) { 505 return _load_at_func(base, offset); 506 } 507 }; 508 509 template <DecoratorSet decorators, typename T> 510 struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG>: AllStatic { 511 typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG>::type func_t; 512 static func_t _atomic_cmpxchg_func; 513 514 static T atomic_cmpxchg_init(T new_value, void* addr, T compare_value); 515 516 static inline T atomic_cmpxchg(T new_value, void* addr, T compare_value) { 517 return _atomic_cmpxchg_func(new_value, addr, compare_value); 518 } 519 }; 520 521 template <DecoratorSet decorators, typename T> 522 struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>: AllStatic { 523 typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::type func_t; 524 static func_t _atomic_cmpxchg_at_func; 525 526 static T atomic_cmpxchg_at_init(T new_value, oop base, ptrdiff_t offset, T compare_value); 527 528 static inline T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) { 529 return _atomic_cmpxchg_at_func(new_value, base, offset, compare_value); 530 } 531 }; 532 533 template <DecoratorSet decorators, typename T> 534 struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG>: AllStatic { 535 typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG>::type func_t; 536 static func_t _atomic_xchg_func; 537 538 static T atomic_xchg_init(T new_value, void* addr); 539 540 static inline T atomic_xchg(T new_value, void* addr) { 541 return _atomic_xchg_func(new_value, addr); 542 } 543 }; 544 545 template <DecoratorSet decorators, typename T> 546 struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>: AllStatic { 547 typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type func_t; 548 static func_t _atomic_xchg_at_func; 549 550 static T atomic_xchg_at_init(T new_value, oop base, ptrdiff_t offset); 551 552 static inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) { 553 return _atomic_xchg_at_func(new_value, base, offset); 554 } 555 }; 556 557 template <DecoratorSet decorators, typename T> 558 struct RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>: AllStatic { 559 typedef typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type func_t; 560 static func_t _arraycopy_func; 561 562 static bool arraycopy_init(arrayOop src_obj, arrayOop dst_obj, T *src, T* dst, size_t length); 563 564 static inline bool arraycopy(arrayOop src_obj, arrayOop dst_obj, T *src, T* dst, size_t length) { 565 return _arraycopy_func(src_obj, dst_obj, src, dst, length); 566 } 567 }; 568 569 template <DecoratorSet decorators, typename T> 570 struct RuntimeDispatch<decorators, T, BARRIER_CLONE>: AllStatic { 571 typedef typename AccessFunction<decorators, T, BARRIER_CLONE>::type func_t; 572 static func_t _clone_func; 573 574 static void clone_init(oop src, oop dst, size_t size); 575 576 static inline void clone(oop src, oop dst, size_t size) { 577 _clone_func(src, dst, size); 578 } 579 }; 580 581 template <DecoratorSet decorators, typename T> 582 struct RuntimeDispatch<decorators, T, BARRIER_RESOLVE>: AllStatic { 583 typedef typename AccessFunction<decorators, T, BARRIER_RESOLVE>::type func_t; 584 static func_t _resolve_func; 585 586 static oop resolve_init(oop obj); 587 588 static inline oop resolve(oop obj) { 589 return _resolve_func(obj); 590 } 591 }; 592 593 template <DecoratorSet decorators, typename T> 594 struct RuntimeDispatch<decorators, T, BARRIER_EQUALS>: AllStatic { 595 typedef typename AccessFunction<decorators, T, BARRIER_EQUALS>::type func_t; 596 static func_t _equals_func; 597 598 static bool equals_init(oop o1, oop o2); 599 600 static inline bool equals(oop o1, oop o2) { 601 return _equals_func(o1, o2); 602 } 603 }; 604 605 // Initialize the function pointers to point to the resolving function. 606 template <DecoratorSet decorators, typename T> 607 typename AccessFunction<decorators, T, BARRIER_STORE>::type 608 RuntimeDispatch<decorators, T, BARRIER_STORE>::_store_func = &store_init; 609 610 template <DecoratorSet decorators, typename T> 611 typename AccessFunction<decorators, T, BARRIER_STORE_AT>::type 612 RuntimeDispatch<decorators, T, BARRIER_STORE_AT>::_store_at_func = &store_at_init; 613 614 template <DecoratorSet decorators, typename T> 615 typename AccessFunction<decorators, T, BARRIER_LOAD>::type 616 RuntimeDispatch<decorators, T, BARRIER_LOAD>::_load_func = &load_init; 617 618 template <DecoratorSet decorators, typename T> 619 typename AccessFunction<decorators, T, BARRIER_LOAD_AT>::type 620 RuntimeDispatch<decorators, T, BARRIER_LOAD_AT>::_load_at_func = &load_at_init; 621 622 template <DecoratorSet decorators, typename T> 623 typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG>::type 624 RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG>::_atomic_cmpxchg_func = &atomic_cmpxchg_init; 625 626 template <DecoratorSet decorators, typename T> 627 typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::type 628 RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::_atomic_cmpxchg_at_func = &atomic_cmpxchg_at_init; 629 630 template <DecoratorSet decorators, typename T> 631 typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG>::type 632 RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG>::_atomic_xchg_func = &atomic_xchg_init; 633 634 template <DecoratorSet decorators, typename T> 635 typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type 636 RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::_atomic_xchg_at_func = &atomic_xchg_at_init; 637 638 template <DecoratorSet decorators, typename T> 639 typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type 640 RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::_arraycopy_func = &arraycopy_init; 641 642 template <DecoratorSet decorators, typename T> 643 typename AccessFunction<decorators, T, BARRIER_CLONE>::type 644 RuntimeDispatch<decorators, T, BARRIER_CLONE>::_clone_func = &clone_init; 645 646 template <DecoratorSet decorators, typename T> 647 typename AccessFunction<decorators, T, BARRIER_RESOLVE>::type 648 RuntimeDispatch<decorators, T, BARRIER_RESOLVE>::_resolve_func = &resolve_init; 649 650 template <DecoratorSet decorators, typename T> 651 typename AccessFunction<decorators, T, BARRIER_EQUALS>::type 652 RuntimeDispatch<decorators, T, BARRIER_EQUALS>::_equals_func = &equals_init; 653 654 // Step 3: Pre-runtime dispatching. 655 // The PreRuntimeDispatch class is responsible for filtering the barrier strength 656 // decorators. That is, for AS_RAW, it hardwires the accesses without a runtime 657 // dispatch point. Otherwise it goes through a runtime check if hardwiring was 658 // not possible. 659 struct PreRuntimeDispatch: AllStatic { 660 template<DecoratorSet decorators> 661 struct CanHardwireRaw: public IntegralConstant< 662 bool, 663 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || // primitive access 664 !HasDecorator<decorators, INTERNAL_CONVERT_COMPRESSED_OOP>::value || // don't care about compressed oops (oop* address) 665 HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value> // we can infer we use compressed oops (narrowOop* address) 666 {}; 667 668 static const DecoratorSet convert_compressed_oops = INTERNAL_RT_USE_COMPRESSED_OOPS | INTERNAL_CONVERT_COMPRESSED_OOP; 669 670 template<DecoratorSet decorators> 671 static bool is_hardwired_primitive() { 672 return !HasDecorator<decorators, INTERNAL_BT_BARRIER_ON_PRIMITIVES>::value && 673 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value; 674 } 675 676 template <DecoratorSet decorators, typename T> 677 inline static typename EnableIf< 678 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value>::type 679 store(void* addr, T value) { 680 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw; 681 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) { 682 Raw::oop_store(addr, value); 683 } else { 684 Raw::store(addr, value); 685 } 686 } 687 688 template <DecoratorSet decorators, typename T> 689 inline static typename EnableIf< 690 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value>::type 691 store(void* addr, T value) { 692 if (UseCompressedOops) { 693 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; 694 PreRuntimeDispatch::store<expanded_decorators>(addr, value); 695 } else { 696 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; 697 PreRuntimeDispatch::store<expanded_decorators>(addr, value); 698 } 699 } 700 701 template <DecoratorSet decorators, typename T> 702 inline static typename EnableIf< 703 !HasDecorator<decorators, AS_RAW>::value>::type 704 store(void* addr, T value) { 705 if (is_hardwired_primitive<decorators>()) { 706 const DecoratorSet expanded_decorators = decorators | AS_RAW; 707 PreRuntimeDispatch::store<expanded_decorators>(addr, value); 708 } else { 709 RuntimeDispatch<decorators, T, BARRIER_STORE>::store(addr, value); 710 } 711 } 712 713 template <DecoratorSet decorators, typename T> 714 inline static typename EnableIf< 715 HasDecorator<decorators, AS_RAW>::value>::type 716 store_at(oop base, ptrdiff_t offset, T value) { 717 store<decorators>(field_addr(base, offset), value); 718 } 719 720 template <DecoratorSet decorators, typename T> 721 inline static typename EnableIf< 722 !HasDecorator<decorators, AS_RAW>::value>::type 723 store_at(oop base, ptrdiff_t offset, T value) { 724 if (is_hardwired_primitive<decorators>()) { 725 const DecoratorSet expanded_decorators = decorators | AS_RAW; 726 PreRuntimeDispatch::store_at<expanded_decorators>(base, offset, value); 727 } else { 728 RuntimeDispatch<decorators, T, BARRIER_STORE_AT>::store_at(base, offset, value); 729 } 730 } 731 732 template <DecoratorSet decorators, typename T> 733 inline static typename EnableIf< 734 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, T>::type 735 load(void* addr) { 736 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw; 737 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) { 738 return Raw::template oop_load<T>(addr); 739 } else { 740 return Raw::template load<T>(addr); 741 } 742 } 743 744 template <DecoratorSet decorators, typename T> 745 inline static typename EnableIf< 746 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, T>::type 747 load(void* addr) { 748 if (UseCompressedOops) { 749 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; 750 return PreRuntimeDispatch::load<expanded_decorators, T>(addr); 751 } else { 752 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; 753 return PreRuntimeDispatch::load<expanded_decorators, T>(addr); 754 } 755 } 756 757 template <DecoratorSet decorators, typename T> 758 inline static typename EnableIf< 759 !HasDecorator<decorators, AS_RAW>::value, T>::type 760 load(void* addr) { 761 if (is_hardwired_primitive<decorators>()) { 762 const DecoratorSet expanded_decorators = decorators | AS_RAW; 763 return PreRuntimeDispatch::load<expanded_decorators, T>(addr); 764 } else { 765 return RuntimeDispatch<decorators, T, BARRIER_LOAD>::load(addr); 766 } 767 } 768 769 template <DecoratorSet decorators, typename T> 770 inline static typename EnableIf< 771 HasDecorator<decorators, AS_RAW>::value, T>::type 772 load_at(oop base, ptrdiff_t offset) { 773 return load<decorators, T>(field_addr(base, offset)); 774 } 775 776 template <DecoratorSet decorators, typename T> 777 inline static typename EnableIf< 778 !HasDecorator<decorators, AS_RAW>::value, T>::type 779 load_at(oop base, ptrdiff_t offset) { 780 if (is_hardwired_primitive<decorators>()) { 781 const DecoratorSet expanded_decorators = decorators | AS_RAW; 782 return PreRuntimeDispatch::load_at<expanded_decorators, T>(base, offset); 783 } else { 784 return RuntimeDispatch<decorators, T, BARRIER_LOAD_AT>::load_at(base, offset); 785 } 786 } 787 788 template <DecoratorSet decorators, typename T> 789 inline static typename EnableIf< 790 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, T>::type 791 atomic_cmpxchg(T new_value, void* addr, T compare_value) { 792 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw; 793 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) { 794 return Raw::oop_atomic_cmpxchg(new_value, addr, compare_value); 795 } else { 796 return Raw::atomic_cmpxchg(new_value, addr, compare_value); 797 } 798 } 799 800 template <DecoratorSet decorators, typename T> 801 inline static typename EnableIf< 802 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, T>::type 803 atomic_cmpxchg(T new_value, void* addr, T compare_value) { 804 if (UseCompressedOops) { 805 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; 806 return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(new_value, addr, compare_value); 807 } else { 808 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; 809 return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(new_value, addr, compare_value); 810 } 811 } 812 813 template <DecoratorSet decorators, typename T> 814 inline static typename EnableIf< 815 !HasDecorator<decorators, AS_RAW>::value, T>::type 816 atomic_cmpxchg(T new_value, void* addr, T compare_value) { 817 if (is_hardwired_primitive<decorators>()) { 818 const DecoratorSet expanded_decorators = decorators | AS_RAW; 819 return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(new_value, addr, compare_value); 820 } else { 821 return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG>::atomic_cmpxchg(new_value, addr, compare_value); 822 } 823 } 824 825 template <DecoratorSet decorators, typename T> 826 inline static typename EnableIf< 827 HasDecorator<decorators, AS_RAW>::value, T>::type 828 atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) { 829 return atomic_cmpxchg<decorators>(new_value, field_addr(base, offset), compare_value); 830 } 831 832 template <DecoratorSet decorators, typename T> 833 inline static typename EnableIf< 834 !HasDecorator<decorators, AS_RAW>::value, T>::type 835 atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) { 836 if (is_hardwired_primitive<decorators>()) { 837 const DecoratorSet expanded_decorators = decorators | AS_RAW; 838 return PreRuntimeDispatch::atomic_cmpxchg_at<expanded_decorators>(new_value, base, offset, compare_value); 839 } else { 840 return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::atomic_cmpxchg_at(new_value, base, offset, compare_value); 841 } 842 } 843 844 template <DecoratorSet decorators, typename T> 845 inline static typename EnableIf< 846 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, T>::type 847 atomic_xchg(T new_value, void* addr) { 848 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw; 849 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) { 850 return Raw::oop_atomic_xchg(new_value, addr); 851 } else { 852 return Raw::atomic_xchg(new_value, addr); 853 } 854 } 855 856 template <DecoratorSet decorators, typename T> 857 inline static typename EnableIf< 858 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, T>::type 859 atomic_xchg(T new_value, void* addr) { 860 if (UseCompressedOops) { 861 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; 862 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, addr); 863 } else { 864 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; 865 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, addr); 866 } 867 } 868 869 template <DecoratorSet decorators, typename T> 870 inline static typename EnableIf< 871 !HasDecorator<decorators, AS_RAW>::value, T>::type 872 atomic_xchg(T new_value, void* addr) { 873 if (is_hardwired_primitive<decorators>()) { 874 const DecoratorSet expanded_decorators = decorators | AS_RAW; 875 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, addr); 876 } else { 877 return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG>::atomic_xchg(new_value, addr); 878 } 879 } 880 881 template <DecoratorSet decorators, typename T> 882 inline static typename EnableIf< 883 HasDecorator<decorators, AS_RAW>::value, T>::type 884 atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) { 885 return atomic_xchg<decorators>(new_value, field_addr(base, offset)); 886 } 887 888 template <DecoratorSet decorators, typename T> 889 inline static typename EnableIf< 890 !HasDecorator<decorators, AS_RAW>::value, T>::type 891 atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) { 892 if (is_hardwired_primitive<decorators>()) { 893 const DecoratorSet expanded_decorators = decorators | AS_RAW; 894 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, base, offset); 895 } else { 896 return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::atomic_xchg_at(new_value, base, offset); 897 } 898 } 899 900 template <DecoratorSet decorators, typename T> 901 inline static typename EnableIf< 902 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, bool>::type 903 arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { 904 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw; 905 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) { 906 return Raw::oop_arraycopy(src_obj, dst_obj, src, dst, length); 907 } else { 908 return Raw::arraycopy(src_obj, dst_obj, src, dst, length); 909 } 910 } 911 912 template <DecoratorSet decorators, typename T> 913 inline static typename EnableIf< 914 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, bool>::type 915 arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { 916 if (UseCompressedOops) { 917 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; 918 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, dst_obj, src, dst, length); 919 } else { 920 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; 921 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, dst_obj, src, dst, length); 922 } 923 } 924 925 template <DecoratorSet decorators, typename T> 926 inline static typename EnableIf< 927 !HasDecorator<decorators, AS_RAW>::value, bool>::type 928 arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { 929 if (is_hardwired_primitive<decorators>()) { 930 const DecoratorSet expanded_decorators = decorators | AS_RAW; 931 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, dst_obj, src, dst, length); 932 } else { 933 return RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::arraycopy(src_obj, dst_obj, src, dst, length); 934 } 935 } 936 937 template <DecoratorSet decorators> 938 inline static typename EnableIf< 939 HasDecorator<decorators, AS_RAW>::value>::type 940 clone(oop src, oop dst, size_t size) { 941 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw; 942 Raw::clone(src, dst, size); 943 } 944 945 template <DecoratorSet decorators> 946 inline static typename EnableIf< 947 !HasDecorator<decorators, AS_RAW>::value>::type 948 clone(oop src, oop dst, size_t size) { 949 RuntimeDispatch<decorators, oop, BARRIER_CLONE>::clone(src, dst, size); 950 } 951 952 template <DecoratorSet decorators> 953 inline static typename EnableIf< 954 HasDecorator<decorators, INTERNAL_BT_TO_SPACE_INVARIANT>::value, oop>::type 955 resolve(oop obj) { 956 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw; 957 return Raw::resolve(obj); 958 } 959 960 template <DecoratorSet decorators> 961 inline static typename EnableIf< 962 !HasDecorator<decorators, INTERNAL_BT_TO_SPACE_INVARIANT>::value, oop>::type 963 resolve(oop obj) { 964 return RuntimeDispatch<decorators, oop, BARRIER_RESOLVE>::resolve(obj); 965 } 966 967 template <DecoratorSet decorators> 968 inline static typename EnableIf< 969 HasDecorator<decorators, INTERNAL_BT_TO_SPACE_INVARIANT>::value, bool>::type 970 equals(oop o1, oop o2) { 971 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw; 972 return Raw::equals(o1, o2); 973 } 974 975 template <DecoratorSet decorators> 976 inline static typename EnableIf< 977 !HasDecorator<decorators, INTERNAL_BT_TO_SPACE_INVARIANT>::value, bool>::type 978 equals(oop o1, oop o2) { 979 return RuntimeDispatch<decorators, oop, BARRIER_EQUALS>::equals(o1, o2); 980 } 981 }; 982 983 // This class adds implied decorators that follow according to decorator rules. 984 // For example adding default reference strength and default memory ordering 985 // semantics. 986 template <DecoratorSet input_decorators> 987 struct DecoratorFixup: AllStatic { 988 // If no reference strength has been picked, then strong will be picked 989 static const DecoratorSet ref_strength_default = input_decorators | 990 (((ON_DECORATOR_MASK & input_decorators) == 0 && (INTERNAL_VALUE_IS_OOP & input_decorators) != 0) ? 991 ON_STRONG_OOP_REF : INTERNAL_EMPTY); 992 // If no memory ordering has been picked, unordered will be picked 993 static const DecoratorSet memory_ordering_default = ref_strength_default | 994 ((MO_DECORATOR_MASK & ref_strength_default) == 0 ? MO_UNORDERED : INTERNAL_EMPTY); 995 // If no barrier strength has been picked, normal will be used 996 static const DecoratorSet barrier_strength_default = memory_ordering_default | 997 ((AS_DECORATOR_MASK & memory_ordering_default) == 0 ? AS_NORMAL : INTERNAL_EMPTY); 998 // Heap array accesses imply it is a heap access 999 static const DecoratorSet heap_array_is_in_heap = barrier_strength_default | 1000 ((IN_HEAP_ARRAY & barrier_strength_default) != 0 ? IN_HEAP : INTERNAL_EMPTY); 1001 static const DecoratorSet conc_root_is_root = heap_array_is_in_heap | 1002 ((IN_CONCURRENT_ROOT & heap_array_is_in_heap) != 0 ? IN_ROOT : INTERNAL_EMPTY); 1003 static const DecoratorSet archive_root_is_root = conc_root_is_root | 1004 ((IN_ARCHIVE_ROOT & conc_root_is_root) != 0 ? IN_ROOT : INTERNAL_EMPTY); 1005 static const DecoratorSet value = archive_root_is_root | BT_BUILDTIME_DECORATORS; 1006 }; 1007 1008 // Step 2: Reduce types. 1009 // Enforce that for non-oop types, T and P have to be strictly the same. 1010 // P is the type of the address and T is the type of the values. 1011 // As for oop types, it is allow to send T in {narrowOop, oop} and 1012 // P in {narrowOop, oop, HeapWord*}. The following rules apply according to 1013 // the subsequent table. (columns are P, rows are T) 1014 // | | HeapWord | oop | narrowOop | 1015 // | oop | rt-comp | hw-none | hw-comp | 1016 // | narrowOop | x | x | hw-none | 1017 // 1018 // x means not allowed 1019 // rt-comp means it must be checked at runtime whether the oop is compressed. 1020 // hw-none means it is statically known the oop will not be compressed. 1021 // hw-comp means it is statically known the oop will be compressed. 1022 1023 template <DecoratorSet decorators, typename T> 1024 inline void store_reduce_types(T* addr, T value) { 1025 PreRuntimeDispatch::store<decorators>(addr, value); 1026 } 1027 1028 template <DecoratorSet decorators> 1029 inline void store_reduce_types(narrowOop* addr, oop value) { 1030 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | 1031 INTERNAL_RT_USE_COMPRESSED_OOPS; 1032 PreRuntimeDispatch::store<expanded_decorators>(addr, value); 1033 } 1034 1035 template <DecoratorSet decorators> 1036 inline void store_reduce_types(narrowOop* addr, narrowOop value) { 1037 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | 1038 INTERNAL_RT_USE_COMPRESSED_OOPS; 1039 PreRuntimeDispatch::store<expanded_decorators>(addr, value); 1040 } 1041 1042 template <DecoratorSet decorators> 1043 inline void store_reduce_types(HeapWord* addr, oop value) { 1044 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; 1045 PreRuntimeDispatch::store<expanded_decorators>(addr, value); 1046 } 1047 1048 template <DecoratorSet decorators, typename T> 1049 inline T atomic_cmpxchg_reduce_types(T new_value, T* addr, T compare_value) { 1050 return PreRuntimeDispatch::atomic_cmpxchg<decorators>(new_value, addr, compare_value); 1051 } 1052 1053 template <DecoratorSet decorators> 1054 inline oop atomic_cmpxchg_reduce_types(oop new_value, narrowOop* addr, oop compare_value) { 1055 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | 1056 INTERNAL_RT_USE_COMPRESSED_OOPS; 1057 return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(new_value, addr, compare_value); 1058 } 1059 1060 template <DecoratorSet decorators> 1061 inline narrowOop atomic_cmpxchg_reduce_types(narrowOop new_value, narrowOop* addr, narrowOop compare_value) { 1062 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | 1063 INTERNAL_RT_USE_COMPRESSED_OOPS; 1064 return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(new_value, addr, compare_value); 1065 } 1066 1067 template <DecoratorSet decorators> 1068 inline oop atomic_cmpxchg_reduce_types(oop new_value, 1069 HeapWord* addr, 1070 oop compare_value) { 1071 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; 1072 return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(new_value, addr, compare_value); 1073 } 1074 1075 template <DecoratorSet decorators, typename T> 1076 inline T atomic_xchg_reduce_types(T new_value, T* addr) { 1077 const DecoratorSet expanded_decorators = decorators; 1078 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, addr); 1079 } 1080 1081 template <DecoratorSet decorators> 1082 inline oop atomic_xchg_reduce_types(oop new_value, narrowOop* addr) { 1083 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | 1084 INTERNAL_RT_USE_COMPRESSED_OOPS; 1085 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, addr); 1086 } 1087 1088 template <DecoratorSet decorators> 1089 inline narrowOop atomic_xchg_reduce_types(narrowOop new_value, narrowOop* addr) { 1090 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | 1091 INTERNAL_RT_USE_COMPRESSED_OOPS; 1092 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, addr); 1093 } 1094 1095 template <DecoratorSet decorators> 1096 inline oop atomic_xchg_reduce_types(oop new_value, HeapWord* addr) { 1097 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; 1098 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, addr); 1099 } 1100 1101 template <DecoratorSet decorators, typename T> 1102 inline T load_reduce_types(T* addr) { 1103 return PreRuntimeDispatch::load<decorators, T>(addr); 1104 } 1105 1106 template <DecoratorSet decorators, typename T> 1107 inline typename OopOrNarrowOop<T>::type load_reduce_types(narrowOop* addr) { 1108 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | 1109 INTERNAL_RT_USE_COMPRESSED_OOPS; 1110 return PreRuntimeDispatch::load<expanded_decorators, typename OopOrNarrowOop<T>::type>(addr); 1111 } 1112 1113 template <DecoratorSet decorators, typename T> 1114 inline oop load_reduce_types(HeapWord* addr) { 1115 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; 1116 return PreRuntimeDispatch::load<expanded_decorators, oop>(addr); 1117 } 1118 1119 template <DecoratorSet decorators, typename T> 1120 inline bool arraycopy_reduce_types(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { 1121 return PreRuntimeDispatch::arraycopy<decorators>(src_obj, dst_obj, src, dst, length); 1122 } 1123 1124 template <DecoratorSet decorators> 1125 inline bool arraycopy_reduce_types(arrayOop src_obj, arrayOop dst_obj, HeapWord* src, HeapWord* dst, size_t length) { 1126 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; 1127 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, dst_obj, src, dst, length); 1128 } 1129 1130 template <DecoratorSet decorators> 1131 inline bool arraycopy_reduce_types(arrayOop src_obj, arrayOop dst_obj, narrowOop* src, narrowOop* dst, size_t length) { 1132 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | 1133 INTERNAL_RT_USE_COMPRESSED_OOPS; 1134 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, dst_obj, src, dst, length); 1135 } 1136 1137 // Step 1: Set default decorators. This step remembers if a type was volatile 1138 // and then sets the MO_VOLATILE decorator by default. Otherwise, a default 1139 // memory ordering is set for the access, and the implied decorator rules 1140 // are applied to select sensible defaults for decorators that have not been 1141 // explicitly set. For example, default object referent strength is set to strong. 1142 // This step also decays the types passed in (e.g. getting rid of CV qualifiers 1143 // and references from the types). This step also perform some type verification 1144 // that the passed in types make sense. 1145 1146 template <DecoratorSet decorators, typename T> 1147 static void verify_types(){ 1148 // If this fails to compile, then you have sent in something that is 1149 // not recognized as a valid primitive type to a primitive Access function. 1150 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || // oops have already been validated 1151 (IsPointer<T>::value || IsIntegral<T>::value) || 1152 IsFloatingPoint<T>::value)); // not allowed primitive type 1153 } 1154 1155 template <DecoratorSet decorators, typename P, typename T> 1156 inline void store(P* addr, T value) { 1157 verify_types<decorators, T>(); 1158 typedef typename Decay<P>::type DecayedP; 1159 typedef typename Decay<T>::type DecayedT; 1160 DecayedT decayed_value = value; 1161 // If a volatile address is passed in but no memory ordering decorator, 1162 // set the memory ordering to MO_VOLATILE by default. 1163 const DecoratorSet expanded_decorators = DecoratorFixup< 1164 (IsVolatile<P>::value && !HasDecorator<decorators, MO_DECORATOR_MASK>::value) ? 1165 (MO_VOLATILE | decorators) : decorators>::value; 1166 store_reduce_types<expanded_decorators>(const_cast<DecayedP*>(addr), decayed_value); 1167 } 1168 1169 template <DecoratorSet decorators, typename T> 1170 inline void store_at(oop base, ptrdiff_t offset, T value) { 1171 verify_types<decorators, T>(); 1172 typedef typename Decay<T>::type DecayedT; 1173 DecayedT decayed_value = value; 1174 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | 1175 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ? 1176 INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY)>::value; 1177 PreRuntimeDispatch::store_at<expanded_decorators>(base, offset, decayed_value); 1178 } 1179 1180 template <DecoratorSet decorators, typename P, typename T> 1181 inline T load(P* addr) { 1182 verify_types<decorators, T>(); 1183 typedef typename Decay<P>::type DecayedP; 1184 typedef typename Conditional<HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value, 1185 typename OopOrNarrowOop<T>::type, 1186 typename Decay<T>::type>::type DecayedT; 1187 // If a volatile address is passed in but no memory ordering decorator, 1188 // set the memory ordering to MO_VOLATILE by default. 1189 const DecoratorSet expanded_decorators = DecoratorFixup< 1190 (IsVolatile<P>::value && !HasDecorator<decorators, MO_DECORATOR_MASK>::value) ? 1191 (MO_VOLATILE | decorators) : decorators>::value; 1192 return load_reduce_types<expanded_decorators, DecayedT>(const_cast<DecayedP*>(addr)); 1193 } 1194 1195 template <DecoratorSet decorators, typename T> 1196 inline T load_at(oop base, ptrdiff_t offset) { 1197 verify_types<decorators, T>(); 1198 typedef typename Conditional<HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value, 1199 typename OopOrNarrowOop<T>::type, 1200 typename Decay<T>::type>::type DecayedT; 1201 // Expand the decorators (figure out sensible defaults) 1202 // Potentially remember if we need compressed oop awareness 1203 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | 1204 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ? 1205 INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY)>::value; 1206 return PreRuntimeDispatch::load_at<expanded_decorators, DecayedT>(base, offset); 1207 } 1208 1209 template <DecoratorSet decorators, typename P, typename T> 1210 inline T atomic_cmpxchg(T new_value, P* addr, T compare_value) { 1211 verify_types<decorators, T>(); 1212 typedef typename Decay<P>::type DecayedP; 1213 typedef typename Decay<T>::type DecayedT; 1214 DecayedT new_decayed_value = new_value; 1215 DecayedT compare_decayed_value = compare_value; 1216 const DecoratorSet expanded_decorators = DecoratorFixup< 1217 (!HasDecorator<decorators, MO_DECORATOR_MASK>::value) ? 1218 (MO_SEQ_CST | decorators) : decorators>::value; 1219 return atomic_cmpxchg_reduce_types<expanded_decorators>(new_decayed_value, 1220 const_cast<DecayedP*>(addr), 1221 compare_decayed_value); 1222 } 1223 1224 template <DecoratorSet decorators, typename T> 1225 inline T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) { 1226 verify_types<decorators, T>(); 1227 typedef typename Decay<T>::type DecayedT; 1228 DecayedT new_decayed_value = new_value; 1229 DecayedT compare_decayed_value = compare_value; 1230 // Determine default memory ordering 1231 const DecoratorSet expanded_decorators = DecoratorFixup< 1232 (!HasDecorator<decorators, MO_DECORATOR_MASK>::value) ? 1233 (MO_SEQ_CST | decorators) : decorators>::value; 1234 // Potentially remember that we need compressed oop awareness 1235 const DecoratorSet final_decorators = expanded_decorators | 1236 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ? 1237 INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY); 1238 return PreRuntimeDispatch::atomic_cmpxchg_at<final_decorators>(new_decayed_value, base, 1239 offset, compare_decayed_value); 1240 } 1241 1242 template <DecoratorSet decorators, typename P, typename T> 1243 inline T atomic_xchg(T new_value, P* addr) { 1244 verify_types<decorators, T>(); 1245 typedef typename Decay<P>::type DecayedP; 1246 typedef typename Decay<T>::type DecayedT; 1247 DecayedT new_decayed_value = new_value; 1248 // atomic_xchg is only available in SEQ_CST flavour. 1249 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST>::value; 1250 return atomic_xchg_reduce_types<expanded_decorators>(new_decayed_value, 1251 const_cast<DecayedP*>(addr)); 1252 } 1253 1254 template <DecoratorSet decorators, typename T> 1255 inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) { 1256 verify_types<decorators, T>(); 1257 typedef typename Decay<T>::type DecayedT; 1258 DecayedT new_decayed_value = new_value; 1259 // atomic_xchg is only available in SEQ_CST flavour. 1260 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST | 1261 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ? 1262 INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY)>::value; 1263 return PreRuntimeDispatch::atomic_xchg_at<expanded_decorators>(new_decayed_value, base, offset); 1264 } 1265 1266 template <DecoratorSet decorators, typename T> 1267 inline bool arraycopy(arrayOop src_obj, arrayOop dst_obj, T* src, T* dst, size_t length) { 1268 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || 1269 (IsSame<T, void>::value || IsIntegral<T>::value) || 1270 IsFloatingPoint<T>::value)); // arraycopy allows type erased void elements 1271 typedef typename Decay<T>::type DecayedT; 1272 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | IN_HEAP_ARRAY | IN_HEAP>::value; 1273 return arraycopy_reduce_types<expanded_decorators>(src_obj, dst_obj, 1274 const_cast<DecayedT*>(src), 1275 const_cast<DecayedT*>(dst), 1276 length); 1277 } 1278 1279 template <DecoratorSet decorators> 1280 inline void clone(oop src, oop dst, size_t size) { 1281 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value; 1282 PreRuntimeDispatch::clone<expanded_decorators>(src, dst, size); 1283 } 1284 1285 template <DecoratorSet decorators> 1286 inline oop resolve(oop obj) { 1287 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value; 1288 return PreRuntimeDispatch::resolve<expanded_decorators>(obj); 1289 } 1290 1291 template <DecoratorSet decorators> 1292 inline bool equals(oop o1, oop o2) { 1293 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value; 1294 return PreRuntimeDispatch::equals<expanded_decorators>(o1, o2); 1295 } 1296 1297 // Infer the type that should be returned from an Access::oop_load. 1298 template <typename P, DecoratorSet decorators> 1299 class OopLoadProxy: public StackObj { 1300 private: 1301 P *const _addr; 1302 public: 1303 OopLoadProxy(P* addr) : _addr(addr) {} 1304 1305 inline operator oop() { 1306 return load<decorators | INTERNAL_VALUE_IS_OOP, P, oop>(_addr); 1307 } 1308 1309 inline operator narrowOop() { 1310 return load<decorators | INTERNAL_VALUE_IS_OOP, P, narrowOop>(_addr); 1311 } 1312 1313 template <typename T> 1314 inline bool operator ==(const T& other) const { 1315 return load<decorators | INTERNAL_VALUE_IS_OOP, P, T>(_addr) == other; 1316 } 1317 1318 template <typename T> 1319 inline bool operator !=(const T& other) const { 1320 return load<decorators | INTERNAL_VALUE_IS_OOP, P, T>(_addr) != other; 1321 } 1322 }; 1323 1324 // Infer the type that should be returned from an Access::load_at. 1325 template <DecoratorSet decorators> 1326 class LoadAtProxy: public StackObj { 1327 private: 1328 const oop _base; 1329 const ptrdiff_t _offset; 1330 public: 1331 LoadAtProxy(oop base, ptrdiff_t offset) : _base(base), _offset(offset) {} 1332 1333 template <typename T> 1334 inline operator T() const { 1335 return load_at<decorators, T>(_base, _offset); 1336 } 1337 1338 template <typename T> 1339 inline bool operator ==(const T& other) const { return load_at<decorators, T>(_base, _offset) == other; } 1340 1341 template <typename T> 1342 inline bool operator !=(const T& other) const { return load_at<decorators, T>(_base, _offset) != other; } 1343 }; 1344 1345 // Infer the type that should be returned from an Access::oop_load_at. 1346 template <DecoratorSet decorators> 1347 class OopLoadAtProxy: public StackObj { 1348 private: 1349 const oop _base; 1350 const ptrdiff_t _offset; 1351 public: 1352 OopLoadAtProxy(oop base, ptrdiff_t offset) : _base(base), _offset(offset) {} 1353 1354 inline operator oop() const { 1355 return load_at<decorators | INTERNAL_VALUE_IS_OOP, oop>(_base, _offset); 1356 } 1357 1358 inline operator narrowOop() const { 1359 return load_at<decorators | INTERNAL_VALUE_IS_OOP, narrowOop>(_base, _offset); 1360 } 1361 1362 template <typename T> 1363 inline bool operator ==(const T& other) const { 1364 return load_at<decorators | INTERNAL_VALUE_IS_OOP, T>(_base, _offset) == other; 1365 } 1366 1367 template <typename T> 1368 inline bool operator !=(const T& other) const { 1369 return load_at<decorators | INTERNAL_VALUE_IS_OOP, T>(_base, _offset) != other; 1370 } 1371 }; 1372 } 1373 1374 #endif // SHARE_OOPS_ACCESSBACKEND_HPP