1 /* 2 * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #ifndef SHARE_OOPS_ACCESSBACKEND_HPP 26 #define SHARE_OOPS_ACCESSBACKEND_HPP 27 28 #include "gc/shared/barrierSetConfig.hpp" 29 #include "memory/allocation.hpp" 30 #include "metaprogramming/conditional.hpp" 31 #include "metaprogramming/decay.hpp" 32 #include "metaprogramming/enableIf.hpp" 33 #include "metaprogramming/integralConstant.hpp" 34 #include "metaprogramming/isFloatingPoint.hpp" 35 #include "metaprogramming/isIntegral.hpp" 36 #include "metaprogramming/isPointer.hpp" 37 #include "metaprogramming/isSame.hpp" 38 #include "metaprogramming/isVolatile.hpp" 39 #include "oops/accessDecorators.hpp" 40 #include "oops/oopsHierarchy.hpp" 41 #include "utilities/debug.hpp" 42 #include "utilities/globalDefinitions.hpp" 43 44 45 // This metafunction returns either oop or narrowOop depending on whether 46 // an access needs to use compressed oops or not. 47 template <DecoratorSet decorators> 48 struct HeapOopType: AllStatic { 49 static const bool needs_oop_compress = HasDecorator<decorators, INTERNAL_CONVERT_COMPRESSED_OOP>::value && 50 HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value; 51 typedef typename Conditional<needs_oop_compress, narrowOop, oop>::type type; 52 }; 53 54 namespace AccessInternal { 55 enum BarrierType { 56 BARRIER_STORE, 57 BARRIER_STORE_AT, 58 BARRIER_LOAD, 59 BARRIER_LOAD_AT, 60 BARRIER_ATOMIC_CMPXCHG, 61 BARRIER_ATOMIC_CMPXCHG_AT, 62 BARRIER_ATOMIC_XCHG, 63 BARRIER_ATOMIC_XCHG_AT, 64 BARRIER_ARRAYCOPY, 65 BARRIER_CLONE, 66 BARRIER_RESOLVE, 67 BARRIER_EQUALS 68 }; 69 70 template <DecoratorSet decorators, typename T> 71 struct MustConvertCompressedOop: public IntegralConstant<bool, 72 HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value && 73 IsSame<typename HeapOopType<decorators>::type, narrowOop>::value && 74 IsSame<T, oop>::value> {}; 75 76 // This metafunction returns an appropriate oop type if the value is oop-like 77 // and otherwise returns the same type T. 78 template <DecoratorSet decorators, typename T> 79 struct EncodedType: AllStatic { 80 typedef typename Conditional< 81 HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value, 82 typename HeapOopType<decorators>::type, T>::type type; 83 }; 84 85 template <DecoratorSet decorators> 86 inline typename HeapOopType<decorators>::type* 87 oop_field_addr(oop base, ptrdiff_t byte_offset) { 88 return reinterpret_cast<typename HeapOopType<decorators>::type*>( 89 reinterpret_cast<intptr_t>((void*)base) + byte_offset); 90 } 91 92 // This metafunction returns whether it is possible for a type T to require 93 // locking to support wide atomics or not. 94 template <typename T> 95 #ifdef SUPPORTS_NATIVE_CX8 96 struct PossiblyLockedAccess: public IntegralConstant<bool, false> {}; 97 #else 98 struct PossiblyLockedAccess: public IntegralConstant<bool, (sizeof(T) > 4)> {}; 99 #endif 100 101 template <DecoratorSet decorators, typename T> 102 struct AccessFunctionTypes { 103 typedef T (*load_at_func_t)(oop base, ptrdiff_t offset); 104 typedef void (*store_at_func_t)(oop base, ptrdiff_t offset, T value); 105 typedef T (*atomic_cmpxchg_at_func_t)(T new_value, oop base, ptrdiff_t offset, T compare_value); 106 typedef T (*atomic_xchg_at_func_t)(T new_value, oop base, ptrdiff_t offset); 107 108 typedef T (*load_func_t)(void* addr); 109 typedef void (*store_func_t)(void* addr, T value); 110 typedef T (*atomic_cmpxchg_func_t)(T new_value, void* addr, T compare_value); 111 typedef T (*atomic_xchg_func_t)(T new_value, void* addr); 112 113 typedef bool (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, 114 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, 115 size_t length); 116 typedef void (*clone_func_t)(oop src, oop dst, size_t size); 117 typedef oop (*resolve_func_t)(oop obj); 118 typedef bool (*equals_func_t)(oop o1, oop o2); 119 }; 120 121 template <DecoratorSet decorators> 122 struct AccessFunctionTypes<decorators, void> { 123 typedef bool (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, void* src, 124 arrayOop dst_obj, size_t dst_offset_in_bytes, void* dst, 125 size_t length); 126 }; 127 128 template <DecoratorSet decorators, typename T, BarrierType barrier> struct AccessFunction {}; 129 130 #define ACCESS_GENERATE_ACCESS_FUNCTION(bt, func) \ 131 template <DecoratorSet decorators, typename T> \ 132 struct AccessFunction<decorators, T, bt>: AllStatic{ \ 133 typedef typename AccessFunctionTypes<decorators, T>::func type; \ 134 } 135 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE, store_func_t); 136 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE_AT, store_at_func_t); 137 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD, load_func_t); 138 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD_AT, load_at_func_t); 139 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG, atomic_cmpxchg_func_t); 140 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG_AT, atomic_cmpxchg_at_func_t); 141 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG, atomic_xchg_func_t); 142 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG_AT, atomic_xchg_at_func_t); 143 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ARRAYCOPY, arraycopy_func_t); 144 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_CLONE, clone_func_t); 145 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_RESOLVE, resolve_func_t); 146 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_EQUALS, equals_func_t); 147 #undef ACCESS_GENERATE_ACCESS_FUNCTION 148 149 template <DecoratorSet decorators, typename T, BarrierType barrier_type> 150 typename AccessFunction<decorators, T, barrier_type>::type resolve_barrier(); 151 152 template <DecoratorSet decorators, typename T, BarrierType barrier_type> 153 typename AccessFunction<decorators, T, barrier_type>::type resolve_oop_barrier(); 154 155 class AccessLocker { 156 public: 157 AccessLocker(); 158 ~AccessLocker(); 159 }; 160 bool wide_atomic_needs_locking(); 161 162 void* field_addr(oop base, ptrdiff_t offset); 163 164 // Forward calls to Copy:: in the cpp file to reduce dependencies and allow 165 // faster build times, given how frequently included access is. 166 void arraycopy_arrayof_conjoint_oops(void* src, void* dst, size_t length); 167 void arraycopy_conjoint_oops(oop* src, oop* dst, size_t length); 168 void arraycopy_conjoint_oops(narrowOop* src, narrowOop* dst, size_t length); 169 170 void arraycopy_disjoint_words(void* src, void* dst, size_t length); 171 void arraycopy_disjoint_words_atomic(void* src, void* dst, size_t length); 172 173 template<typename T> 174 void arraycopy_conjoint(T* src, T* dst, size_t length); 175 template<typename T> 176 void arraycopy_arrayof_conjoint(T* src, T* dst, size_t length); 177 template<typename T> 178 void arraycopy_conjoint_atomic(T* src, T* dst, size_t length); 179 } 180 181 // This mask specifies what decorators are relevant for raw accesses. When passing 182 // accesses to the raw layer, irrelevant decorators are removed. 183 const DecoratorSet RAW_DECORATOR_MASK = INTERNAL_DECORATOR_MASK | MO_DECORATOR_MASK | 184 ARRAYCOPY_DECORATOR_MASK | IS_NOT_NULL; 185 186 // The RawAccessBarrier performs raw accesses with additional knowledge of 187 // memory ordering, so that OrderAccess/Atomic is called when necessary. 188 // It additionally handles compressed oops, and hence is not completely "raw" 189 // strictly speaking. 190 template <DecoratorSet decorators> 191 class RawAccessBarrier: public AllStatic { 192 protected: 193 static inline void* field_addr(oop base, ptrdiff_t byte_offset) { 194 return AccessInternal::field_addr(base, byte_offset); 195 } 196 197 protected: 198 // Only encode if INTERNAL_VALUE_IS_OOP 199 template <DecoratorSet idecorators, typename T> 200 static inline typename EnableIf< 201 AccessInternal::MustConvertCompressedOop<idecorators, T>::value, 202 typename HeapOopType<idecorators>::type>::type 203 encode_internal(T value); 204 205 template <DecoratorSet idecorators, typename T> 206 static inline typename EnableIf< 207 !AccessInternal::MustConvertCompressedOop<idecorators, T>::value, T>::type 208 encode_internal(T value) { 209 return value; 210 } 211 212 template <typename T> 213 static inline typename AccessInternal::EncodedType<decorators, T>::type 214 encode(T value) { 215 return encode_internal<decorators, T>(value); 216 } 217 218 // Only decode if INTERNAL_VALUE_IS_OOP 219 template <DecoratorSet idecorators, typename T> 220 static inline typename EnableIf< 221 AccessInternal::MustConvertCompressedOop<idecorators, T>::value, T>::type 222 decode_internal(typename HeapOopType<idecorators>::type value); 223 224 template <DecoratorSet idecorators, typename T> 225 static inline typename EnableIf< 226 !AccessInternal::MustConvertCompressedOop<idecorators, T>::value, T>::type 227 decode_internal(T value) { 228 return value; 229 } 230 231 template <typename T> 232 static inline T decode(typename AccessInternal::EncodedType<decorators, T>::type value) { 233 return decode_internal<decorators, T>(value); 234 } 235 236 protected: 237 template <DecoratorSet ds, typename T> 238 static typename EnableIf< 239 HasDecorator<ds, MO_SEQ_CST>::value, T>::type 240 load_internal(void* addr); 241 242 template <DecoratorSet ds, typename T> 243 static typename EnableIf< 244 HasDecorator<ds, MO_ACQUIRE>::value, T>::type 245 load_internal(void* addr); 246 247 template <DecoratorSet ds, typename T> 248 static typename EnableIf< 249 HasDecorator<ds, MO_RELAXED>::value, T>::type 250 load_internal(void* addr); 251 252 template <DecoratorSet ds, typename T> 253 static inline typename EnableIf< 254 HasDecorator<ds, MO_VOLATILE>::value, T>::type 255 load_internal(void* addr) { 256 return *reinterpret_cast<const volatile T*>(addr); 257 } 258 259 template <DecoratorSet ds, typename T> 260 static inline typename EnableIf< 261 HasDecorator<ds, MO_UNORDERED>::value, T>::type 262 load_internal(void* addr) { 263 return *reinterpret_cast<T*>(addr); 264 } 265 266 template <DecoratorSet ds, typename T> 267 static typename EnableIf< 268 HasDecorator<ds, MO_SEQ_CST>::value>::type 269 store_internal(void* addr, T value); 270 271 template <DecoratorSet ds, typename T> 272 static typename EnableIf< 273 HasDecorator<ds, MO_RELEASE>::value>::type 274 store_internal(void* addr, T value); 275 276 template <DecoratorSet ds, typename T> 277 static typename EnableIf< 278 HasDecorator<ds, MO_RELAXED>::value>::type 279 store_internal(void* addr, T value); 280 281 template <DecoratorSet ds, typename T> 282 static inline typename EnableIf< 283 HasDecorator<ds, MO_VOLATILE>::value>::type 284 store_internal(void* addr, T value) { 285 (void)const_cast<T&>(*reinterpret_cast<volatile T*>(addr) = value); 286 } 287 288 template <DecoratorSet ds, typename T> 289 static inline typename EnableIf< 290 HasDecorator<ds, MO_UNORDERED>::value>::type 291 store_internal(void* addr, T value) { 292 *reinterpret_cast<T*>(addr) = value; 293 } 294 295 template <DecoratorSet ds, typename T> 296 static typename EnableIf< 297 HasDecorator<ds, MO_SEQ_CST>::value, T>::type 298 atomic_cmpxchg_internal(T new_value, void* addr, T compare_value); 299 300 template <DecoratorSet ds, typename T> 301 static typename EnableIf< 302 HasDecorator<ds, MO_RELAXED>::value, T>::type 303 atomic_cmpxchg_internal(T new_value, void* addr, T compare_value); 304 305 template <DecoratorSet ds, typename T> 306 static typename EnableIf< 307 HasDecorator<ds, MO_SEQ_CST>::value, T>::type 308 atomic_xchg_internal(T new_value, void* addr); 309 310 // The following *_locked mechanisms serve the purpose of handling atomic operations 311 // that are larger than a machine can handle, and then possibly opt for using 312 // a slower path using a mutex to perform the operation. 313 314 template <DecoratorSet ds, typename T> 315 static inline typename EnableIf< 316 !AccessInternal::PossiblyLockedAccess<T>::value, T>::type 317 atomic_cmpxchg_maybe_locked(T new_value, void* addr, T compare_value) { 318 return atomic_cmpxchg_internal<ds>(new_value, addr, compare_value); 319 } 320 321 template <DecoratorSet ds, typename T> 322 static typename EnableIf< 323 AccessInternal::PossiblyLockedAccess<T>::value, T>::type 324 atomic_cmpxchg_maybe_locked(T new_value, void* addr, T compare_value); 325 326 template <DecoratorSet ds, typename T> 327 static inline typename EnableIf< 328 !AccessInternal::PossiblyLockedAccess<T>::value, T>::type 329 atomic_xchg_maybe_locked(T new_value, void* addr) { 330 return atomic_xchg_internal<ds>(new_value, addr); 331 } 332 333 template <DecoratorSet ds, typename T> 334 static typename EnableIf< 335 AccessInternal::PossiblyLockedAccess<T>::value, T>::type 336 atomic_xchg_maybe_locked(T new_value, void* addr); 337 338 public: 339 template <typename T> 340 static inline void store(void* addr, T value) { 341 store_internal<decorators>(addr, value); 342 } 343 344 template <typename T> 345 static inline T load(void* addr) { 346 return load_internal<decorators, T>(addr); 347 } 348 349 template <typename T> 350 static inline T atomic_cmpxchg(T new_value, void* addr, T compare_value) { 351 return atomic_cmpxchg_maybe_locked<decorators>(new_value, addr, compare_value); 352 } 353 354 template <typename T> 355 static inline T atomic_xchg(T new_value, void* addr) { 356 return atomic_xchg_maybe_locked<decorators>(new_value, addr); 357 } 358 359 template <typename T> 360 static bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, 361 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, 362 size_t length); 363 364 template <typename T> 365 static void oop_store(void* addr, T value); 366 template <typename T> 367 static void oop_store_at(oop base, ptrdiff_t offset, T value); 368 369 template <typename T> 370 static T oop_load(void* addr); 371 template <typename T> 372 static T oop_load_at(oop base, ptrdiff_t offset); 373 374 template <typename T> 375 static T oop_atomic_cmpxchg(T new_value, void* addr, T compare_value); 376 template <typename T> 377 static T oop_atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value); 378 379 template <typename T> 380 static T oop_atomic_xchg(T new_value, void* addr); 381 template <typename T> 382 static T oop_atomic_xchg_at(T new_value, oop base, ptrdiff_t offset); 383 384 template <typename T> 385 static void store_at(oop base, ptrdiff_t offset, T value) { 386 store(field_addr(base, offset), value); 387 } 388 389 template <typename T> 390 static T load_at(oop base, ptrdiff_t offset) { 391 return load<T>(field_addr(base, offset)); 392 } 393 394 template <typename T> 395 static T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) { 396 return atomic_cmpxchg(new_value, field_addr(base, offset), compare_value); 397 } 398 399 template <typename T> 400 static T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) { 401 return atomic_xchg(new_value, field_addr(base, offset)); 402 } 403 404 template <typename T> 405 static bool oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, 406 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, 407 size_t length); 408 409 static void clone(oop src, oop dst, size_t size); 410 411 static oop resolve(oop obj) { return obj; } 412 413 static bool equals(oop o1, oop o2) { return (void*)o1 == (void*)o2; } 414 }; 415 416 // Below is the implementation of the first 4 steps of the template pipeline: 417 // * Step 1: Set default decorators and decay types. This step gets rid of CV qualifiers 418 // and sets default decorators to sensible values. 419 // * Step 2: Reduce types. This step makes sure there is only a single T type and not 420 // multiple types. The P type of the address and T type of the value must 421 // match. 422 // * Step 3: Pre-runtime dispatch. This step checks whether a runtime call can be 423 // avoided, and in that case avoids it (calling raw accesses or 424 // primitive accesses in a build that does not require primitive GC barriers) 425 // * Step 4: Runtime-dispatch. This step performs a runtime dispatch to the corresponding 426 // BarrierSet::AccessBarrier accessor that attaches GC-required barriers 427 // to the access. 428 429 namespace AccessInternal { 430 template <typename T> 431 struct OopOrNarrowOopInternal: AllStatic { 432 typedef oop type; 433 }; 434 435 template <> 436 struct OopOrNarrowOopInternal<narrowOop>: AllStatic { 437 typedef narrowOop type; 438 }; 439 440 // This metafunction returns a canonicalized oop/narrowOop type for a passed 441 // in oop-like types passed in from oop_* overloads where the user has sworn 442 // that the passed in values should be oop-like (e.g. oop, oopDesc*, arrayOop, 443 // narrowOoop, instanceOopDesc*, and random other things). 444 // In the oop_* overloads, it must hold that if the passed in type T is not 445 // narrowOop, then it by contract has to be one of many oop-like types implicitly 446 // convertible to oop, and hence returns oop as the canonical oop type. 447 // If it turns out it was not, then the implicit conversion to oop will fail 448 // to compile, as desired. 449 template <typename T> 450 struct OopOrNarrowOop: AllStatic { 451 typedef typename OopOrNarrowOopInternal<typename Decay<T>::type>::type type; 452 }; 453 454 inline void* field_addr(oop base, ptrdiff_t byte_offset) { 455 return reinterpret_cast<void*>(reinterpret_cast<intptr_t>((void*)base) + byte_offset); 456 } 457 // Step 4: Runtime dispatch 458 // The RuntimeDispatch class is responsible for performing a runtime dispatch of the 459 // accessor. This is required when the access either depends on whether compressed oops 460 // is being used, or it depends on which GC implementation was chosen (e.g. requires GC 461 // barriers). The way it works is that a function pointer initially pointing to an 462 // accessor resolution function gets called for each access. Upon first invocation, 463 // it resolves which accessor to be used in future invocations and patches the 464 // function pointer to this new accessor. 465 466 template <DecoratorSet decorators, typename T, BarrierType type> 467 struct RuntimeDispatch: AllStatic {}; 468 469 template <DecoratorSet decorators, typename T> 470 struct RuntimeDispatch<decorators, T, BARRIER_STORE>: AllStatic { 471 typedef typename AccessFunction<decorators, T, BARRIER_STORE>::type func_t; 472 static func_t _store_func; 473 474 static void store_init(void* addr, T value); 475 476 static inline void store(void* addr, T value) { 477 _store_func(addr, value); 478 } 479 }; 480 481 template <DecoratorSet decorators, typename T> 482 struct RuntimeDispatch<decorators, T, BARRIER_STORE_AT>: AllStatic { 483 typedef typename AccessFunction<decorators, T, BARRIER_STORE_AT>::type func_t; 484 static func_t _store_at_func; 485 486 static void store_at_init(oop base, ptrdiff_t offset, T value); 487 488 static inline void store_at(oop base, ptrdiff_t offset, T value) { 489 _store_at_func(base, offset, value); 490 } 491 }; 492 493 template <DecoratorSet decorators, typename T> 494 struct RuntimeDispatch<decorators, T, BARRIER_LOAD>: AllStatic { 495 typedef typename AccessFunction<decorators, T, BARRIER_LOAD>::type func_t; 496 static func_t _load_func; 497 498 static T load_init(void* addr); 499 500 static inline T load(void* addr) { 501 return _load_func(addr); 502 } 503 }; 504 505 template <DecoratorSet decorators, typename T> 506 struct RuntimeDispatch<decorators, T, BARRIER_LOAD_AT>: AllStatic { 507 typedef typename AccessFunction<decorators, T, BARRIER_LOAD_AT>::type func_t; 508 static func_t _load_at_func; 509 510 static T load_at_init(oop base, ptrdiff_t offset); 511 512 static inline T load_at(oop base, ptrdiff_t offset) { 513 return _load_at_func(base, offset); 514 } 515 }; 516 517 template <DecoratorSet decorators, typename T> 518 struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG>: AllStatic { 519 typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG>::type func_t; 520 static func_t _atomic_cmpxchg_func; 521 522 static T atomic_cmpxchg_init(T new_value, void* addr, T compare_value); 523 524 static inline T atomic_cmpxchg(T new_value, void* addr, T compare_value) { 525 return _atomic_cmpxchg_func(new_value, addr, compare_value); 526 } 527 }; 528 529 template <DecoratorSet decorators, typename T> 530 struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>: AllStatic { 531 typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::type func_t; 532 static func_t _atomic_cmpxchg_at_func; 533 534 static T atomic_cmpxchg_at_init(T new_value, oop base, ptrdiff_t offset, T compare_value); 535 536 static inline T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) { 537 return _atomic_cmpxchg_at_func(new_value, base, offset, compare_value); 538 } 539 }; 540 541 template <DecoratorSet decorators, typename T> 542 struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG>: AllStatic { 543 typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG>::type func_t; 544 static func_t _atomic_xchg_func; 545 546 static T atomic_xchg_init(T new_value, void* addr); 547 548 static inline T atomic_xchg(T new_value, void* addr) { 549 return _atomic_xchg_func(new_value, addr); 550 } 551 }; 552 553 template <DecoratorSet decorators, typename T> 554 struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>: AllStatic { 555 typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type func_t; 556 static func_t _atomic_xchg_at_func; 557 558 static T atomic_xchg_at_init(T new_value, oop base, ptrdiff_t offset); 559 560 static inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) { 561 return _atomic_xchg_at_func(new_value, base, offset); 562 } 563 }; 564 565 template <DecoratorSet decorators, typename T> 566 struct RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>: AllStatic { 567 typedef typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type func_t; 568 static func_t _arraycopy_func; 569 570 static bool arraycopy_init(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, 571 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, 572 size_t length); 573 574 static inline bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, 575 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, 576 size_t length) { 577 return _arraycopy_func(src_obj, src_offset_in_bytes, src_raw, 578 dst_obj, dst_offset_in_bytes, dst_raw, 579 length); 580 } 581 }; 582 583 template <DecoratorSet decorators, typename T> 584 struct RuntimeDispatch<decorators, T, BARRIER_CLONE>: AllStatic { 585 typedef typename AccessFunction<decorators, T, BARRIER_CLONE>::type func_t; 586 static func_t _clone_func; 587 588 static void clone_init(oop src, oop dst, size_t size); 589 590 static inline void clone(oop src, oop dst, size_t size) { 591 _clone_func(src, dst, size); 592 } 593 }; 594 595 template <DecoratorSet decorators, typename T> 596 struct RuntimeDispatch<decorators, T, BARRIER_RESOLVE>: AllStatic { 597 typedef typename AccessFunction<decorators, T, BARRIER_RESOLVE>::type func_t; 598 static func_t _resolve_func; 599 600 static oop resolve_init(oop obj); 601 602 static inline oop resolve(oop obj) { 603 return _resolve_func(obj); 604 } 605 }; 606 607 template <DecoratorSet decorators, typename T> 608 struct RuntimeDispatch<decorators, T, BARRIER_EQUALS>: AllStatic { 609 typedef typename AccessFunction<decorators, T, BARRIER_EQUALS>::type func_t; 610 static func_t _equals_func; 611 612 static bool equals_init(oop o1, oop o2); 613 614 static inline bool equals(oop o1, oop o2) { 615 return _equals_func(o1, o2); 616 } 617 }; 618 619 // Initialize the function pointers to point to the resolving function. 620 template <DecoratorSet decorators, typename T> 621 typename AccessFunction<decorators, T, BARRIER_STORE>::type 622 RuntimeDispatch<decorators, T, BARRIER_STORE>::_store_func = &store_init; 623 624 template <DecoratorSet decorators, typename T> 625 typename AccessFunction<decorators, T, BARRIER_STORE_AT>::type 626 RuntimeDispatch<decorators, T, BARRIER_STORE_AT>::_store_at_func = &store_at_init; 627 628 template <DecoratorSet decorators, typename T> 629 typename AccessFunction<decorators, T, BARRIER_LOAD>::type 630 RuntimeDispatch<decorators, T, BARRIER_LOAD>::_load_func = &load_init; 631 632 template <DecoratorSet decorators, typename T> 633 typename AccessFunction<decorators, T, BARRIER_LOAD_AT>::type 634 RuntimeDispatch<decorators, T, BARRIER_LOAD_AT>::_load_at_func = &load_at_init; 635 636 template <DecoratorSet decorators, typename T> 637 typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG>::type 638 RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG>::_atomic_cmpxchg_func = &atomic_cmpxchg_init; 639 640 template <DecoratorSet decorators, typename T> 641 typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::type 642 RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::_atomic_cmpxchg_at_func = &atomic_cmpxchg_at_init; 643 644 template <DecoratorSet decorators, typename T> 645 typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG>::type 646 RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG>::_atomic_xchg_func = &atomic_xchg_init; 647 648 template <DecoratorSet decorators, typename T> 649 typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type 650 RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::_atomic_xchg_at_func = &atomic_xchg_at_init; 651 652 template <DecoratorSet decorators, typename T> 653 typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type 654 RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::_arraycopy_func = &arraycopy_init; 655 656 template <DecoratorSet decorators, typename T> 657 typename AccessFunction<decorators, T, BARRIER_CLONE>::type 658 RuntimeDispatch<decorators, T, BARRIER_CLONE>::_clone_func = &clone_init; 659 660 template <DecoratorSet decorators, typename T> 661 typename AccessFunction<decorators, T, BARRIER_RESOLVE>::type 662 RuntimeDispatch<decorators, T, BARRIER_RESOLVE>::_resolve_func = &resolve_init; 663 664 template <DecoratorSet decorators, typename T> 665 typename AccessFunction<decorators, T, BARRIER_EQUALS>::type 666 RuntimeDispatch<decorators, T, BARRIER_EQUALS>::_equals_func = &equals_init; 667 668 // Step 3: Pre-runtime dispatching. 669 // The PreRuntimeDispatch class is responsible for filtering the barrier strength 670 // decorators. That is, for AS_RAW, it hardwires the accesses without a runtime 671 // dispatch point. Otherwise it goes through a runtime check if hardwiring was 672 // not possible. 673 struct PreRuntimeDispatch: AllStatic { 674 template<DecoratorSet decorators> 675 struct CanHardwireRaw: public IntegralConstant< 676 bool, 677 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || // primitive access 678 !HasDecorator<decorators, INTERNAL_CONVERT_COMPRESSED_OOP>::value || // don't care about compressed oops (oop* address) 679 HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value> // we can infer we use compressed oops (narrowOop* address) 680 {}; 681 682 static const DecoratorSet convert_compressed_oops = INTERNAL_RT_USE_COMPRESSED_OOPS | INTERNAL_CONVERT_COMPRESSED_OOP; 683 684 template<DecoratorSet decorators> 685 static bool is_hardwired_primitive() { 686 return !HasDecorator<decorators, INTERNAL_BT_BARRIER_ON_PRIMITIVES>::value && 687 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value; 688 } 689 690 template <DecoratorSet decorators, typename T> 691 inline static typename EnableIf< 692 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value>::type 693 store(void* addr, T value) { 694 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw; 695 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) { 696 Raw::oop_store(addr, value); 697 } else { 698 Raw::store(addr, value); 699 } 700 } 701 702 template <DecoratorSet decorators, typename T> 703 inline static typename EnableIf< 704 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value>::type 705 store(void* addr, T value) { 706 if (UseCompressedOops) { 707 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; 708 PreRuntimeDispatch::store<expanded_decorators>(addr, value); 709 } else { 710 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; 711 PreRuntimeDispatch::store<expanded_decorators>(addr, value); 712 } 713 } 714 715 template <DecoratorSet decorators, typename T> 716 inline static typename EnableIf< 717 !HasDecorator<decorators, AS_RAW>::value>::type 718 store(void* addr, T value) { 719 if (is_hardwired_primitive<decorators>()) { 720 const DecoratorSet expanded_decorators = decorators | AS_RAW; 721 PreRuntimeDispatch::store<expanded_decorators>(addr, value); 722 } else { 723 RuntimeDispatch<decorators, T, BARRIER_STORE>::store(addr, value); 724 } 725 } 726 727 template <DecoratorSet decorators, typename T> 728 inline static typename EnableIf< 729 HasDecorator<decorators, AS_RAW>::value>::type 730 store_at(oop base, ptrdiff_t offset, T value) { 731 store<decorators>(field_addr(base, offset), value); 732 } 733 734 template <DecoratorSet decorators, typename T> 735 inline static typename EnableIf< 736 !HasDecorator<decorators, AS_RAW>::value>::type 737 store_at(oop base, ptrdiff_t offset, T value) { 738 if (is_hardwired_primitive<decorators>()) { 739 const DecoratorSet expanded_decorators = decorators | AS_RAW; 740 PreRuntimeDispatch::store_at<expanded_decorators>(base, offset, value); 741 } else { 742 RuntimeDispatch<decorators, T, BARRIER_STORE_AT>::store_at(base, offset, value); 743 } 744 } 745 746 template <DecoratorSet decorators, typename T> 747 inline static typename EnableIf< 748 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, T>::type 749 load(void* addr) { 750 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw; 751 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) { 752 return Raw::template oop_load<T>(addr); 753 } else { 754 return Raw::template load<T>(addr); 755 } 756 } 757 758 template <DecoratorSet decorators, typename T> 759 inline static typename EnableIf< 760 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, T>::type 761 load(void* addr) { 762 if (UseCompressedOops) { 763 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; 764 return PreRuntimeDispatch::load<expanded_decorators, T>(addr); 765 } else { 766 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; 767 return PreRuntimeDispatch::load<expanded_decorators, T>(addr); 768 } 769 } 770 771 template <DecoratorSet decorators, typename T> 772 inline static typename EnableIf< 773 !HasDecorator<decorators, AS_RAW>::value, T>::type 774 load(void* addr) { 775 if (is_hardwired_primitive<decorators>()) { 776 const DecoratorSet expanded_decorators = decorators | AS_RAW; 777 return PreRuntimeDispatch::load<expanded_decorators, T>(addr); 778 } else { 779 return RuntimeDispatch<decorators, T, BARRIER_LOAD>::load(addr); 780 } 781 } 782 783 template <DecoratorSet decorators, typename T> 784 inline static typename EnableIf< 785 HasDecorator<decorators, AS_RAW>::value, T>::type 786 load_at(oop base, ptrdiff_t offset) { 787 return load<decorators, T>(field_addr(base, offset)); 788 } 789 790 template <DecoratorSet decorators, typename T> 791 inline static typename EnableIf< 792 !HasDecorator<decorators, AS_RAW>::value, T>::type 793 load_at(oop base, ptrdiff_t offset) { 794 if (is_hardwired_primitive<decorators>()) { 795 const DecoratorSet expanded_decorators = decorators | AS_RAW; 796 return PreRuntimeDispatch::load_at<expanded_decorators, T>(base, offset); 797 } else { 798 return RuntimeDispatch<decorators, T, BARRIER_LOAD_AT>::load_at(base, offset); 799 } 800 } 801 802 template <DecoratorSet decorators, typename T> 803 inline static typename EnableIf< 804 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, T>::type 805 atomic_cmpxchg(T new_value, void* addr, T compare_value) { 806 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw; 807 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) { 808 return Raw::oop_atomic_cmpxchg(new_value, addr, compare_value); 809 } else { 810 return Raw::atomic_cmpxchg(new_value, addr, compare_value); 811 } 812 } 813 814 template <DecoratorSet decorators, typename T> 815 inline static typename EnableIf< 816 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, T>::type 817 atomic_cmpxchg(T new_value, void* addr, T compare_value) { 818 if (UseCompressedOops) { 819 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; 820 return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(new_value, addr, compare_value); 821 } else { 822 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; 823 return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(new_value, addr, compare_value); 824 } 825 } 826 827 template <DecoratorSet decorators, typename T> 828 inline static typename EnableIf< 829 !HasDecorator<decorators, AS_RAW>::value, T>::type 830 atomic_cmpxchg(T new_value, void* addr, T compare_value) { 831 if (is_hardwired_primitive<decorators>()) { 832 const DecoratorSet expanded_decorators = decorators | AS_RAW; 833 return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(new_value, addr, compare_value); 834 } else { 835 return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG>::atomic_cmpxchg(new_value, addr, compare_value); 836 } 837 } 838 839 template <DecoratorSet decorators, typename T> 840 inline static typename EnableIf< 841 HasDecorator<decorators, AS_RAW>::value, T>::type 842 atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) { 843 return atomic_cmpxchg<decorators>(new_value, field_addr(base, offset), compare_value); 844 } 845 846 template <DecoratorSet decorators, typename T> 847 inline static typename EnableIf< 848 !HasDecorator<decorators, AS_RAW>::value, T>::type 849 atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) { 850 if (is_hardwired_primitive<decorators>()) { 851 const DecoratorSet expanded_decorators = decorators | AS_RAW; 852 return PreRuntimeDispatch::atomic_cmpxchg_at<expanded_decorators>(new_value, base, offset, compare_value); 853 } else { 854 return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::atomic_cmpxchg_at(new_value, base, offset, compare_value); 855 } 856 } 857 858 template <DecoratorSet decorators, typename T> 859 inline static typename EnableIf< 860 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, T>::type 861 atomic_xchg(T new_value, void* addr) { 862 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw; 863 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) { 864 return Raw::oop_atomic_xchg(new_value, addr); 865 } else { 866 return Raw::atomic_xchg(new_value, addr); 867 } 868 } 869 870 template <DecoratorSet decorators, typename T> 871 inline static typename EnableIf< 872 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, T>::type 873 atomic_xchg(T new_value, void* addr) { 874 if (UseCompressedOops) { 875 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; 876 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, addr); 877 } else { 878 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; 879 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, addr); 880 } 881 } 882 883 template <DecoratorSet decorators, typename T> 884 inline static typename EnableIf< 885 !HasDecorator<decorators, AS_RAW>::value, T>::type 886 atomic_xchg(T new_value, void* addr) { 887 if (is_hardwired_primitive<decorators>()) { 888 const DecoratorSet expanded_decorators = decorators | AS_RAW; 889 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, addr); 890 } else { 891 return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG>::atomic_xchg(new_value, addr); 892 } 893 } 894 895 template <DecoratorSet decorators, typename T> 896 inline static typename EnableIf< 897 HasDecorator<decorators, AS_RAW>::value, T>::type 898 atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) { 899 return atomic_xchg<decorators>(new_value, field_addr(base, offset)); 900 } 901 902 template <DecoratorSet decorators, typename T> 903 inline static typename EnableIf< 904 !HasDecorator<decorators, AS_RAW>::value, T>::type 905 atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) { 906 if (is_hardwired_primitive<decorators>()) { 907 const DecoratorSet expanded_decorators = decorators | AS_RAW; 908 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, base, offset); 909 } else { 910 return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::atomic_xchg_at(new_value, base, offset); 911 } 912 } 913 914 template <DecoratorSet decorators, typename T> 915 inline static typename EnableIf< 916 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, bool>::type 917 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, 918 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, 919 size_t length) { 920 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw; 921 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) { 922 return Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw, 923 dst_obj, dst_offset_in_bytes, dst_raw, 924 length); 925 } else { 926 return Raw::arraycopy(src_obj, src_offset_in_bytes, src_raw, 927 dst_obj, dst_offset_in_bytes, dst_raw, 928 length); 929 } 930 } 931 932 template <DecoratorSet decorators, typename T> 933 inline static typename EnableIf< 934 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, bool>::type 935 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, 936 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, 937 size_t length) { 938 if (UseCompressedOops) { 939 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; 940 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw, 941 dst_obj, dst_offset_in_bytes, dst_raw, 942 length); 943 } else { 944 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; 945 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw, 946 dst_obj, dst_offset_in_bytes, dst_raw, 947 length); 948 } 949 } 950 951 template <DecoratorSet decorators, typename T> 952 inline static typename EnableIf< 953 !HasDecorator<decorators, AS_RAW>::value, bool>::type 954 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, 955 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, 956 size_t length) { 957 if (is_hardwired_primitive<decorators>()) { 958 const DecoratorSet expanded_decorators = decorators | AS_RAW; 959 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw, 960 dst_obj, dst_offset_in_bytes, dst_raw, 961 length); 962 } else { 963 return RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::arraycopy(src_obj, src_offset_in_bytes, src_raw, 964 dst_obj, dst_offset_in_bytes, dst_raw, 965 length); 966 } 967 } 968 969 template <DecoratorSet decorators> 970 inline static typename EnableIf< 971 HasDecorator<decorators, AS_RAW>::value>::type 972 clone(oop src, oop dst, size_t size) { 973 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw; 974 Raw::clone(src, dst, size); 975 } 976 977 template <DecoratorSet decorators> 978 inline static typename EnableIf< 979 !HasDecorator<decorators, AS_RAW>::value>::type 980 clone(oop src, oop dst, size_t size) { 981 RuntimeDispatch<decorators, oop, BARRIER_CLONE>::clone(src, dst, size); 982 } 983 984 template <DecoratorSet decorators> 985 inline static typename EnableIf< 986 HasDecorator<decorators, INTERNAL_BT_TO_SPACE_INVARIANT>::value, oop>::type 987 resolve(oop obj) { 988 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw; 989 return Raw::resolve(obj); 990 } 991 992 template <DecoratorSet decorators> 993 inline static typename EnableIf< 994 !HasDecorator<decorators, INTERNAL_BT_TO_SPACE_INVARIANT>::value, oop>::type 995 resolve(oop obj) { 996 return RuntimeDispatch<decorators, oop, BARRIER_RESOLVE>::resolve(obj); 997 } 998 999 template <DecoratorSet decorators> 1000 inline static typename EnableIf< 1001 HasDecorator<decorators, AS_RAW>::value || HasDecorator<decorators, INTERNAL_BT_TO_SPACE_INVARIANT>::value, bool>::type 1002 equals(oop o1, oop o2) { 1003 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw; 1004 return Raw::equals(o1, o2); 1005 } 1006 1007 template <DecoratorSet decorators> 1008 inline static typename EnableIf< 1009 !HasDecorator<decorators, AS_RAW>::value && !HasDecorator<decorators, INTERNAL_BT_TO_SPACE_INVARIANT>::value, bool>::type 1010 equals(oop o1, oop o2) { 1011 return RuntimeDispatch<decorators, oop, BARRIER_EQUALS>::equals(o1, o2); 1012 } 1013 }; 1014 1015 // Step 2: Reduce types. 1016 // Enforce that for non-oop types, T and P have to be strictly the same. 1017 // P is the type of the address and T is the type of the values. 1018 // As for oop types, it is allow to send T in {narrowOop, oop} and 1019 // P in {narrowOop, oop, HeapWord*}. The following rules apply according to 1020 // the subsequent table. (columns are P, rows are T) 1021 // | | HeapWord | oop | narrowOop | 1022 // | oop | rt-comp | hw-none | hw-comp | 1023 // | narrowOop | x | x | hw-none | 1024 // 1025 // x means not allowed 1026 // rt-comp means it must be checked at runtime whether the oop is compressed. 1027 // hw-none means it is statically known the oop will not be compressed. 1028 // hw-comp means it is statically known the oop will be compressed. 1029 1030 template <DecoratorSet decorators, typename T> 1031 inline void store_reduce_types(T* addr, T value) { 1032 PreRuntimeDispatch::store<decorators>(addr, value); 1033 } 1034 1035 template <DecoratorSet decorators> 1036 inline void store_reduce_types(narrowOop* addr, oop value) { 1037 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | 1038 INTERNAL_RT_USE_COMPRESSED_OOPS; 1039 PreRuntimeDispatch::store<expanded_decorators>(addr, value); 1040 } 1041 1042 template <DecoratorSet decorators> 1043 inline void store_reduce_types(narrowOop* addr, narrowOop value) { 1044 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | 1045 INTERNAL_RT_USE_COMPRESSED_OOPS; 1046 PreRuntimeDispatch::store<expanded_decorators>(addr, value); 1047 } 1048 1049 template <DecoratorSet decorators> 1050 inline void store_reduce_types(HeapWord* addr, oop value) { 1051 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; 1052 PreRuntimeDispatch::store<expanded_decorators>(addr, value); 1053 } 1054 1055 template <DecoratorSet decorators, typename T> 1056 inline T atomic_cmpxchg_reduce_types(T new_value, T* addr, T compare_value) { 1057 return PreRuntimeDispatch::atomic_cmpxchg<decorators>(new_value, addr, compare_value); 1058 } 1059 1060 template <DecoratorSet decorators> 1061 inline oop atomic_cmpxchg_reduce_types(oop new_value, narrowOop* addr, oop compare_value) { 1062 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | 1063 INTERNAL_RT_USE_COMPRESSED_OOPS; 1064 return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(new_value, addr, compare_value); 1065 } 1066 1067 template <DecoratorSet decorators> 1068 inline narrowOop atomic_cmpxchg_reduce_types(narrowOop new_value, narrowOop* addr, narrowOop compare_value) { 1069 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | 1070 INTERNAL_RT_USE_COMPRESSED_OOPS; 1071 return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(new_value, addr, compare_value); 1072 } 1073 1074 template <DecoratorSet decorators> 1075 inline oop atomic_cmpxchg_reduce_types(oop new_value, 1076 HeapWord* addr, 1077 oop compare_value) { 1078 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; 1079 return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(new_value, addr, compare_value); 1080 } 1081 1082 template <DecoratorSet decorators, typename T> 1083 inline T atomic_xchg_reduce_types(T new_value, T* addr) { 1084 const DecoratorSet expanded_decorators = decorators; 1085 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, addr); 1086 } 1087 1088 template <DecoratorSet decorators> 1089 inline oop atomic_xchg_reduce_types(oop new_value, narrowOop* addr) { 1090 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | 1091 INTERNAL_RT_USE_COMPRESSED_OOPS; 1092 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, addr); 1093 } 1094 1095 template <DecoratorSet decorators> 1096 inline narrowOop atomic_xchg_reduce_types(narrowOop new_value, narrowOop* addr) { 1097 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | 1098 INTERNAL_RT_USE_COMPRESSED_OOPS; 1099 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, addr); 1100 } 1101 1102 template <DecoratorSet decorators> 1103 inline oop atomic_xchg_reduce_types(oop new_value, HeapWord* addr) { 1104 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; 1105 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, addr); 1106 } 1107 1108 template <DecoratorSet decorators, typename T> 1109 inline T load_reduce_types(T* addr) { 1110 return PreRuntimeDispatch::load<decorators, T>(addr); 1111 } 1112 1113 template <DecoratorSet decorators, typename T> 1114 inline typename OopOrNarrowOop<T>::type load_reduce_types(narrowOop* addr) { 1115 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | 1116 INTERNAL_RT_USE_COMPRESSED_OOPS; 1117 return PreRuntimeDispatch::load<expanded_decorators, typename OopOrNarrowOop<T>::type>(addr); 1118 } 1119 1120 template <DecoratorSet decorators, typename T> 1121 inline oop load_reduce_types(HeapWord* addr) { 1122 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; 1123 return PreRuntimeDispatch::load<expanded_decorators, oop>(addr); 1124 } 1125 1126 template <DecoratorSet decorators, typename T> 1127 inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, 1128 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, 1129 size_t length) { 1130 return PreRuntimeDispatch::arraycopy<decorators>(src_obj, src_offset_in_bytes, src_raw, 1131 dst_obj, dst_offset_in_bytes, dst_raw, 1132 length); 1133 } 1134 1135 template <DecoratorSet decorators> 1136 inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, HeapWord* src_raw, 1137 arrayOop dst_obj, size_t dst_offset_in_bytes, HeapWord* dst_raw, 1138 size_t length) { 1139 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; 1140 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw, 1141 dst_obj, dst_offset_in_bytes, dst_raw, 1142 length); 1143 } 1144 1145 template <DecoratorSet decorators> 1146 inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, narrowOop* src_raw, 1147 arrayOop dst_obj, size_t dst_offset_in_bytes, narrowOop* dst_raw, 1148 size_t length) { 1149 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | 1150 INTERNAL_RT_USE_COMPRESSED_OOPS; 1151 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw, 1152 dst_obj, dst_offset_in_bytes, dst_raw, 1153 length); 1154 } 1155 1156 // Step 1: Set default decorators. This step remembers if a type was volatile 1157 // and then sets the MO_VOLATILE decorator by default. Otherwise, a default 1158 // memory ordering is set for the access, and the implied decorator rules 1159 // are applied to select sensible defaults for decorators that have not been 1160 // explicitly set. For example, default object referent strength is set to strong. 1161 // This step also decays the types passed in (e.g. getting rid of CV qualifiers 1162 // and references from the types). This step also perform some type verification 1163 // that the passed in types make sense. 1164 1165 template <DecoratorSet decorators, typename T> 1166 static void verify_types(){ 1167 // If this fails to compile, then you have sent in something that is 1168 // not recognized as a valid primitive type to a primitive Access function. 1169 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || // oops have already been validated 1170 (IsPointer<T>::value || IsIntegral<T>::value) || 1171 IsFloatingPoint<T>::value)); // not allowed primitive type 1172 } 1173 1174 template <DecoratorSet decorators, typename P, typename T> 1175 inline void store(P* addr, T value) { 1176 verify_types<decorators, T>(); 1177 typedef typename Decay<P>::type DecayedP; 1178 typedef typename Decay<T>::type DecayedT; 1179 DecayedT decayed_value = value; 1180 // If a volatile address is passed in but no memory ordering decorator, 1181 // set the memory ordering to MO_VOLATILE by default. 1182 const DecoratorSet expanded_decorators = DecoratorFixup< 1183 (IsVolatile<P>::value && !HasDecorator<decorators, MO_DECORATOR_MASK>::value) ? 1184 (MO_VOLATILE | decorators) : decorators>::value; 1185 store_reduce_types<expanded_decorators>(const_cast<DecayedP*>(addr), decayed_value); 1186 } 1187 1188 template <DecoratorSet decorators, typename T> 1189 inline void store_at(oop base, ptrdiff_t offset, T value) { 1190 verify_types<decorators, T>(); 1191 typedef typename Decay<T>::type DecayedT; 1192 DecayedT decayed_value = value; 1193 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | 1194 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ? 1195 INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY)>::value; 1196 PreRuntimeDispatch::store_at<expanded_decorators>(base, offset, decayed_value); 1197 } 1198 1199 template <DecoratorSet decorators, typename P, typename T> 1200 inline T load(P* addr) { 1201 verify_types<decorators, T>(); 1202 typedef typename Decay<P>::type DecayedP; 1203 typedef typename Conditional<HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value, 1204 typename OopOrNarrowOop<T>::type, 1205 typename Decay<T>::type>::type DecayedT; 1206 // If a volatile address is passed in but no memory ordering decorator, 1207 // set the memory ordering to MO_VOLATILE by default. 1208 const DecoratorSet expanded_decorators = DecoratorFixup< 1209 (IsVolatile<P>::value && !HasDecorator<decorators, MO_DECORATOR_MASK>::value) ? 1210 (MO_VOLATILE | decorators) : decorators>::value; 1211 return load_reduce_types<expanded_decorators, DecayedT>(const_cast<DecayedP*>(addr)); 1212 } 1213 1214 template <DecoratorSet decorators, typename T> 1215 inline T load_at(oop base, ptrdiff_t offset) { 1216 verify_types<decorators, T>(); 1217 typedef typename Conditional<HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value, 1218 typename OopOrNarrowOop<T>::type, 1219 typename Decay<T>::type>::type DecayedT; 1220 // Expand the decorators (figure out sensible defaults) 1221 // Potentially remember if we need compressed oop awareness 1222 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | 1223 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ? 1224 INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY)>::value; 1225 return PreRuntimeDispatch::load_at<expanded_decorators, DecayedT>(base, offset); 1226 } 1227 1228 template <DecoratorSet decorators, typename P, typename T> 1229 inline T atomic_cmpxchg(T new_value, P* addr, T compare_value) { 1230 verify_types<decorators, T>(); 1231 typedef typename Decay<P>::type DecayedP; 1232 typedef typename Decay<T>::type DecayedT; 1233 DecayedT new_decayed_value = new_value; 1234 DecayedT compare_decayed_value = compare_value; 1235 const DecoratorSet expanded_decorators = DecoratorFixup< 1236 (!HasDecorator<decorators, MO_DECORATOR_MASK>::value) ? 1237 (MO_SEQ_CST | decorators) : decorators>::value; 1238 return atomic_cmpxchg_reduce_types<expanded_decorators>(new_decayed_value, 1239 const_cast<DecayedP*>(addr), 1240 compare_decayed_value); 1241 } 1242 1243 template <DecoratorSet decorators, typename T> 1244 inline T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) { 1245 verify_types<decorators, T>(); 1246 typedef typename Decay<T>::type DecayedT; 1247 DecayedT new_decayed_value = new_value; 1248 DecayedT compare_decayed_value = compare_value; 1249 // Determine default memory ordering 1250 const DecoratorSet expanded_decorators = DecoratorFixup< 1251 (!HasDecorator<decorators, MO_DECORATOR_MASK>::value) ? 1252 (MO_SEQ_CST | decorators) : decorators>::value; 1253 // Potentially remember that we need compressed oop awareness 1254 const DecoratorSet final_decorators = expanded_decorators | 1255 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ? 1256 INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY); 1257 return PreRuntimeDispatch::atomic_cmpxchg_at<final_decorators>(new_decayed_value, base, 1258 offset, compare_decayed_value); 1259 } 1260 1261 template <DecoratorSet decorators, typename P, typename T> 1262 inline T atomic_xchg(T new_value, P* addr) { 1263 verify_types<decorators, T>(); 1264 typedef typename Decay<P>::type DecayedP; 1265 typedef typename Decay<T>::type DecayedT; 1266 DecayedT new_decayed_value = new_value; 1267 // atomic_xchg is only available in SEQ_CST flavour. 1268 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST>::value; 1269 return atomic_xchg_reduce_types<expanded_decorators>(new_decayed_value, 1270 const_cast<DecayedP*>(addr)); 1271 } 1272 1273 template <DecoratorSet decorators, typename T> 1274 inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) { 1275 verify_types<decorators, T>(); 1276 typedef typename Decay<T>::type DecayedT; 1277 DecayedT new_decayed_value = new_value; 1278 // atomic_xchg is only available in SEQ_CST flavour. 1279 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST | 1280 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ? 1281 INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY)>::value; 1282 return PreRuntimeDispatch::atomic_xchg_at<expanded_decorators>(new_decayed_value, base, offset); 1283 } 1284 1285 template <DecoratorSet decorators, typename T> 1286 inline bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw, 1287 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, 1288 size_t length) { 1289 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || 1290 (IsSame<T, void>::value || IsIntegral<T>::value) || 1291 IsFloatingPoint<T>::value)); // arraycopy allows type erased void elements 1292 typedef typename Decay<T>::type DecayedT; 1293 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | IS_ARRAY | IN_HEAP>::value; 1294 return arraycopy_reduce_types<expanded_decorators>(src_obj, src_offset_in_bytes, const_cast<DecayedT*>(src_raw), 1295 dst_obj, dst_offset_in_bytes, const_cast<DecayedT*>(dst_raw), 1296 length); 1297 } 1298 1299 template <DecoratorSet decorators> 1300 inline void clone(oop src, oop dst, size_t size) { 1301 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value; 1302 PreRuntimeDispatch::clone<expanded_decorators>(src, dst, size); 1303 } 1304 1305 template <DecoratorSet decorators> 1306 inline oop resolve(oop obj) { 1307 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value; 1308 return PreRuntimeDispatch::resolve<expanded_decorators>(obj); 1309 } 1310 1311 template <DecoratorSet decorators> 1312 inline bool equals(oop o1, oop o2) { 1313 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value; 1314 return PreRuntimeDispatch::equals<expanded_decorators>(o1, o2); 1315 } 1316 1317 // Infer the type that should be returned from an Access::oop_load. 1318 template <typename P, DecoratorSet decorators> 1319 class OopLoadProxy: public StackObj { 1320 private: 1321 P *const _addr; 1322 public: 1323 OopLoadProxy(P* addr) : _addr(addr) {} 1324 1325 inline operator oop() { 1326 return load<decorators | INTERNAL_VALUE_IS_OOP, P, oop>(_addr); 1327 } 1328 1329 inline operator narrowOop() { 1330 return load<decorators | INTERNAL_VALUE_IS_OOP, P, narrowOop>(_addr); 1331 } 1332 1333 template <typename T> 1334 inline bool operator ==(const T& other) const { 1335 return load<decorators | INTERNAL_VALUE_IS_OOP, P, T>(_addr) == other; 1336 } 1337 1338 template <typename T> 1339 inline bool operator !=(const T& other) const { 1340 return load<decorators | INTERNAL_VALUE_IS_OOP, P, T>(_addr) != other; 1341 } 1342 }; 1343 1344 // Infer the type that should be returned from an Access::load_at. 1345 template <DecoratorSet decorators> 1346 class LoadAtProxy: public StackObj { 1347 private: 1348 const oop _base; 1349 const ptrdiff_t _offset; 1350 public: 1351 LoadAtProxy(oop base, ptrdiff_t offset) : _base(base), _offset(offset) {} 1352 1353 template <typename T> 1354 inline operator T() const { 1355 return load_at<decorators, T>(_base, _offset); 1356 } 1357 1358 template <typename T> 1359 inline bool operator ==(const T& other) const { return load_at<decorators, T>(_base, _offset) == other; } 1360 1361 template <typename T> 1362 inline bool operator !=(const T& other) const { return load_at<decorators, T>(_base, _offset) != other; } 1363 }; 1364 1365 // Infer the type that should be returned from an Access::oop_load_at. 1366 template <DecoratorSet decorators> 1367 class OopLoadAtProxy: public StackObj { 1368 private: 1369 const oop _base; 1370 const ptrdiff_t _offset; 1371 public: 1372 OopLoadAtProxy(oop base, ptrdiff_t offset) : _base(base), _offset(offset) {} 1373 1374 inline operator oop() const { 1375 return load_at<decorators | INTERNAL_VALUE_IS_OOP, oop>(_base, _offset); 1376 } 1377 1378 inline operator narrowOop() const { 1379 return load_at<decorators | INTERNAL_VALUE_IS_OOP, narrowOop>(_base, _offset); 1380 } 1381 1382 template <typename T> 1383 inline bool operator ==(const T& other) const { 1384 return load_at<decorators | INTERNAL_VALUE_IS_OOP, T>(_base, _offset) == other; 1385 } 1386 1387 template <typename T> 1388 inline bool operator !=(const T& other) const { 1389 return load_at<decorators | INTERNAL_VALUE_IS_OOP, T>(_base, _offset) != other; 1390 } 1391 }; 1392 } 1393 1394 #endif // SHARE_OOPS_ACCESSBACKEND_HPP