1175 inline void store(P* addr, T value) {
1176 verify_types<decorators, T>();
1177 typedef typename Decay<P>::type DecayedP;
1178 typedef typename Decay<T>::type DecayedT;
1179 DecayedT decayed_value = value;
1180 // If a volatile address is passed in but no memory ordering decorator,
1181 // set the memory ordering to MO_VOLATILE by default.
1182 const DecoratorSet expanded_decorators = DecoratorFixup<
1183 (IsVolatile<P>::value && !HasDecorator<decorators, MO_DECORATOR_MASK>::value) ?
1184 (MO_VOLATILE | decorators) : decorators>::value;
1185 store_reduce_types<expanded_decorators>(const_cast<DecayedP*>(addr), decayed_value);
1186 }
1187
1188 template <DecoratorSet decorators, typename T>
1189 inline void store_at(oop base, ptrdiff_t offset, T value) {
1190 verify_types<decorators, T>();
1191 typedef typename Decay<T>::type DecayedT;
1192 DecayedT decayed_value = value;
1193 const DecoratorSet expanded_decorators = DecoratorFixup<decorators |
1194 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1195 INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY)>::value;
1196 PreRuntimeDispatch::store_at<expanded_decorators>(base, offset, decayed_value);
1197 }
1198
1199 template <DecoratorSet decorators, typename P, typename T>
1200 inline T load(P* addr) {
1201 verify_types<decorators, T>();
1202 typedef typename Decay<P>::type DecayedP;
1203 typedef typename Conditional<HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value,
1204 typename OopOrNarrowOop<T>::type,
1205 typename Decay<T>::type>::type DecayedT;
1206 // If a volatile address is passed in but no memory ordering decorator,
1207 // set the memory ordering to MO_VOLATILE by default.
1208 const DecoratorSet expanded_decorators = DecoratorFixup<
1209 (IsVolatile<P>::value && !HasDecorator<decorators, MO_DECORATOR_MASK>::value) ?
1210 (MO_VOLATILE | decorators) : decorators>::value;
1211 return load_reduce_types<expanded_decorators, DecayedT>(const_cast<DecayedP*>(addr));
1212 }
1213
1214 template <DecoratorSet decorators, typename T>
1215 inline T load_at(oop base, ptrdiff_t offset) {
1216 verify_types<decorators, T>();
1217 typedef typename Conditional<HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value,
1218 typename OopOrNarrowOop<T>::type,
1219 typename Decay<T>::type>::type DecayedT;
1220 // Expand the decorators (figure out sensible defaults)
1221 // Potentially remember if we need compressed oop awareness
1222 const DecoratorSet expanded_decorators = DecoratorFixup<decorators |
1223 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1224 INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY)>::value;
1225 return PreRuntimeDispatch::load_at<expanded_decorators, DecayedT>(base, offset);
1226 }
1227
1228 template <DecoratorSet decorators, typename P, typename T>
1229 inline T atomic_cmpxchg(T new_value, P* addr, T compare_value) {
1230 verify_types<decorators, T>();
1231 typedef typename Decay<P>::type DecayedP;
1232 typedef typename Decay<T>::type DecayedT;
1233 DecayedT new_decayed_value = new_value;
1234 DecayedT compare_decayed_value = compare_value;
1235 const DecoratorSet expanded_decorators = DecoratorFixup<
1236 (!HasDecorator<decorators, MO_DECORATOR_MASK>::value) ?
1237 (MO_SEQ_CST | decorators) : decorators>::value;
1238 return atomic_cmpxchg_reduce_types<expanded_decorators>(new_decayed_value,
1239 const_cast<DecayedP*>(addr),
1240 compare_decayed_value);
1241 }
1242
1243 template <DecoratorSet decorators, typename T>
1244 inline T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) {
1245 verify_types<decorators, T>();
1246 typedef typename Decay<T>::type DecayedT;
1247 DecayedT new_decayed_value = new_value;
1248 DecayedT compare_decayed_value = compare_value;
1249 // Determine default memory ordering
1250 const DecoratorSet expanded_decorators = DecoratorFixup<
1251 (!HasDecorator<decorators, MO_DECORATOR_MASK>::value) ?
1252 (MO_SEQ_CST | decorators) : decorators>::value;
1253 // Potentially remember that we need compressed oop awareness
1254 const DecoratorSet final_decorators = expanded_decorators |
1255 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1256 INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY);
1257 return PreRuntimeDispatch::atomic_cmpxchg_at<final_decorators>(new_decayed_value, base,
1258 offset, compare_decayed_value);
1259 }
1260
1261 template <DecoratorSet decorators, typename P, typename T>
1262 inline T atomic_xchg(T new_value, P* addr) {
1263 verify_types<decorators, T>();
1264 typedef typename Decay<P>::type DecayedP;
1265 typedef typename Decay<T>::type DecayedT;
1266 DecayedT new_decayed_value = new_value;
1267 // atomic_xchg is only available in SEQ_CST flavour.
1268 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST>::value;
1269 return atomic_xchg_reduce_types<expanded_decorators>(new_decayed_value,
1270 const_cast<DecayedP*>(addr));
1271 }
1272
1273 template <DecoratorSet decorators, typename T>
1274 inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
1275 verify_types<decorators, T>();
1276 typedef typename Decay<T>::type DecayedT;
1277 DecayedT new_decayed_value = new_value;
1278 // atomic_xchg is only available in SEQ_CST flavour.
1279 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST |
1280 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1281 INTERNAL_CONVERT_COMPRESSED_OOP : INTERNAL_EMPTY)>::value;
1282 return PreRuntimeDispatch::atomic_xchg_at<expanded_decorators>(new_decayed_value, base, offset);
1283 }
1284
1285 template <DecoratorSet decorators, typename T>
1286 inline bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
1287 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1288 size_t length) {
1289 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ||
1290 (IsSame<T, void>::value || IsIntegral<T>::value) ||
1291 IsFloatingPoint<T>::value)); // arraycopy allows type erased void elements
1292 typedef typename Decay<T>::type DecayedT;
1293 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | IS_ARRAY | IN_HEAP>::value;
1294 return arraycopy_reduce_types<expanded_decorators>(src_obj, src_offset_in_bytes, const_cast<DecayedT*>(src_raw),
1295 dst_obj, dst_offset_in_bytes, const_cast<DecayedT*>(dst_raw),
1296 length);
1297 }
1298
1299 template <DecoratorSet decorators>
1300 inline void clone(oop src, oop dst, size_t size) {
1301 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
|
1175 inline void store(P* addr, T value) {
1176 verify_types<decorators, T>();
1177 typedef typename Decay<P>::type DecayedP;
1178 typedef typename Decay<T>::type DecayedT;
1179 DecayedT decayed_value = value;
1180 // If a volatile address is passed in but no memory ordering decorator,
1181 // set the memory ordering to MO_VOLATILE by default.
1182 const DecoratorSet expanded_decorators = DecoratorFixup<
1183 (IsVolatile<P>::value && !HasDecorator<decorators, MO_DECORATOR_MASK>::value) ?
1184 (MO_VOLATILE | decorators) : decorators>::value;
1185 store_reduce_types<expanded_decorators>(const_cast<DecayedP*>(addr), decayed_value);
1186 }
1187
1188 template <DecoratorSet decorators, typename T>
1189 inline void store_at(oop base, ptrdiff_t offset, T value) {
1190 verify_types<decorators, T>();
1191 typedef typename Decay<T>::type DecayedT;
1192 DecayedT decayed_value = value;
1193 const DecoratorSet expanded_decorators = DecoratorFixup<decorators |
1194 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1195 INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE)>::value;
1196 PreRuntimeDispatch::store_at<expanded_decorators>(base, offset, decayed_value);
1197 }
1198
1199 template <DecoratorSet decorators, typename P, typename T>
1200 inline T load(P* addr) {
1201 verify_types<decorators, T>();
1202 typedef typename Decay<P>::type DecayedP;
1203 typedef typename Conditional<HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value,
1204 typename OopOrNarrowOop<T>::type,
1205 typename Decay<T>::type>::type DecayedT;
1206 // If a volatile address is passed in but no memory ordering decorator,
1207 // set the memory ordering to MO_VOLATILE by default.
1208 const DecoratorSet expanded_decorators = DecoratorFixup<
1209 (IsVolatile<P>::value && !HasDecorator<decorators, MO_DECORATOR_MASK>::value) ?
1210 (MO_VOLATILE | decorators) : decorators>::value;
1211 return load_reduce_types<expanded_decorators, DecayedT>(const_cast<DecayedP*>(addr));
1212 }
1213
1214 template <DecoratorSet decorators, typename T>
1215 inline T load_at(oop base, ptrdiff_t offset) {
1216 verify_types<decorators, T>();
1217 typedef typename Conditional<HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value,
1218 typename OopOrNarrowOop<T>::type,
1219 typename Decay<T>::type>::type DecayedT;
1220 // Expand the decorators (figure out sensible defaults)
1221 // Potentially remember if we need compressed oop awareness
1222 const DecoratorSet expanded_decorators = DecoratorFixup<decorators |
1223 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1224 INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE)>::value;
1225 return PreRuntimeDispatch::load_at<expanded_decorators, DecayedT>(base, offset);
1226 }
1227
1228 template <DecoratorSet decorators, typename P, typename T>
1229 inline T atomic_cmpxchg(T new_value, P* addr, T compare_value) {
1230 verify_types<decorators, T>();
1231 typedef typename Decay<P>::type DecayedP;
1232 typedef typename Decay<T>::type DecayedT;
1233 DecayedT new_decayed_value = new_value;
1234 DecayedT compare_decayed_value = compare_value;
1235 const DecoratorSet expanded_decorators = DecoratorFixup<
1236 (!HasDecorator<decorators, MO_DECORATOR_MASK>::value) ?
1237 (MO_SEQ_CST | decorators) : decorators>::value;
1238 return atomic_cmpxchg_reduce_types<expanded_decorators>(new_decayed_value,
1239 const_cast<DecayedP*>(addr),
1240 compare_decayed_value);
1241 }
1242
1243 template <DecoratorSet decorators, typename T>
1244 inline T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) {
1245 verify_types<decorators, T>();
1246 typedef typename Decay<T>::type DecayedT;
1247 DecayedT new_decayed_value = new_value;
1248 DecayedT compare_decayed_value = compare_value;
1249 // Determine default memory ordering
1250 const DecoratorSet expanded_decorators = DecoratorFixup<
1251 (!HasDecorator<decorators, MO_DECORATOR_MASK>::value) ?
1252 (MO_SEQ_CST | decorators) : decorators>::value;
1253 // Potentially remember that we need compressed oop awareness
1254 const DecoratorSet final_decorators = expanded_decorators |
1255 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1256 INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE);
1257 return PreRuntimeDispatch::atomic_cmpxchg_at<final_decorators>(new_decayed_value, base,
1258 offset, compare_decayed_value);
1259 }
1260
1261 template <DecoratorSet decorators, typename P, typename T>
1262 inline T atomic_xchg(T new_value, P* addr) {
1263 verify_types<decorators, T>();
1264 typedef typename Decay<P>::type DecayedP;
1265 typedef typename Decay<T>::type DecayedT;
1266 DecayedT new_decayed_value = new_value;
1267 // atomic_xchg is only available in SEQ_CST flavour.
1268 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST>::value;
1269 return atomic_xchg_reduce_types<expanded_decorators>(new_decayed_value,
1270 const_cast<DecayedP*>(addr));
1271 }
1272
1273 template <DecoratorSet decorators, typename T>
1274 inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
1275 verify_types<decorators, T>();
1276 typedef typename Decay<T>::type DecayedT;
1277 DecayedT new_decayed_value = new_value;
1278 // atomic_xchg is only available in SEQ_CST flavour.
1279 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST |
1280 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1281 INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE)>::value;
1282 return PreRuntimeDispatch::atomic_xchg_at<expanded_decorators>(new_decayed_value, base, offset);
1283 }
1284
1285 template <DecoratorSet decorators, typename T>
1286 inline bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
1287 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1288 size_t length) {
1289 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ||
1290 (IsSame<T, void>::value || IsIntegral<T>::value) ||
1291 IsFloatingPoint<T>::value)); // arraycopy allows type erased void elements
1292 typedef typename Decay<T>::type DecayedT;
1293 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | IS_ARRAY | IN_HEAP>::value;
1294 return arraycopy_reduce_types<expanded_decorators>(src_obj, src_offset_in_bytes, const_cast<DecayedT*>(src_raw),
1295 dst_obj, dst_offset_in_bytes, const_cast<DecayedT*>(dst_raw),
1296 length);
1297 }
1298
1299 template <DecoratorSet decorators>
1300 inline void clone(oop src, oop dst, size_t size) {
1301 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
|