BarrierSetC2
225 bool inline_math_native(vmIntrinsics::ID id); 226 bool inline_math(vmIntrinsics::ID id); 227 template <typename OverflowOp> 228 bool inline_math_overflow(Node* arg1, Node* arg2); 229 void inline_math_mathExact(Node* math, Node* test); 230 bool inline_math_addExactI(bool is_increment); 231 bool inline_math_addExactL(bool is_increment); 232 bool inline_math_multiplyExactI(); 233 bool inline_math_multiplyExactL(); 234 bool inline_math_multiplyHigh(); 235 bool inline_math_negateExactI(); 236 bool inline_math_negateExactL(); 237 bool inline_math_subtractExactI(bool is_decrement); 238 bool inline_math_subtractExactL(bool is_decrement); 239 bool inline_min_max(vmIntrinsics::ID id); 240 bool inline_notify(vmIntrinsics::ID id); 241 Node* generate_min_max(vmIntrinsics::ID id, Node* x, Node* y); 242 // This returns Type::AnyPtr, RawPtr, or OopPtr. 243 int classify_unsafe_addr(Node* &base, Node* &offset, BasicType type); 244 Node* make_unsafe_address(Node*& base, Node* offset, BasicType type = T_ILLEGAL, bool can_cast = false); 245 // Helper for inline_unsafe_access. 246 // Generates the guards that check whether the result of 247 // Unsafe.getObject should be recorded in an SATB log buffer. 248 void insert_pre_barrier(Node* base_oop, Node* offset, Node* pre_val, bool need_mem_bar); 249 250 typedef enum { Relaxed, Opaque, Volatile, Acquire, Release } AccessKind; 251 bool inline_unsafe_access(bool is_store, BasicType type, AccessKind kind, bool is_unaligned); 252 static bool klass_needs_init_guard(Node* kls); 253 bool inline_unsafe_allocate(); 254 bool inline_unsafe_newArray(bool uninitialized); 255 bool inline_unsafe_copyMemory(); 256 bool inline_native_currentThread(); 257 258 bool inline_native_time_funcs(address method, const char* funcName); 259 #ifdef TRACE_HAVE_INTRINSICS 260 bool inline_native_classID(); 261 bool inline_native_getBufferWriter(); 262 #endif 263 bool inline_native_isInterrupted(); 264 bool inline_native_Class_query(vmIntrinsics::ID id); 265 bool inline_native_subtype_check(); 266 bool inline_native_getLength(); 267 bool inline_array_copyOf(bool is_copyOfRange); 268 bool inline_array_equals(StrIntrinsicNode::ArgEnc ae); 269 bool inline_preconditions_checkIndex(); 270 void copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark); 271 bool inline_native_clone(bool is_virtual); 272 bool inline_native_Reflection_getCallerClass(); 273 // Helper function for inlining native object hash method 274 bool inline_native_hashcode(bool is_virtual, bool is_static); 275 bool inline_native_getClass(); 276 277 // Helper functions for inlining arraycopy 278 bool inline_arraycopy(); 279 AllocateArrayNode* tightly_coupled_allocation(Node* ptr, 280 RegionNode* slow_region); 281 JVMState* arraycopy_restore_alloc_state(AllocateArrayNode* alloc, int& saved_reexecute_sp); 282 void arraycopy_move_allocation_here(AllocateArrayNode* alloc, Node* dest, JVMState* saved_jvms, int saved_reexecute_sp, 283 uint new_idx); 284 285 typedef enum { LS_get_add, LS_get_set, LS_cmp_swap, LS_cmp_swap_weak, LS_cmp_exchange } LoadStoreKind; 286 MemNode::MemOrd access_kind_to_memord_LS(AccessKind access_kind, bool is_store); 287 MemNode::MemOrd access_kind_to_memord(AccessKind access_kind); 288 bool inline_unsafe_load_store(BasicType type, LoadStoreKind kind, AccessKind access_kind); 289 bool inline_unsafe_fence(vmIntrinsics::ID id); 290 bool inline_onspinwait(); 291 bool inline_fp_conversions(vmIntrinsics::ID id); 292 bool inline_number_methods(vmIntrinsics::ID id); 293 bool inline_reference_get(); 294 bool inline_Class_cast(); 295 bool inline_aescrypt_Block(vmIntrinsics::ID id); 296 bool inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id); 297 bool inline_counterMode_AESCrypt(vmIntrinsics::ID id); 298 Node* inline_cipherBlockChaining_AESCrypt_predicate(bool decrypting); 299 Node* inline_counterMode_AESCrypt_predicate(); 300 Node* get_key_start_from_aescrypt_object(Node* aescrypt_object); 301 Node* get_original_key_start_from_aescrypt_object(Node* aescrypt_object); 302 bool inline_ghash_processBlocks(); 303 bool inline_sha_implCompress(vmIntrinsics::ID id); 304 bool inline_digestBase_implCompressMB(int predicate); 305 bool inline_sha_implCompressMB(Node* digestBaseObj, ciInstanceKlass* instklass_SHA, 306 bool long_state, address stubAddr, const char *stubName, |
225 bool inline_math_native(vmIntrinsics::ID id);
226 bool inline_math(vmIntrinsics::ID id);
227 template <typename OverflowOp>
228 bool inline_math_overflow(Node* arg1, Node* arg2);
229 void inline_math_mathExact(Node* math, Node* test);
230 bool inline_math_addExactI(bool is_increment);
231 bool inline_math_addExactL(bool is_increment);
232 bool inline_math_multiplyExactI();
233 bool inline_math_multiplyExactL();
234 bool inline_math_multiplyHigh();
235 bool inline_math_negateExactI();
236 bool inline_math_negateExactL();
237 bool inline_math_subtractExactI(bool is_decrement);
238 bool inline_math_subtractExactL(bool is_decrement);
239 bool inline_min_max(vmIntrinsics::ID id);
240 bool inline_notify(vmIntrinsics::ID id);
241 Node* generate_min_max(vmIntrinsics::ID id, Node* x, Node* y);
242 // This returns Type::AnyPtr, RawPtr, or OopPtr.
243 int classify_unsafe_addr(Node* &base, Node* &offset, BasicType type);
244 Node* make_unsafe_address(Node*& base, Node* offset, BasicType type = T_ILLEGAL, bool can_cast = false);
245
246 typedef enum { Relaxed, Opaque, Volatile, Acquire, Release } AccessKind;
247 DecoratorSet mo_decorator_for_access_kind(AccessKind kind);
248 bool inline_unsafe_access(bool is_store, BasicType type, AccessKind kind, bool is_unaligned);
249 static bool klass_needs_init_guard(Node* kls);
250 bool inline_unsafe_allocate();
251 bool inline_unsafe_newArray(bool uninitialized);
252 bool inline_unsafe_copyMemory();
253 bool inline_native_currentThread();
254
255 bool inline_native_time_funcs(address method, const char* funcName);
256 #ifdef TRACE_HAVE_INTRINSICS
257 bool inline_native_classID();
258 bool inline_native_getBufferWriter();
259 #endif
260 bool inline_native_isInterrupted();
261 bool inline_native_Class_query(vmIntrinsics::ID id);
262 bool inline_native_subtype_check();
263 bool inline_native_getLength();
264 bool inline_array_copyOf(bool is_copyOfRange);
265 bool inline_array_equals(StrIntrinsicNode::ArgEnc ae);
266 bool inline_preconditions_checkIndex();
267 void copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array);
268 bool inline_native_clone(bool is_virtual);
269 bool inline_native_Reflection_getCallerClass();
270 // Helper function for inlining native object hash method
271 bool inline_native_hashcode(bool is_virtual, bool is_static);
272 bool inline_native_getClass();
273
274 // Helper functions for inlining arraycopy
275 bool inline_arraycopy();
276 AllocateArrayNode* tightly_coupled_allocation(Node* ptr,
277 RegionNode* slow_region);
278 JVMState* arraycopy_restore_alloc_state(AllocateArrayNode* alloc, int& saved_reexecute_sp);
279 void arraycopy_move_allocation_here(AllocateArrayNode* alloc, Node* dest, JVMState* saved_jvms, int saved_reexecute_sp,
280 uint new_idx);
281
282 typedef enum { LS_get_add, LS_get_set, LS_cmp_swap, LS_cmp_swap_weak, LS_cmp_exchange } LoadStoreKind;
283 bool inline_unsafe_load_store(BasicType type, LoadStoreKind kind, AccessKind access_kind);
284 bool inline_unsafe_fence(vmIntrinsics::ID id);
285 bool inline_onspinwait();
286 bool inline_fp_conversions(vmIntrinsics::ID id);
287 bool inline_number_methods(vmIntrinsics::ID id);
288 bool inline_reference_get();
289 bool inline_Class_cast();
290 bool inline_aescrypt_Block(vmIntrinsics::ID id);
291 bool inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id);
292 bool inline_counterMode_AESCrypt(vmIntrinsics::ID id);
293 Node* inline_cipherBlockChaining_AESCrypt_predicate(bool decrypting);
294 Node* inline_counterMode_AESCrypt_predicate();
295 Node* get_key_start_from_aescrypt_object(Node* aescrypt_object);
296 Node* get_original_key_start_from_aescrypt_object(Node* aescrypt_object);
297 bool inline_ghash_processBlocks();
298 bool inline_sha_implCompress(vmIntrinsics::ID id);
299 bool inline_digestBase_implCompressMB(int predicate);
300 bool inline_sha_implCompressMB(Node* digestBaseObj, ciInstanceKlass* instklass_SHA,
301 bool long_state, address stubAddr, const char *stubName,
|
2205 Node* n = NULL;
2206 switch (id) {
2207 case vmIntrinsics::_numberOfLeadingZeros_i: n = new CountLeadingZerosINode( arg); break;
2208 case vmIntrinsics::_numberOfLeadingZeros_l: n = new CountLeadingZerosLNode( arg); break;
2209 case vmIntrinsics::_numberOfTrailingZeros_i: n = new CountTrailingZerosINode(arg); break;
2210 case vmIntrinsics::_numberOfTrailingZeros_l: n = new CountTrailingZerosLNode(arg); break;
2211 case vmIntrinsics::_bitCount_i: n = new PopCountINode( arg); break;
2212 case vmIntrinsics::_bitCount_l: n = new PopCountLNode( arg); break;
2213 case vmIntrinsics::_reverseBytes_c: n = new ReverseBytesUSNode(0, arg); break;
2214 case vmIntrinsics::_reverseBytes_s: n = new ReverseBytesSNode( 0, arg); break;
2215 case vmIntrinsics::_reverseBytes_i: n = new ReverseBytesINode( 0, arg); break;
2216 case vmIntrinsics::_reverseBytes_l: n = new ReverseBytesLNode( 0, arg); break;
2217 default: fatal_unexpected_iid(id); break;
2218 }
2219 set_result(_gvn.transform(n));
2220 return true;
2221 }
2222
2223 //----------------------------inline_unsafe_access----------------------------
2224
2225 // Helper that guards and inserts a pre-barrier.
2226 void LibraryCallKit::insert_pre_barrier(Node* base_oop, Node* offset,
2227 Node* pre_val, bool need_mem_bar) {
2228 // We could be accessing the referent field of a reference object. If so, when G1
2229 // is enabled, we need to log the value in the referent field in an SATB buffer.
2230 // This routine performs some compile time filters and generates suitable
2231 // runtime filters that guard the pre-barrier code.
2232 // Also add memory barrier for non volatile load from the referent field
2233 // to prevent commoning of loads across safepoint.
2234 if (!UseG1GC && !need_mem_bar)
2235 return;
2236
2237 // Some compile time checks.
2238
2239 // If offset is a constant, is it java_lang_ref_Reference::_reference_offset?
2240 const TypeX* otype = offset->find_intptr_t_type();
2241 if (otype != NULL && otype->is_con() &&
2242 otype->get_con() != java_lang_ref_Reference::referent_offset) {
2243 // Constant offset but not the reference_offset so just return
2244 return;
2245 }
2246
2247 // We only need to generate the runtime guards for instances.
2248 const TypeOopPtr* btype = base_oop->bottom_type()->isa_oopptr();
2249 if (btype != NULL) {
2250 if (btype->isa_aryptr()) {
2251 // Array type so nothing to do
2252 return;
2253 }
2254
2255 const TypeInstPtr* itype = btype->isa_instptr();
2256 if (itype != NULL) {
2257 // Can the klass of base_oop be statically determined to be
2258 // _not_ a sub-class of Reference and _not_ Object?
2259 ciKlass* klass = itype->klass();
2260 if ( klass->is_loaded() &&
2261 !klass->is_subtype_of(env()->Reference_klass()) &&
2262 !env()->Object_klass()->is_subtype_of(klass)) {
2263 return;
2264 }
2265 }
2266 }
2267
2268 // The compile time filters did not reject base_oop/offset so
2269 // we need to generate the following runtime filters
2270 //
2271 // if (offset == java_lang_ref_Reference::_reference_offset) {
2272 // if (instance_of(base, java.lang.ref.Reference)) {
2273 // pre_barrier(_, pre_val, ...);
2274 // }
2275 // }
2276
2277 float likely = PROB_LIKELY( 0.999);
2278 float unlikely = PROB_UNLIKELY(0.999);
2279
2280 IdealKit ideal(this);
2281 #define __ ideal.
2282
2283 Node* referent_off = __ ConX(java_lang_ref_Reference::referent_offset);
2284
2285 __ if_then(offset, BoolTest::eq, referent_off, unlikely); {
2286 // Update graphKit memory and control from IdealKit.
2287 sync_kit(ideal);
2288
2289 Node* ref_klass_con = makecon(TypeKlassPtr::make(env()->Reference_klass()));
2290 Node* is_instof = gen_instanceof(base_oop, ref_klass_con);
2291
2292 // Update IdealKit memory and control from graphKit.
2293 __ sync_kit(this);
2294
2295 Node* one = __ ConI(1);
2296 // is_instof == 0 if base_oop == NULL
2297 __ if_then(is_instof, BoolTest::eq, one, unlikely); {
2298
2299 // Update graphKit from IdeakKit.
2300 sync_kit(ideal);
2301
2302 // Use the pre-barrier to record the value in the referent field
2303 pre_barrier(false /* do_load */,
2304 __ ctrl(),
2305 NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
2306 pre_val /* pre_val */,
2307 T_OBJECT);
2308 if (need_mem_bar) {
2309 // Add memory barrier to prevent commoning reads from this field
2310 // across safepoint since GC can change its value.
2311 insert_mem_bar(Op_MemBarCPUOrder);
2312 }
2313 // Update IdealKit from graphKit.
2314 __ sync_kit(this);
2315
2316 } __ end_if(); // _ref_type != ref_none
2317 } __ end_if(); // offset == referent_offset
2318
2319 // Final sync IdealKit and GraphKit.
2320 final_sync(ideal);
2321 #undef __
2322 }
2323
2324
2325 const TypeOopPtr* LibraryCallKit::sharpen_unsafe_type(Compile::AliasType* alias_type, const TypePtr *adr_type) {
2326 // Attempt to infer a sharper value type from the offset and base type.
2327 ciKlass* sharpened_klass = NULL;
2328
2329 // See if it is an instance field, with an object type.
2330 if (alias_type->field() != NULL) {
2331 if (alias_type->field()->type()->is_klass()) {
2332 sharpened_klass = alias_type->field()->type()->as_klass();
2333 }
2334 }
2335
2336 // See if it is a narrow oop array.
2337 if (adr_type->isa_aryptr()) {
2338 if (adr_type->offset() >= objArrayOopDesc::base_offset_in_bytes()) {
2339 const TypeOopPtr *elem_type = adr_type->is_aryptr()->elem()->isa_oopptr();
2340 if (elem_type != NULL) {
2341 sharpened_klass = elem_type->klass();
2342 }
2343 }
2344 }
2345
2346 // The sharpened class might be unloaded if there is no class loader
2347 // contraint in place.
2348 if (sharpened_klass != NULL && sharpened_klass->is_loaded()) {
2349 const TypeOopPtr* tjp = TypeOopPtr::make_from_klass(sharpened_klass);
2350
2351 #ifndef PRODUCT
2352 if (C->print_intrinsics() || C->print_inlining()) {
2353 tty->print(" from base type: "); adr_type->dump(); tty->cr();
2354 tty->print(" sharpened value: "); tjp->dump(); tty->cr();
2355 }
2356 #endif
2357 // Sharpen the value type.
2358 return tjp;
2359 }
2360 return NULL;
2361 }
2362
2363 bool LibraryCallKit::inline_unsafe_access(bool is_store, const BasicType type, const AccessKind kind, const bool unaligned) {
2364 if (callee()->is_static()) return false; // caller must have the capability!
2365 guarantee(!is_store || kind != Acquire, "Acquire accesses can be produced only for loads");
2366 guarantee( is_store || kind != Release, "Release accesses can be produced only for stores");
2367 assert(type != T_OBJECT || !unaligned, "unaligned access not supported with object type");
2368
2369 #ifndef PRODUCT
2370 {
2371 ResourceMark rm;
2372 // Check the signatures.
2373 ciSignature* sig = callee()->signature();
2374 #ifdef ASSERT
2375 if (!is_store) {
2376 // Object getObject(Object base, int/long offset), etc.
2377 BasicType rtype = sig->return_type()->basic_type();
2378 assert(rtype == type, "getter must return the expected value");
2379 assert(sig->count() == 2, "oop getter has 2 arguments");
2380 assert(sig->type_at(0)->basic_type() == T_OBJECT, "getter base is object");
2381 assert(sig->type_at(1)->basic_type() == T_LONG, "getter offset is correct");
2382 } else {
2383 // void putObject(Object base, int/long offset, Object x), etc.
2384 assert(sig->return_type()->basic_type() == T_VOID, "putter must not return a value");
2385 assert(sig->count() == 3, "oop putter has 3 arguments");
2386 assert(sig->type_at(0)->basic_type() == T_OBJECT, "putter base is object");
2387 assert(sig->type_at(1)->basic_type() == T_LONG, "putter offset is correct");
|
2200 Node* n = NULL;
2201 switch (id) {
2202 case vmIntrinsics::_numberOfLeadingZeros_i: n = new CountLeadingZerosINode( arg); break;
2203 case vmIntrinsics::_numberOfLeadingZeros_l: n = new CountLeadingZerosLNode( arg); break;
2204 case vmIntrinsics::_numberOfTrailingZeros_i: n = new CountTrailingZerosINode(arg); break;
2205 case vmIntrinsics::_numberOfTrailingZeros_l: n = new CountTrailingZerosLNode(arg); break;
2206 case vmIntrinsics::_bitCount_i: n = new PopCountINode( arg); break;
2207 case vmIntrinsics::_bitCount_l: n = new PopCountLNode( arg); break;
2208 case vmIntrinsics::_reverseBytes_c: n = new ReverseBytesUSNode(0, arg); break;
2209 case vmIntrinsics::_reverseBytes_s: n = new ReverseBytesSNode( 0, arg); break;
2210 case vmIntrinsics::_reverseBytes_i: n = new ReverseBytesINode( 0, arg); break;
2211 case vmIntrinsics::_reverseBytes_l: n = new ReverseBytesLNode( 0, arg); break;
2212 default: fatal_unexpected_iid(id); break;
2213 }
2214 set_result(_gvn.transform(n));
2215 return true;
2216 }
2217
2218 //----------------------------inline_unsafe_access----------------------------
2219
2220 const TypeOopPtr* LibraryCallKit::sharpen_unsafe_type(Compile::AliasType* alias_type, const TypePtr *adr_type) {
2221 // Attempt to infer a sharper value type from the offset and base type.
2222 ciKlass* sharpened_klass = NULL;
2223
2224 // See if it is an instance field, with an object type.
2225 if (alias_type->field() != NULL) {
2226 if (alias_type->field()->type()->is_klass()) {
2227 sharpened_klass = alias_type->field()->type()->as_klass();
2228 }
2229 }
2230
2231 // See if it is a narrow oop array.
2232 if (adr_type->isa_aryptr()) {
2233 if (adr_type->offset() >= objArrayOopDesc::base_offset_in_bytes()) {
2234 const TypeOopPtr *elem_type = adr_type->is_aryptr()->elem()->isa_oopptr();
2235 if (elem_type != NULL) {
2236 sharpened_klass = elem_type->klass();
2237 }
2238 }
2239 }
2240
2241 // The sharpened class might be unloaded if there is no class loader
2242 // contraint in place.
2243 if (sharpened_klass != NULL && sharpened_klass->is_loaded()) {
2244 const TypeOopPtr* tjp = TypeOopPtr::make_from_klass(sharpened_klass);
2245
2246 #ifndef PRODUCT
2247 if (C->print_intrinsics() || C->print_inlining()) {
2248 tty->print(" from base type: "); adr_type->dump(); tty->cr();
2249 tty->print(" sharpened value: "); tjp->dump(); tty->cr();
2250 }
2251 #endif
2252 // Sharpen the value type.
2253 return tjp;
2254 }
2255 return NULL;
2256 }
2257
2258 DecoratorSet LibraryCallKit::mo_decorator_for_access_kind(AccessKind kind) {
2259 switch (kind) {
2260 case Relaxed:
2261 return MO_UNORDERED;
2262 case Opaque:
2263 return MO_RELAXED;
2264 case Acquire:
2265 return MO_ACQUIRE;
2266 case Release:
2267 return MO_RELEASE;
2268 case Volatile:
2269 return MO_SEQ_CST;
2270 default:
2271 ShouldNotReachHere();
2272 return 0;
2273 }
2274 }
2275
2276 bool LibraryCallKit::inline_unsafe_access(bool is_store, const BasicType type, const AccessKind kind, const bool unaligned) {
2277 if (callee()->is_static()) return false; // caller must have the capability!
2278 DecoratorSet decorators = C2_UNSAFE_ACCESS;
2279 guarantee(!is_store || kind != Acquire, "Acquire accesses can be produced only for loads");
2280 guarantee( is_store || kind != Release, "Release accesses can be produced only for stores");
2281 assert(type != T_OBJECT || !unaligned, "unaligned access not supported with object type");
2282
2283 if (type == T_OBJECT || type == T_ARRAY) {
2284 decorators |= ON_UNKNOWN_OOP_REF;
2285 }
2286
2287 if (unaligned) {
2288 decorators |= C2_UNALIGNED;
2289 }
2290
2291 #ifndef PRODUCT
2292 {
2293 ResourceMark rm;
2294 // Check the signatures.
2295 ciSignature* sig = callee()->signature();
2296 #ifdef ASSERT
2297 if (!is_store) {
2298 // Object getObject(Object base, int/long offset), etc.
2299 BasicType rtype = sig->return_type()->basic_type();
2300 assert(rtype == type, "getter must return the expected value");
2301 assert(sig->count() == 2, "oop getter has 2 arguments");
2302 assert(sig->type_at(0)->basic_type() == T_OBJECT, "getter base is object");
2303 assert(sig->type_at(1)->basic_type() == T_LONG, "getter offset is correct");
2304 } else {
2305 // void putObject(Object base, int/long offset, Object x), etc.
2306 assert(sig->return_type()->basic_type() == T_VOID, "putter must not return a value");
2307 assert(sig->count() == 3, "oop putter has 3 arguments");
2308 assert(sig->type_at(0)->basic_type() == T_OBJECT, "putter base is object");
2309 assert(sig->type_at(1)->basic_type() == T_LONG, "putter offset is correct");
|
2407 // The offset is a value produced by Unsafe.staticFieldOffset or Unsafe.objectFieldOffset
2408 offset = argument(2); // type: long
2409 // We currently rely on the cookies produced by Unsafe.xxxFieldOffset
2410 // to be plain byte offsets, which are also the same as those accepted
2411 // by oopDesc::field_addr.
2412 assert(Unsafe_field_offset_to_byte_offset(11) == 11,
2413 "fieldOffset must be byte-scaled");
2414 // 32-bit machines ignore the high half!
2415 offset = ConvL2X(offset);
2416 adr = make_unsafe_address(base, offset, type, kind == Relaxed);
2417
2418 if (_gvn.type(base)->isa_ptr() != TypePtr::NULL_PTR) {
2419 heap_base_oop = base;
2420 } else if (type == T_OBJECT) {
2421 return false; // off-heap oop accesses are not supported
2422 }
2423
2424 // Can base be NULL? Otherwise, always on-heap access.
2425 bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(_gvn.type(heap_base_oop));
2426
2427 val = is_store ? argument(4) : NULL;
2428
2429 const TypePtr *adr_type = _gvn.type(adr)->isa_ptr();
2430
2431 // Try to categorize the address.
2432 Compile::AliasType* alias_type = C->alias_type(adr_type);
2433 assert(alias_type->index() != Compile::AliasIdxBot, "no bare pointers here");
2434
2435 if (alias_type->adr_type() == TypeInstPtr::KLASS ||
2436 alias_type->adr_type() == TypeAryPtr::RANGE) {
2437 return false; // not supported
2438 }
2439
2440 bool mismatched = false;
2441 BasicType bt = alias_type->basic_type();
2442 if (bt != T_ILLEGAL) {
2443 assert(alias_type->adr_type()->is_oopptr(), "should be on-heap access");
2444 if (bt == T_BYTE && adr_type->isa_aryptr()) {
2445 // Alias type doesn't differentiate between byte[] and boolean[]).
2446 // Use address type to get the element type.
2447 bt = adr_type->is_aryptr()->elem()->array_element_basic_type();
2448 }
2449 if (bt == T_ARRAY || bt == T_NARROWOOP) {
2450 // accessing an array field with getObject is not a mismatch
2451 bt = T_OBJECT;
2452 }
2453 if ((bt == T_OBJECT) != (type == T_OBJECT)) {
2454 // Don't intrinsify mismatched object accesses
2455 return false;
2456 }
2457 mismatched = (bt != type);
2458 } else if (alias_type->adr_type()->isa_oopptr()) {
2459 mismatched = true; // conservatively mark all "wide" on-heap accesses as mismatched
2460 }
2461
2462 assert(!mismatched || alias_type->adr_type()->is_oopptr(), "off-heap access can't be mismatched");
2463
2464 // First guess at the value type.
2465 const Type *value_type = Type::get_const_basic_type(type);
2466
2467 // We will need memory barriers unless we can determine a unique
2468 // alias category for this reference. (Note: If for some reason
2469 // the barriers get omitted and the unsafe reference begins to "pollute"
2470 // the alias analysis of the rest of the graph, either Compile::can_alias
2471 // or Compile::must_alias will throw a diagnostic assert.)
2472 bool need_mem_bar = false;
2473 switch (kind) {
2474 case Relaxed:
2475 need_mem_bar = (mismatched && !adr_type->isa_aryptr()) || can_access_non_heap;
2476 break;
2477 case Opaque:
2478 // Opaque uses CPUOrder membars for protection against code movement.
2479 case Acquire:
2480 case Release:
2481 case Volatile:
2482 need_mem_bar = true;
2483 break;
2484 default:
2485 ShouldNotReachHere();
2486 }
2487
2488 // Some accesses require access atomicity for all types, notably longs and doubles.
2489 // When AlwaysAtomicAccesses is enabled, all accesses are atomic.
2490 bool requires_atomic_access = false;
2491 switch (kind) {
2492 case Relaxed:
2493 requires_atomic_access = AlwaysAtomicAccesses;
2494 break;
2495 case Opaque:
2496 // Opaque accesses are atomic.
2497 case Acquire:
2498 case Release:
2499 case Volatile:
2500 requires_atomic_access = true;
2501 break;
2502 default:
2503 ShouldNotReachHere();
2504 }
2505
2506 // Figure out the memory ordering.
2507 // Acquire/Release/Volatile accesses require marking the loads/stores with MemOrd
2508 MemNode::MemOrd mo = access_kind_to_memord_LS(kind, is_store);
2509
2510 // If we are reading the value of the referent field of a Reference
2511 // object (either by using Unsafe directly or through reflection)
2512 // then, if G1 is enabled, we need to record the referent in an
2513 // SATB log buffer using the pre-barrier mechanism.
2514 // Also we need to add memory barrier to prevent commoning reads
2515 // from this field across safepoint since GC can change its value.
2516 bool need_read_barrier = !is_store &&
2517 offset != top() && heap_base_oop != top();
2518
2519 if (!is_store && type == T_OBJECT) {
2520 const TypeOopPtr* tjp = sharpen_unsafe_type(alias_type, adr_type);
2521 if (tjp != NULL) {
2522 value_type = tjp;
2523 }
2524 }
2525
2526 receiver = null_check(receiver);
2527 if (stopped()) {
2528 return true;
2529 }
2530 // Heap pointers get a null-check from the interpreter,
2531 // as a courtesy. However, this is not guaranteed by Unsafe,
2532 // and it is not possible to fully distinguish unintended nulls
2533 // from intended ones in this API.
2534
2535 // We need to emit leading and trailing CPU membars (see below) in
2536 // addition to memory membars for special access modes. This is a little
2537 // too strong, but avoids the need to insert per-alias-type
2538 // volatile membars (for stores; compare Parse::do_put_xxx), which
2539 // we cannot do effectively here because we probably only have a
2540 // rough approximation of type.
2541
2542 switch(kind) {
2543 case Relaxed:
2544 case Opaque:
2545 case Acquire:
2546 break;
2547 case Release:
2548 case Volatile:
2549 if (is_store) {
2550 insert_mem_bar(Op_MemBarRelease);
2551 } else {
2552 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
2553 insert_mem_bar(Op_MemBarVolatile);
2554 }
2555 }
2556 break;
2557 default:
2558 ShouldNotReachHere();
2559 }
2560
2561 // Memory barrier to prevent normal and 'unsafe' accesses from
2562 // bypassing each other. Happens after null checks, so the
2563 // exception paths do not take memory state from the memory barrier,
2564 // so there's no problems making a strong assert about mixing users
2565 // of safe & unsafe memory.
2566 if (need_mem_bar) insert_mem_bar(Op_MemBarCPUOrder);
2567
2568 if (!is_store) {
2569 Node* p = NULL;
2570 // Try to constant fold a load from a constant field
2571 ciField* field = alias_type->field();
2572 if (heap_base_oop != top() && field != NULL && field->is_constant() && !mismatched) {
2573 // final or stable field
2574 p = make_constant_from_field(field, heap_base_oop);
2575 }
2576 if (p == NULL) {
2577 // To be valid, unsafe loads may depend on other conditions than
2578 // the one that guards them: pin the Load node
2579 LoadNode::ControlDependency dep = LoadNode::Pinned;
2580 Node* ctrl = control();
2581 // non volatile loads may be able to float
2582 if (!need_mem_bar && adr_type->isa_instptr()) {
2583 assert(adr_type->meet(TypePtr::NULL_PTR) != adr_type->remove_speculative(), "should be not null");
2584 intptr_t offset = Type::OffsetBot;
2585 AddPNode::Ideal_base_and_offset(adr, &_gvn, offset);
2586 if (offset >= 0) {
2587 int s = Klass::layout_helper_size_in_bytes(adr_type->isa_instptr()->klass()->layout_helper());
2588 if (offset < s) {
2589 // Guaranteed to be a valid access, no need to pin it
2590 dep = LoadNode::DependsOnlyOnTest;
2591 ctrl = NULL;
2592 }
2593 }
2594 }
2595 p = make_load(ctrl, adr, value_type, type, adr_type, mo, dep, requires_atomic_access, unaligned, mismatched);
2596 // load value
2597 switch (type) {
2598 case T_BOOLEAN:
2599 {
2600 // Normalize the value returned by getBoolean in the following cases
2601 if (mismatched ||
2602 heap_base_oop == top() || // - heap_base_oop is NULL or
2603 (can_access_non_heap && alias_type->field() == NULL) // - heap_base_oop is potentially NULL
2604 // and the unsafe access is made to large offset
2605 // (i.e., larger than the maximum offset necessary for any
2606 // field access)
2607 ) {
2608 IdealKit ideal = IdealKit(this);
2609 #define __ ideal.
2610 IdealVariable normalized_result(ideal);
2611 __ declarations_done();
2612 __ set(normalized_result, p);
2613 __ if_then(p, BoolTest::ne, ideal.ConI(0));
2614 __ set(normalized_result, ideal.ConI(1));
2615 ideal.end_if();
2616 final_sync(ideal);
2617 p = __ value(normalized_result);
2618 #undef __
2619 }
2620 }
2621 case T_CHAR:
2622 case T_BYTE:
2623 case T_SHORT:
2624 case T_INT:
2625 case T_LONG:
2626 case T_FLOAT:
2627 case T_DOUBLE:
2628 break;
2629 case T_OBJECT:
2630 if (need_read_barrier) {
2631 // We do not require a mem bar inside pre_barrier if need_mem_bar
2632 // is set: the barriers would be emitted by us.
2633 insert_pre_barrier(heap_base_oop, offset, p, !need_mem_bar);
2634 }
2635 break;
2636 case T_ADDRESS:
2637 // Cast to an int type.
2638 p = _gvn.transform(new CastP2XNode(NULL, p));
2639 p = ConvX2UL(p);
2640 break;
2641 default:
2642 fatal("unexpected type %d: %s", type, type2name(type));
2643 break;
2644 }
2645 }
2646 // The load node has the control of the preceding MemBarCPUOrder. All
2647 // following nodes will have the control of the MemBarCPUOrder inserted at
2648 // the end of this method. So, pushing the load onto the stack at a later
2649 // point is fine.
2650 set_result(p);
2651 } else {
2652 // place effect of store into memory
2653 switch (type) {
2654 case T_DOUBLE:
2655 val = dstore_rounding(val);
2656 break;
2657 case T_ADDRESS:
2658 // Repackage the long as a pointer.
2659 val = ConvL2X(val);
2660 val = _gvn.transform(new CastX2PNode(val));
2661 break;
2662 default:
2663 break;
2664 }
2665
2666 if (type == T_OBJECT) {
2667 store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, type, mo, mismatched);
2668 } else {
2669 store_to_memory(control(), adr, val, type, adr_type, mo, requires_atomic_access, unaligned, mismatched);
2670 }
2671 }
2672
2673 switch(kind) {
2674 case Relaxed:
2675 case Opaque:
2676 case Release:
2677 break;
2678 case Acquire:
2679 case Volatile:
2680 if (!is_store) {
2681 insert_mem_bar(Op_MemBarAcquire);
2682 } else {
2683 if (!support_IRIW_for_not_multiple_copy_atomic_cpu) {
2684 insert_mem_bar(Op_MemBarVolatile);
2685 }
2686 }
2687 break;
2688 default:
2689 ShouldNotReachHere();
2690 }
2691
2692 if (need_mem_bar) insert_mem_bar(Op_MemBarCPUOrder);
2693
2694 return true;
2695 }
2696
2697 //----------------------------inline_unsafe_load_store----------------------------
2698 // This method serves a couple of different customers (depending on LoadStoreKind):
2699 //
2700 // LS_cmp_swap:
2701 //
2702 // boolean compareAndSetObject(Object o, long offset, Object expected, Object x);
2703 // boolean compareAndSetInt( Object o, long offset, int expected, int x);
2704 // boolean compareAndSetLong( Object o, long offset, long expected, long x);
2705 //
2706 // LS_cmp_swap_weak:
2707 //
2708 // boolean weakCompareAndSetObject( Object o, long offset, Object expected, Object x);
2709 // boolean weakCompareAndSetObjectPlain( Object o, long offset, Object expected, Object x);
2710 // boolean weakCompareAndSetObjectAcquire(Object o, long offset, Object expected, Object x);
2711 // boolean weakCompareAndSetObjectRelease(Object o, long offset, Object expected, Object x);
2712 //
|
2329 // The offset is a value produced by Unsafe.staticFieldOffset or Unsafe.objectFieldOffset
2330 offset = argument(2); // type: long
2331 // We currently rely on the cookies produced by Unsafe.xxxFieldOffset
2332 // to be plain byte offsets, which are also the same as those accepted
2333 // by oopDesc::field_addr.
2334 assert(Unsafe_field_offset_to_byte_offset(11) == 11,
2335 "fieldOffset must be byte-scaled");
2336 // 32-bit machines ignore the high half!
2337 offset = ConvL2X(offset);
2338 adr = make_unsafe_address(base, offset, type, kind == Relaxed);
2339
2340 if (_gvn.type(base)->isa_ptr() != TypePtr::NULL_PTR) {
2341 heap_base_oop = base;
2342 } else if (type == T_OBJECT) {
2343 return false; // off-heap oop accesses are not supported
2344 }
2345
2346 // Can base be NULL? Otherwise, always on-heap access.
2347 bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(_gvn.type(heap_base_oop));
2348
2349 if (!can_access_non_heap) {
2350 decorators |= IN_HEAP;
2351 }
2352
2353 val = is_store ? argument(4) : NULL;
2354
2355 const TypePtr *adr_type = _gvn.type(adr)->isa_ptr();
2356
2357 // Try to categorize the address.
2358 Compile::AliasType* alias_type = C->alias_type(adr_type);
2359 assert(alias_type->index() != Compile::AliasIdxBot, "no bare pointers here");
2360
2361 if (alias_type->adr_type() == TypeInstPtr::KLASS ||
2362 alias_type->adr_type() == TypeAryPtr::RANGE) {
2363 return false; // not supported
2364 }
2365
2366 bool mismatched = false;
2367 BasicType bt = alias_type->basic_type();
2368 if (bt != T_ILLEGAL) {
2369 assert(alias_type->adr_type()->is_oopptr(), "should be on-heap access");
2370 if (bt == T_BYTE && adr_type->isa_aryptr()) {
2371 // Alias type doesn't differentiate between byte[] and boolean[]).
2372 // Use address type to get the element type.
2373 bt = adr_type->is_aryptr()->elem()->array_element_basic_type();
2374 }
2375 if (bt == T_ARRAY || bt == T_NARROWOOP) {
2376 // accessing an array field with getObject is not a mismatch
2377 bt = T_OBJECT;
2378 }
2379 if ((bt == T_OBJECT) != (type == T_OBJECT)) {
2380 // Don't intrinsify mismatched object accesses
2381 return false;
2382 }
2383 mismatched = (bt != type);
2384 } else if (alias_type->adr_type()->isa_oopptr()) {
2385 mismatched = true; // conservatively mark all "wide" on-heap accesses as mismatched
2386 }
2387
2388 assert(!mismatched || alias_type->adr_type()->is_oopptr(), "off-heap access can't be mismatched");
2389
2390 if (mismatched) {
2391 decorators |= C2_MISMATCHED;
2392 }
2393
2394 // First guess at the value type.
2395 const Type *value_type = Type::get_const_basic_type(type);
2396
2397 // Figure out the memory ordering.
2398 decorators |= mo_decorator_for_access_kind(kind);
2399
2400 if (!is_store && type == T_OBJECT) {
2401 const TypeOopPtr* tjp = sharpen_unsafe_type(alias_type, adr_type);
2402 if (tjp != NULL) {
2403 value_type = tjp;
2404 }
2405 }
2406
2407 receiver = null_check(receiver);
2408 if (stopped()) {
2409 return true;
2410 }
2411 // Heap pointers get a null-check from the interpreter,
2412 // as a courtesy. However, this is not guaranteed by Unsafe,
2413 // and it is not possible to fully distinguish unintended nulls
2414 // from intended ones in this API.
2415
2416 if (!is_store) {
2417 Node* p = NULL;
2418 // Try to constant fold a load from a constant field
2419 ciField* field = alias_type->field();
2420 if (heap_base_oop != top() && field != NULL && field->is_constant() && !mismatched) {
2421 // final or stable field
2422 p = make_constant_from_field(field, heap_base_oop);
2423 }
2424
2425 if (p == NULL) { // Could not constant fold the load
2426 p = access_load_at(heap_base_oop, adr, adr_type, value_type, type, decorators);
2427 // Normalize the value returned by getBoolean in the following cases
2428 if (type == T_BOOLEAN &&
2429 (mismatched ||
2430 heap_base_oop == top() || // - heap_base_oop is NULL or
2431 (can_access_non_heap && field == NULL)) // - heap_base_oop is potentially NULL
2432 // and the unsafe access is made to large offset
2433 // (i.e., larger than the maximum offset necessary for any
2434 // field access)
2435 ) {
2436 IdealKit ideal = IdealKit(this);
2437 #define __ ideal.
2438 IdealVariable normalized_result(ideal);
2439 __ declarations_done();
2440 __ set(normalized_result, p);
2441 __ if_then(p, BoolTest::ne, ideal.ConI(0));
2442 __ set(normalized_result, ideal.ConI(1));
2443 ideal.end_if();
2444 final_sync(ideal);
2445 p = __ value(normalized_result);
2446 #undef __
2447 }
2448 }
2449 if (type == T_ADDRESS) {
2450 p = gvn().transform(new CastP2XNode(NULL, p));
2451 p = ConvX2UL(p);
2452 }
2453 // The load node has the control of the preceding MemBarCPUOrder. All
2454 // following nodes will have the control of the MemBarCPUOrder inserted at
2455 // the end of this method. So, pushing the load onto the stack at a later
2456 // point is fine.
2457 set_result(p);
2458 } else {
2459 if (bt == T_ADDRESS) {
2460 // Repackage the long as a pointer.
2461 val = ConvL2X(val);
2462 val = gvn().transform(new CastX2PNode(val));
2463 }
2464 access_store_at(control(), heap_base_oop, adr, adr_type, val, value_type, type, decorators);
2465 }
2466
2467 return true;
2468 }
2469
2470 //----------------------------inline_unsafe_load_store----------------------------
2471 // This method serves a couple of different customers (depending on LoadStoreKind):
2472 //
2473 // LS_cmp_swap:
2474 //
2475 // boolean compareAndSetObject(Object o, long offset, Object expected, Object x);
2476 // boolean compareAndSetInt( Object o, long offset, int expected, int x);
2477 // boolean compareAndSetLong( Object o, long offset, long expected, long x);
2478 //
2479 // LS_cmp_swap_weak:
2480 //
2481 // boolean weakCompareAndSetObject( Object o, long offset, Object expected, Object x);
2482 // boolean weakCompareAndSetObjectPlain( Object o, long offset, Object expected, Object x);
2483 // boolean weakCompareAndSetObjectAcquire(Object o, long offset, Object expected, Object x);
2484 // boolean weakCompareAndSetObjectRelease(Object o, long offset, Object expected, Object x);
2485 //
|
2738 //
2739 // int getAndAddInt( Object o, long offset, int delta)
2740 // long getAndAddLong(Object o, long offset, long delta)
2741 //
2742 // LS_get_set:
2743 //
2744 // int getAndSet(Object o, long offset, int newValue)
2745 // long getAndSet(Object o, long offset, long newValue)
2746 // Object getAndSet(Object o, long offset, Object newValue)
2747 //
2748 bool LibraryCallKit::inline_unsafe_load_store(const BasicType type, const LoadStoreKind kind, const AccessKind access_kind) {
2749 // This basic scheme here is the same as inline_unsafe_access, but
2750 // differs in enough details that combining them would make the code
2751 // overly confusing. (This is a true fact! I originally combined
2752 // them, but even I was confused by it!) As much code/comments as
2753 // possible are retained from inline_unsafe_access though to make
2754 // the correspondences clearer. - dl
2755
2756 if (callee()->is_static()) return false; // caller must have the capability!
2757
2758 #ifndef PRODUCT
2759 BasicType rtype;
2760 {
2761 ResourceMark rm;
2762 // Check the signatures.
2763 ciSignature* sig = callee()->signature();
2764 rtype = sig->return_type()->basic_type();
2765 switch(kind) {
2766 case LS_get_add:
2767 case LS_get_set: {
2768 // Check the signatures.
2769 #ifdef ASSERT
2770 assert(rtype == type, "get and set must return the expected type");
2771 assert(sig->count() == 3, "get and set has 3 arguments");
2772 assert(sig->type_at(0)->basic_type() == T_OBJECT, "get and set base is object");
2773 assert(sig->type_at(1)->basic_type() == T_LONG, "get and set offset is long");
2774 assert(sig->type_at(2)->basic_type() == type, "get and set must take expected type as new value/delta");
2775 assert(access_kind == Volatile, "mo is not passed to intrinsic nodes in current implementation");
2776 #endif // ASSERT
|
2511 //
2512 // int getAndAddInt( Object o, long offset, int delta)
2513 // long getAndAddLong(Object o, long offset, long delta)
2514 //
2515 // LS_get_set:
2516 //
2517 // int getAndSet(Object o, long offset, int newValue)
2518 // long getAndSet(Object o, long offset, long newValue)
2519 // Object getAndSet(Object o, long offset, Object newValue)
2520 //
2521 bool LibraryCallKit::inline_unsafe_load_store(const BasicType type, const LoadStoreKind kind, const AccessKind access_kind) {
2522 // This basic scheme here is the same as inline_unsafe_access, but
2523 // differs in enough details that combining them would make the code
2524 // overly confusing. (This is a true fact! I originally combined
2525 // them, but even I was confused by it!) As much code/comments as
2526 // possible are retained from inline_unsafe_access though to make
2527 // the correspondences clearer. - dl
2528
2529 if (callee()->is_static()) return false; // caller must have the capability!
2530
2531 DecoratorSet decorators = C2_UNSAFE_ACCESS;
2532 decorators |= mo_decorator_for_access_kind(access_kind);
2533
2534 #ifndef PRODUCT
2535 BasicType rtype;
2536 {
2537 ResourceMark rm;
2538 // Check the signatures.
2539 ciSignature* sig = callee()->signature();
2540 rtype = sig->return_type()->basic_type();
2541 switch(kind) {
2542 case LS_get_add:
2543 case LS_get_set: {
2544 // Check the signatures.
2545 #ifdef ASSERT
2546 assert(rtype == type, "get and set must return the expected type");
2547 assert(sig->count() == 3, "get and set has 3 arguments");
2548 assert(sig->type_at(0)->basic_type() == T_OBJECT, "get and set base is object");
2549 assert(sig->type_at(1)->basic_type() == T_LONG, "get and set offset is long");
2550 assert(sig->type_at(2)->basic_type() == type, "get and set must take expected type as new value/delta");
2551 assert(access_kind == Volatile, "mo is not passed to intrinsic nodes in current implementation");
2552 #endif // ASSERT
|
2869 }
2870 }
2871 break;
2872 }
2873 case LS_cmp_swap:
2874 case LS_cmp_swap_weak:
2875 case LS_get_add:
2876 break;
2877 default:
2878 ShouldNotReachHere();
2879 }
2880
2881 // Null check receiver.
2882 receiver = null_check(receiver);
2883 if (stopped()) {
2884 return true;
2885 }
2886
2887 int alias_idx = C->get_alias_index(adr_type);
2888
2889 // Memory-model-wise, a LoadStore acts like a little synchronized
2890 // block, so needs barriers on each side. These don't translate
2891 // into actual barriers on most machines, but we still need rest of
2892 // compiler to respect ordering.
2893
2894 switch (access_kind) {
2895 case Relaxed:
2896 case Acquire:
2897 break;
2898 case Release:
2899 insert_mem_bar(Op_MemBarRelease);
2900 break;
2901 case Volatile:
2902 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
2903 insert_mem_bar(Op_MemBarVolatile);
2904 } else {
2905 insert_mem_bar(Op_MemBarRelease);
2906 }
2907 break;
2908 default:
2909 ShouldNotReachHere();
2910 }
2911 insert_mem_bar(Op_MemBarCPUOrder);
2912
2913 // Figure out the memory ordering.
2914 MemNode::MemOrd mo = access_kind_to_memord(access_kind);
2915
2916 // 4984716: MemBars must be inserted before this
2917 // memory node in order to avoid a false
2918 // dependency which will confuse the scheduler.
2919 Node *mem = memory(alias_idx);
2920
2921 // For now, we handle only those cases that actually exist: ints,
2922 // longs, and Object. Adding others should be straightforward.
2923 Node* load_store = NULL;
2924 switch(type) {
2925 case T_BYTE:
2926 switch(kind) {
2927 case LS_get_add:
2928 load_store = _gvn.transform(new GetAndAddBNode(control(), mem, adr, newval, adr_type));
2929 break;
2930 case LS_get_set:
2931 load_store = _gvn.transform(new GetAndSetBNode(control(), mem, adr, newval, adr_type));
2932 break;
2933 case LS_cmp_swap_weak:
2934 load_store = _gvn.transform(new WeakCompareAndSwapBNode(control(), mem, adr, newval, oldval, mo));
2935 break;
2936 case LS_cmp_swap:
2937 load_store = _gvn.transform(new CompareAndSwapBNode(control(), mem, adr, newval, oldval, mo));
2938 break;
2939 case LS_cmp_exchange:
2940 load_store = _gvn.transform(new CompareAndExchangeBNode(control(), mem, adr, newval, oldval, adr_type, mo));
2941 break;
2942 default:
2943 ShouldNotReachHere();
2944 }
2945 break;
2946 case T_SHORT:
2947 switch(kind) {
2948 case LS_get_add:
2949 load_store = _gvn.transform(new GetAndAddSNode(control(), mem, adr, newval, adr_type));
2950 break;
2951 case LS_get_set:
2952 load_store = _gvn.transform(new GetAndSetSNode(control(), mem, adr, newval, adr_type));
2953 break;
2954 case LS_cmp_swap_weak:
2955 load_store = _gvn.transform(new WeakCompareAndSwapSNode(control(), mem, adr, newval, oldval, mo));
2956 break;
2957 case LS_cmp_swap:
2958 load_store = _gvn.transform(new CompareAndSwapSNode(control(), mem, adr, newval, oldval, mo));
2959 break;
2960 case LS_cmp_exchange:
2961 load_store = _gvn.transform(new CompareAndExchangeSNode(control(), mem, adr, newval, oldval, adr_type, mo));
2962 break;
2963 default:
2964 ShouldNotReachHere();
2965 }
2966 break;
2967 case T_INT:
2968 switch(kind) {
2969 case LS_get_add:
2970 load_store = _gvn.transform(new GetAndAddINode(control(), mem, adr, newval, adr_type));
2971 break;
2972 case LS_get_set:
2973 load_store = _gvn.transform(new GetAndSetINode(control(), mem, adr, newval, adr_type));
2974 break;
2975 case LS_cmp_swap_weak:
2976 load_store = _gvn.transform(new WeakCompareAndSwapINode(control(), mem, adr, newval, oldval, mo));
2977 break;
2978 case LS_cmp_swap:
2979 load_store = _gvn.transform(new CompareAndSwapINode(control(), mem, adr, newval, oldval, mo));
2980 break;
2981 case LS_cmp_exchange:
2982 load_store = _gvn.transform(new CompareAndExchangeINode(control(), mem, adr, newval, oldval, adr_type, mo));
2983 break;
2984 default:
2985 ShouldNotReachHere();
2986 }
2987 break;
2988 case T_LONG:
2989 switch(kind) {
2990 case LS_get_add:
2991 load_store = _gvn.transform(new GetAndAddLNode(control(), mem, adr, newval, adr_type));
2992 break;
2993 case LS_get_set:
2994 load_store = _gvn.transform(new GetAndSetLNode(control(), mem, adr, newval, adr_type));
2995 break;
2996 case LS_cmp_swap_weak:
2997 load_store = _gvn.transform(new WeakCompareAndSwapLNode(control(), mem, adr, newval, oldval, mo));
2998 break;
2999 case LS_cmp_swap:
3000 load_store = _gvn.transform(new CompareAndSwapLNode(control(), mem, adr, newval, oldval, mo));
3001 break;
3002 case LS_cmp_exchange:
3003 load_store = _gvn.transform(new CompareAndExchangeLNode(control(), mem, adr, newval, oldval, adr_type, mo));
3004 break;
3005 default:
3006 ShouldNotReachHere();
3007 }
3008 break;
3009 case T_OBJECT:
3010 // Transformation of a value which could be NULL pointer (CastPP #NULL)
3011 // could be delayed during Parse (for example, in adjust_map_after_if()).
3012 // Execute transformation here to avoid barrier generation in such case.
3013 if (_gvn.type(newval) == TypePtr::NULL_PTR)
3014 newval = _gvn.makecon(TypePtr::NULL_PTR);
3015
3016 // Reference stores need a store barrier.
3017 switch(kind) {
3018 case LS_get_set: {
3019 // If pre-barrier must execute before the oop store, old value will require do_load here.
3020 if (!can_move_pre_barrier()) {
3021 pre_barrier(true /* do_load*/,
3022 control(), base, adr, alias_idx, newval, value_type->make_oopptr(),
3023 NULL /* pre_val*/,
3024 T_OBJECT);
3025 } // Else move pre_barrier to use load_store value, see below.
3026 break;
3027 }
3028 case LS_cmp_swap_weak:
3029 case LS_cmp_swap:
3030 case LS_cmp_exchange: {
3031 // Same as for newval above:
3032 if (_gvn.type(oldval) == TypePtr::NULL_PTR) {
3033 oldval = _gvn.makecon(TypePtr::NULL_PTR);
3034 }
3035 // The only known value which might get overwritten is oldval.
3036 pre_barrier(false /* do_load */,
3037 control(), NULL, NULL, max_juint, NULL, NULL,
3038 oldval /* pre_val */,
3039 T_OBJECT);
3040 break;
3041 }
3042 default:
3043 ShouldNotReachHere();
3044 }
3045
3046 #ifdef _LP64
3047 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
3048 Node *newval_enc = _gvn.transform(new EncodePNode(newval, newval->bottom_type()->make_narrowoop()));
3049
3050 switch(kind) {
3051 case LS_get_set:
3052 load_store = _gvn.transform(new GetAndSetNNode(control(), mem, adr, newval_enc, adr_type, value_type->make_narrowoop()));
3053 break;
3054 case LS_cmp_swap_weak: {
3055 Node *oldval_enc = _gvn.transform(new EncodePNode(oldval, oldval->bottom_type()->make_narrowoop()));
3056 load_store = _gvn.transform(new WeakCompareAndSwapNNode(control(), mem, adr, newval_enc, oldval_enc, mo));
3057 break;
3058 }
3059 case LS_cmp_swap: {
3060 Node *oldval_enc = _gvn.transform(new EncodePNode(oldval, oldval->bottom_type()->make_narrowoop()));
3061 load_store = _gvn.transform(new CompareAndSwapNNode(control(), mem, adr, newval_enc, oldval_enc, mo));
3062 break;
3063 }
3064 case LS_cmp_exchange: {
3065 Node *oldval_enc = _gvn.transform(new EncodePNode(oldval, oldval->bottom_type()->make_narrowoop()));
3066 load_store = _gvn.transform(new CompareAndExchangeNNode(control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->
3067 break;
3068 }
3069 default:
3070 ShouldNotReachHere();
3071 }
3072 } else
3073 #endif
3074 switch (kind) {
3075 case LS_get_set:
3076 load_store = _gvn.transform(new GetAndSetPNode(control(), mem, adr, newval, adr_type, value_type->is_oopptr()));
3077 break;
3078 case LS_cmp_swap_weak:
3079 load_store = _gvn.transform(new WeakCompareAndSwapPNode(control(), mem, adr, newval, oldval, mo));
3080 break;
3081 case LS_cmp_swap:
3082 load_store = _gvn.transform(new CompareAndSwapPNode(control(), mem, adr, newval, oldval, mo));
3083 break;
3084 case LS_cmp_exchange:
3085 load_store = _gvn.transform(new CompareAndExchangePNode(control(), mem, adr, newval, oldval, adr_type, value_type->is_oopptr(
3086 break;
3087 default:
3088 ShouldNotReachHere();
3089 }
3090
3091 // Emit the post barrier only when the actual store happened. This makes sense
3092 // to check only for LS_cmp_* that can fail to set the value.
3093 // LS_cmp_exchange does not produce any branches by default, so there is no
3094 // boolean result to piggyback on. TODO: When we merge CompareAndSwap with
3095 // CompareAndExchange and move branches here, it would make sense to conditionalize
3096 // post_barriers for LS_cmp_exchange as well.
3097 //
3098 // CAS success path is marked more likely since we anticipate this is a performance
3099 // critical path, while CAS failure path can use the penalty for going through unlikely
3100 // path as backoff. Which is still better than doing a store barrier there.
3101 switch (kind) {
3102 case LS_get_set:
3103 case LS_cmp_exchange: {
3104 post_barrier(control(), load_store, base, adr, alias_idx, newval, T_OBJECT, true);
3105 break;
3106 }
3107 case LS_cmp_swap_weak:
3108 case LS_cmp_swap: {
3109 IdealKit ideal(this);
3110 ideal.if_then(load_store, BoolTest::ne, ideal.ConI(0), PROB_STATIC_FREQUENT); {
3111 sync_kit(ideal);
3112 post_barrier(ideal.ctrl(), load_store, base, adr, alias_idx, newval, T_OBJECT, true);
3113 ideal.sync_kit(this);
3114 } ideal.end_if();
3115 final_sync(ideal);
3116 break;
3117 }
3118 default:
3119 ShouldNotReachHere();
3120 }
3121 break;
3122 default:
3123 fatal("unexpected type %d: %s", type, type2name(type));
3124 break;
3125 }
3126
3127 // SCMemProjNodes represent the memory state of a LoadStore. Their
3128 // main role is to prevent LoadStore nodes from being optimized away
3129 // when their results aren't used.
3130 Node* proj = _gvn.transform(new SCMemProjNode(load_store));
3131 set_memory(proj, alias_idx);
3132
3133 if (type == T_OBJECT && (kind == LS_get_set || kind == LS_cmp_exchange)) {
3134 #ifdef _LP64
3135 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
3136 load_store = _gvn.transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
3137 }
3138 #endif
3139 if (can_move_pre_barrier() && kind == LS_get_set) {
3140 // Don't need to load pre_val. The old value is returned by load_store.
3141 // The pre_barrier can execute after the xchg as long as no safepoint
3142 // gets inserted between them.
3143 pre_barrier(false /* do_load */,
3144 control(), NULL, NULL, max_juint, NULL, NULL,
3145 load_store /* pre_val */,
3146 T_OBJECT);
3147 }
3148 }
3149
3150 // Add the trailing membar surrounding the access
3151 insert_mem_bar(Op_MemBarCPUOrder);
3152
3153 switch (access_kind) {
3154 case Relaxed:
3155 case Release:
3156 break; // do nothing
3157 case Acquire:
3158 case Volatile:
3159 insert_mem_bar(Op_MemBarAcquire);
3160 // !support_IRIW_for_not_multiple_copy_atomic_cpu handled in platform code
3161 break;
3162 default:
3163 ShouldNotReachHere();
3164 }
3165
3166 assert(type2size[load_store->bottom_type()->basic_type()] == type2size[rtype], "result type should match");
3167 set_result(load_store);
3168 return true;
3169 }
3170
3171 MemNode::MemOrd LibraryCallKit::access_kind_to_memord_LS(AccessKind kind, bool is_store) {
3172 MemNode::MemOrd mo = MemNode::unset;
3173 switch(kind) {
3174 case Opaque:
3175 case Relaxed: mo = MemNode::unordered; break;
3176 case Acquire: mo = MemNode::acquire; break;
3177 case Release: mo = MemNode::release; break;
3178 case Volatile: mo = is_store ? MemNode::release : MemNode::acquire; break;
3179 default:
3180 ShouldNotReachHere();
3181 }
3182 guarantee(mo != MemNode::unset, "Should select memory ordering");
3183 return mo;
3184 }
3185
3186 MemNode::MemOrd LibraryCallKit::access_kind_to_memord(AccessKind kind) {
3187 MemNode::MemOrd mo = MemNode::unset;
3188 switch(kind) {
3189 case Opaque:
3190 case Relaxed: mo = MemNode::unordered; break;
3191 case Acquire: mo = MemNode::acquire; break;
3192 case Release: mo = MemNode::release; break;
3193 case Volatile: mo = MemNode::seqcst; break;
3194 default:
3195 ShouldNotReachHere();
3196 }
3197 guarantee(mo != MemNode::unset, "Should select memory ordering");
3198 return mo;
3199 }
3200
3201 bool LibraryCallKit::inline_unsafe_fence(vmIntrinsics::ID id) {
3202 // Regardless of form, don't allow previous ld/st to move down,
3203 // then issue acquire, release, or volatile mem_bar.
3204 insert_mem_bar(Op_MemBarCPUOrder);
3205 switch(id) {
3206 case vmIntrinsics::_loadFence:
3207 insert_mem_bar(Op_LoadFence);
3208 return true;
3209 case vmIntrinsics::_storeFence:
3210 insert_mem_bar(Op_StoreFence);
3211 return true;
3212 case vmIntrinsics::_fullFence:
3213 insert_mem_bar(Op_MemBarVolatile);
3214 return true;
3215 default:
3216 fatal_unexpected_iid(id);
3217 return false;
3218 }
3219 }
|
2645 }
2646 }
2647 break;
2648 }
2649 case LS_cmp_swap:
2650 case LS_cmp_swap_weak:
2651 case LS_get_add:
2652 break;
2653 default:
2654 ShouldNotReachHere();
2655 }
2656
2657 // Null check receiver.
2658 receiver = null_check(receiver);
2659 if (stopped()) {
2660 return true;
2661 }
2662
2663 int alias_idx = C->get_alias_index(adr_type);
2664
2665 if (type == T_OBJECT || type == T_ARRAY) {
2666 decorators |= IN_HEAP | ON_UNKNOWN_OOP_REF;
2667
2668 // Transformation of a value which could be NULL pointer (CastPP #NULL)
2669 // could be delayed during Parse (for example, in adjust_map_after_if()).
2670 // Execute transformation here to avoid barrier generation in such case.
2671 if (_gvn.type(newval) == TypePtr::NULL_PTR)
2672 newval = _gvn.makecon(TypePtr::NULL_PTR);
2673
2674 if (oldval != NULL && _gvn.type(oldval) == TypePtr::NULL_PTR) {
2675 // Refine the value to a null constant, when it is known to be null
2676 oldval = _gvn.makecon(TypePtr::NULL_PTR);
2677 }
2678 }
2679
2680 Node* result = NULL;
2681 switch (kind) {
2682 case LS_cmp_exchange: {
2683 result = access_atomic_cmpxchg_val_at(control(), base, adr, adr_type, alias_idx,
2684 oldval, newval, value_type, type, decorators);
2685 break;
2686 }
2687 case LS_cmp_swap_weak:
2688 decorators |= C2_WEAK_CMPXCHG;
2689 case LS_cmp_swap: {
2690 result = access_atomic_cmpxchg_bool_at(control(), base, adr, adr_type, alias_idx,
2691 oldval, newval, value_type, type, decorators);
2692 break;
2693 }
2694 case LS_get_set: {
2695 result = access_atomic_xchg_at(control(), base, adr, adr_type, alias_idx,
2696 newval, value_type, type, decorators);
2697 break;
2698 }
2699 case LS_get_add: {
2700 result = access_atomic_add_at(control(), base, adr, adr_type, alias_idx,
2701 newval, value_type, type, decorators);
2702 break;
2703 }
2704 default:
2705 ShouldNotReachHere();
2706 }
2707
2708 assert(type2size[result->bottom_type()->basic_type()] == type2size[rtype], "result type should match");
2709 set_result(result);
2710 return true;
2711 }
2712
2713 bool LibraryCallKit::inline_unsafe_fence(vmIntrinsics::ID id) {
2714 // Regardless of form, don't allow previous ld/st to move down,
2715 // then issue acquire, release, or volatile mem_bar.
2716 insert_mem_bar(Op_MemBarCPUOrder);
2717 switch(id) {
2718 case vmIntrinsics::_loadFence:
2719 insert_mem_bar(Op_LoadFence);
2720 return true;
2721 case vmIntrinsics::_storeFence:
2722 insert_mem_bar(Op_StoreFence);
2723 return true;
2724 case vmIntrinsics::_fullFence:
2725 insert_mem_bar(Op_MemBarVolatile);
2726 return true;
2727 default:
2728 fatal_unexpected_iid(id);
2729 return false;
2730 }
2731 }
|
4617 // Conservatively insert a memory barrier on all memory slices. 4618 // Do not let writes of the copy source or destination float below the copy. 4619 insert_mem_bar(Op_MemBarCPUOrder); 4620 4621 // Call it. Note that the length argument is not scaled. 4622 make_runtime_call(RC_LEAF|RC_NO_FP, 4623 OptoRuntime::fast_arraycopy_Type(), 4624 StubRoutines::unsafe_arraycopy(), 4625 "unsafe_arraycopy", 4626 TypeRawPtr::BOTTOM, 4627 src, dst, size XTOP); 4628 4629 // Do not let reads of the copy destination float above the copy. 4630 insert_mem_bar(Op_MemBarCPUOrder); 4631 4632 return true; 4633 } 4634 4635 //------------------------clone_coping----------------------------------- 4636 // Helper function for inline_native_clone. 4637 void LibraryCallKit::copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark) { 4638 assert(obj_size != NULL, ""); 4639 Node* raw_obj = alloc_obj->in(1); 4640 assert(alloc_obj->is_CheckCastPP() && raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), ""); 4641 4642 AllocateNode* alloc = NULL; 4643 if (ReduceBulkZeroing) { 4644 // We will be completely responsible for initializing this object - 4645 // mark Initialize node as complete. 4646 alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn); 4647 // The object was just allocated - there should be no any stores! 4648 guarantee(alloc != NULL && alloc->maybe_set_complete(&_gvn), ""); 4649 // Mark as complete_with_arraycopy so that on AllocateNode 4650 // expansion, we know this AllocateNode is initialized by an array 4651 // copy and a StoreStore barrier exists after the array copy. 4652 alloc->initialization()->set_complete_with_arraycopy(); 4653 } 4654 4655 // Copy the fastest available way. 4656 // TODO: generate fields copies for small objects instead. 4657 Node* src = obj; 4658 Node* dest = alloc_obj; 4659 Node* size = _gvn.transform(obj_size); 4660 4661 // Exclude the header but include array length to copy by 8 bytes words. 4662 // Can't use base_offset_in_bytes(bt) since basic type is unknown. 4663 int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() : 4664 instanceOopDesc::base_offset_in_bytes(); 4665 // base_off: 4666 // 8 - 32-bit VM 4667 // 12 - 64-bit VM, compressed klass 4668 // 16 - 64-bit VM, normal klass 4669 if (base_off % BytesPerLong != 0) { 4670 assert(UseCompressedClassPointers, ""); 4671 if (is_array) { 4672 // Exclude length to copy by 8 bytes words. 4673 base_off += sizeof(int); 4674 } else { 4675 // Include klass to copy by 8 bytes words. 4676 base_off = instanceOopDesc::klass_offset_in_bytes(); 4677 } 4678 assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment"); 4679 } 4680 src = basic_plus_adr(src, base_off); 4681 dest = basic_plus_adr(dest, base_off); 4682 4683 // Compute the length also, if needed: 4684 Node* countx = size; 4685 countx = _gvn.transform(new SubXNode(countx, MakeConX(base_off))); 4686 countx = _gvn.transform(new URShiftXNode(countx, intcon(LogBytesPerLong) )); 4687 4688 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM; 4689 4690 ArrayCopyNode* ac = ArrayCopyNode::make(this, false, src, NULL, dest, NULL, countx, false, false); 4691 ac->set_clonebasic(); 4692 Node* n = _gvn.transform(ac); 4693 if (n == ac) { 4694 set_predefined_output_for_runtime_call(ac, ac->in(TypeFunc::Memory), raw_adr_type); 4695 } else { 4696 set_all_memory(n); 4697 } 4698 4699 // If necessary, emit some card marks afterwards. (Non-arrays only.) 4700 if (card_mark) { 4701 assert(!is_array, ""); 4702 // Put in store barrier for any and all oops we are sticking 4703 // into this object. (We could avoid this if we could prove 4704 // that the object type contains no oop fields at all.) 4705 Node* no_particular_value = NULL; 4706 Node* no_particular_field = NULL; 4707 int raw_adr_idx = Compile::AliasIdxRaw; 4708 post_barrier(control(), 4709 memory(raw_adr_type), 4710 alloc_obj, 4711 no_particular_field, 4712 raw_adr_idx, 4713 no_particular_value, 4714 T_OBJECT, 4715 false); 4716 } 4717 4718 // Do not let reads from the cloned object float above the arraycopy. 4719 if (alloc != NULL) { 4720 // Do not let stores that initialize this object be reordered with 4721 // a subsequent store that would make this object accessible by 4722 // other threads. 4723 // Record what AllocateNode this StoreStore protects so that 4724 // escape analysis can go from the MemBarStoreStoreNode to the 4725 // AllocateNode and eliminate the MemBarStoreStoreNode if possible 4726 // based on the escape status of the AllocateNode. 4727 insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddress)); 4728 } else { 4729 insert_mem_bar(Op_MemBarCPUOrder); 4730 } 4731 } 4732 4733 //------------------------inline_native_clone---------------------------- 4734 // protected native Object java.lang.Object.clone(); 4735 // |
4129 // Conservatively insert a memory barrier on all memory slices. 4130 // Do not let writes of the copy source or destination float below the copy. 4131 insert_mem_bar(Op_MemBarCPUOrder); 4132 4133 // Call it. Note that the length argument is not scaled. 4134 make_runtime_call(RC_LEAF|RC_NO_FP, 4135 OptoRuntime::fast_arraycopy_Type(), 4136 StubRoutines::unsafe_arraycopy(), 4137 "unsafe_arraycopy", 4138 TypeRawPtr::BOTTOM, 4139 src, dst, size XTOP); 4140 4141 // Do not let reads of the copy destination float above the copy. 4142 insert_mem_bar(Op_MemBarCPUOrder); 4143 4144 return true; 4145 } 4146 4147 //------------------------clone_coping----------------------------------- 4148 // Helper function for inline_native_clone. 4149 void LibraryCallKit::copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array) { 4150 assert(obj_size != NULL, ""); 4151 Node* raw_obj = alloc_obj->in(1); 4152 assert(alloc_obj->is_CheckCastPP() && raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), ""); 4153 4154 AllocateNode* alloc = NULL; 4155 if (ReduceBulkZeroing) { 4156 // We will be completely responsible for initializing this object - 4157 // mark Initialize node as complete. 4158 alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn); 4159 // The object was just allocated - there should be no any stores! 4160 guarantee(alloc != NULL && alloc->maybe_set_complete(&_gvn), ""); 4161 // Mark as complete_with_arraycopy so that on AllocateNode 4162 // expansion, we know this AllocateNode is initialized by an array 4163 // copy and a StoreStore barrier exists after the array copy. 4164 alloc->initialization()->set_complete_with_arraycopy(); 4165 } 4166 4167 // Copy the fastest available way. 4168 // TODO: generate fields copies for small objects instead. 4169 Node* size = _gvn.transform(obj_size); 4170 4171 access_clone(control(), obj, alloc_obj, size, is_array); 4172 4173 // Do not let reads from the cloned object float above the arraycopy. 4174 if (alloc != NULL) { 4175 // Do not let stores that initialize this object be reordered with 4176 // a subsequent store that would make this object accessible by 4177 // other threads. 4178 // Record what AllocateNode this StoreStore protects so that 4179 // escape analysis can go from the MemBarStoreStoreNode to the 4180 // AllocateNode and eliminate the MemBarStoreStoreNode if possible 4181 // based on the escape status of the AllocateNode. 4182 insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddress)); 4183 } else { 4184 insert_mem_bar(Op_MemBarCPUOrder); 4185 } 4186 } 4187 4188 //------------------------inline_native_clone---------------------------- 4189 // protected native Object java.lang.Object.clone(); 4190 // |
4786 : TypeInstPtr::NOTNULL);
4787
4788 // Conservatively insert a memory barrier on all memory slices.
4789 // Do not let writes into the original float below the clone.
4790 insert_mem_bar(Op_MemBarCPUOrder);
4791
4792 // paths into result_reg:
4793 enum {
4794 _slow_path = 1, // out-of-line call to clone method (virtual or not)
4795 _objArray_path, // plain array allocation, plus arrayof_oop_arraycopy
4796 _array_path, // plain array allocation, plus arrayof_long_arraycopy
4797 _instance_path, // plain instance allocation, plus arrayof_long_arraycopy
4798 PATH_LIMIT
4799 };
4800 RegionNode* result_reg = new RegionNode(PATH_LIMIT);
4801 result_val = new PhiNode(result_reg, TypeInstPtr::NOTNULL);
4802 PhiNode* result_i_o = new PhiNode(result_reg, Type::ABIO);
4803 PhiNode* result_mem = new PhiNode(result_reg, Type::MEMORY, TypePtr::BOTTOM);
4804 record_for_igvn(result_reg);
4805
4806 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4807 int raw_adr_idx = Compile::AliasIdxRaw;
4808
4809 Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL);
4810 if (array_ctl != NULL) {
4811 // It's an array.
4812 PreserveJVMState pjvms(this);
4813 set_control(array_ctl);
4814 Node* obj_length = load_array_length(obj);
4815 Node* obj_size = NULL;
4816 Node* alloc_obj = new_array(obj_klass, obj_length, 0, &obj_size); // no arguments to push
4817
4818 if (!use_ReduceInitialCardMarks()) {
4819 // If it is an oop array, it requires very special treatment,
4820 // because card marking is required on each card of the array.
4821 Node* is_obja = generate_objArray_guard(obj_klass, (RegionNode*)NULL);
4822 if (is_obja != NULL) {
4823 PreserveJVMState pjvms2(this);
4824 set_control(is_obja);
4825 // Generate a direct call to the right arraycopy function(s).
4826 Node* alloc = tightly_coupled_allocation(alloc_obj, NULL);
4827 ArrayCopyNode* ac = ArrayCopyNode::make(this, true, obj, intcon(0), alloc_obj, intcon(0), obj_length, alloc != NULL, false)
4828 ac->set_cloneoop();
4829 Node* n = _gvn.transform(ac);
4830 assert(n == ac, "cannot disappear");
4831 ac->connect_outputs(this);
4832
4833 result_reg->init_req(_objArray_path, control());
4834 result_val->init_req(_objArray_path, alloc_obj);
4835 result_i_o ->set_req(_objArray_path, i_o());
4836 result_mem ->set_req(_objArray_path, reset_memory());
4837 }
4838 }
4839 // Otherwise, there are no card marks to worry about.
4840 // (We can dispense with card marks if we know the allocation
4841 // comes out of eden (TLAB)... In fact, ReduceInitialCardMarks
4842 // causes the non-eden paths to take compensating steps to
4843 // simulate a fresh allocation, so that no further
4844 // card marks are required in compiled code to initialize
4845 // the object.)
4846
4847 if (!stopped()) {
4848 copy_to_clone(obj, alloc_obj, obj_size, true, false);
4849
4850 // Present the results of the copy.
4851 result_reg->init_req(_array_path, control());
4852 result_val->init_req(_array_path, alloc_obj);
4853 result_i_o ->set_req(_array_path, i_o());
4854 result_mem ->set_req(_array_path, reset_memory());
4855 }
4856 }
4857
4858 // We only go to the instance fast case code if we pass a number of guards.
4859 // The paths which do not pass are accumulated in the slow_region.
4860 RegionNode* slow_region = new RegionNode(1);
4861 record_for_igvn(slow_region);
4862 if (!stopped()) {
4863 // It's an instance (we did array above). Make the slow-path tests.
4864 // If this is a virtual call, we generate a funny guard. We grab
4865 // the vtable entry corresponding to clone() from the target object.
4866 // If the target method which we are calling happens to be the
4867 // Object clone() method, we pass the guard. We do not need this
|
4241 : TypeInstPtr::NOTNULL);
4242
4243 // Conservatively insert a memory barrier on all memory slices.
4244 // Do not let writes into the original float below the clone.
4245 insert_mem_bar(Op_MemBarCPUOrder);
4246
4247 // paths into result_reg:
4248 enum {
4249 _slow_path = 1, // out-of-line call to clone method (virtual or not)
4250 _objArray_path, // plain array allocation, plus arrayof_oop_arraycopy
4251 _array_path, // plain array allocation, plus arrayof_long_arraycopy
4252 _instance_path, // plain instance allocation, plus arrayof_long_arraycopy
4253 PATH_LIMIT
4254 };
4255 RegionNode* result_reg = new RegionNode(PATH_LIMIT);
4256 result_val = new PhiNode(result_reg, TypeInstPtr::NOTNULL);
4257 PhiNode* result_i_o = new PhiNode(result_reg, Type::ABIO);
4258 PhiNode* result_mem = new PhiNode(result_reg, Type::MEMORY, TypePtr::BOTTOM);
4259 record_for_igvn(result_reg);
4260
4261 Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL);
4262 if (array_ctl != NULL) {
4263 // It's an array.
4264 PreserveJVMState pjvms(this);
4265 set_control(array_ctl);
4266 Node* obj_length = load_array_length(obj);
4267 Node* obj_size = NULL;
4268 Node* alloc_obj = new_array(obj_klass, obj_length, 0, &obj_size); // no arguments to push
4269
4270 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
4271 if (bs->array_copy_requires_gc_barriers(T_OBJECT)) {
4272 // If it is an oop array, it requires very special treatment,
4273 // because gc barriers are required when accessing the array.
4274 Node* is_obja = generate_objArray_guard(obj_klass, (RegionNode*)NULL);
4275 if (is_obja != NULL) {
4276 PreserveJVMState pjvms2(this);
4277 set_control(is_obja);
4278 // Generate a direct call to the right arraycopy function(s).
4279 Node* alloc = tightly_coupled_allocation(alloc_obj, NULL);
4280 ArrayCopyNode* ac = ArrayCopyNode::make(this, true, obj, intcon(0), alloc_obj, intcon(0), obj_length, alloc != NULL, false)
4281 ac->set_cloneoop();
4282 Node* n = _gvn.transform(ac);
4283 assert(n == ac, "cannot disappear");
4284 ac->connect_outputs(this);
4285
4286 result_reg->init_req(_objArray_path, control());
4287 result_val->init_req(_objArray_path, alloc_obj);
4288 result_i_o ->set_req(_objArray_path, i_o());
4289 result_mem ->set_req(_objArray_path, reset_memory());
4290 }
4291 }
4292 // Otherwise, there are no barriers to worry about.
4293 // (We can dispense with card marks if we know the allocation
4294 // comes out of eden (TLAB)... In fact, ReduceInitialCardMarks
4295 // causes the non-eden paths to take compensating steps to
4296 // simulate a fresh allocation, so that no further
4297 // card marks are required in compiled code to initialize
4298 // the object.)
4299
4300 if (!stopped()) {
4301 copy_to_clone(obj, alloc_obj, obj_size, true);
4302
4303 // Present the results of the copy.
4304 result_reg->init_req(_array_path, control());
4305 result_val->init_req(_array_path, alloc_obj);
4306 result_i_o ->set_req(_array_path, i_o());
4307 result_mem ->set_req(_array_path, reset_memory());
4308 }
4309 }
4310
4311 // We only go to the instance fast case code if we pass a number of guards.
4312 // The paths which do not pass are accumulated in the slow_region.
4313 RegionNode* slow_region = new RegionNode(1);
4314 record_for_igvn(slow_region);
4315 if (!stopped()) {
4316 // It's an instance (we did array above). Make the slow-path tests.
4317 // If this is a virtual call, we generate a funny guard. We grab
4318 // the vtable entry corresponding to clone() from the target object.
4319 // If the target method which we are calling happens to be the
4320 // Object clone() method, we pass the guard. We do not need this
|
4874 // The object must be easily cloneable and must not have a finalizer.
4875 // Both of these conditions may be checked in a single test.
4876 // We could optimize the test further, but we don't care.
4877 generate_access_flags_guard(obj_klass,
4878 // Test both conditions:
4879 JVM_ACC_IS_CLONEABLE_FAST | JVM_ACC_HAS_FINALIZER,
4880 // Must be cloneable but not finalizer:
4881 JVM_ACC_IS_CLONEABLE_FAST,
4882 slow_region);
4883 }
4884
4885 if (!stopped()) {
4886 // It's an instance, and it passed the slow-path tests.
4887 PreserveJVMState pjvms(this);
4888 Node* obj_size = NULL;
4889 // Need to deoptimize on exception from allocation since Object.clone intrinsic
4890 // is reexecuted if deoptimization occurs and there could be problems when merging
4891 // exception state between multiple Object.clone versions (reexecute=true vs reexecute=false).
4892 Node* alloc_obj = new_instance(obj_klass, NULL, &obj_size, /*deoptimize_on_exception=*/true);
4893
4894 copy_to_clone(obj, alloc_obj, obj_size, false, !use_ReduceInitialCardMarks());
4895
4896 // Present the results of the slow call.
4897 result_reg->init_req(_instance_path, control());
4898 result_val->init_req(_instance_path, alloc_obj);
4899 result_i_o ->set_req(_instance_path, i_o());
4900 result_mem ->set_req(_instance_path, reset_memory());
4901 }
4902
4903 // Generate code for the slow case. We make a call to clone().
4904 set_control(_gvn.transform(slow_region));
4905 if (!stopped()) {
4906 PreserveJVMState pjvms(this);
4907 CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_clone, is_virtual);
4908 Node* slow_result = set_results_for_java_call(slow_call);
4909 // this->control() comes from set_results_for_java_call
4910 result_reg->init_req(_slow_path, control());
4911 result_val->init_req(_slow_path, slow_result);
4912 result_i_o ->set_req(_slow_path, i_o());
4913 result_mem ->set_req(_slow_path, reset_memory());
|
4327 // The object must be easily cloneable and must not have a finalizer.
4328 // Both of these conditions may be checked in a single test.
4329 // We could optimize the test further, but we don't care.
4330 generate_access_flags_guard(obj_klass,
4331 // Test both conditions:
4332 JVM_ACC_IS_CLONEABLE_FAST | JVM_ACC_HAS_FINALIZER,
4333 // Must be cloneable but not finalizer:
4334 JVM_ACC_IS_CLONEABLE_FAST,
4335 slow_region);
4336 }
4337
4338 if (!stopped()) {
4339 // It's an instance, and it passed the slow-path tests.
4340 PreserveJVMState pjvms(this);
4341 Node* obj_size = NULL;
4342 // Need to deoptimize on exception from allocation since Object.clone intrinsic
4343 // is reexecuted if deoptimization occurs and there could be problems when merging
4344 // exception state between multiple Object.clone versions (reexecute=true vs reexecute=false).
4345 Node* alloc_obj = new_instance(obj_klass, NULL, &obj_size, /*deoptimize_on_exception=*/true);
4346
4347 copy_to_clone(obj, alloc_obj, obj_size, false);
4348
4349 // Present the results of the slow call.
4350 result_reg->init_req(_instance_path, control());
4351 result_val->init_req(_instance_path, alloc_obj);
4352 result_i_o ->set_req(_instance_path, i_o());
4353 result_mem ->set_req(_instance_path, reset_memory());
4354 }
4355
4356 // Generate code for the slow case. We make a call to clone().
4357 set_control(_gvn.transform(slow_region));
4358 if (!stopped()) {
4359 PreserveJVMState pjvms(this);
4360 CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_clone, is_virtual);
4361 Node* slow_result = set_results_for_java_call(slow_call);
4362 // this->control() comes from set_results_for_java_call
4363 result_reg->init_req(_slow_path, control());
4364 result_val->init_req(_slow_path, slow_result);
4365 result_i_o ->set_req(_slow_path, i_o());
4366 result_mem ->set_req(_slow_path, reset_memory());
|
6081
6082 Node* call = make_runtime_call(RC_LEAF, OptoRuntime::updateBytesAdler32_Type(),
6083 stubAddr, stubName, TypePtr::BOTTOM,
6084 crc, src_start, length);
6085
6086 Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms));
6087 set_result(result);
6088 return true;
6089 }
6090
6091 //----------------------------inline_reference_get----------------------------
6092 // public T java.lang.ref.Reference.get();
6093 bool LibraryCallKit::inline_reference_get() {
6094 const int referent_offset = java_lang_ref_Reference::referent_offset;
6095 guarantee(referent_offset > 0, "should have already been set");
6096
6097 // Get the argument:
6098 Node* reference_obj = null_check_receiver();
6099 if (stopped()) return true;
6100
6101 Node* adr = basic_plus_adr(reference_obj, reference_obj, referent_offset);
6102
6103 ciInstanceKlass* klass = env()->Object_klass();
6104 const TypeOopPtr* object_type = TypeOopPtr::make_from_klass(klass);
6105
6106 Node* no_ctrl = NULL;
6107 Node* result = make_load(no_ctrl, adr, object_type, T_OBJECT, MemNode::unordered);
6108
6109 // Use the pre-barrier to record the value in the referent field
6110 pre_barrier(false /* do_load */,
6111 control(),
6112 NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
6113 result /* pre_val */,
6114 T_OBJECT);
6115
6116 // Add memory barrier to prevent commoning reads from this field
6117 // across safepoint since GC can change its value.
6118 insert_mem_bar(Op_MemBarCPUOrder);
6119
6120 set_result(result);
6121 return true;
6122 }
6123
6124
6125 Node * LibraryCallKit::load_field_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString,
6126 bool is_exact=true, bool is_static=false,
6127 ciInstanceKlass * fromKls=NULL) {
6128 if (fromKls == NULL) {
6129 const TypeInstPtr* tinst = _gvn.type(fromObj)->isa_instptr();
6130 assert(tinst != NULL, "obj is null");
6131 assert(tinst->klass()->is_loaded(), "obj is not loaded");
6132 assert(!is_exact || tinst->klass_is_exact(), "klass not exact");
6133 fromKls = tinst->klass()->as_instance_klass();
6134 } else {
|
5534
5535 Node* call = make_runtime_call(RC_LEAF, OptoRuntime::updateBytesAdler32_Type(),
5536 stubAddr, stubName, TypePtr::BOTTOM,
5537 crc, src_start, length);
5538
5539 Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms));
5540 set_result(result);
5541 return true;
5542 }
5543
5544 //----------------------------inline_reference_get----------------------------
5545 // public T java.lang.ref.Reference.get();
5546 bool LibraryCallKit::inline_reference_get() {
5547 const int referent_offset = java_lang_ref_Reference::referent_offset;
5548 guarantee(referent_offset > 0, "should have already been set");
5549
5550 // Get the argument:
5551 Node* reference_obj = null_check_receiver();
5552 if (stopped()) return true;
5553
5554 const TypeInstPtr* tinst = _gvn.type(reference_obj)->isa_instptr();
5555 assert(tinst != NULL, "obj is null");
5556 assert(tinst->klass()->is_loaded(), "obj is not loaded");
5557 ciInstanceKlass* referenceKlass = tinst->klass()->as_instance_klass();
5558 ciField* field = referenceKlass->get_field_by_name(ciSymbol::make("referent"),
5559 ciSymbol::make("Ljava/lang/Object;"),
5560 false);
5561 assert (field != NULL, "undefined field");
5562
5563 Node* adr = basic_plus_adr(reference_obj, reference_obj, referent_offset);
5564 const TypePtr* adr_type = C->alias_type(field)->adr_type();
5565
5566 ciInstanceKlass* klass = env()->Object_klass();
5567 const TypeOopPtr* object_type = TypeOopPtr::make_from_klass(klass);
5568
5569 DecoratorSet decorators = IN_HEAP | ON_WEAK_OOP_REF;
5570 Node* result = access_load_at(reference_obj, adr, adr_type, object_type, T_OBJECT, decorators);
5571 // Add memory barrier to prevent commoning reads from this field
5572 // across safepoint since GC can change its value.
5573 insert_mem_bar(Op_MemBarCPUOrder);
5574
5575 set_result(result);
5576 return true;
5577 }
5578
5579
5580 Node * LibraryCallKit::load_field_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString,
5581 bool is_exact=true, bool is_static=false,
5582 ciInstanceKlass * fromKls=NULL) {
5583 if (fromKls == NULL) {
5584 const TypeInstPtr* tinst = _gvn.type(fromObj)->isa_instptr();
5585 assert(tinst != NULL, "obj is null");
5586 assert(tinst->klass()->is_loaded(), "obj is not loaded");
5587 assert(!is_exact || tinst->klass_is_exact(), "klass not exact");
5588 fromKls = tinst->klass()->as_instance_klass();
5589 } else {
|
6148
6149 // Next code copied from Parse::do_get_xxx():
6150
6151 // Compute address and memory type.
6152 int offset = field->offset_in_bytes();
6153 bool is_vol = field->is_volatile();
6154 ciType* field_klass = field->type();
6155 assert(field_klass->is_loaded(), "should be loaded");
6156 const TypePtr* adr_type = C->alias_type(field)->adr_type();
6157 Node *adr = basic_plus_adr(fromObj, fromObj, offset);
6158 BasicType bt = field->layout_type();
6159
6160 // Build the resultant type of the load
6161 const Type *type;
6162 if (bt == T_OBJECT) {
6163 type = TypeOopPtr::make_from_klass(field_klass->as_klass());
6164 } else {
6165 type = Type::get_const_basic_type(bt);
6166 }
6167
6168 if (support_IRIW_for_not_multiple_copy_atomic_cpu && is_vol) {
6169 insert_mem_bar(Op_MemBarVolatile); // StoreLoad barrier
6170 }
6171 // Build the load.
6172 MemNode::MemOrd mo = is_vol ? MemNode::acquire : MemNode::unordered;
6173 Node* loadedField = make_load(NULL, adr, type, bt, adr_type, mo, LoadNode::DependsOnlyOnTest, is_vol);
6174 // If reference is volatile, prevent following memory ops from
6175 // floating up past the volatile read. Also prevents commoning
6176 // another volatile read.
6177 if (is_vol) {
6178 // Memory barrier includes bogus read of value to force load BEFORE membar
6179 insert_mem_bar(Op_MemBarAcquire, loadedField);
6180 }
6181 return loadedField;
6182 }
6183
6184 Node * LibraryCallKit::field_address_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString,
6185 bool is_exact = true, bool is_static = false,
6186 ciInstanceKlass * fromKls = NULL) {
6187 if (fromKls == NULL) {
6188 const TypeInstPtr* tinst = _gvn.type(fromObj)->isa_instptr();
6189 assert(tinst != NULL, "obj is null");
6190 assert(tinst->klass()->is_loaded(), "obj is not loaded");
6191 assert(!is_exact || tinst->klass_is_exact(), "klass not exact");
6192 fromKls = tinst->klass()->as_instance_klass();
6193 }
6194 else {
6195 assert(is_static, "only for static field access");
6196 }
6197 ciField* field = fromKls->get_field_by_name(ciSymbol::make(fieldName),
6198 ciSymbol::make(fieldTypeString),
6199 is_static);
6200
|
5603
5604 // Next code copied from Parse::do_get_xxx():
5605
5606 // Compute address and memory type.
5607 int offset = field->offset_in_bytes();
5608 bool is_vol = field->is_volatile();
5609 ciType* field_klass = field->type();
5610 assert(field_klass->is_loaded(), "should be loaded");
5611 const TypePtr* adr_type = C->alias_type(field)->adr_type();
5612 Node *adr = basic_plus_adr(fromObj, fromObj, offset);
5613 BasicType bt = field->layout_type();
5614
5615 // Build the resultant type of the load
5616 const Type *type;
5617 if (bt == T_OBJECT) {
5618 type = TypeOopPtr::make_from_klass(field_klass->as_klass());
5619 } else {
5620 type = Type::get_const_basic_type(bt);
5621 }
5622
5623 DecoratorSet decorators = IN_HEAP;
5624
5625 if (is_vol) {
5626 decorators |= MO_SEQ_CST;
5627 }
5628
5629 return access_load_at(fromObj, adr, adr_type, type, bt, decorators);
5630 }
5631
5632 Node * LibraryCallKit::field_address_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString,
5633 bool is_exact = true, bool is_static = false,
5634 ciInstanceKlass * fromKls = NULL) {
5635 if (fromKls == NULL) {
5636 const TypeInstPtr* tinst = _gvn.type(fromObj)->isa_instptr();
5637 assert(tinst != NULL, "obj is null");
5638 assert(tinst->klass()->is_loaded(), "obj is not loaded");
5639 assert(!is_exact || tinst->klass_is_exact(), "klass not exact");
5640 fromKls = tinst->klass()->as_instance_klass();
5641 }
5642 else {
5643 assert(is_static, "only for static field access");
5644 }
5645 ciField* field = fromKls->get_field_by_name(ciSymbol::make(fieldName),
5646 ciSymbol::make(fieldTypeString),
5647 is_static);
5648
|