--- old/src/share/vm/opto/library_call.cpp 2015-10-08 22:15:50.556235021 +0200 +++ new/src/share/vm/opto/library_call.cpp 2015-10-08 22:15:50.504236263 +0200 @@ -28,6 +28,7 @@ #include "classfile/vmSymbols.hpp" #include "compiler/compileBroker.hpp" #include "compiler/compileLog.hpp" +#include "gc/shenandoah/shenandoahRuntime.hpp" #include "oops/objArrayKlass.hpp" #include "opto/addnode.hpp" #include "opto/arraycopynode.hpp" @@ -46,6 +47,7 @@ #include "opto/opaquenode.hpp" #include "opto/parse.hpp" #include "opto/runtime.hpp" +#include "opto/shenandoahSupport.hpp" #include "opto/subnode.hpp" #include "prims/nativeLookup.hpp" #include "runtime/sharedRuntime.hpp" @@ -972,9 +974,19 @@ //------------------------------inline_string_equals------------------------ bool LibraryCallKit::inline_string_equals() { Node* receiver = null_check_receiver(); + + if (ShenandoahVerifyReadsToFromSpace) { + receiver = shenandoah_read_barrier(receiver); + } + // NOTE: Do not null check argument for String.equals() because spec // allows to specify NULL as argument. Node* argument = this->argument(1); + + if (ShenandoahVerifyReadsToFromSpace) { + argument = shenandoah_read_barrier(argument); + } + if (stopped()) { return true; } @@ -1023,6 +1035,11 @@ // Get start addr of receiver Node* receiver_val = load_String_value(no_ctrl, receiver); + + if (ShenandoahVerifyReadsToFromSpace) { + receiver_val = shenandoah_read_barrier(receiver_val); + } + Node* receiver_offset = load_String_offset(no_ctrl, receiver); Node* receiver_start = array_element_address(receiver_val, receiver_offset, T_CHAR); @@ -1031,6 +1048,11 @@ // Get start addr of argument Node* argument_val = load_String_value(no_ctrl, argument); + + if (ShenandoahVerifyReadsToFromSpace) { + argument_val = shenandoah_read_barrier(argument_val); + } + Node* argument_offset = load_String_offset(no_ctrl, argument); Node* argument_start = array_element_address(argument_val, argument_offset, T_CHAR); @@ -1067,6 +1089,10 @@ bool LibraryCallKit::inline_array_equals() { Node* arg1 = argument(0); Node* arg2 = argument(1); + + arg1 = shenandoah_read_barrier(arg1); + arg2 = shenandoah_read_barrier(arg2); + set_result(_gvn.transform(new AryEqNode(control(), memory(TypeAryPtr::CHARS), arg1, arg2))); return true; } @@ -2152,7 +2178,7 @@ // runtime filters that guard the pre-barrier code. // Also add memory barrier for non volatile load from the referent field // to prevent commoning of loads across safepoint. - if (!UseG1GC && !need_mem_bar) + if (!(UseG1GC || UseShenandoahGC) && !need_mem_bar) return; // Some compile time checks. @@ -2341,6 +2367,11 @@ if (!is_native_ptr) { // The base is either a Java object or a value produced by Unsafe.staticFieldBase Node* base = argument(1); // type: oop + if (is_store) { + base = shenandoah_write_barrier(base); + } else { + base = shenandoah_read_barrier(base); + } // The offset is a value produced by Unsafe.staticFieldOffset or Unsafe.objectFieldOffset offset = argument(2); // type: long // We currently rely on the cookies produced by Unsafe.xxxFieldOffset @@ -2492,6 +2523,7 @@ if (type != T_OBJECT ) { (void) store_to_memory(control(), adr, val, type, adr_type, mo, is_volatile); } else { + val = shenandoah_read_barrier_nomem(val); // Possibly an oop being stored to Java heap or native memory if (!TypePtr::NULL_PTR->higher_equal(_gvn.type(heap_base_oop))) { // oop to Java heap. @@ -2621,6 +2653,8 @@ return true; } + base = shenandoah_write_barrier(base); + // Build field offset expression. // We currently rely on the cookies produced by Unsafe.xxxFieldOffset // to be plain byte offsets, which are also the same as those accepted @@ -2662,6 +2696,7 @@ // For now, we handle only those cases that actually exist: ints, // longs, and Object. Adding others should be straightforward. Node* load_store; + Node* result; switch(type) { case T_INT: if (kind == LS_xadd) { @@ -2673,6 +2708,7 @@ } else { ShouldNotReachHere(); } + result = load_store; break; case T_LONG: if (kind == LS_xadd) { @@ -2684,6 +2720,7 @@ } else { ShouldNotReachHere(); } + result = load_store; break; case T_OBJECT: // Transformation of a value which could be NULL pointer (CastPP #NULL) @@ -2692,6 +2729,8 @@ if (_gvn.type(newval) == TypePtr::NULL_PTR) newval = _gvn.makecon(TypePtr::NULL_PTR); + newval = shenandoah_read_barrier_nomem(newval); + // Reference stores need a store barrier. if (kind == LS_xchg) { // If pre-barrier must execute before the oop store, old value will require do_load here. @@ -2727,14 +2766,77 @@ load_store = _gvn.transform(new CompareAndSwapNNode(control(), mem, adr, newval_enc, oldval_enc)); } + result = load_store; } else #endif { if (kind == LS_xchg) { load_store = _gvn.transform(new GetAndSetPNode(control(), mem, adr, newval, adr_type, value_type->is_oopptr())); + result = load_store; } else { assert(kind == LS_cmpxchg, "wrong LoadStore operation"); load_store = _gvn.transform(new CompareAndSwapPNode(control(), mem, adr, newval, oldval)); + result = load_store; + + if (UseShenandoahGC) { + // if (! success) + Node* cmp_true = _gvn.transform(new CmpINode(load_store, intcon(1))); + Node* tst_true = _gvn.transform(new BoolNode(cmp_true, BoolTest::eq)); + IfNode* iff = create_and_map_if(control(), tst_true, PROB_LIKELY_MAG(2), COUNT_UNKNOWN); + Node* iftrue = _gvn.transform(new IfTrueNode(iff)); + Node* iffalse = _gvn.transform(new IfFalseNode(iff)); + + enum { _success_path = 1, _fail_path, _shenandoah_path, PATH_LIMIT }; + RegionNode* region = new RegionNode(PATH_LIMIT); + Node* phi = new PhiNode(region, TypeInt::BOOL); + // success -> return result of CAS1. + region->init_req(_success_path, iftrue); + phi ->init_req(_success_path, load_store); + + // failure + set_control(iffalse); + + // if (read_barrier(expected) == read_barrier(old) + oldval = shenandoah_read_barrier(oldval); + + // Load old value from memory. We shuold really use what we get back from the CAS, + // if we can. + Node* current = make_load(control(), adr, TypeInstPtr::BOTTOM, type, MemNode::unordered); + // read_barrier(old) + Node* new_current = shenandoah_read_barrier(current); + + Node* chk = _gvn.transform(new CmpPNode(new_current, oldval)); + Node* test = _gvn.transform(new BoolNode(chk, BoolTest::eq)); + + IfNode* iff2 = create_and_map_if(control(), test, PROB_UNLIKELY_MAG(2), COUNT_UNKNOWN); + Node* iftrue2 = _gvn.transform(new IfTrueNode(iff2)); + Node* iffalse2 = _gvn.transform(new IfFalseNode(iff2)); + + // If they are not equal, it's a legitimate failure and we return the result of CAS1. + region->init_req(_fail_path, iffalse2); + phi ->init_req(_fail_path, load_store); + + // Otherwise we retry with old. + set_control(iftrue2); + + Node *call = make_runtime_call(RC_LEAF | RC_NO_IO, + OptoRuntime::shenandoah_cas_obj_Type(), + CAST_FROM_FN_PTR(address, ShenandoahRuntime::compare_and_swap_object), + "shenandoah_cas_obj", + NULL, + adr, newval, current); + + Node* retval = _gvn.transform(new ProjNode(call, TypeFunc::Parms + 0)); + + region->init_req(_shenandoah_path, control()); + phi ->init_req(_shenandoah_path, retval); + + set_control(_gvn.transform(region)); + record_for_igvn(region); + phi = _gvn.transform(phi); + result = phi; + } + } } if (kind == LS_cmpxchg) { @@ -2744,14 +2846,14 @@ // critical path, while CAS failure path can use the penalty for going through unlikely // path as backoff. Which is still better than doing a store barrier there. IdealKit ideal(this); - ideal.if_then(load_store, BoolTest::ne, ideal.ConI(0), PROB_STATIC_FREQUENT); { + ideal.if_then(result, BoolTest::ne, ideal.ConI(0), PROB_STATIC_FREQUENT); { sync_kit(ideal); - post_barrier(ideal.ctrl(), load_store, base, adr, alias_idx, newval, T_OBJECT, true); + post_barrier(ideal.ctrl(), result, base, adr, alias_idx, newval, T_OBJECT, true); ideal.sync_kit(this); } ideal.end_if(); final_sync(ideal); } else { - post_barrier(control(), load_store, base, adr, alias_idx, newval, T_OBJECT, true); + post_barrier(control(), result, base, adr, alias_idx, newval, T_OBJECT, true); } break; default: @@ -2768,7 +2870,7 @@ if (type == T_OBJECT && kind == LS_xchg) { #ifdef _LP64 if (adr->bottom_type()->is_ptr_to_narrowoop()) { - load_store = _gvn.transform(new DecodeNNode(load_store, load_store->get_ptr_type())); + result = _gvn.transform(new DecodeNNode(result, result->get_ptr_type())); } #endif if (can_move_pre_barrier()) { @@ -2777,7 +2879,7 @@ // gets inserted between them. pre_barrier(false /* do_load */, control(), NULL, NULL, max_juint, NULL, NULL, - load_store /* pre_val */, + result /* pre_val */, T_OBJECT); } } @@ -2786,8 +2888,8 @@ insert_mem_bar(Op_MemBarCPUOrder); insert_mem_bar(Op_MemBarAcquire); - assert(type2size[load_store->bottom_type()->basic_type()] == type2size[rtype], "result type should match"); - set_result(load_store); + assert(type2size[result->bottom_type()->basic_type()] == type2size[rtype], "result type should match"); + set_result(result); return true; } @@ -2831,6 +2933,8 @@ return true; } + base = shenandoah_write_barrier(base); + // Build field offset expression. assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled"); // 32-bit machines ignore the high half of long offsets @@ -2845,8 +2949,10 @@ // Ensure that the store is atomic for longs: const bool require_atomic_access = true; Node* store; - if (type == T_OBJECT) // reference stores need a store barrier. + if (type == T_OBJECT) { // reference stores need a store barrier. + val = shenandoah_read_barrier_nomem(val); store = store_oop_to_unknown(control(), base, adr, adr_type, val, type, MemNode::release); + } else { store = store_to_memory(control(), adr, val, type, adr_type, MemNode::release, require_atomic_access); } @@ -3168,6 +3274,11 @@ enum { _normal_path = 1, _prim_path = 2, PATH_LIMIT }; Node* mirror = argument(0); + + if (ShenandoahVerifyReadsToFromSpace) { + mirror = shenandoah_read_barrier(mirror); + } + Node* obj = top(); switch (id) { @@ -3175,6 +3286,9 @@ // nothing is an instance of a primitive type prim_return_value = intcon(0); obj = argument(1); + if (ShenandoahVerifyReadsToFromSpace) { + obj = shenandoah_read_barrier(obj); + } break; case vmIntrinsics::_getModifiers: prim_return_value = intcon(JVM_ACC_ABSTRACT | JVM_ACC_FINAL | JVM_ACC_PUBLIC); @@ -3422,6 +3536,7 @@ Node* args[2]; // two java.lang.Class mirrors: superc, subc args[0] = argument(0); args[1] = argument(1); + Node* klasses[2]; // corresponding Klasses: superk, subk klasses[0] = klasses[1] = top(); @@ -3484,6 +3599,7 @@ set_control(region->in(_prim_0_path)); // go back to first null check if (!stopped()) { // Since superc is primitive, make a guard for the superc==subc case. + shenandoah_acmp_barrier(args[0], args[1]); Node* cmp_eq = _gvn.transform(new CmpPNode(args[0], args[1])); Node* bol_eq = _gvn.transform(new BoolNode(cmp_eq, BoolTest::eq)); generate_guard(bol_eq, region, PROB_FAIR); @@ -3728,6 +3844,8 @@ Node* orig_tail = _gvn.transform(new SubINode(orig_length, start)); Node* moved = generate_min_max(vmIntrinsics::_min, orig_tail, length); + original = shenandoah_read_barrier(original); + // Generate a direct call to the right arraycopy function(s). // We know the copy is disjoint but we might not know if the // oop stores need checking. @@ -3909,6 +4027,10 @@ result_val->init_req(_null_path, _gvn.intcon(0)); } + if (ShenandoahVerifyReadsToFromSpace) { + obj = shenandoah_read_barrier(obj); + } + // Unconditionally null? Then return right away. if (stopped()) { set_control( result_reg->in(_null_path)); @@ -4224,6 +4346,9 @@ assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled"); + src_ptr = shenandoah_read_barrier(src_ptr); + dst_ptr = shenandoah_write_barrier(dst_ptr); + Node* src = make_unsafe_address(src_ptr, src_off); Node* dst = make_unsafe_address(dst_ptr, dst_off); @@ -4252,6 +4377,8 @@ Node* raw_obj = alloc_obj->in(1); assert(alloc_obj->is_CheckCastPP() && raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), ""); + obj = shenandoah_read_barrier(obj); + AllocateNode* alloc = NULL; if (ReduceBulkZeroing) { // We will be completely responsible for initializing this object - @@ -4309,6 +4436,15 @@ set_all_memory(n); } + if (UseShenandoahGC) { + // Make sure that references in the cloned object are updated for Shenandoah. + make_runtime_call(RC_LEAF|RC_NO_FP, + OptoRuntime::shenandoah_clone_barrier_Type(), + CAST_FROM_FN_PTR(address, SharedRuntime::shenandoah_clone_barrier), + "shenandoah_clone_barrier", TypePtr::BOTTOM, + alloc_obj); + } + // If necessary, emit some card marks afterwards. (Non-arrays only.) if (card_mark) { assert(!is_array, ""); @@ -4435,6 +4571,9 @@ if (is_obja != NULL) { PreserveJVMState pjvms2(this); set_control(is_obja); + + obj = shenandoah_read_barrier(obj); + // Generate a direct call to the right arraycopy function(s). Node* alloc = tightly_coupled_allocation(alloc_obj, NULL); ArrayCopyNode* ac = ArrayCopyNode::make(this, true, obj, intcon(0), alloc_obj, intcon(0), obj_length, alloc != NULL); @@ -4683,7 +4822,6 @@ Node* dest_offset = argument(3); // type: int Node* length = argument(4); // type: int - // Check for allocation before we add nodes that would confuse // tightly_coupled_allocation() AllocateArrayNode* alloc = tightly_coupled_allocation(dest, NULL); @@ -4893,6 +5031,9 @@ return true; } + src = shenandoah_read_barrier(src); + dest = shenandoah_write_barrier(dest); + ArrayCopyNode* ac = ArrayCopyNode::make(this, true, src, src_offset, dest, dest_offset, length, alloc != NULL, // Create LoadRange and LoadKlass nodes for use during macro expansion here // so the compiler has a chance to eliminate them: during macro expansion, @@ -4922,6 +5063,8 @@ if (stopped()) return NULL; // no fast path if (C->AliasLevel() == 0) return NULL; // no MergeMems around + ptr = ShenandoahBarrierNode::skip_through_barrier(ptr); + AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(ptr, &_gvn); if (alloc == NULL) return NULL; @@ -5001,6 +5144,9 @@ Node *dst_offset = argument(3); Node *length = argument(4); + src = shenandoah_read_barrier(src); + dst = shenandoah_write_barrier(dst); + const Type* src_type = src->Value(&_gvn); const Type* dst_type = dst->Value(&_gvn); const TypeAryPtr* top_src = src_type->isa_aryptr(); @@ -5050,6 +5196,10 @@ Node* ylen = argument(3); Node* z = argument(4); + x = shenandoah_read_barrier(x); + y = shenandoah_read_barrier(y); + z = shenandoah_write_barrier(z); + const Type* x_type = x->Value(&_gvn); const Type* y_type = y->Value(&_gvn); const TypeAryPtr* top_x = x_type->isa_aryptr(); @@ -5150,6 +5300,9 @@ Node* z = argument(2); Node* zlen = argument(3); + x = shenandoah_read_barrier(x); + z = shenandoah_write_barrier(z); + const Type* x_type = x->Value(&_gvn); const Type* z_type = z->Value(&_gvn); const TypeAryPtr* top_x = x_type->isa_aryptr(); @@ -5197,6 +5350,9 @@ Node* len = argument(3); Node* k = argument(4); + in = shenandoah_read_barrier(in); + out = shenandoah_write_barrier(out); + const Type* out_type = out->Value(&_gvn); const Type* in_type = in->Value(&_gvn); const TypeAryPtr* top_out = out_type->isa_aryptr(); @@ -5246,6 +5402,11 @@ Node* inv = argument(4); Node* m = argument(6); + a = shenandoah_read_barrier(a); + b = shenandoah_read_barrier(b); + n = shenandoah_read_barrier(n); + m = shenandoah_write_barrier(m); + const Type* a_type = a->Value(&_gvn); const TypeAryPtr* top_a = a_type->isa_aryptr(); const Type* b_type = b->Value(&_gvn); @@ -5305,6 +5466,10 @@ Node* inv = argument(3); Node* m = argument(5); + a = shenandoah_read_barrier(a); + n = shenandoah_read_barrier(n); + m = shenandoah_write_barrier(m); + const Type* a_type = a->Value(&_gvn); const TypeAryPtr* top_a = a_type->isa_aryptr(); const Type* n_type = a->Value(&_gvn); @@ -5391,6 +5556,8 @@ Node* offset = argument(2); // type: int Node* length = argument(3); // type: int + src = shenandoah_read_barrier(src); + const Type* src_type = src->Value(&_gvn); const TypeAryPtr* top_src = src_type->isa_aryptr(); if (top_src == NULL || top_src->klass() == NULL) { @@ -5493,10 +5660,12 @@ } // 'src_start' points to src array + scaled offset + src = shenandoah_read_barrier(src); Node* src_start = array_element_address(src, offset, src_elem); // static final int[] byteTable in class CRC32C Node* table = get_table_from_crc32c_class(callee()->holder()); + table = shenandoah_read_barrier(table); Node* table_start = array_element_address(table, intcon(0), T_INT); // We assume that range check is done by caller. @@ -5540,6 +5709,7 @@ // static final int[] byteTable in class CRC32C Node* table = get_table_from_crc32c_class(callee()->holder()); + table = shenandoah_read_barrier(table); Node* table_start = array_element_address(table, intcon(0), T_INT); // Call the stub. @@ -5583,6 +5753,7 @@ } // 'src_start' points to src array + scaled offset + src = shenandoah_read_barrier(src); Node* src_start = array_element_address(src, offset, src_elem); // We assume that range check is done by caller. @@ -5645,6 +5816,10 @@ Node* reference_obj = null_check_receiver(); if (stopped()) return true; + if (ShenandoahVerifyReadsToFromSpace) { + reference_obj = shenandoah_read_barrier(reference_obj); + } + Node* adr = basic_plus_adr(reference_obj, reference_obj, referent_offset); ciInstanceKlass* klass = env()->Object_klass(); @@ -5693,6 +5868,8 @@ fromObj = makecon(tip); } + fromObj = shenandoah_read_barrier(fromObj); + // Next code copied from Parse::do_get_xxx(): // Compute address and memory type. @@ -5753,6 +5930,10 @@ Node* dest = argument(3); Node* dest_offset = argument(4); + // Resolve src and dest arrays for ShenandoahGC. + src = shenandoah_read_barrier(src); + dest = shenandoah_write_barrier(dest); + // (1) src and dest are arrays. const Type* src_type = src->Value(&_gvn); const Type* dest_type = dest->Value(&_gvn); @@ -5821,6 +6002,10 @@ Node* dest = argument(4); Node* dest_offset = argument(5); + // Resolve src and dest arrays for ShenandoahGC. + src = shenandoah_read_barrier(src); + dest = shenandoah_write_barrier(dest); + // (1) src and dest are arrays. const Type* src_type = src->Value(&_gvn); const Type* dest_type = dest->Value(&_gvn); @@ -5865,6 +6050,9 @@ // similarly, get the start address of the r vector Node* objRvec = load_field_from_object(cipherBlockChaining_object, "r", "[B", /*is_exact*/ false); + + objRvec = shenandoah_write_barrier(objRvec); + if (objRvec == NULL) return false; Node* r_start = array_element_address(objRvec, intcon(0), T_BYTE); @@ -5900,6 +6088,8 @@ assert (objAESCryptKey != NULL, "wrong version of com.sun.crypto.provider.AESCrypt"); if (objAESCryptKey == NULL) return (Node *) NULL; + objAESCryptKey = shenandoah_read_barrier(objAESCryptKey); + // now have the array, need to get the start address of the K array Node* k_start = array_element_address(objAESCryptKey, intcon(0), T_INT); return k_start;