4081 // Check that the initialization is storing NULL so that no previous store
4082 // has been moved up and directly write a reference
4083 Node* captured_store = st_init->find_captured_store(offset,
4084 type2aelembytes(T_OBJECT),
4085 phase);
4086 if (captured_store == NULL || captured_store == st_init->zero_memory()) {
4087 return true;
4088 }
4089 }
4090 }
4091
4092 // Unless there is an explicit 'continue', we must bail out here,
4093 // because 'mem' is an inscrutable memory state (e.g., a call).
4094 break;
4095 }
4096
4097 return false;
4098 }
4099
4100 static void g1_write_barrier_pre_helper(const GraphKit& kit, Node* adr) {
4101 if (UseShenandoahGC && (ShenandoahSATBBarrier || ShenandoahConditionalSATBBarrier) && adr != NULL) {
4102 Node* c = kit.control();
4103 Node* call = c->in(1)->in(1)->in(1)->in(0);
4104 assert(call->is_g1_wb_pre_call(), "g1_wb_pre call expected");
4105 call->add_req(adr);
4106 }
4107 }
4108
4109 // G1 pre/post barriers
4110 void GraphKit::g1_write_barrier_pre(bool do_load,
4111 Node* obj,
4112 Node* adr,
4113 uint alias_idx,
4114 Node* val,
4115 const TypeOopPtr* val_type,
4116 Node* pre_val,
4117 BasicType bt) {
4118
4119 // Some sanity checks
4120 // Note: val is unused in this routine.
4121
4319 // Final sync IdealKit and GraphKit.
4320 final_sync(ideal);
4321 }
4322
4323 void GraphKit::shenandoah_write_barrier_pre(bool do_load,
4324 Node* obj,
4325 Node* adr,
4326 uint alias_idx,
4327 Node* val,
4328 const TypeOopPtr* val_type,
4329 Node* pre_val,
4330 BasicType bt) {
4331
4332 // Some sanity checks
4333 // Note: val is unused in this routine.
4334
4335 if (val != NULL) {
4336 shenandoah_update_matrix(adr, val);
4337 }
4338
4339 if (ShenandoahConditionalSATBBarrier) {
4340 enum { _set_path = 1, _not_set_path, PATH_LIMIT };
4341 RegionNode* region = new RegionNode(PATH_LIMIT);
4342 Node* prev_mem = memory(Compile::AliasIdxRaw);
4343 Node* memphi = PhiNode::make(region, prev_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
4344
4345 Node* gc_state_addr_p = _gvn.transform(new CastX2PNode(MakeConX((intptr_t) ShenandoahHeap::gc_state_addr())));
4346 Node* gc_state_addr = _gvn.transform(new AddPNode(top(), gc_state_addr_p, MakeConX(0)));
4347 Node* gc_state = _gvn.transform(LoadNode::make(_gvn, control(), memory(Compile::AliasIdxRaw), gc_state_addr, TypeRawPtr::BOTTOM, TypeInt::INT, T_BYTE, MemNode::unordered));
4348 Node* add_set = _gvn.transform(new AddINode(gc_state, intcon(ShenandoahHeap::MARKING)));
4349 Node* cmp_set = _gvn.transform(new CmpINode(add_set, intcon(0)));
4350 Node* cmp_set_bool = _gvn.transform(new BoolNode(cmp_set, BoolTest::eq));
4351 IfNode* cmp_iff = create_and_map_if(control(), cmp_set_bool, PROB_MIN, COUNT_UNKNOWN);
4352 Node* if_not_set = _gvn.transform(new IfTrueNode(cmp_iff));
4353 Node* if_set = _gvn.transform(new IfFalseNode(cmp_iff));
4354
4355 // Conc-mark not in progress. Skip SATB barrier.
4356 set_control(if_not_set);
4357 region->init_req(_not_set_path, control());
4358 memphi->init_req(_not_set_path, prev_mem);
4359
4360 // Conc-mark in progress. Do the SATB barrier.
4361 set_control(if_set);
4362 g1_write_barrier_pre(do_load, obj, adr, alias_idx, val, val_type, pre_val, bt);
4363 region->init_req(_set_path, control());
4364 memphi->init_req(_set_path, memory(Compile::AliasIdxRaw));
4365
4366 // Merge control flow and memory.
4367 set_control(_gvn.transform(region));
4368 record_for_igvn(region);
4369 set_memory(_gvn.transform(memphi), Compile::AliasIdxRaw);
4370
4371 }
4372 if (ShenandoahSATBBarrier) {
4373 g1_write_barrier_pre(do_load, obj, adr, alias_idx, val, val_type, pre_val, bt);
4374 }
4375 }
4376
4377 void GraphKit::shenandoah_update_matrix(Node* adr, Node* val) {
4378 if (!UseShenandoahMatrix) {
4379 return;
4380 }
4381
4382 assert(val != NULL, "checked before");
4383 if (adr == NULL) {
4384 return; // Nothing to do
4385 }
4386 assert(adr != NULL, "must not happen");
4387 if (val->bottom_type()->higher_equal(TypePtr::NULL_PTR)) {
4388 // Nothing to do.
4389 return;
4390 }
4391
4871 }
4872 return NULL;
4873 }
4874
4875 Node* GraphKit::cast_array_to_stable(Node* ary, const TypeAryPtr* ary_type) {
4876 // Reify the property as a CastPP node in Ideal graph to comply with monotonicity
4877 // assumption of CCP analysis.
4878 return _gvn.transform(new CastPPNode(ary, ary_type->cast_to_stable(true)));
4879 }
4880
4881 Node* GraphKit::shenandoah_read_barrier(Node* obj) {
4882 if (UseShenandoahGC && ShenandoahReadBarrier) {
4883 return shenandoah_read_barrier_impl(obj, false, true, true);
4884 } else {
4885 return obj;
4886 }
4887 }
4888
4889 Node* GraphKit::shenandoah_storeval_barrier(Node* obj) {
4890 if (UseShenandoahGC) {
4891 if (ShenandoahStoreValWriteBarrier || ShenandoahStoreValEnqueueBarrier) {
4892 obj = shenandoah_write_barrier(obj);
4893 }
4894 if (ShenandoahStoreValEnqueueBarrier && !ShenandoahMWF) {
4895 shenandoah_enqueue_barrier(obj);
4896 }
4897 if (ShenandoahStoreValReadBarrier) {
4898 obj = shenandoah_read_barrier_impl(obj, true, false, false);
4899 }
4900 }
4901 return obj;
4902 }
4903
4904 Node* GraphKit::shenandoah_read_barrier_acmp(Node* obj) {
4905 return shenandoah_read_barrier_impl(obj, true, true, false);
4906 }
4907
4908 Node* GraphKit::shenandoah_read_barrier_impl(Node* obj, bool use_ctrl, bool use_mem, bool allow_fromspace) {
4909
4910 const Type* obj_type = obj->bottom_type();
4911 if (obj_type->higher_equal(TypePtr::NULL_PTR)) {
4912 return obj;
4913 }
4914 const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type);
4950 Node* n = _gvn.transform(rb);
4951 record_for_igvn(n);
4952 return n;
4953 }
4954 }
4955
4956 Node* GraphKit::shenandoah_write_barrier_helper(GraphKit& kit, Node* obj, const TypePtr* adr_type) {
4957 ShenandoahWriteBarrierNode* wb = new ShenandoahWriteBarrierNode(kit.C, kit.control(), kit.memory(adr_type), obj);
4958 Node* n = kit.gvn().transform(wb);
4959 if (n == wb) { // New barrier needs memory projection.
4960 Node* proj = kit.gvn().transform(new ShenandoahWBMemProjNode(n));
4961 kit.set_memory(proj, adr_type);
4962 }
4963
4964 return n;
4965 }
4966
4967 Node* GraphKit::shenandoah_write_barrier(Node* obj) {
4968
4969 if (UseShenandoahGC && ShenandoahWriteBarrier) {
4970 obj = shenandoah_write_barrier_impl(obj);
4971 if (ShenandoahStoreValEnqueueBarrier && ShenandoahMWF) {
4972 shenandoah_enqueue_barrier(obj);
4973 }
4974 return obj;
4975 } else {
4976 return obj;
4977 }
4978 }
4979
4980 Node* GraphKit::shenandoah_write_barrier_impl(Node* obj) {
4981 if (! ShenandoahBarrierNode::needs_barrier(&_gvn, NULL, obj, NULL, true)) {
4982 return obj;
4983 }
4984 const Type* obj_type = obj->bottom_type();
4985 const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type);
4986 if (obj_type->meet(TypePtr::NULL_PTR) == obj_type->remove_speculative()) {
4987 // We don't know if it's null or not. Need null-check.
4988 enum { _not_null_path = 1, _null_path, PATH_LIMIT };
4989 RegionNode* region = new RegionNode(PATH_LIMIT);
4990 Node* phi = new PhiNode(region, obj_type);
4991 Node* memphi = PhiNode::make(region, memory(adr_type), Type::MEMORY, C->alias_type(adr_type)->adr_type());
4992
4993 Node* prev_mem = memory(adr_type);
4994 Node* null_ctrl = top();
|
4081 // Check that the initialization is storing NULL so that no previous store
4082 // has been moved up and directly write a reference
4083 Node* captured_store = st_init->find_captured_store(offset,
4084 type2aelembytes(T_OBJECT),
4085 phase);
4086 if (captured_store == NULL || captured_store == st_init->zero_memory()) {
4087 return true;
4088 }
4089 }
4090 }
4091
4092 // Unless there is an explicit 'continue', we must bail out here,
4093 // because 'mem' is an inscrutable memory state (e.g., a call).
4094 break;
4095 }
4096
4097 return false;
4098 }
4099
4100 static void g1_write_barrier_pre_helper(const GraphKit& kit, Node* adr) {
4101 if (UseShenandoahGC && ShenandoahSATBBarrier && adr != NULL) {
4102 Node* c = kit.control();
4103 Node* call = c->in(1)->in(1)->in(1)->in(0);
4104 assert(call->is_g1_wb_pre_call(), "g1_wb_pre call expected");
4105 call->add_req(adr);
4106 }
4107 }
4108
4109 // G1 pre/post barriers
4110 void GraphKit::g1_write_barrier_pre(bool do_load,
4111 Node* obj,
4112 Node* adr,
4113 uint alias_idx,
4114 Node* val,
4115 const TypeOopPtr* val_type,
4116 Node* pre_val,
4117 BasicType bt) {
4118
4119 // Some sanity checks
4120 // Note: val is unused in this routine.
4121
4319 // Final sync IdealKit and GraphKit.
4320 final_sync(ideal);
4321 }
4322
4323 void GraphKit::shenandoah_write_barrier_pre(bool do_load,
4324 Node* obj,
4325 Node* adr,
4326 uint alias_idx,
4327 Node* val,
4328 const TypeOopPtr* val_type,
4329 Node* pre_val,
4330 BasicType bt) {
4331
4332 // Some sanity checks
4333 // Note: val is unused in this routine.
4334
4335 if (val != NULL) {
4336 shenandoah_update_matrix(adr, val);
4337 }
4338
4339 if (ShenandoahSATBBarrier) {
4340 g1_write_barrier_pre(do_load, obj, adr, alias_idx, val, val_type, pre_val, bt);
4341 }
4342 }
4343
4344 void GraphKit::shenandoah_update_matrix(Node* adr, Node* val) {
4345 if (!UseShenandoahMatrix) {
4346 return;
4347 }
4348
4349 assert(val != NULL, "checked before");
4350 if (adr == NULL) {
4351 return; // Nothing to do
4352 }
4353 assert(adr != NULL, "must not happen");
4354 if (val->bottom_type()->higher_equal(TypePtr::NULL_PTR)) {
4355 // Nothing to do.
4356 return;
4357 }
4358
4838 }
4839 return NULL;
4840 }
4841
4842 Node* GraphKit::cast_array_to_stable(Node* ary, const TypeAryPtr* ary_type) {
4843 // Reify the property as a CastPP node in Ideal graph to comply with monotonicity
4844 // assumption of CCP analysis.
4845 return _gvn.transform(new CastPPNode(ary, ary_type->cast_to_stable(true)));
4846 }
4847
4848 Node* GraphKit::shenandoah_read_barrier(Node* obj) {
4849 if (UseShenandoahGC && ShenandoahReadBarrier) {
4850 return shenandoah_read_barrier_impl(obj, false, true, true);
4851 } else {
4852 return obj;
4853 }
4854 }
4855
4856 Node* GraphKit::shenandoah_storeval_barrier(Node* obj) {
4857 if (UseShenandoahGC) {
4858 if (ShenandoahStoreValEnqueueBarrier) {
4859 obj = shenandoah_write_barrier(obj);
4860 shenandoah_enqueue_barrier(obj);
4861 }
4862 if (ShenandoahStoreValReadBarrier) {
4863 obj = shenandoah_read_barrier_impl(obj, true, false, false);
4864 }
4865 }
4866 return obj;
4867 }
4868
4869 Node* GraphKit::shenandoah_read_barrier_acmp(Node* obj) {
4870 return shenandoah_read_barrier_impl(obj, true, true, false);
4871 }
4872
4873 Node* GraphKit::shenandoah_read_barrier_impl(Node* obj, bool use_ctrl, bool use_mem, bool allow_fromspace) {
4874
4875 const Type* obj_type = obj->bottom_type();
4876 if (obj_type->higher_equal(TypePtr::NULL_PTR)) {
4877 return obj;
4878 }
4879 const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type);
4915 Node* n = _gvn.transform(rb);
4916 record_for_igvn(n);
4917 return n;
4918 }
4919 }
4920
4921 Node* GraphKit::shenandoah_write_barrier_helper(GraphKit& kit, Node* obj, const TypePtr* adr_type) {
4922 ShenandoahWriteBarrierNode* wb = new ShenandoahWriteBarrierNode(kit.C, kit.control(), kit.memory(adr_type), obj);
4923 Node* n = kit.gvn().transform(wb);
4924 if (n == wb) { // New barrier needs memory projection.
4925 Node* proj = kit.gvn().transform(new ShenandoahWBMemProjNode(n));
4926 kit.set_memory(proj, adr_type);
4927 }
4928
4929 return n;
4930 }
4931
4932 Node* GraphKit::shenandoah_write_barrier(Node* obj) {
4933
4934 if (UseShenandoahGC && ShenandoahWriteBarrier) {
4935 return shenandoah_write_barrier_impl(obj);
4936 } else {
4937 return obj;
4938 }
4939 }
4940
4941 Node* GraphKit::shenandoah_write_barrier_impl(Node* obj) {
4942 if (! ShenandoahBarrierNode::needs_barrier(&_gvn, NULL, obj, NULL, true)) {
4943 return obj;
4944 }
4945 const Type* obj_type = obj->bottom_type();
4946 const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type);
4947 if (obj_type->meet(TypePtr::NULL_PTR) == obj_type->remove_speculative()) {
4948 // We don't know if it's null or not. Need null-check.
4949 enum { _not_null_path = 1, _null_path, PATH_LIMIT };
4950 RegionNode* region = new RegionNode(PATH_LIMIT);
4951 Node* phi = new PhiNode(region, obj_type);
4952 Node* memphi = PhiNode::make(region, memory(adr_type), Type::MEMORY, C->alias_type(adr_type)->adr_type());
4953
4954 Node* prev_mem = memory(adr_type);
4955 Node* null_ctrl = top();
|