BarrierSetC2
0 /* 1 * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved. 2 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 3 * 4 * This code is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License version 2 only, as 6 * published by the Free Software Foundation. 7 * 8 * This code is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 * version 2 for more details (a copy is included in the LICENSE file that 12 * accompanied this code). 13 * 14 * You should have received a copy of the GNU General Public License version 15 * 2 along with this work; if not, write to the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 19 * or visit www.oracle.com if you need additional information or have any 20 * questions. 21 * 22 */ 23 24 #include "precompiled.hpp" 25 #include "opto/arraycopynode.hpp" 26 #include "opto/graphKit.hpp" 27 #include "runtime/sharedRuntime.hpp" 28 29 ArrayCopyNode::ArrayCopyNode(Compile* C, bool alloc_tightly_coupled, bool has_negative_length_guard) 30 : CallNode(arraycopy_type(), NULL, TypeRawPtr::BOTTOM), 31 _alloc_tightly_coupled(alloc_tightly_coupled), 32 _has_negative_length_guard(has_negative_length_guard), 33 _kind(None), 34 _arguments_validated(false), 35 _src_type(TypeOopPtr::BOTTOM), 36 _dest_type(TypeOopPtr::BOTTOM) { 37 init_class_id(Class_ArrayCopy); 38 init_flags(Flag_is_macro); 39 C->add_macro_node(this); 40 } 41 42 uint ArrayCopyNode::size_of() const { return sizeof(*this); } 43 44 ArrayCopyNode* ArrayCopyNode::make(GraphKit* kit, bool may_throw, 45 Node* src, Node* src_offset, 46 Node* dest, Node* dest_offset, |
0 /* 1 * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved. 2 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 3 * 4 * This code is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License version 2 only, as 6 * published by the Free Software Foundation. 7 * 8 * This code is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 * version 2 for more details (a copy is included in the LICENSE file that 12 * accompanied this code). 13 * 14 * You should have received a copy of the GNU General Public License version 15 * 2 along with this work; if not, write to the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 19 * or visit www.oracle.com if you need additional information or have any 20 * questions. 21 * 22 */ 23 24 #include "precompiled.hpp" 25 #include "gc/shared/barrierSet.hpp" 26 #include "gc/shared/c2/barrierSetC2.hpp" 27 #include "gc/shared/c2/cardTableBarrierSetC2.hpp" 28 #include "opto/arraycopynode.hpp" 29 #include "opto/graphKit.hpp" 30 #include "runtime/sharedRuntime.hpp" 31 #include "utilities/macros.hpp" 32 33 ArrayCopyNode::ArrayCopyNode(Compile* C, bool alloc_tightly_coupled, bool has_negative_length_guard) 34 : CallNode(arraycopy_type(), NULL, TypeRawPtr::BOTTOM), 35 _alloc_tightly_coupled(alloc_tightly_coupled), 36 _has_negative_length_guard(has_negative_length_guard), 37 _kind(None), 38 _arguments_validated(false), 39 _src_type(TypeOopPtr::BOTTOM), 40 _dest_type(TypeOopPtr::BOTTOM) { 41 init_class_id(Class_ArrayCopy); 42 init_flags(Flag_is_macro); 43 C->add_macro_node(this); 44 } 45 46 uint ArrayCopyNode::size_of() const { return sizeof(*this); } 47 48 ArrayCopyNode* ArrayCopyNode::make(GraphKit* kit, bool may_throw, 49 Node* src, Node* src_offset, 50 Node* dest, Node* dest_offset, |
234
235 // newly allocated object is guaranteed to not overlap with source object
236 disjoint_bases = is_alloc_tightly_coupled();
237
238 if (ary_src == NULL || ary_src->klass() == NULL ||
239 ary_dest == NULL || ary_dest->klass() == NULL) {
240 // We don't know if arguments are arrays
241 return false;
242 }
243
244 BasicType src_elem = ary_src->klass()->as_array_klass()->element_type()->basic_type();
245 BasicType dest_elem = ary_dest->klass()->as_array_klass()->element_type()->basic_type();
246 if (src_elem == T_ARRAY) src_elem = T_OBJECT;
247 if (dest_elem == T_ARRAY) dest_elem = T_OBJECT;
248
249 if (src_elem != dest_elem || dest_elem == T_VOID) {
250 // We don't know if arguments are arrays of the same type
251 return false;
252 }
253
254 if (dest_elem == T_OBJECT && (!is_alloc_tightly_coupled() || !GraphKit::use_ReduceInitialCardMarks())) {
255 // It's an object array copy but we can't emit the card marking
256 // that is needed
257 return false;
258 }
259
260 value_type = ary_src->elem();
261
262 base_src = src;
263 base_dest = dest;
264
265 uint shift = exact_log2(type2aelembytes(dest_elem));
266 uint header = arrayOopDesc::base_offset_in_bytes(dest_elem);
267
268 adr_src = src;
269 adr_dest = dest;
270
271 src_offset = Compile::conv_I2X_index(phase, src_offset, ary_src->size());
272 dest_offset = Compile::conv_I2X_index(phase, dest_offset, ary_dest->size());
273
|
238
239 // newly allocated object is guaranteed to not overlap with source object
240 disjoint_bases = is_alloc_tightly_coupled();
241
242 if (ary_src == NULL || ary_src->klass() == NULL ||
243 ary_dest == NULL || ary_dest->klass() == NULL) {
244 // We don't know if arguments are arrays
245 return false;
246 }
247
248 BasicType src_elem = ary_src->klass()->as_array_klass()->element_type()->basic_type();
249 BasicType dest_elem = ary_dest->klass()->as_array_klass()->element_type()->basic_type();
250 if (src_elem == T_ARRAY) src_elem = T_OBJECT;
251 if (dest_elem == T_ARRAY) dest_elem = T_OBJECT;
252
253 if (src_elem != dest_elem || dest_elem == T_VOID) {
254 // We don't know if arguments are arrays of the same type
255 return false;
256 }
257
258 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
259 if (dest_elem == T_OBJECT && (!is_alloc_tightly_coupled() ||
260 bs->array_copy_requires_gc_barriers(T_OBJECT))) {
261 // It's an object array copy but we can't emit the card marking
262 // that is needed
263 return false;
264 }
265
266 value_type = ary_src->elem();
267
268 base_src = src;
269 base_dest = dest;
270
271 uint shift = exact_log2(type2aelembytes(dest_elem));
272 uint header = arrayOopDesc::base_offset_in_bytes(dest_elem);
273
274 adr_src = src;
275 adr_dest = dest;
276
277 src_offset = Compile::conv_I2X_index(phase, src_offset, ary_src->size());
278 dest_offset = Compile::conv_I2X_index(phase, dest_offset, ary_dest->size());
279
|
416 v = phase->transform(v);
417 mem = StoreNode::make(*phase, backward_ctl, mem, adr_dest, atp_dest, v, copy_type, MemNode::unordered);
418 mem = phase->transform(mem);
419 } else if(can_reshape) {
420 PhaseIterGVN* igvn = phase->is_IterGVN();
421 igvn->_worklist.push(adr_src);
422 igvn->_worklist.push(adr_dest);
423 }
424 }
425 return mem;
426 }
427
428 bool ArrayCopyNode::finish_transform(PhaseGVN *phase, bool can_reshape,
429 Node* ctl, Node *mem) {
430 if (can_reshape) {
431 PhaseIterGVN* igvn = phase->is_IterGVN();
432 igvn->set_delay_transform(false);
433 if (is_clonebasic()) {
434 Node* out_mem = proj_out(TypeFunc::Memory);
435
436 if (out_mem->outcnt() != 1 || !out_mem->raw_out(0)->is_MergeMem() ||
437 out_mem->raw_out(0)->outcnt() != 1 || !out_mem->raw_out(0)->raw_out(0)->is_MemBar()) {
438 assert(!GraphKit::use_ReduceInitialCardMarks(), "can only happen with card marking");
439 return false;
440 }
441
442 igvn->replace_node(out_mem->raw_out(0), mem);
443
444 Node* out_ctl = proj_out(TypeFunc::Control);
445 igvn->replace_node(out_ctl, ctl);
446 } else {
447 // replace fallthrough projections of the ArrayCopyNode by the
448 // new memory, control and the input IO.
449 CallProjections callprojs;
450 extract_projections(&callprojs, true, false);
451
452 if (callprojs.fallthrough_ioproj != NULL) {
453 igvn->replace_node(callprojs.fallthrough_ioproj, in(TypeFunc::I_O));
454 }
455 if (callprojs.fallthrough_memproj != NULL) {
456 igvn->replace_node(callprojs.fallthrough_memproj, mem);
457 }
|
422 v = phase->transform(v);
423 mem = StoreNode::make(*phase, backward_ctl, mem, adr_dest, atp_dest, v, copy_type, MemNode::unordered);
424 mem = phase->transform(mem);
425 } else if(can_reshape) {
426 PhaseIterGVN* igvn = phase->is_IterGVN();
427 igvn->_worklist.push(adr_src);
428 igvn->_worklist.push(adr_dest);
429 }
430 }
431 return mem;
432 }
433
434 bool ArrayCopyNode::finish_transform(PhaseGVN *phase, bool can_reshape,
435 Node* ctl, Node *mem) {
436 if (can_reshape) {
437 PhaseIterGVN* igvn = phase->is_IterGVN();
438 igvn->set_delay_transform(false);
439 if (is_clonebasic()) {
440 Node* out_mem = proj_out(TypeFunc::Memory);
441
442 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
443 if (out_mem->outcnt() != 1 || !out_mem->raw_out(0)->is_MergeMem() ||
444 out_mem->raw_out(0)->outcnt() != 1 || !out_mem->raw_out(0)->raw_out(0)->is_MemBar()) {
445 assert(bs->array_copy_requires_gc_barriers(T_OBJECT), "can only happen with card marking");
446 return false;
447 }
448
449 igvn->replace_node(out_mem->raw_out(0), mem);
450
451 Node* out_ctl = proj_out(TypeFunc::Control);
452 igvn->replace_node(out_ctl, ctl);
453 } else {
454 // replace fallthrough projections of the ArrayCopyNode by the
455 // new memory, control and the input IO.
456 CallProjections callprojs;
457 extract_projections(&callprojs, true, false);
458
459 if (callprojs.fallthrough_ioproj != NULL) {
460 igvn->replace_node(callprojs.fallthrough_ioproj, in(TypeFunc::I_O));
461 }
462 if (callprojs.fallthrough_memproj != NULL) {
463 igvn->replace_node(callprojs.fallthrough_memproj, mem);
464 }
|
625
626 if (_dest_type != TypeOopPtr::BOTTOM || t_oop->is_known_instance()) {
627 assert(_dest_type == TypeOopPtr::BOTTOM || _dest_type->is_known_instance(), "result of EA is known instance");
628 return t_oop->instance_id() == _dest_type->instance_id();
629 }
630
631 return CallNode::may_modify_arraycopy_helper(dest_t, t_oop, phase);
632 }
633
634 bool ArrayCopyNode::may_modify_helper(const TypeOopPtr *t_oop, Node* n, PhaseTransform *phase, CallNode*& call) {
635 if (n != NULL &&
636 n->is_Call() &&
637 n->as_Call()->may_modify(t_oop, phase) &&
638 (n->as_Call()->is_ArrayCopy() || n->as_Call()->is_call_to_arraycopystub())) {
639 call = n->as_Call();
640 return true;
641 }
642 return false;
643 }
644
645 static Node* step_over_gc_barrier(Node* c) {
646 if (UseG1GC && !GraphKit::use_ReduceInitialCardMarks() &&
647 c != NULL && c->is_Region() && c->req() == 3) {
648 for (uint i = 1; i < c->req(); i++) {
649 if (c->in(i) != NULL && c->in(i)->is_Region() &&
650 c->in(i)->req() == 3) {
651 Node* r = c->in(i);
652 for (uint j = 1; j < r->req(); j++) {
653 if (r->in(j) != NULL && r->in(j)->is_Proj() &&
654 r->in(j)->in(0) != NULL &&
655 r->in(j)->in(0)->Opcode() == Op_CallLeaf &&
656 r->in(j)->in(0)->as_Call()->entry_point() == CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post)) {
657 Node* call = r->in(j)->in(0);
658 c = c->in(i == 1 ? 2 : 1);
659 if (c != NULL) {
660 c = c->in(0);
661 if (c != NULL) {
662 c = c->in(0);
663 assert(call->in(0) == NULL ||
664 call->in(0)->in(0) == NULL ||
665 call->in(0)->in(0)->in(0) == NULL ||
666 call->in(0)->in(0)->in(0)->in(0) == NULL ||
667 call->in(0)->in(0)->in(0)->in(0)->in(0) == NULL ||
668 c == call->in(0)->in(0)->in(0)->in(0)->in(0), "bad barrier shape");
669 return c;
670 }
671 }
672 }
673 }
674 }
675 }
676 }
677 return c;
678 }
679
680 bool ArrayCopyNode::may_modify(const TypeOopPtr *t_oop, MemBarNode* mb, PhaseTransform *phase, ArrayCopyNode*& ac) {
681
682 Node* c = mb->in(0);
683
684 // step over g1 gc barrier if we're at a clone with ReduceInitialCardMarks off
685 c = step_over_gc_barrier(c);
686
687 CallNode* call = NULL;
688 if (c != NULL && c->is_Region()) {
689 for (uint i = 1; i < c->req(); i++) {
690 if (c->in(i) != NULL) {
691 Node* n = c->in(i)->in(0);
692 if (may_modify_helper(t_oop, n, phase, call)) {
693 ac = call->isa_ArrayCopy();
694 assert(c == mb->in(0), "only for clone");
695 return true;
696 }
697 }
698 }
699 } else if (may_modify_helper(t_oop, c->in(0), phase, call)) {
700 ac = call->isa_ArrayCopy();
701 assert(c == mb->in(0) || (ac != NULL && ac->is_clonebasic() && !GraphKit::use_ReduceInitialCardMarks()), "only for clone");
702 return true;
703 }
704
705 return false;
706 }
707
708 // Does this array copy modify offsets between offset_lo and offset_hi
709 // in the destination array
710 // if must_modify is false, return true if the copy could write
711 // between offset_lo and offset_hi
712 // if must_modify is true, return true if the copy is guaranteed to
713 // write between offset_lo and offset_hi
714 bool ArrayCopyNode::modifies(intptr_t offset_lo, intptr_t offset_hi, PhaseTransform* phase, bool must_modify) const {
715 assert(_kind == ArrayCopy || _kind == CopyOf || _kind == CopyOfRange, "only for real array copies");
716
717 Node* dest = in(Dest);
718 Node* dest_pos = in(DestPos);
719 Node* len = in(Length);
720
|
632
633 if (_dest_type != TypeOopPtr::BOTTOM || t_oop->is_known_instance()) {
634 assert(_dest_type == TypeOopPtr::BOTTOM || _dest_type->is_known_instance(), "result of EA is known instance");
635 return t_oop->instance_id() == _dest_type->instance_id();
636 }
637
638 return CallNode::may_modify_arraycopy_helper(dest_t, t_oop, phase);
639 }
640
641 bool ArrayCopyNode::may_modify_helper(const TypeOopPtr *t_oop, Node* n, PhaseTransform *phase, CallNode*& call) {
642 if (n != NULL &&
643 n->is_Call() &&
644 n->as_Call()->may_modify(t_oop, phase) &&
645 (n->as_Call()->is_ArrayCopy() || n->as_Call()->is_call_to_arraycopystub())) {
646 call = n->as_Call();
647 return true;
648 }
649 return false;
650 }
651
652 bool ArrayCopyNode::may_modify(const TypeOopPtr *t_oop, MemBarNode* mb, PhaseTransform *phase, ArrayCopyNode*& ac) {
653
654 Node* c = mb->in(0);
655
656 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
657 // step over g1 gc barrier if we're at e.g. a clone with ReduceInitialCardMarks off
658 c = bs->step_over_gc_barrier(c);
659
660 CallNode* call = NULL;
661 if (c != NULL && c->is_Region()) {
662 for (uint i = 1; i < c->req(); i++) {
663 if (c->in(i) != NULL) {
664 Node* n = c->in(i)->in(0);
665 if (may_modify_helper(t_oop, n, phase, call)) {
666 ac = call->isa_ArrayCopy();
667 assert(c == mb->in(0), "only for clone");
668 return true;
669 }
670 }
671 }
672 } else if (may_modify_helper(t_oop, c->in(0), phase, call)) {
673 ac = call->isa_ArrayCopy();
674 #ifdef ASSERT
675 bool use_ReduceInitialCardMarks = BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
676 static_cast<CardTableBarrierSetC2*>(bs)->use_ReduceInitialCardMarks();
677 assert(c == mb->in(0) || (ac != NULL && ac->is_clonebasic() && !use_ReduceInitialCardMarks), "only for clone");
678 #endif
679 return true;
680 }
681
682 return false;
683 }
684
685 // Does this array copy modify offsets between offset_lo and offset_hi
686 // in the destination array
687 // if must_modify is false, return true if the copy could write
688 // between offset_lo and offset_hi
689 // if must_modify is true, return true if the copy is guaranteed to
690 // write between offset_lo and offset_hi
691 bool ArrayCopyNode::modifies(intptr_t offset_lo, intptr_t offset_hi, PhaseTransform* phase, bool must_modify) const {
692 assert(_kind == ArrayCopy || _kind == CopyOf || _kind == CopyOfRange, "only for real array copies");
693
694 Node* dest = in(Dest);
695 Node* dest_pos = in(DestPos);
696 Node* len = in(Length);
697
|
729 BasicType ary_elem = ary_t->klass()->as_array_klass()->element_type()->basic_type();
730 uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
731 uint elemsize = type2aelembytes(ary_elem);
732
733 jlong dest_pos_plus_len_lo = (((jlong)dest_pos_t->_lo) + len_t->_lo) * elemsize + header;
734 jlong dest_pos_plus_len_hi = (((jlong)dest_pos_t->_hi) + len_t->_hi) * elemsize + header;
735 jlong dest_pos_lo = ((jlong)dest_pos_t->_lo) * elemsize + header;
736 jlong dest_pos_hi = ((jlong)dest_pos_t->_hi) * elemsize + header;
737
738 if (must_modify) {
739 if (offset_lo >= dest_pos_hi && offset_hi < dest_pos_plus_len_lo) {
740 return true;
741 }
742 } else {
743 if (offset_hi >= dest_pos_lo && offset_lo < dest_pos_plus_len_hi) {
744 return true;
745 }
746 }
747 return false;
748 }
749
|
706 BasicType ary_elem = ary_t->klass()->as_array_klass()->element_type()->basic_type();
707 uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
708 uint elemsize = type2aelembytes(ary_elem);
709
710 jlong dest_pos_plus_len_lo = (((jlong)dest_pos_t->_lo) + len_t->_lo) * elemsize + header;
711 jlong dest_pos_plus_len_hi = (((jlong)dest_pos_t->_hi) + len_t->_hi) * elemsize + header;
712 jlong dest_pos_lo = ((jlong)dest_pos_t->_lo) * elemsize + header;
713 jlong dest_pos_hi = ((jlong)dest_pos_t->_hi) * elemsize + header;
714
715 if (must_modify) {
716 if (offset_lo >= dest_pos_hi && offset_hi < dest_pos_plus_len_lo) {
717 return true;
718 }
719 } else {
720 if (offset_hi >= dest_pos_lo && offset_lo < dest_pos_plus_len_hi) {
721 return true;
722 }
723 }
724 return false;
725 }
|