1014 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
1015 "sha512_implCompress",
1016 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
1017 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
1018 "sha1_implCompressMB",
1019 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
1020 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
1021 "sha256_implCompressMB",
1022 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
1023 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
1024 "sha512_implCompressMB",
1025 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
1026 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
1027 "encodeBlock",
1028 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
1029 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
1030 };
1031
1032 if (call->is_call_to_arraycopystub()) {
1033 Node* dest = NULL;
1034 const TypeTuple* args = n->as_Call()->_tf->domain();
1035 for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {
1036 if (args->field_at(i)->isa_ptr()) {
1037 j++;
1038 if (j == 2) {
1039 dest = n->in(i);
1040 break;
1041 }
1042 }
1043 }
1044 if (!ShenandoahBarrierNode::verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
1045 !ShenandoahBarrierNode::verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
1046 report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
1047 }
1048 } else if (strlen(call->_name) > 5 &&
1049 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
1050 if (!ShenandoahBarrierNode::verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
1051 report_verify_failure("Shenandoah verification: _fill should have barriers", n);
1052 }
1053 } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
1054 // skip
1140 { { 2, ShenandoahLoad }, { 4, ShenandoahLoad } },
1141 Op_StrEquals,
1142 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } },
1143 Op_EncodeISOArray,
1144 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } },
1145 Op_HasNegatives,
1146 { { 2, ShenandoahLoad }, { -1, ShenandoahNone} },
1147 Op_CastP2X,
1148 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} },
1149 Op_StrIndexOfChar,
1150 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
1151 };
1152
1153 const int others_len = sizeof(others) / sizeof(others[0]);
1154 int i = 0;
1155 for (; i < others_len; i++) {
1156 if (others[i].opcode == n->Opcode()) {
1157 break;
1158 }
1159 }
1160 uint stop = n->is_Call() ? n->as_Call()->tf()->domain()->cnt() : n->req();
1161 if (i != others_len) {
1162 const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]);
1163 for (uint j = 0; j < inputs_len; j++) {
1164 int pos = others[i].inputs[j].pos;
1165 if (pos == -1) {
1166 break;
1167 }
1168 if (!ShenandoahBarrierNode::verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
1169 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
1170 }
1171 }
1172 for (uint j = 1; j < stop; j++) {
1173 if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() &&
1174 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
1175 uint k = 0;
1176 for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++);
1177 if (k == inputs_len) {
1178 fatal("arg %d for node %s not covered", j, n->Name());
1179 }
1180 }
2297 }
2298 }
2299 }
2300
2301 Node* ShenandoahWriteBarrierNode::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
2302 Node* mem = NULL;
2303 Node* c = ctrl;
2304 do {
2305 if (c->is_Region()) {
2306 Node* phi_bottom = NULL;
2307 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == NULL; i++) {
2308 Node* u = c->fast_out(i);
2309 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
2310 if (u->adr_type() == TypePtr::BOTTOM) {
2311 mem = u;
2312 }
2313 }
2314 }
2315 } else {
2316 if (c->is_Call() && c->as_Call()->adr_type() != NULL) {
2317 CallProjections projs;
2318 c->as_Call()->extract_projections(&projs, true, false);
2319 if (projs.fallthrough_memproj != NULL) {
2320 if (projs.fallthrough_memproj->adr_type() == TypePtr::BOTTOM) {
2321 if (projs.catchall_memproj == NULL) {
2322 mem = projs.fallthrough_memproj;
2323 } else {
2324 if (phase->is_dominator(projs.fallthrough_catchproj, ctrl)) {
2325 mem = projs.fallthrough_memproj;
2326 } else {
2327 assert(phase->is_dominator(projs.catchall_catchproj, ctrl), "one proj must dominate barrier");
2328 mem = projs.catchall_memproj;
2329 }
2330 }
2331 }
2332 } else {
2333 Node* proj = c->as_Call()->proj_out(TypeFunc::Memory);
2334 if (proj != NULL &&
2335 proj->adr_type() == TypePtr::BOTTOM) {
2336 mem = proj;
2337 }
2338 }
2339 } else {
2340 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
2341 Node* u = c->fast_out(i);
2342 if (u->is_Proj() &&
2343 u->bottom_type() == Type::MEMORY &&
2344 u->adr_type() == TypePtr::BOTTOM) {
2345 assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), "");
2346 assert(mem == NULL, "only one proj");
2347 mem = u;
2348 }
3741 Node* u = c->fast_out(i);
3742 if (u->is_Phi() && u->bottom_type() == Type::MEMORY &&
3743 u != n) {
3744 if (u->adr_type() == TypePtr::BOTTOM) {
3745 fix_memory_uses(u, n, n, c);
3746 } else if (_phase->C->get_alias_index(u->adr_type()) == _alias) {
3747 _phase->lazy_replace(u, n);
3748 --i; --imax;
3749 }
3750 }
3751 }
3752 }
3753 }
3754 }
3755
3756 Node* MemoryGraphFixer::get_ctrl(Node* n) const {
3757 Node* c = _phase->get_ctrl(n);
3758 if (n->is_Proj() && n->in(0) != NULL && n->in(0)->is_Call()) {
3759 assert(c == n->in(0), "");
3760 CallNode* call = c->as_Call();
3761 CallProjections projs;
3762 call->extract_projections(&projs, true, false);
3763 if (projs.catchall_memproj != NULL) {
3764 if (projs.fallthrough_memproj == n) {
3765 c = projs.fallthrough_catchproj;
3766 } else {
3767 assert(projs.catchall_memproj == n, "");
3768 c = projs.catchall_catchproj;
3769 }
3770 }
3771 }
3772 return c;
3773 }
3774
3775 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const {
3776 if (_phase->has_ctrl(n))
3777 return get_ctrl(n);
3778 else {
3779 assert (n->is_CFG(), "must be a CFG node");
3780 return n;
3781 }
3782 }
3783
3784 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const {
3785 return m != NULL && get_ctrl(m) == c;
3786 }
3787
3788 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const {
|
1014 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
1015 "sha512_implCompress",
1016 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
1017 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
1018 "sha1_implCompressMB",
1019 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
1020 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
1021 "sha256_implCompressMB",
1022 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
1023 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
1024 "sha512_implCompressMB",
1025 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { -1, ShenandoahNone },
1026 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
1027 "encodeBlock",
1028 { { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahStore }, { -1, ShenandoahNone },
1029 { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
1030 };
1031
1032 if (call->is_call_to_arraycopystub()) {
1033 Node* dest = NULL;
1034 const TypeTuple* args = n->as_Call()->_tf->domain_sig();
1035 for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {
1036 if (args->field_at(i)->isa_ptr()) {
1037 j++;
1038 if (j == 2) {
1039 dest = n->in(i);
1040 break;
1041 }
1042 }
1043 }
1044 if (!ShenandoahBarrierNode::verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||
1045 !ShenandoahBarrierNode::verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {
1046 report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);
1047 }
1048 } else if (strlen(call->_name) > 5 &&
1049 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {
1050 if (!ShenandoahBarrierNode::verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {
1051 report_verify_failure("Shenandoah verification: _fill should have barriers", n);
1052 }
1053 } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {
1054 // skip
1140 { { 2, ShenandoahLoad }, { 4, ShenandoahLoad } },
1141 Op_StrEquals,
1142 { { 2, ShenandoahLoad }, { 3, ShenandoahLoad } },
1143 Op_EncodeISOArray,
1144 { { 2, ShenandoahLoad }, { 3, ShenandoahStore } },
1145 Op_HasNegatives,
1146 { { 2, ShenandoahLoad }, { -1, ShenandoahNone} },
1147 Op_CastP2X,
1148 { { 1, ShenandoahLoad }, { -1, ShenandoahNone} },
1149 Op_StrIndexOfChar,
1150 { { 2, ShenandoahLoad }, { -1, ShenandoahNone } },
1151 };
1152
1153 const int others_len = sizeof(others) / sizeof(others[0]);
1154 int i = 0;
1155 for (; i < others_len; i++) {
1156 if (others[i].opcode == n->Opcode()) {
1157 break;
1158 }
1159 }
1160 uint stop = n->is_Call() ? n->as_Call()->tf()->domain_sig()->cnt() : n->req();
1161 if (i != others_len) {
1162 const uint inputs_len = sizeof(others[0].inputs) / sizeof(others[0].inputs[0]);
1163 for (uint j = 0; j < inputs_len; j++) {
1164 int pos = others[i].inputs[j].pos;
1165 if (pos == -1) {
1166 break;
1167 }
1168 if (!ShenandoahBarrierNode::verify_helper(n->in(pos), phis, visited, others[i].inputs[j].t, trace, barriers_used)) {
1169 report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);
1170 }
1171 }
1172 for (uint j = 1; j < stop; j++) {
1173 if (n->in(j) != NULL && n->in(j)->bottom_type()->make_ptr() &&
1174 n->in(j)->bottom_type()->make_ptr()->make_oopptr()) {
1175 uint k = 0;
1176 for (; k < inputs_len && others[i].inputs[k].pos != (int)j; k++);
1177 if (k == inputs_len) {
1178 fatal("arg %d for node %s not covered", j, n->Name());
1179 }
1180 }
2297 }
2298 }
2299 }
2300
2301 Node* ShenandoahWriteBarrierNode::find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase) {
2302 Node* mem = NULL;
2303 Node* c = ctrl;
2304 do {
2305 if (c->is_Region()) {
2306 Node* phi_bottom = NULL;
2307 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax && mem == NULL; i++) {
2308 Node* u = c->fast_out(i);
2309 if (u->is_Phi() && u->bottom_type() == Type::MEMORY) {
2310 if (u->adr_type() == TypePtr::BOTTOM) {
2311 mem = u;
2312 }
2313 }
2314 }
2315 } else {
2316 if (c->is_Call() && c->as_Call()->adr_type() != NULL) {
2317 CallProjections* projs = c->as_Call()->extract_projections(true, false);
2318 if (projs->fallthrough_memproj != NULL) {
2319 if (projs->fallthrough_memproj->adr_type() == TypePtr::BOTTOM) {
2320 if (projs->catchall_memproj == NULL) {
2321 mem = projs->fallthrough_memproj;
2322 } else {
2323 if (phase->is_dominator(projs->fallthrough_catchproj, ctrl)) {
2324 mem = projs->fallthrough_memproj;
2325 } else {
2326 assert(phase->is_dominator(projs->catchall_catchproj, ctrl), "one proj must dominate barrier");
2327 mem = projs->catchall_memproj;
2328 }
2329 }
2330 }
2331 } else {
2332 Node* proj = c->as_Call()->proj_out(TypeFunc::Memory);
2333 if (proj != NULL &&
2334 proj->adr_type() == TypePtr::BOTTOM) {
2335 mem = proj;
2336 }
2337 }
2338 } else {
2339 for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
2340 Node* u = c->fast_out(i);
2341 if (u->is_Proj() &&
2342 u->bottom_type() == Type::MEMORY &&
2343 u->adr_type() == TypePtr::BOTTOM) {
2344 assert(c->is_SafePoint() || c->is_MemBar() || c->is_Start(), "");
2345 assert(mem == NULL, "only one proj");
2346 mem = u;
2347 }
3740 Node* u = c->fast_out(i);
3741 if (u->is_Phi() && u->bottom_type() == Type::MEMORY &&
3742 u != n) {
3743 if (u->adr_type() == TypePtr::BOTTOM) {
3744 fix_memory_uses(u, n, n, c);
3745 } else if (_phase->C->get_alias_index(u->adr_type()) == _alias) {
3746 _phase->lazy_replace(u, n);
3747 --i; --imax;
3748 }
3749 }
3750 }
3751 }
3752 }
3753 }
3754
3755 Node* MemoryGraphFixer::get_ctrl(Node* n) const {
3756 Node* c = _phase->get_ctrl(n);
3757 if (n->is_Proj() && n->in(0) != NULL && n->in(0)->is_Call()) {
3758 assert(c == n->in(0), "");
3759 CallNode* call = c->as_Call();
3760 CallProjections* projs = call->extract_projections(true, false);
3761 if (projs->catchall_memproj != NULL) {
3762 if (projs->fallthrough_memproj == n) {
3763 c = projs->fallthrough_catchproj;
3764 } else {
3765 assert(projs->catchall_memproj == n, "");
3766 c = projs->catchall_catchproj;
3767 }
3768 }
3769 }
3770 return c;
3771 }
3772
3773 Node* MemoryGraphFixer::ctrl_or_self(Node* n) const {
3774 if (_phase->has_ctrl(n))
3775 return get_ctrl(n);
3776 else {
3777 assert (n->is_CFG(), "must be a CFG node");
3778 return n;
3779 }
3780 }
3781
3782 bool MemoryGraphFixer::mem_is_valid(Node* m, Node* c) const {
3783 return m != NULL && get_ctrl(m) == c;
3784 }
3785
3786 Node* MemoryGraphFixer::find_mem(Node* ctrl, Node* n) const {
|