164 if (n->Opcode() == Op_LoadP ||
165 n->Opcode() == Op_LoadN ||
166 n->Opcode() == Op_GetAndSetP ||
167 n->Opcode() == Op_CompareAndExchangeP ||
168 n->Opcode() == Op_GetAndSetN ||
169 n->Opcode() == Op_CompareAndExchangeN) {
170 return true;
171 }
172 if (n->Opcode() == Op_DecodeN ||
173 n->Opcode() == Op_EncodeP) {
174 return needs_barrier_impl(phase, orig, n->in(1), rb_mem, allow_fromspace, visited);
175 }
176
177 #ifdef ASSERT
178 tty->print("need barrier on?: "); n->dump();
179 ShouldNotReachHere();
180 #endif
181 return true;
182 }
183
184 /**
185 * In Shenandoah, we need barriers on acmp (and similar instructions that compare two
186 * oops) to avoid false negatives. If it compares a from-space and a to-space
187 * copy of an object, a regular acmp would return false, even though both are
188 * the same. The acmp barrier compares the two objects, and when they are
189 * *not equal* it does a read-barrier on both, and compares them again. When it
190 * failed because of different copies of the object, we know that the object
191 * must already have been evacuated (and therefore doesn't require a write-barrier).
192 */
193 void ShenandoahBarrierNode::do_cmpp_if(GraphKit& kit, Node*& taken_branch, Node*& untaken_branch, Node*& taken_memory, Node*& untaken_memory) {
194 assert(taken_memory == NULL && untaken_memory == NULL, "unexpected memory inputs");
195 if (!UseShenandoahGC || !ShenandoahAcmpBarrier || ShenandoahVerifyOptoBarriers) {
196 return;
197 }
198 if (taken_branch->is_top() || untaken_branch->is_top()) {
199 // one of the branches is known to be untaken
200 return;
201 }
202 assert(taken_branch->is_IfProj() && untaken_branch->is_IfProj(), "if projections only");
203 assert(taken_branch->in(0) == untaken_branch->in(0), "should come from same if");
204 IfNode* iff = taken_branch->in(0)->as_If();
205 BoolNode* bol = iff->in(1)->as_Bool();
206 Node* cmp = bol->in(1);
207 if (cmp->Opcode() != Op_CmpP) {
208 return;
209 }
210 Node* a = cmp->in(1);
211 Node* b = cmp->in(2);
212 const Type* a_type = kit.gvn().type(a);
213 const Type* b_type = kit.gvn().type(b);
214 if (a_type->higher_equal(TypePtr::NULL_PTR) || b_type->higher_equal(TypePtr::NULL_PTR)) {
215 // We know one arg is gonna be null. No need for barriers.
216 return;
217 }
218
219 const TypePtr* a_adr_type = ShenandoahBarrierNode::brooks_pointer_type(a_type);
220 const TypePtr* b_adr_type = ShenandoahBarrierNode::brooks_pointer_type(b_type);
221 if ((! ShenandoahBarrierNode::needs_barrier(&kit.gvn(), NULL, a, kit.memory(a_adr_type), false)) &&
222 (! ShenandoahBarrierNode::needs_barrier(&kit.gvn(), NULL, b, kit.memory(b_adr_type), false))) {
223 // We know both args are in to-space already. No acmp barrier needed.
224 return;
225 }
226
227 Node* equal_path = iff->proj_out(true);
228 Node* not_equal_path = iff->proj_out(false);
229
230 if (bol->_test._test == BoolTest::ne) {
231 swap(equal_path, not_equal_path);
232 }
233
234 Node* init_equal_path = equal_path;
235 Node* init_not_equal_path = not_equal_path;
236
237 uint alias_a = kit.C->get_alias_index(a_adr_type);
238 uint alias_b = kit.C->get_alias_index(b_adr_type);
239
240 Node* equal_memory = NULL;
241 Node* not_equal_memory = NULL;
242
243 RegionNode* region = new RegionNode(3);
244 region->init_req(1, equal_path);
245 PhiNode* mem_phi = NULL;
246 if (alias_a == alias_b) {
247 mem_phi = PhiNode::make(region, kit.memory(alias_a), Type::MEMORY, kit.C->get_adr_type(alias_a));
248 } else {
249 Node* mem = kit.reset_memory();
250 mem_phi = PhiNode::make(region, mem, Type::MEMORY, TypePtr::BOTTOM);
251 kit.set_all_memory(mem);
252 }
253
254 kit.set_control(not_equal_path);
255
256 Node* mb = NULL;
257 if (alias_a == alias_b) {
258 Node* mem = kit.reset_memory();
259 mb = MemBarNode::make(kit.C, Op_MemBarAcquire, alias_a);
260 mb->init_req(TypeFunc::Control, kit.control());
261 mb->init_req(TypeFunc::Memory, mem);
262 Node* membar = kit.gvn().transform(mb);
263 kit.set_control(kit.gvn().transform(new ProjNode(membar, TypeFunc::Control)));
264 Node* newmem = kit.gvn().transform(new ProjNode(membar, TypeFunc::Memory));
265 kit.set_all_memory(mem);
266 kit.set_memory(newmem, alias_a);
267 } else {
268 mb = kit.insert_mem_bar(Op_MemBarAcquire);
269 }
270
271 ShenandoahBarrierSetC2* bs = (ShenandoahBarrierSetC2*) BarrierSet::barrier_set()->barrier_set_c2();
272 a = bs->shenandoah_read_barrier_acmp(&kit, a);
273 b = bs->shenandoah_read_barrier_acmp(&kit, b);
274
275 Node* cmp2 = kit.gvn().transform(new CmpPNode(a, b));
276 Node* bol2 = bol->clone();
277 bol2->set_req(1, cmp2);
278 bol2 = kit.gvn().transform(bol2);
279 Node* iff2 = iff->clone();
280 iff2->set_req(0, kit.control());
281 iff2->set_req(1, bol2);
282 kit.gvn().set_type(iff2, kit.gvn().type(iff));
283 Node* equal_path2 = equal_path->clone();
284 equal_path2->set_req(0, iff2);
285 equal_path2 = kit.gvn().transform(equal_path2);
286 Node* not_equal_path2 = not_equal_path->clone();
287 not_equal_path2->set_req(0, iff2);
288 not_equal_path2 = kit.gvn().transform(not_equal_path2);
289
290 region->init_req(2, equal_path2);
291 not_equal_memory = kit.reset_memory();
292 not_equal_path = not_equal_path2;
293
294 kit.set_all_memory(not_equal_memory);
295
296 if (alias_a == alias_b) {
297 mem_phi->init_req(2, kit.memory(alias_a));
298 kit.set_memory(mem_phi, alias_a);
299 } else {
300 mem_phi->init_req(2, kit.reset_memory());
301 }
302
303 kit.record_for_igvn(mem_phi);
304 kit.gvn().set_type(mem_phi, Type::MEMORY);
305
306 if (alias_a == alias_b) {
307 equal_memory = kit.reset_memory();
308 } else {
309 equal_memory = mem_phi;
310 }
311
312 assert(kit.map()->memory() == NULL, "no live memory state");
313 equal_path = kit.gvn().transform(region);
314
315 if (taken_branch == init_equal_path) {
316 assert(untaken_branch == init_not_equal_path, "inconsistent");
317 taken_branch = equal_path;
318 untaken_branch = not_equal_path;
319 taken_memory = equal_memory;
320 untaken_memory = not_equal_memory;
321 } else {
322 assert(taken_branch == init_not_equal_path, "inconsistent");
323 assert(untaken_branch == init_equal_path, "inconsistent");
324 taken_branch = not_equal_path;
325 untaken_branch = equal_path;
326 taken_memory = not_equal_memory;
327 untaken_memory = equal_memory;
328 }
329 }
330
331 bool ShenandoahReadBarrierNode::dominates_memory_rb_impl(PhaseGVN* phase,
332 Node* b1,
333 Node* b2,
334 Node* current,
335 bool linear) {
336 ResourceMark rm;
337 VectorSet visited(Thread::current()->resource_area());
338 Node_Stack phis(0);
339
340 for(int i = 0; i < 10; i++) {
341 if (current == NULL) {
342 return false;
343 } else if (visited.test_set(current->_idx) || current->is_top() || current == b1) {
344 current = NULL;
345 while (phis.is_nonempty() && current == NULL) {
346 uint idx = phis.index();
347 Node* phi = phis.node();
348 if (idx >= phi->req()) {
349 phis.pop();
350 } else {
|
164 if (n->Opcode() == Op_LoadP ||
165 n->Opcode() == Op_LoadN ||
166 n->Opcode() == Op_GetAndSetP ||
167 n->Opcode() == Op_CompareAndExchangeP ||
168 n->Opcode() == Op_GetAndSetN ||
169 n->Opcode() == Op_CompareAndExchangeN) {
170 return true;
171 }
172 if (n->Opcode() == Op_DecodeN ||
173 n->Opcode() == Op_EncodeP) {
174 return needs_barrier_impl(phase, orig, n->in(1), rb_mem, allow_fromspace, visited);
175 }
176
177 #ifdef ASSERT
178 tty->print("need barrier on?: "); n->dump();
179 ShouldNotReachHere();
180 #endif
181 return true;
182 }
183
184 bool ShenandoahReadBarrierNode::dominates_memory_rb_impl(PhaseGVN* phase,
185 Node* b1,
186 Node* b2,
187 Node* current,
188 bool linear) {
189 ResourceMark rm;
190 VectorSet visited(Thread::current()->resource_area());
191 Node_Stack phis(0);
192
193 for(int i = 0; i < 10; i++) {
194 if (current == NULL) {
195 return false;
196 } else if (visited.test_set(current->_idx) || current->is_top() || current == b1) {
197 current = NULL;
198 while (phis.is_nonempty() && current == NULL) {
199 uint idx = phis.index();
200 Node* phi = phis.node();
201 if (idx >= phi->req()) {
202 phis.pop();
203 } else {
|