23
24 #include "precompiled.hpp"
25 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
26 #include "gc/shenandoah/shenandoahHeap.hpp"
27 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
28 #include "gc/shenandoah/shenandoahHeuristics.hpp"
29 #include "gc/shenandoah/shenandoahRuntime.hpp"
30 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
31 #include "interpreter/interpreter.hpp"
32 #include "interpreter/interp_masm.hpp"
33 #include "runtime/sharedRuntime.hpp"
34 #include "runtime/thread.hpp"
35 #ifdef COMPILER1
36 #include "c1/c1_LIRAssembler.hpp"
37 #include "c1/c1_MacroAssembler.hpp"
38 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
39 #endif
40
41 #define __ masm->
42
43 address ShenandoahBarrierSetAssembler::_shenandoah_wb = NULL;
44
45 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
46 Register addr, Register count, RegSet saved_regs) {
47 if (is_oop) {
48 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
49 if (ShenandoahSATBBarrier && !dest_uninitialized && !ShenandoahHeap::heap()->heuristics()->can_do_traversal_gc()) {
50
51 Label done;
52
53 // Avoid calling runtime if count == 0
54 __ cbz(count, done);
55
56 // Is marking active?
57 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
58 __ ldrb(rscratch1, gc_state);
59 __ tbz(rscratch1, ShenandoahHeap::MARKING_BITPOS, done);
60
61 __ push(saved_regs, sp);
62 if (count == c_rarg0) {
63 if (addr == c_rarg1) {
181 // If we care generating the pre-barrier without a frame (e.g. in the
182 // intrinsified Reference.get() routine) then ebp might be pointing to
183 // the caller frame and so this check will most likely fail at runtime.
184 //
185 // Expanding the call directly bypasses the generation of the check.
186 // So when we do not have have a full interpreter frame on the stack
187 // expand_call should be passed true.
188
189 if (expand_call) {
190 assert(pre_val != c_rarg1, "smashed arg");
191 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
192 } else {
193 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
194 }
195
196 __ pop(saved, sp);
197
198 __ bind(done);
199 }
200
201 void ShenandoahBarrierSetAssembler::read_barrier(MacroAssembler* masm, Register dst) {
202 if (ShenandoahReadBarrier) {
203 read_barrier_impl(masm, dst);
204 }
205 }
206
207 void ShenandoahBarrierSetAssembler::read_barrier_impl(MacroAssembler* masm, Register dst) {
208 assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled");
209 Label is_null;
210 __ cbz(dst, is_null);
211 read_barrier_not_null_impl(masm, dst);
212 __ bind(is_null);
213 }
214
215 void ShenandoahBarrierSetAssembler::read_barrier_not_null(MacroAssembler* masm, Register dst) {
216 if (ShenandoahReadBarrier) {
217 read_barrier_not_null_impl(masm, dst);
218 }
219 }
220
221
222 void ShenandoahBarrierSetAssembler::read_barrier_not_null_impl(MacroAssembler* masm, Register dst) {
223 assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled");
224 __ ldr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset()));
225 }
226
227 void ShenandoahBarrierSetAssembler::write_barrier(MacroAssembler* masm, Register dst) {
228 if (ShenandoahWriteBarrier) {
229 write_barrier_impl(masm, dst);
230 }
231 }
232
233 void ShenandoahBarrierSetAssembler::write_barrier_impl(MacroAssembler* masm, Register dst) {
234 assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "Should be enabled");
235 assert(dst != rscratch1, "need rscratch1");
236 assert(dst != rscratch2, "need rscratch2");
237
238 Label done;
239
240 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
241 __ ldrb(rscratch1, gc_state);
242
243 // Check for heap stability
244 __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
245 __ tst(rscratch1, rscratch2);
246 __ br(Assembler::EQ, done);
247
248 // Heap is unstable, need to perform the read-barrier even if WB is inactive
249 __ ldr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset()));
250
251 // Check for evacuation-in-progress and jump to WB slow-path if needed
252 __ mov(rscratch2, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
253 __ tst(rscratch1, rscratch2);
254 __ br(Assembler::EQ, done);
255
256 RegSet to_save = RegSet::of(r0);
257 if (dst != r0) {
258 __ push(to_save, sp);
259 __ mov(r0, dst);
260 }
261
262 __ far_call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_wb())));
263
264 if (dst != r0) {
265 __ mov(dst, r0);
266 __ pop(to_save, sp);
267 }
268
269 __ bind(done);
270 }
271
272 void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
273 if (ShenandoahStoreValEnqueueBarrier) {
274 Label is_null;
275 __ cbz(dst, is_null);
276 write_barrier_impl(masm, dst);
277 __ bind(is_null);
278 // Save possibly live regs.
279 RegSet live_regs = RegSet::range(r0, r4) - dst;
280 __ push(live_regs, sp);
281 __ strd(v0, __ pre(sp, 2 * -wordSize));
282
283 satb_write_barrier_pre(masm, noreg, dst, rthread, tmp, true, false);
284
285 // Restore possibly live regs.
286 __ ldrd(v0, __ post(sp, 2 * wordSize));
287 __ pop(live_regs, sp);
288 }
289 if (ShenandoahStoreValReadBarrier) {
290 read_barrier_impl(masm, dst);
291 }
292 }
293
294 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
295 Register dst, Address src, Register tmp1, Register tmp_thread) {
296 bool on_oop = type == T_OBJECT || type == T_ARRAY;
297 bool in_heap = (decorators & IN_HEAP) != 0;
298 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
299 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
300 bool on_reference = on_weak || on_phantom;
301
302 if (in_heap) {
303 read_barrier_not_null(masm, src.base());
304 }
305
306 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
307 if (ShenandoahKeepAliveBarrier && on_oop && on_reference) {
308 __ enter();
309 satb_write_barrier_pre(masm /* masm */,
310 noreg /* obj */,
311 dst /* pre_val */,
312 rthread /* thread */,
313 tmp1 /* tmp */,
314 true /* tosca_live */,
315 true /* expand_call */);
316 __ leave();
317 }
318 }
319
320 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
321 Address dst, Register val, Register tmp1, Register tmp2) {
322 bool on_oop = type == T_OBJECT || type == T_ARRAY;
323 bool in_heap = (decorators & IN_HEAP) != 0;
324 if (in_heap) {
325 write_barrier(masm, dst.base());
326 }
327 if (!on_oop) {
328 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
329 return;
330 }
331
332 // flatten object address if needed
333 if (dst.index() == noreg && dst.offset() == 0) {
334 if (dst.base() != r3) {
335 __ mov(r3, dst.base());
336 }
337 } else {
338 __ lea(r3, dst);
339 }
340
341 shenandoah_write_barrier_pre(masm,
342 r3 /* obj */,
343 tmp2 /* pre_val */,
344 rthread /* thread */,
345 tmp1 /* tmp */,
346 val != noreg /* tosca_live */,
347 false /* expand_call */);
348
349 if (val == noreg) {
350 BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), noreg, noreg, noreg);
351 } else {
352 storeval_barrier(masm, val, tmp1);
353 // G1 barrier needs uncompressed oop for region cross check.
354 Register new_val = val;
355 if (UseCompressedOops) {
356 new_val = rscratch2;
357 __ mov(new_val, val);
358 }
359 BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), val, noreg, noreg);
360 }
361
362 }
363
364 void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, Register op1, Register op2) {
365 __ cmp(op1, op2);
366 if (ShenandoahAcmpBarrier) {
367 Label done;
368 __ br(Assembler::EQ, done);
369 // The object may have been evacuated, but we won't see it without a
370 // membar here.
371 __ membar(Assembler::LoadStore| Assembler::LoadLoad);
372 read_barrier(masm, op1);
373 read_barrier(masm, op2);
374 __ cmp(op1, op2);
375 __ bind(done);
376 }
377 }
378
379 void ShenandoahBarrierSetAssembler::tlab_allocate(MacroAssembler* masm, Register obj,
380 Register var_size_in_bytes,
381 int con_size_in_bytes,
382 Register t1,
383 Register t2,
384 Label& slow_case) {
385
386 assert_different_registers(obj, t2);
387 assert_different_registers(obj, var_size_in_bytes);
388 Register end = t2;
389
390 __ ldr(obj, Address(rthread, JavaThread::tlab_top_offset()));
391 if (var_size_in_bytes == noreg) {
392 __ lea(end, Address(obj, (int) (con_size_in_bytes + ShenandoahBrooksPointer::byte_size())));
393 } else {
394 __ add(var_size_in_bytes, var_size_in_bytes, ShenandoahBrooksPointer::byte_size());
395 __ lea(end, Address(obj, var_size_in_bytes));
396 }
397 __ ldr(rscratch1, Address(rthread, JavaThread::tlab_end_offset()));
398 __ cmp(end, rscratch1);
399 __ br(Assembler::HI, slow_case);
400
401 // update the tlab top pointer
402 __ str(end, Address(rthread, JavaThread::tlab_top_offset()));
403
404 __ add(obj, obj, ShenandoahBrooksPointer::byte_size());
405 __ str(obj, Address(obj, ShenandoahBrooksPointer::byte_offset()));
406
407 // recover var_size_in_bytes if necessary
408 if (var_size_in_bytes == end) {
409 __ sub(var_size_in_bytes, var_size_in_bytes, obj);
410 }
411 }
412
413 void ShenandoahBarrierSetAssembler::resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj) {
414 bool oop_not_null = (decorators & IS_NOT_NULL) != 0;
415 bool is_write = (decorators & ACCESS_WRITE) != 0;
416 if (is_write) {
417 if (oop_not_null) {
418 write_barrier(masm, obj);
419 } else {
420 Label done;
421 __ cbz(obj, done);
422 write_barrier(masm, obj);
423 __ bind(done);
424 }
425 } else {
426 if (oop_not_null) {
427 read_barrier_not_null(masm, obj);
428 } else {
429 read_barrier(masm, obj);
430 }
431 }
432 }
433
434 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val,
435 bool acquire, bool release, bool weak, bool is_cae,
436 Register result) {
437 Register tmp1 = rscratch1;
438 Register tmp2 = rscratch2;
439 bool is_narrow = UseCompressedOops;
440 Assembler::operand_size size = is_narrow ? Assembler::word : Assembler::xword;
441
442 assert_different_registers(addr, expected, new_val, tmp1, tmp2);
443
444 Label retry, done, fail;
445
446 // CAS, using LL/SC pair.
447 __ bind(retry);
448 __ load_exclusive(tmp1, addr, size, acquire);
449 if (is_narrow) {
450 __ cmpw(tmp1, expected);
451 } else {
452 __ cmp(tmp1, expected);
453 }
454 __ br(Assembler::NE, fail);
455 __ store_exclusive(tmp2, new_val, addr, size, release);
456 if (weak) {
457 __ cmpw(tmp2, 0u); // If the store fails, return NE to our caller
458 } else {
459 __ cbnzw(tmp2, retry);
460 }
461 __ b(done);
462
463 __ bind(fail);
464 // Check if rb(expected)==rb(tmp1)
465 // Shuffle registers so that we have memory value ready for next expected.
466 __ mov(tmp2, expected);
467 __ mov(expected, tmp1);
468 if (is_narrow) {
469 __ decode_heap_oop(tmp1, tmp1);
470 __ decode_heap_oop(tmp2, tmp2);
471 }
472 read_barrier_impl(masm, tmp1);
473 read_barrier_impl(masm, tmp2);
474 __ cmp(tmp1, tmp2);
475 // Retry with expected now being the value we just loaded from addr.
476 __ br(Assembler::EQ, retry);
477 if (is_cae && is_narrow) {
478 // For cmp-and-exchange and narrow oops, we need to restore
479 // the compressed old-value. We moved it to 'expected' a few lines up.
480 __ mov(tmp1, expected);
481 }
482 __ bind(done);
483
484 if (is_cae) {
485 __ mov(result, tmp1);
486 } else {
487 __ cset(result, Assembler::EQ);
488 }
489 }
490
491 #ifdef COMPILER1
492
493 #undef __
498 // At this point we know that marking is in progress.
499 // If do_load() is true then we have to emit the
500 // load of the previous value; otherwise it has already
501 // been loaded into _pre_val.
502
503 __ bind(*stub->entry());
504
505 assert(stub->pre_val()->is_register(), "Precondition.");
506
507 Register pre_val_reg = stub->pre_val()->as_register();
508
509 if (stub->do_load()) {
510 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/);
511 }
512 __ cbz(pre_val_reg, *stub->continuation());
513 ce->store_parameter(stub->pre_val()->as_register(), 0);
514 __ far_call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
515 __ b(*stub->continuation());
516 }
517
518 void ShenandoahBarrierSetAssembler::gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub) {
519
520 Register obj = stub->obj()->as_register();
521 Register res = stub->result()->as_register();
522
523 Label done;
524
525 __ bind(*stub->entry());
526
527 if (res != obj) {
528 __ mov(res, obj);
529 }
530 // Check for null.
531 if (stub->needs_null_check()) {
532 __ cbz(res, done);
533 }
534
535 write_barrier(ce->masm(), res);
536
537 __ bind(done);
538 __ b(*stub->continuation());
539 }
540
541 #undef __
542
543 #define __ sasm->
544
545 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
546 __ prologue("shenandoah_pre_barrier", false);
547
548 // arg0 : previous value of memory
549
550 BarrierSet* bs = BarrierSet::barrier_set();
551
552 const Register pre_val = r0;
553 const Register thread = rthread;
554 const Register tmp = rscratch1;
555
575 __ ldr(rscratch2, buffer);
576 __ add(tmp, tmp, rscratch2);
577 __ load_parameter(0, rscratch2);
578 __ str(rscratch2, Address(tmp, 0));
579 __ b(done);
580
581 __ bind(runtime);
582 __ push_call_clobbered_registers();
583 __ load_parameter(0, pre_val);
584 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
585 __ pop_call_clobbered_registers();
586 __ bind(done);
587
588 __ epilogue();
589 }
590
591 #undef __
592
593 #endif // COMPILER1
594
595 address ShenandoahBarrierSetAssembler::shenandoah_wb() {
596 assert(_shenandoah_wb != NULL, "need write barrier stub");
597 return _shenandoah_wb;
598 }
599
600 #define __ cgen->assembler()->
601
602 // Shenandoah write barrier.
603 //
604 // Input:
605 // r0: OOP to evacuate. Not null.
606 //
607 // Output:
608 // r0: Pointer to evacuated OOP.
609 //
610 // Trash rscratch1, rscratch2. Preserve everything else.
611 address ShenandoahBarrierSetAssembler::generate_shenandoah_wb(StubCodeGenerator* cgen) {
612
613 __ align(6);
614 StubCodeMark mark(cgen, "StubRoutines", "shenandoah_wb");
615 address start = __ pc();
616
617 Label work;
618 __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
619 __ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint());
620 __ ldrb(rscratch2, Address(rscratch2, rscratch1));
621 __ tbnz(rscratch2, 0, work);
622 __ ret(lr);
623 __ bind(work);
624
625 Register obj = r0;
626
627 __ enter(); // required for proper stackwalking of RuntimeStub frame
628
629 __ push_call_clobbered_registers();
630
631 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT));
632 __ blrt(lr, 1, 0, MacroAssembler::ret_type_integral);
633 __ mov(rscratch1, obj);
634 __ pop_call_clobbered_registers();
635 __ mov(obj, rscratch1);
636
637 __ leave(); // required for proper stackwalking of RuntimeStub frame
638 __ ret(lr);
639
640 return start;
641 }
642
643 #undef __
644
645 void ShenandoahBarrierSetAssembler::barrier_stubs_init() {
646 if (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier) {
647 int stub_code_size = 2048;
648 ResourceMark rm;
649 BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size);
650 CodeBuffer buf(bb);
651 StubCodeGenerator cgen(&buf);
652 _shenandoah_wb = generate_shenandoah_wb(&cgen);
653 }
654 }
|
23
24 #include "precompiled.hpp"
25 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
26 #include "gc/shenandoah/shenandoahHeap.hpp"
27 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
28 #include "gc/shenandoah/shenandoahHeuristics.hpp"
29 #include "gc/shenandoah/shenandoahRuntime.hpp"
30 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
31 #include "interpreter/interpreter.hpp"
32 #include "interpreter/interp_masm.hpp"
33 #include "runtime/sharedRuntime.hpp"
34 #include "runtime/thread.hpp"
35 #ifdef COMPILER1
36 #include "c1/c1_LIRAssembler.hpp"
37 #include "c1/c1_MacroAssembler.hpp"
38 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
39 #endif
40
41 #define __ masm->
42
43 address ShenandoahBarrierSetAssembler::_shenandoah_lrb = NULL;
44
45 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
46 Register addr, Register count, RegSet saved_regs) {
47 if (is_oop) {
48 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
49 if (ShenandoahSATBBarrier && !dest_uninitialized && !ShenandoahHeap::heap()->heuristics()->can_do_traversal_gc()) {
50
51 Label done;
52
53 // Avoid calling runtime if count == 0
54 __ cbz(count, done);
55
56 // Is marking active?
57 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
58 __ ldrb(rscratch1, gc_state);
59 __ tbz(rscratch1, ShenandoahHeap::MARKING_BITPOS, done);
60
61 __ push(saved_regs, sp);
62 if (count == c_rarg0) {
63 if (addr == c_rarg1) {
181 // If we care generating the pre-barrier without a frame (e.g. in the
182 // intrinsified Reference.get() routine) then ebp might be pointing to
183 // the caller frame and so this check will most likely fail at runtime.
184 //
185 // Expanding the call directly bypasses the generation of the check.
186 // So when we do not have have a full interpreter frame on the stack
187 // expand_call should be passed true.
188
189 if (expand_call) {
190 assert(pre_val != c_rarg1, "smashed arg");
191 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
192 } else {
193 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
194 }
195
196 __ pop(saved, sp);
197
198 __ bind(done);
199 }
200
201 void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst) {
202 assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled");
203 Label is_null;
204 __ cbz(dst, is_null);
205 resolve_forward_pointer_not_null(masm, dst);
206 __ bind(is_null);
207 }
208
209 void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst) {
210 assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled");
211 __ ldr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset()));
212 }
213
214 void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst, Register tmp) {
215 assert(ShenandoahLoadRefBarrier, "Should be enabled");
216 assert(dst != rscratch2, "need rscratch2");
217 if (tmp == noreg) {
218 assert(dst != rscratch1, "need rscratch1");
219 tmp = rscratch1;
220 }
221
222 Label done;
223
224 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
225 __ ldrb(tmp, gc_state);
226
227 // Check for heap stability
228 __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
229 __ tst(tmp, rscratch2);
230 __ br(Assembler::EQ, done);
231
232 // Heap is unstable, need to perform the resolve even if LRB is inactive
233 resolve_forward_pointer_not_null(masm, dst);
234
235 // Check for evacuation-in-progress and jump to LRB slow-path if needed
236 __ mov(rscratch2, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
237 __ tst(tmp, rscratch2);
238 __ br(Assembler::EQ, done);
239
240 RegSet to_save = RegSet::of(r0);
241 if (dst != r0) {
242 __ push(to_save, sp);
243 __ mov(r0, dst);
244 }
245
246 __ far_call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_lrb())));
247
248 if (dst != r0) {
249 __ mov(dst, r0);
250 __ pop(to_save, sp);
251 }
252
253 __ bind(done);
254 }
255
256 void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
257 if (ShenandoahStoreValEnqueueBarrier) {
258 // Save possibly live regs.
259 RegSet live_regs = RegSet::range(r0, r4) - dst;
260 __ push(live_regs, sp);
261 __ strd(v0, __ pre(sp, 2 * -wordSize));
262
263 satb_write_barrier_pre(masm, noreg, dst, rthread, tmp, true, false);
264
265 // Restore possibly live regs.
266 __ ldrd(v0, __ post(sp, 2 * wordSize));
267 __ pop(live_regs, sp);
268 }
269 }
270
271 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Register tmp) {
272 if (ShenandoahLoadRefBarrier) {
273 Label is_null;
274 __ cbz(dst, is_null);
275 load_reference_barrier_not_null(masm, dst, tmp);
276 __ bind(is_null);
277 }
278 }
279
280 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
281 Register dst, Address src, Register tmp1, Register tmp_thread) {
282 bool on_oop = type == T_OBJECT || type == T_ARRAY;
283 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
284 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
285 bool on_reference = on_weak || on_phantom;
286
287 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
288 if (on_oop) {
289 load_reference_barrier(masm, dst, tmp1);
290
291 if (ShenandoahKeepAliveBarrier && on_reference) {
292 __ enter();
293 satb_write_barrier_pre(masm /* masm */,
294 noreg /* obj */,
295 dst /* pre_val */,
296 rthread /* thread */,
297 tmp1 /* tmp */,
298 true /* tosca_live */,
299 true /* expand_call */);
300 __ leave();
301 }
302 }
303 }
304
305 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
306 Address dst, Register val, Register tmp1, Register tmp2) {
307 bool on_oop = type == T_OBJECT || type == T_ARRAY;
308 if (!on_oop) {
309 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
310 return;
311 }
312
313 // flatten object address if needed
314 if (dst.index() == noreg && dst.offset() == 0) {
315 if (dst.base() != r3) {
316 __ mov(r3, dst.base());
317 }
318 } else {
319 __ lea(r3, dst);
320 }
321
322 shenandoah_write_barrier_pre(masm,
323 r3 /* obj */,
324 tmp2 /* pre_val */,
325 rthread /* thread */,
326 tmp1 /* tmp */,
327 val != noreg /* tosca_live */,
328 false /* expand_call */);
329
330 if (val == noreg) {
331 BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), noreg, noreg, noreg);
332 } else {
333 storeval_barrier(masm, val, tmp1);
334 // G1 barrier needs uncompressed oop for region cross check.
335 Register new_val = val;
336 if (UseCompressedOops) {
337 new_val = rscratch2;
338 __ mov(new_val, val);
339 }
340 BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), val, noreg, noreg);
341 }
342
343 }
344
345 void ShenandoahBarrierSetAssembler::tlab_allocate(MacroAssembler* masm, Register obj,
346 Register var_size_in_bytes,
347 int con_size_in_bytes,
348 Register t1,
349 Register t2,
350 Label& slow_case) {
351
352 assert_different_registers(obj, t2);
353 assert_different_registers(obj, var_size_in_bytes);
354 Register end = t2;
355
356 __ ldr(obj, Address(rthread, JavaThread::tlab_top_offset()));
357 if (var_size_in_bytes == noreg) {
358 __ lea(end, Address(obj, (int) (con_size_in_bytes + ShenandoahBrooksPointer::byte_size())));
359 } else {
360 __ add(var_size_in_bytes, var_size_in_bytes, ShenandoahBrooksPointer::byte_size());
361 __ lea(end, Address(obj, var_size_in_bytes));
362 }
363 __ ldr(rscratch1, Address(rthread, JavaThread::tlab_end_offset()));
364 __ cmp(end, rscratch1);
365 __ br(Assembler::HI, slow_case);
366
367 // update the tlab top pointer
368 __ str(end, Address(rthread, JavaThread::tlab_top_offset()));
369
370 __ add(obj, obj, ShenandoahBrooksPointer::byte_size());
371 __ str(obj, Address(obj, ShenandoahBrooksPointer::byte_offset()));
372
373 // recover var_size_in_bytes if necessary
374 if (var_size_in_bytes == end) {
375 __ sub(var_size_in_bytes, var_size_in_bytes, obj);
376 }
377 }
378
379 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val,
380 bool acquire, bool release, bool weak, bool is_cae,
381 Register result) {
382 Register tmp1 = rscratch1;
383 Register tmp2 = rscratch2;
384 bool is_narrow = UseCompressedOops;
385 Assembler::operand_size size = is_narrow ? Assembler::word : Assembler::xword;
386
387 assert_different_registers(addr, expected, new_val, tmp1, tmp2);
388
389 Label retry, done, fail;
390
391 // CAS, using LL/SC pair.
392 __ bind(retry);
393 __ load_exclusive(tmp1, addr, size, acquire);
394 if (is_narrow) {
395 __ cmpw(tmp1, expected);
396 } else {
397 __ cmp(tmp1, expected);
398 }
399 __ br(Assembler::NE, fail);
400 __ store_exclusive(tmp2, new_val, addr, size, release);
401 if (weak) {
402 __ cmpw(tmp2, 0u); // If the store fails, return NE to our caller
403 } else {
404 __ cbnzw(tmp2, retry);
405 }
406 __ b(done);
407
408 __ bind(fail);
409 // Check if rb(expected)==rb(tmp1)
410 // Shuffle registers so that we have memory value ready for next expected.
411 __ mov(tmp2, expected);
412 __ mov(expected, tmp1);
413 if (is_narrow) {
414 __ decode_heap_oop(tmp1, tmp1);
415 __ decode_heap_oop(tmp2, tmp2);
416 }
417 resolve_forward_pointer(masm, tmp1);
418 resolve_forward_pointer(masm, tmp2);
419 __ cmp(tmp1, tmp2);
420 // Retry with expected now being the value we just loaded from addr.
421 __ br(Assembler::EQ, retry);
422 if (is_cae && is_narrow) {
423 // For cmp-and-exchange and narrow oops, we need to restore
424 // the compressed old-value. We moved it to 'expected' a few lines up.
425 __ mov(tmp1, expected);
426 }
427 __ bind(done);
428
429 if (is_cae) {
430 __ mov(result, tmp1);
431 } else {
432 __ cset(result, Assembler::EQ);
433 }
434 }
435
436 #ifdef COMPILER1
437
438 #undef __
443 // At this point we know that marking is in progress.
444 // If do_load() is true then we have to emit the
445 // load of the previous value; otherwise it has already
446 // been loaded into _pre_val.
447
448 __ bind(*stub->entry());
449
450 assert(stub->pre_val()->is_register(), "Precondition.");
451
452 Register pre_val_reg = stub->pre_val()->as_register();
453
454 if (stub->do_load()) {
455 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/);
456 }
457 __ cbz(pre_val_reg, *stub->continuation());
458 ce->store_parameter(stub->pre_val()->as_register(), 0);
459 __ far_call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
460 __ b(*stub->continuation());
461 }
462
463 void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
464
465 Register obj = stub->obj()->as_register();
466 Register res = stub->result()->as_register();
467
468 Label done;
469
470 __ bind(*stub->entry());
471
472 if (res != obj) {
473 __ mov(res, obj);
474 }
475 // Check for null.
476 if (stub->needs_null_check()) {
477 __ cbz(res, done);
478 }
479
480 load_reference_barrier_not_null(ce->masm(), res, rscratch1);
481
482 __ bind(done);
483 __ b(*stub->continuation());
484 }
485
486 #undef __
487
488 #define __ sasm->
489
490 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
491 __ prologue("shenandoah_pre_barrier", false);
492
493 // arg0 : previous value of memory
494
495 BarrierSet* bs = BarrierSet::barrier_set();
496
497 const Register pre_val = r0;
498 const Register thread = rthread;
499 const Register tmp = rscratch1;
500
520 __ ldr(rscratch2, buffer);
521 __ add(tmp, tmp, rscratch2);
522 __ load_parameter(0, rscratch2);
523 __ str(rscratch2, Address(tmp, 0));
524 __ b(done);
525
526 __ bind(runtime);
527 __ push_call_clobbered_registers();
528 __ load_parameter(0, pre_val);
529 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
530 __ pop_call_clobbered_registers();
531 __ bind(done);
532
533 __ epilogue();
534 }
535
536 #undef __
537
538 #endif // COMPILER1
539
540 address ShenandoahBarrierSetAssembler::shenandoah_lrb() {
541 assert(_shenandoah_lrb != NULL, "need load reference barrier stub");
542 return _shenandoah_lrb;
543 }
544
545 #define __ cgen->assembler()->
546
547 // Shenandoah load reference barrier.
548 //
549 // Input:
550 // r0: OOP to evacuate. Not null.
551 //
552 // Output:
553 // r0: Pointer to evacuated OOP.
554 //
555 // Trash rscratch1, rscratch2. Preserve everything else.
556 address ShenandoahBarrierSetAssembler::generate_shenandoah_lrb(StubCodeGenerator* cgen) {
557
558 __ align(6);
559 StubCodeMark mark(cgen, "StubRoutines", "shenandoah_lrb");
560 address start = __ pc();
561
562 Label work;
563 __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
564 __ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint());
565 __ ldrb(rscratch2, Address(rscratch2, rscratch1));
566 __ tbnz(rscratch2, 0, work);
567 __ ret(lr);
568 __ bind(work);
569
570 Register obj = r0;
571
572 __ enter(); // required for proper stackwalking of RuntimeStub frame
573
574 __ push_call_clobbered_registers();
575
576 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT));
577 __ blrt(lr, 1, 0, MacroAssembler::ret_type_integral);
578 __ mov(rscratch1, obj);
579 __ pop_call_clobbered_registers();
580 __ mov(obj, rscratch1);
581
582 __ leave(); // required for proper stackwalking of RuntimeStub frame
583 __ ret(lr);
584
585 return start;
586 }
587
588 #undef __
589
590 void ShenandoahBarrierSetAssembler::barrier_stubs_init() {
591 if (ShenandoahLoadRefBarrier) {
592 int stub_code_size = 2048;
593 ResourceMark rm;
594 BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size);
595 CodeBuffer buf(bb);
596 StubCodeGenerator cgen(&buf);
597 _shenandoah_lrb = generate_shenandoah_lrb(&cgen);
598 }
599 }
|