144 __ mov(c_rarg0, dst);
145 __ mov(c_rarg1, count);
146 }
147 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::write_ref_array_post_entry), 2);
148 #else
149 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::write_ref_array_post_entry),
150 dst, count);
151 #endif
152 __ popa();
153 }
154 }
155
156 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
157 Register obj,
158 Register pre_val,
159 Register thread,
160 Register tmp,
161 bool tosca_live,
162 bool expand_call) {
163
164 if (ShenandoahConditionalSATBBarrier) {
165 Label done;
166 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
167 __ testb(gc_state, ShenandoahHeap::MARKING);
168 __ jcc(Assembler::zero, done); // Skip SATB barrier when conc-mark is not active
169 satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
170 __ bind(done);
171 }
172 if (ShenandoahSATBBarrier) {
173 satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
174 }
175 }
176
177 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
178 Register obj,
179 Register pre_val,
180 Register thread,
181 Register tmp,
182 bool tosca_live,
183 bool expand_call) {
184 // If expand_call is true then we expand the call_VM_leaf macro
185 // directly to skip generating the check by
186 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
187
188 #ifdef _LP64
189 assert(thread == r15_thread, "must be");
190 #endif // _LP64
191
345
346 void ShenandoahBarrierSetAssembler::read_barrier_not_null(MacroAssembler* masm, Register dst) {
347 if (ShenandoahReadBarrier) {
348 read_barrier_not_null_impl(masm, dst);
349 }
350 }
351
352 void ShenandoahBarrierSetAssembler::read_barrier_not_null_impl(MacroAssembler* masm, Register dst) {
353 assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier), "should be enabled");
354 __ movptr(dst, Address(dst, BrooksPointer::byte_offset()));
355 }
356
357
358 void ShenandoahBarrierSetAssembler::write_barrier(MacroAssembler* masm, Register dst) {
359 if (ShenandoahWriteBarrier) {
360 write_barrier_impl(masm, dst);
361 }
362 }
363
364 void ShenandoahBarrierSetAssembler::write_barrier_impl(MacroAssembler* masm, Register dst) {
365 assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValWriteBarrier || ShenandoahStoreValEnqueueBarrier), "should be enabled");
366 #ifdef _LP64
367 assert(dst != rscratch1, "different regs");
368
369 Label done;
370
371 Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
372 __ testb(gc_state, ShenandoahHeap::EVACUATION | ShenandoahHeap::PARTIAL | ShenandoahHeap::TRAVERSAL);
373
374 // Now check if evacuation is in progress.
375 read_barrier_not_null(masm, dst);
376
377 __ jcc(Assembler::zero, done);
378 __ push(rscratch1);
379 __ push(rscratch2);
380
381 __ movptr(rscratch1, dst);
382 __ shrptr(rscratch1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
383 __ movptr(rscratch2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
384 __ movbool(rscratch2, Address(rscratch2, rscratch1, Address::times_1));
385 __ testb(rscratch2, 0x1);
386
387 __ pop(rscratch2);
388 __ pop(rscratch1);
389
390 __ jcc(Assembler::zero, done);
391
392 __ push(rscratch1);
430 }
431 if (dst != rbx) {
432 __ pop(rbx);
433 }
434 if (dst != rax) {
435 __ pop(rax);
436 }
437
438 // Move result into dst reg.
439 __ mov(dst, rscratch1);
440
441 __ pop(rscratch1);
442
443 __ bind(done);
444 #else
445 Unimplemented();
446 #endif
447 }
448
449 void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
450 if (ShenandoahStoreValReadBarrier || ShenandoahStoreValWriteBarrier || ShenandoahStoreValEnqueueBarrier) {
451 storeval_barrier_impl(masm, dst, tmp);
452 }
453 }
454
455 void ShenandoahBarrierSetAssembler::storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp) {
456 assert(UseShenandoahGC && (ShenandoahStoreValReadBarrier || ShenandoahStoreValWriteBarrier || ShenandoahStoreValEnqueueBarrier), "should be enabled");
457
458 if (dst == noreg) return;
459
460 #ifdef _LP64
461 if (ShenandoahStoreValWriteBarrier || ShenandoahStoreValEnqueueBarrier) {
462 Label is_null;
463 __ testptr(dst, dst);
464 __ jcc(Assembler::zero, is_null);
465 write_barrier_impl(masm, dst);
466 __ bind(is_null);
467 }
468
469 if (ShenandoahStoreValEnqueueBarrier) {
470 // The set of registers to be saved+restored is the same as in the write-barrier above.
471 // Those are the commonly used registers in the interpreter.
472 __ push(rbx);
473 __ push(rcx);
474 __ push(rdx);
475 __ push(c_rarg1);
476 __ subptr(rsp, 2 * Interpreter::stackElementSize);
477 __ movdbl(Address(rsp, 0), xmm0);
478
479 satb_write_barrier_pre(masm, noreg, dst, r15_thread, tmp, true, false);
480 __ movdbl(xmm0, Address(rsp, 0));
481 __ addptr(rsp, 2 * Interpreter::stackElementSize);
482 __ pop(c_rarg1);
483 __ pop(rdx);
484 __ pop(rcx);
485 __ pop(rbx);
486 }
487 if (ShenandoahStoreValReadBarrier) {
488 read_barrier_impl(masm, dst);
489 }
|
144 __ mov(c_rarg0, dst);
145 __ mov(c_rarg1, count);
146 }
147 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::write_ref_array_post_entry), 2);
148 #else
149 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::write_ref_array_post_entry),
150 dst, count);
151 #endif
152 __ popa();
153 }
154 }
155
156 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
157 Register obj,
158 Register pre_val,
159 Register thread,
160 Register tmp,
161 bool tosca_live,
162 bool expand_call) {
163
164 if (ShenandoahSATBBarrier) {
165 satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
166 }
167 }
168
169 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
170 Register obj,
171 Register pre_val,
172 Register thread,
173 Register tmp,
174 bool tosca_live,
175 bool expand_call) {
176 // If expand_call is true then we expand the call_VM_leaf macro
177 // directly to skip generating the check by
178 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
179
180 #ifdef _LP64
181 assert(thread == r15_thread, "must be");
182 #endif // _LP64
183
337
338 void ShenandoahBarrierSetAssembler::read_barrier_not_null(MacroAssembler* masm, Register dst) {
339 if (ShenandoahReadBarrier) {
340 read_barrier_not_null_impl(masm, dst);
341 }
342 }
343
344 void ShenandoahBarrierSetAssembler::read_barrier_not_null_impl(MacroAssembler* masm, Register dst) {
345 assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier), "should be enabled");
346 __ movptr(dst, Address(dst, BrooksPointer::byte_offset()));
347 }
348
349
350 void ShenandoahBarrierSetAssembler::write_barrier(MacroAssembler* masm, Register dst) {
351 if (ShenandoahWriteBarrier) {
352 write_barrier_impl(masm, dst);
353 }
354 }
355
356 void ShenandoahBarrierSetAssembler::write_barrier_impl(MacroAssembler* masm, Register dst) {
357 assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "should be enabled");
358 #ifdef _LP64
359 assert(dst != rscratch1, "different regs");
360
361 Label done;
362
363 Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
364 __ testb(gc_state, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
365
366 // Now check if evacuation is in progress.
367 read_barrier_not_null(masm, dst);
368
369 __ jcc(Assembler::zero, done);
370 __ push(rscratch1);
371 __ push(rscratch2);
372
373 __ movptr(rscratch1, dst);
374 __ shrptr(rscratch1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
375 __ movptr(rscratch2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
376 __ movbool(rscratch2, Address(rscratch2, rscratch1, Address::times_1));
377 __ testb(rscratch2, 0x1);
378
379 __ pop(rscratch2);
380 __ pop(rscratch1);
381
382 __ jcc(Assembler::zero, done);
383
384 __ push(rscratch1);
422 }
423 if (dst != rbx) {
424 __ pop(rbx);
425 }
426 if (dst != rax) {
427 __ pop(rax);
428 }
429
430 // Move result into dst reg.
431 __ mov(dst, rscratch1);
432
433 __ pop(rscratch1);
434
435 __ bind(done);
436 #else
437 Unimplemented();
438 #endif
439 }
440
441 void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
442 if (ShenandoahStoreValReadBarrier || ShenandoahStoreValEnqueueBarrier) {
443 storeval_barrier_impl(masm, dst, tmp);
444 }
445 }
446
447 void ShenandoahBarrierSetAssembler::storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp) {
448 assert(UseShenandoahGC && (ShenandoahStoreValReadBarrier || ShenandoahStoreValEnqueueBarrier), "should be enabled");
449
450 if (dst == noreg) return;
451
452 #ifdef _LP64
453 if (ShenandoahStoreValEnqueueBarrier) {
454 Label is_null;
455 __ testptr(dst, dst);
456 __ jcc(Assembler::zero, is_null);
457 write_barrier_impl(masm, dst);
458 __ bind(is_null);
459
460 // The set of registers to be saved+restored is the same as in the write-barrier above.
461 // Those are the commonly used registers in the interpreter.
462 __ push(rbx);
463 __ push(rcx);
464 __ push(rdx);
465 __ push(c_rarg1);
466 __ subptr(rsp, 2 * Interpreter::stackElementSize);
467 __ movdbl(Address(rsp, 0), xmm0);
468
469 satb_write_barrier_pre(masm, noreg, dst, r15_thread, tmp, true, false);
470 __ movdbl(xmm0, Address(rsp, 0));
471 __ addptr(rsp, 2 * Interpreter::stackElementSize);
472 __ pop(c_rarg1);
473 __ pop(rdx);
474 __ pop(rcx);
475 __ pop(rbx);
476 }
477 if (ShenandoahStoreValReadBarrier) {
478 read_barrier_impl(masm, dst);
479 }
|