5568 Register var_size_in_bytes,
5569 int con_size_in_bytes,
5570 Register t1,
5571 Register t2,
5572 Label& slow_case) {
5573 assert_different_registers(obj, t1, t2);
5574 assert_different_registers(obj, var_size_in_bytes, t1);
5575 Register end = t2;
5576 Register thread = NOT_LP64(t1) LP64_ONLY(r15_thread);
5577
5578 verify_tlab();
5579
5580 NOT_LP64(get_thread(thread));
5581
5582 movptr(obj, Address(thread, JavaThread::tlab_top_offset()));
5583 if (var_size_in_bytes == noreg) {
5584 lea(end, Address(obj, con_size_in_bytes));
5585 } else {
5586 lea(end, Address(obj, var_size_in_bytes, Address::times_1));
5587 }
5588 cmpptr(end, Address(thread, JavaThread::tlab_end_offset()));
5589 jcc(Assembler::above, slow_case);
5590
5591 // update the tlab top pointer
5592 movptr(Address(thread, JavaThread::tlab_top_offset()), end);
5593
5594 // recover var_size_in_bytes if necessary
5595 if (var_size_in_bytes == end) {
5596 subptr(var_size_in_bytes, obj);
5597 }
5598 verify_tlab();
5599 }
5600
5601 // Preserves rbx, and rdx.
5602 Register MacroAssembler::tlab_refill(Label& retry,
5603 Label& try_eden,
5604 Label& slow_case) {
5605 Register top = rax;
5606 Register t1 = rcx; // object size
5607 Register t2 = rsi;
5608 Register thread_reg = NOT_LP64(rdi) LP64_ONLY(r15_thread);
5609 assert_different_registers(top, thread_reg, t1, t2, /* preserve: */ rbx, rdx);
5610 Label do_refill, discard_tlab;
5611
5612 if (!Universe::heap()->supports_inline_contig_alloc()) {
5613 // No allocation in the shared eden.
5614 jmp(slow_case);
5615 }
5616
5617 NOT_LP64(get_thread(thread_reg));
5618
5619 movptr(top, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
5620 movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())));
5621
5622 // calculate amount of free space
5623 subptr(t1, top);
5624 shrptr(t1, LogHeapWordSize);
5625
5626 // Retain tlab and allocate object in shared space if
5627 // the amount free in the tlab is too large to discard.
5628 cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_refill_waste_limit_offset())));
5629 jcc(Assembler::lessEqual, discard_tlab);
5630
5631 // Retain
5632 // %%% yuck as movptr...
5633 movptr(t2, (int32_t) ThreadLocalAllocBuffer::refill_waste_limit_increment());
5634 addptr(Address(thread_reg, in_bytes(JavaThread::tlab_refill_waste_limit_offset())), t2);
5635 if (TLABStats) {
5636 // increment number of slow_allocations
5637 addl(Address(thread_reg, in_bytes(JavaThread::tlab_slow_allocations_offset())), 1);
5638 }
5639 jmp(try_eden);
5640
5681 if (UseTLAB) {
5682 Label ok;
5683 Register tsize = rsi;
5684 assert_different_registers(tsize, thread_reg, t1);
5685 push(tsize);
5686 movptr(tsize, Address(thread_reg, in_bytes(JavaThread::tlab_size_offset())));
5687 shlptr(tsize, LogHeapWordSize);
5688 cmpptr(t1, tsize);
5689 jcc(Assembler::equal, ok);
5690 STOP("assert(t1 != tlab size)");
5691 should_not_reach_here();
5692
5693 bind(ok);
5694 pop(tsize);
5695 }
5696 #endif
5697 movptr(Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())), top);
5698 movptr(Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())), top);
5699 addptr(top, t1);
5700 subptr(top, (int32_t)ThreadLocalAllocBuffer::alignment_reserve_in_bytes());
5701 movptr(Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())), top);
5702
5703 if (ZeroTLAB) {
5704 // This is a fast TLAB refill, therefore the GC is not notified of it.
5705 // So compiled code must fill the new TLAB with zeroes.
5706 movptr(top, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())));
5707 zero_memory(top, t1, 0, t2);
5708 }
5709
5710 verify_tlab();
5711 jmp(retry);
5712
5713 return thread_reg; // for use by caller
5714 }
5715
5716 // Preserves the contents of address, destroys the contents length_in_bytes and temp.
5717 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) {
5718 assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different");
5719 assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
5720 Label done;
5721
6242 }
6243
6244 void MacroAssembler::verify_tlab() {
6245 #ifdef ASSERT
6246 if (UseTLAB && VerifyOops) {
6247 Label next, ok;
6248 Register t1 = rsi;
6249 Register thread_reg = NOT_LP64(rbx) LP64_ONLY(r15_thread);
6250
6251 push(t1);
6252 NOT_LP64(push(thread_reg));
6253 NOT_LP64(get_thread(thread_reg));
6254
6255 movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
6256 cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())));
6257 jcc(Assembler::aboveEqual, next);
6258 STOP("assert(top >= start)");
6259 should_not_reach_here();
6260
6261 bind(next);
6262 movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())));
6263 cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
6264 jcc(Assembler::aboveEqual, ok);
6265 STOP("assert(top <= end)");
6266 should_not_reach_here();
6267
6268 bind(ok);
6269 NOT_LP64(pop(thread_reg));
6270 pop(t1);
6271 }
6272 #endif
6273 }
6274
6275 class ControlWord {
6276 public:
6277 int32_t _value;
6278
6279 int rounding_control() const { return (_value >> 10) & 3 ; }
6280 int precision_control() const { return (_value >> 8) & 3 ; }
6281 bool precision() const { return ((_value >> 5) & 1) != 0; }
6282 bool underflow() const { return ((_value >> 4) & 1) != 0; }
|
5568 Register var_size_in_bytes,
5569 int con_size_in_bytes,
5570 Register t1,
5571 Register t2,
5572 Label& slow_case) {
5573 assert_different_registers(obj, t1, t2);
5574 assert_different_registers(obj, var_size_in_bytes, t1);
5575 Register end = t2;
5576 Register thread = NOT_LP64(t1) LP64_ONLY(r15_thread);
5577
5578 verify_tlab();
5579
5580 NOT_LP64(get_thread(thread));
5581
5582 movptr(obj, Address(thread, JavaThread::tlab_top_offset()));
5583 if (var_size_in_bytes == noreg) {
5584 lea(end, Address(obj, con_size_in_bytes));
5585 } else {
5586 lea(end, Address(obj, var_size_in_bytes, Address::times_1));
5587 }
5588 cmpptr(end, Address(thread, JavaThread::tlab_current_end_offset()));
5589 jcc(Assembler::above, slow_case);
5590
5591 // update the tlab top pointer
5592 movptr(Address(thread, JavaThread::tlab_top_offset()), end);
5593
5594 // recover var_size_in_bytes if necessary
5595 if (var_size_in_bytes == end) {
5596 subptr(var_size_in_bytes, obj);
5597 }
5598 verify_tlab();
5599 }
5600
5601 // Preserves rbx, and rdx.
5602 Register MacroAssembler::tlab_refill(Label& retry,
5603 Label& try_eden,
5604 Label& slow_case) {
5605 Register top = rax;
5606 Register t1 = rcx; // object size
5607 Register t2 = rsi;
5608 Register thread_reg = NOT_LP64(rdi) LP64_ONLY(r15_thread);
5609 assert_different_registers(top, thread_reg, t1, t2, /* preserve: */ rbx, rdx);
5610 Label do_refill, discard_tlab;
5611
5612 if (!Universe::heap()->supports_inline_contig_alloc()) {
5613 // No allocation in the shared eden.
5614 jmp(slow_case);
5615 }
5616
5617 NOT_LP64(get_thread(thread_reg));
5618
5619 movptr(top, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
5620 movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_current_end_offset())));
5621
5622 // calculate amount of free space
5623 subptr(t1, top);
5624 shrptr(t1, LogHeapWordSize);
5625
5626 // Retain tlab and allocate object in shared space if
5627 // the amount free in the tlab is too large to discard.
5628 cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_refill_waste_limit_offset())));
5629 jcc(Assembler::lessEqual, discard_tlab);
5630
5631 // Retain
5632 // %%% yuck as movptr...
5633 movptr(t2, (int32_t) ThreadLocalAllocBuffer::refill_waste_limit_increment());
5634 addptr(Address(thread_reg, in_bytes(JavaThread::tlab_refill_waste_limit_offset())), t2);
5635 if (TLABStats) {
5636 // increment number of slow_allocations
5637 addl(Address(thread_reg, in_bytes(JavaThread::tlab_slow_allocations_offset())), 1);
5638 }
5639 jmp(try_eden);
5640
5681 if (UseTLAB) {
5682 Label ok;
5683 Register tsize = rsi;
5684 assert_different_registers(tsize, thread_reg, t1);
5685 push(tsize);
5686 movptr(tsize, Address(thread_reg, in_bytes(JavaThread::tlab_size_offset())));
5687 shlptr(tsize, LogHeapWordSize);
5688 cmpptr(t1, tsize);
5689 jcc(Assembler::equal, ok);
5690 STOP("assert(t1 != tlab size)");
5691 should_not_reach_here();
5692
5693 bind(ok);
5694 pop(tsize);
5695 }
5696 #endif
5697 movptr(Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())), top);
5698 movptr(Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())), top);
5699 addptr(top, t1);
5700 subptr(top, (int32_t)ThreadLocalAllocBuffer::alignment_reserve_in_bytes());
5701 movptr(Address(thread_reg, in_bytes(JavaThread::tlab_current_end_offset())), top);
5702
5703 if (ZeroTLAB) {
5704 // This is a fast TLAB refill, therefore the GC is not notified of it.
5705 // So compiled code must fill the new TLAB with zeroes.
5706 movptr(top, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())));
5707 zero_memory(top, t1, 0, t2);
5708 }
5709
5710 verify_tlab();
5711 jmp(retry);
5712
5713 return thread_reg; // for use by caller
5714 }
5715
5716 // Preserves the contents of address, destroys the contents length_in_bytes and temp.
5717 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) {
5718 assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different");
5719 assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
5720 Label done;
5721
6242 }
6243
6244 void MacroAssembler::verify_tlab() {
6245 #ifdef ASSERT
6246 if (UseTLAB && VerifyOops) {
6247 Label next, ok;
6248 Register t1 = rsi;
6249 Register thread_reg = NOT_LP64(rbx) LP64_ONLY(r15_thread);
6250
6251 push(t1);
6252 NOT_LP64(push(thread_reg));
6253 NOT_LP64(get_thread(thread_reg));
6254
6255 movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
6256 cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())));
6257 jcc(Assembler::aboveEqual, next);
6258 STOP("assert(top >= start)");
6259 should_not_reach_here();
6260
6261 bind(next);
6262 movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_current_end_offset())));
6263 cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
6264 jcc(Assembler::aboveEqual, ok);
6265 STOP("assert(top <= end)");
6266 should_not_reach_here();
6267
6268 bind(ok);
6269 NOT_LP64(pop(thread_reg));
6270 pop(t1);
6271 }
6272 #endif
6273 }
6274
6275 class ControlWord {
6276 public:
6277 int32_t _value;
6278
6279 int rounding_control() const { return (_value >> 10) & 3 ; }
6280 int precision_control() const { return (_value >> 8) & 3 ; }
6281 bool precision() const { return ((_value >> 5) & 1) != 0; }
6282 bool underflow() const { return ((_value >> 4) & 1) != 0; }
|