5411 lea(end, Address(obj, var_size_in_bytes, Address::times_1));
5412 }
5413 cmpptr(end, Address(thread, JavaThread::tlab_end_offset()));
5414 jcc(Assembler::above, slow_case);
5415
5416 // update the tlab top pointer
5417 movptr(Address(thread, JavaThread::tlab_top_offset()), end);
5418
5419 // recover var_size_in_bytes if necessary
5420 if (var_size_in_bytes == end) {
5421 subptr(var_size_in_bytes, obj);
5422 }
5423 verify_tlab();
5424 }
5425
5426 // Preserves rbx, and rdx.
5427 Register MacroAssembler::tlab_refill(Label& retry,
5428 Label& try_eden,
5429 Label& slow_case) {
5430 Register top = rax;
5431 Register t1 = rcx;
5432 Register t2 = rsi;
5433 Register thread_reg = NOT_LP64(rdi) LP64_ONLY(r15_thread);
5434 assert_different_registers(top, thread_reg, t1, t2, /* preserve: */ rbx, rdx);
5435 Label do_refill, discard_tlab;
5436
5437 if (!Universe::heap()->supports_inline_contig_alloc()) {
5438 // No allocation in the shared eden.
5439 jmp(slow_case);
5440 }
5441
5442 NOT_LP64(get_thread(thread_reg));
5443
5444 movptr(top, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
5445 movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())));
5446
5447 // calculate amount of free space
5448 subptr(t1, top);
5449 shrptr(t1, LogHeapWordSize);
5450
5451 // Retain tlab and allocate object in shared space if
5507 Label ok;
5508 Register tsize = rsi;
5509 assert_different_registers(tsize, thread_reg, t1);
5510 push(tsize);
5511 movptr(tsize, Address(thread_reg, in_bytes(JavaThread::tlab_size_offset())));
5512 shlptr(tsize, LogHeapWordSize);
5513 cmpptr(t1, tsize);
5514 jcc(Assembler::equal, ok);
5515 STOP("assert(t1 != tlab size)");
5516 should_not_reach_here();
5517
5518 bind(ok);
5519 pop(tsize);
5520 }
5521 #endif
5522 movptr(Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())), top);
5523 movptr(Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())), top);
5524 addptr(top, t1);
5525 subptr(top, (int32_t)ThreadLocalAllocBuffer::alignment_reserve_in_bytes());
5526 movptr(Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())), top);
5527 verify_tlab();
5528 jmp(retry);
5529
5530 return thread_reg; // for use by caller
5531 }
5532
5533 void MacroAssembler::incr_allocated_bytes(Register thread,
5534 Register var_size_in_bytes,
5535 int con_size_in_bytes,
5536 Register t1) {
5537 if (!thread->is_valid()) {
5538 #ifdef _LP64
5539 thread = r15_thread;
5540 #else
5541 assert(t1->is_valid(), "need temp reg");
5542 thread = t1;
5543 get_thread(thread);
5544 #endif
5545 }
5546
5547 #ifdef _LP64
5548 if (var_size_in_bytes->is_valid()) {
5549 addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes);
5550 } else {
|
5411 lea(end, Address(obj, var_size_in_bytes, Address::times_1));
5412 }
5413 cmpptr(end, Address(thread, JavaThread::tlab_end_offset()));
5414 jcc(Assembler::above, slow_case);
5415
5416 // update the tlab top pointer
5417 movptr(Address(thread, JavaThread::tlab_top_offset()), end);
5418
5419 // recover var_size_in_bytes if necessary
5420 if (var_size_in_bytes == end) {
5421 subptr(var_size_in_bytes, obj);
5422 }
5423 verify_tlab();
5424 }
5425
5426 // Preserves rbx, and rdx.
5427 Register MacroAssembler::tlab_refill(Label& retry,
5428 Label& try_eden,
5429 Label& slow_case) {
5430 Register top = rax;
5431 Register t1 = rcx; // object size
5432 Register t2 = rsi;
5433 Register thread_reg = NOT_LP64(rdi) LP64_ONLY(r15_thread);
5434 assert_different_registers(top, thread_reg, t1, t2, /* preserve: */ rbx, rdx);
5435 Label do_refill, discard_tlab;
5436
5437 if (!Universe::heap()->supports_inline_contig_alloc()) {
5438 // No allocation in the shared eden.
5439 jmp(slow_case);
5440 }
5441
5442 NOT_LP64(get_thread(thread_reg));
5443
5444 movptr(top, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
5445 movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())));
5446
5447 // calculate amount of free space
5448 subptr(t1, top);
5449 shrptr(t1, LogHeapWordSize);
5450
5451 // Retain tlab and allocate object in shared space if
5507 Label ok;
5508 Register tsize = rsi;
5509 assert_different_registers(tsize, thread_reg, t1);
5510 push(tsize);
5511 movptr(tsize, Address(thread_reg, in_bytes(JavaThread::tlab_size_offset())));
5512 shlptr(tsize, LogHeapWordSize);
5513 cmpptr(t1, tsize);
5514 jcc(Assembler::equal, ok);
5515 STOP("assert(t1 != tlab size)");
5516 should_not_reach_here();
5517
5518 bind(ok);
5519 pop(tsize);
5520 }
5521 #endif
5522 movptr(Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())), top);
5523 movptr(Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())), top);
5524 addptr(top, t1);
5525 subptr(top, (int32_t)ThreadLocalAllocBuffer::alignment_reserve_in_bytes());
5526 movptr(Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())), top);
5527
5528 if (ZeroTLAB) {
5529 // This is a fast TLAB refill, therefore the GC is not notified of it.
5530 // So compiled code must fill the new TLAB with zeroes.
5531 movptr(top, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())));
5532 zero_memory(top, t1, 0, t2);
5533 }
5534
5535 verify_tlab();
5536 jmp(retry);
5537
5538 return thread_reg; // for use by caller
5539 }
5540
5541 // Preserves the contents of address, destroys the contents length_in_bytes and temp.
5542 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) {
5543 assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different");
5544 assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
5545 Label done;
5546
5547 testptr(length_in_bytes, length_in_bytes);
5548 jcc(Assembler::zero, done);
5549
5550 // initialize topmost word, divide index by 2, check if odd and test if zero
5551 // note: for the remaining code to work, index must be a multiple of BytesPerWord
5552 #ifdef ASSERT
5553 {
5554 Label L;
5555 testptr(length_in_bytes, BytesPerWord - 1);
5556 jcc(Assembler::zero, L);
5557 stop("length must be a multiple of BytesPerWord");
5558 bind(L);
5559 }
5560 #endif
5561 Register index = length_in_bytes;
5562 xorptr(temp, temp); // use _zero reg to clear memory (shorter code)
5563 if (UseIncDec) {
5564 shrptr(index, 3); // divide by 8/16 and set carry flag if bit 2 was set
5565 } else {
5566 shrptr(index, 2); // use 2 instructions to avoid partial flag stall
5567 shrptr(index, 1);
5568 }
5569 #ifndef _LP64
5570 // index could have not been a multiple of 8 (i.e., bit 2 was set)
5571 {
5572 Label even;
5573 // note: if index was a multiple of 8, then it cannot
5574 // be 0 now otherwise it must have been 0 before
5575 // => if it is even, we don't need to check for 0 again
5576 jcc(Assembler::carryClear, even);
5577 // clear topmost word (no jump would be needed if conditional assignment worked here)
5578 movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp);
5579 // index could be 0 now, must check again
5580 jcc(Assembler::zero, done);
5581 bind(even);
5582 }
5583 #endif // !_LP64
5584 // initialize remaining object fields: index is a multiple of 2 now
5585 {
5586 Label loop;
5587 bind(loop);
5588 movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp);
5589 NOT_LP64(movptr(Address(address, index, Address::times_8, offset_in_bytes - 2*BytesPerWord), temp);)
5590 decrement(index);
5591 jcc(Assembler::notZero, loop);
5592 }
5593
5594 bind(done);
5595 }
5596
5597 void MacroAssembler::incr_allocated_bytes(Register thread,
5598 Register var_size_in_bytes,
5599 int con_size_in_bytes,
5600 Register t1) {
5601 if (!thread->is_valid()) {
5602 #ifdef _LP64
5603 thread = r15_thread;
5604 #else
5605 assert(t1->is_valid(), "need temp reg");
5606 thread = t1;
5607 get_thread(thread);
5608 #endif
5609 }
5610
5611 #ifdef _LP64
5612 if (var_size_in_bytes->is_valid()) {
5613 addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes);
5614 } else {
|