5402 lea(end, Address(obj, var_size_in_bytes, Address::times_1));
5403 }
5404 cmpptr(end, Address(thread, JavaThread::tlab_end_offset()));
5405 jcc(Assembler::above, slow_case);
5406
5407 // update the tlab top pointer
5408 movptr(Address(thread, JavaThread::tlab_top_offset()), end);
5409
5410 // recover var_size_in_bytes if necessary
5411 if (var_size_in_bytes == end) {
5412 subptr(var_size_in_bytes, obj);
5413 }
5414 verify_tlab();
5415 }
5416
5417 // Preserves rbx, and rdx.
5418 Register MacroAssembler::tlab_refill(Label& retry,
5419 Label& try_eden,
5420 Label& slow_case) {
5421 Register top = rax;
5422 Register t1 = rcx;
5423 Register t2 = rsi;
5424 Register thread_reg = NOT_LP64(rdi) LP64_ONLY(r15_thread);
5425 assert_different_registers(top, thread_reg, t1, t2, /* preserve: */ rbx, rdx);
5426 Label do_refill, discard_tlab;
5427
5428 if (!Universe::heap()->supports_inline_contig_alloc()) {
5429 // No allocation in the shared eden.
5430 jmp(slow_case);
5431 }
5432
5433 NOT_LP64(get_thread(thread_reg));
5434
5435 movptr(top, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
5436 movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())));
5437
5438 // calculate amount of free space
5439 subptr(t1, top);
5440 shrptr(t1, LogHeapWordSize);
5441
5442 // Retain tlab and allocate object in shared space if
5498 Label ok;
5499 Register tsize = rsi;
5500 assert_different_registers(tsize, thread_reg, t1);
5501 push(tsize);
5502 movptr(tsize, Address(thread_reg, in_bytes(JavaThread::tlab_size_offset())));
5503 shlptr(tsize, LogHeapWordSize);
5504 cmpptr(t1, tsize);
5505 jcc(Assembler::equal, ok);
5506 STOP("assert(t1 != tlab size)");
5507 should_not_reach_here();
5508
5509 bind(ok);
5510 pop(tsize);
5511 }
5512 #endif
5513 movptr(Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())), top);
5514 movptr(Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())), top);
5515 addptr(top, t1);
5516 subptr(top, (int32_t)ThreadLocalAllocBuffer::alignment_reserve_in_bytes());
5517 movptr(Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())), top);
5518 verify_tlab();
5519 jmp(retry);
5520
5521 return thread_reg; // for use by caller
5522 }
5523
5524 void MacroAssembler::incr_allocated_bytes(Register thread,
5525 Register var_size_in_bytes,
5526 int con_size_in_bytes,
5527 Register t1) {
5528 if (!thread->is_valid()) {
5529 #ifdef _LP64
5530 thread = r15_thread;
5531 #else
5532 assert(t1->is_valid(), "need temp reg");
5533 thread = t1;
5534 get_thread(thread);
5535 #endif
5536 }
5537
5538 #ifdef _LP64
5539 if (var_size_in_bytes->is_valid()) {
5540 addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes);
5541 } else {
|
5402 lea(end, Address(obj, var_size_in_bytes, Address::times_1));
5403 }
5404 cmpptr(end, Address(thread, JavaThread::tlab_end_offset()));
5405 jcc(Assembler::above, slow_case);
5406
5407 // update the tlab top pointer
5408 movptr(Address(thread, JavaThread::tlab_top_offset()), end);
5409
5410 // recover var_size_in_bytes if necessary
5411 if (var_size_in_bytes == end) {
5412 subptr(var_size_in_bytes, obj);
5413 }
5414 verify_tlab();
5415 }
5416
5417 // Preserves rbx, and rdx.
5418 Register MacroAssembler::tlab_refill(Label& retry,
5419 Label& try_eden,
5420 Label& slow_case) {
5421 Register top = rax;
5422 Register t1 = rcx; // object size
5423 Register t2 = rsi;
5424 Register thread_reg = NOT_LP64(rdi) LP64_ONLY(r15_thread);
5425 assert_different_registers(top, thread_reg, t1, t2, /* preserve: */ rbx, rdx);
5426 Label do_refill, discard_tlab;
5427
5428 if (!Universe::heap()->supports_inline_contig_alloc()) {
5429 // No allocation in the shared eden.
5430 jmp(slow_case);
5431 }
5432
5433 NOT_LP64(get_thread(thread_reg));
5434
5435 movptr(top, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
5436 movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())));
5437
5438 // calculate amount of free space
5439 subptr(t1, top);
5440 shrptr(t1, LogHeapWordSize);
5441
5442 // Retain tlab and allocate object in shared space if
5498 Label ok;
5499 Register tsize = rsi;
5500 assert_different_registers(tsize, thread_reg, t1);
5501 push(tsize);
5502 movptr(tsize, Address(thread_reg, in_bytes(JavaThread::tlab_size_offset())));
5503 shlptr(tsize, LogHeapWordSize);
5504 cmpptr(t1, tsize);
5505 jcc(Assembler::equal, ok);
5506 STOP("assert(t1 != tlab size)");
5507 should_not_reach_here();
5508
5509 bind(ok);
5510 pop(tsize);
5511 }
5512 #endif
5513 movptr(Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())), top);
5514 movptr(Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())), top);
5515 addptr(top, t1);
5516 subptr(top, (int32_t)ThreadLocalAllocBuffer::alignment_reserve_in_bytes());
5517 movptr(Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())), top);
5518
5519 if (ZeroTLAB) {
5520 // This is a fast TLAB refill, therefore the GC is not notified of it.
5521 // So compiled code must fill the new TLAB with zeroes.
5522 movptr(top, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())));
5523 zero_memory(top, t1, 0, t2);
5524 }
5525
5526 verify_tlab();
5527 jmp(retry);
5528
5529 return thread_reg; // for use by caller
5530 }
5531
5532 // Preserves the contents of address, destroys the contents length_in_bytes and temp.
5533 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) {
5534 assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different");
5535 assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
5536 Label done;
5537
5538 testptr(length_in_bytes, length_in_bytes);
5539 jcc(Assembler::zero, done);
5540
5541 // initialize topmost word, divide index by 2, check if odd and test if zero
5542 // note: for the remaining code to work, index must be a multiple of BytesPerWord
5543 #ifdef ASSERT
5544 {
5545 Label L;
5546 testptr(length_in_bytes, BytesPerWord - 1);
5547 jcc(Assembler::zero, L);
5548 stop("length must be a multiple of BytesPerWord");
5549 bind(L);
5550 }
5551 #endif
5552 Register index = length_in_bytes;
5553 xorptr(temp, temp); // use _zero reg to clear memory (shorter code)
5554 if (UseIncDec) {
5555 shrptr(index, 3); // divide by 8/16 and set carry flag if bit 2 was set
5556 } else {
5557 shrptr(index, 2); // use 2 instructions to avoid partial flag stall
5558 shrptr(index, 1);
5559 }
5560 #ifndef _LP64
5561 // index could have not been a multiple of 8 (i.e., bit 2 was set)
5562 {
5563 Label even;
5564 // note: if index was a multiple of 8, then it cannot
5565 // be 0 now otherwise it must have been 0 before
5566 // => if it is even, we don't need to check for 0 again
5567 jcc(Assembler::carryClear, even);
5568 // clear topmost word (no jump would be needed if conditional assignment worked here)
5569 movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp);
5570 // index could be 0 now, must check again
5571 jcc(Assembler::zero, done);
5572 bind(even);
5573 }
5574 #endif // !_LP64
5575 // initialize remaining object fields: index is a multiple of 2 now
5576 {
5577 Label loop;
5578 bind(loop);
5579 movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp);
5580 NOT_LP64(movptr(Address(address, index, Address::times_8, offset_in_bytes - 2*BytesPerWord), temp);)
5581 decrement(index);
5582 jcc(Assembler::notZero, loop);
5583 }
5584
5585 bind(done);
5586 }
5587
5588 void MacroAssembler::incr_allocated_bytes(Register thread,
5589 Register var_size_in_bytes,
5590 int con_size_in_bytes,
5591 Register t1) {
5592 if (!thread->is_valid()) {
5593 #ifdef _LP64
5594 thread = r15_thread;
5595 #else
5596 assert(t1->is_valid(), "need temp reg");
5597 thread = t1;
5598 get_thread(thread);
5599 #endif
5600 }
5601
5602 #ifdef _LP64
5603 if (var_size_in_bytes->is_valid()) {
5604 addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes);
5605 } else {
|