4059
4060 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
4061 void MacroAssembler::tlab_allocate(Register obj,
4062 Register var_size_in_bytes,
4063 int con_size_in_bytes,
4064 Register t1,
4065 Register t2,
4066 Label& slow_case) {
4067 assert_different_registers(obj, t2);
4068 assert_different_registers(obj, var_size_in_bytes);
4069 Register end = t2;
4070
4071 // verify_tlab();
4072
4073 ldr(obj, Address(rthread, JavaThread::tlab_top_offset()));
4074 if (var_size_in_bytes == noreg) {
4075 lea(end, Address(obj, con_size_in_bytes));
4076 } else {
4077 lea(end, Address(obj, var_size_in_bytes));
4078 }
4079 ldr(rscratch1, Address(rthread, JavaThread::tlab_end_offset()));
4080 cmp(end, rscratch1);
4081 br(Assembler::HI, slow_case);
4082
4083 // update the tlab top pointer
4084 str(end, Address(rthread, JavaThread::tlab_top_offset()));
4085
4086 // recover var_size_in_bytes if necessary
4087 if (var_size_in_bytes == end) {
4088 sub(var_size_in_bytes, var_size_in_bytes, obj);
4089 }
4090 // verify_tlab();
4091 }
4092
4093 // Preserves r19, and r3.
4094 Register MacroAssembler::tlab_refill(Label& retry,
4095 Label& try_eden,
4096 Label& slow_case) {
4097 Register top = r0;
4098 Register t1 = r2;
4099 Register t2 = r4;
4100 assert_different_registers(top, rthread, t1, t2, /* preserve: */ r19, r3);
4101 Label do_refill, discard_tlab;
4102
4103 if (!Universe::heap()->supports_inline_contig_alloc()) {
4104 // No allocation in the shared eden.
4105 b(slow_case);
4106 }
4107
4108 ldr(top, Address(rthread, in_bytes(JavaThread::tlab_top_offset())));
4109 ldr(t1, Address(rthread, in_bytes(JavaThread::tlab_end_offset())));
4110
4111 // calculate amount of free space
4112 sub(t1, t1, top);
4113 lsr(t1, t1, LogHeapWordSize);
4114
4115 // Retain tlab and allocate object in shared space if
4116 // the amount free in the tlab is too large to discard.
4117
4118 ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_refill_waste_limit_offset())));
4119 cmp(t1, rscratch1);
4120 br(Assembler::LE, discard_tlab);
4121
4122 // Retain
4123 // ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_refill_waste_limit_offset())));
4124 mov(t2, (int32_t) ThreadLocalAllocBuffer::refill_waste_limit_increment());
4125 add(rscratch1, rscratch1, t2);
4126 str(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_refill_waste_limit_offset())));
4127
4128 if (TLABStats) {
4129 // increment number of slow_allocations
4183 if (UseTLAB) {
4184 Label ok;
4185 Register tsize = r4;
4186 assert_different_registers(tsize, rthread, t1);
4187 str(tsize, Address(pre(sp, -16)));
4188 ldr(tsize, Address(rthread, in_bytes(JavaThread::tlab_size_offset())));
4189 lsl(tsize, tsize, LogHeapWordSize);
4190 cmp(t1, tsize);
4191 br(Assembler::EQ, ok);
4192 STOP("assert(t1 != tlab size)");
4193 should_not_reach_here();
4194
4195 bind(ok);
4196 ldr(tsize, Address(post(sp, 16)));
4197 }
4198 #endif
4199 str(top, Address(rthread, in_bytes(JavaThread::tlab_start_offset())));
4200 str(top, Address(rthread, in_bytes(JavaThread::tlab_top_offset())));
4201 add(top, top, t1);
4202 sub(top, top, (int32_t)ThreadLocalAllocBuffer::alignment_reserve_in_bytes());
4203 str(top, Address(rthread, in_bytes(JavaThread::tlab_end_offset())));
4204
4205 if (ZeroTLAB) {
4206 // This is a fast TLAB refill, therefore the GC is not notified of it.
4207 // So compiled code must fill the new TLAB with zeroes.
4208 ldr(top, Address(rthread, in_bytes(JavaThread::tlab_start_offset())));
4209 zero_memory(top,t1,t2);
4210 }
4211
4212 verify_tlab();
4213 b(retry);
4214
4215 return rthread; // for use by caller
4216 }
4217
4218 // Zero words; len is in bytes
4219 // Destroys all registers except addr
4220 // len must be a nonzero multiple of wordSize
4221 void MacroAssembler::zero_memory(Register addr, Register len, Register t1) {
4222 assert_different_registers(addr, len, t1, rscratch1, rscratch2);
4223
4330 stlxr(rscratch2, end, rscratch1);
4331 cbnzw(rscratch2, retry);
4332 }
4333 }
4334
4335 void MacroAssembler::verify_tlab() {
4336 #ifdef ASSERT
4337 if (UseTLAB && VerifyOops) {
4338 Label next, ok;
4339
4340 stp(rscratch2, rscratch1, Address(pre(sp, -16)));
4341
4342 ldr(rscratch2, Address(rthread, in_bytes(JavaThread::tlab_top_offset())));
4343 ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_start_offset())));
4344 cmp(rscratch2, rscratch1);
4345 br(Assembler::HS, next);
4346 STOP("assert(top >= start)");
4347 should_not_reach_here();
4348
4349 bind(next);
4350 ldr(rscratch2, Address(rthread, in_bytes(JavaThread::tlab_end_offset())));
4351 ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_top_offset())));
4352 cmp(rscratch2, rscratch1);
4353 br(Assembler::HS, ok);
4354 STOP("assert(top <= end)");
4355 should_not_reach_here();
4356
4357 bind(ok);
4358 ldp(rscratch2, rscratch1, Address(post(sp, 16)));
4359 }
4360 #endif
4361 }
4362
4363 // Writes to stack successive pages until offset reached to check for
4364 // stack overflow + shadow pages. This clobbers tmp.
4365 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
4366 assert_different_registers(tmp, size, rscratch1);
4367 mov(tmp, sp);
4368 // Bang stack for total size given plus shadow page size.
4369 // Bang one page at a time because large size can bang beyond yellow and
4370 // red zones.
|
4059
4060 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
4061 void MacroAssembler::tlab_allocate(Register obj,
4062 Register var_size_in_bytes,
4063 int con_size_in_bytes,
4064 Register t1,
4065 Register t2,
4066 Label& slow_case) {
4067 assert_different_registers(obj, t2);
4068 assert_different_registers(obj, var_size_in_bytes);
4069 Register end = t2;
4070
4071 // verify_tlab();
4072
4073 ldr(obj, Address(rthread, JavaThread::tlab_top_offset()));
4074 if (var_size_in_bytes == noreg) {
4075 lea(end, Address(obj, con_size_in_bytes));
4076 } else {
4077 lea(end, Address(obj, var_size_in_bytes));
4078 }
4079 ldr(rscratch1, Address(rthread, JavaThread::tlab_current_end_offset()));
4080 cmp(end, rscratch1);
4081 br(Assembler::HI, slow_case);
4082
4083 // update the tlab top pointer
4084 str(end, Address(rthread, JavaThread::tlab_top_offset()));
4085
4086 // recover var_size_in_bytes if necessary
4087 if (var_size_in_bytes == end) {
4088 sub(var_size_in_bytes, var_size_in_bytes, obj);
4089 }
4090 // verify_tlab();
4091 }
4092
4093 // Preserves r19, and r3.
4094 Register MacroAssembler::tlab_refill(Label& retry,
4095 Label& try_eden,
4096 Label& slow_case) {
4097 Register top = r0;
4098 Register t1 = r2;
4099 Register t2 = r4;
4100 assert_different_registers(top, rthread, t1, t2, /* preserve: */ r19, r3);
4101 Label do_refill, discard_tlab;
4102
4103 if (!Universe::heap()->supports_inline_contig_alloc()) {
4104 // No allocation in the shared eden.
4105 b(slow_case);
4106 }
4107
4108 ldr(top, Address(rthread, in_bytes(JavaThread::tlab_top_offset())));
4109 ldr(t1, Address(rthread, in_bytes(JavaThread::tlab_current_end_offset())));
4110
4111 // calculate amount of free space
4112 sub(t1, t1, top);
4113 lsr(t1, t1, LogHeapWordSize);
4114
4115 // Retain tlab and allocate object in shared space if
4116 // the amount free in the tlab is too large to discard.
4117
4118 ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_refill_waste_limit_offset())));
4119 cmp(t1, rscratch1);
4120 br(Assembler::LE, discard_tlab);
4121
4122 // Retain
4123 // ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_refill_waste_limit_offset())));
4124 mov(t2, (int32_t) ThreadLocalAllocBuffer::refill_waste_limit_increment());
4125 add(rscratch1, rscratch1, t2);
4126 str(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_refill_waste_limit_offset())));
4127
4128 if (TLABStats) {
4129 // increment number of slow_allocations
4183 if (UseTLAB) {
4184 Label ok;
4185 Register tsize = r4;
4186 assert_different_registers(tsize, rthread, t1);
4187 str(tsize, Address(pre(sp, -16)));
4188 ldr(tsize, Address(rthread, in_bytes(JavaThread::tlab_size_offset())));
4189 lsl(tsize, tsize, LogHeapWordSize);
4190 cmp(t1, tsize);
4191 br(Assembler::EQ, ok);
4192 STOP("assert(t1 != tlab size)");
4193 should_not_reach_here();
4194
4195 bind(ok);
4196 ldr(tsize, Address(post(sp, 16)));
4197 }
4198 #endif
4199 str(top, Address(rthread, in_bytes(JavaThread::tlab_start_offset())));
4200 str(top, Address(rthread, in_bytes(JavaThread::tlab_top_offset())));
4201 add(top, top, t1);
4202 sub(top, top, (int32_t)ThreadLocalAllocBuffer::alignment_reserve_in_bytes());
4203 str(top, Address(rthread, in_bytes(JavaThread::tlab_current_end_offset())));
4204
4205 if (ZeroTLAB) {
4206 // This is a fast TLAB refill, therefore the GC is not notified of it.
4207 // So compiled code must fill the new TLAB with zeroes.
4208 ldr(top, Address(rthread, in_bytes(JavaThread::tlab_start_offset())));
4209 zero_memory(top,t1,t2);
4210 }
4211
4212 verify_tlab();
4213 b(retry);
4214
4215 return rthread; // for use by caller
4216 }
4217
4218 // Zero words; len is in bytes
4219 // Destroys all registers except addr
4220 // len must be a nonzero multiple of wordSize
4221 void MacroAssembler::zero_memory(Register addr, Register len, Register t1) {
4222 assert_different_registers(addr, len, t1, rscratch1, rscratch2);
4223
4330 stlxr(rscratch2, end, rscratch1);
4331 cbnzw(rscratch2, retry);
4332 }
4333 }
4334
4335 void MacroAssembler::verify_tlab() {
4336 #ifdef ASSERT
4337 if (UseTLAB && VerifyOops) {
4338 Label next, ok;
4339
4340 stp(rscratch2, rscratch1, Address(pre(sp, -16)));
4341
4342 ldr(rscratch2, Address(rthread, in_bytes(JavaThread::tlab_top_offset())));
4343 ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_start_offset())));
4344 cmp(rscratch2, rscratch1);
4345 br(Assembler::HS, next);
4346 STOP("assert(top >= start)");
4347 should_not_reach_here();
4348
4349 bind(next);
4350 ldr(rscratch2, Address(rthread, in_bytes(JavaThread::tlab_current_end_offset())));
4351 ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_top_offset())));
4352 cmp(rscratch2, rscratch1);
4353 br(Assembler::HS, ok);
4354 STOP("assert(top <= end)");
4355 should_not_reach_here();
4356
4357 bind(ok);
4358 ldp(rscratch2, rscratch1, Address(post(sp, 16)));
4359 }
4360 #endif
4361 }
4362
4363 // Writes to stack successive pages until offset reached to check for
4364 // stack overflow + shadow pages. This clobbers tmp.
4365 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
4366 assert_different_registers(tmp, size, rscratch1);
4367 mov(tmp, sp);
4368 // Bang stack for total size given plus shadow page size.
4369 // Bang one page at a time because large size can bang beyond yellow and
4370 // red zones.
|