2423 result = different(prev, newv, addr) ? prev : rscratch2; \
2424 \
2425 Label retry_load; \
2426 if ((VM_Version::features() & VM_Version::CPU_STXR_PREFETCH)) \
2427 prfm(Address(addr), PSTL1STRM); \
2428 bind(retry_load); \
2429 LDXR(result, addr); \
2430 STXR(rscratch1, newv, addr); \
2431 cbnzw(rscratch1, retry_load); \
2432 if (prev->is_valid() && prev != result) \
2433 mov(prev, result); \
2434 }
2435
2436 ATOMIC_XCHG(xchg, swp, ldxr, stxr, Assembler::xword)
2437 ATOMIC_XCHG(xchgw, swp, ldxrw, stxrw, Assembler::word)
2438 ATOMIC_XCHG(xchgal, swpal, ldaxr, stlxr, Assembler::xword)
2439 ATOMIC_XCHG(xchgalw, swpal, ldaxrw, stlxrw, Assembler::word)
2440
2441 #undef ATOMIC_XCHG
2442
2443 void MacroAssembler::incr_allocated_bytes(Register thread,
2444 Register var_size_in_bytes,
2445 int con_size_in_bytes,
2446 Register t1) {
2447 if (!thread->is_valid()) {
2448 thread = rthread;
2449 }
2450 assert(t1->is_valid(), "need temp reg");
2451
2452 ldr(t1, Address(thread, in_bytes(JavaThread::allocated_bytes_offset())));
2453 if (var_size_in_bytes->is_valid()) {
2454 add(t1, t1, var_size_in_bytes);
2455 } else {
2456 add(t1, t1, con_size_in_bytes);
2457 }
2458 str(t1, Address(thread, in_bytes(JavaThread::allocated_bytes_offset())));
2459 }
2460
2461 #ifndef PRODUCT
2462 extern "C" void findpc(intptr_t x);
2463 #endif
2464
2465 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[])
2466 {
2467 // In order to get locks to work, we need to fake a in_VM state
2468 if (ShowMessageBoxOnError ) {
2469 JavaThread* thread = JavaThread::current();
2470 JavaThreadState saved_state = thread->thread_state();
2471 thread->set_thread_state(_thread_in_vm);
2472 #ifndef PRODUCT
2473 if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
2474 ttyLocker ttyl;
2475 BytecodeCounter::print();
2476 }
2477 #endif
2478 if (os::message_box(msg, "Execution stopped, print registers?")) {
2479 ttyLocker ttyl;
2480 tty->print_cr(" pc = 0x%016lx", pc);
4075
4076 Address MacroAssembler::constant_oop_address(jobject obj) {
4077 #ifdef ASSERT
4078 {
4079 ThreadInVMfromUnknown tiv;
4080 assert(oop_recorder() != NULL, "this assembler needs an OopRecorder");
4081 assert(Universe::heap()->is_in_reserved(JNIHandles::resolve(obj)), "not an oop");
4082 }
4083 #endif
4084 int oop_index = oop_recorder()->find_index(obj);
4085 return Address((address)obj, oop_Relocation::spec(oop_index));
4086 }
4087
4088 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
4089 void MacroAssembler::tlab_allocate(Register obj,
4090 Register var_size_in_bytes,
4091 int con_size_in_bytes,
4092 Register t1,
4093 Register t2,
4094 Label& slow_case) {
4095 assert_different_registers(obj, t2);
4096 assert_different_registers(obj, var_size_in_bytes);
4097 Register end = t2;
4098
4099 // verify_tlab();
4100
4101 ldr(obj, Address(rthread, JavaThread::tlab_top_offset()));
4102 if (var_size_in_bytes == noreg) {
4103 lea(end, Address(obj, con_size_in_bytes));
4104 } else {
4105 lea(end, Address(obj, var_size_in_bytes));
4106 }
4107 ldr(rscratch1, Address(rthread, JavaThread::tlab_end_offset()));
4108 cmp(end, rscratch1);
4109 br(Assembler::HI, slow_case);
4110
4111 // update the tlab top pointer
4112 str(end, Address(rthread, JavaThread::tlab_top_offset()));
4113
4114 // recover var_size_in_bytes if necessary
4115 if (var_size_in_bytes == end) {
4116 sub(var_size_in_bytes, var_size_in_bytes, obj);
4117 }
4118 // verify_tlab();
4119 }
4120
4121 // Zero words; len is in bytes
4122 // Destroys all registers except addr
4123 // len must be a nonzero multiple of wordSize
4124 void MacroAssembler::zero_memory(Register addr, Register len, Register t1) {
4125 assert_different_registers(addr, len, t1, rscratch1, rscratch2);
4126
4127 #ifdef ASSERT
4128 { Label L;
4129 tst(len, BytesPerWord - 1);
4130 br(Assembler::EQ, L);
4131 stop("len is not a multiple of BytesPerWord");
4132 bind(L);
4133 }
4134 #endif
4135
4136 #ifndef PRODUCT
4137 block_comment("zero memory");
4138 #endif
4161 // } while (cnt);
4162 // }
4163
4164 const int unroll = 8; // Number of str(zr) instructions we'll unroll
4165
4166 lsr(len, len, LogBytesPerWord);
4167 andr(rscratch1, len, unroll - 1); // tmp1 = cnt % unroll
4168 sub(len, len, rscratch1); // cnt -= unroll
4169 // t1 always points to the end of the region we're about to zero
4170 add(t1, addr, rscratch1, Assembler::LSL, LogBytesPerWord);
4171 adr(rscratch2, entry);
4172 sub(rscratch2, rscratch2, rscratch1, Assembler::LSL, 2);
4173 br(rscratch2);
4174 bind(loop);
4175 sub(len, len, unroll);
4176 for (int i = -unroll; i < 0; i++)
4177 Assembler::str(zr, Address(t1, i * wordSize));
4178 bind(entry);
4179 add(t1, t1, unroll * wordSize);
4180 cbnz(len, loop);
4181 }
4182
4183 // Defines obj, preserves var_size_in_bytes
4184 void MacroAssembler::eden_allocate(Register obj,
4185 Register var_size_in_bytes,
4186 int con_size_in_bytes,
4187 Register t1,
4188 Label& slow_case) {
4189 assert_different_registers(obj, var_size_in_bytes, t1);
4190 if (!Universe::heap()->supports_inline_contig_alloc()) {
4191 b(slow_case);
4192 } else {
4193 Register end = t1;
4194 Register heap_end = rscratch2;
4195 Label retry;
4196 bind(retry);
4197 {
4198 unsigned long offset;
4199 adrp(rscratch1, ExternalAddress((address) Universe::heap()->end_addr()), offset);
4200 ldr(heap_end, Address(rscratch1, offset));
4201 }
4202
4203 ExternalAddress heap_top((address) Universe::heap()->top_addr());
4204
4205 // Get the current top of the heap
4206 {
4207 unsigned long offset;
4208 adrp(rscratch1, heap_top, offset);
4209 // Use add() here after ARDP, rather than lea().
4210 // lea() does not generate anything if its offset is zero.
4211 // However, relocs expect to find either an ADD or a load/store
4212 // insn after an ADRP. add() always generates an ADD insn, even
4213 // for add(Rn, Rn, 0).
4214 add(rscratch1, rscratch1, offset);
4215 ldaxr(obj, rscratch1);
4216 }
4217
4218 // Adjust it my the size of our new object
4219 if (var_size_in_bytes == noreg) {
4220 lea(end, Address(obj, con_size_in_bytes));
4221 } else {
4222 lea(end, Address(obj, var_size_in_bytes));
4223 }
4224
4225 // if end < obj then we wrapped around high memory
4226 cmp(end, obj);
4227 br(Assembler::LO, slow_case);
4228
4229 cmp(end, heap_end);
4230 br(Assembler::HI, slow_case);
4231
4232 // If heap_top hasn't been changed by some other thread, update it.
4233 stlxr(rscratch2, end, rscratch1);
4234 cbnzw(rscratch2, retry);
4235 }
4236 }
4237
4238 void MacroAssembler::verify_tlab() {
4239 #ifdef ASSERT
4240 if (UseTLAB && VerifyOops) {
4241 Label next, ok;
4242
4243 stp(rscratch2, rscratch1, Address(pre(sp, -16)));
4244
4245 ldr(rscratch2, Address(rthread, in_bytes(JavaThread::tlab_top_offset())));
4246 ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_start_offset())));
4247 cmp(rscratch2, rscratch1);
4248 br(Assembler::HS, next);
4249 STOP("assert(top >= start)");
4250 should_not_reach_here();
4251
4252 bind(next);
4253 ldr(rscratch2, Address(rthread, in_bytes(JavaThread::tlab_end_offset())));
4254 ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_top_offset())));
4255 cmp(rscratch2, rscratch1);
|
2423 result = different(prev, newv, addr) ? prev : rscratch2; \
2424 \
2425 Label retry_load; \
2426 if ((VM_Version::features() & VM_Version::CPU_STXR_PREFETCH)) \
2427 prfm(Address(addr), PSTL1STRM); \
2428 bind(retry_load); \
2429 LDXR(result, addr); \
2430 STXR(rscratch1, newv, addr); \
2431 cbnzw(rscratch1, retry_load); \
2432 if (prev->is_valid() && prev != result) \
2433 mov(prev, result); \
2434 }
2435
2436 ATOMIC_XCHG(xchg, swp, ldxr, stxr, Assembler::xword)
2437 ATOMIC_XCHG(xchgw, swp, ldxrw, stxrw, Assembler::word)
2438 ATOMIC_XCHG(xchgal, swpal, ldaxr, stlxr, Assembler::xword)
2439 ATOMIC_XCHG(xchgalw, swpal, ldaxrw, stlxrw, Assembler::word)
2440
2441 #undef ATOMIC_XCHG
2442
2443 #ifndef PRODUCT
2444 extern "C" void findpc(intptr_t x);
2445 #endif
2446
2447 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[])
2448 {
2449 // In order to get locks to work, we need to fake a in_VM state
2450 if (ShowMessageBoxOnError ) {
2451 JavaThread* thread = JavaThread::current();
2452 JavaThreadState saved_state = thread->thread_state();
2453 thread->set_thread_state(_thread_in_vm);
2454 #ifndef PRODUCT
2455 if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
2456 ttyLocker ttyl;
2457 BytecodeCounter::print();
2458 }
2459 #endif
2460 if (os::message_box(msg, "Execution stopped, print registers?")) {
2461 ttyLocker ttyl;
2462 tty->print_cr(" pc = 0x%016lx", pc);
4057
4058 Address MacroAssembler::constant_oop_address(jobject obj) {
4059 #ifdef ASSERT
4060 {
4061 ThreadInVMfromUnknown tiv;
4062 assert(oop_recorder() != NULL, "this assembler needs an OopRecorder");
4063 assert(Universe::heap()->is_in_reserved(JNIHandles::resolve(obj)), "not an oop");
4064 }
4065 #endif
4066 int oop_index = oop_recorder()->find_index(obj);
4067 return Address((address)obj, oop_Relocation::spec(oop_index));
4068 }
4069
4070 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
4071 void MacroAssembler::tlab_allocate(Register obj,
4072 Register var_size_in_bytes,
4073 int con_size_in_bytes,
4074 Register t1,
4075 Register t2,
4076 Label& slow_case) {
4077 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
4078 bs->tlab_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
4079 }
4080
4081 // Defines obj, preserves var_size_in_bytes
4082 void MacroAssembler::eden_allocate(Register obj,
4083 Register var_size_in_bytes,
4084 int con_size_in_bytes,
4085 Register t1,
4086 Label& slow_case) {
4087 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
4088 bs->eden_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case);
4089 }
4090
4091 // Zero words; len is in bytes
4092 // Destroys all registers except addr
4093 // len must be a nonzero multiple of wordSize
4094 void MacroAssembler::zero_memory(Register addr, Register len, Register t1) {
4095 assert_different_registers(addr, len, t1, rscratch1, rscratch2);
4096
4097 #ifdef ASSERT
4098 { Label L;
4099 tst(len, BytesPerWord - 1);
4100 br(Assembler::EQ, L);
4101 stop("len is not a multiple of BytesPerWord");
4102 bind(L);
4103 }
4104 #endif
4105
4106 #ifndef PRODUCT
4107 block_comment("zero memory");
4108 #endif
4131 // } while (cnt);
4132 // }
4133
4134 const int unroll = 8; // Number of str(zr) instructions we'll unroll
4135
4136 lsr(len, len, LogBytesPerWord);
4137 andr(rscratch1, len, unroll - 1); // tmp1 = cnt % unroll
4138 sub(len, len, rscratch1); // cnt -= unroll
4139 // t1 always points to the end of the region we're about to zero
4140 add(t1, addr, rscratch1, Assembler::LSL, LogBytesPerWord);
4141 adr(rscratch2, entry);
4142 sub(rscratch2, rscratch2, rscratch1, Assembler::LSL, 2);
4143 br(rscratch2);
4144 bind(loop);
4145 sub(len, len, unroll);
4146 for (int i = -unroll; i < 0; i++)
4147 Assembler::str(zr, Address(t1, i * wordSize));
4148 bind(entry);
4149 add(t1, t1, unroll * wordSize);
4150 cbnz(len, loop);
4151 }
4152
4153 void MacroAssembler::verify_tlab() {
4154 #ifdef ASSERT
4155 if (UseTLAB && VerifyOops) {
4156 Label next, ok;
4157
4158 stp(rscratch2, rscratch1, Address(pre(sp, -16)));
4159
4160 ldr(rscratch2, Address(rthread, in_bytes(JavaThread::tlab_top_offset())));
4161 ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_start_offset())));
4162 cmp(rscratch2, rscratch1);
4163 br(Assembler::HS, next);
4164 STOP("assert(top >= start)");
4165 should_not_reach_here();
4166
4167 bind(next);
4168 ldr(rscratch2, Address(rthread, in_bytes(JavaThread::tlab_end_offset())));
4169 ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_top_offset())));
4170 cmp(rscratch2, rscratch1);
|