3057
3058
3059 void MacroAssembler::verify_tlab() {
3060 #ifdef ASSERT
3061 if (UseTLAB && VerifyOops) {
3062 Label next, next2, ok;
3063 Register t1 = L0;
3064 Register t2 = L1;
3065 Register t3 = L2;
3066
3067 save_frame(0);
3068 ld_ptr(G2_thread, in_bytes(JavaThread::tlab_top_offset()), t1);
3069 ld_ptr(G2_thread, in_bytes(JavaThread::tlab_start_offset()), t2);
3070 or3(t1, t2, t3);
3071 cmp_and_br_short(t1, t2, Assembler::greaterEqual, Assembler::pn, next);
3072 STOP("assert(top >= start)");
3073 should_not_reach_here();
3074
3075 bind(next);
3076 ld_ptr(G2_thread, in_bytes(JavaThread::tlab_top_offset()), t1);
3077 ld_ptr(G2_thread, in_bytes(JavaThread::tlab_end_offset()), t2);
3078 or3(t3, t2, t3);
3079 cmp_and_br_short(t1, t2, Assembler::lessEqual, Assembler::pn, next2);
3080 STOP("assert(top <= end)");
3081 should_not_reach_here();
3082
3083 bind(next2);
3084 and3(t3, MinObjAlignmentInBytesMask, t3);
3085 cmp_and_br_short(t3, 0, Assembler::lessEqual, Assembler::pn, ok);
3086 STOP("assert(aligned)");
3087 should_not_reach_here();
3088
3089 bind(ok);
3090 restore();
3091 }
3092 #endif
3093 }
3094
3095
3096 void MacroAssembler::eden_allocate(
3097 Register obj, // result: pointer to object after successful allocation
3179
3180 void MacroAssembler::tlab_allocate(
3181 Register obj, // result: pointer to object after successful allocation
3182 Register var_size_in_bytes, // object size in bytes if unknown at compile time; invalid otherwise
3183 int con_size_in_bytes, // object size in bytes if known at compile time
3184 Register t1, // temp register
3185 Label& slow_case // continuation point if fast allocation fails
3186 ){
3187 // make sure arguments make sense
3188 assert_different_registers(obj, var_size_in_bytes, t1);
3189 assert(0 <= con_size_in_bytes && is_simm13(con_size_in_bytes), "illegal object size");
3190 assert((con_size_in_bytes & MinObjAlignmentInBytesMask) == 0, "object size is not multiple of alignment");
3191
3192 const Register free = t1;
3193
3194 verify_tlab();
3195
3196 ld_ptr(G2_thread, in_bytes(JavaThread::tlab_top_offset()), obj);
3197
3198 // calculate amount of free space
3199 ld_ptr(G2_thread, in_bytes(JavaThread::tlab_end_offset()), free);
3200 sub(free, obj, free);
3201
3202 Label done;
3203 if (var_size_in_bytes == noreg) {
3204 cmp(free, con_size_in_bytes);
3205 } else {
3206 cmp(free, var_size_in_bytes);
3207 }
3208 br(Assembler::less, false, Assembler::pn, slow_case);
3209 // calculate the new top pointer
3210 if (var_size_in_bytes == noreg) {
3211 delayed()->add(obj, con_size_in_bytes, free);
3212 } else {
3213 delayed()->add(obj, var_size_in_bytes, free);
3214 }
3215
3216 bind(done);
3217
3218 #ifdef ASSERT
3219 // make sure new free pointer is properly aligned
3231 st_ptr(free, G2_thread, in_bytes(JavaThread::tlab_top_offset()));
3232 verify_tlab();
3233 }
3234
3235
3236 void MacroAssembler::tlab_refill(Label& retry, Label& try_eden, Label& slow_case) {
3237 Register top = O0;
3238 Register t1 = G1;
3239 Register t2 = G3;
3240 Register t3 = O1;
3241 assert_different_registers(top, t1, t2, t3, G4, G5 /* preserve G4 and G5 */);
3242 Label do_refill, discard_tlab;
3243
3244 if (!Universe::heap()->supports_inline_contig_alloc()) {
3245 // No allocation in the shared eden.
3246 ba(slow_case);
3247 delayed()->nop();
3248 }
3249
3250 ld_ptr(G2_thread, in_bytes(JavaThread::tlab_top_offset()), top);
3251 ld_ptr(G2_thread, in_bytes(JavaThread::tlab_end_offset()), t1);
3252 ld_ptr(G2_thread, in_bytes(JavaThread::tlab_refill_waste_limit_offset()), t2);
3253
3254 // calculate amount of free space
3255 sub(t1, top, t1);
3256 srl_ptr(t1, LogHeapWordSize, t1);
3257
3258 // Retain tlab and allocate object in shared space if
3259 // the amount free in the tlab is too large to discard.
3260 cmp(t1, t2);
3261
3262 brx(Assembler::lessEqual, false, Assembler::pt, discard_tlab);
3263 // increment waste limit to prevent getting stuck on this slow path
3264 if (Assembler::is_simm13(ThreadLocalAllocBuffer::refill_waste_limit_increment())) {
3265 delayed()->add(t2, ThreadLocalAllocBuffer::refill_waste_limit_increment(), t2);
3266 } else {
3267 delayed()->nop();
3268 // set64 does not use the temp register if the given constant is 32 bit. So
3269 // we can just use any register; using G0 results in ignoring of the upper 32 bit
3270 // of that value.
3271 set64(ThreadLocalAllocBuffer::refill_waste_limit_increment(), t3, G0);
3323 // allocate new tlab, address returned in top
3324 eden_allocate(top, t1, 0, t2, t3, slow_case);
3325
3326 st_ptr(top, G2_thread, in_bytes(JavaThread::tlab_start_offset()));
3327 st_ptr(top, G2_thread, in_bytes(JavaThread::tlab_top_offset()));
3328 #ifdef ASSERT
3329 // check that tlab_size (t1) is still valid
3330 {
3331 Label ok;
3332 ld_ptr(G2_thread, in_bytes(JavaThread::tlab_size_offset()), t2);
3333 sll_ptr(t2, LogHeapWordSize, t2);
3334 cmp_and_br_short(t1, t2, Assembler::equal, Assembler::pt, ok);
3335 STOP("assert(t1 == tlab_size)");
3336 should_not_reach_here();
3337
3338 bind(ok);
3339 }
3340 #endif // ASSERT
3341 add(top, t1, top); // t1 is tlab_size
3342 sub(top, ThreadLocalAllocBuffer::alignment_reserve_in_bytes(), top);
3343 st_ptr(top, G2_thread, in_bytes(JavaThread::tlab_end_offset()));
3344
3345 if (ZeroTLAB) {
3346 // This is a fast TLAB refill, therefore the GC is not notified of it.
3347 // So compiled code must fill the new TLAB with zeroes.
3348 ld_ptr(G2_thread, in_bytes(JavaThread::tlab_start_offset()), t2);
3349 zero_memory(t2, t1);
3350 }
3351 verify_tlab();
3352 ba(retry);
3353 delayed()->nop();
3354 }
3355
3356 void MacroAssembler::zero_memory(Register base, Register index) {
3357 assert_different_registers(base, index);
3358 Label loop;
3359 bind(loop);
3360 subcc(index, HeapWordSize, index);
3361 brx(Assembler::greaterEqual, true, Assembler::pt, loop);
3362 delayed()->st_ptr(G0, base, index);
3363 }
|
3057
3058
3059 void MacroAssembler::verify_tlab() {
3060 #ifdef ASSERT
3061 if (UseTLAB && VerifyOops) {
3062 Label next, next2, ok;
3063 Register t1 = L0;
3064 Register t2 = L1;
3065 Register t3 = L2;
3066
3067 save_frame(0);
3068 ld_ptr(G2_thread, in_bytes(JavaThread::tlab_top_offset()), t1);
3069 ld_ptr(G2_thread, in_bytes(JavaThread::tlab_start_offset()), t2);
3070 or3(t1, t2, t3);
3071 cmp_and_br_short(t1, t2, Assembler::greaterEqual, Assembler::pn, next);
3072 STOP("assert(top >= start)");
3073 should_not_reach_here();
3074
3075 bind(next);
3076 ld_ptr(G2_thread, in_bytes(JavaThread::tlab_top_offset()), t1);
3077 ld_ptr(G2_thread, in_bytes(JavaThread::tlab_current_end_offset()), t2);
3078 or3(t3, t2, t3);
3079 cmp_and_br_short(t1, t2, Assembler::lessEqual, Assembler::pn, next2);
3080 STOP("assert(top <= end)");
3081 should_not_reach_here();
3082
3083 bind(next2);
3084 and3(t3, MinObjAlignmentInBytesMask, t3);
3085 cmp_and_br_short(t3, 0, Assembler::lessEqual, Assembler::pn, ok);
3086 STOP("assert(aligned)");
3087 should_not_reach_here();
3088
3089 bind(ok);
3090 restore();
3091 }
3092 #endif
3093 }
3094
3095
3096 void MacroAssembler::eden_allocate(
3097 Register obj, // result: pointer to object after successful allocation
3179
3180 void MacroAssembler::tlab_allocate(
3181 Register obj, // result: pointer to object after successful allocation
3182 Register var_size_in_bytes, // object size in bytes if unknown at compile time; invalid otherwise
3183 int con_size_in_bytes, // object size in bytes if known at compile time
3184 Register t1, // temp register
3185 Label& slow_case // continuation point if fast allocation fails
3186 ){
3187 // make sure arguments make sense
3188 assert_different_registers(obj, var_size_in_bytes, t1);
3189 assert(0 <= con_size_in_bytes && is_simm13(con_size_in_bytes), "illegal object size");
3190 assert((con_size_in_bytes & MinObjAlignmentInBytesMask) == 0, "object size is not multiple of alignment");
3191
3192 const Register free = t1;
3193
3194 verify_tlab();
3195
3196 ld_ptr(G2_thread, in_bytes(JavaThread::tlab_top_offset()), obj);
3197
3198 // calculate amount of free space
3199 ld_ptr(G2_thread, in_bytes(JavaThread::tlab_current_end_offset()), free);
3200 sub(free, obj, free);
3201
3202 Label done;
3203 if (var_size_in_bytes == noreg) {
3204 cmp(free, con_size_in_bytes);
3205 } else {
3206 cmp(free, var_size_in_bytes);
3207 }
3208 br(Assembler::less, false, Assembler::pn, slow_case);
3209 // calculate the new top pointer
3210 if (var_size_in_bytes == noreg) {
3211 delayed()->add(obj, con_size_in_bytes, free);
3212 } else {
3213 delayed()->add(obj, var_size_in_bytes, free);
3214 }
3215
3216 bind(done);
3217
3218 #ifdef ASSERT
3219 // make sure new free pointer is properly aligned
3231 st_ptr(free, G2_thread, in_bytes(JavaThread::tlab_top_offset()));
3232 verify_tlab();
3233 }
3234
3235
3236 void MacroAssembler::tlab_refill(Label& retry, Label& try_eden, Label& slow_case) {
3237 Register top = O0;
3238 Register t1 = G1;
3239 Register t2 = G3;
3240 Register t3 = O1;
3241 assert_different_registers(top, t1, t2, t3, G4, G5 /* preserve G4 and G5 */);
3242 Label do_refill, discard_tlab;
3243
3244 if (!Universe::heap()->supports_inline_contig_alloc()) {
3245 // No allocation in the shared eden.
3246 ba(slow_case);
3247 delayed()->nop();
3248 }
3249
3250 ld_ptr(G2_thread, in_bytes(JavaThread::tlab_top_offset()), top);
3251 ld_ptr(G2_thread, in_bytes(JavaThread::tlab_current_end_offset()), t1);
3252 ld_ptr(G2_thread, in_bytes(JavaThread::tlab_refill_waste_limit_offset()), t2);
3253
3254 // calculate amount of free space
3255 sub(t1, top, t1);
3256 srl_ptr(t1, LogHeapWordSize, t1);
3257
3258 // Retain tlab and allocate object in shared space if
3259 // the amount free in the tlab is too large to discard.
3260 cmp(t1, t2);
3261
3262 brx(Assembler::lessEqual, false, Assembler::pt, discard_tlab);
3263 // increment waste limit to prevent getting stuck on this slow path
3264 if (Assembler::is_simm13(ThreadLocalAllocBuffer::refill_waste_limit_increment())) {
3265 delayed()->add(t2, ThreadLocalAllocBuffer::refill_waste_limit_increment(), t2);
3266 } else {
3267 delayed()->nop();
3268 // set64 does not use the temp register if the given constant is 32 bit. So
3269 // we can just use any register; using G0 results in ignoring of the upper 32 bit
3270 // of that value.
3271 set64(ThreadLocalAllocBuffer::refill_waste_limit_increment(), t3, G0);
3323 // allocate new tlab, address returned in top
3324 eden_allocate(top, t1, 0, t2, t3, slow_case);
3325
3326 st_ptr(top, G2_thread, in_bytes(JavaThread::tlab_start_offset()));
3327 st_ptr(top, G2_thread, in_bytes(JavaThread::tlab_top_offset()));
3328 #ifdef ASSERT
3329 // check that tlab_size (t1) is still valid
3330 {
3331 Label ok;
3332 ld_ptr(G2_thread, in_bytes(JavaThread::tlab_size_offset()), t2);
3333 sll_ptr(t2, LogHeapWordSize, t2);
3334 cmp_and_br_short(t1, t2, Assembler::equal, Assembler::pt, ok);
3335 STOP("assert(t1 == tlab_size)");
3336 should_not_reach_here();
3337
3338 bind(ok);
3339 }
3340 #endif // ASSERT
3341 add(top, t1, top); // t1 is tlab_size
3342 sub(top, ThreadLocalAllocBuffer::alignment_reserve_in_bytes(), top);
3343 st_ptr(top, G2_thread, in_bytes(JavaThread::tlab_current_end_offset()));
3344
3345 if (ZeroTLAB) {
3346 // This is a fast TLAB refill, therefore the GC is not notified of it.
3347 // So compiled code must fill the new TLAB with zeroes.
3348 ld_ptr(G2_thread, in_bytes(JavaThread::tlab_start_offset()), t2);
3349 zero_memory(t2, t1);
3350 }
3351 verify_tlab();
3352 ba(retry);
3353 delayed()->nop();
3354 }
3355
3356 void MacroAssembler::zero_memory(Register base, Register index) {
3357 assert_different_registers(base, index);
3358 Label loop;
3359 bind(loop);
3360 subcc(index, HeapWordSize, index);
3361 brx(Assembler::greaterEqual, true, Assembler::pt, loop);
3362 delayed()->st_ptr(G0, base, index);
3363 }
|