3019
3020 // on success:
3021 restore();
3022 ba_short(L_success);
3023
3024 // on failure:
3025 bind(L_pop_to_failure);
3026 restore();
3027 bind(L_failure);
3028 }
3029
3030
3031 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
3032 Register super_klass,
3033 Register temp_reg,
3034 Register temp2_reg,
3035 Label* L_success,
3036 Label* L_failure,
3037 Label* L_slow_path,
3038 RegisterOrConstant super_check_offset) {
3039 int sc_offset = (klassOopDesc::header_size() * HeapWordSize +
3040 Klass::secondary_super_cache_offset_in_bytes());
3041 int sco_offset = (klassOopDesc::header_size() * HeapWordSize +
3042 Klass::super_check_offset_offset_in_bytes());
3043
3044 bool must_load_sco = (super_check_offset.constant_or_zero() == -1);
3045 bool need_slow_path = (must_load_sco ||
3046 super_check_offset.constant_or_zero() == sco_offset);
3047
3048 assert_different_registers(sub_klass, super_klass, temp_reg);
3049 if (super_check_offset.is_register()) {
3050 assert_different_registers(sub_klass, super_klass, temp_reg,
3051 super_check_offset.as_register());
3052 } else if (must_load_sco) {
3053 assert(temp2_reg != noreg, "supply either a temp or a register offset");
3054 }
3055
3056 Label L_fallthrough;
3057 int label_nulls = 0;
3058 if (L_success == NULL) { L_success = &L_fallthrough; label_nulls++; }
3059 if (L_failure == NULL) { L_failure = &L_fallthrough; label_nulls++; }
3060 if (L_slow_path == NULL) { L_slow_path = &L_fallthrough; label_nulls++; }
3061 assert(label_nulls <= 1 ||
3062 (L_slow_path == &L_fallthrough && label_nulls <= 2 && !need_slow_path),
3142
3143
3144 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
3145 Register super_klass,
3146 Register count_temp,
3147 Register scan_temp,
3148 Register scratch_reg,
3149 Register coop_reg,
3150 Label* L_success,
3151 Label* L_failure) {
3152 assert_different_registers(sub_klass, super_klass,
3153 count_temp, scan_temp, scratch_reg, coop_reg);
3154
3155 Label L_fallthrough, L_loop;
3156 int label_nulls = 0;
3157 if (L_success == NULL) { L_success = &L_fallthrough; label_nulls++; }
3158 if (L_failure == NULL) { L_failure = &L_fallthrough; label_nulls++; }
3159 assert(label_nulls <= 1, "at most one NULL in the batch");
3160
3161 // a couple of useful fields in sub_klass:
3162 int ss_offset = (klassOopDesc::header_size() * HeapWordSize +
3163 Klass::secondary_supers_offset_in_bytes());
3164 int sc_offset = (klassOopDesc::header_size() * HeapWordSize +
3165 Klass::secondary_super_cache_offset_in_bytes());
3166
3167 // Do a linear scan of the secondary super-klass chain.
3168 // This code is rarely used, so simplicity is a virtue here.
3169
3170 #ifndef PRODUCT
3171 int* pst_counter = &SharedRuntime::_partial_subtype_ctr;
3172 inc_counter((address) pst_counter, count_temp, scan_temp);
3173 #endif
3174
3175 // We will consult the secondary-super array.
3176 ld_ptr(sub_klass, ss_offset, scan_temp);
3177
3178 // Compress superclass if necessary.
3179 Register search_key = super_klass;
3180 bool decode_super_klass = false;
3181 if (UseCompressedOops) {
3182 if (coop_reg != noreg) {
3183 encode_heap_oop_not_null(super_klass, coop_reg);
3184 search_key = coop_reg;
3185 } else {
3319 assert(UseBiasedLocking, "why call this otherwise?");
3320
3321 if (PrintBiasedLockingStatistics) {
3322 assert_different_registers(obj_reg, mark_reg, temp_reg, O7);
3323 if (counters == NULL)
3324 counters = BiasedLocking::counters();
3325 }
3326
3327 Label cas_label;
3328
3329 // Biased locking
3330 // See whether the lock is currently biased toward our thread and
3331 // whether the epoch is still valid
3332 // Note that the runtime guarantees sufficient alignment of JavaThread
3333 // pointers to allow age to be placed into low bits
3334 assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits, "biased locking makes assumptions about bit layout");
3335 and3(mark_reg, markOopDesc::biased_lock_mask_in_place, temp_reg);
3336 cmp_and_brx_short(temp_reg, markOopDesc::biased_lock_pattern, Assembler::notEqual, Assembler::pn, cas_label);
3337
3338 load_klass(obj_reg, temp_reg);
3339 ld_ptr(Address(temp_reg, Klass::prototype_header_offset_in_bytes() + klassOopDesc::klass_part_offset_in_bytes()), temp_reg);
3340 or3(G2_thread, temp_reg, temp_reg);
3341 xor3(mark_reg, temp_reg, temp_reg);
3342 andcc(temp_reg, ~((int) markOopDesc::age_mask_in_place), temp_reg);
3343 if (counters != NULL) {
3344 cond_inc(Assembler::equal, (address) counters->biased_lock_entry_count_addr(), mark_reg, temp_reg);
3345 // Reload mark_reg as we may need it later
3346 ld_ptr(Address(obj_reg, oopDesc::mark_offset_in_bytes()), mark_reg);
3347 }
3348 brx(Assembler::equal, true, Assembler::pt, done);
3349 delayed()->nop();
3350
3351 Label try_revoke_bias;
3352 Label try_rebias;
3353 Address mark_addr = Address(obj_reg, oopDesc::mark_offset_in_bytes());
3354 assert(mark_addr.disp() == 0, "cas must take a zero displacement");
3355
3356 // At this point we know that the header has the bias pattern and
3357 // that we are not the bias owner in the current epoch. We need to
3358 // figure out more details about the state of the header in order to
3359 // know what operations can be legally performed on the object's
3396 if (counters != NULL) {
3397 cond_inc(Assembler::zero, (address) counters->anonymously_biased_lock_entry_count_addr(), mark_reg, temp_reg);
3398 }
3399 if (slow_case != NULL) {
3400 brx(Assembler::notEqual, true, Assembler::pn, *slow_case);
3401 delayed()->nop();
3402 }
3403 ba_short(done);
3404
3405 bind(try_rebias);
3406 // At this point we know the epoch has expired, meaning that the
3407 // current "bias owner", if any, is actually invalid. Under these
3408 // circumstances _only_, we are allowed to use the current header's
3409 // value as the comparison value when doing the cas to acquire the
3410 // bias in the current epoch. In other words, we allow transfer of
3411 // the bias from one thread to another directly in this situation.
3412 //
3413 // FIXME: due to a lack of registers we currently blow away the age
3414 // bits in this situation. Should attempt to preserve them.
3415 load_klass(obj_reg, temp_reg);
3416 ld_ptr(Address(temp_reg, Klass::prototype_header_offset_in_bytes() + klassOopDesc::klass_part_offset_in_bytes()), temp_reg);
3417 or3(G2_thread, temp_reg, temp_reg);
3418 casn(mark_addr.base(), mark_reg, temp_reg);
3419 // If the biasing toward our thread failed, this means that
3420 // another thread succeeded in biasing it toward itself and we
3421 // need to revoke that bias. The revocation will occur in the
3422 // interpreter runtime in the slow case.
3423 cmp(mark_reg, temp_reg);
3424 if (counters != NULL) {
3425 cond_inc(Assembler::zero, (address) counters->rebiased_lock_entry_count_addr(), mark_reg, temp_reg);
3426 }
3427 if (slow_case != NULL) {
3428 brx(Assembler::notEqual, true, Assembler::pn, *slow_case);
3429 delayed()->nop();
3430 }
3431 ba_short(done);
3432
3433 bind(try_revoke_bias);
3434 // The prototype mark in the klass doesn't have the bias bit set any
3435 // more, indicating that objects of this data type are not supposed
3436 // to be biased any more. We are going to try to reset the mark of
3437 // this object to the prototype value and fall through to the
3438 // CAS-based locking scheme. Note that if our CAS fails, it means
3439 // that another thread raced us for the privilege of revoking the
3440 // bias of this particular object, so it's okay to continue in the
3441 // normal locking code.
3442 //
3443 // FIXME: due to a lack of registers we currently blow away the age
3444 // bits in this situation. Should attempt to preserve them.
3445 load_klass(obj_reg, temp_reg);
3446 ld_ptr(Address(temp_reg, Klass::prototype_header_offset_in_bytes() + klassOopDesc::klass_part_offset_in_bytes()), temp_reg);
3447 casn(mark_addr.base(), mark_reg, temp_reg);
3448 // Fall through to the normal CAS-based lock, because no matter what
3449 // the result of the above CAS, some thread must have succeeded in
3450 // removing the bias bit from the object's header.
3451 if (counters != NULL) {
3452 cmp(mark_reg, temp_reg);
3453 cond_inc(Assembler::zero, (address) counters->revoked_lock_entry_count_addr(), mark_reg, temp_reg);
3454 }
3455
3456 bind(cas_label);
3457 }
3458
3459 void MacroAssembler::biased_locking_exit (Address mark_addr, Register temp_reg, Label& done,
3460 bool allow_delay_slot_filling) {
3461 // Check for biased locking unlock case, which is a no-op
3462 // Note: we do not have to check the thread ID for two reasons.
3463 // First, the interpreter checks for IllegalMonitorStateException at
3464 // a higher level. Second, if the bias was revoked while we held the
3465 // lock, the object could not be rebiased toward another thread, so
3466 // the bias bit would be clear.
|
3019
3020 // on success:
3021 restore();
3022 ba_short(L_success);
3023
3024 // on failure:
3025 bind(L_pop_to_failure);
3026 restore();
3027 bind(L_failure);
3028 }
3029
3030
3031 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
3032 Register super_klass,
3033 Register temp_reg,
3034 Register temp2_reg,
3035 Label* L_success,
3036 Label* L_failure,
3037 Label* L_slow_path,
3038 RegisterOrConstant super_check_offset) {
3039 int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
3040 int sco_offset = in_bytes(Klass::super_check_offset_offset());
3041
3042 bool must_load_sco = (super_check_offset.constant_or_zero() == -1);
3043 bool need_slow_path = (must_load_sco ||
3044 super_check_offset.constant_or_zero() == sco_offset);
3045
3046 assert_different_registers(sub_klass, super_klass, temp_reg);
3047 if (super_check_offset.is_register()) {
3048 assert_different_registers(sub_klass, super_klass, temp_reg,
3049 super_check_offset.as_register());
3050 } else if (must_load_sco) {
3051 assert(temp2_reg != noreg, "supply either a temp or a register offset");
3052 }
3053
3054 Label L_fallthrough;
3055 int label_nulls = 0;
3056 if (L_success == NULL) { L_success = &L_fallthrough; label_nulls++; }
3057 if (L_failure == NULL) { L_failure = &L_fallthrough; label_nulls++; }
3058 if (L_slow_path == NULL) { L_slow_path = &L_fallthrough; label_nulls++; }
3059 assert(label_nulls <= 1 ||
3060 (L_slow_path == &L_fallthrough && label_nulls <= 2 && !need_slow_path),
3140
3141
3142 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
3143 Register super_klass,
3144 Register count_temp,
3145 Register scan_temp,
3146 Register scratch_reg,
3147 Register coop_reg,
3148 Label* L_success,
3149 Label* L_failure) {
3150 assert_different_registers(sub_klass, super_klass,
3151 count_temp, scan_temp, scratch_reg, coop_reg);
3152
3153 Label L_fallthrough, L_loop;
3154 int label_nulls = 0;
3155 if (L_success == NULL) { L_success = &L_fallthrough; label_nulls++; }
3156 if (L_failure == NULL) { L_failure = &L_fallthrough; label_nulls++; }
3157 assert(label_nulls <= 1, "at most one NULL in the batch");
3158
3159 // a couple of useful fields in sub_klass:
3160 int ss_offset = in_bytes(Klass::secondary_supers_offset());
3161 int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
3162
3163 // Do a linear scan of the secondary super-klass chain.
3164 // This code is rarely used, so simplicity is a virtue here.
3165
3166 #ifndef PRODUCT
3167 int* pst_counter = &SharedRuntime::_partial_subtype_ctr;
3168 inc_counter((address) pst_counter, count_temp, scan_temp);
3169 #endif
3170
3171 // We will consult the secondary-super array.
3172 ld_ptr(sub_klass, ss_offset, scan_temp);
3173
3174 // Compress superclass if necessary.
3175 Register search_key = super_klass;
3176 bool decode_super_klass = false;
3177 if (UseCompressedOops) {
3178 if (coop_reg != noreg) {
3179 encode_heap_oop_not_null(super_klass, coop_reg);
3180 search_key = coop_reg;
3181 } else {
3315 assert(UseBiasedLocking, "why call this otherwise?");
3316
3317 if (PrintBiasedLockingStatistics) {
3318 assert_different_registers(obj_reg, mark_reg, temp_reg, O7);
3319 if (counters == NULL)
3320 counters = BiasedLocking::counters();
3321 }
3322
3323 Label cas_label;
3324
3325 // Biased locking
3326 // See whether the lock is currently biased toward our thread and
3327 // whether the epoch is still valid
3328 // Note that the runtime guarantees sufficient alignment of JavaThread
3329 // pointers to allow age to be placed into low bits
3330 assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits, "biased locking makes assumptions about bit layout");
3331 and3(mark_reg, markOopDesc::biased_lock_mask_in_place, temp_reg);
3332 cmp_and_brx_short(temp_reg, markOopDesc::biased_lock_pattern, Assembler::notEqual, Assembler::pn, cas_label);
3333
3334 load_klass(obj_reg, temp_reg);
3335 ld_ptr(Address(temp_reg, Klass::prototype_header_offset()), temp_reg);
3336 or3(G2_thread, temp_reg, temp_reg);
3337 xor3(mark_reg, temp_reg, temp_reg);
3338 andcc(temp_reg, ~((int) markOopDesc::age_mask_in_place), temp_reg);
3339 if (counters != NULL) {
3340 cond_inc(Assembler::equal, (address) counters->biased_lock_entry_count_addr(), mark_reg, temp_reg);
3341 // Reload mark_reg as we may need it later
3342 ld_ptr(Address(obj_reg, oopDesc::mark_offset_in_bytes()), mark_reg);
3343 }
3344 brx(Assembler::equal, true, Assembler::pt, done);
3345 delayed()->nop();
3346
3347 Label try_revoke_bias;
3348 Label try_rebias;
3349 Address mark_addr = Address(obj_reg, oopDesc::mark_offset_in_bytes());
3350 assert(mark_addr.disp() == 0, "cas must take a zero displacement");
3351
3352 // At this point we know that the header has the bias pattern and
3353 // that we are not the bias owner in the current epoch. We need to
3354 // figure out more details about the state of the header in order to
3355 // know what operations can be legally performed on the object's
3392 if (counters != NULL) {
3393 cond_inc(Assembler::zero, (address) counters->anonymously_biased_lock_entry_count_addr(), mark_reg, temp_reg);
3394 }
3395 if (slow_case != NULL) {
3396 brx(Assembler::notEqual, true, Assembler::pn, *slow_case);
3397 delayed()->nop();
3398 }
3399 ba_short(done);
3400
3401 bind(try_rebias);
3402 // At this point we know the epoch has expired, meaning that the
3403 // current "bias owner", if any, is actually invalid. Under these
3404 // circumstances _only_, we are allowed to use the current header's
3405 // value as the comparison value when doing the cas to acquire the
3406 // bias in the current epoch. In other words, we allow transfer of
3407 // the bias from one thread to another directly in this situation.
3408 //
3409 // FIXME: due to a lack of registers we currently blow away the age
3410 // bits in this situation. Should attempt to preserve them.
3411 load_klass(obj_reg, temp_reg);
3412 ld_ptr(Address(temp_reg, Klass::prototype_header_offset()), temp_reg);
3413 or3(G2_thread, temp_reg, temp_reg);
3414 casn(mark_addr.base(), mark_reg, temp_reg);
3415 // If the biasing toward our thread failed, this means that
3416 // another thread succeeded in biasing it toward itself and we
3417 // need to revoke that bias. The revocation will occur in the
3418 // interpreter runtime in the slow case.
3419 cmp(mark_reg, temp_reg);
3420 if (counters != NULL) {
3421 cond_inc(Assembler::zero, (address) counters->rebiased_lock_entry_count_addr(), mark_reg, temp_reg);
3422 }
3423 if (slow_case != NULL) {
3424 brx(Assembler::notEqual, true, Assembler::pn, *slow_case);
3425 delayed()->nop();
3426 }
3427 ba_short(done);
3428
3429 bind(try_revoke_bias);
3430 // The prototype mark in the klass doesn't have the bias bit set any
3431 // more, indicating that objects of this data type are not supposed
3432 // to be biased any more. We are going to try to reset the mark of
3433 // this object to the prototype value and fall through to the
3434 // CAS-based locking scheme. Note that if our CAS fails, it means
3435 // that another thread raced us for the privilege of revoking the
3436 // bias of this particular object, so it's okay to continue in the
3437 // normal locking code.
3438 //
3439 // FIXME: due to a lack of registers we currently blow away the age
3440 // bits in this situation. Should attempt to preserve them.
3441 load_klass(obj_reg, temp_reg);
3442 ld_ptr(Address(temp_reg, Klass::prototype_header_offset()), temp_reg);
3443 casn(mark_addr.base(), mark_reg, temp_reg);
3444 // Fall through to the normal CAS-based lock, because no matter what
3445 // the result of the above CAS, some thread must have succeeded in
3446 // removing the bias bit from the object's header.
3447 if (counters != NULL) {
3448 cmp(mark_reg, temp_reg);
3449 cond_inc(Assembler::zero, (address) counters->revoked_lock_entry_count_addr(), mark_reg, temp_reg);
3450 }
3451
3452 bind(cas_label);
3453 }
3454
3455 void MacroAssembler::biased_locking_exit (Address mark_addr, Register temp_reg, Label& done,
3456 bool allow_delay_slot_filling) {
3457 // Check for biased locking unlock case, which is a no-op
3458 // Note: we do not have to check the thread ID for two reasons.
3459 // First, the interpreter checks for IllegalMonitorStateException at
3460 // a higher level. Second, if the bias was revoked while we held the
3461 // lock, the object could not be rebiased toward another thread, so
3462 // the bias bit would be clear.
|