2920 // !defined(COMPILER2) is because of stupid core builds
2921 #if !defined(_LP64) || defined(COMPILER1) || !defined(COMPILER2) || INCLUDE_JVMCI
2922 void MacroAssembler::empty_FPU_stack() {
2923 if (VM_Version::supports_mmx()) {
2924 emms();
2925 } else {
2926 for (int i = 8; i-- > 0; ) ffree(i);
2927 }
2928 }
2929 #endif // !LP64 || C1 || !C2 || INCLUDE_JVMCI
2930
2931
2932 // Defines obj, preserves var_size_in_bytes
2933 void MacroAssembler::eden_allocate(Register obj,
2934 Register var_size_in_bytes,
2935 int con_size_in_bytes,
2936 Register t1,
2937 Label& slow_case) {
2938 assert(obj == rax, "obj must be in rax, for cmpxchg");
2939 assert_different_registers(obj, var_size_in_bytes, t1);
2940 if (!Universe::heap()->supports_inline_contig_alloc()) {
2941 jmp(slow_case);
2942 } else {
2943 Register end = t1;
2944 Label retry;
2945 bind(retry);
2946 ExternalAddress heap_top((address) Universe::heap()->top_addr());
2947 movptr(obj, heap_top);
2948 if (var_size_in_bytes == noreg) {
2949 lea(end, Address(obj, con_size_in_bytes));
2950 } else {
2951 lea(end, Address(obj, var_size_in_bytes, Address::times_1));
2952 }
2953 // if end < obj then we wrapped around => object too long => slow case
2954 cmpptr(end, obj);
2955 jcc(Assembler::below, slow_case);
2956 cmpptr(end, ExternalAddress((address) Universe::heap()->end_addr()));
2957 jcc(Assembler::above, slow_case);
2958 // Compare obj with the top addr, and if still equal, store the new top addr in
2959 // end at the address of the top addr pointer. Sets ZF if was equal, and clears
2960 // it otherwise. Use lock prefix for atomicity on MPs.
2961 locked_cmpxchgptr(end, heap_top);
2962 jcc(Assembler::notEqual, retry);
2963 }
2964 }
2965
2966 void MacroAssembler::enter() {
2967 push(rbp);
2968 mov(rbp, rsp);
2969 }
2970
2971 // A 5 byte nop that is safe for patching (see patch_verified_entry)
2972 void MacroAssembler::fat_nop() {
2973 if (UseAddressNop) {
2974 addr_nop_5();
2975 } else {
2976 emit_int8(0x26); // es:
5220 if(tosca_live) pop(rax);
5221
5222 bind(done);
5223 }
5224
5225 void MacroAssembler::g1_write_barrier_post(Register store_addr,
5226 Register new_val,
5227 Register thread,
5228 Register tmp,
5229 Register tmp2) {
5230 #ifdef _LP64
5231 assert(thread == r15_thread, "must be");
5232 #endif // _LP64
5233
5234 Address queue_index(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
5235 DirtyCardQueue::byte_offset_of_index()));
5236 Address buffer(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
5237 DirtyCardQueue::byte_offset_of_buf()));
5238
5239 CardTableModRefBS* ct =
5240 barrier_set_cast<CardTableModRefBS>(Universe::heap()->barrier_set());
5241 assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
5242
5243 Label done;
5244 Label runtime;
5245
5246 // Does store cross heap regions?
5247
5248 movptr(tmp, store_addr);
5249 xorptr(tmp, new_val);
5250 shrptr(tmp, HeapRegion::LogOfHRGrainBytes);
5251 jcc(Assembler::equal, done);
5252
5253 // crosses regions, storing NULL?
5254
5255 cmpptr(new_val, (int32_t) NULL_WORD);
5256 jcc(Assembler::equal, done);
5257
5258 // storing region crossing non-NULL, is card already dirty?
5259
5260 const Register card_addr = tmp;
5305 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, thread);
5306 pop(thread);
5307 #endif
5308 pop(new_val);
5309 pop(store_addr);
5310
5311 bind(done);
5312 }
5313
5314 #endif // INCLUDE_ALL_GCS
5315 //////////////////////////////////////////////////////////////////////////////////
5316
5317
5318 void MacroAssembler::store_check(Register obj, Address dst) {
5319 store_check(obj);
5320 }
5321
5322 void MacroAssembler::store_check(Register obj) {
5323 // Does a store check for the oop in register obj. The content of
5324 // register obj is destroyed afterwards.
5325 BarrierSet* bs = Universe::heap()->barrier_set();
5326 assert(bs->kind() == BarrierSet::CardTableForRS ||
5327 bs->kind() == BarrierSet::CardTableExtension,
5328 "Wrong barrier set kind");
5329
5330 CardTableModRefBS* ct = barrier_set_cast<CardTableModRefBS>(bs);
5331 assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
5332
5333 shrptr(obj, CardTableModRefBS::card_shift);
5334
5335 Address card_addr;
5336
5337 // The calculation for byte_map_base is as follows:
5338 // byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift);
5339 // So this essentially converts an address to a displacement and it will
5340 // never need to be relocated. On 64bit however the value may be too
5341 // large for a 32bit displacement.
5342 intptr_t disp = (intptr_t) ct->byte_map_base;
5343 if (is_simm32(disp)) {
5344 card_addr = Address(noreg, obj, Address::times_1, disp);
5345 } else {
5427 movptr(Address(thread, JavaThread::tlab_top_offset()), end);
5428
5429 // recover var_size_in_bytes if necessary
5430 if (var_size_in_bytes == end) {
5431 subptr(var_size_in_bytes, obj);
5432 }
5433 verify_tlab();
5434 }
5435
5436 // Preserves rbx, and rdx.
5437 Register MacroAssembler::tlab_refill(Label& retry,
5438 Label& try_eden,
5439 Label& slow_case) {
5440 Register top = rax;
5441 Register t1 = rcx; // object size
5442 Register t2 = rsi;
5443 Register thread_reg = NOT_LP64(rdi) LP64_ONLY(r15_thread);
5444 assert_different_registers(top, thread_reg, t1, t2, /* preserve: */ rbx, rdx);
5445 Label do_refill, discard_tlab;
5446
5447 if (!Universe::heap()->supports_inline_contig_alloc()) {
5448 // No allocation in the shared eden.
5449 jmp(slow_case);
5450 }
5451
5452 NOT_LP64(get_thread(thread_reg));
5453
5454 movptr(top, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
5455 movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())));
5456
5457 // calculate amount of free space
5458 subptr(t1, top);
5459 shrptr(t1, LogHeapWordSize);
5460
5461 // Retain tlab and allocate object in shared space if
5462 // the amount free in the tlab is too large to discard.
5463 cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_refill_waste_limit_offset())));
5464 jcc(Assembler::lessEqual, discard_tlab);
5465
5466 // Retain
5467 // %%% yuck as movptr...
6570 movl(dst, (int32_t)NULL_WORD);
6571 } else {
6572 movslq(dst, (int32_t)NULL_WORD);
6573 }
6574 #else
6575 movl(dst, (int32_t)NULL_WORD);
6576 #endif
6577 }
6578
6579 #ifdef _LP64
6580 void MacroAssembler::store_klass_gap(Register dst, Register src) {
6581 if (UseCompressedClassPointers) {
6582 // Store to klass gap in destination
6583 movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src);
6584 }
6585 }
6586
6587 #ifdef ASSERT
6588 void MacroAssembler::verify_heapbase(const char* msg) {
6589 assert (UseCompressedOops, "should be compressed");
6590 assert (Universe::heap() != NULL, "java heap should be initialized");
6591 if (CheckCompressedOops) {
6592 Label ok;
6593 push(rscratch1); // cmpptr trashes rscratch1
6594 cmpptr(r12_heapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr()));
6595 jcc(Assembler::equal, ok);
6596 STOP(msg);
6597 bind(ok);
6598 pop(rscratch1);
6599 }
6600 }
6601 #endif
6602
6603 // Algorithm must match oop.inline.hpp encode_heap_oop.
6604 void MacroAssembler::encode_heap_oop(Register r) {
6605 #ifdef ASSERT
6606 verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
6607 #endif
6608 verify_oop(r, "broken oop in encode_heap_oop");
6609 if (Universe::narrow_oop_base() == NULL) {
6610 if (Universe::narrow_oop_shift() != 0) {
6669 verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?");
6670 #endif
6671 if (Universe::narrow_oop_base() == NULL) {
6672 if (Universe::narrow_oop_shift() != 0) {
6673 assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6674 shlq(r, LogMinObjAlignmentInBytes);
6675 }
6676 } else {
6677 Label done;
6678 shlq(r, LogMinObjAlignmentInBytes);
6679 jccb(Assembler::equal, done);
6680 addq(r, r12_heapbase);
6681 bind(done);
6682 }
6683 verify_oop(r, "broken oop in decode_heap_oop");
6684 }
6685
6686 void MacroAssembler::decode_heap_oop_not_null(Register r) {
6687 // Note: it will change flags
6688 assert (UseCompressedOops, "should only be used for compressed headers");
6689 assert (Universe::heap() != NULL, "java heap should be initialized");
6690 // Cannot assert, unverified entry point counts instructions (see .ad file)
6691 // vtableStubs also counts instructions in pd_code_size_limit.
6692 // Also do not verify_oop as this is called by verify_oop.
6693 if (Universe::narrow_oop_shift() != 0) {
6694 assert(LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6695 shlq(r, LogMinObjAlignmentInBytes);
6696 if (Universe::narrow_oop_base() != NULL) {
6697 addq(r, r12_heapbase);
6698 }
6699 } else {
6700 assert (Universe::narrow_oop_base() == NULL, "sanity");
6701 }
6702 }
6703
6704 void MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
6705 // Note: it will change flags
6706 assert (UseCompressedOops, "should only be used for compressed headers");
6707 assert (Universe::heap() != NULL, "java heap should be initialized");
6708 // Cannot assert, unverified entry point counts instructions (see .ad file)
6709 // vtableStubs also counts instructions in pd_code_size_limit.
6710 // Also do not verify_oop as this is called by verify_oop.
6711 if (Universe::narrow_oop_shift() != 0) {
6712 assert(LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6713 if (LogMinObjAlignmentInBytes == Address::times_8) {
6714 leaq(dst, Address(r12_heapbase, src, Address::times_8, 0));
6715 } else {
6716 if (dst != src) {
6717 movq(dst, src);
6718 }
6719 shlq(dst, LogMinObjAlignmentInBytes);
6720 if (Universe::narrow_oop_base() != NULL) {
6721 addq(dst, r12_heapbase);
6722 }
6723 }
6724 } else {
6725 assert (Universe::narrow_oop_base() == NULL, "sanity");
6726 if (dst != src) {
6727 movq(dst, src);
6748 void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
6749 if (dst == src) {
6750 encode_klass_not_null(src);
6751 } else {
6752 if (Universe::narrow_klass_base() != NULL) {
6753 mov64(dst, (int64_t)Universe::narrow_klass_base());
6754 negq(dst);
6755 addq(dst, src);
6756 } else {
6757 movptr(dst, src);
6758 }
6759 if (Universe::narrow_klass_shift() != 0) {
6760 assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
6761 shrq(dst, LogKlassAlignmentInBytes);
6762 }
6763 }
6764 }
6765
6766 // Function instr_size_for_decode_klass_not_null() counts the instructions
6767 // generated by decode_klass_not_null(register r) and reinit_heapbase(),
6768 // when (Universe::heap() != NULL). Hence, if the instructions they
6769 // generate change, then this method needs to be updated.
6770 int MacroAssembler::instr_size_for_decode_klass_not_null() {
6771 assert (UseCompressedClassPointers, "only for compressed klass ptrs");
6772 if (Universe::narrow_klass_base() != NULL) {
6773 // mov64 + addq + shlq? + mov64 (for reinit_heapbase()).
6774 return (Universe::narrow_klass_shift() == 0 ? 20 : 24);
6775 } else {
6776 // longest load decode klass function, mov64, leaq
6777 return 16;
6778 }
6779 }
6780
6781 // !!! If the instructions that get generated here change then function
6782 // instr_size_for_decode_klass_not_null() needs to get updated.
6783 void MacroAssembler::decode_klass_not_null(Register r) {
6784 // Note: it will change flags
6785 assert (UseCompressedClassPointers, "should only be used for compressed headers");
6786 assert(r != r12_heapbase, "Decoding a klass in r12");
6787 // Cannot assert, unverified entry point counts instructions (see .ad file)
6788 // vtableStubs also counts instructions in pd_code_size_limit.
6804 assert (UseCompressedClassPointers, "should only be used for compressed headers");
6805 if (dst == src) {
6806 decode_klass_not_null(dst);
6807 } else {
6808 // Cannot assert, unverified entry point counts instructions (see .ad file)
6809 // vtableStubs also counts instructions in pd_code_size_limit.
6810 // Also do not verify_oop as this is called by verify_oop.
6811 mov64(dst, (int64_t)Universe::narrow_klass_base());
6812 if (Universe::narrow_klass_shift() != 0) {
6813 assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
6814 assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?");
6815 leaq(dst, Address(dst, src, Address::times_8, 0));
6816 } else {
6817 addq(dst, src);
6818 }
6819 }
6820 }
6821
6822 void MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
6823 assert (UseCompressedOops, "should only be used for compressed headers");
6824 assert (Universe::heap() != NULL, "java heap should be initialized");
6825 assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6826 int oop_index = oop_recorder()->find_index(obj);
6827 RelocationHolder rspec = oop_Relocation::spec(oop_index);
6828 mov_narrow_oop(dst, oop_index, rspec);
6829 }
6830
6831 void MacroAssembler::set_narrow_oop(Address dst, jobject obj) {
6832 assert (UseCompressedOops, "should only be used for compressed headers");
6833 assert (Universe::heap() != NULL, "java heap should be initialized");
6834 assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6835 int oop_index = oop_recorder()->find_index(obj);
6836 RelocationHolder rspec = oop_Relocation::spec(oop_index);
6837 mov_narrow_oop(dst, oop_index, rspec);
6838 }
6839
6840 void MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
6841 assert (UseCompressedClassPointers, "should only be used for compressed headers");
6842 assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6843 int klass_index = oop_recorder()->find_index(k);
6844 RelocationHolder rspec = metadata_Relocation::spec(klass_index);
6845 mov_narrow_oop(dst, Klass::encode_klass(k), rspec);
6846 }
6847
6848 void MacroAssembler::set_narrow_klass(Address dst, Klass* k) {
6849 assert (UseCompressedClassPointers, "should only be used for compressed headers");
6850 assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6851 int klass_index = oop_recorder()->find_index(k);
6852 RelocationHolder rspec = metadata_Relocation::spec(klass_index);
6853 mov_narrow_oop(dst, Klass::encode_klass(k), rspec);
6854 }
6855
6856 void MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) {
6857 assert (UseCompressedOops, "should only be used for compressed headers");
6858 assert (Universe::heap() != NULL, "java heap should be initialized");
6859 assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6860 int oop_index = oop_recorder()->find_index(obj);
6861 RelocationHolder rspec = oop_Relocation::spec(oop_index);
6862 Assembler::cmp_narrow_oop(dst, oop_index, rspec);
6863 }
6864
6865 void MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) {
6866 assert (UseCompressedOops, "should only be used for compressed headers");
6867 assert (Universe::heap() != NULL, "java heap should be initialized");
6868 assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6869 int oop_index = oop_recorder()->find_index(obj);
6870 RelocationHolder rspec = oop_Relocation::spec(oop_index);
6871 Assembler::cmp_narrow_oop(dst, oop_index, rspec);
6872 }
6873
6874 void MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) {
6875 assert (UseCompressedClassPointers, "should only be used for compressed headers");
6876 assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6877 int klass_index = oop_recorder()->find_index(k);
6878 RelocationHolder rspec = metadata_Relocation::spec(klass_index);
6879 Assembler::cmp_narrow_oop(dst, Klass::encode_klass(k), rspec);
6880 }
6881
6882 void MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) {
6883 assert (UseCompressedClassPointers, "should only be used for compressed headers");
6884 assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6885 int klass_index = oop_recorder()->find_index(k);
6886 RelocationHolder rspec = metadata_Relocation::spec(klass_index);
6887 Assembler::cmp_narrow_oop(dst, Klass::encode_klass(k), rspec);
6888 }
6889
6890 void MacroAssembler::reinit_heapbase() {
6891 if (UseCompressedOops || UseCompressedClassPointers) {
6892 if (Universe::heap() != NULL) {
6893 if (Universe::narrow_oop_base() == NULL) {
6894 MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
6895 } else {
6896 mov64(r12_heapbase, (int64_t)Universe::narrow_ptrs_base());
6897 }
6898 } else {
6899 movptr(r12_heapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr()));
6900 }
6901 }
6902 }
6903
6904 #endif // _LP64
6905
6906
6907 // C2 compiled method's prolog code.
6908 void MacroAssembler::verified_entry(int framesize, int stack_bang_size, bool fp_mode_24b) {
6909
6910 // WARNING: Initial instruction MUST be 5 bytes or longer so that
6911 // NativeJump::patch_verified_entry will be able to patch out the entry
6912 // code safely. The push to verify stack depth is ok at 5 bytes,
|
2920 // !defined(COMPILER2) is because of stupid core builds
2921 #if !defined(_LP64) || defined(COMPILER1) || !defined(COMPILER2) || INCLUDE_JVMCI
2922 void MacroAssembler::empty_FPU_stack() {
2923 if (VM_Version::supports_mmx()) {
2924 emms();
2925 } else {
2926 for (int i = 8; i-- > 0; ) ffree(i);
2927 }
2928 }
2929 #endif // !LP64 || C1 || !C2 || INCLUDE_JVMCI
2930
2931
2932 // Defines obj, preserves var_size_in_bytes
2933 void MacroAssembler::eden_allocate(Register obj,
2934 Register var_size_in_bytes,
2935 int con_size_in_bytes,
2936 Register t1,
2937 Label& slow_case) {
2938 assert(obj == rax, "obj must be in rax, for cmpxchg");
2939 assert_different_registers(obj, var_size_in_bytes, t1);
2940 if (!GC::gc()->heap()->supports_inline_contig_alloc()) {
2941 jmp(slow_case);
2942 } else {
2943 Register end = t1;
2944 Label retry;
2945 bind(retry);
2946 ExternalAddress heap_top((address) GC::gc()->heap()->top_addr());
2947 movptr(obj, heap_top);
2948 if (var_size_in_bytes == noreg) {
2949 lea(end, Address(obj, con_size_in_bytes));
2950 } else {
2951 lea(end, Address(obj, var_size_in_bytes, Address::times_1));
2952 }
2953 // if end < obj then we wrapped around => object too long => slow case
2954 cmpptr(end, obj);
2955 jcc(Assembler::below, slow_case);
2956 cmpptr(end, ExternalAddress((address) GC::gc()->heap()->end_addr()));
2957 jcc(Assembler::above, slow_case);
2958 // Compare obj with the top addr, and if still equal, store the new top addr in
2959 // end at the address of the top addr pointer. Sets ZF if was equal, and clears
2960 // it otherwise. Use lock prefix for atomicity on MPs.
2961 locked_cmpxchgptr(end, heap_top);
2962 jcc(Assembler::notEqual, retry);
2963 }
2964 }
2965
2966 void MacroAssembler::enter() {
2967 push(rbp);
2968 mov(rbp, rsp);
2969 }
2970
2971 // A 5 byte nop that is safe for patching (see patch_verified_entry)
2972 void MacroAssembler::fat_nop() {
2973 if (UseAddressNop) {
2974 addr_nop_5();
2975 } else {
2976 emit_int8(0x26); // es:
5220 if(tosca_live) pop(rax);
5221
5222 bind(done);
5223 }
5224
5225 void MacroAssembler::g1_write_barrier_post(Register store_addr,
5226 Register new_val,
5227 Register thread,
5228 Register tmp,
5229 Register tmp2) {
5230 #ifdef _LP64
5231 assert(thread == r15_thread, "must be");
5232 #endif // _LP64
5233
5234 Address queue_index(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
5235 DirtyCardQueue::byte_offset_of_index()));
5236 Address buffer(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
5237 DirtyCardQueue::byte_offset_of_buf()));
5238
5239 CardTableModRefBS* ct =
5240 barrier_set_cast<CardTableModRefBS>(GC::gc()->heap()->barrier_set());
5241 assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
5242
5243 Label done;
5244 Label runtime;
5245
5246 // Does store cross heap regions?
5247
5248 movptr(tmp, store_addr);
5249 xorptr(tmp, new_val);
5250 shrptr(tmp, HeapRegion::LogOfHRGrainBytes);
5251 jcc(Assembler::equal, done);
5252
5253 // crosses regions, storing NULL?
5254
5255 cmpptr(new_val, (int32_t) NULL_WORD);
5256 jcc(Assembler::equal, done);
5257
5258 // storing region crossing non-NULL, is card already dirty?
5259
5260 const Register card_addr = tmp;
5305 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, thread);
5306 pop(thread);
5307 #endif
5308 pop(new_val);
5309 pop(store_addr);
5310
5311 bind(done);
5312 }
5313
5314 #endif // INCLUDE_ALL_GCS
5315 //////////////////////////////////////////////////////////////////////////////////
5316
5317
5318 void MacroAssembler::store_check(Register obj, Address dst) {
5319 store_check(obj);
5320 }
5321
5322 void MacroAssembler::store_check(Register obj) {
5323 // Does a store check for the oop in register obj. The content of
5324 // register obj is destroyed afterwards.
5325 BarrierSet* bs = GC::gc()->heap()->barrier_set();
5326 assert(bs->kind() == BarrierSet::CardTableForRS ||
5327 bs->kind() == BarrierSet::CardTableExtension,
5328 "Wrong barrier set kind");
5329
5330 CardTableModRefBS* ct = barrier_set_cast<CardTableModRefBS>(bs);
5331 assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
5332
5333 shrptr(obj, CardTableModRefBS::card_shift);
5334
5335 Address card_addr;
5336
5337 // The calculation for byte_map_base is as follows:
5338 // byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift);
5339 // So this essentially converts an address to a displacement and it will
5340 // never need to be relocated. On 64bit however the value may be too
5341 // large for a 32bit displacement.
5342 intptr_t disp = (intptr_t) ct->byte_map_base;
5343 if (is_simm32(disp)) {
5344 card_addr = Address(noreg, obj, Address::times_1, disp);
5345 } else {
5427 movptr(Address(thread, JavaThread::tlab_top_offset()), end);
5428
5429 // recover var_size_in_bytes if necessary
5430 if (var_size_in_bytes == end) {
5431 subptr(var_size_in_bytes, obj);
5432 }
5433 verify_tlab();
5434 }
5435
5436 // Preserves rbx, and rdx.
5437 Register MacroAssembler::tlab_refill(Label& retry,
5438 Label& try_eden,
5439 Label& slow_case) {
5440 Register top = rax;
5441 Register t1 = rcx; // object size
5442 Register t2 = rsi;
5443 Register thread_reg = NOT_LP64(rdi) LP64_ONLY(r15_thread);
5444 assert_different_registers(top, thread_reg, t1, t2, /* preserve: */ rbx, rdx);
5445 Label do_refill, discard_tlab;
5446
5447 if (!GC::gc()->heap()->supports_inline_contig_alloc()) {
5448 // No allocation in the shared eden.
5449 jmp(slow_case);
5450 }
5451
5452 NOT_LP64(get_thread(thread_reg));
5453
5454 movptr(top, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
5455 movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())));
5456
5457 // calculate amount of free space
5458 subptr(t1, top);
5459 shrptr(t1, LogHeapWordSize);
5460
5461 // Retain tlab and allocate object in shared space if
5462 // the amount free in the tlab is too large to discard.
5463 cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_refill_waste_limit_offset())));
5464 jcc(Assembler::lessEqual, discard_tlab);
5465
5466 // Retain
5467 // %%% yuck as movptr...
6570 movl(dst, (int32_t)NULL_WORD);
6571 } else {
6572 movslq(dst, (int32_t)NULL_WORD);
6573 }
6574 #else
6575 movl(dst, (int32_t)NULL_WORD);
6576 #endif
6577 }
6578
6579 #ifdef _LP64
6580 void MacroAssembler::store_klass_gap(Register dst, Register src) {
6581 if (UseCompressedClassPointers) {
6582 // Store to klass gap in destination
6583 movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src);
6584 }
6585 }
6586
6587 #ifdef ASSERT
6588 void MacroAssembler::verify_heapbase(const char* msg) {
6589 assert (UseCompressedOops, "should be compressed");
6590 assert (GC::gc()->heap() != NULL, "java heap should be initialized");
6591 if (CheckCompressedOops) {
6592 Label ok;
6593 push(rscratch1); // cmpptr trashes rscratch1
6594 cmpptr(r12_heapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr()));
6595 jcc(Assembler::equal, ok);
6596 STOP(msg);
6597 bind(ok);
6598 pop(rscratch1);
6599 }
6600 }
6601 #endif
6602
6603 // Algorithm must match oop.inline.hpp encode_heap_oop.
6604 void MacroAssembler::encode_heap_oop(Register r) {
6605 #ifdef ASSERT
6606 verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
6607 #endif
6608 verify_oop(r, "broken oop in encode_heap_oop");
6609 if (Universe::narrow_oop_base() == NULL) {
6610 if (Universe::narrow_oop_shift() != 0) {
6669 verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?");
6670 #endif
6671 if (Universe::narrow_oop_base() == NULL) {
6672 if (Universe::narrow_oop_shift() != 0) {
6673 assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6674 shlq(r, LogMinObjAlignmentInBytes);
6675 }
6676 } else {
6677 Label done;
6678 shlq(r, LogMinObjAlignmentInBytes);
6679 jccb(Assembler::equal, done);
6680 addq(r, r12_heapbase);
6681 bind(done);
6682 }
6683 verify_oop(r, "broken oop in decode_heap_oop");
6684 }
6685
6686 void MacroAssembler::decode_heap_oop_not_null(Register r) {
6687 // Note: it will change flags
6688 assert (UseCompressedOops, "should only be used for compressed headers");
6689 assert (GC::gc()->heap() != NULL, "java heap should be initialized");
6690 // Cannot assert, unverified entry point counts instructions (see .ad file)
6691 // vtableStubs also counts instructions in pd_code_size_limit.
6692 // Also do not verify_oop as this is called by verify_oop.
6693 if (Universe::narrow_oop_shift() != 0) {
6694 assert(LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6695 shlq(r, LogMinObjAlignmentInBytes);
6696 if (Universe::narrow_oop_base() != NULL) {
6697 addq(r, r12_heapbase);
6698 }
6699 } else {
6700 assert (Universe::narrow_oop_base() == NULL, "sanity");
6701 }
6702 }
6703
6704 void MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
6705 // Note: it will change flags
6706 assert (UseCompressedOops, "should only be used for compressed headers");
6707 assert (GC::gc()->heap() != NULL, "java heap should be initialized");
6708 // Cannot assert, unverified entry point counts instructions (see .ad file)
6709 // vtableStubs also counts instructions in pd_code_size_limit.
6710 // Also do not verify_oop as this is called by verify_oop.
6711 if (Universe::narrow_oop_shift() != 0) {
6712 assert(LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6713 if (LogMinObjAlignmentInBytes == Address::times_8) {
6714 leaq(dst, Address(r12_heapbase, src, Address::times_8, 0));
6715 } else {
6716 if (dst != src) {
6717 movq(dst, src);
6718 }
6719 shlq(dst, LogMinObjAlignmentInBytes);
6720 if (Universe::narrow_oop_base() != NULL) {
6721 addq(dst, r12_heapbase);
6722 }
6723 }
6724 } else {
6725 assert (Universe::narrow_oop_base() == NULL, "sanity");
6726 if (dst != src) {
6727 movq(dst, src);
6748 void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
6749 if (dst == src) {
6750 encode_klass_not_null(src);
6751 } else {
6752 if (Universe::narrow_klass_base() != NULL) {
6753 mov64(dst, (int64_t)Universe::narrow_klass_base());
6754 negq(dst);
6755 addq(dst, src);
6756 } else {
6757 movptr(dst, src);
6758 }
6759 if (Universe::narrow_klass_shift() != 0) {
6760 assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
6761 shrq(dst, LogKlassAlignmentInBytes);
6762 }
6763 }
6764 }
6765
6766 // Function instr_size_for_decode_klass_not_null() counts the instructions
6767 // generated by decode_klass_not_null(register r) and reinit_heapbase(),
6768 // when (GC::gc()->heap() != NULL). Hence, if the instructions they
6769 // generate change, then this method needs to be updated.
6770 int MacroAssembler::instr_size_for_decode_klass_not_null() {
6771 assert (UseCompressedClassPointers, "only for compressed klass ptrs");
6772 if (Universe::narrow_klass_base() != NULL) {
6773 // mov64 + addq + shlq? + mov64 (for reinit_heapbase()).
6774 return (Universe::narrow_klass_shift() == 0 ? 20 : 24);
6775 } else {
6776 // longest load decode klass function, mov64, leaq
6777 return 16;
6778 }
6779 }
6780
6781 // !!! If the instructions that get generated here change then function
6782 // instr_size_for_decode_klass_not_null() needs to get updated.
6783 void MacroAssembler::decode_klass_not_null(Register r) {
6784 // Note: it will change flags
6785 assert (UseCompressedClassPointers, "should only be used for compressed headers");
6786 assert(r != r12_heapbase, "Decoding a klass in r12");
6787 // Cannot assert, unverified entry point counts instructions (see .ad file)
6788 // vtableStubs also counts instructions in pd_code_size_limit.
6804 assert (UseCompressedClassPointers, "should only be used for compressed headers");
6805 if (dst == src) {
6806 decode_klass_not_null(dst);
6807 } else {
6808 // Cannot assert, unverified entry point counts instructions (see .ad file)
6809 // vtableStubs also counts instructions in pd_code_size_limit.
6810 // Also do not verify_oop as this is called by verify_oop.
6811 mov64(dst, (int64_t)Universe::narrow_klass_base());
6812 if (Universe::narrow_klass_shift() != 0) {
6813 assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
6814 assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?");
6815 leaq(dst, Address(dst, src, Address::times_8, 0));
6816 } else {
6817 addq(dst, src);
6818 }
6819 }
6820 }
6821
6822 void MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
6823 assert (UseCompressedOops, "should only be used for compressed headers");
6824 assert (GC::gc()->heap() != NULL, "java heap should be initialized");
6825 assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6826 int oop_index = oop_recorder()->find_index(obj);
6827 RelocationHolder rspec = oop_Relocation::spec(oop_index);
6828 mov_narrow_oop(dst, oop_index, rspec);
6829 }
6830
6831 void MacroAssembler::set_narrow_oop(Address dst, jobject obj) {
6832 assert (UseCompressedOops, "should only be used for compressed headers");
6833 assert (GC::gc()->heap() != NULL, "java heap should be initialized");
6834 assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6835 int oop_index = oop_recorder()->find_index(obj);
6836 RelocationHolder rspec = oop_Relocation::spec(oop_index);
6837 mov_narrow_oop(dst, oop_index, rspec);
6838 }
6839
6840 void MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
6841 assert (UseCompressedClassPointers, "should only be used for compressed headers");
6842 assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6843 int klass_index = oop_recorder()->find_index(k);
6844 RelocationHolder rspec = metadata_Relocation::spec(klass_index);
6845 mov_narrow_oop(dst, Klass::encode_klass(k), rspec);
6846 }
6847
6848 void MacroAssembler::set_narrow_klass(Address dst, Klass* k) {
6849 assert (UseCompressedClassPointers, "should only be used for compressed headers");
6850 assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6851 int klass_index = oop_recorder()->find_index(k);
6852 RelocationHolder rspec = metadata_Relocation::spec(klass_index);
6853 mov_narrow_oop(dst, Klass::encode_klass(k), rspec);
6854 }
6855
6856 void MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) {
6857 assert (UseCompressedOops, "should only be used for compressed headers");
6858 assert (GC::gc()->heap() != NULL, "java heap should be initialized");
6859 assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6860 int oop_index = oop_recorder()->find_index(obj);
6861 RelocationHolder rspec = oop_Relocation::spec(oop_index);
6862 Assembler::cmp_narrow_oop(dst, oop_index, rspec);
6863 }
6864
6865 void MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) {
6866 assert (UseCompressedOops, "should only be used for compressed headers");
6867 assert (GC::gc()->heap() != NULL, "java heap should be initialized");
6868 assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6869 int oop_index = oop_recorder()->find_index(obj);
6870 RelocationHolder rspec = oop_Relocation::spec(oop_index);
6871 Assembler::cmp_narrow_oop(dst, oop_index, rspec);
6872 }
6873
6874 void MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) {
6875 assert (UseCompressedClassPointers, "should only be used for compressed headers");
6876 assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6877 int klass_index = oop_recorder()->find_index(k);
6878 RelocationHolder rspec = metadata_Relocation::spec(klass_index);
6879 Assembler::cmp_narrow_oop(dst, Klass::encode_klass(k), rspec);
6880 }
6881
6882 void MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) {
6883 assert (UseCompressedClassPointers, "should only be used for compressed headers");
6884 assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6885 int klass_index = oop_recorder()->find_index(k);
6886 RelocationHolder rspec = metadata_Relocation::spec(klass_index);
6887 Assembler::cmp_narrow_oop(dst, Klass::encode_klass(k), rspec);
6888 }
6889
6890 void MacroAssembler::reinit_heapbase() {
6891 if (UseCompressedOops || UseCompressedClassPointers) {
6892 if (GC::is_initialized()) {
6893 if (Universe::narrow_oop_base() == NULL) {
6894 MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
6895 } else {
6896 mov64(r12_heapbase, (int64_t)Universe::narrow_ptrs_base());
6897 }
6898 } else {
6899 movptr(r12_heapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr()));
6900 }
6901 }
6902 }
6903
6904 #endif // _LP64
6905
6906
6907 // C2 compiled method's prolog code.
6908 void MacroAssembler::verified_entry(int framesize, int stack_bang_size, bool fp_mode_24b) {
6909
6910 // WARNING: Initial instruction MUST be 5 bytes or longer so that
6911 // NativeJump::patch_verified_entry will be able to patch out the entry
6912 // code safely. The push to verify stack depth is ok at 5 bytes,
|