80 } else if (Instruction_aarch64::extract(insn, 30, 25) == 0b011011) {
81 // Test & branch (immediate)
82 Instruction_aarch64::spatch(branch, 18, 5, offset);
83 } else if (Instruction_aarch64::extract(insn, 28, 24) == 0b10000) {
84 // PC-rel. addressing
85 offset = target-branch;
86 int shift = Instruction_aarch64::extract(insn, 31, 31);
87 if (shift) {
88 u_int64_t dest = (u_int64_t)target;
89 uint64_t pc_page = (uint64_t)branch >> 12;
90 uint64_t adr_page = (uint64_t)target >> 12;
91 unsigned offset_lo = dest & 0xfff;
92 offset = adr_page - pc_page;
93
94 // We handle 3 types of PC relative addressing
95 // 1 - adrp Rx, target_page
96 // ldr/str Ry, [Rx, #offset_in_page]
97 // 2 - adrp Rx, target_page
98 // add Ry, Rx, #offset_in_page
99 // 3 - adrp Rx, target_page (page aligned reloc, offset == 0)
100 // In the first 2 cases we must check that Rx is the same in the adrp and the
101 // subsequent ldr/str or add instruction. Otherwise we could accidentally end
102 // up treating a type 3 relocation as a type 1 or 2 just because it happened
103 // to be followed by a random unrelated ldr/str or add instruction.
104 //
105 // In the case of a type 3 relocation, we know that these are only generated
106 // for the safepoint polling page, or for the card type byte map base so we
107 // assert as much and of course that the offset is 0.
108 //
109 unsigned insn2 = ((unsigned*)branch)[1];
110 if (Instruction_aarch64::extract(insn2, 29, 24) == 0b111001 &&
111 Instruction_aarch64::extract(insn, 4, 0) ==
112 Instruction_aarch64::extract(insn2, 9, 5)) {
113 // Load/store register (unsigned immediate)
114 unsigned size = Instruction_aarch64::extract(insn2, 31, 30);
115 Instruction_aarch64::patch(branch + sizeof (unsigned),
116 21, 10, offset_lo >> size);
117 guarantee(((dest >> size) << size) == dest, "misaligned target");
118 instructions = 2;
119 } else if (Instruction_aarch64::extract(insn2, 31, 22) == 0b1001000100 &&
120 Instruction_aarch64::extract(insn, 4, 0) ==
121 Instruction_aarch64::extract(insn2, 4, 0)) {
122 // add (immediate)
123 Instruction_aarch64::patch(branch + sizeof (unsigned),
124 21, 10, offset_lo);
125 instructions = 2;
126 } else {
127 assert((jbyte *)target ==
128 ((CardTableModRefBS*)(Universe::heap()->barrier_set()))->byte_map_base ||
129 target == StubRoutines::crc_table_addr() ||
130 (address)target == os::get_polling_page(),
131 "adrp must be polling page or byte map base");
132 assert(offset_lo == 0, "offset must be 0 for polling page or byte map base");
133 }
134 }
135 int offset_lo = offset & 3;
136 offset >>= 2;
137 Instruction_aarch64::spatch(branch, 23, 5, offset);
138 Instruction_aarch64::patch(branch, 30, 29, offset_lo);
139 } else if (Instruction_aarch64::extract(insn, 31, 21) == 0b11010010100) {
140 u_int64_t dest = (u_int64_t)target;
141 // Move wide constant
142 assert(nativeInstruction_at(branch+4)->is_movk(), "wrong insns in patch");
143 assert(nativeInstruction_at(branch+8)->is_movk(), "wrong insns in patch");
144 Instruction_aarch64::patch(branch, 20, 5, dest & 0xffff);
145 Instruction_aarch64::patch(branch+4, 20, 5, (dest >>= 16) & 0xffff);
146 Instruction_aarch64::patch(branch+8, 20, 5, (dest >>= 16) & 0xffff);
147 assert(target_addr_for_insn(branch) == target, "should be");
148 instructions = 3;
149 } else if (Instruction_aarch64::extract(insn, 31, 22) == 0b1011100101 &&
150 Instruction_aarch64::extract(insn, 4, 0) == 0b11111) {
151 // nothing to do
152 assert(target == 0, "did not expect to relocate target for polling page load");
198 // Compare & branch (immediate)
199 offset = Instruction_aarch64::sextract(insn, 23, 5);
200 } else if (Instruction_aarch64::extract(insn, 30, 25) == 0b011011) {
201 // Test & branch (immediate)
202 offset = Instruction_aarch64::sextract(insn, 18, 5);
203 } else if (Instruction_aarch64::extract(insn, 28, 24) == 0b10000) {
204 // PC-rel. addressing
205 offset = Instruction_aarch64::extract(insn, 30, 29);
206 offset |= Instruction_aarch64::sextract(insn, 23, 5) << 2;
207 int shift = Instruction_aarch64::extract(insn, 31, 31) ? 12 : 0;
208 if (shift) {
209 offset <<= shift;
210 uint64_t target_page = ((uint64_t)insn_addr) + offset;
211 target_page &= ((uint64_t)-1) << shift;
212 // Return the target address for the following sequences
213 // 1 - adrp Rx, target_page
214 // ldr/str Ry, [Rx, #offset_in_page]
215 // 2 - adrp Rx, target_page ]
216 // add Ry, Rx, #offset_in_page
217 // 3 - adrp Rx, target_page (page aligned reloc, offset == 0)
218 //
219 // In the first two cases we check that the register is the same and
220 // return the target_page + the offset within the page.
221 // Otherwise we assume it is a page aligned relocation and return
222 // the target page only. The only cases this is generated is for
223 // the safepoint polling page or for the card table byte map base so
224 // we assert as much.
225 //
226 unsigned insn2 = ((unsigned*)insn_addr)[1];
227 if (Instruction_aarch64::extract(insn2, 29, 24) == 0b111001 &&
228 Instruction_aarch64::extract(insn, 4, 0) ==
229 Instruction_aarch64::extract(insn2, 9, 5)) {
230 // Load/store register (unsigned immediate)
231 unsigned int byte_offset = Instruction_aarch64::extract(insn2, 21, 10);
232 unsigned int size = Instruction_aarch64::extract(insn2, 31, 30);
233 return address(target_page + (byte_offset << size));
234 } else if (Instruction_aarch64::extract(insn2, 31, 22) == 0b1001000100 &&
235 Instruction_aarch64::extract(insn, 4, 0) ==
236 Instruction_aarch64::extract(insn2, 4, 0)) {
237 // add (immediate)
238 unsigned int byte_offset = Instruction_aarch64::extract(insn2, 21, 10);
239 return address(target_page + byte_offset);
240 } else {
241 assert((jbyte *)target_page ==
242 ((CardTableModRefBS*)(Universe::heap()->barrier_set()))->byte_map_base ||
243 (address)target_page == os::get_polling_page(),
244 "adrp must be polling page or byte map base");
245 return (address)target_page;
246 }
247 } else {
248 ShouldNotReachHere();
249 }
250 } else if (Instruction_aarch64::extract(insn, 31, 23) == 0b110100101) {
251 u_int32_t *insns = (u_int32_t *)insn_addr;
252 // Move wide constant: movz, movk, movk. See movptr().
253 assert(nativeInstruction_at(insns+1)->is_movk(), "wrong insns in patch");
254 assert(nativeInstruction_at(insns+2)->is_movk(), "wrong insns in patch");
255 return address(u_int64_t(Instruction_aarch64::extract(insns[0], 20, 5))
256 + (u_int64_t(Instruction_aarch64::extract(insns[1], 20, 5)) << 16)
257 + (u_int64_t(Instruction_aarch64::extract(insns[2], 20, 5)) << 32));
258 } else if (Instruction_aarch64::extract(insn, 31, 22) == 0b1011100101 &&
259 Instruction_aarch64::extract(insn, 4, 0) == 0b11111) {
260 return 0;
261 } else {
262 ShouldNotReachHere();
263 }
264 return address(((uint64_t)insn_addr + (offset << 2)));
265 }
3056 void MacroAssembler::addptr(const Address &dst, int32_t src) {
3057 Address adr;
3058 switch(dst.getMode()) {
3059 case Address::base_plus_offset:
3060 // This is the expected mode, although we allow all the other
3061 // forms below.
3062 adr = form_address(rscratch2, dst.base(), dst.offset(), LogBytesPerWord);
3063 break;
3064 default:
3065 lea(rscratch2, dst);
3066 adr = Address(rscratch2);
3067 break;
3068 }
3069 ldr(rscratch1, adr);
3070 add(rscratch1, rscratch1, src);
3071 str(rscratch1, adr);
3072 }
3073
3074 void MacroAssembler::cmpptr(Register src1, Address src2) {
3075 unsigned long offset;
3076 adrp(rscratch1, src2, offset);
3077 ldr(rscratch1, Address(rscratch1, offset));
3078 cmp(src1, rscratch1);
3079 }
3080
3081 void MacroAssembler::store_check(Register obj, Address dst) {
3082 store_check(obj);
3083 }
3084
3085 void MacroAssembler::store_check(Register obj) {
3086 // Does a store check for the oop in register obj. The content of
3087 // register obj is destroyed afterwards.
3088
3089 BarrierSet* bs = Universe::heap()->barrier_set();
3090 assert(bs->kind() == BarrierSet::CardTableForRS ||
3091 bs->kind() == BarrierSet::CardTableExtension,
3092 "Wrong barrier set kind");
3093
3094 CardTableModRefBS* ct = barrier_set_cast<CardTableModRefBS>(bs);
3095 assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
3096
3097 lsr(obj, obj, CardTableModRefBS::card_shift);
3098
3099 assert(CardTableModRefBS::dirty_card_val() == 0, "must be");
3100
3101 {
3102 ExternalAddress cardtable((address) ct->byte_map_base);
3103 unsigned long offset;
3104 adrp(rscratch1, cardtable, offset);
3105 assert(offset == 0, "byte_map_base is misaligned");
3106 }
3107
3108 if (UseCondCardMark) {
3109 Label L_already_dirty;
3110 membar(StoreLoad);
3111 ldrb(rscratch2, Address(obj, rscratch1));
3112 cbz(rscratch2, L_already_dirty);
3113 strb(zr, Address(obj, rscratch1));
3114 bind(L_already_dirty);
3115 } else {
3116 if (UseConcMarkSweepGC && CMSPrecleaningEnabled) {
3117 membar(StoreStore);
3118 }
3119 strb(zr, Address(obj, rscratch1));
3120 }
3121 }
3122
3123 void MacroAssembler::load_klass(Register dst, Register src) {
3124 if (UseCompressedClassPointers) {
3579
3580 // Does store cross heap regions?
3581
3582 eor(tmp, store_addr, new_val);
3583 lsr(tmp, tmp, HeapRegion::LogOfHRGrainBytes);
3584 cbz(tmp, done);
3585
3586 // crosses regions, storing NULL?
3587
3588 cbz(new_val, done);
3589
3590 // storing region crossing non-NULL, is card already dirty?
3591
3592 ExternalAddress cardtable((address) ct->byte_map_base);
3593 assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
3594 const Register card_addr = tmp;
3595
3596 lsr(card_addr, store_addr, CardTableModRefBS::card_shift);
3597
3598 unsigned long offset;
3599 adrp(tmp2, cardtable, offset);
3600
3601 // get the address of the card
3602 add(card_addr, card_addr, tmp2);
3603 ldrb(tmp2, Address(card_addr, offset));
3604 cmpw(tmp2, (int)G1SATBCardTableModRefBS::g1_young_card_val());
3605 br(Assembler::EQ, done);
3606
3607 assert((int)CardTableModRefBS::dirty_card_val() == 0, "must be 0");
3608
3609 membar(Assembler::StoreLoad);
3610
3611 ldrb(tmp2, Address(card_addr, offset));
3612 cbzw(tmp2, done);
3613
3614 // storing a region crossing, non-NULL oop, card is clean.
3615 // dirty card and log.
3616
3617 strb(zr, Address(card_addr, offset));
3618
3619 ldr(rscratch1, queue_index);
3766 addmw(Address(rthread, in_bytes(JavaThread::tlab_fast_refill_waste_offset())), t1,
3767 rscratch1);
3768 }
3769
3770 // if tlab is currently allocated (top or end != null) then
3771 // fill [top, end + alignment_reserve) with array object
3772 cbz(top, do_refill);
3773
3774 // set up the mark word
3775 mov(rscratch1, (intptr_t)markOopDesc::prototype()->copy_set_hash(0x2));
3776 str(rscratch1, Address(top, oopDesc::mark_offset_in_bytes()));
3777 // set the length to the remaining space
3778 sub(t1, t1, typeArrayOopDesc::header_size(T_INT));
3779 add(t1, t1, (int32_t)ThreadLocalAllocBuffer::alignment_reserve());
3780 lsl(t1, t1, log2_intptr(HeapWordSize/sizeof(jint)));
3781 strw(t1, Address(top, arrayOopDesc::length_offset_in_bytes()));
3782 // set klass to intArrayKlass
3783 {
3784 unsigned long offset;
3785 // dubious reloc why not an oop reloc?
3786 adrp(rscratch1, ExternalAddress((address)Universe::intArrayKlassObj_addr()),
3787 offset);
3788 ldr(t1, Address(rscratch1, offset));
3789 }
3790 // store klass last. concurrent gcs assumes klass length is valid if
3791 // klass field is not null.
3792 store_klass(top, t1);
3793
3794 mov(t1, top);
3795 ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_start_offset())));
3796 sub(t1, t1, rscratch1);
3797 incr_allocated_bytes(rthread, t1, 0, rscratch1);
3798
3799 // refill the tlab with an eden allocation
3800 bind(do_refill);
3801 ldr(t1, Address(rthread, in_bytes(JavaThread::tlab_size_offset())));
3802 lsl(t1, t1, LogHeapWordSize);
3803 // allocate new tlab, address returned in top
3804 eden_allocate(top, t1, 0, t2, slow_case);
3805
3806 // Check that t1 was preserved in eden_allocate.
3831
3832 return rthread; // for use by caller
3833 }
3834
3835 // Defines obj, preserves var_size_in_bytes
3836 void MacroAssembler::eden_allocate(Register obj,
3837 Register var_size_in_bytes,
3838 int con_size_in_bytes,
3839 Register t1,
3840 Label& slow_case) {
3841 assert_different_registers(obj, var_size_in_bytes, t1);
3842 if (!Universe::heap()->supports_inline_contig_alloc()) {
3843 b(slow_case);
3844 } else {
3845 Register end = t1;
3846 Register heap_end = rscratch2;
3847 Label retry;
3848 bind(retry);
3849 {
3850 unsigned long offset;
3851 adrp(rscratch1, ExternalAddress((address) Universe::heap()->end_addr()), offset);
3852 ldr(heap_end, Address(rscratch1, offset));
3853 }
3854
3855 ExternalAddress heap_top((address) Universe::heap()->top_addr());
3856
3857 // Get the current top of the heap
3858 {
3859 unsigned long offset;
3860 adrp(rscratch1, heap_top, offset);
3861 // Use add() here after ARDP, rather than lea().
3862 // lea() does not generate anything if its offset is zero.
3863 // However, relocs expect to find either an ADD or a load/store
3864 // insn after an ADRP. add() always generates an ADD insn, even
3865 // for add(Rn, Rn, 0).
3866 add(rscratch1, rscratch1, offset);
3867 ldaxr(obj, rscratch1);
3868 }
3869
3870 // Adjust it my the size of our new object
3871 if (var_size_in_bytes == noreg) {
3872 lea(end, Address(obj, con_size_in_bytes));
3873 } else {
3874 lea(end, Address(obj, var_size_in_bytes));
3875 }
3876
3877 // if end < obj then we wrapped around high memory
3878 cmp(end, obj);
3879 br(Assembler::LO, slow_case);
3880
3931 str(size, Address(tmp));
3932 br(Assembler::GT, loop);
3933
3934 // Bang down shadow pages too.
3935 // At this point, (tmp-0) is the last address touched, so don't
3936 // touch it again. (It was touched as (tmp-pagesize) but then tmp
3937 // was post-decremented.) Skip this address by starting at i=1, and
3938 // touch a few more pages below. N.B. It is important to touch all
3939 // the way down to and including i=StackShadowPages.
3940 for (int i = 0; i< StackShadowPages-1; i++) {
3941 // this could be any sized move but this is can be a debugging crumb
3942 // so the bigger the better.
3943 lea(tmp, Address(tmp, -os::vm_page_size()));
3944 str(size, Address(tmp));
3945 }
3946 }
3947
3948
3949 address MacroAssembler::read_polling_page(Register r, address page, relocInfo::relocType rtype) {
3950 unsigned long off;
3951 adrp(r, Address(page, rtype), off);
3952 InstructionMark im(this);
3953 code_section()->relocate(inst_mark(), rtype);
3954 ldrw(zr, Address(r, off));
3955 return inst_mark();
3956 }
3957
3958 address MacroAssembler::read_polling_page(Register r, relocInfo::relocType rtype) {
3959 InstructionMark im(this);
3960 code_section()->relocate(inst_mark(), rtype);
3961 ldrw(zr, Address(r, 0));
3962 return inst_mark();
3963 }
3964
3965 void MacroAssembler::adrp(Register reg1, const Address &dest, unsigned long &byte_offset) {
3966 relocInfo::relocType rtype = dest.rspec().reloc()->type();
3967 if (uabs(pc() - dest.target()) >= (1LL << 32)) {
3968 guarantee(rtype == relocInfo::none
3969 || rtype == relocInfo::external_word_type
3970 || rtype == relocInfo::poll_type
3971 || rtype == relocInfo::poll_return_type,
3972 "can only use a fixed address with an ADRP");
3973 // Out of range. This doesn't happen very often, but we have to
3974 // handle it
3975 mov(reg1, dest);
3976 byte_offset = 0;
3977 } else {
3978 InstructionMark im(this);
3979 code_section()->relocate(inst_mark(), dest.rspec());
3980 byte_offset = (uint64_t)dest.target() & 0xfff;
3981 _adrp(reg1, dest.target());
3982 }
3983 }
3984
3985 void MacroAssembler::build_frame(int framesize) {
3986 assert(framesize > 0, "framesize must be > 0");
3987 if (framesize < ((1 << 9) + 2 * wordSize)) {
3988 sub(sp, sp, framesize);
3989 stp(rfp, lr, Address(sp, framesize - 2 * wordSize));
3990 if (PreserveFramePointer) add(rfp, sp, framesize - 2 * wordSize);
3991 } else {
3992 stp(rfp, lr, Address(pre(sp, -2 * wordSize)));
3993 if (PreserveFramePointer) mov(rfp, sp);
3994 if (framesize < ((1 << 12) + 2 * wordSize))
3995 sub(sp, sp, framesize - 2 * wordSize);
3996 else {
3997 mov(rscratch1, framesize - 2 * wordSize);
3998 sub(sp, sp, rscratch1);
3999 }
4000 }
4001 }
4002
|
80 } else if (Instruction_aarch64::extract(insn, 30, 25) == 0b011011) {
81 // Test & branch (immediate)
82 Instruction_aarch64::spatch(branch, 18, 5, offset);
83 } else if (Instruction_aarch64::extract(insn, 28, 24) == 0b10000) {
84 // PC-rel. addressing
85 offset = target-branch;
86 int shift = Instruction_aarch64::extract(insn, 31, 31);
87 if (shift) {
88 u_int64_t dest = (u_int64_t)target;
89 uint64_t pc_page = (uint64_t)branch >> 12;
90 uint64_t adr_page = (uint64_t)target >> 12;
91 unsigned offset_lo = dest & 0xfff;
92 offset = adr_page - pc_page;
93
94 // We handle 3 types of PC relative addressing
95 // 1 - adrp Rx, target_page
96 // ldr/str Ry, [Rx, #offset_in_page]
97 // 2 - adrp Rx, target_page
98 // add Ry, Rx, #offset_in_page
99 // 3 - adrp Rx, target_page (page aligned reloc, offset == 0)
100 // nop/movk Rx, #imm12<<32
101 // In all cases we check that Rx is the same in the adrp and the subsequent
102 // ldr/str, add or movk.
103 //
104 unsigned insn2 = ((unsigned*)branch)[1];
105 if (Instruction_aarch64::extract(insn2, 29, 24) == 0b111001 &&
106 Instruction_aarch64::extract(insn, 4, 0) ==
107 Instruction_aarch64::extract(insn2, 9, 5)) {
108 // Load/store register (unsigned immediate)
109 unsigned size = Instruction_aarch64::extract(insn2, 31, 30);
110 Instruction_aarch64::patch(branch + sizeof (unsigned),
111 21, 10, offset_lo >> size);
112 guarantee(((dest >> size) << size) == dest, "misaligned target");
113 instructions = 2;
114 } else if (Instruction_aarch64::extract(insn2, 31, 22) == 0b1001000100 &&
115 Instruction_aarch64::extract(insn, 4, 0) ==
116 Instruction_aarch64::extract(insn2, 4, 0)) {
117 // add (immediate)
118 Instruction_aarch64::patch(branch + sizeof (unsigned),
119 21, 10, offset_lo);
120 instructions = 2;
121 } else if (insn2 == aarch64_NOP || // NOP or MOVK Rx, #imm12 << 32
122 (Instruction_aarch64::extract(insn2, 31, 21) == 0b11110010110) &&
123 Instruction_aarch64::extract(insn, 4, 0) ==
124 Instruction_aarch64::extract(insn2, 4, 0)) {
125 if (offset >= -(1<<20) && offset < (1<<20)) {
126 *(unsigned *)(branch + 4) = aarch64_NOP;
127 } else {
128 Instruction_aarch64::patch_movk(branch, 4, 20, 5, (uint64_t)target, 32);
129 offset &= (1<<20)-1;
130 }
131 } else {
132 ShouldNotReachHere();
133 }
134 }
135 int offset_lo = offset & 3;
136 offset >>= 2;
137 Instruction_aarch64::spatch(branch, 23, 5, offset);
138 Instruction_aarch64::patch(branch, 30, 29, offset_lo);
139 } else if (Instruction_aarch64::extract(insn, 31, 21) == 0b11010010100) {
140 u_int64_t dest = (u_int64_t)target;
141 // Move wide constant
142 assert(nativeInstruction_at(branch+4)->is_movk(), "wrong insns in patch");
143 assert(nativeInstruction_at(branch+8)->is_movk(), "wrong insns in patch");
144 Instruction_aarch64::patch(branch, 20, 5, dest & 0xffff);
145 Instruction_aarch64::patch(branch+4, 20, 5, (dest >>= 16) & 0xffff);
146 Instruction_aarch64::patch(branch+8, 20, 5, (dest >>= 16) & 0xffff);
147 assert(target_addr_for_insn(branch) == target, "should be");
148 instructions = 3;
149 } else if (Instruction_aarch64::extract(insn, 31, 22) == 0b1011100101 &&
150 Instruction_aarch64::extract(insn, 4, 0) == 0b11111) {
151 // nothing to do
152 assert(target == 0, "did not expect to relocate target for polling page load");
198 // Compare & branch (immediate)
199 offset = Instruction_aarch64::sextract(insn, 23, 5);
200 } else if (Instruction_aarch64::extract(insn, 30, 25) == 0b011011) {
201 // Test & branch (immediate)
202 offset = Instruction_aarch64::sextract(insn, 18, 5);
203 } else if (Instruction_aarch64::extract(insn, 28, 24) == 0b10000) {
204 // PC-rel. addressing
205 offset = Instruction_aarch64::extract(insn, 30, 29);
206 offset |= Instruction_aarch64::sextract(insn, 23, 5) << 2;
207 int shift = Instruction_aarch64::extract(insn, 31, 31) ? 12 : 0;
208 if (shift) {
209 offset <<= shift;
210 uint64_t target_page = ((uint64_t)insn_addr) + offset;
211 target_page &= ((uint64_t)-1) << shift;
212 // Return the target address for the following sequences
213 // 1 - adrp Rx, target_page
214 // ldr/str Ry, [Rx, #offset_in_page]
215 // 2 - adrp Rx, target_page ]
216 // add Ry, Rx, #offset_in_page
217 // 3 - adrp Rx, target_page (page aligned reloc, offset == 0)
218 // nop/movk Rx, #imm12<<32
219 //
220 // In the first two cases we check that the register is the same and
221 // return the target_page + the offset within the page.
222 //
223 // In the third case we return just the target_page
224 //
225 unsigned insn2 = ((unsigned*)insn_addr)[1];
226 if (Instruction_aarch64::extract(insn2, 29, 24) == 0b111001 &&
227 Instruction_aarch64::extract(insn, 4, 0) ==
228 Instruction_aarch64::extract(insn2, 9, 5)) {
229 // Load/store register (unsigned immediate)
230 unsigned int byte_offset = Instruction_aarch64::extract(insn2, 21, 10);
231 unsigned int size = Instruction_aarch64::extract(insn2, 31, 30);
232 return address(target_page + (byte_offset << size));
233 } else if (Instruction_aarch64::extract(insn2, 31, 22) == 0b1001000100 &&
234 Instruction_aarch64::extract(insn, 4, 0) ==
235 Instruction_aarch64::extract(insn2, 4, 0)) {
236 // add (immediate)
237 unsigned int byte_offset = Instruction_aarch64::extract(insn2, 21, 10);
238 return address(target_page + byte_offset);
239 } else if (insn2 == aarch64_NOP || // NOP or MOVK Rx, #imm12 << 32
240 (Instruction_aarch64::extract(insn2, 31, 21) == 0b11110010110) &&
241 Instruction_aarch64::extract(insn, 4, 0) ==
242 Instruction_aarch64::extract(insn2, 4, 0)) {
243 if (insn2 != aarch64_NOP) {
244 target_page = (target_page & 0xffffffff) |
245 ((uint64_t)Instruction_aarch64::extract(insn2, 20, 5) << 32);
246 }
247 return (address)target_page;
248 } else {
249 ShouldNotReachHere();
250 }
251 } else {
252 ShouldNotReachHere();
253 }
254 } else if (Instruction_aarch64::extract(insn, 31, 23) == 0b110100101) {
255 u_int32_t *insns = (u_int32_t *)insn_addr;
256 // Move wide constant: movz, movk, movk. See movptr().
257 assert(nativeInstruction_at(insns+1)->is_movk(), "wrong insns in patch");
258 assert(nativeInstruction_at(insns+2)->is_movk(), "wrong insns in patch");
259 return address(u_int64_t(Instruction_aarch64::extract(insns[0], 20, 5))
260 + (u_int64_t(Instruction_aarch64::extract(insns[1], 20, 5)) << 16)
261 + (u_int64_t(Instruction_aarch64::extract(insns[2], 20, 5)) << 32));
262 } else if (Instruction_aarch64::extract(insn, 31, 22) == 0b1011100101 &&
263 Instruction_aarch64::extract(insn, 4, 0) == 0b11111) {
264 return 0;
265 } else {
266 ShouldNotReachHere();
267 }
268 return address(((uint64_t)insn_addr + (offset << 2)));
269 }
3060 void MacroAssembler::addptr(const Address &dst, int32_t src) {
3061 Address adr;
3062 switch(dst.getMode()) {
3063 case Address::base_plus_offset:
3064 // This is the expected mode, although we allow all the other
3065 // forms below.
3066 adr = form_address(rscratch2, dst.base(), dst.offset(), LogBytesPerWord);
3067 break;
3068 default:
3069 lea(rscratch2, dst);
3070 adr = Address(rscratch2);
3071 break;
3072 }
3073 ldr(rscratch1, adr);
3074 add(rscratch1, rscratch1, src);
3075 str(rscratch1, adr);
3076 }
3077
3078 void MacroAssembler::cmpptr(Register src1, Address src2) {
3079 unsigned long offset;
3080 far_adrp(rscratch1, src2, offset);
3081 ldr(rscratch1, Address(rscratch1, offset));
3082 cmp(src1, rscratch1);
3083 }
3084
3085 void MacroAssembler::store_check(Register obj, Address dst) {
3086 store_check(obj);
3087 }
3088
3089 void MacroAssembler::store_check(Register obj) {
3090 // Does a store check for the oop in register obj. The content of
3091 // register obj is destroyed afterwards.
3092
3093 BarrierSet* bs = Universe::heap()->barrier_set();
3094 assert(bs->kind() == BarrierSet::CardTableForRS ||
3095 bs->kind() == BarrierSet::CardTableExtension,
3096 "Wrong barrier set kind");
3097
3098 CardTableModRefBS* ct = barrier_set_cast<CardTableModRefBS>(bs);
3099 assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
3100
3101 lsr(obj, obj, CardTableModRefBS::card_shift);
3102
3103 assert(CardTableModRefBS::dirty_card_val() == 0, "must be");
3104
3105 {
3106 ExternalAddress cardtable((address) ct->byte_map_base);
3107 unsigned long offset;
3108 far_adrp(rscratch1, cardtable, offset);
3109 assert(offset == 0, "byte_map_base is misaligned");
3110 }
3111
3112 if (UseCondCardMark) {
3113 Label L_already_dirty;
3114 membar(StoreLoad);
3115 ldrb(rscratch2, Address(obj, rscratch1));
3116 cbz(rscratch2, L_already_dirty);
3117 strb(zr, Address(obj, rscratch1));
3118 bind(L_already_dirty);
3119 } else {
3120 if (UseConcMarkSweepGC && CMSPrecleaningEnabled) {
3121 membar(StoreStore);
3122 }
3123 strb(zr, Address(obj, rscratch1));
3124 }
3125 }
3126
3127 void MacroAssembler::load_klass(Register dst, Register src) {
3128 if (UseCompressedClassPointers) {
3583
3584 // Does store cross heap regions?
3585
3586 eor(tmp, store_addr, new_val);
3587 lsr(tmp, tmp, HeapRegion::LogOfHRGrainBytes);
3588 cbz(tmp, done);
3589
3590 // crosses regions, storing NULL?
3591
3592 cbz(new_val, done);
3593
3594 // storing region crossing non-NULL, is card already dirty?
3595
3596 ExternalAddress cardtable((address) ct->byte_map_base);
3597 assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
3598 const Register card_addr = tmp;
3599
3600 lsr(card_addr, store_addr, CardTableModRefBS::card_shift);
3601
3602 unsigned long offset;
3603 far_adrp(tmp2, cardtable, offset);
3604
3605 // get the address of the card
3606 add(card_addr, card_addr, tmp2);
3607 ldrb(tmp2, Address(card_addr, offset));
3608 cmpw(tmp2, (int)G1SATBCardTableModRefBS::g1_young_card_val());
3609 br(Assembler::EQ, done);
3610
3611 assert((int)CardTableModRefBS::dirty_card_val() == 0, "must be 0");
3612
3613 membar(Assembler::StoreLoad);
3614
3615 ldrb(tmp2, Address(card_addr, offset));
3616 cbzw(tmp2, done);
3617
3618 // storing a region crossing, non-NULL oop, card is clean.
3619 // dirty card and log.
3620
3621 strb(zr, Address(card_addr, offset));
3622
3623 ldr(rscratch1, queue_index);
3770 addmw(Address(rthread, in_bytes(JavaThread::tlab_fast_refill_waste_offset())), t1,
3771 rscratch1);
3772 }
3773
3774 // if tlab is currently allocated (top or end != null) then
3775 // fill [top, end + alignment_reserve) with array object
3776 cbz(top, do_refill);
3777
3778 // set up the mark word
3779 mov(rscratch1, (intptr_t)markOopDesc::prototype()->copy_set_hash(0x2));
3780 str(rscratch1, Address(top, oopDesc::mark_offset_in_bytes()));
3781 // set the length to the remaining space
3782 sub(t1, t1, typeArrayOopDesc::header_size(T_INT));
3783 add(t1, t1, (int32_t)ThreadLocalAllocBuffer::alignment_reserve());
3784 lsl(t1, t1, log2_intptr(HeapWordSize/sizeof(jint)));
3785 strw(t1, Address(top, arrayOopDesc::length_offset_in_bytes()));
3786 // set klass to intArrayKlass
3787 {
3788 unsigned long offset;
3789 // dubious reloc why not an oop reloc?
3790 far_adrp(rscratch1, ExternalAddress((address)Universe::intArrayKlassObj_addr()),
3791 offset);
3792 ldr(t1, Address(rscratch1, offset));
3793 }
3794 // store klass last. concurrent gcs assumes klass length is valid if
3795 // klass field is not null.
3796 store_klass(top, t1);
3797
3798 mov(t1, top);
3799 ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_start_offset())));
3800 sub(t1, t1, rscratch1);
3801 incr_allocated_bytes(rthread, t1, 0, rscratch1);
3802
3803 // refill the tlab with an eden allocation
3804 bind(do_refill);
3805 ldr(t1, Address(rthread, in_bytes(JavaThread::tlab_size_offset())));
3806 lsl(t1, t1, LogHeapWordSize);
3807 // allocate new tlab, address returned in top
3808 eden_allocate(top, t1, 0, t2, slow_case);
3809
3810 // Check that t1 was preserved in eden_allocate.
3835
3836 return rthread; // for use by caller
3837 }
3838
3839 // Defines obj, preserves var_size_in_bytes
3840 void MacroAssembler::eden_allocate(Register obj,
3841 Register var_size_in_bytes,
3842 int con_size_in_bytes,
3843 Register t1,
3844 Label& slow_case) {
3845 assert_different_registers(obj, var_size_in_bytes, t1);
3846 if (!Universe::heap()->supports_inline_contig_alloc()) {
3847 b(slow_case);
3848 } else {
3849 Register end = t1;
3850 Register heap_end = rscratch2;
3851 Label retry;
3852 bind(retry);
3853 {
3854 unsigned long offset;
3855 far_adrp(rscratch1, ExternalAddress((address) Universe::heap()->end_addr()), offset);
3856 ldr(heap_end, Address(rscratch1, offset));
3857 }
3858
3859 ExternalAddress heap_top((address) Universe::heap()->top_addr());
3860
3861 // Get the current top of the heap
3862 {
3863 unsigned long offset;
3864 far_adrp(rscratch1, heap_top, offset);
3865 // Use add() here after ARDP, rather than lea().
3866 // lea() does not generate anything if its offset is zero.
3867 // However, relocs expect to find either an ADD or a load/store
3868 // insn after an ADRP. add() always generates an ADD insn, even
3869 // for add(Rn, Rn, 0).
3870 add(rscratch1, rscratch1, offset);
3871 ldaxr(obj, rscratch1);
3872 }
3873
3874 // Adjust it my the size of our new object
3875 if (var_size_in_bytes == noreg) {
3876 lea(end, Address(obj, con_size_in_bytes));
3877 } else {
3878 lea(end, Address(obj, var_size_in_bytes));
3879 }
3880
3881 // if end < obj then we wrapped around high memory
3882 cmp(end, obj);
3883 br(Assembler::LO, slow_case);
3884
3935 str(size, Address(tmp));
3936 br(Assembler::GT, loop);
3937
3938 // Bang down shadow pages too.
3939 // At this point, (tmp-0) is the last address touched, so don't
3940 // touch it again. (It was touched as (tmp-pagesize) but then tmp
3941 // was post-decremented.) Skip this address by starting at i=1, and
3942 // touch a few more pages below. N.B. It is important to touch all
3943 // the way down to and including i=StackShadowPages.
3944 for (int i = 0; i< StackShadowPages-1; i++) {
3945 // this could be any sized move but this is can be a debugging crumb
3946 // so the bigger the better.
3947 lea(tmp, Address(tmp, -os::vm_page_size()));
3948 str(size, Address(tmp));
3949 }
3950 }
3951
3952
3953 address MacroAssembler::read_polling_page(Register r, address page, relocInfo::relocType rtype) {
3954 unsigned long off;
3955 far_adrp(r, Address(page, rtype), off);
3956 InstructionMark im(this);
3957 code_section()->relocate(inst_mark(), rtype);
3958 ldrw(zr, Address(r, off));
3959 return inst_mark();
3960 }
3961
3962 address MacroAssembler::read_polling_page(Register r, relocInfo::relocType rtype) {
3963 InstructionMark im(this);
3964 code_section()->relocate(inst_mark(), rtype);
3965 ldrw(zr, Address(r, 0));
3966 return inst_mark();
3967 }
3968
3969 void MacroAssembler::far_adrp(Register reg1, const Address &dest, unsigned long &byte_offset) {
3970 uint64_t pc_page = (uint64_t)pc() >> 12;
3971 uint64_t adr_page = (uint64_t)dest.target() >> 12;
3972 int64_t offset = adr_page - pc_page;
3973
3974 InstructionMark im(this);
3975 code_section()->relocate(inst_mark(), dest.rspec());
3976 if (offset >= -(1<<20) && offset < (1<<20)) {
3977 _adrp(reg1, dest.target());
3978 nop();
3979 } else {
3980 offset = (offset & ((1<<20)-1)) << 12;
3981 _adrp(reg1, pc()+offset);
3982 movk(reg1, ((uint64_t)dest.target() >> 32) & 0xffff, 32);
3983 }
3984 byte_offset = (uint64_t)dest.target() & 0xfff;
3985 }
3986
3987 void MacroAssembler::adrp(Register reg1, const Address &dest, unsigned long &byte_offset) {
3988 uint64_t pc_page = (uint64_t)pc() >> 12;
3989 uint64_t adr_page = (uint64_t)dest.target() >> 12;
3990 int64_t offset = adr_page - pc_page;
3991 guarantee(offset >= -(1<<20) && offset < (1<<20), "adrp out of range, use far_adrp");
3992 InstructionMark im(this);
3993 code_section()->relocate(inst_mark(), dest.rspec());
3994 byte_offset = (uint64_t)dest.target() & 0xfff;
3995 _adrp(reg1, dest.target());
3996 }
3997
3998 void MacroAssembler::build_frame(int framesize) {
3999 assert(framesize > 0, "framesize must be > 0");
4000 if (framesize < ((1 << 9) + 2 * wordSize)) {
4001 sub(sp, sp, framesize);
4002 stp(rfp, lr, Address(sp, framesize - 2 * wordSize));
4003 if (PreserveFramePointer) add(rfp, sp, framesize - 2 * wordSize);
4004 } else {
4005 stp(rfp, lr, Address(pre(sp, -2 * wordSize)));
4006 if (PreserveFramePointer) mov(rfp, sp);
4007 if (framesize < ((1 << 12) + 2 * wordSize))
4008 sub(sp, sp, framesize - 2 * wordSize);
4009 else {
4010 mov(rscratch1, framesize - 2 * wordSize);
4011 sub(sp, sp, rscratch1);
4012 }
4013 }
4014 }
4015
|