1 /*
2 * Copyright (c) 1997, 2018, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, 2015, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include <sys/types.h>
27
28 #include "precompiled.hpp"
29 #include "jvm.h"
30 #include "asm/assembler.hpp"
31 #include "asm/assembler.inline.hpp"
32 #include "gc/shared/barrierSet.hpp"
33 #include "gc/shared/cardTable.hpp"
34 #include "gc/shared/barrierSetAssembler.hpp"
35 #include "gc/shared/cardTableBarrierSet.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "compiler/disassembler.hpp"
38 #include "memory/resourceArea.hpp"
39 #include "nativeInst_aarch64.hpp"
40 #include "oops/accessDecorators.hpp"
41 #include "oops/compressedOops.inline.hpp"
42 #include "oops/klass.inline.hpp"
43 #include "oops/oop.hpp"
44 #include "opto/compile.hpp"
45 #include "opto/intrinsicnode.hpp"
46 #include "opto/node.hpp"
47 #include "runtime/biasedLocking.hpp"
48 #include "runtime/icache.hpp"
49 #include "runtime/interfaceSupport.inline.hpp"
50 #include "runtime/jniHandles.inline.hpp"
51 #include "runtime/sharedRuntime.hpp"
52 #include "runtime/thread.hpp"
53
54 #ifdef PRODUCT
55 #define BLOCK_COMMENT(str) /* nothing */
56 #define STOP(error) stop(error)
57 #else
58 #define BLOCK_COMMENT(str) block_comment(str)
59 #define STOP(error) block_comment(error); stop(error)
60 #endif
61
62 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
63
64 // Patch any kind of instruction; there may be several instructions.
65 // Return the total length (in bytes) of the instructions.
66 int MacroAssembler::pd_patch_instruction_size(address branch, address target) {
67 int instructions = 1;
68 assert((uint64_t)target < (1ul << 48), "48-bit overflow in address constant");
69 long offset = (target - branch) >> 2;
70 unsigned insn = *(unsigned*)branch;
71 if ((Instruction_aarch64::extract(insn, 29, 24) & 0b111011) == 0b011000) {
72 // Load register (literal)
73 Instruction_aarch64::spatch(branch, 23, 5, offset);
74 } else if (Instruction_aarch64::extract(insn, 30, 26) == 0b00101) {
75 // Unconditional branch (immediate)
76 Instruction_aarch64::spatch(branch, 25, 0, offset);
77 } else if (Instruction_aarch64::extract(insn, 31, 25) == 0b0101010) {
78 // Conditional branch (immediate)
79 Instruction_aarch64::spatch(branch, 23, 5, offset);
80 } else if (Instruction_aarch64::extract(insn, 30, 25) == 0b011010) {
81 // Compare & branch (immediate)
82 Instruction_aarch64::spatch(branch, 23, 5, offset);
83 } else if (Instruction_aarch64::extract(insn, 30, 25) == 0b011011) {
84 // Test & branch (immediate)
85 Instruction_aarch64::spatch(branch, 18, 5, offset);
86 } else if (Instruction_aarch64::extract(insn, 28, 24) == 0b10000) {
87 // PC-rel. addressing
88 offset = target-branch;
89 int shift = Instruction_aarch64::extract(insn, 31, 31);
90 if (shift) {
91 u_int64_t dest = (u_int64_t)target;
92 uint64_t pc_page = (uint64_t)branch >> 12;
93 uint64_t adr_page = (uint64_t)target >> 12;
94 unsigned offset_lo = dest & 0xfff;
95 offset = adr_page - pc_page;
96
97 // We handle 4 types of PC relative addressing
98 // 1 - adrp Rx, target_page
99 // ldr/str Ry, [Rx, #offset_in_page]
100 // 2 - adrp Rx, target_page
101 // add Ry, Rx, #offset_in_page
102 // 3 - adrp Rx, target_page (page aligned reloc, offset == 0)
103 // movk Rx, #imm16<<32
104 // 4 - adrp Rx, target_page (page aligned reloc, offset == 0)
105 // In the first 3 cases we must check that Rx is the same in the adrp and the
106 // subsequent ldr/str, add or movk instruction. Otherwise we could accidentally end
107 // up treating a type 4 relocation as a type 1, 2 or 3 just because it happened
108 // to be followed by a random unrelated ldr/str, add or movk instruction.
109 //
110 unsigned insn2 = ((unsigned*)branch)[1];
111 if (Instruction_aarch64::extract(insn2, 29, 24) == 0b111001 &&
112 Instruction_aarch64::extract(insn, 4, 0) ==
113 Instruction_aarch64::extract(insn2, 9, 5)) {
114 // Load/store register (unsigned immediate)
115 unsigned size = Instruction_aarch64::extract(insn2, 31, 30);
116 Instruction_aarch64::patch(branch + sizeof (unsigned),
117 21, 10, offset_lo >> size);
118 guarantee(((dest >> size) << size) == dest, "misaligned target");
119 instructions = 2;
120 } else if (Instruction_aarch64::extract(insn2, 31, 22) == 0b1001000100 &&
121 Instruction_aarch64::extract(insn, 4, 0) ==
122 Instruction_aarch64::extract(insn2, 4, 0)) {
123 // add (immediate)
124 Instruction_aarch64::patch(branch + sizeof (unsigned),
125 21, 10, offset_lo);
126 instructions = 2;
127 } else if (Instruction_aarch64::extract(insn2, 31, 21) == 0b11110010110 &&
128 Instruction_aarch64::extract(insn, 4, 0) ==
129 Instruction_aarch64::extract(insn2, 4, 0)) {
130 // movk #imm16<<32
131 Instruction_aarch64::patch(branch + 4, 20, 5, (uint64_t)target >> 32);
132 long dest = ((long)target & 0xffffffffL) | ((long)branch & 0xffff00000000L);
133 long pc_page = (long)branch >> 12;
134 long adr_page = (long)dest >> 12;
135 offset = adr_page - pc_page;
136 instructions = 2;
137 }
138 }
139 int offset_lo = offset & 3;
140 offset >>= 2;
141 Instruction_aarch64::spatch(branch, 23, 5, offset);
142 Instruction_aarch64::patch(branch, 30, 29, offset_lo);
143 } else if (Instruction_aarch64::extract(insn, 31, 21) == 0b11010010100) {
144 u_int64_t dest = (u_int64_t)target;
145 // Move wide constant
146 assert(nativeInstruction_at(branch+4)->is_movk(), "wrong insns in patch");
147 assert(nativeInstruction_at(branch+8)->is_movk(), "wrong insns in patch");
148 Instruction_aarch64::patch(branch, 20, 5, dest & 0xffff);
149 Instruction_aarch64::patch(branch+4, 20, 5, (dest >>= 16) & 0xffff);
150 Instruction_aarch64::patch(branch+8, 20, 5, (dest >>= 16) & 0xffff);
151 assert(target_addr_for_insn(branch) == target, "should be");
152 instructions = 3;
153 } else if (Instruction_aarch64::extract(insn, 31, 22) == 0b1011100101 &&
154 Instruction_aarch64::extract(insn, 4, 0) == 0b11111) {
155 // nothing to do
156 assert(target == 0, "did not expect to relocate target for polling page load");
157 } else {
158 ShouldNotReachHere();
159 }
160 return instructions * NativeInstruction::instruction_size;
161 }
162
163 int MacroAssembler::patch_oop(address insn_addr, address o) {
164 int instructions;
165 unsigned insn = *(unsigned*)insn_addr;
166 assert(nativeInstruction_at(insn_addr+4)->is_movk(), "wrong insns in patch");
167
168 // OOPs are either narrow (32 bits) or wide (48 bits). We encode
169 // narrow OOPs by setting the upper 16 bits in the first
170 // instruction.
171 if (Instruction_aarch64::extract(insn, 31, 21) == 0b11010010101) {
172 // Move narrow OOP
173 narrowOop n = CompressedOops::encode((oop)o);
174 Instruction_aarch64::patch(insn_addr, 20, 5, n >> 16);
175 Instruction_aarch64::patch(insn_addr+4, 20, 5, n & 0xffff);
176 instructions = 2;
177 } else {
178 // Move wide OOP
179 assert(nativeInstruction_at(insn_addr+8)->is_movk(), "wrong insns in patch");
180 uintptr_t dest = (uintptr_t)o;
181 Instruction_aarch64::patch(insn_addr, 20, 5, dest & 0xffff);
182 Instruction_aarch64::patch(insn_addr+4, 20, 5, (dest >>= 16) & 0xffff);
183 Instruction_aarch64::patch(insn_addr+8, 20, 5, (dest >>= 16) & 0xffff);
184 instructions = 3;
185 }
186 return instructions * NativeInstruction::instruction_size;
187 }
188
189 int MacroAssembler::patch_narrow_klass(address insn_addr, narrowKlass n) {
190 // Metatdata pointers are either narrow (32 bits) or wide (48 bits).
191 // We encode narrow ones by setting the upper 16 bits in the first
192 // instruction.
193 NativeInstruction *insn = nativeInstruction_at(insn_addr);
194 assert(Instruction_aarch64::extract(insn->encoding(), 31, 21) == 0b11010010101 &&
195 nativeInstruction_at(insn_addr+4)->is_movk(), "wrong insns in patch");
196
197 Instruction_aarch64::patch(insn_addr, 20, 5, n >> 16);
198 Instruction_aarch64::patch(insn_addr+4, 20, 5, n & 0xffff);
199 return 2 * NativeInstruction::instruction_size;
200 }
201
202 address MacroAssembler::target_addr_for_insn(address insn_addr, unsigned insn) {
203 long offset = 0;
204 if ((Instruction_aarch64::extract(insn, 29, 24) & 0b011011) == 0b00011000) {
205 // Load register (literal)
206 offset = Instruction_aarch64::sextract(insn, 23, 5);
207 return address(((uint64_t)insn_addr + (offset << 2)));
208 } else if (Instruction_aarch64::extract(insn, 30, 26) == 0b00101) {
209 // Unconditional branch (immediate)
210 offset = Instruction_aarch64::sextract(insn, 25, 0);
211 } else if (Instruction_aarch64::extract(insn, 31, 25) == 0b0101010) {
212 // Conditional branch (immediate)
213 offset = Instruction_aarch64::sextract(insn, 23, 5);
214 } else if (Instruction_aarch64::extract(insn, 30, 25) == 0b011010) {
215 // Compare & branch (immediate)
216 offset = Instruction_aarch64::sextract(insn, 23, 5);
217 } else if (Instruction_aarch64::extract(insn, 30, 25) == 0b011011) {
218 // Test & branch (immediate)
219 offset = Instruction_aarch64::sextract(insn, 18, 5);
220 } else if (Instruction_aarch64::extract(insn, 28, 24) == 0b10000) {
221 // PC-rel. addressing
222 offset = Instruction_aarch64::extract(insn, 30, 29);
223 offset |= Instruction_aarch64::sextract(insn, 23, 5) << 2;
224 int shift = Instruction_aarch64::extract(insn, 31, 31) ? 12 : 0;
225 if (shift) {
226 offset <<= shift;
227 uint64_t target_page = ((uint64_t)insn_addr) + offset;
228 target_page &= ((uint64_t)-1) << shift;
229 // Return the target address for the following sequences
230 // 1 - adrp Rx, target_page
231 // ldr/str Ry, [Rx, #offset_in_page]
232 // 2 - adrp Rx, target_page
233 // add Ry, Rx, #offset_in_page
234 // 3 - adrp Rx, target_page (page aligned reloc, offset == 0)
235 // movk Rx, #imm12<<32
236 // 4 - adrp Rx, target_page (page aligned reloc, offset == 0)
237 //
238 // In the first two cases we check that the register is the same and
239 // return the target_page + the offset within the page.
240 // Otherwise we assume it is a page aligned relocation and return
241 // the target page only.
242 //
243 unsigned insn2 = ((unsigned*)insn_addr)[1];
244 if (Instruction_aarch64::extract(insn2, 29, 24) == 0b111001 &&
245 Instruction_aarch64::extract(insn, 4, 0) ==
246 Instruction_aarch64::extract(insn2, 9, 5)) {
247 // Load/store register (unsigned immediate)
248 unsigned int byte_offset = Instruction_aarch64::extract(insn2, 21, 10);
249 unsigned int size = Instruction_aarch64::extract(insn2, 31, 30);
250 return address(target_page + (byte_offset << size));
251 } else if (Instruction_aarch64::extract(insn2, 31, 22) == 0b1001000100 &&
252 Instruction_aarch64::extract(insn, 4, 0) ==
253 Instruction_aarch64::extract(insn2, 4, 0)) {
254 // add (immediate)
255 unsigned int byte_offset = Instruction_aarch64::extract(insn2, 21, 10);
256 return address(target_page + byte_offset);
257 } else {
258 if (Instruction_aarch64::extract(insn2, 31, 21) == 0b11110010110 &&
259 Instruction_aarch64::extract(insn, 4, 0) ==
260 Instruction_aarch64::extract(insn2, 4, 0)) {
261 target_page = (target_page & 0xffffffff) |
262 ((uint64_t)Instruction_aarch64::extract(insn2, 20, 5) << 32);
263 }
264 return (address)target_page;
265 }
266 } else {
267 ShouldNotReachHere();
268 }
269 } else if (Instruction_aarch64::extract(insn, 31, 23) == 0b110100101) {
270 u_int32_t *insns = (u_int32_t *)insn_addr;
271 // Move wide constant: movz, movk, movk. See movptr().
272 assert(nativeInstruction_at(insns+1)->is_movk(), "wrong insns in patch");
273 assert(nativeInstruction_at(insns+2)->is_movk(), "wrong insns in patch");
274 return address(u_int64_t(Instruction_aarch64::extract(insns[0], 20, 5))
275 + (u_int64_t(Instruction_aarch64::extract(insns[1], 20, 5)) << 16)
276 + (u_int64_t(Instruction_aarch64::extract(insns[2], 20, 5)) << 32));
277 } else if (Instruction_aarch64::extract(insn, 31, 22) == 0b1011100101 &&
278 Instruction_aarch64::extract(insn, 4, 0) == 0b11111) {
279 return 0;
280 } else {
281 ShouldNotReachHere();
282 }
283 return address(((uint64_t)insn_addr + (offset << 2)));
284 }
285
286 void MacroAssembler::serialize_memory(Register thread, Register tmp) {
287 dsb(Assembler::SY);
288 }
289
290 void MacroAssembler::safepoint_poll(Label& slow_path) {
291 if (SafepointMechanism::uses_thread_local_poll()) {
292 ldr(rscratch1, Address(rthread, Thread::polling_page_offset()));
293 tbnz(rscratch1, exact_log2(SafepointMechanism::poll_bit()), slow_path);
294 } else {
295 unsigned long offset;
296 adrp(rscratch1, ExternalAddress(SafepointSynchronize::address_of_state()), offset);
297 ldrw(rscratch1, Address(rscratch1, offset));
298 assert(SafepointSynchronize::_not_synchronized == 0, "rewrite this code");
299 cbnz(rscratch1, slow_path);
300 }
301 }
302
303 // Just like safepoint_poll, but use an acquiring load for thread-
304 // local polling.
305 //
306 // We need an acquire here to ensure that any subsequent load of the
307 // global SafepointSynchronize::_state flag is ordered after this load
308 // of the local Thread::_polling page. We don't want this poll to
309 // return false (i.e. not safepointing) and a later poll of the global
310 // SafepointSynchronize::_state spuriously to return true.
311 //
312 // This is to avoid a race when we're in a native->Java transition
313 // racing the code which wakes up from a safepoint.
314 //
315 void MacroAssembler::safepoint_poll_acquire(Label& slow_path) {
316 if (SafepointMechanism::uses_thread_local_poll()) {
317 lea(rscratch1, Address(rthread, Thread::polling_page_offset()));
318 ldar(rscratch1, rscratch1);
319 tbnz(rscratch1, exact_log2(SafepointMechanism::poll_bit()), slow_path);
320 } else {
321 safepoint_poll(slow_path);
322 }
323 }
324
325 void MacroAssembler::reset_last_Java_frame(bool clear_fp) {
326 // we must set sp to zero to clear frame
327 str(zr, Address(rthread, JavaThread::last_Java_sp_offset()));
328
329 // must clear fp, so that compiled frames are not confused; it is
330 // possible that we need it only for debugging
331 if (clear_fp) {
332 str(zr, Address(rthread, JavaThread::last_Java_fp_offset()));
333 }
334
335 // Always clear the pc because it could have been set by make_walkable()
336 str(zr, Address(rthread, JavaThread::last_Java_pc_offset()));
337 }
338
339 // Calls to C land
340 //
341 // When entering C land, the rfp, & resp of the last Java frame have to be recorded
342 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp
343 // has to be reset to 0. This is required to allow proper stack traversal.
344 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
345 Register last_java_fp,
346 Register last_java_pc,
347 Register scratch) {
348
349 if (last_java_pc->is_valid()) {
350 str(last_java_pc, Address(rthread,
351 JavaThread::frame_anchor_offset()
352 + JavaFrameAnchor::last_Java_pc_offset()));
353 }
354
355 // determine last_java_sp register
356 if (last_java_sp == sp) {
357 mov(scratch, sp);
358 last_java_sp = scratch;
359 } else if (!last_java_sp->is_valid()) {
360 last_java_sp = esp;
361 }
362
363 str(last_java_sp, Address(rthread, JavaThread::last_Java_sp_offset()));
364
365 // last_java_fp is optional
366 if (last_java_fp->is_valid()) {
367 str(last_java_fp, Address(rthread, JavaThread::last_Java_fp_offset()));
368 }
369 }
370
371 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
372 Register last_java_fp,
373 address last_java_pc,
374 Register scratch) {
375 if (last_java_pc != NULL) {
376 adr(scratch, last_java_pc);
377 } else {
378 // FIXME: This is almost never correct. We should delete all
379 // cases of set_last_Java_frame with last_java_pc=NULL and use the
380 // correct return address instead.
381 adr(scratch, pc());
382 }
383
384 str(scratch, Address(rthread,
385 JavaThread::frame_anchor_offset()
386 + JavaFrameAnchor::last_Java_pc_offset()));
387
388 set_last_Java_frame(last_java_sp, last_java_fp, noreg, scratch);
389 }
390
391 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
392 Register last_java_fp,
393 Label &L,
394 Register scratch) {
395 if (L.is_bound()) {
396 set_last_Java_frame(last_java_sp, last_java_fp, target(L), scratch);
397 } else {
398 InstructionMark im(this);
399 L.add_patch_at(code(), locator());
400 set_last_Java_frame(last_java_sp, last_java_fp, (address)NULL, scratch);
401 }
402 }
403
404 void MacroAssembler::far_call(Address entry, CodeBuffer *cbuf, Register tmp) {
405 assert(ReservedCodeCacheSize < 4*G, "branch out of range");
406 assert(CodeCache::find_blob(entry.target()) != NULL,
407 "destination of far call not found in code cache");
408 if (far_branches()) {
409 unsigned long offset;
410 // We can use ADRP here because we know that the total size of
411 // the code cache cannot exceed 2Gb.
412 adrp(tmp, entry, offset);
413 add(tmp, tmp, offset);
414 if (cbuf) cbuf->set_insts_mark();
415 blr(tmp);
416 } else {
417 if (cbuf) cbuf->set_insts_mark();
418 bl(entry);
419 }
420 }
421
422 void MacroAssembler::far_jump(Address entry, CodeBuffer *cbuf, Register tmp) {
423 assert(ReservedCodeCacheSize < 4*G, "branch out of range");
424 assert(CodeCache::find_blob(entry.target()) != NULL,
425 "destination of far call not found in code cache");
426 if (far_branches()) {
427 unsigned long offset;
428 // We can use ADRP here because we know that the total size of
429 // the code cache cannot exceed 2Gb.
430 adrp(tmp, entry, offset);
431 add(tmp, tmp, offset);
432 if (cbuf) cbuf->set_insts_mark();
433 br(tmp);
434 } else {
435 if (cbuf) cbuf->set_insts_mark();
436 b(entry);
437 }
438 }
439
440 void MacroAssembler::reserved_stack_check() {
441 // testing if reserved zone needs to be enabled
442 Label no_reserved_zone_enabling;
443
444 ldr(rscratch1, Address(rthread, JavaThread::reserved_stack_activation_offset()));
445 cmp(sp, rscratch1);
446 br(Assembler::LO, no_reserved_zone_enabling);
447
448 enter(); // LR and FP are live.
449 lea(rscratch1, CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone));
450 mov(c_rarg0, rthread);
451 blr(rscratch1);
452 leave();
453
454 // We have already removed our own frame.
455 // throw_delayed_StackOverflowError will think that it's been
456 // called by our caller.
457 lea(rscratch1, RuntimeAddress(StubRoutines::throw_delayed_StackOverflowError_entry()));
458 br(rscratch1);
459 should_not_reach_here();
460
461 bind(no_reserved_zone_enabling);
462 }
463
464 int MacroAssembler::biased_locking_enter(Register lock_reg,
465 Register obj_reg,
466 Register swap_reg,
467 Register tmp_reg,
468 bool swap_reg_contains_mark,
469 Label& done,
470 Label* slow_case,
471 BiasedLockingCounters* counters) {
472 assert(UseBiasedLocking, "why call this otherwise?");
473 assert_different_registers(lock_reg, obj_reg, swap_reg);
474
475 if (PrintBiasedLockingStatistics && counters == NULL)
476 counters = BiasedLocking::counters();
477
478 assert_different_registers(lock_reg, obj_reg, swap_reg, tmp_reg, rscratch1, rscratch2, noreg);
479 assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits, "biased locking makes assumptions about bit layout");
480 Address mark_addr (obj_reg, oopDesc::mark_offset_in_bytes());
481 Address klass_addr (obj_reg, oopDesc::klass_offset_in_bytes());
482 Address saved_mark_addr(lock_reg, 0);
483
484 // Biased locking
485 // See whether the lock is currently biased toward our thread and
486 // whether the epoch is still valid
487 // Note that the runtime guarantees sufficient alignment of JavaThread
488 // pointers to allow age to be placed into low bits
489 // First check to see whether biasing is even enabled for this object
490 Label cas_label;
491 int null_check_offset = -1;
492 if (!swap_reg_contains_mark) {
493 null_check_offset = offset();
494 ldr(swap_reg, mark_addr);
495 }
496 andr(tmp_reg, swap_reg, markOopDesc::biased_lock_mask_in_place);
497 cmp(tmp_reg, markOopDesc::biased_lock_pattern);
498 br(Assembler::NE, cas_label);
499 // The bias pattern is present in the object's header. Need to check
500 // whether the bias owner and the epoch are both still current.
501 load_prototype_header(tmp_reg, obj_reg);
502 orr(tmp_reg, tmp_reg, rthread);
503 eor(tmp_reg, swap_reg, tmp_reg);
504 andr(tmp_reg, tmp_reg, ~((int) markOopDesc::age_mask_in_place));
505 if (counters != NULL) {
506 Label around;
507 cbnz(tmp_reg, around);
508 atomic_incw(Address((address)counters->biased_lock_entry_count_addr()), tmp_reg, rscratch1, rscratch2);
509 b(done);
510 bind(around);
511 } else {
512 cbz(tmp_reg, done);
513 }
514
515 Label try_revoke_bias;
516 Label try_rebias;
517
518 // At this point we know that the header has the bias pattern and
519 // that we are not the bias owner in the current epoch. We need to
520 // figure out more details about the state of the header in order to
521 // know what operations can be legally performed on the object's
522 // header.
523
524 // If the low three bits in the xor result aren't clear, that means
525 // the prototype header is no longer biased and we have to revoke
526 // the bias on this object.
527 andr(rscratch1, tmp_reg, markOopDesc::biased_lock_mask_in_place);
528 cbnz(rscratch1, try_revoke_bias);
529
530 // Biasing is still enabled for this data type. See whether the
531 // epoch of the current bias is still valid, meaning that the epoch
532 // bits of the mark word are equal to the epoch bits of the
533 // prototype header. (Note that the prototype header's epoch bits
534 // only change at a safepoint.) If not, attempt to rebias the object
535 // toward the current thread. Note that we must be absolutely sure
536 // that the current epoch is invalid in order to do this because
537 // otherwise the manipulations it performs on the mark word are
538 // illegal.
539 andr(rscratch1, tmp_reg, markOopDesc::epoch_mask_in_place);
540 cbnz(rscratch1, try_rebias);
541
542 // The epoch of the current bias is still valid but we know nothing
543 // about the owner; it might be set or it might be clear. Try to
544 // acquire the bias of the object using an atomic operation. If this
545 // fails we will go in to the runtime to revoke the object's bias.
546 // Note that we first construct the presumed unbiased header so we
547 // don't accidentally blow away another thread's valid bias.
548 {
549 Label here;
550 mov(rscratch1, markOopDesc::biased_lock_mask_in_place | markOopDesc::age_mask_in_place | markOopDesc::epoch_mask_in_place);
551 andr(swap_reg, swap_reg, rscratch1);
552 orr(tmp_reg, swap_reg, rthread);
553 cmpxchg_obj_header(swap_reg, tmp_reg, obj_reg, rscratch1, here, slow_case);
554 // If the biasing toward our thread failed, this means that
555 // another thread succeeded in biasing it toward itself and we
556 // need to revoke that bias. The revocation will occur in the
557 // interpreter runtime in the slow case.
558 bind(here);
559 if (counters != NULL) {
560 atomic_incw(Address((address)counters->anonymously_biased_lock_entry_count_addr()),
561 tmp_reg, rscratch1, rscratch2);
562 }
563 }
564 b(done);
565
566 bind(try_rebias);
567 // At this point we know the epoch has expired, meaning that the
568 // current "bias owner", if any, is actually invalid. Under these
569 // circumstances _only_, we are allowed to use the current header's
570 // value as the comparison value when doing the cas to acquire the
571 // bias in the current epoch. In other words, we allow transfer of
572 // the bias from one thread to another directly in this situation.
573 //
574 // FIXME: due to a lack of registers we currently blow away the age
575 // bits in this situation. Should attempt to preserve them.
576 {
577 Label here;
578 load_prototype_header(tmp_reg, obj_reg);
579 orr(tmp_reg, rthread, tmp_reg);
580 cmpxchg_obj_header(swap_reg, tmp_reg, obj_reg, rscratch1, here, slow_case);
581 // If the biasing toward our thread failed, then another thread
582 // succeeded in biasing it toward itself and we need to revoke that
583 // bias. The revocation will occur in the runtime in the slow case.
584 bind(here);
585 if (counters != NULL) {
586 atomic_incw(Address((address)counters->rebiased_lock_entry_count_addr()),
587 tmp_reg, rscratch1, rscratch2);
588 }
589 }
590 b(done);
591
592 bind(try_revoke_bias);
593 // The prototype mark in the klass doesn't have the bias bit set any
594 // more, indicating that objects of this data type are not supposed
595 // to be biased any more. We are going to try to reset the mark of
596 // this object to the prototype value and fall through to the
597 // CAS-based locking scheme. Note that if our CAS fails, it means
598 // that another thread raced us for the privilege of revoking the
599 // bias of this particular object, so it's okay to continue in the
600 // normal locking code.
601 //
602 // FIXME: due to a lack of registers we currently blow away the age
603 // bits in this situation. Should attempt to preserve them.
604 {
605 Label here, nope;
606 load_prototype_header(tmp_reg, obj_reg);
607 cmpxchg_obj_header(swap_reg, tmp_reg, obj_reg, rscratch1, here, &nope);
608 bind(here);
609
610 // Fall through to the normal CAS-based lock, because no matter what
611 // the result of the above CAS, some thread must have succeeded in
612 // removing the bias bit from the object's header.
613 if (counters != NULL) {
614 atomic_incw(Address((address)counters->revoked_lock_entry_count_addr()), tmp_reg,
615 rscratch1, rscratch2);
616 }
617 bind(nope);
618 }
619
620 bind(cas_label);
621
622 return null_check_offset;
623 }
624
625 void MacroAssembler::biased_locking_exit(Register obj_reg, Register temp_reg, Label& done) {
626 assert(UseBiasedLocking, "why call this otherwise?");
627
628 // Check for biased locking unlock case, which is a no-op
629 // Note: we do not have to check the thread ID for two reasons.
630 // First, the interpreter checks for IllegalMonitorStateException at
631 // a higher level. Second, if the bias was revoked while we held the
632 // lock, the object could not be rebiased toward another thread, so
633 // the bias bit would be clear.
634 ldr(temp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
635 andr(temp_reg, temp_reg, markOopDesc::biased_lock_mask_in_place);
636 cmp(temp_reg, markOopDesc::biased_lock_pattern);
637 br(Assembler::EQ, done);
638 }
639
640 static void pass_arg0(MacroAssembler* masm, Register arg) {
641 if (c_rarg0 != arg ) {
642 masm->mov(c_rarg0, arg);
643 }
644 }
645
646 static void pass_arg1(MacroAssembler* masm, Register arg) {
647 if (c_rarg1 != arg ) {
648 masm->mov(c_rarg1, arg);
649 }
650 }
651
652 static void pass_arg2(MacroAssembler* masm, Register arg) {
653 if (c_rarg2 != arg ) {
654 masm->mov(c_rarg2, arg);
655 }
656 }
657
658 static void pass_arg3(MacroAssembler* masm, Register arg) {
659 if (c_rarg3 != arg ) {
660 masm->mov(c_rarg3, arg);
661 }
662 }
663
664 void MacroAssembler::call_VM_base(Register oop_result,
665 Register java_thread,
666 Register last_java_sp,
667 address entry_point,
668 int number_of_arguments,
669 bool check_exceptions) {
670 // determine java_thread register
671 if (!java_thread->is_valid()) {
672 java_thread = rthread;
673 }
674
675 // determine last_java_sp register
676 if (!last_java_sp->is_valid()) {
677 last_java_sp = esp;
678 }
679
680 // debugging support
681 assert(number_of_arguments >= 0 , "cannot have negative number of arguments");
682 assert(java_thread == rthread, "unexpected register");
683 #ifdef ASSERT
684 // TraceBytecodes does not use r12 but saves it over the call, so don't verify
685 // if ((UseCompressedOops || UseCompressedClassPointers) && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");
686 #endif // ASSERT
687
688 assert(java_thread != oop_result , "cannot use the same register for java_thread & oop_result");
689 assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp");
690
691 // push java thread (becomes first argument of C function)
692
693 mov(c_rarg0, java_thread);
694
695 // set last Java frame before call
696 assert(last_java_sp != rfp, "can't use rfp");
697
698 Label l;
699 set_last_Java_frame(last_java_sp, rfp, l, rscratch1);
700
701 // do the call, remove parameters
702 MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments, &l);
703
704 // reset last Java frame
705 // Only interpreter should have to clear fp
706 reset_last_Java_frame(true);
707
708 // C++ interp handles this in the interpreter
709 check_and_handle_popframe(java_thread);
710 check_and_handle_earlyret(java_thread);
711
712 if (check_exceptions) {
713 // check for pending exceptions (java_thread is set upon return)
714 ldr(rscratch1, Address(java_thread, in_bytes(Thread::pending_exception_offset())));
715 Label ok;
716 cbz(rscratch1, ok);
717 lea(rscratch1, RuntimeAddress(StubRoutines::forward_exception_entry()));
718 br(rscratch1);
719 bind(ok);
720 }
721
722 // get oop result if there is one and reset the value in the thread
723 if (oop_result->is_valid()) {
724 get_vm_result(oop_result, java_thread);
725 }
726 }
727
728 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
729 call_VM_base(oop_result, noreg, noreg, entry_point, number_of_arguments, check_exceptions);
730 }
731
732 // Maybe emit a call via a trampoline. If the code cache is small
733 // trampolines won't be emitted.
734
735 address MacroAssembler::trampoline_call(Address entry, CodeBuffer *cbuf) {
736 assert(JavaThread::current()->is_Compiler_thread(), "just checking");
737 assert(entry.rspec().type() == relocInfo::runtime_call_type
738 || entry.rspec().type() == relocInfo::opt_virtual_call_type
739 || entry.rspec().type() == relocInfo::static_call_type
740 || entry.rspec().type() == relocInfo::virtual_call_type, "wrong reloc type");
741
742 // We need a trampoline if branches are far.
743 if (far_branches()) {
744 // We don't want to emit a trampoline if C2 is generating dummy
745 // code during its branch shortening phase.
746 CompileTask* task = ciEnv::current()->task();
747 bool in_scratch_emit_size =
748 (task != NULL && is_c2_compile(task->comp_level()) &&
749 Compile::current()->in_scratch_emit_size());
750 if (!in_scratch_emit_size) {
751 address stub = emit_trampoline_stub(offset(), entry.target());
752 if (stub == NULL) {
753 return NULL; // CodeCache is full
754 }
755 }
756 }
757
758 if (cbuf) cbuf->set_insts_mark();
759 relocate(entry.rspec());
760 if (!far_branches()) {
761 bl(entry.target());
762 } else {
763 bl(pc());
764 }
765 // just need to return a non-null address
766 return pc();
767 }
768
769
770 // Emit a trampoline stub for a call to a target which is too far away.
771 //
772 // code sequences:
773 //
774 // call-site:
775 // branch-and-link to <destination> or <trampoline stub>
776 //
777 // Related trampoline stub for this call site in the stub section:
778 // load the call target from the constant pool
779 // branch (LR still points to the call site above)
780
781 address MacroAssembler::emit_trampoline_stub(int insts_call_instruction_offset,
782 address dest) {
783 address stub = start_a_stub(Compile::MAX_stubs_size/2);
784 if (stub == NULL) {
785 return NULL; // CodeBuffer::expand failed
786 }
787
788 // Create a trampoline stub relocation which relates this trampoline stub
789 // with the call instruction at insts_call_instruction_offset in the
790 // instructions code-section.
791 align(wordSize);
792 relocate(trampoline_stub_Relocation::spec(code()->insts()->start()
793 + insts_call_instruction_offset));
794 const int stub_start_offset = offset();
795
796 // Now, create the trampoline stub's code:
797 // - load the call
798 // - call
799 Label target;
800 ldr(rscratch1, target);
801 br(rscratch1);
802 bind(target);
803 assert(offset() - stub_start_offset == NativeCallTrampolineStub::data_offset,
804 "should be");
805 emit_int64((int64_t)dest);
806
807 const address stub_start_addr = addr_at(stub_start_offset);
808
809 assert(is_NativeCallTrampolineStub_at(stub_start_addr), "doesn't look like a trampoline");
810
811 end_a_stub();
812 return stub_start_addr;
813 }
814
815 address MacroAssembler::ic_call(address entry, jint method_index) {
816 RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
817 // address const_ptr = long_constant((jlong)Universe::non_oop_word());
818 // unsigned long offset;
819 // ldr_constant(rscratch2, const_ptr);
820 movptr(rscratch2, (uintptr_t)Universe::non_oop_word());
821 return trampoline_call(Address(entry, rh));
822 }
823
824 // Implementation of call_VM versions
825
826 void MacroAssembler::call_VM(Register oop_result,
827 address entry_point,
828 bool check_exceptions) {
829 call_VM_helper(oop_result, entry_point, 0, check_exceptions);
830 }
831
832 void MacroAssembler::call_VM(Register oop_result,
833 address entry_point,
834 Register arg_1,
835 bool check_exceptions) {
836 pass_arg1(this, arg_1);
837 call_VM_helper(oop_result, entry_point, 1, check_exceptions);
838 }
839
840 void MacroAssembler::call_VM(Register oop_result,
841 address entry_point,
842 Register arg_1,
843 Register arg_2,
844 bool check_exceptions) {
845 assert(arg_1 != c_rarg2, "smashed arg");
846 pass_arg2(this, arg_2);
847 pass_arg1(this, arg_1);
848 call_VM_helper(oop_result, entry_point, 2, check_exceptions);
849 }
850
851 void MacroAssembler::call_VM(Register oop_result,
852 address entry_point,
853 Register arg_1,
854 Register arg_2,
855 Register arg_3,
856 bool check_exceptions) {
857 assert(arg_1 != c_rarg3, "smashed arg");
858 assert(arg_2 != c_rarg3, "smashed arg");
859 pass_arg3(this, arg_3);
860
861 assert(arg_1 != c_rarg2, "smashed arg");
862 pass_arg2(this, arg_2);
863
864 pass_arg1(this, arg_1);
865 call_VM_helper(oop_result, entry_point, 3, check_exceptions);
866 }
867
868 void MacroAssembler::call_VM(Register oop_result,
869 Register last_java_sp,
870 address entry_point,
871 int number_of_arguments,
872 bool check_exceptions) {
873 call_VM_base(oop_result, rthread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
874 }
875
876 void MacroAssembler::call_VM(Register oop_result,
877 Register last_java_sp,
878 address entry_point,
879 Register arg_1,
880 bool check_exceptions) {
881 pass_arg1(this, arg_1);
882 call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
883 }
884
885 void MacroAssembler::call_VM(Register oop_result,
886 Register last_java_sp,
887 address entry_point,
888 Register arg_1,
889 Register arg_2,
890 bool check_exceptions) {
891
892 assert(arg_1 != c_rarg2, "smashed arg");
893 pass_arg2(this, arg_2);
894 pass_arg1(this, arg_1);
895 call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
896 }
897
898 void MacroAssembler::call_VM(Register oop_result,
899 Register last_java_sp,
900 address entry_point,
901 Register arg_1,
902 Register arg_2,
903 Register arg_3,
904 bool check_exceptions) {
905 assert(arg_1 != c_rarg3, "smashed arg");
906 assert(arg_2 != c_rarg3, "smashed arg");
907 pass_arg3(this, arg_3);
908 assert(arg_1 != c_rarg2, "smashed arg");
909 pass_arg2(this, arg_2);
910 pass_arg1(this, arg_1);
911 call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
912 }
913
914
915 void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) {
916 ldr(oop_result, Address(java_thread, JavaThread::vm_result_offset()));
917 str(zr, Address(java_thread, JavaThread::vm_result_offset()));
918 verify_oop(oop_result, "broken oop in call_VM_base");
919 }
920
921 void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) {
922 ldr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset()));
923 str(zr, Address(java_thread, JavaThread::vm_result_2_offset()));
924 }
925
926 void MacroAssembler::align(int modulus) {
927 while (offset() % modulus != 0) nop();
928 }
929
930 // these are no-ops overridden by InterpreterMacroAssembler
931
932 void MacroAssembler::check_and_handle_earlyret(Register java_thread) { }
933
934 void MacroAssembler::check_and_handle_popframe(Register java_thread) { }
935
936
937 RegisterOrConstant MacroAssembler::delayed_value_impl(intptr_t* delayed_value_addr,
938 Register tmp,
939 int offset) {
940 intptr_t value = *delayed_value_addr;
941 if (value != 0)
942 return RegisterOrConstant(value + offset);
943
944 // load indirectly to solve generation ordering problem
945 ldr(tmp, ExternalAddress((address) delayed_value_addr));
946
947 if (offset != 0)
948 add(tmp, tmp, offset);
949
950 return RegisterOrConstant(tmp);
951 }
952
953
954 void MacroAssembler:: notify(int type) {
955 if (type == bytecode_start) {
956 // set_last_Java_frame(esp, rfp, (address)NULL);
957 Assembler:: notify(type);
958 // reset_last_Java_frame(true);
959 }
960 else
961 Assembler:: notify(type);
962 }
963
964 // Look up the method for a megamorphic invokeinterface call.
965 // The target method is determined by <intf_klass, itable_index>.
966 // The receiver klass is in recv_klass.
967 // On success, the result will be in method_result, and execution falls through.
968 // On failure, execution transfers to the given label.
969 void MacroAssembler::lookup_interface_method(Register recv_klass,
970 Register intf_klass,
971 RegisterOrConstant itable_index,
972 Register method_result,
973 Register scan_temp,
974 Label& L_no_such_interface,
975 bool return_method) {
976 assert_different_registers(recv_klass, intf_klass, scan_temp);
977 assert_different_registers(method_result, intf_klass, scan_temp);
978 assert(recv_klass != method_result || !return_method,
979 "recv_klass can be destroyed when method isn't needed");
980 assert(itable_index.is_constant() || itable_index.as_register() == method_result,
981 "caller must use same register for non-constant itable index as for method");
982
983 // Compute start of first itableOffsetEntry (which is at the end of the vtable)
984 int vtable_base = in_bytes(Klass::vtable_start_offset());
985 int itentry_off = itableMethodEntry::method_offset_in_bytes();
986 int scan_step = itableOffsetEntry::size() * wordSize;
987 int vte_size = vtableEntry::size_in_bytes();
988 assert(vte_size == wordSize, "else adjust times_vte_scale");
989
990 ldrw(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
991
992 // %%% Could store the aligned, prescaled offset in the klassoop.
993 // lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base));
994 lea(scan_temp, Address(recv_klass, scan_temp, Address::lsl(3)));
995 add(scan_temp, scan_temp, vtable_base);
996
997 if (return_method) {
998 // Adjust recv_klass by scaled itable_index, so we can free itable_index.
999 assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
1000 // lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off));
1001 lea(recv_klass, Address(recv_klass, itable_index, Address::lsl(3)));
1002 if (itentry_off)
1003 add(recv_klass, recv_klass, itentry_off);
1004 }
1005
1006 // for (scan = klass->itable(); scan->interface() != NULL; scan += scan_step) {
1007 // if (scan->interface() == intf) {
1008 // result = (klass + scan->offset() + itable_index);
1009 // }
1010 // }
1011 Label search, found_method;
1012
1013 for (int peel = 1; peel >= 0; peel--) {
1014 ldr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset_in_bytes()));
1015 cmp(intf_klass, method_result);
1016
1017 if (peel) {
1018 br(Assembler::EQ, found_method);
1019 } else {
1020 br(Assembler::NE, search);
1021 // (invert the test to fall through to found_method...)
1022 }
1023
1024 if (!peel) break;
1025
1026 bind(search);
1027
1028 // Check that the previous entry is non-null. A null entry means that
1029 // the receiver class doesn't implement the interface, and wasn't the
1030 // same as when the caller was compiled.
1031 cbz(method_result, L_no_such_interface);
1032 add(scan_temp, scan_temp, scan_step);
1033 }
1034
1035 bind(found_method);
1036
1037 // Got a hit.
1038 if (return_method) {
1039 ldrw(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset_in_bytes()));
1040 ldr(method_result, Address(recv_klass, scan_temp, Address::uxtw(0)));
1041 }
1042 }
1043
1044 // virtual method calling
1045 void MacroAssembler::lookup_virtual_method(Register recv_klass,
1046 RegisterOrConstant vtable_index,
1047 Register method_result) {
1048 const int base = in_bytes(Klass::vtable_start_offset());
1049 assert(vtableEntry::size() * wordSize == 8,
1050 "adjust the scaling in the code below");
1051 int vtable_offset_in_bytes = base + vtableEntry::method_offset_in_bytes();
1052
1053 if (vtable_index.is_register()) {
1054 lea(method_result, Address(recv_klass,
1055 vtable_index.as_register(),
1056 Address::lsl(LogBytesPerWord)));
1057 ldr(method_result, Address(method_result, vtable_offset_in_bytes));
1058 } else {
1059 vtable_offset_in_bytes += vtable_index.as_constant() * wordSize;
1060 ldr(method_result,
1061 form_address(rscratch1, recv_klass, vtable_offset_in_bytes, 0));
1062 }
1063 }
1064
1065 void MacroAssembler::check_klass_subtype(Register sub_klass,
1066 Register super_klass,
1067 Register temp_reg,
1068 Label& L_success) {
1069 Label L_failure;
1070 check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg, &L_success, &L_failure, NULL);
1071 check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, NULL);
1072 bind(L_failure);
1073 }
1074
1075
1076 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
1077 Register super_klass,
1078 Register temp_reg,
1079 Label* L_success,
1080 Label* L_failure,
1081 Label* L_slow_path,
1082 RegisterOrConstant super_check_offset) {
1083 assert_different_registers(sub_klass, super_klass, temp_reg);
1084 bool must_load_sco = (super_check_offset.constant_or_zero() == -1);
1085 if (super_check_offset.is_register()) {
1086 assert_different_registers(sub_klass, super_klass,
1087 super_check_offset.as_register());
1088 } else if (must_load_sco) {
1089 assert(temp_reg != noreg, "supply either a temp or a register offset");
1090 }
1091
1092 Label L_fallthrough;
1093 int label_nulls = 0;
1094 if (L_success == NULL) { L_success = &L_fallthrough; label_nulls++; }
1095 if (L_failure == NULL) { L_failure = &L_fallthrough; label_nulls++; }
1096 if (L_slow_path == NULL) { L_slow_path = &L_fallthrough; label_nulls++; }
1097 assert(label_nulls <= 1, "at most one NULL in the batch");
1098
1099 int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
1100 int sco_offset = in_bytes(Klass::super_check_offset_offset());
1101 Address super_check_offset_addr(super_klass, sco_offset);
1102
1103 // Hacked jmp, which may only be used just before L_fallthrough.
1104 #define final_jmp(label) \
1105 if (&(label) == &L_fallthrough) { /*do nothing*/ } \
1106 else b(label) /*omit semi*/
1107
1108 // If the pointers are equal, we are done (e.g., String[] elements).
1109 // This self-check enables sharing of secondary supertype arrays among
1110 // non-primary types such as array-of-interface. Otherwise, each such
1111 // type would need its own customized SSA.
1112 // We move this check to the front of the fast path because many
1113 // type checks are in fact trivially successful in this manner,
1114 // so we get a nicely predicted branch right at the start of the check.
1115 cmp(sub_klass, super_klass);
1116 br(Assembler::EQ, *L_success);
1117
1118 // Check the supertype display:
1119 if (must_load_sco) {
1120 ldrw(temp_reg, super_check_offset_addr);
1121 super_check_offset = RegisterOrConstant(temp_reg);
1122 }
1123 Address super_check_addr(sub_klass, super_check_offset);
1124 ldr(rscratch1, super_check_addr);
1125 cmp(super_klass, rscratch1); // load displayed supertype
1126
1127 // This check has worked decisively for primary supers.
1128 // Secondary supers are sought in the super_cache ('super_cache_addr').
1129 // (Secondary supers are interfaces and very deeply nested subtypes.)
1130 // This works in the same check above because of a tricky aliasing
1131 // between the super_cache and the primary super display elements.
1132 // (The 'super_check_addr' can address either, as the case requires.)
1133 // Note that the cache is updated below if it does not help us find
1134 // what we need immediately.
1135 // So if it was a primary super, we can just fail immediately.
1136 // Otherwise, it's the slow path for us (no success at this point).
1137
1138 if (super_check_offset.is_register()) {
1139 br(Assembler::EQ, *L_success);
1140 cmp(super_check_offset.as_register(), sc_offset);
1141 if (L_failure == &L_fallthrough) {
1142 br(Assembler::EQ, *L_slow_path);
1143 } else {
1144 br(Assembler::NE, *L_failure);
1145 final_jmp(*L_slow_path);
1146 }
1147 } else if (super_check_offset.as_constant() == sc_offset) {
1148 // Need a slow path; fast failure is impossible.
1149 if (L_slow_path == &L_fallthrough) {
1150 br(Assembler::EQ, *L_success);
1151 } else {
1152 br(Assembler::NE, *L_slow_path);
1153 final_jmp(*L_success);
1154 }
1155 } else {
1156 // No slow path; it's a fast decision.
1157 if (L_failure == &L_fallthrough) {
1158 br(Assembler::EQ, *L_success);
1159 } else {
1160 br(Assembler::NE, *L_failure);
1161 final_jmp(*L_success);
1162 }
1163 }
1164
1165 bind(L_fallthrough);
1166
1167 #undef final_jmp
1168 }
1169
1170 // These two are taken from x86, but they look generally useful
1171
1172 // scans count pointer sized words at [addr] for occurence of value,
1173 // generic
1174 void MacroAssembler::repne_scan(Register addr, Register value, Register count,
1175 Register scratch) {
1176 Label Lloop, Lexit;
1177 cbz(count, Lexit);
1178 bind(Lloop);
1179 ldr(scratch, post(addr, wordSize));
1180 cmp(value, scratch);
1181 br(EQ, Lexit);
1182 sub(count, count, 1);
1183 cbnz(count, Lloop);
1184 bind(Lexit);
1185 }
1186
1187 // scans count 4 byte words at [addr] for occurence of value,
1188 // generic
1189 void MacroAssembler::repne_scanw(Register addr, Register value, Register count,
1190 Register scratch) {
1191 Label Lloop, Lexit;
1192 cbz(count, Lexit);
1193 bind(Lloop);
1194 ldrw(scratch, post(addr, wordSize));
1195 cmpw(value, scratch);
1196 br(EQ, Lexit);
1197 sub(count, count, 1);
1198 cbnz(count, Lloop);
1199 bind(Lexit);
1200 }
1201
1202 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
1203 Register super_klass,
1204 Register temp_reg,
1205 Register temp2_reg,
1206 Label* L_success,
1207 Label* L_failure,
1208 bool set_cond_codes) {
1209 assert_different_registers(sub_klass, super_klass, temp_reg);
1210 if (temp2_reg != noreg)
1211 assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg, rscratch1);
1212 #define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg)
1213
1214 Label L_fallthrough;
1215 int label_nulls = 0;
1216 if (L_success == NULL) { L_success = &L_fallthrough; label_nulls++; }
1217 if (L_failure == NULL) { L_failure = &L_fallthrough; label_nulls++; }
1218 assert(label_nulls <= 1, "at most one NULL in the batch");
1219
1220 // a couple of useful fields in sub_klass:
1221 int ss_offset = in_bytes(Klass::secondary_supers_offset());
1222 int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
1223 Address secondary_supers_addr(sub_klass, ss_offset);
1224 Address super_cache_addr( sub_klass, sc_offset);
1225
1226 BLOCK_COMMENT("check_klass_subtype_slow_path");
1227
1228 // Do a linear scan of the secondary super-klass chain.
1229 // This code is rarely used, so simplicity is a virtue here.
1230 // The repne_scan instruction uses fixed registers, which we must spill.
1231 // Don't worry too much about pre-existing connections with the input regs.
1232
1233 assert(sub_klass != r0, "killed reg"); // killed by mov(r0, super)
1234 assert(sub_klass != r2, "killed reg"); // killed by lea(r2, &pst_counter)
1235
1236 RegSet pushed_registers;
1237 if (!IS_A_TEMP(r2)) pushed_registers += r2;
1238 if (!IS_A_TEMP(r5)) pushed_registers += r5;
1239
1240 if (super_klass != r0 || UseCompressedOops) {
1241 if (!IS_A_TEMP(r0)) pushed_registers += r0;
1242 }
1243
1244 push(pushed_registers, sp);
1245
1246 // Get super_klass value into r0 (even if it was in r5 or r2).
1247 if (super_klass != r0) {
1248 mov(r0, super_klass);
1249 }
1250
1251 #ifndef PRODUCT
1252 mov(rscratch2, (address)&SharedRuntime::_partial_subtype_ctr);
1253 Address pst_counter_addr(rscratch2);
1254 ldr(rscratch1, pst_counter_addr);
1255 add(rscratch1, rscratch1, 1);
1256 str(rscratch1, pst_counter_addr);
1257 #endif //PRODUCT
1258
1259 // We will consult the secondary-super array.
1260 ldr(r5, secondary_supers_addr);
1261 // Load the array length.
1262 ldrw(r2, Address(r5, Array<Klass*>::length_offset_in_bytes()));
1263 // Skip to start of data.
1264 add(r5, r5, Array<Klass*>::base_offset_in_bytes());
1265
1266 cmp(sp, zr); // Clear Z flag; SP is never zero
1267 // Scan R2 words at [R5] for an occurrence of R0.
1268 // Set NZ/Z based on last compare.
1269 repne_scan(r5, r0, r2, rscratch1);
1270
1271 // Unspill the temp. registers:
1272 pop(pushed_registers, sp);
1273
1274 br(Assembler::NE, *L_failure);
1275
1276 // Success. Cache the super we found and proceed in triumph.
1277 str(super_klass, super_cache_addr);
1278
1279 if (L_success != &L_fallthrough) {
1280 b(*L_success);
1281 }
1282
1283 #undef IS_A_TEMP
1284
1285 bind(L_fallthrough);
1286 }
1287
1288
1289 void MacroAssembler::verify_oop(Register reg, const char* s) {
1290 if (!VerifyOops) return;
1291
1292 // Pass register number to verify_oop_subroutine
1293 const char* b = NULL;
1294 {
1295 ResourceMark rm;
1296 stringStream ss;
1297 ss.print("verify_oop: %s: %s", reg->name(), s);
1298 b = code_string(ss.as_string());
1299 }
1300 BLOCK_COMMENT("verify_oop {");
1301
1302 stp(r0, rscratch1, Address(pre(sp, -2 * wordSize)));
1303 stp(rscratch2, lr, Address(pre(sp, -2 * wordSize)));
1304
1305 mov(r0, reg);
1306 mov(rscratch1, (address)b);
1307
1308 // call indirectly to solve generation ordering problem
1309 lea(rscratch2, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
1310 ldr(rscratch2, Address(rscratch2));
1311 blr(rscratch2);
1312
1313 ldp(rscratch2, lr, Address(post(sp, 2 * wordSize)));
1314 ldp(r0, rscratch1, Address(post(sp, 2 * wordSize)));
1315
1316 BLOCK_COMMENT("} verify_oop");
1317 }
1318
1319 void MacroAssembler::verify_oop_addr(Address addr, const char* s) {
1320 if (!VerifyOops) return;
1321
1322 const char* b = NULL;
1323 {
1324 ResourceMark rm;
1325 stringStream ss;
1326 ss.print("verify_oop_addr: %s", s);
1327 b = code_string(ss.as_string());
1328 }
1329 BLOCK_COMMENT("verify_oop_addr {");
1330
1331 stp(r0, rscratch1, Address(pre(sp, -2 * wordSize)));
1332 stp(rscratch2, lr, Address(pre(sp, -2 * wordSize)));
1333
1334 // addr may contain sp so we will have to adjust it based on the
1335 // pushes that we just did.
1336 if (addr.uses(sp)) {
1337 lea(r0, addr);
1338 ldr(r0, Address(r0, 4 * wordSize));
1339 } else {
1340 ldr(r0, addr);
1341 }
1342 mov(rscratch1, (address)b);
1343
1344 // call indirectly to solve generation ordering problem
1345 lea(rscratch2, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
1346 ldr(rscratch2, Address(rscratch2));
1347 blr(rscratch2);
1348
1349 ldp(rscratch2, lr, Address(post(sp, 2 * wordSize)));
1350 ldp(r0, rscratch1, Address(post(sp, 2 * wordSize)));
1351
1352 BLOCK_COMMENT("} verify_oop_addr");
1353 }
1354
1355 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot,
1356 int extra_slot_offset) {
1357 // cf. TemplateTable::prepare_invoke(), if (load_receiver).
1358 int stackElementSize = Interpreter::stackElementSize;
1359 int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0);
1360 #ifdef ASSERT
1361 int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1);
1362 assert(offset1 - offset == stackElementSize, "correct arithmetic");
1363 #endif
1364 if (arg_slot.is_constant()) {
1365 return Address(esp, arg_slot.as_constant() * stackElementSize
1366 + offset);
1367 } else {
1368 add(rscratch1, esp, arg_slot.as_register(),
1369 ext::uxtx, exact_log2(stackElementSize));
1370 return Address(rscratch1, offset);
1371 }
1372 }
1373
1374 void MacroAssembler::call_VM_leaf_base(address entry_point,
1375 int number_of_arguments,
1376 Label *retaddr) {
1377 call_VM_leaf_base1(entry_point, number_of_arguments, 0, ret_type_integral, retaddr);
1378 }
1379
1380 void MacroAssembler::call_VM_leaf_base1(address entry_point,
1381 int number_of_gp_arguments,
1382 int number_of_fp_arguments,
1383 ret_type type,
1384 Label *retaddr) {
1385 Label E, L;
1386
1387 stp(rscratch1, rmethod, Address(pre(sp, -2 * wordSize)));
1388
1389 // We add 1 to number_of_arguments because the thread in arg0 is
1390 // not counted
1391 mov(rscratch1, entry_point);
1392 blrt(rscratch1, number_of_gp_arguments + 1, number_of_fp_arguments, type);
1393 if (retaddr)
1394 bind(*retaddr);
1395
1396 ldp(rscratch1, rmethod, Address(post(sp, 2 * wordSize)));
1397 maybe_isb();
1398 }
1399
1400 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
1401 call_VM_leaf_base(entry_point, number_of_arguments);
1402 }
1403
1404 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {
1405 pass_arg0(this, arg_0);
1406 call_VM_leaf_base(entry_point, 1);
1407 }
1408
1409 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
1410 pass_arg0(this, arg_0);
1411 pass_arg1(this, arg_1);
1412 call_VM_leaf_base(entry_point, 2);
1413 }
1414
1415 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0,
1416 Register arg_1, Register arg_2) {
1417 pass_arg0(this, arg_0);
1418 pass_arg1(this, arg_1);
1419 pass_arg2(this, arg_2);
1420 call_VM_leaf_base(entry_point, 3);
1421 }
1422
1423 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
1424 pass_arg0(this, arg_0);
1425 MacroAssembler::call_VM_leaf_base(entry_point, 1);
1426 }
1427
1428 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
1429
1430 assert(arg_0 != c_rarg1, "smashed arg");
1431 pass_arg1(this, arg_1);
1432 pass_arg0(this, arg_0);
1433 MacroAssembler::call_VM_leaf_base(entry_point, 2);
1434 }
1435
1436 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
1437 assert(arg_0 != c_rarg2, "smashed arg");
1438 assert(arg_1 != c_rarg2, "smashed arg");
1439 pass_arg2(this, arg_2);
1440 assert(arg_0 != c_rarg1, "smashed arg");
1441 pass_arg1(this, arg_1);
1442 pass_arg0(this, arg_0);
1443 MacroAssembler::call_VM_leaf_base(entry_point, 3);
1444 }
1445
1446 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
1447 assert(arg_0 != c_rarg3, "smashed arg");
1448 assert(arg_1 != c_rarg3, "smashed arg");
1449 assert(arg_2 != c_rarg3, "smashed arg");
1450 pass_arg3(this, arg_3);
1451 assert(arg_0 != c_rarg2, "smashed arg");
1452 assert(arg_1 != c_rarg2, "smashed arg");
1453 pass_arg2(this, arg_2);
1454 assert(arg_0 != c_rarg1, "smashed arg");
1455 pass_arg1(this, arg_1);
1456 pass_arg0(this, arg_0);
1457 MacroAssembler::call_VM_leaf_base(entry_point, 4);
1458 }
1459
1460 void MacroAssembler::null_check(Register reg, int offset) {
1461 if (needs_explicit_null_check(offset)) {
1462 // provoke OS NULL exception if reg = NULL by
1463 // accessing M[reg] w/o changing any registers
1464 // NOTE: this is plenty to provoke a segv
1465 ldr(zr, Address(reg));
1466 } else {
1467 // nothing to do, (later) access of M[reg + offset]
1468 // will provoke OS NULL exception if reg = NULL
1469 }
1470 }
1471
1472 // MacroAssembler protected routines needed to implement
1473 // public methods
1474
1475 void MacroAssembler::mov(Register r, Address dest) {
1476 code_section()->relocate(pc(), dest.rspec());
1477 u_int64_t imm64 = (u_int64_t)dest.target();
1478 movptr(r, imm64);
1479 }
1480
1481 // Move a constant pointer into r. In AArch64 mode the virtual
1482 // address space is 48 bits in size, so we only need three
1483 // instructions to create a patchable instruction sequence that can
1484 // reach anywhere.
1485 void MacroAssembler::movptr(Register r, uintptr_t imm64) {
1486 #ifndef PRODUCT
1487 {
1488 char buffer[64];
1489 snprintf(buffer, sizeof(buffer), "0x%"PRIX64, imm64);
1490 block_comment(buffer);
1491 }
1492 #endif
1493 assert(imm64 < (1ul << 48), "48-bit overflow in address constant");
1494 movz(r, imm64 & 0xffff);
1495 imm64 >>= 16;
1496 movk(r, imm64 & 0xffff, 16);
1497 imm64 >>= 16;
1498 movk(r, imm64 & 0xffff, 32);
1499 }
1500
1501 // Macro to mov replicated immediate to vector register.
1502 // Vd will get the following values for different arrangements in T
1503 // imm32 == hex 000000gh T8B: Vd = ghghghghghghghgh
1504 // imm32 == hex 000000gh T16B: Vd = ghghghghghghghghghghghghghghghgh
1505 // imm32 == hex 0000efgh T4H: Vd = efghefghefghefgh
1506 // imm32 == hex 0000efgh T8H: Vd = efghefghefghefghefghefghefghefgh
1507 // imm32 == hex abcdefgh T2S: Vd = abcdefghabcdefgh
1508 // imm32 == hex abcdefgh T4S: Vd = abcdefghabcdefghabcdefghabcdefgh
1509 // T1D/T2D: invalid
1510 void MacroAssembler::mov(FloatRegister Vd, SIMD_Arrangement T, u_int32_t imm32) {
1511 assert(T != T1D && T != T2D, "invalid arrangement");
1512 if (T == T8B || T == T16B) {
1513 assert((imm32 & ~0xff) == 0, "extraneous bits in unsigned imm32 (T8B/T16B)");
1514 movi(Vd, T, imm32 & 0xff, 0);
1515 return;
1516 }
1517 u_int32_t nimm32 = ~imm32;
1518 if (T == T4H || T == T8H) {
1519 assert((imm32 & ~0xffff) == 0, "extraneous bits in unsigned imm32 (T4H/T8H)");
1520 imm32 &= 0xffff;
1521 nimm32 &= 0xffff;
1522 }
1523 u_int32_t x = imm32;
1524 int movi_cnt = 0;
1525 int movn_cnt = 0;
1526 while (x) { if (x & 0xff) movi_cnt++; x >>= 8; }
1527 x = nimm32;
1528 while (x) { if (x & 0xff) movn_cnt++; x >>= 8; }
1529 if (movn_cnt < movi_cnt) imm32 = nimm32;
1530 unsigned lsl = 0;
1531 while (imm32 && (imm32 & 0xff) == 0) { lsl += 8; imm32 >>= 8; }
1532 if (movn_cnt < movi_cnt)
1533 mvni(Vd, T, imm32 & 0xff, lsl);
1534 else
1535 movi(Vd, T, imm32 & 0xff, lsl);
1536 imm32 >>= 8; lsl += 8;
1537 while (imm32) {
1538 while ((imm32 & 0xff) == 0) { lsl += 8; imm32 >>= 8; }
1539 if (movn_cnt < movi_cnt)
1540 bici(Vd, T, imm32 & 0xff, lsl);
1541 else
1542 orri(Vd, T, imm32 & 0xff, lsl);
1543 lsl += 8; imm32 >>= 8;
1544 }
1545 }
1546
1547 void MacroAssembler::mov_immediate64(Register dst, u_int64_t imm64)
1548 {
1549 #ifndef PRODUCT
1550 {
1551 char buffer[64];
1552 snprintf(buffer, sizeof(buffer), "0x%"PRIX64, imm64);
1553 block_comment(buffer);
1554 }
1555 #endif
1556 if (operand_valid_for_logical_immediate(false, imm64)) {
1557 orr(dst, zr, imm64);
1558 } else {
1559 // we can use a combination of MOVZ or MOVN with
1560 // MOVK to build up the constant
1561 u_int64_t imm_h[4];
1562 int zero_count = 0;
1563 int neg_count = 0;
1564 int i;
1565 for (i = 0; i < 4; i++) {
1566 imm_h[i] = ((imm64 >> (i * 16)) & 0xffffL);
1567 if (imm_h[i] == 0) {
1568 zero_count++;
1569 } else if (imm_h[i] == 0xffffL) {
1570 neg_count++;
1571 }
1572 }
1573 if (zero_count == 4) {
1574 // one MOVZ will do
1575 movz(dst, 0);
1576 } else if (neg_count == 4) {
1577 // one MOVN will do
1578 movn(dst, 0);
1579 } else if (zero_count == 3) {
1580 for (i = 0; i < 4; i++) {
1581 if (imm_h[i] != 0L) {
1582 movz(dst, (u_int32_t)imm_h[i], (i << 4));
1583 break;
1584 }
1585 }
1586 } else if (neg_count == 3) {
1587 // one MOVN will do
1588 for (int i = 0; i < 4; i++) {
1589 if (imm_h[i] != 0xffffL) {
1590 movn(dst, (u_int32_t)imm_h[i] ^ 0xffffL, (i << 4));
1591 break;
1592 }
1593 }
1594 } else if (zero_count == 2) {
1595 // one MOVZ and one MOVK will do
1596 for (i = 0; i < 3; i++) {
1597 if (imm_h[i] != 0L) {
1598 movz(dst, (u_int32_t)imm_h[i], (i << 4));
1599 i++;
1600 break;
1601 }
1602 }
1603 for (;i < 4; i++) {
1604 if (imm_h[i] != 0L) {
1605 movk(dst, (u_int32_t)imm_h[i], (i << 4));
1606 }
1607 }
1608 } else if (neg_count == 2) {
1609 // one MOVN and one MOVK will do
1610 for (i = 0; i < 4; i++) {
1611 if (imm_h[i] != 0xffffL) {
1612 movn(dst, (u_int32_t)imm_h[i] ^ 0xffffL, (i << 4));
1613 i++;
1614 break;
1615 }
1616 }
1617 for (;i < 4; i++) {
1618 if (imm_h[i] != 0xffffL) {
1619 movk(dst, (u_int32_t)imm_h[i], (i << 4));
1620 }
1621 }
1622 } else if (zero_count == 1) {
1623 // one MOVZ and two MOVKs will do
1624 for (i = 0; i < 4; i++) {
1625 if (imm_h[i] != 0L) {
1626 movz(dst, (u_int32_t)imm_h[i], (i << 4));
1627 i++;
1628 break;
1629 }
1630 }
1631 for (;i < 4; i++) {
1632 if (imm_h[i] != 0x0L) {
1633 movk(dst, (u_int32_t)imm_h[i], (i << 4));
1634 }
1635 }
1636 } else if (neg_count == 1) {
1637 // one MOVN and two MOVKs will do
1638 for (i = 0; i < 4; i++) {
1639 if (imm_h[i] != 0xffffL) {
1640 movn(dst, (u_int32_t)imm_h[i] ^ 0xffffL, (i << 4));
1641 i++;
1642 break;
1643 }
1644 }
1645 for (;i < 4; i++) {
1646 if (imm_h[i] != 0xffffL) {
1647 movk(dst, (u_int32_t)imm_h[i], (i << 4));
1648 }
1649 }
1650 } else {
1651 // use a MOVZ and 3 MOVKs (makes it easier to debug)
1652 movz(dst, (u_int32_t)imm_h[0], 0);
1653 for (i = 1; i < 4; i++) {
1654 movk(dst, (u_int32_t)imm_h[i], (i << 4));
1655 }
1656 }
1657 }
1658 }
1659
1660 void MacroAssembler::mov_immediate32(Register dst, u_int32_t imm32)
1661 {
1662 #ifndef PRODUCT
1663 {
1664 char buffer[64];
1665 snprintf(buffer, sizeof(buffer), "0x%"PRIX32, imm32);
1666 block_comment(buffer);
1667 }
1668 #endif
1669 if (operand_valid_for_logical_immediate(true, imm32)) {
1670 orrw(dst, zr, imm32);
1671 } else {
1672 // we can use MOVZ, MOVN or two calls to MOVK to build up the
1673 // constant
1674 u_int32_t imm_h[2];
1675 imm_h[0] = imm32 & 0xffff;
1676 imm_h[1] = ((imm32 >> 16) & 0xffff);
1677 if (imm_h[0] == 0) {
1678 movzw(dst, imm_h[1], 16);
1679 } else if (imm_h[0] == 0xffff) {
1680 movnw(dst, imm_h[1] ^ 0xffff, 16);
1681 } else if (imm_h[1] == 0) {
1682 movzw(dst, imm_h[0], 0);
1683 } else if (imm_h[1] == 0xffff) {
1684 movnw(dst, imm_h[0] ^ 0xffff, 0);
1685 } else {
1686 // use a MOVZ and MOVK (makes it easier to debug)
1687 movzw(dst, imm_h[0], 0);
1688 movkw(dst, imm_h[1], 16);
1689 }
1690 }
1691 }
1692
1693 // Form an address from base + offset in Rd. Rd may or may
1694 // not actually be used: you must use the Address that is returned.
1695 // It is up to you to ensure that the shift provided matches the size
1696 // of your data.
1697 Address MacroAssembler::form_address(Register Rd, Register base, long byte_offset, int shift) {
1698 if (Address::offset_ok_for_immed(byte_offset, shift))
1699 // It fits; no need for any heroics
1700 return Address(base, byte_offset);
1701
1702 // Don't do anything clever with negative or misaligned offsets
1703 unsigned mask = (1 << shift) - 1;
1704 if (byte_offset < 0 || byte_offset & mask) {
1705 mov(Rd, byte_offset);
1706 add(Rd, base, Rd);
1707 return Address(Rd);
1708 }
1709
1710 // See if we can do this with two 12-bit offsets
1711 {
1712 unsigned long word_offset = byte_offset >> shift;
1713 unsigned long masked_offset = word_offset & 0xfff000;
1714 if (Address::offset_ok_for_immed(word_offset - masked_offset)
1715 && Assembler::operand_valid_for_add_sub_immediate(masked_offset << shift)) {
1716 add(Rd, base, masked_offset << shift);
1717 word_offset -= masked_offset;
1718 return Address(Rd, word_offset << shift);
1719 }
1720 }
1721
1722 // Do it the hard way
1723 mov(Rd, byte_offset);
1724 add(Rd, base, Rd);
1725 return Address(Rd);
1726 }
1727
1728 void MacroAssembler::atomic_incw(Register counter_addr, Register tmp, Register tmp2) {
1729 if (UseLSE) {
1730 mov(tmp, 1);
1731 ldadd(Assembler::word, tmp, zr, counter_addr);
1732 return;
1733 }
1734 Label retry_load;
1735 if ((VM_Version::features() & VM_Version::CPU_STXR_PREFETCH))
1736 prfm(Address(counter_addr), PSTL1STRM);
1737 bind(retry_load);
1738 // flush and load exclusive from the memory location
1739 ldxrw(tmp, counter_addr);
1740 addw(tmp, tmp, 1);
1741 // if we store+flush with no intervening write tmp wil be zero
1742 stxrw(tmp2, tmp, counter_addr);
1743 cbnzw(tmp2, retry_load);
1744 }
1745
1746
1747 int MacroAssembler::corrected_idivl(Register result, Register ra, Register rb,
1748 bool want_remainder, Register scratch)
1749 {
1750 // Full implementation of Java idiv and irem. The function
1751 // returns the (pc) offset of the div instruction - may be needed
1752 // for implicit exceptions.
1753 //
1754 // constraint : ra/rb =/= scratch
1755 // normal case
1756 //
1757 // input : ra: dividend
1758 // rb: divisor
1759 //
1760 // result: either
1761 // quotient (= ra idiv rb)
1762 // remainder (= ra irem rb)
1763
1764 assert(ra != scratch && rb != scratch, "reg cannot be scratch");
1765
1766 int idivl_offset = offset();
1767 if (! want_remainder) {
1768 sdivw(result, ra, rb);
1769 } else {
1770 sdivw(scratch, ra, rb);
1771 Assembler::msubw(result, scratch, rb, ra);
1772 }
1773
1774 return idivl_offset;
1775 }
1776
1777 int MacroAssembler::corrected_idivq(Register result, Register ra, Register rb,
1778 bool want_remainder, Register scratch)
1779 {
1780 // Full implementation of Java ldiv and lrem. The function
1781 // returns the (pc) offset of the div instruction - may be needed
1782 // for implicit exceptions.
1783 //
1784 // constraint : ra/rb =/= scratch
1785 // normal case
1786 //
1787 // input : ra: dividend
1788 // rb: divisor
1789 //
1790 // result: either
1791 // quotient (= ra idiv rb)
1792 // remainder (= ra irem rb)
1793
1794 assert(ra != scratch && rb != scratch, "reg cannot be scratch");
1795
1796 int idivq_offset = offset();
1797 if (! want_remainder) {
1798 sdiv(result, ra, rb);
1799 } else {
1800 sdiv(scratch, ra, rb);
1801 Assembler::msub(result, scratch, rb, ra);
1802 }
1803
1804 return idivq_offset;
1805 }
1806
1807 void MacroAssembler::membar(Membar_mask_bits order_constraint) {
1808 address prev = pc() - NativeMembar::instruction_size;
1809 address last = code()->last_insn();
1810 if (last != NULL && nativeInstruction_at(last)->is_Membar() && prev == last) {
1811 NativeMembar *bar = NativeMembar_at(prev);
1812 // We are merging two memory barrier instructions. On AArch64 we
1813 // can do this simply by ORing them together.
1814 bar->set_kind(bar->get_kind() | order_constraint);
1815 BLOCK_COMMENT("merged membar");
1816 } else {
1817 code()->set_last_insn(pc());
1818 dmb(Assembler::barrier(order_constraint));
1819 }
1820 }
1821
1822 bool MacroAssembler::try_merge_ldst(Register rt, const Address &adr, size_t size_in_bytes, bool is_store) {
1823 if (ldst_can_merge(rt, adr, size_in_bytes, is_store)) {
1824 merge_ldst(rt, adr, size_in_bytes, is_store);
1825 code()->clear_last_insn();
1826 return true;
1827 } else {
1828 assert(size_in_bytes == 8 || size_in_bytes == 4, "only 8 bytes or 4 bytes load/store is supported.");
1829 const unsigned mask = size_in_bytes - 1;
1830 if (adr.getMode() == Address::base_plus_offset &&
1831 (adr.offset() & mask) == 0) { // only supports base_plus_offset.
1832 code()->set_last_insn(pc());
1833 }
1834 return false;
1835 }
1836 }
1837
1838 void MacroAssembler::ldr(Register Rx, const Address &adr) {
1839 // We always try to merge two adjacent loads into one ldp.
1840 if (!try_merge_ldst(Rx, adr, 8, false)) {
1841 Assembler::ldr(Rx, adr);
1842 }
1843 }
1844
1845 void MacroAssembler::ldrw(Register Rw, const Address &adr) {
1846 // We always try to merge two adjacent loads into one ldp.
1847 if (!try_merge_ldst(Rw, adr, 4, false)) {
1848 Assembler::ldrw(Rw, adr);
1849 }
1850 }
1851
1852 void MacroAssembler::str(Register Rx, const Address &adr) {
1853 // We always try to merge two adjacent stores into one stp.
1854 if (!try_merge_ldst(Rx, adr, 8, true)) {
1855 Assembler::str(Rx, adr);
1856 }
1857 }
1858
1859 void MacroAssembler::strw(Register Rw, const Address &adr) {
1860 // We always try to merge two adjacent stores into one stp.
1861 if (!try_merge_ldst(Rw, adr, 4, true)) {
1862 Assembler::strw(Rw, adr);
1863 }
1864 }
1865
1866 // MacroAssembler routines found actually to be needed
1867
1868 void MacroAssembler::push(Register src)
1869 {
1870 str(src, Address(pre(esp, -1 * wordSize)));
1871 }
1872
1873 void MacroAssembler::pop(Register dst)
1874 {
1875 ldr(dst, Address(post(esp, 1 * wordSize)));
1876 }
1877
1878 // Note: load_unsigned_short used to be called load_unsigned_word.
1879 int MacroAssembler::load_unsigned_short(Register dst, Address src) {
1880 int off = offset();
1881 ldrh(dst, src);
1882 return off;
1883 }
1884
1885 int MacroAssembler::load_unsigned_byte(Register dst, Address src) {
1886 int off = offset();
1887 ldrb(dst, src);
1888 return off;
1889 }
1890
1891 int MacroAssembler::load_signed_short(Register dst, Address src) {
1892 int off = offset();
1893 ldrsh(dst, src);
1894 return off;
1895 }
1896
1897 int MacroAssembler::load_signed_byte(Register dst, Address src) {
1898 int off = offset();
1899 ldrsb(dst, src);
1900 return off;
1901 }
1902
1903 int MacroAssembler::load_signed_short32(Register dst, Address src) {
1904 int off = offset();
1905 ldrshw(dst, src);
1906 return off;
1907 }
1908
1909 int MacroAssembler::load_signed_byte32(Register dst, Address src) {
1910 int off = offset();
1911 ldrsbw(dst, src);
1912 return off;
1913 }
1914
1915 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) {
1916 switch (size_in_bytes) {
1917 case 8: ldr(dst, src); break;
1918 case 4: ldrw(dst, src); break;
1919 case 2: is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break;
1920 case 1: is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break;
1921 default: ShouldNotReachHere();
1922 }
1923 }
1924
1925 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2) {
1926 switch (size_in_bytes) {
1927 case 8: str(src, dst); break;
1928 case 4: strw(src, dst); break;
1929 case 2: strh(src, dst); break;
1930 case 1: strb(src, dst); break;
1931 default: ShouldNotReachHere();
1932 }
1933 }
1934
1935 void MacroAssembler::decrementw(Register reg, int value)
1936 {
1937 if (value < 0) { incrementw(reg, -value); return; }
1938 if (value == 0) { return; }
1939 if (value < (1 << 12)) { subw(reg, reg, value); return; }
1940 /* else */ {
1941 guarantee(reg != rscratch2, "invalid dst for register decrement");
1942 movw(rscratch2, (unsigned)value);
1943 subw(reg, reg, rscratch2);
1944 }
1945 }
1946
1947 void MacroAssembler::decrement(Register reg, int value)
1948 {
1949 if (value < 0) { increment(reg, -value); return; }
1950 if (value == 0) { return; }
1951 if (value < (1 << 12)) { sub(reg, reg, value); return; }
1952 /* else */ {
1953 assert(reg != rscratch2, "invalid dst for register decrement");
1954 mov(rscratch2, (unsigned long)value);
1955 sub(reg, reg, rscratch2);
1956 }
1957 }
1958
1959 void MacroAssembler::decrementw(Address dst, int value)
1960 {
1961 assert(!dst.uses(rscratch1), "invalid dst for address decrement");
1962 if (dst.getMode() == Address::literal) {
1963 assert(abs(value) < (1 << 12), "invalid value and address mode combination");
1964 lea(rscratch2, dst);
1965 dst = Address(rscratch2);
1966 }
1967 ldrw(rscratch1, dst);
1968 decrementw(rscratch1, value);
1969 strw(rscratch1, dst);
1970 }
1971
1972 void MacroAssembler::decrement(Address dst, int value)
1973 {
1974 assert(!dst.uses(rscratch1), "invalid address for decrement");
1975 if (dst.getMode() == Address::literal) {
1976 assert(abs(value) < (1 << 12), "invalid value and address mode combination");
1977 lea(rscratch2, dst);
1978 dst = Address(rscratch2);
1979 }
1980 ldr(rscratch1, dst);
1981 decrement(rscratch1, value);
1982 str(rscratch1, dst);
1983 }
1984
1985 void MacroAssembler::incrementw(Register reg, int value)
1986 {
1987 if (value < 0) { decrementw(reg, -value); return; }
1988 if (value == 0) { return; }
1989 if (value < (1 << 12)) { addw(reg, reg, value); return; }
1990 /* else */ {
1991 assert(reg != rscratch2, "invalid dst for register increment");
1992 movw(rscratch2, (unsigned)value);
1993 addw(reg, reg, rscratch2);
1994 }
1995 }
1996
1997 void MacroAssembler::increment(Register reg, int value)
1998 {
1999 if (value < 0) { decrement(reg, -value); return; }
2000 if (value == 0) { return; }
2001 if (value < (1 << 12)) { add(reg, reg, value); return; }
2002 /* else */ {
2003 assert(reg != rscratch2, "invalid dst for register increment");
2004 movw(rscratch2, (unsigned)value);
2005 add(reg, reg, rscratch2);
2006 }
2007 }
2008
2009 void MacroAssembler::incrementw(Address dst, int value)
2010 {
2011 assert(!dst.uses(rscratch1), "invalid dst for address increment");
2012 if (dst.getMode() == Address::literal) {
2013 assert(abs(value) < (1 << 12), "invalid value and address mode combination");
2014 lea(rscratch2, dst);
2015 dst = Address(rscratch2);
2016 }
2017 ldrw(rscratch1, dst);
2018 incrementw(rscratch1, value);
2019 strw(rscratch1, dst);
2020 }
2021
2022 void MacroAssembler::increment(Address dst, int value)
2023 {
2024 assert(!dst.uses(rscratch1), "invalid dst for address increment");
2025 if (dst.getMode() == Address::literal) {
2026 assert(abs(value) < (1 << 12), "invalid value and address mode combination");
2027 lea(rscratch2, dst);
2028 dst = Address(rscratch2);
2029 }
2030 ldr(rscratch1, dst);
2031 increment(rscratch1, value);
2032 str(rscratch1, dst);
2033 }
2034
2035
2036 void MacroAssembler::pusha() {
2037 push(0x7fffffff, sp);
2038 }
2039
2040 void MacroAssembler::popa() {
2041 pop(0x7fffffff, sp);
2042 }
2043
2044 // Push lots of registers in the bit set supplied. Don't push sp.
2045 // Return the number of words pushed
2046 int MacroAssembler::push(unsigned int bitset, Register stack) {
2047 int words_pushed = 0;
2048
2049 // Scan bitset to accumulate register pairs
2050 unsigned char regs[32];
2051 int count = 0;
2052 for (int reg = 0; reg <= 30; reg++) {
2053 if (1 & bitset)
2054 regs[count++] = reg;
2055 bitset >>= 1;
2056 }
2057 regs[count++] = zr->encoding_nocheck();
2058 count &= ~1; // Only push an even nuber of regs
2059
2060 if (count) {
2061 stp(as_Register(regs[0]), as_Register(regs[1]),
2062 Address(pre(stack, -count * wordSize)));
2063 words_pushed += 2;
2064 }
2065 for (int i = 2; i < count; i += 2) {
2066 stp(as_Register(regs[i]), as_Register(regs[i+1]),
2067 Address(stack, i * wordSize));
2068 words_pushed += 2;
2069 }
2070
2071 assert(words_pushed == count, "oops, pushed != count");
2072
2073 return count;
2074 }
2075
2076 int MacroAssembler::pop(unsigned int bitset, Register stack) {
2077 int words_pushed = 0;
2078
2079 // Scan bitset to accumulate register pairs
2080 unsigned char regs[32];
2081 int count = 0;
2082 for (int reg = 0; reg <= 30; reg++) {
2083 if (1 & bitset)
2084 regs[count++] = reg;
2085 bitset >>= 1;
2086 }
2087 regs[count++] = zr->encoding_nocheck();
2088 count &= ~1;
2089
2090 for (int i = 2; i < count; i += 2) {
2091 ldp(as_Register(regs[i]), as_Register(regs[i+1]),
2092 Address(stack, i * wordSize));
2093 words_pushed += 2;
2094 }
2095 if (count) {
2096 ldp(as_Register(regs[0]), as_Register(regs[1]),
2097 Address(post(stack, count * wordSize)));
2098 words_pushed += 2;
2099 }
2100
2101 assert(words_pushed == count, "oops, pushed != count");
2102
2103 return count;
2104 }
2105 #ifdef ASSERT
2106 void MacroAssembler::verify_heapbase(const char* msg) {
2107 #if 0
2108 assert (UseCompressedOops || UseCompressedClassPointers, "should be compressed");
2109 assert (Universe::heap() != NULL, "java heap should be initialized");
2110 if (CheckCompressedOops) {
2111 Label ok;
2112 push(1 << rscratch1->encoding(), sp); // cmpptr trashes rscratch1
2113 cmpptr(rheapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr()));
2114 br(Assembler::EQ, ok);
2115 stop(msg);
2116 bind(ok);
2117 pop(1 << rscratch1->encoding(), sp);
2118 }
2119 #endif
2120 }
2121 #endif
2122
2123 void MacroAssembler::resolve_jobject(Register value, Register thread, Register tmp) {
2124 Label done, not_weak;
2125 cbz(value, done); // Use NULL as-is.
2126
2127 STATIC_ASSERT(JNIHandles::weak_tag_mask == 1u);
2128 tbz(r0, 0, not_weak); // Test for jweak tag.
2129
2130 // Resolve jweak.
2131 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, value,
2132 Address(value, -JNIHandles::weak_tag_value), tmp, thread);
2133 verify_oop(value);
2134 b(done);
2135
2136 bind(not_weak);
2137 // Resolve (untagged) jobject.
2138 access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, 0), tmp, thread);
2139 verify_oop(value);
2140 bind(done);
2141 }
2142
2143 void MacroAssembler::stop(const char* msg) {
2144 address ip = pc();
2145 pusha();
2146 mov(c_rarg0, (address)msg);
2147 mov(c_rarg1, (address)ip);
2148 mov(c_rarg2, sp);
2149 mov(c_rarg3, CAST_FROM_FN_PTR(address, MacroAssembler::debug64));
2150 // call(c_rarg3);
2151 blrt(c_rarg3, 3, 0, 1);
2152 hlt(0);
2153 }
2154
2155 void MacroAssembler::unimplemented(const char* what) {
2156 const char* buf = NULL;
2157 {
2158 ResourceMark rm;
2159 stringStream ss;
2160 ss.print("unimplemented: %s", what);
2161 buf = code_string(ss.as_string());
2162 }
2163 stop(buf);
2164 }
2165
2166 // If a constant does not fit in an immediate field, generate some
2167 // number of MOV instructions and then perform the operation.
2168 void MacroAssembler::wrap_add_sub_imm_insn(Register Rd, Register Rn, unsigned imm,
2169 add_sub_imm_insn insn1,
2170 add_sub_reg_insn insn2) {
2171 assert(Rd != zr, "Rd = zr and not setting flags?");
2172 if (operand_valid_for_add_sub_immediate((int)imm)) {
2173 (this->*insn1)(Rd, Rn, imm);
2174 } else {
2175 if (uabs(imm) < (1 << 24)) {
2176 (this->*insn1)(Rd, Rn, imm & -(1 << 12));
2177 (this->*insn1)(Rd, Rd, imm & ((1 << 12)-1));
2178 } else {
2179 assert_different_registers(Rd, Rn);
2180 mov(Rd, (uint64_t)imm);
2181 (this->*insn2)(Rd, Rn, Rd, LSL, 0);
2182 }
2183 }
2184 }
2185
2186 // Seperate vsn which sets the flags. Optimisations are more restricted
2187 // because we must set the flags correctly.
2188 void MacroAssembler::wrap_adds_subs_imm_insn(Register Rd, Register Rn, unsigned imm,
2189 add_sub_imm_insn insn1,
2190 add_sub_reg_insn insn2) {
2191 if (operand_valid_for_add_sub_immediate((int)imm)) {
2192 (this->*insn1)(Rd, Rn, imm);
2193 } else {
2194 assert_different_registers(Rd, Rn);
2195 assert(Rd != zr, "overflow in immediate operand");
2196 mov(Rd, (uint64_t)imm);
2197 (this->*insn2)(Rd, Rn, Rd, LSL, 0);
2198 }
2199 }
2200
2201
2202 void MacroAssembler::add(Register Rd, Register Rn, RegisterOrConstant increment) {
2203 if (increment.is_register()) {
2204 add(Rd, Rn, increment.as_register());
2205 } else {
2206 add(Rd, Rn, increment.as_constant());
2207 }
2208 }
2209
2210 void MacroAssembler::addw(Register Rd, Register Rn, RegisterOrConstant increment) {
2211 if (increment.is_register()) {
2212 addw(Rd, Rn, increment.as_register());
2213 } else {
2214 addw(Rd, Rn, increment.as_constant());
2215 }
2216 }
2217
2218 void MacroAssembler::sub(Register Rd, Register Rn, RegisterOrConstant decrement) {
2219 if (decrement.is_register()) {
2220 sub(Rd, Rn, decrement.as_register());
2221 } else {
2222 sub(Rd, Rn, decrement.as_constant());
2223 }
2224 }
2225
2226 void MacroAssembler::subw(Register Rd, Register Rn, RegisterOrConstant decrement) {
2227 if (decrement.is_register()) {
2228 subw(Rd, Rn, decrement.as_register());
2229 } else {
2230 subw(Rd, Rn, decrement.as_constant());
2231 }
2232 }
2233
2234 void MacroAssembler::reinit_heapbase()
2235 {
2236 if (UseCompressedOops) {
2237 if (Universe::is_fully_initialized()) {
2238 mov(rheapbase, Universe::narrow_ptrs_base());
2239 } else {
2240 lea(rheapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr()));
2241 ldr(rheapbase, Address(rheapbase));
2242 }
2243 }
2244 }
2245
2246 // this simulates the behaviour of the x86 cmpxchg instruction using a
2247 // load linked/store conditional pair. we use the acquire/release
2248 // versions of these instructions so that we flush pending writes as
2249 // per Java semantics.
2250
2251 // n.b the x86 version assumes the old value to be compared against is
2252 // in rax and updates rax with the value located in memory if the
2253 // cmpxchg fails. we supply a register for the old value explicitly
2254
2255 // the aarch64 load linked/store conditional instructions do not
2256 // accept an offset. so, unlike x86, we must provide a plain register
2257 // to identify the memory word to be compared/exchanged rather than a
2258 // register+offset Address.
2259
2260 void MacroAssembler::cmpxchgptr(Register oldv, Register newv, Register addr, Register tmp,
2261 Label &succeed, Label *fail) {
2262 // oldv holds comparison value
2263 // newv holds value to write in exchange
2264 // addr identifies memory word to compare against/update
2265 if (UseLSE) {
2266 mov(tmp, oldv);
2267 casal(Assembler::xword, oldv, newv, addr);
2268 cmp(tmp, oldv);
2269 br(Assembler::EQ, succeed);
2270 membar(AnyAny);
2271 } else {
2272 Label retry_load, nope;
2273 if ((VM_Version::features() & VM_Version::CPU_STXR_PREFETCH))
2274 prfm(Address(addr), PSTL1STRM);
2275 bind(retry_load);
2276 // flush and load exclusive from the memory location
2277 // and fail if it is not what we expect
2278 ldaxr(tmp, addr);
2279 cmp(tmp, oldv);
2280 br(Assembler::NE, nope);
2281 // if we store+flush with no intervening write tmp wil be zero
2282 stlxr(tmp, newv, addr);
2283 cbzw(tmp, succeed);
2284 // retry so we only ever return after a load fails to compare
2285 // ensures we don't return a stale value after a failed write.
2286 b(retry_load);
2287 // if the memory word differs we return it in oldv and signal a fail
2288 bind(nope);
2289 membar(AnyAny);
2290 mov(oldv, tmp);
2291 }
2292 if (fail)
2293 b(*fail);
2294 }
2295
2296 void MacroAssembler::cmpxchg_obj_header(Register oldv, Register newv, Register obj, Register tmp,
2297 Label &succeed, Label *fail) {
2298 assert(oopDesc::mark_offset_in_bytes() == 0, "assumption");
2299 cmpxchgptr(oldv, newv, obj, tmp, succeed, fail);
2300 }
2301
2302 void MacroAssembler::cmpxchgw(Register oldv, Register newv, Register addr, Register tmp,
2303 Label &succeed, Label *fail) {
2304 // oldv holds comparison value
2305 // newv holds value to write in exchange
2306 // addr identifies memory word to compare against/update
2307 // tmp returns 0/1 for success/failure
2308 if (UseLSE) {
2309 mov(tmp, oldv);
2310 casal(Assembler::word, oldv, newv, addr);
2311 cmp(tmp, oldv);
2312 br(Assembler::EQ, succeed);
2313 membar(AnyAny);
2314 } else {
2315 Label retry_load, nope;
2316 if ((VM_Version::features() & VM_Version::CPU_STXR_PREFETCH))
2317 prfm(Address(addr), PSTL1STRM);
2318 bind(retry_load);
2319 // flush and load exclusive from the memory location
2320 // and fail if it is not what we expect
2321 ldaxrw(tmp, addr);
2322 cmp(tmp, oldv);
2323 br(Assembler::NE, nope);
2324 // if we store+flush with no intervening write tmp wil be zero
2325 stlxrw(tmp, newv, addr);
2326 cbzw(tmp, succeed);
2327 // retry so we only ever return after a load fails to compare
2328 // ensures we don't return a stale value after a failed write.
2329 b(retry_load);
2330 // if the memory word differs we return it in oldv and signal a fail
2331 bind(nope);
2332 membar(AnyAny);
2333 mov(oldv, tmp);
2334 }
2335 if (fail)
2336 b(*fail);
2337 }
2338
2339 // A generic CAS; success or failure is in the EQ flag. A weak CAS
2340 // doesn't retry and may fail spuriously. If the oldval is wanted,
2341 // Pass a register for the result, otherwise pass noreg.
2342
2343 // Clobbers rscratch1
2344 void MacroAssembler::cmpxchg(Register addr, Register expected,
2345 Register new_val,
2346 enum operand_size size,
2347 bool acquire, bool release,
2348 bool weak,
2349 Register result) {
2350 if (result == noreg) result = rscratch1;
2351 if (UseLSE) {
2352 mov(result, expected);
2353 lse_cas(result, new_val, addr, size, acquire, release, /*not_pair*/ true);
2354 cmp(result, expected);
2355 } else {
2356 BLOCK_COMMENT("cmpxchg {");
2357 Label retry_load, done;
2358 if ((VM_Version::features() & VM_Version::CPU_STXR_PREFETCH))
2359 prfm(Address(addr), PSTL1STRM);
2360 bind(retry_load);
2361 load_exclusive(result, addr, size, acquire);
2362 if (size == xword)
2363 cmp(result, expected);
2364 else
2365 cmpw(result, expected);
2366 br(Assembler::NE, done);
2367 store_exclusive(rscratch1, new_val, addr, size, release);
2368 if (weak) {
2369 cmpw(rscratch1, 0u); // If the store fails, return NE to our caller.
2370 } else {
2371 cbnzw(rscratch1, retry_load);
2372 }
2373 bind(done);
2374 BLOCK_COMMENT("} cmpxchg");
2375 }
2376 }
2377
2378 static bool different(Register a, RegisterOrConstant b, Register c) {
2379 if (b.is_constant())
2380 return a != c;
2381 else
2382 return a != b.as_register() && a != c && b.as_register() != c;
2383 }
2384
2385 #define ATOMIC_OP(NAME, LDXR, OP, IOP, AOP, STXR, sz) \
2386 void MacroAssembler::atomic_##NAME(Register prev, RegisterOrConstant incr, Register addr) { \
2387 if (UseLSE) { \
2388 prev = prev->is_valid() ? prev : zr; \
2389 if (incr.is_register()) { \
2390 AOP(sz, incr.as_register(), prev, addr); \
2391 } else { \
2392 mov(rscratch2, incr.as_constant()); \
2393 AOP(sz, rscratch2, prev, addr); \
2394 } \
2395 return; \
2396 } \
2397 Register result = rscratch2; \
2398 if (prev->is_valid()) \
2399 result = different(prev, incr, addr) ? prev : rscratch2; \
2400 \
2401 Label retry_load; \
2402 if ((VM_Version::features() & VM_Version::CPU_STXR_PREFETCH)) \
2403 prfm(Address(addr), PSTL1STRM); \
2404 bind(retry_load); \
2405 LDXR(result, addr); \
2406 OP(rscratch1, result, incr); \
2407 STXR(rscratch2, rscratch1, addr); \
2408 cbnzw(rscratch2, retry_load); \
2409 if (prev->is_valid() && prev != result) { \
2410 IOP(prev, rscratch1, incr); \
2411 } \
2412 }
2413
2414 ATOMIC_OP(add, ldxr, add, sub, ldadd, stxr, Assembler::xword)
2415 ATOMIC_OP(addw, ldxrw, addw, subw, ldadd, stxrw, Assembler::word)
2416 ATOMIC_OP(addal, ldaxr, add, sub, ldaddal, stlxr, Assembler::xword)
2417 ATOMIC_OP(addalw, ldaxrw, addw, subw, ldaddal, stlxrw, Assembler::word)
2418
2419 #undef ATOMIC_OP
2420
2421 #define ATOMIC_XCHG(OP, AOP, LDXR, STXR, sz) \
2422 void MacroAssembler::atomic_##OP(Register prev, Register newv, Register addr) { \
2423 if (UseLSE) { \
2424 prev = prev->is_valid() ? prev : zr; \
2425 AOP(sz, newv, prev, addr); \
2426 return; \
2427 } \
2428 Register result = rscratch2; \
2429 if (prev->is_valid()) \
2430 result = different(prev, newv, addr) ? prev : rscratch2; \
2431 \
2432 Label retry_load; \
2433 if ((VM_Version::features() & VM_Version::CPU_STXR_PREFETCH)) \
2434 prfm(Address(addr), PSTL1STRM); \
2435 bind(retry_load); \
2436 LDXR(result, addr); \
2437 STXR(rscratch1, newv, addr); \
2438 cbnzw(rscratch1, retry_load); \
2439 if (prev->is_valid() && prev != result) \
2440 mov(prev, result); \
2441 }
2442
2443 ATOMIC_XCHG(xchg, swp, ldxr, stxr, Assembler::xword)
2444 ATOMIC_XCHG(xchgw, swp, ldxrw, stxrw, Assembler::word)
2445 ATOMIC_XCHG(xchgal, swpal, ldaxr, stlxr, Assembler::xword)
2446 ATOMIC_XCHG(xchgalw, swpal, ldaxrw, stlxrw, Assembler::word)
2447
2448 #undef ATOMIC_XCHG
2449
2450 #ifndef PRODUCT
2451 extern "C" void findpc(intptr_t x);
2452 #endif
2453
2454 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[])
2455 {
2456 // In order to get locks to work, we need to fake a in_VM state
2457 if (ShowMessageBoxOnError ) {
2458 JavaThread* thread = JavaThread::current();
2459 JavaThreadState saved_state = thread->thread_state();
2460 thread->set_thread_state(_thread_in_vm);
2461 #ifndef PRODUCT
2462 if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
2463 ttyLocker ttyl;
2464 BytecodeCounter::print();
2465 }
2466 #endif
2467 if (os::message_box(msg, "Execution stopped, print registers?")) {
2468 ttyLocker ttyl;
2469 tty->print_cr(" pc = 0x%016lx", pc);
2470 #ifndef PRODUCT
2471 tty->cr();
2472 findpc(pc);
2473 tty->cr();
2474 #endif
2475 tty->print_cr(" r0 = 0x%016lx", regs[0]);
2476 tty->print_cr(" r1 = 0x%016lx", regs[1]);
2477 tty->print_cr(" r2 = 0x%016lx", regs[2]);
2478 tty->print_cr(" r3 = 0x%016lx", regs[3]);
2479 tty->print_cr(" r4 = 0x%016lx", regs[4]);
2480 tty->print_cr(" r5 = 0x%016lx", regs[5]);
2481 tty->print_cr(" r6 = 0x%016lx", regs[6]);
2482 tty->print_cr(" r7 = 0x%016lx", regs[7]);
2483 tty->print_cr(" r8 = 0x%016lx", regs[8]);
2484 tty->print_cr(" r9 = 0x%016lx", regs[9]);
2485 tty->print_cr("r10 = 0x%016lx", regs[10]);
2486 tty->print_cr("r11 = 0x%016lx", regs[11]);
2487 tty->print_cr("r12 = 0x%016lx", regs[12]);
2488 tty->print_cr("r13 = 0x%016lx", regs[13]);
2489 tty->print_cr("r14 = 0x%016lx", regs[14]);
2490 tty->print_cr("r15 = 0x%016lx", regs[15]);
2491 tty->print_cr("r16 = 0x%016lx", regs[16]);
2492 tty->print_cr("r17 = 0x%016lx", regs[17]);
2493 tty->print_cr("r18 = 0x%016lx", regs[18]);
2494 tty->print_cr("r19 = 0x%016lx", regs[19]);
2495 tty->print_cr("r20 = 0x%016lx", regs[20]);
2496 tty->print_cr("r21 = 0x%016lx", regs[21]);
2497 tty->print_cr("r22 = 0x%016lx", regs[22]);
2498 tty->print_cr("r23 = 0x%016lx", regs[23]);
2499 tty->print_cr("r24 = 0x%016lx", regs[24]);
2500 tty->print_cr("r25 = 0x%016lx", regs[25]);
2501 tty->print_cr("r26 = 0x%016lx", regs[26]);
2502 tty->print_cr("r27 = 0x%016lx", regs[27]);
2503 tty->print_cr("r28 = 0x%016lx", regs[28]);
2504 tty->print_cr("r30 = 0x%016lx", regs[30]);
2505 tty->print_cr("r31 = 0x%016lx", regs[31]);
2506 BREAKPOINT;
2507 }
2508 ThreadStateTransition::transition(thread, _thread_in_vm, saved_state);
2509 } else {
2510 ttyLocker ttyl;
2511 ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n",
2512 msg);
2513 assert(false, "DEBUG MESSAGE: %s", msg);
2514 }
2515 }
2516
2517 #ifdef BUILTIN_SIM
2518 // routine to generate an x86 prolog for a stub function which
2519 // bootstraps into the generated ARM code which directly follows the
2520 // stub
2521 //
2522 // the argument encodes the number of general and fp registers
2523 // passed by the caller and the callng convention (currently just
2524 // the number of general registers and assumes C argument passing)
2525
2526 extern "C" {
2527 int aarch64_stub_prolog_size();
2528 void aarch64_stub_prolog();
2529 void aarch64_prolog();
2530 }
2531
2532 void MacroAssembler::c_stub_prolog(int gp_arg_count, int fp_arg_count, int ret_type,
2533 address *prolog_ptr)
2534 {
2535 int calltype = (((ret_type & 0x3) << 8) |
2536 ((fp_arg_count & 0xf) << 4) |
2537 (gp_arg_count & 0xf));
2538
2539 // the addresses for the x86 to ARM entry code we need to use
2540 address start = pc();
2541 // printf("start = %lx\n", start);
2542 int byteCount = aarch64_stub_prolog_size();
2543 // printf("byteCount = %x\n", byteCount);
2544 int instructionCount = (byteCount + 3)/ 4;
2545 // printf("instructionCount = %x\n", instructionCount);
2546 for (int i = 0; i < instructionCount; i++) {
2547 nop();
2548 }
2549
2550 memcpy(start, (void*)aarch64_stub_prolog, byteCount);
2551
2552 // write the address of the setup routine and the call format at the
2553 // end of into the copied code
2554 u_int64_t *patch_end = (u_int64_t *)(start + byteCount);
2555 if (prolog_ptr)
2556 patch_end[-2] = (u_int64_t)prolog_ptr;
2557 patch_end[-1] = calltype;
2558 }
2559 #endif
2560
2561 void MacroAssembler::push_call_clobbered_registers() {
2562 int step = 4 * wordSize;
2563 push(RegSet::range(r0, r18) - RegSet::of(rscratch1, rscratch2), sp);
2564 sub(sp, sp, step);
2565 mov(rscratch1, -step);
2566 // Push v0-v7, v16-v31.
2567 for (int i = 31; i>= 4; i -= 4) {
2568 if (i <= v7->encoding() || i >= v16->encoding())
2569 st1(as_FloatRegister(i-3), as_FloatRegister(i-2), as_FloatRegister(i-1),
2570 as_FloatRegister(i), T1D, Address(post(sp, rscratch1)));
2571 }
2572 st1(as_FloatRegister(0), as_FloatRegister(1), as_FloatRegister(2),
2573 as_FloatRegister(3), T1D, Address(sp));
2574 }
2575
2576 void MacroAssembler::pop_call_clobbered_registers() {
2577 for (int i = 0; i < 32; i += 4) {
2578 if (i <= v7->encoding() || i >= v16->encoding())
2579 ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
2580 as_FloatRegister(i+3), T1D, Address(post(sp, 4 * wordSize)));
2581 }
2582
2583 pop(RegSet::range(r0, r18) - RegSet::of(rscratch1, rscratch2), sp);
2584 }
2585
2586 void MacroAssembler::push_CPU_state(bool save_vectors) {
2587 int step = (save_vectors ? 8 : 4) * wordSize;
2588 push(0x3fffffff, sp); // integer registers except lr & sp
2589 mov(rscratch1, -step);
2590 sub(sp, sp, step);
2591 for (int i = 28; i >= 4; i -= 4) {
2592 st1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
2593 as_FloatRegister(i+3), save_vectors ? T2D : T1D, Address(post(sp, rscratch1)));
2594 }
2595 st1(v0, v1, v2, v3, save_vectors ? T2D : T1D, sp);
2596 }
2597
2598 void MacroAssembler::pop_CPU_state(bool restore_vectors) {
2599 int step = (restore_vectors ? 8 : 4) * wordSize;
2600 for (int i = 0; i <= 28; i += 4)
2601 ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
2602 as_FloatRegister(i+3), restore_vectors ? T2D : T1D, Address(post(sp, step)));
2603 pop(0x3fffffff, sp); // integer registers except lr & sp
2604 }
2605
2606 /**
2607 * Helpers for multiply_to_len().
2608 */
2609 void MacroAssembler::add2_with_carry(Register final_dest_hi, Register dest_hi, Register dest_lo,
2610 Register src1, Register src2) {
2611 adds(dest_lo, dest_lo, src1);
2612 adc(dest_hi, dest_hi, zr);
2613 adds(dest_lo, dest_lo, src2);
2614 adc(final_dest_hi, dest_hi, zr);
2615 }
2616
2617 // Generate an address from (r + r1 extend offset). "size" is the
2618 // size of the operand. The result may be in rscratch2.
2619 Address MacroAssembler::offsetted_address(Register r, Register r1,
2620 Address::extend ext, int offset, int size) {
2621 if (offset || (ext.shift() % size != 0)) {
2622 lea(rscratch2, Address(r, r1, ext));
2623 return Address(rscratch2, offset);
2624 } else {
2625 return Address(r, r1, ext);
2626 }
2627 }
2628
2629 Address MacroAssembler::spill_address(int size, int offset, Register tmp)
2630 {
2631 assert(offset >= 0, "spill to negative address?");
2632 // Offset reachable ?
2633 // Not aligned - 9 bits signed offset
2634 // Aligned - 12 bits unsigned offset shifted
2635 Register base = sp;
2636 if ((offset & (size-1)) && offset >= (1<<8)) {
2637 add(tmp, base, offset & ((1<<12)-1));
2638 base = tmp;
2639 offset &= -1<<12;
2640 }
2641
2642 if (offset >= (1<<12) * size) {
2643 add(tmp, base, offset & (((1<<12)-1)<<12));
2644 base = tmp;
2645 offset &= ~(((1<<12)-1)<<12);
2646 }
2647
2648 return Address(base, offset);
2649 }
2650
2651 // Checks whether offset is aligned.
2652 // Returns true if it is, else false.
2653 bool MacroAssembler::merge_alignment_check(Register base,
2654 size_t size,
2655 long cur_offset,
2656 long prev_offset) const {
2657 if (AvoidUnalignedAccesses) {
2658 if (base == sp) {
2659 // Checks whether low offset if aligned to pair of registers.
2660 long pair_mask = size * 2 - 1;
2661 long offset = prev_offset > cur_offset ? cur_offset : prev_offset;
2662 return (offset & pair_mask) == 0;
2663 } else { // If base is not sp, we can't guarantee the access is aligned.
2664 return false;
2665 }
2666 } else {
2667 long mask = size - 1;
2668 // Load/store pair instruction only supports element size aligned offset.
2669 return (cur_offset & mask) == 0 && (prev_offset & mask) == 0;
2670 }
2671 }
2672
2673 // Checks whether current and previous loads/stores can be merged.
2674 // Returns true if it can be merged, else false.
2675 bool MacroAssembler::ldst_can_merge(Register rt,
2676 const Address &adr,
2677 size_t cur_size_in_bytes,
2678 bool is_store) const {
2679 address prev = pc() - NativeInstruction::instruction_size;
2680 address last = code()->last_insn();
2681
2682 if (last == NULL || !nativeInstruction_at(last)->is_Imm_LdSt()) {
2683 return false;
2684 }
2685
2686 if (adr.getMode() != Address::base_plus_offset || prev != last) {
2687 return false;
2688 }
2689
2690 NativeLdSt* prev_ldst = NativeLdSt_at(prev);
2691 size_t prev_size_in_bytes = prev_ldst->size_in_bytes();
2692
2693 assert(prev_size_in_bytes == 4 || prev_size_in_bytes == 8, "only supports 64/32bit merging.");
2694 assert(cur_size_in_bytes == 4 || cur_size_in_bytes == 8, "only supports 64/32bit merging.");
2695
2696 if (cur_size_in_bytes != prev_size_in_bytes || is_store != prev_ldst->is_store()) {
2697 return false;
2698 }
2699
2700 long max_offset = 63 * prev_size_in_bytes;
2701 long min_offset = -64 * prev_size_in_bytes;
2702
2703 assert(prev_ldst->is_not_pre_post_index(), "pre-index or post-index is not supported to be merged.");
2704
2705 // Only same base can be merged.
2706 if (adr.base() != prev_ldst->base()) {
2707 return false;
2708 }
2709
2710 long cur_offset = adr.offset();
2711 long prev_offset = prev_ldst->offset();
2712 size_t diff = abs(cur_offset - prev_offset);
2713 if (diff != prev_size_in_bytes) {
2714 return false;
2715 }
2716
2717 // Following cases can not be merged:
2718 // ldr x2, [x2, #8]
2719 // ldr x3, [x2, #16]
2720 // or:
2721 // ldr x2, [x3, #8]
2722 // ldr x2, [x3, #16]
2723 // If t1 and t2 is the same in "ldp t1, t2, [xn, #imm]", we'll get SIGILL.
2724 if (!is_store && (adr.base() == prev_ldst->target() || rt == prev_ldst->target())) {
2725 return false;
2726 }
2727
2728 long low_offset = prev_offset > cur_offset ? cur_offset : prev_offset;
2729 // Offset range must be in ldp/stp instruction's range.
2730 if (low_offset > max_offset || low_offset < min_offset) {
2731 return false;
2732 }
2733
2734 if (merge_alignment_check(adr.base(), prev_size_in_bytes, cur_offset, prev_offset)) {
2735 return true;
2736 }
2737
2738 return false;
2739 }
2740
2741 // Merge current load/store with previous load/store into ldp/stp.
2742 void MacroAssembler::merge_ldst(Register rt,
2743 const Address &adr,
2744 size_t cur_size_in_bytes,
2745 bool is_store) {
2746
2747 assert(ldst_can_merge(rt, adr, cur_size_in_bytes, is_store) == true, "cur and prev must be able to be merged.");
2748
2749 Register rt_low, rt_high;
2750 address prev = pc() - NativeInstruction::instruction_size;
2751 NativeLdSt* prev_ldst = NativeLdSt_at(prev);
2752
2753 long offset;
2754
2755 if (adr.offset() < prev_ldst->offset()) {
2756 offset = adr.offset();
2757 rt_low = rt;
2758 rt_high = prev_ldst->target();
2759 } else {
2760 offset = prev_ldst->offset();
2761 rt_low = prev_ldst->target();
2762 rt_high = rt;
2763 }
2764
2765 Address adr_p = Address(prev_ldst->base(), offset);
2766 // Overwrite previous generated binary.
2767 code_section()->set_end(prev);
2768
2769 const int sz = prev_ldst->size_in_bytes();
2770 assert(sz == 8 || sz == 4, "only supports 64/32bit merging.");
2771 if (!is_store) {
2772 BLOCK_COMMENT("merged ldr pair");
2773 if (sz == 8) {
2774 ldp(rt_low, rt_high, adr_p);
2775 } else {
2776 ldpw(rt_low, rt_high, adr_p);
2777 }
2778 } else {
2779 BLOCK_COMMENT("merged str pair");
2780 if (sz == 8) {
2781 stp(rt_low, rt_high, adr_p);
2782 } else {
2783 stpw(rt_low, rt_high, adr_p);
2784 }
2785 }
2786 }
2787
2788 /**
2789 * Multiply 64 bit by 64 bit first loop.
2790 */
2791 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
2792 Register y, Register y_idx, Register z,
2793 Register carry, Register product,
2794 Register idx, Register kdx) {
2795 //
2796 // jlong carry, x[], y[], z[];
2797 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
2798 // huge_128 product = y[idx] * x[xstart] + carry;
2799 // z[kdx] = (jlong)product;
2800 // carry = (jlong)(product >>> 64);
2801 // }
2802 // z[xstart] = carry;
2803 //
2804
2805 Label L_first_loop, L_first_loop_exit;
2806 Label L_one_x, L_one_y, L_multiply;
2807
2808 subsw(xstart, xstart, 1);
2809 br(Assembler::MI, L_one_x);
2810
2811 lea(rscratch1, Address(x, xstart, Address::lsl(LogBytesPerInt)));
2812 ldr(x_xstart, Address(rscratch1));
2813 ror(x_xstart, x_xstart, 32); // convert big-endian to little-endian
2814
2815 bind(L_first_loop);
2816 subsw(idx, idx, 1);
2817 br(Assembler::MI, L_first_loop_exit);
2818 subsw(idx, idx, 1);
2819 br(Assembler::MI, L_one_y);
2820 lea(rscratch1, Address(y, idx, Address::uxtw(LogBytesPerInt)));
2821 ldr(y_idx, Address(rscratch1));
2822 ror(y_idx, y_idx, 32); // convert big-endian to little-endian
2823 bind(L_multiply);
2824
2825 // AArch64 has a multiply-accumulate instruction that we can't use
2826 // here because it has no way to process carries, so we have to use
2827 // separate add and adc instructions. Bah.
2828 umulh(rscratch1, x_xstart, y_idx); // x_xstart * y_idx -> rscratch1:product
2829 mul(product, x_xstart, y_idx);
2830 adds(product, product, carry);
2831 adc(carry, rscratch1, zr); // x_xstart * y_idx + carry -> carry:product
2832
2833 subw(kdx, kdx, 2);
2834 ror(product, product, 32); // back to big-endian
2835 str(product, offsetted_address(z, kdx, Address::uxtw(LogBytesPerInt), 0, BytesPerLong));
2836
2837 b(L_first_loop);
2838
2839 bind(L_one_y);
2840 ldrw(y_idx, Address(y, 0));
2841 b(L_multiply);
2842
2843 bind(L_one_x);
2844 ldrw(x_xstart, Address(x, 0));
2845 b(L_first_loop);
2846
2847 bind(L_first_loop_exit);
2848 }
2849
2850 /**
2851 * Multiply 128 bit by 128. Unrolled inner loop.
2852 *
2853 */
2854 void MacroAssembler::multiply_128_x_128_loop(Register y, Register z,
2855 Register carry, Register carry2,
2856 Register idx, Register jdx,
2857 Register yz_idx1, Register yz_idx2,
2858 Register tmp, Register tmp3, Register tmp4,
2859 Register tmp6, Register product_hi) {
2860
2861 // jlong carry, x[], y[], z[];
2862 // int kdx = ystart+1;
2863 // for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
2864 // huge_128 tmp3 = (y[idx+1] * product_hi) + z[kdx+idx+1] + carry;
2865 // jlong carry2 = (jlong)(tmp3 >>> 64);
2866 // huge_128 tmp4 = (y[idx] * product_hi) + z[kdx+idx] + carry2;
2867 // carry = (jlong)(tmp4 >>> 64);
2868 // z[kdx+idx+1] = (jlong)tmp3;
2869 // z[kdx+idx] = (jlong)tmp4;
2870 // }
2871 // idx += 2;
2872 // if (idx > 0) {
2873 // yz_idx1 = (y[idx] * product_hi) + z[kdx+idx] + carry;
2874 // z[kdx+idx] = (jlong)yz_idx1;
2875 // carry = (jlong)(yz_idx1 >>> 64);
2876 // }
2877 //
2878
2879 Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
2880
2881 lsrw(jdx, idx, 2);
2882
2883 bind(L_third_loop);
2884
2885 subsw(jdx, jdx, 1);
2886 br(Assembler::MI, L_third_loop_exit);
2887 subw(idx, idx, 4);
2888
2889 lea(rscratch1, Address(y, idx, Address::uxtw(LogBytesPerInt)));
2890
2891 ldp(yz_idx2, yz_idx1, Address(rscratch1, 0));
2892
2893 lea(tmp6, Address(z, idx, Address::uxtw(LogBytesPerInt)));
2894
2895 ror(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian
2896 ror(yz_idx2, yz_idx2, 32);
2897
2898 ldp(rscratch2, rscratch1, Address(tmp6, 0));
2899
2900 mul(tmp3, product_hi, yz_idx1); // yz_idx1 * product_hi -> tmp4:tmp3
2901 umulh(tmp4, product_hi, yz_idx1);
2902
2903 ror(rscratch1, rscratch1, 32); // convert big-endian to little-endian
2904 ror(rscratch2, rscratch2, 32);
2905
2906 mul(tmp, product_hi, yz_idx2); // yz_idx2 * product_hi -> carry2:tmp
2907 umulh(carry2, product_hi, yz_idx2);
2908
2909 // propagate sum of both multiplications into carry:tmp4:tmp3
2910 adds(tmp3, tmp3, carry);
2911 adc(tmp4, tmp4, zr);
2912 adds(tmp3, tmp3, rscratch1);
2913 adcs(tmp4, tmp4, tmp);
2914 adc(carry, carry2, zr);
2915 adds(tmp4, tmp4, rscratch2);
2916 adc(carry, carry, zr);
2917
2918 ror(tmp3, tmp3, 32); // convert little-endian to big-endian
2919 ror(tmp4, tmp4, 32);
2920 stp(tmp4, tmp3, Address(tmp6, 0));
2921
2922 b(L_third_loop);
2923 bind (L_third_loop_exit);
2924
2925 andw (idx, idx, 0x3);
2926 cbz(idx, L_post_third_loop_done);
2927
2928 Label L_check_1;
2929 subsw(idx, idx, 2);
2930 br(Assembler::MI, L_check_1);
2931
2932 lea(rscratch1, Address(y, idx, Address::uxtw(LogBytesPerInt)));
2933 ldr(yz_idx1, Address(rscratch1, 0));
2934 ror(yz_idx1, yz_idx1, 32);
2935 mul(tmp3, product_hi, yz_idx1); // yz_idx1 * product_hi -> tmp4:tmp3
2936 umulh(tmp4, product_hi, yz_idx1);
2937 lea(rscratch1, Address(z, idx, Address::uxtw(LogBytesPerInt)));
2938 ldr(yz_idx2, Address(rscratch1, 0));
2939 ror(yz_idx2, yz_idx2, 32);
2940
2941 add2_with_carry(carry, tmp4, tmp3, carry, yz_idx2);
2942
2943 ror(tmp3, tmp3, 32);
2944 str(tmp3, Address(rscratch1, 0));
2945
2946 bind (L_check_1);
2947
2948 andw (idx, idx, 0x1);
2949 subsw(idx, idx, 1);
2950 br(Assembler::MI, L_post_third_loop_done);
2951 ldrw(tmp4, Address(y, idx, Address::uxtw(LogBytesPerInt)));
2952 mul(tmp3, tmp4, product_hi); // tmp4 * product_hi -> carry2:tmp3
2953 umulh(carry2, tmp4, product_hi);
2954 ldrw(tmp4, Address(z, idx, Address::uxtw(LogBytesPerInt)));
2955
2956 add2_with_carry(carry2, tmp3, tmp4, carry);
2957
2958 strw(tmp3, Address(z, idx, Address::uxtw(LogBytesPerInt)));
2959 extr(carry, carry2, tmp3, 32);
2960
2961 bind(L_post_third_loop_done);
2962 }
2963
2964 /**
2965 * Code for BigInteger::multiplyToLen() instrinsic.
2966 *
2967 * r0: x
2968 * r1: xlen
2969 * r2: y
2970 * r3: ylen
2971 * r4: z
2972 * r5: zlen
2973 * r10: tmp1
2974 * r11: tmp2
2975 * r12: tmp3
2976 * r13: tmp4
2977 * r14: tmp5
2978 * r15: tmp6
2979 * r16: tmp7
2980 *
2981 */
2982 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen,
2983 Register z, Register zlen,
2984 Register tmp1, Register tmp2, Register tmp3, Register tmp4,
2985 Register tmp5, Register tmp6, Register product_hi) {
2986
2987 assert_different_registers(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6);
2988
2989 const Register idx = tmp1;
2990 const Register kdx = tmp2;
2991 const Register xstart = tmp3;
2992
2993 const Register y_idx = tmp4;
2994 const Register carry = tmp5;
2995 const Register product = xlen;
2996 const Register x_xstart = zlen; // reuse register
2997
2998 // First Loop.
2999 //
3000 // final static long LONG_MASK = 0xffffffffL;
3001 // int xstart = xlen - 1;
3002 // int ystart = ylen - 1;
3003 // long carry = 0;
3004 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
3005 // long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry;
3006 // z[kdx] = (int)product;
3007 // carry = product >>> 32;
3008 // }
3009 // z[xstart] = (int)carry;
3010 //
3011
3012 movw(idx, ylen); // idx = ylen;
3013 movw(kdx, zlen); // kdx = xlen+ylen;
3014 mov(carry, zr); // carry = 0;
3015
3016 Label L_done;
3017
3018 movw(xstart, xlen);
3019 subsw(xstart, xstart, 1);
3020 br(Assembler::MI, L_done);
3021
3022 multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx);
3023
3024 Label L_second_loop;
3025 cbzw(kdx, L_second_loop);
3026
3027 Label L_carry;
3028 subw(kdx, kdx, 1);
3029 cbzw(kdx, L_carry);
3030
3031 strw(carry, Address(z, kdx, Address::uxtw(LogBytesPerInt)));
3032 lsr(carry, carry, 32);
3033 subw(kdx, kdx, 1);
3034
3035 bind(L_carry);
3036 strw(carry, Address(z, kdx, Address::uxtw(LogBytesPerInt)));
3037
3038 // Second and third (nested) loops.
3039 //
3040 // for (int i = xstart-1; i >= 0; i--) { // Second loop
3041 // carry = 0;
3042 // for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop
3043 // long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) +
3044 // (z[k] & LONG_MASK) + carry;
3045 // z[k] = (int)product;
3046 // carry = product >>> 32;
3047 // }
3048 // z[i] = (int)carry;
3049 // }
3050 //
3051 // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = product_hi
3052
3053 const Register jdx = tmp1;
3054
3055 bind(L_second_loop);
3056 mov(carry, zr); // carry = 0;
3057 movw(jdx, ylen); // j = ystart+1
3058
3059 subsw(xstart, xstart, 1); // i = xstart-1;
3060 br(Assembler::MI, L_done);
3061
3062 str(z, Address(pre(sp, -4 * wordSize)));
3063
3064 Label L_last_x;
3065 lea(z, offsetted_address(z, xstart, Address::uxtw(LogBytesPerInt), 4, BytesPerInt)); // z = z + k - j
3066 subsw(xstart, xstart, 1); // i = xstart-1;
3067 br(Assembler::MI, L_last_x);
3068
3069 lea(rscratch1, Address(x, xstart, Address::uxtw(LogBytesPerInt)));
3070 ldr(product_hi, Address(rscratch1));
3071 ror(product_hi, product_hi, 32); // convert big-endian to little-endian
3072
3073 Label L_third_loop_prologue;
3074 bind(L_third_loop_prologue);
3075
3076 str(ylen, Address(sp, wordSize));
3077 stp(x, xstart, Address(sp, 2 * wordSize));
3078 multiply_128_x_128_loop(y, z, carry, x, jdx, ylen, product,
3079 tmp2, x_xstart, tmp3, tmp4, tmp6, product_hi);
3080 ldp(z, ylen, Address(post(sp, 2 * wordSize)));
3081 ldp(x, xlen, Address(post(sp, 2 * wordSize))); // copy old xstart -> xlen
3082
3083 addw(tmp3, xlen, 1);
3084 strw(carry, Address(z, tmp3, Address::uxtw(LogBytesPerInt)));
3085 subsw(tmp3, tmp3, 1);
3086 br(Assembler::MI, L_done);
3087
3088 lsr(carry, carry, 32);
3089 strw(carry, Address(z, tmp3, Address::uxtw(LogBytesPerInt)));
3090 b(L_second_loop);
3091
3092 // Next infrequent code is moved outside loops.
3093 bind(L_last_x);
3094 ldrw(product_hi, Address(x, 0));
3095 b(L_third_loop_prologue);
3096
3097 bind(L_done);
3098 }
3099
3100 // Code for BigInteger::mulAdd instrinsic
3101 // out = r0
3102 // in = r1
3103 // offset = r2 (already out.length-offset)
3104 // len = r3
3105 // k = r4
3106 //
3107 // pseudo code from java implementation:
3108 // carry = 0;
3109 // offset = out.length-offset - 1;
3110 // for (int j=len-1; j >= 0; j--) {
3111 // product = (in[j] & LONG_MASK) * kLong + (out[offset] & LONG_MASK) + carry;
3112 // out[offset--] = (int)product;
3113 // carry = product >>> 32;
3114 // }
3115 // return (int)carry;
3116 void MacroAssembler::mul_add(Register out, Register in, Register offset,
3117 Register len, Register k) {
3118 Label LOOP, END;
3119 // pre-loop
3120 cmp(len, zr); // cmp, not cbz/cbnz: to use condition twice => less branches
3121 csel(out, zr, out, Assembler::EQ);
3122 br(Assembler::EQ, END);
3123 add(in, in, len, LSL, 2); // in[j+1] address
3124 add(offset, out, offset, LSL, 2); // out[offset + 1] address
3125 mov(out, zr); // used to keep carry now
3126 BIND(LOOP);
3127 ldrw(rscratch1, Address(pre(in, -4)));
3128 madd(rscratch1, rscratch1, k, out);
3129 ldrw(rscratch2, Address(pre(offset, -4)));
3130 add(rscratch1, rscratch1, rscratch2);
3131 strw(rscratch1, Address(offset));
3132 lsr(out, rscratch1, 32);
3133 subs(len, len, 1);
3134 br(Assembler::NE, LOOP);
3135 BIND(END);
3136 }
3137
3138 /**
3139 * Emits code to update CRC-32 with a byte value according to constants in table
3140 *
3141 * @param [in,out]crc Register containing the crc.
3142 * @param [in]val Register containing the byte to fold into the CRC.
3143 * @param [in]table Register containing the table of crc constants.
3144 *
3145 * uint32_t crc;
3146 * val = crc_table[(val ^ crc) & 0xFF];
3147 * crc = val ^ (crc >> 8);
3148 *
3149 */
3150 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) {
3151 eor(val, val, crc);
3152 andr(val, val, 0xff);
3153 ldrw(val, Address(table, val, Address::lsl(2)));
3154 eor(crc, val, crc, Assembler::LSR, 8);
3155 }
3156
3157 /**
3158 * Emits code to update CRC-32 with a 32-bit value according to tables 0 to 3
3159 *
3160 * @param [in,out]crc Register containing the crc.
3161 * @param [in]v Register containing the 32-bit to fold into the CRC.
3162 * @param [in]table0 Register containing table 0 of crc constants.
3163 * @param [in]table1 Register containing table 1 of crc constants.
3164 * @param [in]table2 Register containing table 2 of crc constants.
3165 * @param [in]table3 Register containing table 3 of crc constants.
3166 *
3167 * uint32_t crc;
3168 * v = crc ^ v
3169 * crc = table3[v&0xff]^table2[(v>>8)&0xff]^table1[(v>>16)&0xff]^table0[v>>24]
3170 *
3171 */
3172 void MacroAssembler::update_word_crc32(Register crc, Register v, Register tmp,
3173 Register table0, Register table1, Register table2, Register table3,
3174 bool upper) {
3175 eor(v, crc, v, upper ? LSR:LSL, upper ? 32:0);
3176 uxtb(tmp, v);
3177 ldrw(crc, Address(table3, tmp, Address::lsl(2)));
3178 ubfx(tmp, v, 8, 8);
3179 ldrw(tmp, Address(table2, tmp, Address::lsl(2)));
3180 eor(crc, crc, tmp);
3181 ubfx(tmp, v, 16, 8);
3182 ldrw(tmp, Address(table1, tmp, Address::lsl(2)));
3183 eor(crc, crc, tmp);
3184 ubfx(tmp, v, 24, 8);
3185 ldrw(tmp, Address(table0, tmp, Address::lsl(2)));
3186 eor(crc, crc, tmp);
3187 }
3188
3189 void MacroAssembler::kernel_crc32_using_crc32(Register crc, Register buf,
3190 Register len, Register tmp0, Register tmp1, Register tmp2,
3191 Register tmp3) {
3192 Label CRC_by64_loop, CRC_by4_loop, CRC_by1_loop, CRC_less64, CRC_by64_pre, CRC_by32_loop, CRC_less32, L_exit;
3193 assert_different_registers(crc, buf, len, tmp0, tmp1, tmp2, tmp3);
3194
3195 mvnw(crc, crc);
3196
3197 subs(len, len, 128);
3198 br(Assembler::GE, CRC_by64_pre);
3199 BIND(CRC_less64);
3200 adds(len, len, 128-32);
3201 br(Assembler::GE, CRC_by32_loop);
3202 BIND(CRC_less32);
3203 adds(len, len, 32-4);
3204 br(Assembler::GE, CRC_by4_loop);
3205 adds(len, len, 4);
3206 br(Assembler::GT, CRC_by1_loop);
3207 b(L_exit);
3208
3209 BIND(CRC_by32_loop);
3210 ldp(tmp0, tmp1, Address(post(buf, 16)));
3211 subs(len, len, 32);
3212 crc32x(crc, crc, tmp0);
3213 ldr(tmp2, Address(post(buf, 8)));
3214 crc32x(crc, crc, tmp1);
3215 ldr(tmp3, Address(post(buf, 8)));
3216 crc32x(crc, crc, tmp2);
3217 crc32x(crc, crc, tmp3);
3218 br(Assembler::GE, CRC_by32_loop);
3219 cmn(len, 32);
3220 br(Assembler::NE, CRC_less32);
3221 b(L_exit);
3222
3223 BIND(CRC_by4_loop);
3224 ldrw(tmp0, Address(post(buf, 4)));
3225 subs(len, len, 4);
3226 crc32w(crc, crc, tmp0);
3227 br(Assembler::GE, CRC_by4_loop);
3228 adds(len, len, 4);
3229 br(Assembler::LE, L_exit);
3230 BIND(CRC_by1_loop);
3231 ldrb(tmp0, Address(post(buf, 1)));
3232 subs(len, len, 1);
3233 crc32b(crc, crc, tmp0);
3234 br(Assembler::GT, CRC_by1_loop);
3235 b(L_exit);
3236
3237 BIND(CRC_by64_pre);
3238 sub(buf, buf, 8);
3239 ldp(tmp0, tmp1, Address(buf, 8));
3240 crc32x(crc, crc, tmp0);
3241 ldr(tmp2, Address(buf, 24));
3242 crc32x(crc, crc, tmp1);
3243 ldr(tmp3, Address(buf, 32));
3244 crc32x(crc, crc, tmp2);
3245 ldr(tmp0, Address(buf, 40));
3246 crc32x(crc, crc, tmp3);
3247 ldr(tmp1, Address(buf, 48));
3248 crc32x(crc, crc, tmp0);
3249 ldr(tmp2, Address(buf, 56));
3250 crc32x(crc, crc, tmp1);
3251 ldr(tmp3, Address(pre(buf, 64)));
3252
3253 b(CRC_by64_loop);
3254
3255 align(CodeEntryAlignment);
3256 BIND(CRC_by64_loop);
3257 subs(len, len, 64);
3258 crc32x(crc, crc, tmp2);
3259 ldr(tmp0, Address(buf, 8));
3260 crc32x(crc, crc, tmp3);
3261 ldr(tmp1, Address(buf, 16));
3262 crc32x(crc, crc, tmp0);
3263 ldr(tmp2, Address(buf, 24));
3264 crc32x(crc, crc, tmp1);
3265 ldr(tmp3, Address(buf, 32));
3266 crc32x(crc, crc, tmp2);
3267 ldr(tmp0, Address(buf, 40));
3268 crc32x(crc, crc, tmp3);
3269 ldr(tmp1, Address(buf, 48));
3270 crc32x(crc, crc, tmp0);
3271 ldr(tmp2, Address(buf, 56));
3272 crc32x(crc, crc, tmp1);
3273 ldr(tmp3, Address(pre(buf, 64)));
3274 br(Assembler::GE, CRC_by64_loop);
3275
3276 // post-loop
3277 crc32x(crc, crc, tmp2);
3278 crc32x(crc, crc, tmp3);
3279
3280 sub(len, len, 64);
3281 add(buf, buf, 8);
3282 cmn(len, 128);
3283 br(Assembler::NE, CRC_less64);
3284 BIND(L_exit);
3285 mvnw(crc, crc);
3286 }
3287
3288 /**
3289 * @param crc register containing existing CRC (32-bit)
3290 * @param buf register pointing to input byte buffer (byte*)
3291 * @param len register containing number of bytes
3292 * @param table register that will contain address of CRC table
3293 * @param tmp scratch register
3294 */
3295 void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len,
3296 Register table0, Register table1, Register table2, Register table3,
3297 Register tmp, Register tmp2, Register tmp3) {
3298 Label L_by16, L_by16_loop, L_by4, L_by4_loop, L_by1, L_by1_loop, L_exit;
3299 unsigned long offset;
3300
3301 if (UseCRC32) {
3302 kernel_crc32_using_crc32(crc, buf, len, table0, table1, table2, table3);
3303 return;
3304 }
3305
3306 mvnw(crc, crc);
3307
3308 adrp(table0, ExternalAddress(StubRoutines::crc_table_addr()), offset);
3309 if (offset) add(table0, table0, offset);
3310 add(table1, table0, 1*256*sizeof(juint));
3311 add(table2, table0, 2*256*sizeof(juint));
3312 add(table3, table0, 3*256*sizeof(juint));
3313
3314 if (UseNeon) {
3315 cmp(len, 64);
3316 br(Assembler::LT, L_by16);
3317 eor(v16, T16B, v16, v16);
3318
3319 Label L_fold;
3320
3321 add(tmp, table0, 4*256*sizeof(juint)); // Point at the Neon constants
3322
3323 ld1(v0, v1, T2D, post(buf, 32));
3324 ld1r(v4, T2D, post(tmp, 8));
3325 ld1r(v5, T2D, post(tmp, 8));
3326 ld1r(v6, T2D, post(tmp, 8));
3327 ld1r(v7, T2D, post(tmp, 8));
3328 mov(v16, T4S, 0, crc);
3329
3330 eor(v0, T16B, v0, v16);
3331 sub(len, len, 64);
3332
3333 BIND(L_fold);
3334 pmull(v22, T8H, v0, v5, T8B);
3335 pmull(v20, T8H, v0, v7, T8B);
3336 pmull(v23, T8H, v0, v4, T8B);
3337 pmull(v21, T8H, v0, v6, T8B);
3338
3339 pmull2(v18, T8H, v0, v5, T16B);
3340 pmull2(v16, T8H, v0, v7, T16B);
3341 pmull2(v19, T8H, v0, v4, T16B);
3342 pmull2(v17, T8H, v0, v6, T16B);
3343
3344 uzp1(v24, T8H, v20, v22);
3345 uzp2(v25, T8H, v20, v22);
3346 eor(v20, T16B, v24, v25);
3347
3348 uzp1(v26, T8H, v16, v18);
3349 uzp2(v27, T8H, v16, v18);
3350 eor(v16, T16B, v26, v27);
3351
3352 ushll2(v22, T4S, v20, T8H, 8);
3353 ushll(v20, T4S, v20, T4H, 8);
3354
3355 ushll2(v18, T4S, v16, T8H, 8);
3356 ushll(v16, T4S, v16, T4H, 8);
3357
3358 eor(v22, T16B, v23, v22);
3359 eor(v18, T16B, v19, v18);
3360 eor(v20, T16B, v21, v20);
3361 eor(v16, T16B, v17, v16);
3362
3363 uzp1(v17, T2D, v16, v20);
3364 uzp2(v21, T2D, v16, v20);
3365 eor(v17, T16B, v17, v21);
3366
3367 ushll2(v20, T2D, v17, T4S, 16);
3368 ushll(v16, T2D, v17, T2S, 16);
3369
3370 eor(v20, T16B, v20, v22);
3371 eor(v16, T16B, v16, v18);
3372
3373 uzp1(v17, T2D, v20, v16);
3374 uzp2(v21, T2D, v20, v16);
3375 eor(v28, T16B, v17, v21);
3376
3377 pmull(v22, T8H, v1, v5, T8B);
3378 pmull(v20, T8H, v1, v7, T8B);
3379 pmull(v23, T8H, v1, v4, T8B);
3380 pmull(v21, T8H, v1, v6, T8B);
3381
3382 pmull2(v18, T8H, v1, v5, T16B);
3383 pmull2(v16, T8H, v1, v7, T16B);
3384 pmull2(v19, T8H, v1, v4, T16B);
3385 pmull2(v17, T8H, v1, v6, T16B);
3386
3387 ld1(v0, v1, T2D, post(buf, 32));
3388
3389 uzp1(v24, T8H, v20, v22);
3390 uzp2(v25, T8H, v20, v22);
3391 eor(v20, T16B, v24, v25);
3392
3393 uzp1(v26, T8H, v16, v18);
3394 uzp2(v27, T8H, v16, v18);
3395 eor(v16, T16B, v26, v27);
3396
3397 ushll2(v22, T4S, v20, T8H, 8);
3398 ushll(v20, T4S, v20, T4H, 8);
3399
3400 ushll2(v18, T4S, v16, T8H, 8);
3401 ushll(v16, T4S, v16, T4H, 8);
3402
3403 eor(v22, T16B, v23, v22);
3404 eor(v18, T16B, v19, v18);
3405 eor(v20, T16B, v21, v20);
3406 eor(v16, T16B, v17, v16);
3407
3408 uzp1(v17, T2D, v16, v20);
3409 uzp2(v21, T2D, v16, v20);
3410 eor(v16, T16B, v17, v21);
3411
3412 ushll2(v20, T2D, v16, T4S, 16);
3413 ushll(v16, T2D, v16, T2S, 16);
3414
3415 eor(v20, T16B, v22, v20);
3416 eor(v16, T16B, v16, v18);
3417
3418 uzp1(v17, T2D, v20, v16);
3419 uzp2(v21, T2D, v20, v16);
3420 eor(v20, T16B, v17, v21);
3421
3422 shl(v16, T2D, v28, 1);
3423 shl(v17, T2D, v20, 1);
3424
3425 eor(v0, T16B, v0, v16);
3426 eor(v1, T16B, v1, v17);
3427
3428 subs(len, len, 32);
3429 br(Assembler::GE, L_fold);
3430
3431 mov(crc, 0);
3432 mov(tmp, v0, T1D, 0);
3433 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, false);
3434 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, true);
3435 mov(tmp, v0, T1D, 1);
3436 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, false);
3437 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, true);
3438 mov(tmp, v1, T1D, 0);
3439 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, false);
3440 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, true);
3441 mov(tmp, v1, T1D, 1);
3442 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, false);
3443 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, true);
3444
3445 add(len, len, 32);
3446 }
3447
3448 BIND(L_by16);
3449 subs(len, len, 16);
3450 br(Assembler::GE, L_by16_loop);
3451 adds(len, len, 16-4);
3452 br(Assembler::GE, L_by4_loop);
3453 adds(len, len, 4);
3454 br(Assembler::GT, L_by1_loop);
3455 b(L_exit);
3456
3457 BIND(L_by4_loop);
3458 ldrw(tmp, Address(post(buf, 4)));
3459 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3);
3460 subs(len, len, 4);
3461 br(Assembler::GE, L_by4_loop);
3462 adds(len, len, 4);
3463 br(Assembler::LE, L_exit);
3464 BIND(L_by1_loop);
3465 subs(len, len, 1);
3466 ldrb(tmp, Address(post(buf, 1)));
3467 update_byte_crc32(crc, tmp, table0);
3468 br(Assembler::GT, L_by1_loop);
3469 b(L_exit);
3470
3471 align(CodeEntryAlignment);
3472 BIND(L_by16_loop);
3473 subs(len, len, 16);
3474 ldp(tmp, tmp3, Address(post(buf, 16)));
3475 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, false);
3476 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, true);
3477 update_word_crc32(crc, tmp3, tmp2, table0, table1, table2, table3, false);
3478 update_word_crc32(crc, tmp3, tmp2, table0, table1, table2, table3, true);
3479 br(Assembler::GE, L_by16_loop);
3480 adds(len, len, 16-4);
3481 br(Assembler::GE, L_by4_loop);
3482 adds(len, len, 4);
3483 br(Assembler::GT, L_by1_loop);
3484 BIND(L_exit);
3485 mvnw(crc, crc);
3486 }
3487
3488 void MacroAssembler::kernel_crc32c_using_crc32c(Register crc, Register buf,
3489 Register len, Register tmp0, Register tmp1, Register tmp2,
3490 Register tmp3) {
3491 Label CRC_by64_loop, CRC_by4_loop, CRC_by1_loop, CRC_less64, CRC_by64_pre, CRC_by32_loop, CRC_less32, L_exit;
3492 assert_different_registers(crc, buf, len, tmp0, tmp1, tmp2, tmp3);
3493
3494 subs(len, len, 128);
3495 br(Assembler::GE, CRC_by64_pre);
3496 BIND(CRC_less64);
3497 adds(len, len, 128-32);
3498 br(Assembler::GE, CRC_by32_loop);
3499 BIND(CRC_less32);
3500 adds(len, len, 32-4);
3501 br(Assembler::GE, CRC_by4_loop);
3502 adds(len, len, 4);
3503 br(Assembler::GT, CRC_by1_loop);
3504 b(L_exit);
3505
3506 BIND(CRC_by32_loop);
3507 ldp(tmp0, tmp1, Address(post(buf, 16)));
3508 subs(len, len, 32);
3509 crc32cx(crc, crc, tmp0);
3510 ldr(tmp2, Address(post(buf, 8)));
3511 crc32cx(crc, crc, tmp1);
3512 ldr(tmp3, Address(post(buf, 8)));
3513 crc32cx(crc, crc, tmp2);
3514 crc32cx(crc, crc, tmp3);
3515 br(Assembler::GE, CRC_by32_loop);
3516 cmn(len, 32);
3517 br(Assembler::NE, CRC_less32);
3518 b(L_exit);
3519
3520 BIND(CRC_by4_loop);
3521 ldrw(tmp0, Address(post(buf, 4)));
3522 subs(len, len, 4);
3523 crc32cw(crc, crc, tmp0);
3524 br(Assembler::GE, CRC_by4_loop);
3525 adds(len, len, 4);
3526 br(Assembler::LE, L_exit);
3527 BIND(CRC_by1_loop);
3528 ldrb(tmp0, Address(post(buf, 1)));
3529 subs(len, len, 1);
3530 crc32cb(crc, crc, tmp0);
3531 br(Assembler::GT, CRC_by1_loop);
3532 b(L_exit);
3533
3534 BIND(CRC_by64_pre);
3535 sub(buf, buf, 8);
3536 ldp(tmp0, tmp1, Address(buf, 8));
3537 crc32cx(crc, crc, tmp0);
3538 ldr(tmp2, Address(buf, 24));
3539 crc32cx(crc, crc, tmp1);
3540 ldr(tmp3, Address(buf, 32));
3541 crc32cx(crc, crc, tmp2);
3542 ldr(tmp0, Address(buf, 40));
3543 crc32cx(crc, crc, tmp3);
3544 ldr(tmp1, Address(buf, 48));
3545 crc32cx(crc, crc, tmp0);
3546 ldr(tmp2, Address(buf, 56));
3547 crc32cx(crc, crc, tmp1);
3548 ldr(tmp3, Address(pre(buf, 64)));
3549
3550 b(CRC_by64_loop);
3551
3552 align(CodeEntryAlignment);
3553 BIND(CRC_by64_loop);
3554 subs(len, len, 64);
3555 crc32cx(crc, crc, tmp2);
3556 ldr(tmp0, Address(buf, 8));
3557 crc32cx(crc, crc, tmp3);
3558 ldr(tmp1, Address(buf, 16));
3559 crc32cx(crc, crc, tmp0);
3560 ldr(tmp2, Address(buf, 24));
3561 crc32cx(crc, crc, tmp1);
3562 ldr(tmp3, Address(buf, 32));
3563 crc32cx(crc, crc, tmp2);
3564 ldr(tmp0, Address(buf, 40));
3565 crc32cx(crc, crc, tmp3);
3566 ldr(tmp1, Address(buf, 48));
3567 crc32cx(crc, crc, tmp0);
3568 ldr(tmp2, Address(buf, 56));
3569 crc32cx(crc, crc, tmp1);
3570 ldr(tmp3, Address(pre(buf, 64)));
3571 br(Assembler::GE, CRC_by64_loop);
3572
3573 // post-loop
3574 crc32cx(crc, crc, tmp2);
3575 crc32cx(crc, crc, tmp3);
3576
3577 sub(len, len, 64);
3578 add(buf, buf, 8);
3579 cmn(len, 128);
3580 br(Assembler::NE, CRC_less64);
3581 BIND(L_exit);
3582 }
3583
3584 /**
3585 * @param crc register containing existing CRC (32-bit)
3586 * @param buf register pointing to input byte buffer (byte*)
3587 * @param len register containing number of bytes
3588 * @param table register that will contain address of CRC table
3589 * @param tmp scratch register
3590 */
3591 void MacroAssembler::kernel_crc32c(Register crc, Register buf, Register len,
3592 Register table0, Register table1, Register table2, Register table3,
3593 Register tmp, Register tmp2, Register tmp3) {
3594 kernel_crc32c_using_crc32c(crc, buf, len, table0, table1, table2, table3);
3595 }
3596
3597
3598 SkipIfEqual::SkipIfEqual(
3599 MacroAssembler* masm, const bool* flag_addr, bool value) {
3600 _masm = masm;
3601 unsigned long offset;
3602 _masm->adrp(rscratch1, ExternalAddress((address)flag_addr), offset);
3603 _masm->ldrb(rscratch1, Address(rscratch1, offset));
3604 _masm->cbzw(rscratch1, _label);
3605 }
3606
3607 SkipIfEqual::~SkipIfEqual() {
3608 _masm->bind(_label);
3609 }
3610
3611 void MacroAssembler::addptr(const Address &dst, int32_t src) {
3612 Address adr;
3613 switch(dst.getMode()) {
3614 case Address::base_plus_offset:
3615 // This is the expected mode, although we allow all the other
3616 // forms below.
3617 adr = form_address(rscratch2, dst.base(), dst.offset(), LogBytesPerWord);
3618 break;
3619 default:
3620 lea(rscratch2, dst);
3621 adr = Address(rscratch2);
3622 break;
3623 }
3624 ldr(rscratch1, adr);
3625 add(rscratch1, rscratch1, src);
3626 str(rscratch1, adr);
3627 }
3628
3629 void MacroAssembler::cmpptr(Register src1, Address src2) {
3630 unsigned long offset;
3631 adrp(rscratch1, src2, offset);
3632 ldr(rscratch1, Address(rscratch1, offset));
3633 cmp(src1, rscratch1);
3634 }
3635
3636 void MacroAssembler::cmpoop(Register obj1, Register obj2) {
3637 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3638 bs->obj_equals(this, obj1, obj2);
3639 }
3640
3641 void MacroAssembler::load_klass(Register dst, Register src) {
3642 if (UseCompressedClassPointers) {
3643 ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3644 decode_klass_not_null(dst);
3645 } else {
3646 ldr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3647 }
3648 }
3649
3650 // ((OopHandle)result).resolve();
3651 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) {
3652 // OopHandle::resolve is an indirection.
3653 access_load_at(T_OBJECT, IN_NATIVE, result, Address(result, 0), tmp, noreg);
3654 }
3655
3656 void MacroAssembler::load_mirror(Register dst, Register method, Register tmp) {
3657 const int mirror_offset = in_bytes(Klass::java_mirror_offset());
3658 ldr(dst, Address(rmethod, Method::const_offset()));
3659 ldr(dst, Address(dst, ConstMethod::constants_offset()));
3660 ldr(dst, Address(dst, ConstantPool::pool_holder_offset_in_bytes()));
3661 ldr(dst, Address(dst, mirror_offset));
3662 resolve_oop_handle(dst, tmp);
3663 }
3664
3665 void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp) {
3666 if (UseCompressedClassPointers) {
3667 ldrw(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3668 if (Universe::narrow_klass_base() == NULL) {
3669 cmp(trial_klass, tmp, LSL, Universe::narrow_klass_shift());
3670 return;
3671 } else if (((uint64_t)Universe::narrow_klass_base() & 0xffffffff) == 0
3672 && Universe::narrow_klass_shift() == 0) {
3673 // Only the bottom 32 bits matter
3674 cmpw(trial_klass, tmp);
3675 return;
3676 }
3677 decode_klass_not_null(tmp);
3678 } else {
3679 ldr(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3680 }
3681 cmp(trial_klass, tmp);
3682 }
3683
3684 void MacroAssembler::load_prototype_header(Register dst, Register src) {
3685 load_klass(dst, src);
3686 ldr(dst, Address(dst, Klass::prototype_header_offset()));
3687 }
3688
3689 void MacroAssembler::store_klass(Register dst, Register src) {
3690 // FIXME: Should this be a store release? concurrent gcs assumes
3691 // klass length is valid if klass field is not null.
3692 if (UseCompressedClassPointers) {
3693 encode_klass_not_null(src);
3694 strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3695 } else {
3696 str(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3697 }
3698 }
3699
3700 void MacroAssembler::store_klass_gap(Register dst, Register src) {
3701 if (UseCompressedClassPointers) {
3702 // Store to klass gap in destination
3703 strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
3704 }
3705 }
3706
3707 // Algorithm must match CompressedOops::encode.
3708 void MacroAssembler::encode_heap_oop(Register d, Register s) {
3709 #ifdef ASSERT
3710 verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
3711 #endif
3712 verify_oop(s, "broken oop in encode_heap_oop");
3713 if (Universe::narrow_oop_base() == NULL) {
3714 if (Universe::narrow_oop_shift() != 0) {
3715 assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
3716 lsr(d, s, LogMinObjAlignmentInBytes);
3717 } else {
3718 mov(d, s);
3719 }
3720 } else {
3721 subs(d, s, rheapbase);
3722 csel(d, d, zr, Assembler::HS);
3723 lsr(d, d, LogMinObjAlignmentInBytes);
3724
3725 /* Old algorithm: is this any worse?
3726 Label nonnull;
3727 cbnz(r, nonnull);
3728 sub(r, r, rheapbase);
3729 bind(nonnull);
3730 lsr(r, r, LogMinObjAlignmentInBytes);
3731 */
3732 }
3733 }
3734
3735 void MacroAssembler::encode_heap_oop_not_null(Register r) {
3736 #ifdef ASSERT
3737 verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?");
3738 if (CheckCompressedOops) {
3739 Label ok;
3740 cbnz(r, ok);
3741 stop("null oop passed to encode_heap_oop_not_null");
3742 bind(ok);
3743 }
3744 #endif
3745 verify_oop(r, "broken oop in encode_heap_oop_not_null");
3746 if (Universe::narrow_oop_base() != NULL) {
3747 sub(r, r, rheapbase);
3748 }
3749 if (Universe::narrow_oop_shift() != 0) {
3750 assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
3751 lsr(r, r, LogMinObjAlignmentInBytes);
3752 }
3753 }
3754
3755 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) {
3756 #ifdef ASSERT
3757 verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?");
3758 if (CheckCompressedOops) {
3759 Label ok;
3760 cbnz(src, ok);
3761 stop("null oop passed to encode_heap_oop_not_null2");
3762 bind(ok);
3763 }
3764 #endif
3765 verify_oop(src, "broken oop in encode_heap_oop_not_null2");
3766
3767 Register data = src;
3768 if (Universe::narrow_oop_base() != NULL) {
3769 sub(dst, src, rheapbase);
3770 data = dst;
3771 }
3772 if (Universe::narrow_oop_shift() != 0) {
3773 assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
3774 lsr(dst, data, LogMinObjAlignmentInBytes);
3775 data = dst;
3776 }
3777 if (data == src)
3778 mov(dst, src);
3779 }
3780
3781 void MacroAssembler::decode_heap_oop(Register d, Register s) {
3782 #ifdef ASSERT
3783 verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?");
3784 #endif
3785 if (Universe::narrow_oop_base() == NULL) {
3786 if (Universe::narrow_oop_shift() != 0 || d != s) {
3787 lsl(d, s, Universe::narrow_oop_shift());
3788 }
3789 } else {
3790 Label done;
3791 if (d != s)
3792 mov(d, s);
3793 cbz(s, done);
3794 add(d, rheapbase, s, Assembler::LSL, LogMinObjAlignmentInBytes);
3795 bind(done);
3796 }
3797 verify_oop(d, "broken oop in decode_heap_oop");
3798 }
3799
3800 void MacroAssembler::decode_heap_oop_not_null(Register r) {
3801 assert (UseCompressedOops, "should only be used for compressed headers");
3802 assert (Universe::heap() != NULL, "java heap should be initialized");
3803 // Cannot assert, unverified entry point counts instructions (see .ad file)
3804 // vtableStubs also counts instructions in pd_code_size_limit.
3805 // Also do not verify_oop as this is called by verify_oop.
3806 if (Universe::narrow_oop_shift() != 0) {
3807 assert(LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
3808 if (Universe::narrow_oop_base() != NULL) {
3809 add(r, rheapbase, r, Assembler::LSL, LogMinObjAlignmentInBytes);
3810 } else {
3811 add(r, zr, r, Assembler::LSL, LogMinObjAlignmentInBytes);
3812 }
3813 } else {
3814 assert (Universe::narrow_oop_base() == NULL, "sanity");
3815 }
3816 }
3817
3818 void MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
3819 assert (UseCompressedOops, "should only be used for compressed headers");
3820 assert (Universe::heap() != NULL, "java heap should be initialized");
3821 // Cannot assert, unverified entry point counts instructions (see .ad file)
3822 // vtableStubs also counts instructions in pd_code_size_limit.
3823 // Also do not verify_oop as this is called by verify_oop.
3824 if (Universe::narrow_oop_shift() != 0) {
3825 assert(LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
3826 if (Universe::narrow_oop_base() != NULL) {
3827 add(dst, rheapbase, src, Assembler::LSL, LogMinObjAlignmentInBytes);
3828 } else {
3829 add(dst, zr, src, Assembler::LSL, LogMinObjAlignmentInBytes);
3830 }
3831 } else {
3832 assert (Universe::narrow_oop_base() == NULL, "sanity");
3833 if (dst != src) {
3834 mov(dst, src);
3835 }
3836 }
3837 }
3838
3839 void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
3840 if (Universe::narrow_klass_base() == NULL) {
3841 if (Universe::narrow_klass_shift() != 0) {
3842 assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
3843 lsr(dst, src, LogKlassAlignmentInBytes);
3844 } else {
3845 if (dst != src) mov(dst, src);
3846 }
3847 return;
3848 }
3849
3850 if (use_XOR_for_compressed_class_base) {
3851 if (Universe::narrow_klass_shift() != 0) {
3852 eor(dst, src, (uint64_t)Universe::narrow_klass_base());
3853 lsr(dst, dst, LogKlassAlignmentInBytes);
3854 } else {
3855 eor(dst, src, (uint64_t)Universe::narrow_klass_base());
3856 }
3857 return;
3858 }
3859
3860 if (((uint64_t)Universe::narrow_klass_base() & 0xffffffff) == 0
3861 && Universe::narrow_klass_shift() == 0) {
3862 movw(dst, src);
3863 return;
3864 }
3865
3866 #ifdef ASSERT
3867 verify_heapbase("MacroAssembler::encode_klass_not_null2: heap base corrupted?");
3868 #endif
3869
3870 Register rbase = dst;
3871 if (dst == src) rbase = rheapbase;
3872 mov(rbase, (uint64_t)Universe::narrow_klass_base());
3873 sub(dst, src, rbase);
3874 if (Universe::narrow_klass_shift() != 0) {
3875 assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
3876 lsr(dst, dst, LogKlassAlignmentInBytes);
3877 }
3878 if (dst == src) reinit_heapbase();
3879 }
3880
3881 void MacroAssembler::encode_klass_not_null(Register r) {
3882 encode_klass_not_null(r, r);
3883 }
3884
3885 void MacroAssembler::decode_klass_not_null(Register dst, Register src) {
3886 Register rbase = dst;
3887 assert (UseCompressedClassPointers, "should only be used for compressed headers");
3888
3889 if (Universe::narrow_klass_base() == NULL) {
3890 if (Universe::narrow_klass_shift() != 0) {
3891 assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
3892 lsl(dst, src, LogKlassAlignmentInBytes);
3893 } else {
3894 if (dst != src) mov(dst, src);
3895 }
3896 return;
3897 }
3898
3899 if (use_XOR_for_compressed_class_base) {
3900 if (Universe::narrow_klass_shift() != 0) {
3901 lsl(dst, src, LogKlassAlignmentInBytes);
3902 eor(dst, dst, (uint64_t)Universe::narrow_klass_base());
3903 } else {
3904 eor(dst, src, (uint64_t)Universe::narrow_klass_base());
3905 }
3906 return;
3907 }
3908
3909 if (((uint64_t)Universe::narrow_klass_base() & 0xffffffff) == 0
3910 && Universe::narrow_klass_shift() == 0) {
3911 if (dst != src)
3912 movw(dst, src);
3913 movk(dst, (uint64_t)Universe::narrow_klass_base() >> 32, 32);
3914 return;
3915 }
3916
3917 // Cannot assert, unverified entry point counts instructions (see .ad file)
3918 // vtableStubs also counts instructions in pd_code_size_limit.
3919 // Also do not verify_oop as this is called by verify_oop.
3920 if (dst == src) rbase = rheapbase;
3921 mov(rbase, (uint64_t)Universe::narrow_klass_base());
3922 if (Universe::narrow_klass_shift() != 0) {
3923 assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
3924 add(dst, rbase, src, Assembler::LSL, LogKlassAlignmentInBytes);
3925 } else {
3926 add(dst, rbase, src);
3927 }
3928 if (dst == src) reinit_heapbase();
3929 }
3930
3931 void MacroAssembler::decode_klass_not_null(Register r) {
3932 decode_klass_not_null(r, r);
3933 }
3934
3935 void MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
3936 #ifdef ASSERT
3937 {
3938 ThreadInVMfromUnknown tiv;
3939 assert (UseCompressedOops, "should only be used for compressed oops");
3940 assert (Universe::heap() != NULL, "java heap should be initialized");
3941 assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
3942 assert(Universe::heap()->is_in_reserved(JNIHandles::resolve(obj)), "should be real oop");
3943 }
3944 #endif
3945 int oop_index = oop_recorder()->find_index(obj);
3946 InstructionMark im(this);
3947 RelocationHolder rspec = oop_Relocation::spec(oop_index);
3948 code_section()->relocate(inst_mark(), rspec);
3949 movz(dst, 0xDEAD, 16);
3950 movk(dst, 0xBEEF);
3951 }
3952
3953 void MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
3954 assert (UseCompressedClassPointers, "should only be used for compressed headers");
3955 assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
3956 int index = oop_recorder()->find_index(k);
3957 assert(! Universe::heap()->is_in_reserved(k), "should not be an oop");
3958
3959 InstructionMark im(this);
3960 RelocationHolder rspec = metadata_Relocation::spec(index);
3961 code_section()->relocate(inst_mark(), rspec);
3962 narrowKlass nk = Klass::encode_klass(k);
3963 movz(dst, (nk >> 16), 16);
3964 movk(dst, nk & 0xffff);
3965 }
3966
3967 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators,
3968 Register dst, Address src,
3969 Register tmp1, Register thread_tmp) {
3970 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
3971 decorators = AccessInternal::decorator_fixup(decorators);
3972 bool as_raw = (decorators & AS_RAW) != 0;
3973 if (as_raw) {
3974 bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
3975 } else {
3976 bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
3977 }
3978 }
3979
3980 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators,
3981 Address dst, Register src,
3982 Register tmp1, Register thread_tmp) {
3983 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
3984 decorators = AccessInternal::decorator_fixup(decorators);
3985 bool as_raw = (decorators & AS_RAW) != 0;
3986 if (as_raw) {
3987 bs->BarrierSetAssembler::store_at(this, decorators, type, dst, src, tmp1, thread_tmp);
3988 } else {
3989 bs->store_at(this, decorators, type, dst, src, tmp1, thread_tmp);
3990 }
3991 }
3992
3993 void MacroAssembler::resolve(DecoratorSet decorators, Register obj) {
3994 // Use stronger ACCESS_WRITE|ACCESS_READ by default.
3995 if ((decorators & (ACCESS_READ | ACCESS_WRITE)) == 0) {
3996 decorators |= ACCESS_READ | ACCESS_WRITE;
3997 }
3998 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3999 return bs->resolve(this, decorators, obj);
4000 }
4001
4002 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,
4003 Register thread_tmp, DecoratorSet decorators) {
4004 access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp);
4005 }
4006
4007 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1,
4008 Register thread_tmp, DecoratorSet decorators) {
4009 access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, thread_tmp);
4010 }
4011
4012 void MacroAssembler::store_heap_oop(Address dst, Register src, Register tmp1,
4013 Register thread_tmp, DecoratorSet decorators) {
4014 access_store_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp);
4015 }
4016
4017 // Used for storing NULLs.
4018 void MacroAssembler::store_heap_oop_null(Address dst) {
4019 access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg);
4020 }
4021
4022 Address MacroAssembler::allocate_metadata_address(Metadata* obj) {
4023 assert(oop_recorder() != NULL, "this assembler needs a Recorder");
4024 int index = oop_recorder()->allocate_metadata_index(obj);
4025 RelocationHolder rspec = metadata_Relocation::spec(index);
4026 return Address((address)obj, rspec);
4027 }
4028
4029 // Move an oop into a register. immediate is true if we want
4030 // immediate instrcutions, i.e. we are not going to patch this
4031 // instruction while the code is being executed by another thread. In
4032 // that case we can use move immediates rather than the constant pool.
4033 void MacroAssembler::movoop(Register dst, jobject obj, bool immediate) {
4034 int oop_index;
4035 if (obj == NULL) {
4036 oop_index = oop_recorder()->allocate_oop_index(obj);
4037 } else {
4038 #ifdef ASSERT
4039 {
4040 ThreadInVMfromUnknown tiv;
4041 assert(Universe::heap()->is_in_reserved(JNIHandles::resolve(obj)), "should be real oop");
4042 }
4043 #endif
4044 oop_index = oop_recorder()->find_index(obj);
4045 }
4046 RelocationHolder rspec = oop_Relocation::spec(oop_index);
4047 if (! immediate) {
4048 address dummy = address(uintptr_t(pc()) & -wordSize); // A nearby aligned address
4049 ldr_constant(dst, Address(dummy, rspec));
4050 } else
4051 mov(dst, Address((address)obj, rspec));
4052 }
4053
4054 // Move a metadata address into a register.
4055 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
4056 int oop_index;
4057 if (obj == NULL) {
4058 oop_index = oop_recorder()->allocate_metadata_index(obj);
4059 } else {
4060 oop_index = oop_recorder()->find_index(obj);
4061 }
4062 RelocationHolder rspec = metadata_Relocation::spec(oop_index);
4063 mov(dst, Address((address)obj, rspec));
4064 }
4065
4066 Address MacroAssembler::constant_oop_address(jobject obj) {
4067 #ifdef ASSERT
4068 {
4069 ThreadInVMfromUnknown tiv;
4070 assert(oop_recorder() != NULL, "this assembler needs an OopRecorder");
4071 assert(Universe::heap()->is_in_reserved(JNIHandles::resolve(obj)), "not an oop");
4072 }
4073 #endif
4074 int oop_index = oop_recorder()->find_index(obj);
4075 return Address((address)obj, oop_Relocation::spec(oop_index));
4076 }
4077
4078 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
4079 void MacroAssembler::tlab_allocate(Register obj,
4080 Register var_size_in_bytes,
4081 int con_size_in_bytes,
4082 Register t1,
4083 Register t2,
4084 Label& slow_case) {
4085 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
4086 bs->tlab_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
4087 }
4088
4089 // Defines obj, preserves var_size_in_bytes
4090 void MacroAssembler::eden_allocate(Register obj,
4091 Register var_size_in_bytes,
4092 int con_size_in_bytes,
4093 Register t1,
4094 Label& slow_case) {
4095 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
4096 bs->eden_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case);
4097 }
4098
4099 // Zero words; len is in bytes
4100 // Destroys all registers except addr
4101 // len must be a nonzero multiple of wordSize
4102 void MacroAssembler::zero_memory(Register addr, Register len, Register t1) {
4103 assert_different_registers(addr, len, t1, rscratch1, rscratch2);
4104
4105 #ifdef ASSERT
4106 { Label L;
4107 tst(len, BytesPerWord - 1);
4108 br(Assembler::EQ, L);
4109 stop("len is not a multiple of BytesPerWord");
4110 bind(L);
4111 }
4112 #endif
4113
4114 #ifndef PRODUCT
4115 block_comment("zero memory");
4116 #endif
4117
4118 Label loop;
4119 Label entry;
4120
4121 // Algorithm:
4122 //
4123 // scratch1 = cnt & 7;
4124 // cnt -= scratch1;
4125 // p += scratch1;
4126 // switch (scratch1) {
4127 // do {
4128 // cnt -= 8;
4129 // p[-8] = 0;
4130 // case 7:
4131 // p[-7] = 0;
4132 // case 6:
4133 // p[-6] = 0;
4134 // // ...
4135 // case 1:
4136 // p[-1] = 0;
4137 // case 0:
4138 // p += 8;
4139 // } while (cnt);
4140 // }
4141
4142 const int unroll = 8; // Number of str(zr) instructions we'll unroll
4143
4144 lsr(len, len, LogBytesPerWord);
4145 andr(rscratch1, len, unroll - 1); // tmp1 = cnt % unroll
4146 sub(len, len, rscratch1); // cnt -= unroll
4147 // t1 always points to the end of the region we're about to zero
4148 add(t1, addr, rscratch1, Assembler::LSL, LogBytesPerWord);
4149 adr(rscratch2, entry);
4150 sub(rscratch2, rscratch2, rscratch1, Assembler::LSL, 2);
4151 br(rscratch2);
4152 bind(loop);
4153 sub(len, len, unroll);
4154 for (int i = -unroll; i < 0; i++)
4155 Assembler::str(zr, Address(t1, i * wordSize));
4156 bind(entry);
4157 add(t1, t1, unroll * wordSize);
4158 cbnz(len, loop);
4159 }
4160
4161 void MacroAssembler::verify_tlab() {
4162 #ifdef ASSERT
4163 if (UseTLAB && VerifyOops) {
4164 Label next, ok;
4165
4166 stp(rscratch2, rscratch1, Address(pre(sp, -16)));
4167
4168 ldr(rscratch2, Address(rthread, in_bytes(JavaThread::tlab_top_offset())));
4169 ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_start_offset())));
4170 cmp(rscratch2, rscratch1);
4171 br(Assembler::HS, next);
4172 STOP("assert(top >= start)");
4173 should_not_reach_here();
4174
4175 bind(next);
4176 ldr(rscratch2, Address(rthread, in_bytes(JavaThread::tlab_end_offset())));
4177 ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_top_offset())));
4178 cmp(rscratch2, rscratch1);
4179 br(Assembler::HS, ok);
4180 STOP("assert(top <= end)");
4181 should_not_reach_here();
4182
4183 bind(ok);
4184 ldp(rscratch2, rscratch1, Address(post(sp, 16)));
4185 }
4186 #endif
4187 }
4188
4189 // Writes to stack successive pages until offset reached to check for
4190 // stack overflow + shadow pages. This clobbers tmp.
4191 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
4192 assert_different_registers(tmp, size, rscratch1);
4193 mov(tmp, sp);
4194 // Bang stack for total size given plus shadow page size.
4195 // Bang one page at a time because large size can bang beyond yellow and
4196 // red zones.
4197 Label loop;
4198 mov(rscratch1, os::vm_page_size());
4199 bind(loop);
4200 lea(tmp, Address(tmp, -os::vm_page_size()));
4201 subsw(size, size, rscratch1);
4202 str(size, Address(tmp));
4203 br(Assembler::GT, loop);
4204
4205 // Bang down shadow pages too.
4206 // At this point, (tmp-0) is the last address touched, so don't
4207 // touch it again. (It was touched as (tmp-pagesize) but then tmp
4208 // was post-decremented.) Skip this address by starting at i=1, and
4209 // touch a few more pages below. N.B. It is important to touch all
4210 // the way down to and including i=StackShadowPages.
4211 for (int i = 0; i < (int)(JavaThread::stack_shadow_zone_size() / os::vm_page_size()) - 1; i++) {
4212 // this could be any sized move but this is can be a debugging crumb
4213 // so the bigger the better.
4214 lea(tmp, Address(tmp, -os::vm_page_size()));
4215 str(size, Address(tmp));
4216 }
4217 }
4218
4219
4220 // Move the address of the polling page into dest.
4221 void MacroAssembler::get_polling_page(Register dest, address page, relocInfo::relocType rtype) {
4222 if (SafepointMechanism::uses_thread_local_poll()) {
4223 ldr(dest, Address(rthread, Thread::polling_page_offset()));
4224 } else {
4225 unsigned long off;
4226 adrp(dest, Address(page, rtype), off);
4227 assert(off == 0, "polling page must be page aligned");
4228 }
4229 }
4230
4231 // Move the address of the polling page into r, then read the polling
4232 // page.
4233 address MacroAssembler::read_polling_page(Register r, address page, relocInfo::relocType rtype) {
4234 get_polling_page(r, page, rtype);
4235 return read_polling_page(r, rtype);
4236 }
4237
4238 // Read the polling page. The address of the polling page must
4239 // already be in r.
4240 address MacroAssembler::read_polling_page(Register r, relocInfo::relocType rtype) {
4241 InstructionMark im(this);
4242 code_section()->relocate(inst_mark(), rtype);
4243 ldrw(zr, Address(r, 0));
4244 return inst_mark();
4245 }
4246
4247 void MacroAssembler::adrp(Register reg1, const Address &dest, unsigned long &byte_offset) {
4248 relocInfo::relocType rtype = dest.rspec().reloc()->type();
4249 unsigned long low_page = (unsigned long)CodeCache::low_bound() >> 12;
4250 unsigned long high_page = (unsigned long)(CodeCache::high_bound()-1) >> 12;
4251 unsigned long dest_page = (unsigned long)dest.target() >> 12;
4252 long offset_low = dest_page - low_page;
4253 long offset_high = dest_page - high_page;
4254
4255 assert(is_valid_AArch64_address(dest.target()), "bad address");
4256 assert(dest.getMode() == Address::literal, "ADRP must be applied to a literal address");
4257
4258 InstructionMark im(this);
4259 code_section()->relocate(inst_mark(), dest.rspec());
4260 // 8143067: Ensure that the adrp can reach the dest from anywhere within
4261 // the code cache so that if it is relocated we know it will still reach
4262 if (offset_high >= -(1<<20) && offset_low < (1<<20)) {
4263 _adrp(reg1, dest.target());
4264 } else {
4265 unsigned long target = (unsigned long)dest.target();
4266 unsigned long adrp_target
4267 = (target & 0xffffffffUL) | ((unsigned long)pc() & 0xffff00000000UL);
4268
4269 _adrp(reg1, (address)adrp_target);
4270 movk(reg1, target >> 32, 32);
4271 }
4272 byte_offset = (unsigned long)dest.target() & 0xfff;
4273 }
4274
4275 void MacroAssembler::load_byte_map_base(Register reg) {
4276 jbyte *byte_map_base =
4277 ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base();
4278
4279 if (is_valid_AArch64_address((address)byte_map_base)) {
4280 // Strictly speaking the byte_map_base isn't an address at all,
4281 // and it might even be negative.
4282 unsigned long offset;
4283 adrp(reg, ExternalAddress((address)byte_map_base), offset);
4284 // We expect offset to be zero with most collectors.
4285 if (offset != 0) {
4286 add(reg, reg, offset);
4287 }
4288 } else {
4289 mov(reg, (uint64_t)byte_map_base);
4290 }
4291 }
4292
4293 void MacroAssembler::build_frame(int framesize) {
4294 assert(framesize > 0, "framesize must be > 0");
4295 if (framesize < ((1 << 9) + 2 * wordSize)) {
4296 sub(sp, sp, framesize);
4297 stp(rfp, lr, Address(sp, framesize - 2 * wordSize));
4298 if (PreserveFramePointer) add(rfp, sp, framesize - 2 * wordSize);
4299 } else {
4300 stp(rfp, lr, Address(pre(sp, -2 * wordSize)));
4301 if (PreserveFramePointer) mov(rfp, sp);
4302 if (framesize < ((1 << 12) + 2 * wordSize))
4303 sub(sp, sp, framesize - 2 * wordSize);
4304 else {
4305 mov(rscratch1, framesize - 2 * wordSize);
4306 sub(sp, sp, rscratch1);
4307 }
4308 }
4309 }
4310
4311 void MacroAssembler::remove_frame(int framesize) {
4312 assert(framesize > 0, "framesize must be > 0");
4313 if (framesize < ((1 << 9) + 2 * wordSize)) {
4314 ldp(rfp, lr, Address(sp, framesize - 2 * wordSize));
4315 add(sp, sp, framesize);
4316 } else {
4317 if (framesize < ((1 << 12) + 2 * wordSize))
4318 add(sp, sp, framesize - 2 * wordSize);
4319 else {
4320 mov(rscratch1, framesize - 2 * wordSize);
4321 add(sp, sp, rscratch1);
4322 }
4323 ldp(rfp, lr, Address(post(sp, 2 * wordSize)));
4324 }
4325 }
4326
4327 typedef void (MacroAssembler::* chr_insn)(Register Rt, const Address &adr);
4328
4329 // Search for str1 in str2 and return index or -1
4330 void MacroAssembler::string_indexof(Register str2, Register str1,
4331 Register cnt2, Register cnt1,
4332 Register tmp1, Register tmp2,
4333 Register tmp3, Register tmp4,
4334 Register tmp5, Register tmp6,
4335 int icnt1, Register result, int ae) {
4336 // NOTE: tmp5, tmp6 can be zr depending on specific method version
4337 Label LINEARSEARCH, LINEARSTUB, LINEAR_MEDIUM, DONE, NOMATCH, MATCH;
4338
4339 Register ch1 = rscratch1;
4340 Register ch2 = rscratch2;
4341 Register cnt1tmp = tmp1;
4342 Register cnt2tmp = tmp2;
4343 Register cnt1_neg = cnt1;
4344 Register cnt2_neg = cnt2;
4345 Register result_tmp = tmp4;
4346
4347 bool isL = ae == StrIntrinsicNode::LL;
4348
4349 bool str1_isL = ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UL;
4350 bool str2_isL = ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::LU;
4351 int str1_chr_shift = str1_isL ? 0:1;
4352 int str2_chr_shift = str2_isL ? 0:1;
4353 int str1_chr_size = str1_isL ? 1:2;
4354 int str2_chr_size = str2_isL ? 1:2;
4355 chr_insn str1_load_1chr = str1_isL ? (chr_insn)&MacroAssembler::ldrb :
4356 (chr_insn)&MacroAssembler::ldrh;
4357 chr_insn str2_load_1chr = str2_isL ? (chr_insn)&MacroAssembler::ldrb :
4358 (chr_insn)&MacroAssembler::ldrh;
4359 chr_insn load_2chr = isL ? (chr_insn)&MacroAssembler::ldrh : (chr_insn)&MacroAssembler::ldrw;
4360 chr_insn load_4chr = isL ? (chr_insn)&MacroAssembler::ldrw : (chr_insn)&MacroAssembler::ldr;
4361
4362 // Note, inline_string_indexOf() generates checks:
4363 // if (substr.count > string.count) return -1;
4364 // if (substr.count == 0) return 0;
4365
4366 // We have two strings, a source string in str2, cnt2 and a pattern string
4367 // in str1, cnt1. Find the 1st occurence of pattern in source or return -1.
4368
4369 // For larger pattern and source we use a simplified Boyer Moore algorithm.
4370 // With a small pattern and source we use linear scan.
4371
4372 if (icnt1 == -1) {
4373 sub(result_tmp, cnt2, cnt1);
4374 cmp(cnt1, 8); // Use Linear Scan if cnt1 < 8 || cnt1 >= 256
4375 br(LT, LINEARSEARCH);
4376 dup(v0, T16B, cnt1); // done in separate FPU pipeline. Almost no penalty
4377 cmp(cnt1, 256);
4378 lsr(tmp1, cnt2, 2);
4379 ccmp(cnt1, tmp1, 0b0000, LT); // Source must be 4 * pattern for BM
4380 br(GE, LINEARSTUB);
4381 }
4382
4383 // The Boyer Moore alogorithm is based on the description here:-
4384 //
4385 // http://en.wikipedia.org/wiki/Boyer%E2%80%93Moore_string_search_algorithm
4386 //
4387 // This describes and algorithm with 2 shift rules. The 'Bad Character' rule
4388 // and the 'Good Suffix' rule.
4389 //
4390 // These rules are essentially heuristics for how far we can shift the
4391 // pattern along the search string.
4392 //
4393 // The implementation here uses the 'Bad Character' rule only because of the
4394 // complexity of initialisation for the 'Good Suffix' rule.
4395 //
4396 // This is also known as the Boyer-Moore-Horspool algorithm:-
4397 //
4398 // http://en.wikipedia.org/wiki/Boyer-Moore-Horspool_algorithm
4399 //
4400 // This particular implementation has few java-specific optimizations.
4401 //
4402 // #define ASIZE 256
4403 //
4404 // int bm(unsigned char *x, int m, unsigned char *y, int n) {
4405 // int i, j;
4406 // unsigned c;
4407 // unsigned char bc[ASIZE];
4408 //
4409 // /* Preprocessing */
4410 // for (i = 0; i < ASIZE; ++i)
4411 // bc[i] = m;
4412 // for (i = 0; i < m - 1; ) {
4413 // c = x[i];
4414 // ++i;
4415 // // c < 256 for Latin1 string, so, no need for branch
4416 // #ifdef PATTERN_STRING_IS_LATIN1
4417 // bc[c] = m - i;
4418 // #else
4419 // if (c < ASIZE) bc[c] = m - i;
4420 // #endif
4421 // }
4422 //
4423 // /* Searching */
4424 // j = 0;
4425 // while (j <= n - m) {
4426 // c = y[i+j];
4427 // if (x[m-1] == c)
4428 // for (i = m - 2; i >= 0 && x[i] == y[i + j]; --i);
4429 // if (i < 0) return j;
4430 // // c < 256 for Latin1 string, so, no need for branch
4431 // #ifdef SOURCE_STRING_IS_LATIN1
4432 // // LL case: (c< 256) always true. Remove branch
4433 // j += bc[y[j+m-1]];
4434 // #endif
4435 // #ifndef PATTERN_STRING_IS_UTF
4436 // // UU case: need if (c<ASIZE) check. Skip 1 character if not.
4437 // if (c < ASIZE)
4438 // j += bc[y[j+m-1]];
4439 // else
4440 // j += 1
4441 // #endif
4442 // #ifdef PATTERN_IS_LATIN1_AND_SOURCE_IS_UTF
4443 // // UL case: need if (c<ASIZE) check. Skip <pattern length> if not.
4444 // if (c < ASIZE)
4445 // j += bc[y[j+m-1]];
4446 // else
4447 // j += m
4448 // #endif
4449 // }
4450 // }
4451
4452 if (icnt1 == -1) {
4453 Label BCLOOP, BCSKIP, BMLOOPSTR2, BMLOOPSTR1, BMSKIP, BMADV, BMMATCH,
4454 BMLOOPSTR1_LASTCMP, BMLOOPSTR1_CMP, BMLOOPSTR1_AFTER_LOAD, BM_INIT_LOOP;
4455 Register cnt1end = tmp2;
4456 Register str2end = cnt2;
4457 Register skipch = tmp2;
4458
4459 // str1 length is >=8, so, we can read at least 1 register for cases when
4460 // UTF->Latin1 conversion is not needed(8 LL or 4UU) and half register for
4461 // UL case. We'll re-read last character in inner pre-loop code to have
4462 // single outer pre-loop load
4463 const int firstStep = isL ? 7 : 3;
4464
4465 const int ASIZE = 256;
4466 const int STORED_BYTES = 32; // amount of bytes stored per instruction
4467 sub(sp, sp, ASIZE);
4468 mov(tmp5, ASIZE/STORED_BYTES); // loop iterations
4469 mov(ch1, sp);
4470 BIND(BM_INIT_LOOP);
4471 stpq(v0, v0, Address(post(ch1, STORED_BYTES)));
4472 subs(tmp5, tmp5, 1);
4473 br(GT, BM_INIT_LOOP);
4474
4475 sub(cnt1tmp, cnt1, 1);
4476 mov(tmp5, str2);
4477 add(str2end, str2, result_tmp, LSL, str2_chr_shift);
4478 sub(ch2, cnt1, 1);
4479 mov(tmp3, str1);
4480 BIND(BCLOOP);
4481 (this->*str1_load_1chr)(ch1, Address(post(tmp3, str1_chr_size)));
4482 if (!str1_isL) {
4483 cmp(ch1, ASIZE);
4484 br(HS, BCSKIP);
4485 }
4486 strb(ch2, Address(sp, ch1));
4487 BIND(BCSKIP);
4488 subs(ch2, ch2, 1);
4489 br(GT, BCLOOP);
4490
4491 add(tmp6, str1, cnt1, LSL, str1_chr_shift); // address after str1
4492 if (str1_isL == str2_isL) {
4493 // load last 8 bytes (8LL/4UU symbols)
4494 ldr(tmp6, Address(tmp6, -wordSize));
4495 } else {
4496 ldrw(tmp6, Address(tmp6, -wordSize/2)); // load last 4 bytes(4 symbols)
4497 // convert Latin1 to UTF. We'll have to wait until load completed, but
4498 // it's still faster than per-character loads+checks
4499 lsr(tmp3, tmp6, BitsPerByte * (wordSize/2 - str1_chr_size)); // str1[N-1]
4500 ubfx(ch1, tmp6, 8, 8); // str1[N-2]
4501 ubfx(ch2, tmp6, 16, 8); // str1[N-3]
4502 andr(tmp6, tmp6, 0xFF); // str1[N-4]
4503 orr(ch2, ch1, ch2, LSL, 16);
4504 orr(tmp6, tmp6, tmp3, LSL, 48);
4505 orr(tmp6, tmp6, ch2, LSL, 16);
4506 }
4507 BIND(BMLOOPSTR2);
4508 (this->*str2_load_1chr)(skipch, Address(str2, cnt1tmp, Address::lsl(str2_chr_shift)));
4509 sub(cnt1tmp, cnt1tmp, firstStep); // cnt1tmp is positive here, because cnt1 >= 8
4510 if (str1_isL == str2_isL) {
4511 // re-init tmp3. It's for free because it's executed in parallel with
4512 // load above. Alternative is to initialize it before loop, but it'll
4513 // affect performance on in-order systems with 2 or more ld/st pipelines
4514 lsr(tmp3, tmp6, BitsPerByte * (wordSize - str1_chr_size));
4515 }
4516 if (!isL) { // UU/UL case
4517 lsl(ch2, cnt1tmp, 1); // offset in bytes
4518 }
4519 cmp(tmp3, skipch);
4520 br(NE, BMSKIP);
4521 ldr(ch2, Address(str2, isL ? cnt1tmp : ch2));
4522 mov(ch1, tmp6);
4523 if (isL) {
4524 b(BMLOOPSTR1_AFTER_LOAD);
4525 } else {
4526 sub(cnt1tmp, cnt1tmp, 1); // no need to branch for UU/UL case. cnt1 >= 8
4527 b(BMLOOPSTR1_CMP);
4528 }
4529 BIND(BMLOOPSTR1);
4530 (this->*str1_load_1chr)(ch1, Address(str1, cnt1tmp, Address::lsl(str1_chr_shift)));
4531 (this->*str2_load_1chr)(ch2, Address(str2, cnt1tmp, Address::lsl(str2_chr_shift)));
4532 BIND(BMLOOPSTR1_AFTER_LOAD);
4533 subs(cnt1tmp, cnt1tmp, 1);
4534 br(LT, BMLOOPSTR1_LASTCMP);
4535 BIND(BMLOOPSTR1_CMP);
4536 cmp(ch1, ch2);
4537 br(EQ, BMLOOPSTR1);
4538 BIND(BMSKIP);
4539 if (!isL) {
4540 // if we've met UTF symbol while searching Latin1 pattern, then we can
4541 // skip cnt1 symbols
4542 if (str1_isL != str2_isL) {
4543 mov(result_tmp, cnt1);
4544 } else {
4545 mov(result_tmp, 1);
4546 }
4547 cmp(skipch, ASIZE);
4548 br(HS, BMADV);
4549 }
4550 ldrb(result_tmp, Address(sp, skipch)); // load skip distance
4551 BIND(BMADV);
4552 sub(cnt1tmp, cnt1, 1);
4553 add(str2, str2, result_tmp, LSL, str2_chr_shift);
4554 cmp(str2, str2end);
4555 br(LE, BMLOOPSTR2);
4556 add(sp, sp, ASIZE);
4557 b(NOMATCH);
4558 BIND(BMLOOPSTR1_LASTCMP);
4559 cmp(ch1, ch2);
4560 br(NE, BMSKIP);
4561 BIND(BMMATCH);
4562 sub(result, str2, tmp5);
4563 if (!str2_isL) lsr(result, result, 1);
4564 add(sp, sp, ASIZE);
4565 b(DONE);
4566
4567 BIND(LINEARSTUB);
4568 cmp(cnt1, 16); // small patterns still should be handled by simple algorithm
4569 br(LT, LINEAR_MEDIUM);
4570 mov(result, zr);
4571 RuntimeAddress stub = NULL;
4572 if (isL) {
4573 stub = RuntimeAddress(StubRoutines::aarch64::string_indexof_linear_ll());
4574 assert(stub.target() != NULL, "string_indexof_linear_ll stub has not been generated");
4575 } else if (str1_isL) {
4576 stub = RuntimeAddress(StubRoutines::aarch64::string_indexof_linear_ul());
4577 assert(stub.target() != NULL, "string_indexof_linear_ul stub has not been generated");
4578 } else {
4579 stub = RuntimeAddress(StubRoutines::aarch64::string_indexof_linear_uu());
4580 assert(stub.target() != NULL, "string_indexof_linear_uu stub has not been generated");
4581 }
4582 trampoline_call(stub);
4583 b(DONE);
4584 }
4585
4586 BIND(LINEARSEARCH);
4587 {
4588 Label DO1, DO2, DO3;
4589
4590 Register str2tmp = tmp2;
4591 Register first = tmp3;
4592
4593 if (icnt1 == -1)
4594 {
4595 Label DOSHORT, FIRST_LOOP, STR2_NEXT, STR1_LOOP, STR1_NEXT;
4596
4597 cmp(cnt1, str1_isL == str2_isL ? 4 : 2);
4598 br(LT, DOSHORT);
4599 BIND(LINEAR_MEDIUM);
4600 (this->*str1_load_1chr)(first, Address(str1));
4601 lea(str1, Address(str1, cnt1, Address::lsl(str1_chr_shift)));
4602 sub(cnt1_neg, zr, cnt1, LSL, str1_chr_shift);
4603 lea(str2, Address(str2, result_tmp, Address::lsl(str2_chr_shift)));
4604 sub(cnt2_neg, zr, result_tmp, LSL, str2_chr_shift);
4605
4606 BIND(FIRST_LOOP);
4607 (this->*str2_load_1chr)(ch2, Address(str2, cnt2_neg));
4608 cmp(first, ch2);
4609 br(EQ, STR1_LOOP);
4610 BIND(STR2_NEXT);
4611 adds(cnt2_neg, cnt2_neg, str2_chr_size);
4612 br(LE, FIRST_LOOP);
4613 b(NOMATCH);
4614
4615 BIND(STR1_LOOP);
4616 adds(cnt1tmp, cnt1_neg, str1_chr_size);
4617 add(cnt2tmp, cnt2_neg, str2_chr_size);
4618 br(GE, MATCH);
4619
4620 BIND(STR1_NEXT);
4621 (this->*str1_load_1chr)(ch1, Address(str1, cnt1tmp));
4622 (this->*str2_load_1chr)(ch2, Address(str2, cnt2tmp));
4623 cmp(ch1, ch2);
4624 br(NE, STR2_NEXT);
4625 adds(cnt1tmp, cnt1tmp, str1_chr_size);
4626 add(cnt2tmp, cnt2tmp, str2_chr_size);
4627 br(LT, STR1_NEXT);
4628 b(MATCH);
4629
4630 BIND(DOSHORT);
4631 if (str1_isL == str2_isL) {
4632 cmp(cnt1, 2);
4633 br(LT, DO1);
4634 br(GT, DO3);
4635 }
4636 }
4637
4638 if (icnt1 == 4) {
4639 Label CH1_LOOP;
4640
4641 (this->*load_4chr)(ch1, str1);
4642 sub(result_tmp, cnt2, 4);
4643 lea(str2, Address(str2, result_tmp, Address::lsl(str2_chr_shift)));
4644 sub(cnt2_neg, zr, result_tmp, LSL, str2_chr_shift);
4645
4646 BIND(CH1_LOOP);
4647 (this->*load_4chr)(ch2, Address(str2, cnt2_neg));
4648 cmp(ch1, ch2);
4649 br(EQ, MATCH);
4650 adds(cnt2_neg, cnt2_neg, str2_chr_size);
4651 br(LE, CH1_LOOP);
4652 b(NOMATCH);
4653 }
4654
4655 if ((icnt1 == -1 && str1_isL == str2_isL) || icnt1 == 2) {
4656 Label CH1_LOOP;
4657
4658 BIND(DO2);
4659 (this->*load_2chr)(ch1, str1);
4660 if (icnt1 == 2) {
4661 sub(result_tmp, cnt2, 2);
4662 }
4663 lea(str2, Address(str2, result_tmp, Address::lsl(str2_chr_shift)));
4664 sub(cnt2_neg, zr, result_tmp, LSL, str2_chr_shift);
4665 BIND(CH1_LOOP);
4666 (this->*load_2chr)(ch2, Address(str2, cnt2_neg));
4667 cmp(ch1, ch2);
4668 br(EQ, MATCH);
4669 adds(cnt2_neg, cnt2_neg, str2_chr_size);
4670 br(LE, CH1_LOOP);
4671 b(NOMATCH);
4672 }
4673
4674 if ((icnt1 == -1 && str1_isL == str2_isL) || icnt1 == 3) {
4675 Label FIRST_LOOP, STR2_NEXT, STR1_LOOP;
4676
4677 BIND(DO3);
4678 (this->*load_2chr)(first, str1);
4679 (this->*str1_load_1chr)(ch1, Address(str1, 2*str1_chr_size));
4680 if (icnt1 == 3) {
4681 sub(result_tmp, cnt2, 3);
4682 }
4683 lea(str2, Address(str2, result_tmp, Address::lsl(str2_chr_shift)));
4684 sub(cnt2_neg, zr, result_tmp, LSL, str2_chr_shift);
4685 BIND(FIRST_LOOP);
4686 (this->*load_2chr)(ch2, Address(str2, cnt2_neg));
4687 cmpw(first, ch2);
4688 br(EQ, STR1_LOOP);
4689 BIND(STR2_NEXT);
4690 adds(cnt2_neg, cnt2_neg, str2_chr_size);
4691 br(LE, FIRST_LOOP);
4692 b(NOMATCH);
4693
4694 BIND(STR1_LOOP);
4695 add(cnt2tmp, cnt2_neg, 2*str2_chr_size);
4696 (this->*str2_load_1chr)(ch2, Address(str2, cnt2tmp));
4697 cmp(ch1, ch2);
4698 br(NE, STR2_NEXT);
4699 b(MATCH);
4700 }
4701
4702 if (icnt1 == -1 || icnt1 == 1) {
4703 Label CH1_LOOP, HAS_ZERO, DO1_SHORT, DO1_LOOP;
4704
4705 BIND(DO1);
4706 (this->*str1_load_1chr)(ch1, str1);
4707 cmp(cnt2, 8);
4708 br(LT, DO1_SHORT);
4709
4710 sub(result_tmp, cnt2, 8/str2_chr_size);
4711 sub(cnt2_neg, zr, result_tmp, LSL, str2_chr_shift);
4712 mov(tmp3, str2_isL ? 0x0101010101010101 : 0x0001000100010001);
4713 lea(str2, Address(str2, result_tmp, Address::lsl(str2_chr_shift)));
4714
4715 if (str2_isL) {
4716 orr(ch1, ch1, ch1, LSL, 8);
4717 }
4718 orr(ch1, ch1, ch1, LSL, 16);
4719 orr(ch1, ch1, ch1, LSL, 32);
4720 BIND(CH1_LOOP);
4721 ldr(ch2, Address(str2, cnt2_neg));
4722 eor(ch2, ch1, ch2);
4723 sub(tmp1, ch2, tmp3);
4724 orr(tmp2, ch2, str2_isL ? 0x7f7f7f7f7f7f7f7f : 0x7fff7fff7fff7fff);
4725 bics(tmp1, tmp1, tmp2);
4726 br(NE, HAS_ZERO);
4727 adds(cnt2_neg, cnt2_neg, 8);
4728 br(LT, CH1_LOOP);
4729
4730 cmp(cnt2_neg, 8);
4731 mov(cnt2_neg, 0);
4732 br(LT, CH1_LOOP);
4733 b(NOMATCH);
4734
4735 BIND(HAS_ZERO);
4736 rev(tmp1, tmp1);
4737 clz(tmp1, tmp1);
4738 add(cnt2_neg, cnt2_neg, tmp1, LSR, 3);
4739 b(MATCH);
4740
4741 BIND(DO1_SHORT);
4742 mov(result_tmp, cnt2);
4743 lea(str2, Address(str2, cnt2, Address::lsl(str2_chr_shift)));
4744 sub(cnt2_neg, zr, cnt2, LSL, str2_chr_shift);
4745 BIND(DO1_LOOP);
4746 (this->*str2_load_1chr)(ch2, Address(str2, cnt2_neg));
4747 cmpw(ch1, ch2);
4748 br(EQ, MATCH);
4749 adds(cnt2_neg, cnt2_neg, str2_chr_size);
4750 br(LT, DO1_LOOP);
4751 }
4752 }
4753 BIND(NOMATCH);
4754 mov(result, -1);
4755 b(DONE);
4756 BIND(MATCH);
4757 add(result, result_tmp, cnt2_neg, ASR, str2_chr_shift);
4758 BIND(DONE);
4759 }
4760
4761 typedef void (MacroAssembler::* chr_insn)(Register Rt, const Address &adr);
4762 typedef void (MacroAssembler::* uxt_insn)(Register Rd, Register Rn);
4763
4764 void MacroAssembler::string_indexof_char(Register str1, Register cnt1,
4765 Register ch, Register result,
4766 Register tmp1, Register tmp2, Register tmp3)
4767 {
4768 Label CH1_LOOP, HAS_ZERO, DO1_SHORT, DO1_LOOP, MATCH, NOMATCH, DONE;
4769 Register cnt1_neg = cnt1;
4770 Register ch1 = rscratch1;
4771 Register result_tmp = rscratch2;
4772
4773 cmp(cnt1, 4);
4774 br(LT, DO1_SHORT);
4775
4776 orr(ch, ch, ch, LSL, 16);
4777 orr(ch, ch, ch, LSL, 32);
4778
4779 sub(cnt1, cnt1, 4);
4780 mov(result_tmp, cnt1);
4781 lea(str1, Address(str1, cnt1, Address::uxtw(1)));
4782 sub(cnt1_neg, zr, cnt1, LSL, 1);
4783
4784 mov(tmp3, 0x0001000100010001);
4785
4786 BIND(CH1_LOOP);
4787 ldr(ch1, Address(str1, cnt1_neg));
4788 eor(ch1, ch, ch1);
4789 sub(tmp1, ch1, tmp3);
4790 orr(tmp2, ch1, 0x7fff7fff7fff7fff);
4791 bics(tmp1, tmp1, tmp2);
4792 br(NE, HAS_ZERO);
4793 adds(cnt1_neg, cnt1_neg, 8);
4794 br(LT, CH1_LOOP);
4795
4796 cmp(cnt1_neg, 8);
4797 mov(cnt1_neg, 0);
4798 br(LT, CH1_LOOP);
4799 b(NOMATCH);
4800
4801 BIND(HAS_ZERO);
4802 rev(tmp1, tmp1);
4803 clz(tmp1, tmp1);
4804 add(cnt1_neg, cnt1_neg, tmp1, LSR, 3);
4805 b(MATCH);
4806
4807 BIND(DO1_SHORT);
4808 mov(result_tmp, cnt1);
4809 lea(str1, Address(str1, cnt1, Address::uxtw(1)));
4810 sub(cnt1_neg, zr, cnt1, LSL, 1);
4811 BIND(DO1_LOOP);
4812 ldrh(ch1, Address(str1, cnt1_neg));
4813 cmpw(ch, ch1);
4814 br(EQ, MATCH);
4815 adds(cnt1_neg, cnt1_neg, 2);
4816 br(LT, DO1_LOOP);
4817 BIND(NOMATCH);
4818 mov(result, -1);
4819 b(DONE);
4820 BIND(MATCH);
4821 add(result, result_tmp, cnt1_neg, ASR, 1);
4822 BIND(DONE);
4823 }
4824
4825 // Compare strings.
4826 void MacroAssembler::string_compare(Register str1, Register str2,
4827 Register cnt1, Register cnt2, Register result, Register tmp1, Register tmp2,
4828 FloatRegister vtmp1, FloatRegister vtmp2, FloatRegister vtmp3, int ae) {
4829 Label DONE, SHORT_LOOP, SHORT_STRING, SHORT_LAST, TAIL, STUB,
4830 DIFFERENCE, NEXT_WORD, SHORT_LOOP_TAIL, SHORT_LAST2, SHORT_LAST_INIT,
4831 SHORT_LOOP_START, TAIL_CHECK;
4832
4833 const int STUB_THRESHOLD = 64 + 8;
4834 bool isLL = ae == StrIntrinsicNode::LL;
4835 bool isLU = ae == StrIntrinsicNode::LU;
4836 bool isUL = ae == StrIntrinsicNode::UL;
4837
4838 bool str1_isL = isLL || isLU;
4839 bool str2_isL = isLL || isUL;
4840
4841 int str1_chr_shift = str1_isL ? 0 : 1;
4842 int str2_chr_shift = str2_isL ? 0 : 1;
4843 int str1_chr_size = str1_isL ? 1 : 2;
4844 int str2_chr_size = str2_isL ? 1 : 2;
4845 int minCharsInWord = isLL ? wordSize : wordSize/2;
4846
4847 FloatRegister vtmpZ = vtmp1, vtmp = vtmp2;
4848 chr_insn str1_load_chr = str1_isL ? (chr_insn)&MacroAssembler::ldrb :
4849 (chr_insn)&MacroAssembler::ldrh;
4850 chr_insn str2_load_chr = str2_isL ? (chr_insn)&MacroAssembler::ldrb :
4851 (chr_insn)&MacroAssembler::ldrh;
4852 uxt_insn ext_chr = isLL ? (uxt_insn)&MacroAssembler::uxtbw :
4853 (uxt_insn)&MacroAssembler::uxthw;
4854
4855 BLOCK_COMMENT("string_compare {");
4856
4857 // Bizzarely, the counts are passed in bytes, regardless of whether they
4858 // are L or U strings, however the result is always in characters.
4859 if (!str1_isL) asrw(cnt1, cnt1, 1);
4860 if (!str2_isL) asrw(cnt2, cnt2, 1);
4861
4862 // Compute the minimum of the string lengths and save the difference.
4863 subsw(result, cnt1, cnt2);
4864 cselw(cnt2, cnt1, cnt2, Assembler::LE); // min
4865
4866 // A very short string
4867 cmpw(cnt2, minCharsInWord);
4868 br(Assembler::LT, SHORT_STRING);
4869
4870 // Compare longwords
4871 // load first parts of strings and finish initialization while loading
4872 {
4873 if (str1_isL == str2_isL) { // LL or UU
4874 ldr(tmp1, Address(str1));
4875 cmp(str1, str2);
4876 br(Assembler::EQ, DONE);
4877 ldr(tmp2, Address(str2));
4878 cmp(cnt2, STUB_THRESHOLD);
4879 br(GE, STUB);
4880 subsw(cnt2, cnt2, minCharsInWord);
4881 br(EQ, TAIL_CHECK);
4882 lea(str2, Address(str2, cnt2, Address::uxtw(str2_chr_shift)));
4883 lea(str1, Address(str1, cnt2, Address::uxtw(str1_chr_shift)));
4884 sub(cnt2, zr, cnt2, LSL, str2_chr_shift);
4885 } else if (isLU) {
4886 ldrs(vtmp, Address(str1));
4887 cmp(str1, str2);
4888 br(Assembler::EQ, DONE);
4889 ldr(tmp2, Address(str2));
4890 cmp(cnt2, STUB_THRESHOLD);
4891 br(GE, STUB);
4892 subsw(cnt2, cnt2, 4);
4893 br(EQ, TAIL_CHECK);
4894 eor(vtmpZ, T16B, vtmpZ, vtmpZ);
4895 lea(str1, Address(str1, cnt2, Address::uxtw(str1_chr_shift)));
4896 lea(str2, Address(str2, cnt2, Address::uxtw(str2_chr_shift)));
4897 zip1(vtmp, T8B, vtmp, vtmpZ);
4898 sub(cnt1, zr, cnt2, LSL, str1_chr_shift);
4899 sub(cnt2, zr, cnt2, LSL, str2_chr_shift);
4900 add(cnt1, cnt1, 4);
4901 fmovd(tmp1, vtmp);
4902 } else { // UL case
4903 ldr(tmp1, Address(str1));
4904 cmp(str1, str2);
4905 br(Assembler::EQ, DONE);
4906 ldrs(vtmp, Address(str2));
4907 cmp(cnt2, STUB_THRESHOLD);
4908 br(GE, STUB);
4909 subsw(cnt2, cnt2, 4);
4910 br(EQ, TAIL_CHECK);
4911 lea(str1, Address(str1, cnt2, Address::uxtw(str1_chr_shift)));
4912 eor(vtmpZ, T16B, vtmpZ, vtmpZ);
4913 lea(str2, Address(str2, cnt2, Address::uxtw(str2_chr_shift)));
4914 sub(cnt1, zr, cnt2, LSL, str1_chr_shift);
4915 zip1(vtmp, T8B, vtmp, vtmpZ);
4916 sub(cnt2, zr, cnt2, LSL, str2_chr_shift);
4917 add(cnt1, cnt1, 8);
4918 fmovd(tmp2, vtmp);
4919 }
4920 adds(cnt2, cnt2, isUL ? 4 : 8);
4921 br(GE, TAIL);
4922 eor(rscratch2, tmp1, tmp2);
4923 cbnz(rscratch2, DIFFERENCE);
4924 // main loop
4925 bind(NEXT_WORD);
4926 if (str1_isL == str2_isL) {
4927 ldr(tmp1, Address(str1, cnt2));
4928 ldr(tmp2, Address(str2, cnt2));
4929 adds(cnt2, cnt2, 8);
4930 } else if (isLU) {
4931 ldrs(vtmp, Address(str1, cnt1));
4932 ldr(tmp2, Address(str2, cnt2));
4933 add(cnt1, cnt1, 4);
4934 zip1(vtmp, T8B, vtmp, vtmpZ);
4935 fmovd(tmp1, vtmp);
4936 adds(cnt2, cnt2, 8);
4937 } else { // UL
4938 ldrs(vtmp, Address(str2, cnt2));
4939 ldr(tmp1, Address(str1, cnt1));
4940 zip1(vtmp, T8B, vtmp, vtmpZ);
4941 add(cnt1, cnt1, 8);
4942 fmovd(tmp2, vtmp);
4943 adds(cnt2, cnt2, 4);
4944 }
4945 br(GE, TAIL);
4946
4947 eor(rscratch2, tmp1, tmp2);
4948 cbz(rscratch2, NEXT_WORD);
4949 b(DIFFERENCE);
4950 bind(TAIL);
4951 eor(rscratch2, tmp1, tmp2);
4952 cbnz(rscratch2, DIFFERENCE);
4953 // Last longword. In the case where length == 4 we compare the
4954 // same longword twice, but that's still faster than another
4955 // conditional branch.
4956 if (str1_isL == str2_isL) {
4957 ldr(tmp1, Address(str1));
4958 ldr(tmp2, Address(str2));
4959 } else if (isLU) {
4960 ldrs(vtmp, Address(str1));
4961 ldr(tmp2, Address(str2));
4962 zip1(vtmp, T8B, vtmp, vtmpZ);
4963 fmovd(tmp1, vtmp);
4964 } else { // UL
4965 ldrs(vtmp, Address(str2));
4966 ldr(tmp1, Address(str1));
4967 zip1(vtmp, T8B, vtmp, vtmpZ);
4968 fmovd(tmp2, vtmp);
4969 }
4970 bind(TAIL_CHECK);
4971 eor(rscratch2, tmp1, tmp2);
4972 cbz(rscratch2, DONE);
4973
4974 // Find the first different characters in the longwords and
4975 // compute their difference.
4976 bind(DIFFERENCE);
4977 rev(rscratch2, rscratch2);
4978 clz(rscratch2, rscratch2);
4979 andr(rscratch2, rscratch2, isLL ? -8 : -16);
4980 lsrv(tmp1, tmp1, rscratch2);
4981 (this->*ext_chr)(tmp1, tmp1);
4982 lsrv(tmp2, tmp2, rscratch2);
4983 (this->*ext_chr)(tmp2, tmp2);
4984 subw(result, tmp1, tmp2);
4985 b(DONE);
4986 }
4987
4988 bind(STUB);
4989 RuntimeAddress stub = NULL;
4990 switch(ae) {
4991 case StrIntrinsicNode::LL:
4992 stub = RuntimeAddress(StubRoutines::aarch64::compare_long_string_LL());
4993 break;
4994 case StrIntrinsicNode::UU:
4995 stub = RuntimeAddress(StubRoutines::aarch64::compare_long_string_UU());
4996 break;
4997 case StrIntrinsicNode::LU:
4998 stub = RuntimeAddress(StubRoutines::aarch64::compare_long_string_LU());
4999 break;
5000 case StrIntrinsicNode::UL:
5001 stub = RuntimeAddress(StubRoutines::aarch64::compare_long_string_UL());
5002 break;
5003 default:
5004 ShouldNotReachHere();
5005 }
5006 assert(stub.target() != NULL, "compare_long_string stub has not been generated");
5007 trampoline_call(stub);
5008 b(DONE);
5009
5010 bind(SHORT_STRING);
5011 // Is the minimum length zero?
5012 cbz(cnt2, DONE);
5013 // arrange code to do most branches while loading and loading next characters
5014 // while comparing previous
5015 (this->*str1_load_chr)(tmp1, Address(post(str1, str1_chr_size)));
5016 subs(cnt2, cnt2, 1);
5017 br(EQ, SHORT_LAST_INIT);
5018 (this->*str2_load_chr)(cnt1, Address(post(str2, str2_chr_size)));
5019 b(SHORT_LOOP_START);
5020 bind(SHORT_LOOP);
5021 subs(cnt2, cnt2, 1);
5022 br(EQ, SHORT_LAST);
5023 bind(SHORT_LOOP_START);
5024 (this->*str1_load_chr)(tmp2, Address(post(str1, str1_chr_size)));
5025 (this->*str2_load_chr)(rscratch1, Address(post(str2, str2_chr_size)));
5026 cmp(tmp1, cnt1);
5027 br(NE, SHORT_LOOP_TAIL);
5028 subs(cnt2, cnt2, 1);
5029 br(EQ, SHORT_LAST2);
5030 (this->*str1_load_chr)(tmp1, Address(post(str1, str1_chr_size)));
5031 (this->*str2_load_chr)(cnt1, Address(post(str2, str2_chr_size)));
5032 cmp(tmp2, rscratch1);
5033 br(EQ, SHORT_LOOP);
5034 sub(result, tmp2, rscratch1);
5035 b(DONE);
5036 bind(SHORT_LOOP_TAIL);
5037 sub(result, tmp1, cnt1);
5038 b(DONE);
5039 bind(SHORT_LAST2);
5040 cmp(tmp2, rscratch1);
5041 br(EQ, DONE);
5042 sub(result, tmp2, rscratch1);
5043
5044 b(DONE);
5045 bind(SHORT_LAST_INIT);
5046 (this->*str2_load_chr)(cnt1, Address(post(str2, str2_chr_size)));
5047 bind(SHORT_LAST);
5048 cmp(tmp1, cnt1);
5049 br(EQ, DONE);
5050 sub(result, tmp1, cnt1);
5051
5052 bind(DONE);
5053
5054 BLOCK_COMMENT("} string_compare");
5055 }
5056
5057 // This method checks if provided byte array contains byte with highest bit set.
5058 void MacroAssembler::has_negatives(Register ary1, Register len, Register result) {
5059 // Simple and most common case of aligned small array which is not at the
5060 // end of memory page is placed here. All other cases are in stub.
5061 Label LOOP, END, STUB, STUB_LONG, SET_RESULT, DONE;
5062 const uint64_t UPPER_BIT_MASK=0x8080808080808080;
5063 assert_different_registers(ary1, len, result);
5064
5065 cmpw(len, 0);
5066 br(LE, SET_RESULT);
5067 cmpw(len, 4 * wordSize);
5068 br(GE, STUB_LONG); // size > 32 then go to stub
5069
5070 int shift = 64 - exact_log2(os::vm_page_size());
5071 lsl(rscratch1, ary1, shift);
5072 mov(rscratch2, (size_t)(4 * wordSize) << shift);
5073 adds(rscratch2, rscratch1, rscratch2); // At end of page?
5074 br(CS, STUB); // at the end of page then go to stub
5075 subs(len, len, wordSize);
5076 br(LT, END);
5077
5078 BIND(LOOP);
5079 ldr(rscratch1, Address(post(ary1, wordSize)));
5080 tst(rscratch1, UPPER_BIT_MASK);
5081 br(NE, SET_RESULT);
5082 subs(len, len, wordSize);
5083 br(GE, LOOP);
5084 cmpw(len, -wordSize);
5085 br(EQ, SET_RESULT);
5086
5087 BIND(END);
5088 ldr(result, Address(ary1));
5089 sub(len, zr, len, LSL, 3); // LSL 3 is to get bits from bytes
5090 lslv(result, result, len);
5091 tst(result, UPPER_BIT_MASK);
5092 b(SET_RESULT);
5093
5094 BIND(STUB);
5095 RuntimeAddress has_neg = RuntimeAddress(StubRoutines::aarch64::has_negatives());
5096 assert(has_neg.target() != NULL, "has_negatives stub has not been generated");
5097 trampoline_call(has_neg);
5098 b(DONE);
5099
5100 BIND(STUB_LONG);
5101 RuntimeAddress has_neg_long = RuntimeAddress(
5102 StubRoutines::aarch64::has_negatives_long());
5103 assert(has_neg_long.target() != NULL, "has_negatives stub has not been generated");
5104 trampoline_call(has_neg_long);
5105 b(DONE);
5106
5107 BIND(SET_RESULT);
5108 cset(result, NE); // set true or false
5109
5110 BIND(DONE);
5111 }
5112
5113 void MacroAssembler::arrays_equals(Register a1, Register a2, Register tmp3,
5114 Register tmp4, Register tmp5, Register result,
5115 Register cnt1, int elem_size) {
5116 Label DONE, SAME;
5117 Register tmp1 = rscratch1;
5118 Register tmp2 = rscratch2;
5119 Register cnt2 = tmp2; // cnt2 only used in array length compare
5120 int elem_per_word = wordSize/elem_size;
5121 int log_elem_size = exact_log2(elem_size);
5122 int length_offset = arrayOopDesc::length_offset_in_bytes();
5123 int base_offset
5124 = arrayOopDesc::base_offset_in_bytes(elem_size == 2 ? T_CHAR : T_BYTE);
5125 int stubBytesThreshold = 3 * 64 + (UseSIMDForArrayEquals ? 0 : 16);
5126
5127 assert(elem_size == 1 || elem_size == 2, "must be char or byte");
5128 assert_different_registers(a1, a2, result, cnt1, rscratch1, rscratch2);
5129
5130 #ifndef PRODUCT
5131 {
5132 const char kind = (elem_size == 2) ? 'U' : 'L';
5133 char comment[64];
5134 snprintf(comment, sizeof comment, "array_equals%c{", kind);
5135 BLOCK_COMMENT(comment);
5136 }
5137 #endif
5138
5139 // if (a1 == a2)
5140 // return true;
5141 cmpoop(a1, a2); // May have read barriers for a1 and a2.
5142 br(EQ, SAME);
5143
5144 if (UseSimpleArrayEquals) {
5145 Label NEXT_WORD, SHORT, TAIL03, TAIL01, A_MIGHT_BE_NULL, A_IS_NOT_NULL;
5146 // if (a1 == null || a2 == null)
5147 // return false;
5148 // a1 & a2 == 0 means (some-pointer is null) or
5149 // (very-rare-or-even-probably-impossible-pointer-values)
5150 // so, we can save one branch in most cases
5151 tst(a1, a2);
5152 mov(result, false);
5153 br(EQ, A_MIGHT_BE_NULL);
5154 // if (a1.length != a2.length)
5155 // return false;
5156 bind(A_IS_NOT_NULL);
5157 ldrw(cnt1, Address(a1, length_offset));
5158 ldrw(cnt2, Address(a2, length_offset));
5159 eorw(tmp5, cnt1, cnt2);
5160 cbnzw(tmp5, DONE);
5161 lea(a1, Address(a1, base_offset));
5162 lea(a2, Address(a2, base_offset));
5163 // Check for short strings, i.e. smaller than wordSize.
5164 subs(cnt1, cnt1, elem_per_word);
5165 br(Assembler::LT, SHORT);
5166 // Main 8 byte comparison loop.
5167 bind(NEXT_WORD); {
5168 ldr(tmp1, Address(post(a1, wordSize)));
5169 ldr(tmp2, Address(post(a2, wordSize)));
5170 subs(cnt1, cnt1, elem_per_word);
5171 eor(tmp5, tmp1, tmp2);
5172 cbnz(tmp5, DONE);
5173 } br(GT, NEXT_WORD);
5174 // Last longword. In the case where length == 4 we compare the
5175 // same longword twice, but that's still faster than another
5176 // conditional branch.
5177 // cnt1 could be 0, -1, -2, -3, -4 for chars; -4 only happens when
5178 // length == 4.
5179 if (log_elem_size > 0)
5180 lsl(cnt1, cnt1, log_elem_size);
5181 ldr(tmp3, Address(a1, cnt1));
5182 ldr(tmp4, Address(a2, cnt1));
5183 eor(tmp5, tmp3, tmp4);
5184 cbnz(tmp5, DONE);
5185 b(SAME);
5186 bind(A_MIGHT_BE_NULL);
5187 // in case both a1 and a2 are not-null, proceed with loads
5188 cbz(a1, DONE);
5189 cbz(a2, DONE);
5190 b(A_IS_NOT_NULL);
5191 bind(SHORT);
5192
5193 tbz(cnt1, 2 - log_elem_size, TAIL03); // 0-7 bytes left.
5194 {
5195 ldrw(tmp1, Address(post(a1, 4)));
5196 ldrw(tmp2, Address(post(a2, 4)));
5197 eorw(tmp5, tmp1, tmp2);
5198 cbnzw(tmp5, DONE);
5199 }
5200 bind(TAIL03);
5201 tbz(cnt1, 1 - log_elem_size, TAIL01); // 0-3 bytes left.
5202 {
5203 ldrh(tmp3, Address(post(a1, 2)));
5204 ldrh(tmp4, Address(post(a2, 2)));
5205 eorw(tmp5, tmp3, tmp4);
5206 cbnzw(tmp5, DONE);
5207 }
5208 bind(TAIL01);
5209 if (elem_size == 1) { // Only needed when comparing byte arrays.
5210 tbz(cnt1, 0, SAME); // 0-1 bytes left.
5211 {
5212 ldrb(tmp1, a1);
5213 ldrb(tmp2, a2);
5214 eorw(tmp5, tmp1, tmp2);
5215 cbnzw(tmp5, DONE);
5216 }
5217 }
5218 } else {
5219 Label NEXT_DWORD, SHORT, TAIL, TAIL2, STUB, EARLY_OUT,
5220 CSET_EQ, LAST_CHECK;
5221 mov(result, false);
5222 cbz(a1, DONE);
5223 ldrw(cnt1, Address(a1, length_offset));
5224 cbz(a2, DONE);
5225 ldrw(cnt2, Address(a2, length_offset));
5226 // on most CPUs a2 is still "locked"(surprisingly) in ldrw and it's
5227 // faster to perform another branch before comparing a1 and a2
5228 cmp(cnt1, elem_per_word);
5229 br(LE, SHORT); // short or same
5230 ldr(tmp3, Address(pre(a1, base_offset)));
5231 cmp(cnt1, stubBytesThreshold);
5232 br(GE, STUB);
5233 ldr(tmp4, Address(pre(a2, base_offset)));
5234 sub(tmp5, zr, cnt1, LSL, 3 + log_elem_size);
5235 cmp(cnt2, cnt1);
5236 br(NE, DONE);
5237
5238 // Main 16 byte comparison loop with 2 exits
5239 bind(NEXT_DWORD); {
5240 ldr(tmp1, Address(pre(a1, wordSize)));
5241 ldr(tmp2, Address(pre(a2, wordSize)));
5242 subs(cnt1, cnt1, 2 * elem_per_word);
5243 br(LE, TAIL);
5244 eor(tmp4, tmp3, tmp4);
5245 cbnz(tmp4, DONE);
5246 ldr(tmp3, Address(pre(a1, wordSize)));
5247 ldr(tmp4, Address(pre(a2, wordSize)));
5248 cmp(cnt1, elem_per_word);
5249 br(LE, TAIL2);
5250 cmp(tmp1, tmp2);
5251 } br(EQ, NEXT_DWORD);
5252 b(DONE);
5253
5254 bind(TAIL);
5255 eor(tmp4, tmp3, tmp4);
5256 eor(tmp2, tmp1, tmp2);
5257 lslv(tmp2, tmp2, tmp5);
5258 orr(tmp5, tmp4, tmp2);
5259 cmp(tmp5, zr);
5260 b(CSET_EQ);
5261
5262 bind(TAIL2);
5263 eor(tmp2, tmp1, tmp2);
5264 cbnz(tmp2, DONE);
5265 b(LAST_CHECK);
5266
5267 bind(STUB);
5268 ldr(tmp4, Address(pre(a2, base_offset)));
5269 cmp(cnt2, cnt1);
5270 br(NE, DONE);
5271 if (elem_size == 2) { // convert to byte counter
5272 lsl(cnt1, cnt1, 1);
5273 }
5274 eor(tmp5, tmp3, tmp4);
5275 cbnz(tmp5, DONE);
5276 RuntimeAddress stub = RuntimeAddress(StubRoutines::aarch64::large_array_equals());
5277 assert(stub.target() != NULL, "array_equals_long stub has not been generated");
5278 trampoline_call(stub);
5279 b(DONE);
5280
5281 bind(EARLY_OUT);
5282 // (a1 != null && a2 == null) || (a1 != null && a2 != null && a1 == a2)
5283 // so, if a2 == null => return false(0), else return true, so we can return a2
5284 mov(result, a2);
5285 b(DONE);
5286 bind(SHORT);
5287 cmp(cnt2, cnt1);
5288 br(NE, DONE);
5289 cbz(cnt1, SAME);
5290 sub(tmp5, zr, cnt1, LSL, 3 + log_elem_size);
5291 ldr(tmp3, Address(a1, base_offset));
5292 ldr(tmp4, Address(a2, base_offset));
5293 bind(LAST_CHECK);
5294 eor(tmp4, tmp3, tmp4);
5295 lslv(tmp5, tmp4, tmp5);
5296 cmp(tmp5, zr);
5297 bind(CSET_EQ);
5298 cset(result, EQ);
5299 b(DONE);
5300 }
5301
5302 bind(SAME);
5303 mov(result, true);
5304 // That's it.
5305 bind(DONE);
5306
5307 BLOCK_COMMENT("} array_equals");
5308 }
5309
5310 // Compare Strings
5311
5312 // For Strings we're passed the address of the first characters in a1
5313 // and a2 and the length in cnt1.
5314 // elem_size is the element size in bytes: either 1 or 2.
5315 // There are two implementations. For arrays >= 8 bytes, all
5316 // comparisons (including the final one, which may overlap) are
5317 // performed 8 bytes at a time. For strings < 8 bytes, we compare a
5318 // halfword, then a short, and then a byte.
5319
5320 void MacroAssembler::string_equals(Register a1, Register a2,
5321 Register result, Register cnt1, int elem_size)
5322 {
5323 Label SAME, DONE, SHORT, NEXT_WORD;
5324 Register tmp1 = rscratch1;
5325 Register tmp2 = rscratch2;
5326 Register cnt2 = tmp2; // cnt2 only used in array length compare
5327
5328 assert(elem_size == 1 || elem_size == 2, "must be 2 or 1 byte");
5329 assert_different_registers(a1, a2, result, cnt1, rscratch1, rscratch2);
5330
5331 #ifndef PRODUCT
5332 {
5333 const char kind = (elem_size == 2) ? 'U' : 'L';
5334 char comment[64];
5335 snprintf(comment, sizeof comment, "{string_equals%c", kind);
5336 BLOCK_COMMENT(comment);
5337 }
5338 #endif
5339
5340 mov(result, false);
5341
5342 // Check for short strings, i.e. smaller than wordSize.
5343 subs(cnt1, cnt1, wordSize);
5344 br(Assembler::LT, SHORT);
5345 // Main 8 byte comparison loop.
5346 bind(NEXT_WORD); {
5347 ldr(tmp1, Address(post(a1, wordSize)));
5348 ldr(tmp2, Address(post(a2, wordSize)));
5349 subs(cnt1, cnt1, wordSize);
5350 eor(tmp1, tmp1, tmp2);
5351 cbnz(tmp1, DONE);
5352 } br(GT, NEXT_WORD);
5353 // Last longword. In the case where length == 4 we compare the
5354 // same longword twice, but that's still faster than another
5355 // conditional branch.
5356 // cnt1 could be 0, -1, -2, -3, -4 for chars; -4 only happens when
5357 // length == 4.
5358 ldr(tmp1, Address(a1, cnt1));
5359 ldr(tmp2, Address(a2, cnt1));
5360 eor(tmp2, tmp1, tmp2);
5361 cbnz(tmp2, DONE);
5362 b(SAME);
5363
5364 bind(SHORT);
5365 Label TAIL03, TAIL01;
5366
5367 tbz(cnt1, 2, TAIL03); // 0-7 bytes left.
5368 {
5369 ldrw(tmp1, Address(post(a1, 4)));
5370 ldrw(tmp2, Address(post(a2, 4)));
5371 eorw(tmp1, tmp1, tmp2);
5372 cbnzw(tmp1, DONE);
5373 }
5374 bind(TAIL03);
5375 tbz(cnt1, 1, TAIL01); // 0-3 bytes left.
5376 {
5377 ldrh(tmp1, Address(post(a1, 2)));
5378 ldrh(tmp2, Address(post(a2, 2)));
5379 eorw(tmp1, tmp1, tmp2);
5380 cbnzw(tmp1, DONE);
5381 }
5382 bind(TAIL01);
5383 if (elem_size == 1) { // Only needed when comparing 1-byte elements
5384 tbz(cnt1, 0, SAME); // 0-1 bytes left.
5385 {
5386 ldrb(tmp1, a1);
5387 ldrb(tmp2, a2);
5388 eorw(tmp1, tmp1, tmp2);
5389 cbnzw(tmp1, DONE);
5390 }
5391 }
5392 // Arrays are equal.
5393 bind(SAME);
5394 mov(result, true);
5395
5396 // That's it.
5397 bind(DONE);
5398 BLOCK_COMMENT("} string_equals");
5399 }
5400
5401
5402 // The size of the blocks erased by the zero_blocks stub. We must
5403 // handle anything smaller than this ourselves in zero_words().
5404 const int MacroAssembler::zero_words_block_size = 8;
5405
5406 // zero_words() is used by C2 ClearArray patterns. It is as small as
5407 // possible, handling small word counts locally and delegating
5408 // anything larger to the zero_blocks stub. It is expanded many times
5409 // in compiled code, so it is important to keep it short.
5410
5411 // ptr: Address of a buffer to be zeroed.
5412 // cnt: Count in HeapWords.
5413 //
5414 // ptr, cnt, rscratch1, and rscratch2 are clobbered.
5415 void MacroAssembler::zero_words(Register ptr, Register cnt)
5416 {
5417 assert(is_power_of_2(zero_words_block_size), "adjust this");
5418 assert(ptr == r10 && cnt == r11, "mismatch in register usage");
5419
5420 BLOCK_COMMENT("zero_words {");
5421 cmp(cnt, zero_words_block_size);
5422 Label around, done, done16;
5423 br(LO, around);
5424 {
5425 RuntimeAddress zero_blocks = RuntimeAddress(StubRoutines::aarch64::zero_blocks());
5426 assert(zero_blocks.target() != NULL, "zero_blocks stub has not been generated");
5427 if (StubRoutines::aarch64::complete()) {
5428 trampoline_call(zero_blocks);
5429 } else {
5430 bl(zero_blocks);
5431 }
5432 }
5433 bind(around);
5434 for (int i = zero_words_block_size >> 1; i > 1; i >>= 1) {
5435 Label l;
5436 tbz(cnt, exact_log2(i), l);
5437 for (int j = 0; j < i; j += 2) {
5438 stp(zr, zr, post(ptr, 16));
5439 }
5440 bind(l);
5441 }
5442 {
5443 Label l;
5444 tbz(cnt, 0, l);
5445 str(zr, Address(ptr));
5446 bind(l);
5447 }
5448 BLOCK_COMMENT("} zero_words");
5449 }
5450
5451 // base: Address of a buffer to be zeroed, 8 bytes aligned.
5452 // cnt: Immediate count in HeapWords.
5453 #define SmallArraySize (18 * BytesPerLong)
5454 void MacroAssembler::zero_words(Register base, u_int64_t cnt)
5455 {
5456 BLOCK_COMMENT("zero_words {");
5457 int i = cnt & 1; // store any odd word to start
5458 if (i) str(zr, Address(base));
5459
5460 if (cnt <= SmallArraySize / BytesPerLong) {
5461 for (; i < (int)cnt; i += 2)
5462 stp(zr, zr, Address(base, i * wordSize));
5463 } else {
5464 const int unroll = 4; // Number of stp(zr, zr) instructions we'll unroll
5465 int remainder = cnt % (2 * unroll);
5466 for (; i < remainder; i += 2)
5467 stp(zr, zr, Address(base, i * wordSize));
5468
5469 Label loop;
5470 Register cnt_reg = rscratch1;
5471 Register loop_base = rscratch2;
5472 cnt = cnt - remainder;
5473 mov(cnt_reg, cnt);
5474 // adjust base and prebias by -2 * wordSize so we can pre-increment
5475 add(loop_base, base, (remainder - 2) * wordSize);
5476 bind(loop);
5477 sub(cnt_reg, cnt_reg, 2 * unroll);
5478 for (i = 1; i < unroll; i++)
5479 stp(zr, zr, Address(loop_base, 2 * i * wordSize));
5480 stp(zr, zr, Address(pre(loop_base, 2 * unroll * wordSize)));
5481 cbnz(cnt_reg, loop);
5482 }
5483 BLOCK_COMMENT("} zero_words");
5484 }
5485
5486 // Zero blocks of memory by using DC ZVA.
5487 //
5488 // Aligns the base address first sufficently for DC ZVA, then uses
5489 // DC ZVA repeatedly for every full block. cnt is the size to be
5490 // zeroed in HeapWords. Returns the count of words left to be zeroed
5491 // in cnt.
5492 //
5493 // NOTE: This is intended to be used in the zero_blocks() stub. If
5494 // you want to use it elsewhere, note that cnt must be >= 2*zva_length.
5495 void MacroAssembler::zero_dcache_blocks(Register base, Register cnt) {
5496 Register tmp = rscratch1;
5497 Register tmp2 = rscratch2;
5498 int zva_length = VM_Version::zva_length();
5499 Label initial_table_end, loop_zva;
5500 Label fini;
5501
5502 // Base must be 16 byte aligned. If not just return and let caller handle it
5503 tst(base, 0x0f);
5504 br(Assembler::NE, fini);
5505 // Align base with ZVA length.
5506 neg(tmp, base);
5507 andr(tmp, tmp, zva_length - 1);
5508
5509 // tmp: the number of bytes to be filled to align the base with ZVA length.
5510 add(base, base, tmp);
5511 sub(cnt, cnt, tmp, Assembler::ASR, 3);
5512 adr(tmp2, initial_table_end);
5513 sub(tmp2, tmp2, tmp, Assembler::LSR, 2);
5514 br(tmp2);
5515
5516 for (int i = -zva_length + 16; i < 0; i += 16)
5517 stp(zr, zr, Address(base, i));
5518 bind(initial_table_end);
5519
5520 sub(cnt, cnt, zva_length >> 3);
5521 bind(loop_zva);
5522 dc(Assembler::ZVA, base);
5523 subs(cnt, cnt, zva_length >> 3);
5524 add(base, base, zva_length);
5525 br(Assembler::GE, loop_zva);
5526 add(cnt, cnt, zva_length >> 3); // count not zeroed by DC ZVA
5527 bind(fini);
5528 }
5529
5530 // base: Address of a buffer to be filled, 8 bytes aligned.
5531 // cnt: Count in 8-byte unit.
5532 // value: Value to be filled with.
5533 // base will point to the end of the buffer after filling.
5534 void MacroAssembler::fill_words(Register base, Register cnt, Register value)
5535 {
5536 // Algorithm:
5537 //
5538 // scratch1 = cnt & 7;
5539 // cnt -= scratch1;
5540 // p += scratch1;
5541 // switch (scratch1) {
5542 // do {
5543 // cnt -= 8;
5544 // p[-8] = v;
5545 // case 7:
5546 // p[-7] = v;
5547 // case 6:
5548 // p[-6] = v;
5549 // // ...
5550 // case 1:
5551 // p[-1] = v;
5552 // case 0:
5553 // p += 8;
5554 // } while (cnt);
5555 // }
5556
5557 assert_different_registers(base, cnt, value, rscratch1, rscratch2);
5558
5559 Label fini, skip, entry, loop;
5560 const int unroll = 8; // Number of stp instructions we'll unroll
5561
5562 cbz(cnt, fini);
5563 tbz(base, 3, skip);
5564 str(value, Address(post(base, 8)));
5565 sub(cnt, cnt, 1);
5566 bind(skip);
5567
5568 andr(rscratch1, cnt, (unroll-1) * 2);
5569 sub(cnt, cnt, rscratch1);
5570 add(base, base, rscratch1, Assembler::LSL, 3);
5571 adr(rscratch2, entry);
5572 sub(rscratch2, rscratch2, rscratch1, Assembler::LSL, 1);
5573 br(rscratch2);
5574
5575 bind(loop);
5576 add(base, base, unroll * 16);
5577 for (int i = -unroll; i < 0; i++)
5578 stp(value, value, Address(base, i * 16));
5579 bind(entry);
5580 subs(cnt, cnt, unroll * 2);
5581 br(Assembler::GE, loop);
5582
5583 tbz(cnt, 0, fini);
5584 str(value, Address(post(base, 8)));
5585 bind(fini);
5586 }
5587
5588 // Intrinsic for sun/nio/cs/ISO_8859_1$Encoder.implEncodeISOArray and
5589 // java/lang/StringUTF16.compress.
5590 void MacroAssembler::encode_iso_array(Register src, Register dst,
5591 Register len, Register result,
5592 FloatRegister Vtmp1, FloatRegister Vtmp2,
5593 FloatRegister Vtmp3, FloatRegister Vtmp4)
5594 {
5595 Label DONE, SET_RESULT, NEXT_32, NEXT_32_PRFM, LOOP_8, NEXT_8, LOOP_1, NEXT_1,
5596 NEXT_32_START, NEXT_32_PRFM_START;
5597 Register tmp1 = rscratch1, tmp2 = rscratch2;
5598
5599 mov(result, len); // Save initial len
5600
5601 #ifndef BUILTIN_SIM
5602 cmp(len, 8); // handle shortest strings first
5603 br(LT, LOOP_1);
5604 cmp(len, 32);
5605 br(LT, NEXT_8);
5606 // The following code uses the SIMD 'uzp1' and 'uzp2' instructions
5607 // to convert chars to bytes
5608 if (SoftwarePrefetchHintDistance >= 0) {
5609 ld1(Vtmp1, Vtmp2, Vtmp3, Vtmp4, T8H, src);
5610 cmp(len, SoftwarePrefetchHintDistance/2 + 16);
5611 br(LE, NEXT_32_START);
5612 b(NEXT_32_PRFM_START);
5613 BIND(NEXT_32_PRFM);
5614 ld1(Vtmp1, Vtmp2, Vtmp3, Vtmp4, T8H, src);
5615 BIND(NEXT_32_PRFM_START);
5616 prfm(Address(src, SoftwarePrefetchHintDistance));
5617 orr(v4, T16B, Vtmp1, Vtmp2);
5618 orr(v5, T16B, Vtmp3, Vtmp4);
5619 uzp1(Vtmp1, T16B, Vtmp1, Vtmp2);
5620 uzp1(Vtmp3, T16B, Vtmp3, Vtmp4);
5621 stpq(Vtmp1, Vtmp3, dst);
5622 uzp2(v5, T16B, v4, v5); // high bytes
5623 umov(tmp2, v5, D, 1);
5624 fmovd(tmp1, v5);
5625 orr(tmp1, tmp1, tmp2);
5626 cbnz(tmp1, LOOP_8);
5627 sub(len, len, 32);
5628 add(dst, dst, 32);
5629 add(src, src, 64);
5630 cmp(len, SoftwarePrefetchHintDistance/2 + 16);
5631 br(GE, NEXT_32_PRFM);
5632 cmp(len, 32);
5633 br(LT, LOOP_8);
5634 BIND(NEXT_32);
5635 ld1(Vtmp1, Vtmp2, Vtmp3, Vtmp4, T8H, src);
5636 BIND(NEXT_32_START);
5637 } else {
5638 BIND(NEXT_32);
5639 ld1(Vtmp1, Vtmp2, Vtmp3, Vtmp4, T8H, src);
5640 }
5641 prfm(Address(src, SoftwarePrefetchHintDistance));
5642 uzp1(v4, T16B, Vtmp1, Vtmp2);
5643 uzp1(v5, T16B, Vtmp3, Vtmp4);
5644 stpq(v4, v5, dst);
5645 orr(Vtmp1, T16B, Vtmp1, Vtmp2);
5646 orr(Vtmp3, T16B, Vtmp3, Vtmp4);
5647 uzp2(Vtmp1, T16B, Vtmp1, Vtmp3); // high bytes
5648 umov(tmp2, Vtmp1, D, 1);
5649 fmovd(tmp1, Vtmp1);
5650 orr(tmp1, tmp1, tmp2);
5651 cbnz(tmp1, LOOP_8);
5652 sub(len, len, 32);
5653 add(dst, dst, 32);
5654 add(src, src, 64);
5655 cmp(len, 32);
5656 br(GE, NEXT_32);
5657 cbz(len, DONE);
5658
5659 BIND(LOOP_8);
5660 cmp(len, 8);
5661 br(LT, LOOP_1);
5662 BIND(NEXT_8);
5663 ld1(Vtmp1, T8H, src);
5664 uzp1(Vtmp2, T16B, Vtmp1, Vtmp1); // low bytes
5665 uzp2(Vtmp3, T16B, Vtmp1, Vtmp1); // high bytes
5666 strd(Vtmp2, dst);
5667 fmovd(tmp1, Vtmp3);
5668 cbnz(tmp1, NEXT_1);
5669
5670 sub(len, len, 8);
5671 add(dst, dst, 8);
5672 add(src, src, 16);
5673 cmp(len, 8);
5674 br(GE, NEXT_8);
5675
5676 BIND(LOOP_1);
5677 #endif
5678 cbz(len, DONE);
5679 BIND(NEXT_1);
5680 ldrh(tmp1, Address(post(src, 2)));
5681 strb(tmp1, Address(post(dst, 1)));
5682 tst(tmp1, 0xff00);
5683 br(NE, SET_RESULT);
5684 subs(len, len, 1);
5685 br(GT, NEXT_1);
5686
5687 BIND(SET_RESULT);
5688 sub(result, result, len); // Return index where we stopped
5689 // Return len == 0 if we processed all
5690 // characters
5691 BIND(DONE);
5692 }
5693
5694
5695 // Inflate byte[] array to char[].
5696 void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len,
5697 FloatRegister vtmp1, FloatRegister vtmp2, FloatRegister vtmp3,
5698 Register tmp4) {
5699 Label big, done, after_init, to_stub;
5700
5701 assert_different_registers(src, dst, len, tmp4, rscratch1);
5702
5703 fmovd(vtmp1, zr);
5704 lsrw(tmp4, len, 3);
5705 bind(after_init);
5706 cbnzw(tmp4, big);
5707 // Short string: less than 8 bytes.
5708 {
5709 Label loop, tiny;
5710
5711 cmpw(len, 4);
5712 br(LT, tiny);
5713 // Use SIMD to do 4 bytes.
5714 ldrs(vtmp2, post(src, 4));
5715 zip1(vtmp3, T8B, vtmp2, vtmp1);
5716 subw(len, len, 4);
5717 strd(vtmp3, post(dst, 8));
5718
5719 cbzw(len, done);
5720
5721 // Do the remaining bytes by steam.
5722 bind(loop);
5723 ldrb(tmp4, post(src, 1));
5724 strh(tmp4, post(dst, 2));
5725 subw(len, len, 1);
5726
5727 bind(tiny);
5728 cbnz(len, loop);
5729
5730 b(done);
5731 }
5732
5733 if (SoftwarePrefetchHintDistance >= 0) {
5734 bind(to_stub);
5735 RuntimeAddress stub = RuntimeAddress(StubRoutines::aarch64::large_byte_array_inflate());
5736 assert(stub.target() != NULL, "large_byte_array_inflate stub has not been generated");
5737 trampoline_call(stub);
5738 b(after_init);
5739 }
5740
5741 // Unpack the bytes 8 at a time.
5742 bind(big);
5743 {
5744 Label loop, around, loop_last, loop_start;
5745
5746 if (SoftwarePrefetchHintDistance >= 0) {
5747 const int large_loop_threshold = (64 + 16)/8;
5748 ldrd(vtmp2, post(src, 8));
5749 andw(len, len, 7);
5750 cmp(tmp4, large_loop_threshold);
5751 br(GE, to_stub);
5752 b(loop_start);
5753
5754 bind(loop);
5755 ldrd(vtmp2, post(src, 8));
5756 bind(loop_start);
5757 subs(tmp4, tmp4, 1);
5758 br(EQ, loop_last);
5759 zip1(vtmp2, T16B, vtmp2, vtmp1);
5760 ldrd(vtmp3, post(src, 8));
5761 st1(vtmp2, T8H, post(dst, 16));
5762 subs(tmp4, tmp4, 1);
5763 zip1(vtmp3, T16B, vtmp3, vtmp1);
5764 st1(vtmp3, T8H, post(dst, 16));
5765 br(NE, loop);
5766 b(around);
5767 bind(loop_last);
5768 zip1(vtmp2, T16B, vtmp2, vtmp1);
5769 st1(vtmp2, T8H, post(dst, 16));
5770 bind(around);
5771 cbz(len, done);
5772 } else {
5773 andw(len, len, 7);
5774 bind(loop);
5775 ldrd(vtmp2, post(src, 8));
5776 sub(tmp4, tmp4, 1);
5777 zip1(vtmp3, T16B, vtmp2, vtmp1);
5778 st1(vtmp3, T8H, post(dst, 16));
5779 cbnz(tmp4, loop);
5780 }
5781 }
5782
5783 // Do the tail of up to 8 bytes.
5784 add(src, src, len);
5785 ldrd(vtmp3, Address(src, -8));
5786 add(dst, dst, len, ext::uxtw, 1);
5787 zip1(vtmp3, T16B, vtmp3, vtmp1);
5788 strq(vtmp3, Address(dst, -16));
5789
5790 bind(done);
5791 }
5792
5793 // Compress char[] array to byte[].
5794 void MacroAssembler::char_array_compress(Register src, Register dst, Register len,
5795 FloatRegister tmp1Reg, FloatRegister tmp2Reg,
5796 FloatRegister tmp3Reg, FloatRegister tmp4Reg,
5797 Register result) {
5798 encode_iso_array(src, dst, len, result,
5799 tmp1Reg, tmp2Reg, tmp3Reg, tmp4Reg);
5800 cmp(len, zr);
5801 csel(result, result, zr, EQ);
5802 }
5803
5804 // get_thread() can be called anywhere inside generated code so we
5805 // need to save whatever non-callee save context might get clobbered
5806 // by the call to JavaThread::aarch64_get_thread_helper() or, indeed,
5807 // the call setup code.
5808 //
5809 // aarch64_get_thread_helper() clobbers only r0, r1, and flags.
5810 //
5811 void MacroAssembler::get_thread(Register dst) {
5812 RegSet saved_regs = RegSet::range(r0, r1) + lr - dst;
5813 push(saved_regs, sp);
5814
5815 mov(lr, CAST_FROM_FN_PTR(address, JavaThread::aarch64_get_thread_helper));
5816 blrt(lr, 1, 0, 1);
5817 if (dst != c_rarg0) {
5818 mov(dst, c_rarg0);
5819 }
5820
5821 pop(saved_regs, sp);
5822 }