1 /*
2 * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, 2019, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include <sys/types.h>
27
28 #include "precompiled.hpp"
29 #include "jvm.h"
30 #include "asm/assembler.hpp"
31 #include "asm/assembler.inline.hpp"
32 #include "gc/shared/barrierSet.hpp"
33 #include "gc/shared/cardTable.hpp"
34 #include "gc/shared/barrierSetAssembler.hpp"
35 #include "gc/shared/cardTableBarrierSet.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "compiler/disassembler.hpp"
38 #include "memory/resourceArea.hpp"
39 #include "memory/universe.hpp"
40 #include "nativeInst_aarch64.hpp"
41 #include "oops/accessDecorators.hpp"
42 #include "oops/compressedOops.inline.hpp"
43 #include "oops/klass.inline.hpp"
44 #include "runtime/biasedLocking.hpp"
45 #include "runtime/icache.hpp"
46 #include "runtime/interfaceSupport.inline.hpp"
47 #include "runtime/jniHandles.inline.hpp"
48 #include "runtime/sharedRuntime.hpp"
49 #include "runtime/thread.hpp"
50 #ifdef COMPILER1
51 #include "c1/c1_LIRAssembler.hpp"
52 #endif
53 #ifdef COMPILER2
54 #include "oops/oop.hpp"
55 #include "opto/compile.hpp"
56 #include "opto/intrinsicnode.hpp"
57 #include "opto/node.hpp"
58 #endif
59
60 #ifdef PRODUCT
61 #define BLOCK_COMMENT(str) /* nothing */
62 #define STOP(error) stop(error)
63 #else
64 #define BLOCK_COMMENT(str) block_comment(str)
65 #define STOP(error) block_comment(error); stop(error)
66 #endif
67
68 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
69
70 // Patch any kind of instruction; there may be several instructions.
71 // Return the total length (in bytes) of the instructions.
72 int MacroAssembler::pd_patch_instruction_size(address branch, address target) {
73 int instructions = 1;
74 assert((uint64_t)target < (1ul << 48), "48-bit overflow in address constant");
75 long offset = (target - branch) >> 2;
76 unsigned insn = *(unsigned*)branch;
77 if ((Instruction_aarch64::extract(insn, 29, 24) & 0b111011) == 0b011000) {
78 // Load register (literal)
79 Instruction_aarch64::spatch(branch, 23, 5, offset);
80 } else if (Instruction_aarch64::extract(insn, 30, 26) == 0b00101) {
81 // Unconditional branch (immediate)
82 Instruction_aarch64::spatch(branch, 25, 0, offset);
83 } else if (Instruction_aarch64::extract(insn, 31, 25) == 0b0101010) {
84 // Conditional branch (immediate)
85 Instruction_aarch64::spatch(branch, 23, 5, offset);
86 } else if (Instruction_aarch64::extract(insn, 30, 25) == 0b011010) {
87 // Compare & branch (immediate)
88 Instruction_aarch64::spatch(branch, 23, 5, offset);
89 } else if (Instruction_aarch64::extract(insn, 30, 25) == 0b011011) {
90 // Test & branch (immediate)
91 Instruction_aarch64::spatch(branch, 18, 5, offset);
92 } else if (Instruction_aarch64::extract(insn, 28, 24) == 0b10000) {
93 // PC-rel. addressing
94 offset = target-branch;
95 int shift = Instruction_aarch64::extract(insn, 31, 31);
96 if (shift) {
97 u_int64_t dest = (u_int64_t)target;
98 uint64_t pc_page = (uint64_t)branch >> 12;
99 uint64_t adr_page = (uint64_t)target >> 12;
100 unsigned offset_lo = dest & 0xfff;
101 offset = adr_page - pc_page;
102
103 // We handle 4 types of PC relative addressing
104 // 1 - adrp Rx, target_page
105 // ldr/str Ry, [Rx, #offset_in_page]
106 // 2 - adrp Rx, target_page
107 // add Ry, Rx, #offset_in_page
108 // 3 - adrp Rx, target_page (page aligned reloc, offset == 0)
109 // movk Rx, #imm16<<32
110 // 4 - adrp Rx, target_page (page aligned reloc, offset == 0)
111 // In the first 3 cases we must check that Rx is the same in the adrp and the
112 // subsequent ldr/str, add or movk instruction. Otherwise we could accidentally end
113 // up treating a type 4 relocation as a type 1, 2 or 3 just because it happened
114 // to be followed by a random unrelated ldr/str, add or movk instruction.
115 //
116 unsigned insn2 = ((unsigned*)branch)[1];
117 if (Instruction_aarch64::extract(insn2, 29, 24) == 0b111001 &&
118 Instruction_aarch64::extract(insn, 4, 0) ==
119 Instruction_aarch64::extract(insn2, 9, 5)) {
120 // Load/store register (unsigned immediate)
121 unsigned size = Instruction_aarch64::extract(insn2, 31, 30);
122 Instruction_aarch64::patch(branch + sizeof (unsigned),
123 21, 10, offset_lo >> size);
124 guarantee(((dest >> size) << size) == dest, "misaligned target");
125 instructions = 2;
126 } else if (Instruction_aarch64::extract(insn2, 31, 22) == 0b1001000100 &&
127 Instruction_aarch64::extract(insn, 4, 0) ==
128 Instruction_aarch64::extract(insn2, 4, 0)) {
129 // add (immediate)
130 Instruction_aarch64::patch(branch + sizeof (unsigned),
131 21, 10, offset_lo);
132 instructions = 2;
133 } else if (Instruction_aarch64::extract(insn2, 31, 21) == 0b11110010110 &&
134 Instruction_aarch64::extract(insn, 4, 0) ==
135 Instruction_aarch64::extract(insn2, 4, 0)) {
136 // movk #imm16<<32
137 Instruction_aarch64::patch(branch + 4, 20, 5, (uint64_t)target >> 32);
138 long dest = ((long)target & 0xffffffffL) | ((long)branch & 0xffff00000000L);
139 long pc_page = (long)branch >> 12;
140 long adr_page = (long)dest >> 12;
141 offset = adr_page - pc_page;
142 instructions = 2;
143 }
144 }
145 int offset_lo = offset & 3;
146 offset >>= 2;
147 Instruction_aarch64::spatch(branch, 23, 5, offset);
148 Instruction_aarch64::patch(branch, 30, 29, offset_lo);
149 } else if (Instruction_aarch64::extract(insn, 31, 21) == 0b11010010100) {
150 u_int64_t dest = (u_int64_t)target;
151 // Move wide constant
152 assert(nativeInstruction_at(branch+4)->is_movk(), "wrong insns in patch");
153 assert(nativeInstruction_at(branch+8)->is_movk(), "wrong insns in patch");
154 Instruction_aarch64::patch(branch, 20, 5, dest & 0xffff);
155 Instruction_aarch64::patch(branch+4, 20, 5, (dest >>= 16) & 0xffff);
156 Instruction_aarch64::patch(branch+8, 20, 5, (dest >>= 16) & 0xffff);
157 assert(target_addr_for_insn(branch) == target, "should be");
158 instructions = 3;
159 } else if (Instruction_aarch64::extract(insn, 31, 22) == 0b1011100101 &&
160 Instruction_aarch64::extract(insn, 4, 0) == 0b11111) {
161 // nothing to do
162 assert(target == 0, "did not expect to relocate target for polling page load");
163 } else {
164 ShouldNotReachHere();
165 }
166 return instructions * NativeInstruction::instruction_size;
167 }
168
169 int MacroAssembler::patch_oop(address insn_addr, address o) {
170 int instructions;
171 unsigned insn = *(unsigned*)insn_addr;
172 assert(nativeInstruction_at(insn_addr+4)->is_movk(), "wrong insns in patch");
173
174 // OOPs are either narrow (32 bits) or wide (48 bits). We encode
175 // narrow OOPs by setting the upper 16 bits in the first
176 // instruction.
177 if (Instruction_aarch64::extract(insn, 31, 21) == 0b11010010101) {
178 // Move narrow OOP
179 narrowOop n = CompressedOops::encode((oop)o);
180 Instruction_aarch64::patch(insn_addr, 20, 5, n >> 16);
181 Instruction_aarch64::patch(insn_addr+4, 20, 5, n & 0xffff);
182 instructions = 2;
183 } else {
184 // Move wide OOP
185 assert(nativeInstruction_at(insn_addr+8)->is_movk(), "wrong insns in patch");
186 uintptr_t dest = (uintptr_t)o;
187 Instruction_aarch64::patch(insn_addr, 20, 5, dest & 0xffff);
188 Instruction_aarch64::patch(insn_addr+4, 20, 5, (dest >>= 16) & 0xffff);
189 Instruction_aarch64::patch(insn_addr+8, 20, 5, (dest >>= 16) & 0xffff);
190 instructions = 3;
191 }
192 return instructions * NativeInstruction::instruction_size;
193 }
194
195 int MacroAssembler::patch_narrow_klass(address insn_addr, narrowKlass n) {
196 // Metatdata pointers are either narrow (32 bits) or wide (48 bits).
197 // We encode narrow ones by setting the upper 16 bits in the first
198 // instruction.
199 NativeInstruction *insn = nativeInstruction_at(insn_addr);
200 assert(Instruction_aarch64::extract(insn->encoding(), 31, 21) == 0b11010010101 &&
201 nativeInstruction_at(insn_addr+4)->is_movk(), "wrong insns in patch");
202
203 Instruction_aarch64::patch(insn_addr, 20, 5, n >> 16);
204 Instruction_aarch64::patch(insn_addr+4, 20, 5, n & 0xffff);
205 return 2 * NativeInstruction::instruction_size;
206 }
207
208 address MacroAssembler::target_addr_for_insn(address insn_addr, unsigned insn) {
209 long offset = 0;
210 if ((Instruction_aarch64::extract(insn, 29, 24) & 0b011011) == 0b00011000) {
211 // Load register (literal)
212 offset = Instruction_aarch64::sextract(insn, 23, 5);
213 return address(((uint64_t)insn_addr + (offset << 2)));
214 } else if (Instruction_aarch64::extract(insn, 30, 26) == 0b00101) {
215 // Unconditional branch (immediate)
216 offset = Instruction_aarch64::sextract(insn, 25, 0);
217 } else if (Instruction_aarch64::extract(insn, 31, 25) == 0b0101010) {
218 // Conditional branch (immediate)
219 offset = Instruction_aarch64::sextract(insn, 23, 5);
220 } else if (Instruction_aarch64::extract(insn, 30, 25) == 0b011010) {
221 // Compare & branch (immediate)
222 offset = Instruction_aarch64::sextract(insn, 23, 5);
223 } else if (Instruction_aarch64::extract(insn, 30, 25) == 0b011011) {
224 // Test & branch (immediate)
225 offset = Instruction_aarch64::sextract(insn, 18, 5);
226 } else if (Instruction_aarch64::extract(insn, 28, 24) == 0b10000) {
227 // PC-rel. addressing
228 offset = Instruction_aarch64::extract(insn, 30, 29);
229 offset |= Instruction_aarch64::sextract(insn, 23, 5) << 2;
230 int shift = Instruction_aarch64::extract(insn, 31, 31) ? 12 : 0;
231 if (shift) {
232 offset <<= shift;
233 uint64_t target_page = ((uint64_t)insn_addr) + offset;
234 target_page &= ((uint64_t)-1) << shift;
235 // Return the target address for the following sequences
236 // 1 - adrp Rx, target_page
237 // ldr/str Ry, [Rx, #offset_in_page]
238 // 2 - adrp Rx, target_page
239 // add Ry, Rx, #offset_in_page
240 // 3 - adrp Rx, target_page (page aligned reloc, offset == 0)
241 // movk Rx, #imm12<<32
242 // 4 - adrp Rx, target_page (page aligned reloc, offset == 0)
243 //
244 // In the first two cases we check that the register is the same and
245 // return the target_page + the offset within the page.
246 // Otherwise we assume it is a page aligned relocation and return
247 // the target page only.
248 //
249 unsigned insn2 = ((unsigned*)insn_addr)[1];
250 if (Instruction_aarch64::extract(insn2, 29, 24) == 0b111001 &&
251 Instruction_aarch64::extract(insn, 4, 0) ==
252 Instruction_aarch64::extract(insn2, 9, 5)) {
253 // Load/store register (unsigned immediate)
254 unsigned int byte_offset = Instruction_aarch64::extract(insn2, 21, 10);
255 unsigned int size = Instruction_aarch64::extract(insn2, 31, 30);
256 return address(target_page + (byte_offset << size));
257 } else if (Instruction_aarch64::extract(insn2, 31, 22) == 0b1001000100 &&
258 Instruction_aarch64::extract(insn, 4, 0) ==
259 Instruction_aarch64::extract(insn2, 4, 0)) {
260 // add (immediate)
261 unsigned int byte_offset = Instruction_aarch64::extract(insn2, 21, 10);
262 return address(target_page + byte_offset);
263 } else {
264 if (Instruction_aarch64::extract(insn2, 31, 21) == 0b11110010110 &&
265 Instruction_aarch64::extract(insn, 4, 0) ==
266 Instruction_aarch64::extract(insn2, 4, 0)) {
267 target_page = (target_page & 0xffffffff) |
268 ((uint64_t)Instruction_aarch64::extract(insn2, 20, 5) << 32);
269 }
270 return (address)target_page;
271 }
272 } else {
273 ShouldNotReachHere();
274 }
275 } else if (Instruction_aarch64::extract(insn, 31, 23) == 0b110100101) {
276 u_int32_t *insns = (u_int32_t *)insn_addr;
277 // Move wide constant: movz, movk, movk. See movptr().
278 assert(nativeInstruction_at(insns+1)->is_movk(), "wrong insns in patch");
279 assert(nativeInstruction_at(insns+2)->is_movk(), "wrong insns in patch");
280 return address(u_int64_t(Instruction_aarch64::extract(insns[0], 20, 5))
281 + (u_int64_t(Instruction_aarch64::extract(insns[1], 20, 5)) << 16)
282 + (u_int64_t(Instruction_aarch64::extract(insns[2], 20, 5)) << 32));
283 } else if (Instruction_aarch64::extract(insn, 31, 22) == 0b1011100101 &&
284 Instruction_aarch64::extract(insn, 4, 0) == 0b11111) {
285 return 0;
286 } else {
287 ShouldNotReachHere();
288 }
289 return address(((uint64_t)insn_addr + (offset << 2)));
290 }
291
292 void MacroAssembler::safepoint_poll(Label& slow_path) {
293 if (SafepointMechanism::uses_thread_local_poll()) {
294 ldr(rscratch1, Address(rthread, Thread::polling_page_offset()));
295 tbnz(rscratch1, exact_log2(SafepointMechanism::poll_bit()), slow_path);
296 } else {
297 unsigned long offset;
298 adrp(rscratch1, ExternalAddress(SafepointSynchronize::address_of_state()), offset);
299 ldrw(rscratch1, Address(rscratch1, offset));
300 assert(SafepointSynchronize::_not_synchronized == 0, "rewrite this code");
301 cbnz(rscratch1, slow_path);
302 }
303 }
304
305 // Just like safepoint_poll, but use an acquiring load for thread-
306 // local polling.
307 //
308 // We need an acquire here to ensure that any subsequent load of the
309 // global SafepointSynchronize::_state flag is ordered after this load
310 // of the local Thread::_polling page. We don't want this poll to
311 // return false (i.e. not safepointing) and a later poll of the global
312 // SafepointSynchronize::_state spuriously to return true.
313 //
314 // This is to avoid a race when we're in a native->Java transition
315 // racing the code which wakes up from a safepoint.
316 //
317 void MacroAssembler::safepoint_poll_acquire(Label& slow_path) {
318 if (SafepointMechanism::uses_thread_local_poll()) {
319 lea(rscratch1, Address(rthread, Thread::polling_page_offset()));
320 ldar(rscratch1, rscratch1);
321 tbnz(rscratch1, exact_log2(SafepointMechanism::poll_bit()), slow_path);
322 } else {
323 safepoint_poll(slow_path);
324 }
325 }
326
327 void MacroAssembler::reset_last_Java_frame(bool clear_fp) {
328 // we must set sp to zero to clear frame
329 str(zr, Address(rthread, JavaThread::last_Java_sp_offset()));
330
331 // must clear fp, so that compiled frames are not confused; it is
332 // possible that we need it only for debugging
333 if (clear_fp) {
334 str(zr, Address(rthread, JavaThread::last_Java_fp_offset()));
335 }
336
337 // Always clear the pc because it could have been set by make_walkable()
338 str(zr, Address(rthread, JavaThread::last_Java_pc_offset()));
339 }
340
341 // Calls to C land
342 //
343 // When entering C land, the rfp, & resp of the last Java frame have to be recorded
344 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp
345 // has to be reset to 0. This is required to allow proper stack traversal.
346 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
347 Register last_java_fp,
348 Register last_java_pc,
349 Register scratch) {
350
351 if (last_java_pc->is_valid()) {
352 str(last_java_pc, Address(rthread,
353 JavaThread::frame_anchor_offset()
354 + JavaFrameAnchor::last_Java_pc_offset()));
355 }
356
357 // determine last_java_sp register
358 if (last_java_sp == sp) {
359 mov(scratch, sp);
360 last_java_sp = scratch;
361 } else if (!last_java_sp->is_valid()) {
362 last_java_sp = esp;
363 }
364
365 str(last_java_sp, Address(rthread, JavaThread::last_Java_sp_offset()));
366
367 // last_java_fp is optional
368 if (last_java_fp->is_valid()) {
369 str(last_java_fp, Address(rthread, JavaThread::last_Java_fp_offset()));
370 }
371 }
372
373 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
374 Register last_java_fp,
375 address last_java_pc,
376 Register scratch) {
377 assert(last_java_pc != NULL, "must provide a valid PC");
378
379 adr(scratch, last_java_pc);
380 str(scratch, Address(rthread,
381 JavaThread::frame_anchor_offset()
382 + JavaFrameAnchor::last_Java_pc_offset()));
383
384 set_last_Java_frame(last_java_sp, last_java_fp, noreg, scratch);
385 }
386
387 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
388 Register last_java_fp,
389 Label &L,
390 Register scratch) {
391 if (L.is_bound()) {
392 set_last_Java_frame(last_java_sp, last_java_fp, target(L), scratch);
393 } else {
394 InstructionMark im(this);
395 L.add_patch_at(code(), locator());
396 set_last_Java_frame(last_java_sp, last_java_fp, pc() /* Patched later */, scratch);
397 }
398 }
399
400 void MacroAssembler::far_call(Address entry, CodeBuffer *cbuf, Register tmp) {
401 assert(ReservedCodeCacheSize < 4*G, "branch out of range");
402 assert(CodeCache::find_blob(entry.target()) != NULL,
403 "destination of far call not found in code cache");
404 if (far_branches()) {
405 unsigned long offset;
406 // We can use ADRP here because we know that the total size of
407 // the code cache cannot exceed 2Gb.
408 adrp(tmp, entry, offset);
409 add(tmp, tmp, offset);
410 if (cbuf) cbuf->set_insts_mark();
411 blr(tmp);
412 } else {
413 if (cbuf) cbuf->set_insts_mark();
414 bl(entry);
415 }
416 }
417
418 void MacroAssembler::far_jump(Address entry, CodeBuffer *cbuf, Register tmp) {
419 assert(ReservedCodeCacheSize < 4*G, "branch out of range");
420 assert(CodeCache::find_blob(entry.target()) != NULL,
421 "destination of far call not found in code cache");
422 if (far_branches()) {
423 unsigned long offset;
424 // We can use ADRP here because we know that the total size of
425 // the code cache cannot exceed 2Gb.
426 adrp(tmp, entry, offset);
427 add(tmp, tmp, offset);
428 if (cbuf) cbuf->set_insts_mark();
429 br(tmp);
430 } else {
431 if (cbuf) cbuf->set_insts_mark();
432 b(entry);
433 }
434 }
435
436 void MacroAssembler::reserved_stack_check() {
437 // testing if reserved zone needs to be enabled
438 Label no_reserved_zone_enabling;
439
440 ldr(rscratch1, Address(rthread, JavaThread::reserved_stack_activation_offset()));
441 cmp(sp, rscratch1);
442 br(Assembler::LO, no_reserved_zone_enabling);
443
444 enter(); // LR and FP are live.
445 lea(rscratch1, CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone));
446 mov(c_rarg0, rthread);
447 blr(rscratch1);
448 leave();
449
450 // We have already removed our own frame.
451 // throw_delayed_StackOverflowError will think that it's been
452 // called by our caller.
453 lea(rscratch1, RuntimeAddress(StubRoutines::throw_delayed_StackOverflowError_entry()));
454 br(rscratch1);
455 should_not_reach_here();
456
457 bind(no_reserved_zone_enabling);
458 }
459
460 int MacroAssembler::biased_locking_enter(Register lock_reg,
461 Register obj_reg,
462 Register swap_reg,
463 Register tmp_reg,
464 bool swap_reg_contains_mark,
465 Label& done,
466 Label* slow_case,
467 BiasedLockingCounters* counters) {
468 assert(UseBiasedLocking, "why call this otherwise?");
469 assert_different_registers(lock_reg, obj_reg, swap_reg);
470
471 if (PrintBiasedLockingStatistics && counters == NULL)
472 counters = BiasedLocking::counters();
473
474 assert_different_registers(lock_reg, obj_reg, swap_reg, tmp_reg, rscratch1, rscratch2, noreg);
475 assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits, "biased locking makes assumptions about bit layout");
476 Address mark_addr (obj_reg, oopDesc::mark_offset_in_bytes());
477 Address klass_addr (obj_reg, oopDesc::klass_offset_in_bytes());
478 Address saved_mark_addr(lock_reg, 0);
479
480 // Biased locking
481 // See whether the lock is currently biased toward our thread and
482 // whether the epoch is still valid
483 // Note that the runtime guarantees sufficient alignment of JavaThread
484 // pointers to allow age to be placed into low bits
485 // First check to see whether biasing is even enabled for this object
486 Label cas_label;
487 int null_check_offset = -1;
488 if (!swap_reg_contains_mark) {
489 null_check_offset = offset();
490 ldr(swap_reg, mark_addr);
491 }
492 andr(tmp_reg, swap_reg, markOopDesc::biased_lock_mask_in_place);
493 cmp(tmp_reg, (u1)markOopDesc::biased_lock_pattern);
494 br(Assembler::NE, cas_label);
495 // The bias pattern is present in the object's header. Need to check
496 // whether the bias owner and the epoch are both still current.
497 load_prototype_header(tmp_reg, obj_reg);
498 orr(tmp_reg, tmp_reg, rthread);
499 eor(tmp_reg, swap_reg, tmp_reg);
500 andr(tmp_reg, tmp_reg, ~((int) markOopDesc::age_mask_in_place));
501 if (counters != NULL) {
502 Label around;
503 cbnz(tmp_reg, around);
504 atomic_incw(Address((address)counters->biased_lock_entry_count_addr()), tmp_reg, rscratch1, rscratch2);
505 b(done);
506 bind(around);
507 } else {
508 cbz(tmp_reg, done);
509 }
510
511 Label try_revoke_bias;
512 Label try_rebias;
513
514 // At this point we know that the header has the bias pattern and
515 // that we are not the bias owner in the current epoch. We need to
516 // figure out more details about the state of the header in order to
517 // know what operations can be legally performed on the object's
518 // header.
519
520 // If the low three bits in the xor result aren't clear, that means
521 // the prototype header is no longer biased and we have to revoke
522 // the bias on this object.
523 andr(rscratch1, tmp_reg, markOopDesc::biased_lock_mask_in_place);
524 cbnz(rscratch1, try_revoke_bias);
525
526 // Biasing is still enabled for this data type. See whether the
527 // epoch of the current bias is still valid, meaning that the epoch
528 // bits of the mark word are equal to the epoch bits of the
529 // prototype header. (Note that the prototype header's epoch bits
530 // only change at a safepoint.) If not, attempt to rebias the object
531 // toward the current thread. Note that we must be absolutely sure
532 // that the current epoch is invalid in order to do this because
533 // otherwise the manipulations it performs on the mark word are
534 // illegal.
535 andr(rscratch1, tmp_reg, markOopDesc::epoch_mask_in_place);
536 cbnz(rscratch1, try_rebias);
537
538 // The epoch of the current bias is still valid but we know nothing
539 // about the owner; it might be set or it might be clear. Try to
540 // acquire the bias of the object using an atomic operation. If this
541 // fails we will go in to the runtime to revoke the object's bias.
542 // Note that we first construct the presumed unbiased header so we
543 // don't accidentally blow away another thread's valid bias.
544 {
545 Label here;
546 mov(rscratch1, markOopDesc::biased_lock_mask_in_place | markOopDesc::age_mask_in_place | markOopDesc::epoch_mask_in_place);
547 andr(swap_reg, swap_reg, rscratch1);
548 orr(tmp_reg, swap_reg, rthread);
549 cmpxchg_obj_header(swap_reg, tmp_reg, obj_reg, rscratch1, here, slow_case);
550 // If the biasing toward our thread failed, this means that
551 // another thread succeeded in biasing it toward itself and we
552 // need to revoke that bias. The revocation will occur in the
553 // interpreter runtime in the slow case.
554 bind(here);
555 if (counters != NULL) {
556 atomic_incw(Address((address)counters->anonymously_biased_lock_entry_count_addr()),
557 tmp_reg, rscratch1, rscratch2);
558 }
559 }
560 b(done);
561
562 bind(try_rebias);
563 // At this point we know the epoch has expired, meaning that the
564 // current "bias owner", if any, is actually invalid. Under these
565 // circumstances _only_, we are allowed to use the current header's
566 // value as the comparison value when doing the cas to acquire the
567 // bias in the current epoch. In other words, we allow transfer of
568 // the bias from one thread to another directly in this situation.
569 //
570 // FIXME: due to a lack of registers we currently blow away the age
571 // bits in this situation. Should attempt to preserve them.
572 {
573 Label here;
574 load_prototype_header(tmp_reg, obj_reg);
575 orr(tmp_reg, rthread, tmp_reg);
576 cmpxchg_obj_header(swap_reg, tmp_reg, obj_reg, rscratch1, here, slow_case);
577 // If the biasing toward our thread failed, then another thread
578 // succeeded in biasing it toward itself and we need to revoke that
579 // bias. The revocation will occur in the runtime in the slow case.
580 bind(here);
581 if (counters != NULL) {
582 atomic_incw(Address((address)counters->rebiased_lock_entry_count_addr()),
583 tmp_reg, rscratch1, rscratch2);
584 }
585 }
586 b(done);
587
588 bind(try_revoke_bias);
589 // The prototype mark in the klass doesn't have the bias bit set any
590 // more, indicating that objects of this data type are not supposed
591 // to be biased any more. We are going to try to reset the mark of
592 // this object to the prototype value and fall through to the
593 // CAS-based locking scheme. Note that if our CAS fails, it means
594 // that another thread raced us for the privilege of revoking the
595 // bias of this particular object, so it's okay to continue in the
596 // normal locking code.
597 //
598 // FIXME: due to a lack of registers we currently blow away the age
599 // bits in this situation. Should attempt to preserve them.
600 {
601 Label here, nope;
602 load_prototype_header(tmp_reg, obj_reg);
603 cmpxchg_obj_header(swap_reg, tmp_reg, obj_reg, rscratch1, here, &nope);
604 bind(here);
605
606 // Fall through to the normal CAS-based lock, because no matter what
607 // the result of the above CAS, some thread must have succeeded in
608 // removing the bias bit from the object's header.
609 if (counters != NULL) {
610 atomic_incw(Address((address)counters->revoked_lock_entry_count_addr()), tmp_reg,
611 rscratch1, rscratch2);
612 }
613 bind(nope);
614 }
615
616 bind(cas_label);
617
618 return null_check_offset;
619 }
620
621 void MacroAssembler::biased_locking_exit(Register obj_reg, Register temp_reg, Label& done) {
622 assert(UseBiasedLocking, "why call this otherwise?");
623
624 // Check for biased locking unlock case, which is a no-op
625 // Note: we do not have to check the thread ID for two reasons.
626 // First, the interpreter checks for IllegalMonitorStateException at
627 // a higher level. Second, if the bias was revoked while we held the
628 // lock, the object could not be rebiased toward another thread, so
629 // the bias bit would be clear.
630 ldr(temp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
631 andr(temp_reg, temp_reg, markOopDesc::biased_lock_mask_in_place);
632 cmp(temp_reg, (u1)markOopDesc::biased_lock_pattern);
633 br(Assembler::EQ, done);
634 }
635
636 static void pass_arg0(MacroAssembler* masm, Register arg) {
637 if (c_rarg0 != arg ) {
638 masm->mov(c_rarg0, arg);
639 }
640 }
641
642 static void pass_arg1(MacroAssembler* masm, Register arg) {
643 if (c_rarg1 != arg ) {
644 masm->mov(c_rarg1, arg);
645 }
646 }
647
648 static void pass_arg2(MacroAssembler* masm, Register arg) {
649 if (c_rarg2 != arg ) {
650 masm->mov(c_rarg2, arg);
651 }
652 }
653
654 static void pass_arg3(MacroAssembler* masm, Register arg) {
655 if (c_rarg3 != arg ) {
656 masm->mov(c_rarg3, arg);
657 }
658 }
659
660 void MacroAssembler::call_VM_base(Register oop_result,
661 Register java_thread,
662 Register last_java_sp,
663 address entry_point,
664 int number_of_arguments,
665 bool check_exceptions) {
666 // determine java_thread register
667 if (!java_thread->is_valid()) {
668 java_thread = rthread;
669 }
670
671 // determine last_java_sp register
672 if (!last_java_sp->is_valid()) {
673 last_java_sp = esp;
674 }
675
676 // debugging support
677 assert(number_of_arguments >= 0 , "cannot have negative number of arguments");
678 assert(java_thread == rthread, "unexpected register");
679 #ifdef ASSERT
680 // TraceBytecodes does not use r12 but saves it over the call, so don't verify
681 // if ((UseCompressedOops || UseCompressedClassPointers) && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");
682 #endif // ASSERT
683
684 assert(java_thread != oop_result , "cannot use the same register for java_thread & oop_result");
685 assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp");
686
687 // push java thread (becomes first argument of C function)
688
689 mov(c_rarg0, java_thread);
690
691 // set last Java frame before call
692 assert(last_java_sp != rfp, "can't use rfp");
693
694 Label l;
695 set_last_Java_frame(last_java_sp, rfp, l, rscratch1);
696
697 // do the call, remove parameters
698 MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments, &l);
699
700 // reset last Java frame
701 // Only interpreter should have to clear fp
702 reset_last_Java_frame(true);
703
704 // C++ interp handles this in the interpreter
705 check_and_handle_popframe(java_thread);
706 check_and_handle_earlyret(java_thread);
707
708 if (check_exceptions) {
709 // check for pending exceptions (java_thread is set upon return)
710 ldr(rscratch1, Address(java_thread, in_bytes(Thread::pending_exception_offset())));
711 Label ok;
712 cbz(rscratch1, ok);
713 lea(rscratch1, RuntimeAddress(StubRoutines::forward_exception_entry()));
714 br(rscratch1);
715 bind(ok);
716 }
717
718 // get oop result if there is one and reset the value in the thread
719 if (oop_result->is_valid()) {
720 get_vm_result(oop_result, java_thread);
721 }
722 }
723
724 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
725 call_VM_base(oop_result, noreg, noreg, entry_point, number_of_arguments, check_exceptions);
726 }
727
728 // Maybe emit a call via a trampoline. If the code cache is small
729 // trampolines won't be emitted.
730
731 address MacroAssembler::trampoline_call(Address entry, CodeBuffer *cbuf) {
732 assert(JavaThread::current()->is_Compiler_thread(), "just checking");
733 assert(entry.rspec().type() == relocInfo::runtime_call_type
734 || entry.rspec().type() == relocInfo::opt_virtual_call_type
735 || entry.rspec().type() == relocInfo::static_call_type
736 || entry.rspec().type() == relocInfo::virtual_call_type, "wrong reloc type");
737
738 // We need a trampoline if branches are far.
739 if (far_branches()) {
740 bool in_scratch_emit_size = false;
741 #ifdef COMPILER2
742 // We don't want to emit a trampoline if C2 is generating dummy
743 // code during its branch shortening phase.
744 CompileTask* task = ciEnv::current()->task();
745 in_scratch_emit_size =
746 (task != NULL && is_c2_compile(task->comp_level()) &&
747 Compile::current()->in_scratch_emit_size());
748 #endif
749 if (!in_scratch_emit_size) {
750 address stub = emit_trampoline_stub(offset(), entry.target());
751 if (stub == NULL) {
752 return NULL; // CodeCache is full
753 }
754 }
755 }
756
757 if (cbuf) cbuf->set_insts_mark();
758 relocate(entry.rspec());
759 if (!far_branches()) {
760 bl(entry.target());
761 } else {
762 bl(pc());
763 }
764 // just need to return a non-null address
765 return pc();
766 }
767
768
769 // Emit a trampoline stub for a call to a target which is too far away.
770 //
771 // code sequences:
772 //
773 // call-site:
774 // branch-and-link to <destination> or <trampoline stub>
775 //
776 // Related trampoline stub for this call site in the stub section:
777 // load the call target from the constant pool
778 // branch (LR still points to the call site above)
779
780 address MacroAssembler::emit_trampoline_stub(int insts_call_instruction_offset,
781 address dest) {
782 // Max stub size: alignment nop, TrampolineStub.
783 address stub = start_a_stub(NativeInstruction::instruction_size
784 + NativeCallTrampolineStub::instruction_size);
785 if (stub == NULL) {
786 return NULL; // CodeBuffer::expand failed
787 }
788
789 // Create a trampoline stub relocation which relates this trampoline stub
790 // with the call instruction at insts_call_instruction_offset in the
791 // instructions code-section.
792 align(wordSize);
793 relocate(trampoline_stub_Relocation::spec(code()->insts()->start()
794 + insts_call_instruction_offset));
795 const int stub_start_offset = offset();
796
797 // Now, create the trampoline stub's code:
798 // - load the call
799 // - call
800 Label target;
801 ldr(rscratch1, target);
802 br(rscratch1);
803 bind(target);
804 assert(offset() - stub_start_offset == NativeCallTrampolineStub::data_offset,
805 "should be");
806 emit_int64((int64_t)dest);
807
808 const address stub_start_addr = addr_at(stub_start_offset);
809
810 assert(is_NativeCallTrampolineStub_at(stub_start_addr), "doesn't look like a trampoline");
811
812 end_a_stub();
813 return stub_start_addr;
814 }
815
816 void MacroAssembler::emit_static_call_stub() {
817 // CompiledDirectStaticCall::set_to_interpreted knows the
818 // exact layout of this stub.
819
820 isb();
821 mov_metadata(rmethod, (Metadata*)NULL);
822
823 // Jump to the entry point of the i2c stub.
824 movptr(rscratch1, 0);
825 br(rscratch1);
826 }
827
828 void MacroAssembler::c2bool(Register x) {
829 // implements x == 0 ? 0 : 1
830 // note: must only look at least-significant byte of x
831 // since C-style booleans are stored in one byte
832 // only! (was bug)
833 tst(x, 0xff);
834 cset(x, Assembler::NE);
835 }
836
837 address MacroAssembler::ic_call(address entry, jint method_index) {
838 RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
839 // address const_ptr = long_constant((jlong)Universe::non_oop_word());
840 // unsigned long offset;
841 // ldr_constant(rscratch2, const_ptr);
842 movptr(rscratch2, (uintptr_t)Universe::non_oop_word());
843 return trampoline_call(Address(entry, rh));
844 }
845
846 // Implementation of call_VM versions
847
848 void MacroAssembler::call_VM(Register oop_result,
849 address entry_point,
850 bool check_exceptions) {
851 call_VM_helper(oop_result, entry_point, 0, check_exceptions);
852 }
853
854 void MacroAssembler::call_VM(Register oop_result,
855 address entry_point,
856 Register arg_1,
857 bool check_exceptions) {
858 pass_arg1(this, arg_1);
859 call_VM_helper(oop_result, entry_point, 1, check_exceptions);
860 }
861
862 void MacroAssembler::call_VM(Register oop_result,
863 address entry_point,
864 Register arg_1,
865 Register arg_2,
866 bool check_exceptions) {
867 assert(arg_1 != c_rarg2, "smashed arg");
868 pass_arg2(this, arg_2);
869 pass_arg1(this, arg_1);
870 call_VM_helper(oop_result, entry_point, 2, check_exceptions);
871 }
872
873 void MacroAssembler::call_VM(Register oop_result,
874 address entry_point,
875 Register arg_1,
876 Register arg_2,
877 Register arg_3,
878 bool check_exceptions) {
879 assert(arg_1 != c_rarg3, "smashed arg");
880 assert(arg_2 != c_rarg3, "smashed arg");
881 pass_arg3(this, arg_3);
882
883 assert(arg_1 != c_rarg2, "smashed arg");
884 pass_arg2(this, arg_2);
885
886 pass_arg1(this, arg_1);
887 call_VM_helper(oop_result, entry_point, 3, check_exceptions);
888 }
889
890 void MacroAssembler::call_VM(Register oop_result,
891 Register last_java_sp,
892 address entry_point,
893 int number_of_arguments,
894 bool check_exceptions) {
895 call_VM_base(oop_result, rthread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
896 }
897
898 void MacroAssembler::call_VM(Register oop_result,
899 Register last_java_sp,
900 address entry_point,
901 Register arg_1,
902 bool check_exceptions) {
903 pass_arg1(this, arg_1);
904 call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
905 }
906
907 void MacroAssembler::call_VM(Register oop_result,
908 Register last_java_sp,
909 address entry_point,
910 Register arg_1,
911 Register arg_2,
912 bool check_exceptions) {
913
914 assert(arg_1 != c_rarg2, "smashed arg");
915 pass_arg2(this, arg_2);
916 pass_arg1(this, arg_1);
917 call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
918 }
919
920 void MacroAssembler::call_VM(Register oop_result,
921 Register last_java_sp,
922 address entry_point,
923 Register arg_1,
924 Register arg_2,
925 Register arg_3,
926 bool check_exceptions) {
927 assert(arg_1 != c_rarg3, "smashed arg");
928 assert(arg_2 != c_rarg3, "smashed arg");
929 pass_arg3(this, arg_3);
930 assert(arg_1 != c_rarg2, "smashed arg");
931 pass_arg2(this, arg_2);
932 pass_arg1(this, arg_1);
933 call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
934 }
935
936
937 void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) {
938 ldr(oop_result, Address(java_thread, JavaThread::vm_result_offset()));
939 str(zr, Address(java_thread, JavaThread::vm_result_offset()));
940 verify_oop(oop_result, "broken oop in call_VM_base");
941 }
942
943 void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) {
944 ldr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset()));
945 str(zr, Address(java_thread, JavaThread::vm_result_2_offset()));
946 }
947
948 void MacroAssembler::align(int modulus) {
949 while (offset() % modulus != 0) nop();
950 }
951
952 // these are no-ops overridden by InterpreterMacroAssembler
953
954 void MacroAssembler::check_and_handle_earlyret(Register java_thread) { }
955
956 void MacroAssembler::check_and_handle_popframe(Register java_thread) { }
957
958
959 RegisterOrConstant MacroAssembler::delayed_value_impl(intptr_t* delayed_value_addr,
960 Register tmp,
961 int offset) {
962 intptr_t value = *delayed_value_addr;
963 if (value != 0)
964 return RegisterOrConstant(value + offset);
965
966 // load indirectly to solve generation ordering problem
967 ldr(tmp, ExternalAddress((address) delayed_value_addr));
968
969 if (offset != 0)
970 add(tmp, tmp, offset);
971
972 return RegisterOrConstant(tmp);
973 }
974
975
976 void MacroAssembler:: notify(int type) {
977 if (type == bytecode_start) {
978 // set_last_Java_frame(esp, rfp, (address)NULL);
979 Assembler:: notify(type);
980 // reset_last_Java_frame(true);
981 }
982 else
983 Assembler:: notify(type);
984 }
985
986 // Look up the method for a megamorphic invokeinterface call.
987 // The target method is determined by <intf_klass, itable_index>.
988 // The receiver klass is in recv_klass.
989 // On success, the result will be in method_result, and execution falls through.
990 // On failure, execution transfers to the given label.
991 void MacroAssembler::lookup_interface_method(Register recv_klass,
992 Register intf_klass,
993 RegisterOrConstant itable_index,
994 Register method_result,
995 Register scan_temp,
996 Label& L_no_such_interface,
997 bool return_method) {
998 assert_different_registers(recv_klass, intf_klass, scan_temp);
999 assert_different_registers(method_result, intf_klass, scan_temp);
1000 assert(recv_klass != method_result || !return_method,
1001 "recv_klass can be destroyed when method isn't needed");
1002 assert(itable_index.is_constant() || itable_index.as_register() == method_result,
1003 "caller must use same register for non-constant itable index as for method");
1004
1005 // Compute start of first itableOffsetEntry (which is at the end of the vtable)
1006 int vtable_base = in_bytes(Klass::vtable_start_offset());
1007 int itentry_off = itableMethodEntry::method_offset_in_bytes();
1008 int scan_step = itableOffsetEntry::size() * wordSize;
1009 int vte_size = vtableEntry::size_in_bytes();
1010 assert(vte_size == wordSize, "else adjust times_vte_scale");
1011
1012 ldrw(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
1013
1014 // %%% Could store the aligned, prescaled offset in the klassoop.
1015 // lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base));
1016 lea(scan_temp, Address(recv_klass, scan_temp, Address::lsl(3)));
1017 add(scan_temp, scan_temp, vtable_base);
1018
1019 if (return_method) {
1020 // Adjust recv_klass by scaled itable_index, so we can free itable_index.
1021 assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
1022 // lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off));
1023 lea(recv_klass, Address(recv_klass, itable_index, Address::lsl(3)));
1024 if (itentry_off)
1025 add(recv_klass, recv_klass, itentry_off);
1026 }
1027
1028 // for (scan = klass->itable(); scan->interface() != NULL; scan += scan_step) {
1029 // if (scan->interface() == intf) {
1030 // result = (klass + scan->offset() + itable_index);
1031 // }
1032 // }
1033 Label search, found_method;
1034
1035 for (int peel = 1; peel >= 0; peel--) {
1036 ldr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset_in_bytes()));
1037 cmp(intf_klass, method_result);
1038
1039 if (peel) {
1040 br(Assembler::EQ, found_method);
1041 } else {
1042 br(Assembler::NE, search);
1043 // (invert the test to fall through to found_method...)
1044 }
1045
1046 if (!peel) break;
1047
1048 bind(search);
1049
1050 // Check that the previous entry is non-null. A null entry means that
1051 // the receiver class doesn't implement the interface, and wasn't the
1052 // same as when the caller was compiled.
1053 cbz(method_result, L_no_such_interface);
1054 add(scan_temp, scan_temp, scan_step);
1055 }
1056
1057 bind(found_method);
1058
1059 // Got a hit.
1060 if (return_method) {
1061 ldrw(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset_in_bytes()));
1062 ldr(method_result, Address(recv_klass, scan_temp, Address::uxtw(0)));
1063 }
1064 }
1065
1066 // virtual method calling
1067 void MacroAssembler::lookup_virtual_method(Register recv_klass,
1068 RegisterOrConstant vtable_index,
1069 Register method_result) {
1070 const int base = in_bytes(Klass::vtable_start_offset());
1071 assert(vtableEntry::size() * wordSize == 8,
1072 "adjust the scaling in the code below");
1073 int vtable_offset_in_bytes = base + vtableEntry::method_offset_in_bytes();
1074
1075 if (vtable_index.is_register()) {
1076 lea(method_result, Address(recv_klass,
1077 vtable_index.as_register(),
1078 Address::lsl(LogBytesPerWord)));
1079 ldr(method_result, Address(method_result, vtable_offset_in_bytes));
1080 } else {
1081 vtable_offset_in_bytes += vtable_index.as_constant() * wordSize;
1082 ldr(method_result,
1083 form_address(rscratch1, recv_klass, vtable_offset_in_bytes, 0));
1084 }
1085 }
1086
1087 void MacroAssembler::check_klass_subtype(Register sub_klass,
1088 Register super_klass,
1089 Register temp_reg,
1090 Label& L_success) {
1091 Label L_failure;
1092 check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg, &L_success, &L_failure, NULL);
1093 check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, NULL);
1094 bind(L_failure);
1095 }
1096
1097
1098 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
1099 Register super_klass,
1100 Register temp_reg,
1101 Label* L_success,
1102 Label* L_failure,
1103 Label* L_slow_path,
1104 RegisterOrConstant super_check_offset) {
1105 assert_different_registers(sub_klass, super_klass, temp_reg);
1106 bool must_load_sco = (super_check_offset.constant_or_zero() == -1);
1107 if (super_check_offset.is_register()) {
1108 assert_different_registers(sub_klass, super_klass,
1109 super_check_offset.as_register());
1110 } else if (must_load_sco) {
1111 assert(temp_reg != noreg, "supply either a temp or a register offset");
1112 }
1113
1114 Label L_fallthrough;
1115 int label_nulls = 0;
1116 if (L_success == NULL) { L_success = &L_fallthrough; label_nulls++; }
1117 if (L_failure == NULL) { L_failure = &L_fallthrough; label_nulls++; }
1118 if (L_slow_path == NULL) { L_slow_path = &L_fallthrough; label_nulls++; }
1119 assert(label_nulls <= 1, "at most one NULL in the batch");
1120
1121 int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
1122 int sco_offset = in_bytes(Klass::super_check_offset_offset());
1123 Address super_check_offset_addr(super_klass, sco_offset);
1124
1125 // Hacked jmp, which may only be used just before L_fallthrough.
1126 #define final_jmp(label) \
1127 if (&(label) == &L_fallthrough) { /*do nothing*/ } \
1128 else b(label) /*omit semi*/
1129
1130 // If the pointers are equal, we are done (e.g., String[] elements).
1131 // This self-check enables sharing of secondary supertype arrays among
1132 // non-primary types such as array-of-interface. Otherwise, each such
1133 // type would need its own customized SSA.
1134 // We move this check to the front of the fast path because many
1135 // type checks are in fact trivially successful in this manner,
1136 // so we get a nicely predicted branch right at the start of the check.
1137 cmp(sub_klass, super_klass);
1138 br(Assembler::EQ, *L_success);
1139
1140 // Check the supertype display:
1141 if (must_load_sco) {
1142 ldrw(temp_reg, super_check_offset_addr);
1143 super_check_offset = RegisterOrConstant(temp_reg);
1144 }
1145 Address super_check_addr(sub_klass, super_check_offset);
1146 ldr(rscratch1, super_check_addr);
1147 cmp(super_klass, rscratch1); // load displayed supertype
1148
1149 // This check has worked decisively for primary supers.
1150 // Secondary supers are sought in the super_cache ('super_cache_addr').
1151 // (Secondary supers are interfaces and very deeply nested subtypes.)
1152 // This works in the same check above because of a tricky aliasing
1153 // between the super_cache and the primary super display elements.
1154 // (The 'super_check_addr' can address either, as the case requires.)
1155 // Note that the cache is updated below if it does not help us find
1156 // what we need immediately.
1157 // So if it was a primary super, we can just fail immediately.
1158 // Otherwise, it's the slow path for us (no success at this point).
1159
1160 if (super_check_offset.is_register()) {
1161 br(Assembler::EQ, *L_success);
1162 subs(zr, super_check_offset.as_register(), sc_offset);
1163 if (L_failure == &L_fallthrough) {
1164 br(Assembler::EQ, *L_slow_path);
1165 } else {
1166 br(Assembler::NE, *L_failure);
1167 final_jmp(*L_slow_path);
1168 }
1169 } else if (super_check_offset.as_constant() == sc_offset) {
1170 // Need a slow path; fast failure is impossible.
1171 if (L_slow_path == &L_fallthrough) {
1172 br(Assembler::EQ, *L_success);
1173 } else {
1174 br(Assembler::NE, *L_slow_path);
1175 final_jmp(*L_success);
1176 }
1177 } else {
1178 // No slow path; it's a fast decision.
1179 if (L_failure == &L_fallthrough) {
1180 br(Assembler::EQ, *L_success);
1181 } else {
1182 br(Assembler::NE, *L_failure);
1183 final_jmp(*L_success);
1184 }
1185 }
1186
1187 bind(L_fallthrough);
1188
1189 #undef final_jmp
1190 }
1191
1192 // These two are taken from x86, but they look generally useful
1193
1194 // scans count pointer sized words at [addr] for occurence of value,
1195 // generic
1196 void MacroAssembler::repne_scan(Register addr, Register value, Register count,
1197 Register scratch) {
1198 Label Lloop, Lexit;
1199 cbz(count, Lexit);
1200 bind(Lloop);
1201 ldr(scratch, post(addr, wordSize));
1202 cmp(value, scratch);
1203 br(EQ, Lexit);
1204 sub(count, count, 1);
1205 cbnz(count, Lloop);
1206 bind(Lexit);
1207 }
1208
1209 // scans count 4 byte words at [addr] for occurence of value,
1210 // generic
1211 void MacroAssembler::repne_scanw(Register addr, Register value, Register count,
1212 Register scratch) {
1213 Label Lloop, Lexit;
1214 cbz(count, Lexit);
1215 bind(Lloop);
1216 ldrw(scratch, post(addr, wordSize));
1217 cmpw(value, scratch);
1218 br(EQ, Lexit);
1219 sub(count, count, 1);
1220 cbnz(count, Lloop);
1221 bind(Lexit);
1222 }
1223
1224 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
1225 Register super_klass,
1226 Register temp_reg,
1227 Register temp2_reg,
1228 Label* L_success,
1229 Label* L_failure,
1230 bool set_cond_codes) {
1231 assert_different_registers(sub_klass, super_klass, temp_reg);
1232 if (temp2_reg != noreg)
1233 assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg, rscratch1);
1234 #define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg)
1235
1236 Label L_fallthrough;
1237 int label_nulls = 0;
1238 if (L_success == NULL) { L_success = &L_fallthrough; label_nulls++; }
1239 if (L_failure == NULL) { L_failure = &L_fallthrough; label_nulls++; }
1240 assert(label_nulls <= 1, "at most one NULL in the batch");
1241
1242 // a couple of useful fields in sub_klass:
1243 int ss_offset = in_bytes(Klass::secondary_supers_offset());
1244 int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
1245 Address secondary_supers_addr(sub_klass, ss_offset);
1246 Address super_cache_addr( sub_klass, sc_offset);
1247
1248 BLOCK_COMMENT("check_klass_subtype_slow_path");
1249
1250 // Do a linear scan of the secondary super-klass chain.
1251 // This code is rarely used, so simplicity is a virtue here.
1252 // The repne_scan instruction uses fixed registers, which we must spill.
1253 // Don't worry too much about pre-existing connections with the input regs.
1254
1255 assert(sub_klass != r0, "killed reg"); // killed by mov(r0, super)
1256 assert(sub_klass != r2, "killed reg"); // killed by lea(r2, &pst_counter)
1257
1258 RegSet pushed_registers;
1259 if (!IS_A_TEMP(r2)) pushed_registers += r2;
1260 if (!IS_A_TEMP(r5)) pushed_registers += r5;
1261
1262 if (super_klass != r0 || UseCompressedOops) {
1263 if (!IS_A_TEMP(r0)) pushed_registers += r0;
1264 }
1265
1266 push(pushed_registers, sp);
1267
1268 // Get super_klass value into r0 (even if it was in r5 or r2).
1269 if (super_klass != r0) {
1270 mov(r0, super_klass);
1271 }
1272
1273 #ifndef PRODUCT
1274 mov(rscratch2, (address)&SharedRuntime::_partial_subtype_ctr);
1275 Address pst_counter_addr(rscratch2);
1276 ldr(rscratch1, pst_counter_addr);
1277 add(rscratch1, rscratch1, 1);
1278 str(rscratch1, pst_counter_addr);
1279 #endif //PRODUCT
1280
1281 // We will consult the secondary-super array.
1282 ldr(r5, secondary_supers_addr);
1283 // Load the array length.
1284 ldrw(r2, Address(r5, Array<Klass*>::length_offset_in_bytes()));
1285 // Skip to start of data.
1286 add(r5, r5, Array<Klass*>::base_offset_in_bytes());
1287
1288 cmp(sp, zr); // Clear Z flag; SP is never zero
1289 // Scan R2 words at [R5] for an occurrence of R0.
1290 // Set NZ/Z based on last compare.
1291 repne_scan(r5, r0, r2, rscratch1);
1292
1293 // Unspill the temp. registers:
1294 pop(pushed_registers, sp);
1295
1296 br(Assembler::NE, *L_failure);
1297
1298 // Success. Cache the super we found and proceed in triumph.
1299 str(super_klass, super_cache_addr);
1300
1301 if (L_success != &L_fallthrough) {
1302 b(*L_success);
1303 }
1304
1305 #undef IS_A_TEMP
1306
1307 bind(L_fallthrough);
1308 }
1309
1310
1311 void MacroAssembler::verify_oop(Register reg, const char* s) {
1312 if (!VerifyOops) return;
1313
1314 // Pass register number to verify_oop_subroutine
1315 const char* b = NULL;
1316 {
1317 ResourceMark rm;
1318 stringStream ss;
1319 ss.print("verify_oop: %s: %s", reg->name(), s);
1320 b = code_string(ss.as_string());
1321 }
1322 BLOCK_COMMENT("verify_oop {");
1323
1324 stp(r0, rscratch1, Address(pre(sp, -2 * wordSize)));
1325 stp(rscratch2, lr, Address(pre(sp, -2 * wordSize)));
1326
1327 mov(r0, reg);
1328 mov(rscratch1, (address)b);
1329
1330 // call indirectly to solve generation ordering problem
1331 lea(rscratch2, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
1332 ldr(rscratch2, Address(rscratch2));
1333 blr(rscratch2);
1334
1335 ldp(rscratch2, lr, Address(post(sp, 2 * wordSize)));
1336 ldp(r0, rscratch1, Address(post(sp, 2 * wordSize)));
1337
1338 BLOCK_COMMENT("} verify_oop");
1339 }
1340
1341 void MacroAssembler::verify_oop_addr(Address addr, const char* s) {
1342 if (!VerifyOops) return;
1343
1344 const char* b = NULL;
1345 {
1346 ResourceMark rm;
1347 stringStream ss;
1348 ss.print("verify_oop_addr: %s", s);
1349 b = code_string(ss.as_string());
1350 }
1351 BLOCK_COMMENT("verify_oop_addr {");
1352
1353 stp(r0, rscratch1, Address(pre(sp, -2 * wordSize)));
1354 stp(rscratch2, lr, Address(pre(sp, -2 * wordSize)));
1355
1356 // addr may contain sp so we will have to adjust it based on the
1357 // pushes that we just did.
1358 if (addr.uses(sp)) {
1359 lea(r0, addr);
1360 ldr(r0, Address(r0, 4 * wordSize));
1361 } else {
1362 ldr(r0, addr);
1363 }
1364 mov(rscratch1, (address)b);
1365
1366 // call indirectly to solve generation ordering problem
1367 lea(rscratch2, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
1368 ldr(rscratch2, Address(rscratch2));
1369 blr(rscratch2);
1370
1371 ldp(rscratch2, lr, Address(post(sp, 2 * wordSize)));
1372 ldp(r0, rscratch1, Address(post(sp, 2 * wordSize)));
1373
1374 BLOCK_COMMENT("} verify_oop_addr");
1375 }
1376
1377 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot,
1378 int extra_slot_offset) {
1379 // cf. TemplateTable::prepare_invoke(), if (load_receiver).
1380 int stackElementSize = Interpreter::stackElementSize;
1381 int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0);
1382 #ifdef ASSERT
1383 int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1);
1384 assert(offset1 - offset == stackElementSize, "correct arithmetic");
1385 #endif
1386 if (arg_slot.is_constant()) {
1387 return Address(esp, arg_slot.as_constant() * stackElementSize
1388 + offset);
1389 } else {
1390 add(rscratch1, esp, arg_slot.as_register(),
1391 ext::uxtx, exact_log2(stackElementSize));
1392 return Address(rscratch1, offset);
1393 }
1394 }
1395
1396 void MacroAssembler::call_VM_leaf_base(address entry_point,
1397 int number_of_arguments,
1398 Label *retaddr) {
1399 call_VM_leaf_base1(entry_point, number_of_arguments, 0, ret_type_integral, retaddr);
1400 }
1401
1402 void MacroAssembler::call_VM_leaf_base1(address entry_point,
1403 int number_of_gp_arguments,
1404 int number_of_fp_arguments,
1405 ret_type type,
1406 Label *retaddr) {
1407 Label E, L;
1408
1409 stp(rscratch1, rmethod, Address(pre(sp, -2 * wordSize)));
1410
1411 // We add 1 to number_of_arguments because the thread in arg0 is
1412 // not counted
1413 mov(rscratch1, entry_point);
1414 blrt(rscratch1, number_of_gp_arguments + 1, number_of_fp_arguments, type);
1415 if (retaddr)
1416 bind(*retaddr);
1417
1418 ldp(rscratch1, rmethod, Address(post(sp, 2 * wordSize)));
1419 maybe_isb();
1420 }
1421
1422 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
1423 call_VM_leaf_base(entry_point, number_of_arguments);
1424 }
1425
1426 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {
1427 pass_arg0(this, arg_0);
1428 call_VM_leaf_base(entry_point, 1);
1429 }
1430
1431 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
1432 pass_arg0(this, arg_0);
1433 pass_arg1(this, arg_1);
1434 call_VM_leaf_base(entry_point, 2);
1435 }
1436
1437 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0,
1438 Register arg_1, Register arg_2) {
1439 pass_arg0(this, arg_0);
1440 pass_arg1(this, arg_1);
1441 pass_arg2(this, arg_2);
1442 call_VM_leaf_base(entry_point, 3);
1443 }
1444
1445 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
1446 pass_arg0(this, arg_0);
1447 MacroAssembler::call_VM_leaf_base(entry_point, 1);
1448 }
1449
1450 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
1451
1452 assert(arg_0 != c_rarg1, "smashed arg");
1453 pass_arg1(this, arg_1);
1454 pass_arg0(this, arg_0);
1455 MacroAssembler::call_VM_leaf_base(entry_point, 2);
1456 }
1457
1458 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
1459 assert(arg_0 != c_rarg2, "smashed arg");
1460 assert(arg_1 != c_rarg2, "smashed arg");
1461 pass_arg2(this, arg_2);
1462 assert(arg_0 != c_rarg1, "smashed arg");
1463 pass_arg1(this, arg_1);
1464 pass_arg0(this, arg_0);
1465 MacroAssembler::call_VM_leaf_base(entry_point, 3);
1466 }
1467
1468 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
1469 assert(arg_0 != c_rarg3, "smashed arg");
1470 assert(arg_1 != c_rarg3, "smashed arg");
1471 assert(arg_2 != c_rarg3, "smashed arg");
1472 pass_arg3(this, arg_3);
1473 assert(arg_0 != c_rarg2, "smashed arg");
1474 assert(arg_1 != c_rarg2, "smashed arg");
1475 pass_arg2(this, arg_2);
1476 assert(arg_0 != c_rarg1, "smashed arg");
1477 pass_arg1(this, arg_1);
1478 pass_arg0(this, arg_0);
1479 MacroAssembler::call_VM_leaf_base(entry_point, 4);
1480 }
1481
1482 void MacroAssembler::null_check(Register reg, int offset) {
1483 if (needs_explicit_null_check(offset)) {
1484 // provoke OS NULL exception if reg = NULL by
1485 // accessing M[reg] w/o changing any registers
1486 // NOTE: this is plenty to provoke a segv
1487 ldr(zr, Address(reg));
1488 } else {
1489 // nothing to do, (later) access of M[reg + offset]
1490 // will provoke OS NULL exception if reg = NULL
1491 }
1492 }
1493
1494 // MacroAssembler protected routines needed to implement
1495 // public methods
1496
1497 void MacroAssembler::mov(Register r, Address dest) {
1498 code_section()->relocate(pc(), dest.rspec());
1499 u_int64_t imm64 = (u_int64_t)dest.target();
1500 movptr(r, imm64);
1501 }
1502
1503 // Move a constant pointer into r. In AArch64 mode the virtual
1504 // address space is 48 bits in size, so we only need three
1505 // instructions to create a patchable instruction sequence that can
1506 // reach anywhere.
1507 void MacroAssembler::movptr(Register r, uintptr_t imm64) {
1508 #ifndef PRODUCT
1509 {
1510 char buffer[64];
1511 snprintf(buffer, sizeof(buffer), "0x%" PRIX64, imm64);
1512 block_comment(buffer);
1513 }
1514 #endif
1515 assert(imm64 < (1ul << 48), "48-bit overflow in address constant");
1516 movz(r, imm64 & 0xffff);
1517 imm64 >>= 16;
1518 movk(r, imm64 & 0xffff, 16);
1519 imm64 >>= 16;
1520 movk(r, imm64 & 0xffff, 32);
1521 }
1522
1523 // Macro to mov replicated immediate to vector register.
1524 // Vd will get the following values for different arrangements in T
1525 // imm32 == hex 000000gh T8B: Vd = ghghghghghghghgh
1526 // imm32 == hex 000000gh T16B: Vd = ghghghghghghghghghghghghghghghgh
1527 // imm32 == hex 0000efgh T4H: Vd = efghefghefghefgh
1528 // imm32 == hex 0000efgh T8H: Vd = efghefghefghefghefghefghefghefgh
1529 // imm32 == hex abcdefgh T2S: Vd = abcdefghabcdefgh
1530 // imm32 == hex abcdefgh T4S: Vd = abcdefghabcdefghabcdefghabcdefgh
1531 // T1D/T2D: invalid
1532 void MacroAssembler::mov(FloatRegister Vd, SIMD_Arrangement T, u_int32_t imm32) {
1533 assert(T != T1D && T != T2D, "invalid arrangement");
1534 if (T == T8B || T == T16B) {
1535 assert((imm32 & ~0xff) == 0, "extraneous bits in unsigned imm32 (T8B/T16B)");
1536 movi(Vd, T, imm32 & 0xff, 0);
1537 return;
1538 }
1539 u_int32_t nimm32 = ~imm32;
1540 if (T == T4H || T == T8H) {
1541 assert((imm32 & ~0xffff) == 0, "extraneous bits in unsigned imm32 (T4H/T8H)");
1542 imm32 &= 0xffff;
1543 nimm32 &= 0xffff;
1544 }
1545 u_int32_t x = imm32;
1546 int movi_cnt = 0;
1547 int movn_cnt = 0;
1548 while (x) { if (x & 0xff) movi_cnt++; x >>= 8; }
1549 x = nimm32;
1550 while (x) { if (x & 0xff) movn_cnt++; x >>= 8; }
1551 if (movn_cnt < movi_cnt) imm32 = nimm32;
1552 unsigned lsl = 0;
1553 while (imm32 && (imm32 & 0xff) == 0) { lsl += 8; imm32 >>= 8; }
1554 if (movn_cnt < movi_cnt)
1555 mvni(Vd, T, imm32 & 0xff, lsl);
1556 else
1557 movi(Vd, T, imm32 & 0xff, lsl);
1558 imm32 >>= 8; lsl += 8;
1559 while (imm32) {
1560 while ((imm32 & 0xff) == 0) { lsl += 8; imm32 >>= 8; }
1561 if (movn_cnt < movi_cnt)
1562 bici(Vd, T, imm32 & 0xff, lsl);
1563 else
1564 orri(Vd, T, imm32 & 0xff, lsl);
1565 lsl += 8; imm32 >>= 8;
1566 }
1567 }
1568
1569 void MacroAssembler::mov_immediate64(Register dst, u_int64_t imm64)
1570 {
1571 #ifndef PRODUCT
1572 {
1573 char buffer[64];
1574 snprintf(buffer, sizeof(buffer), "0x%" PRIX64, imm64);
1575 block_comment(buffer);
1576 }
1577 #endif
1578 if (operand_valid_for_logical_immediate(false, imm64)) {
1579 orr(dst, zr, imm64);
1580 } else {
1581 // we can use a combination of MOVZ or MOVN with
1582 // MOVK to build up the constant
1583 u_int64_t imm_h[4];
1584 int zero_count = 0;
1585 int neg_count = 0;
1586 int i;
1587 for (i = 0; i < 4; i++) {
1588 imm_h[i] = ((imm64 >> (i * 16)) & 0xffffL);
1589 if (imm_h[i] == 0) {
1590 zero_count++;
1591 } else if (imm_h[i] == 0xffffL) {
1592 neg_count++;
1593 }
1594 }
1595 if (zero_count == 4) {
1596 // one MOVZ will do
1597 movz(dst, 0);
1598 } else if (neg_count == 4) {
1599 // one MOVN will do
1600 movn(dst, 0);
1601 } else if (zero_count == 3) {
1602 for (i = 0; i < 4; i++) {
1603 if (imm_h[i] != 0L) {
1604 movz(dst, (u_int32_t)imm_h[i], (i << 4));
1605 break;
1606 }
1607 }
1608 } else if (neg_count == 3) {
1609 // one MOVN will do
1610 for (int i = 0; i < 4; i++) {
1611 if (imm_h[i] != 0xffffL) {
1612 movn(dst, (u_int32_t)imm_h[i] ^ 0xffffL, (i << 4));
1613 break;
1614 }
1615 }
1616 } else if (zero_count == 2) {
1617 // one MOVZ and one MOVK will do
1618 for (i = 0; i < 3; i++) {
1619 if (imm_h[i] != 0L) {
1620 movz(dst, (u_int32_t)imm_h[i], (i << 4));
1621 i++;
1622 break;
1623 }
1624 }
1625 for (;i < 4; i++) {
1626 if (imm_h[i] != 0L) {
1627 movk(dst, (u_int32_t)imm_h[i], (i << 4));
1628 }
1629 }
1630 } else if (neg_count == 2) {
1631 // one MOVN and one MOVK will do
1632 for (i = 0; i < 4; i++) {
1633 if (imm_h[i] != 0xffffL) {
1634 movn(dst, (u_int32_t)imm_h[i] ^ 0xffffL, (i << 4));
1635 i++;
1636 break;
1637 }
1638 }
1639 for (;i < 4; i++) {
1640 if (imm_h[i] != 0xffffL) {
1641 movk(dst, (u_int32_t)imm_h[i], (i << 4));
1642 }
1643 }
1644 } else if (zero_count == 1) {
1645 // one MOVZ and two MOVKs will do
1646 for (i = 0; i < 4; i++) {
1647 if (imm_h[i] != 0L) {
1648 movz(dst, (u_int32_t)imm_h[i], (i << 4));
1649 i++;
1650 break;
1651 }
1652 }
1653 for (;i < 4; i++) {
1654 if (imm_h[i] != 0x0L) {
1655 movk(dst, (u_int32_t)imm_h[i], (i << 4));
1656 }
1657 }
1658 } else if (neg_count == 1) {
1659 // one MOVN and two MOVKs will do
1660 for (i = 0; i < 4; i++) {
1661 if (imm_h[i] != 0xffffL) {
1662 movn(dst, (u_int32_t)imm_h[i] ^ 0xffffL, (i << 4));
1663 i++;
1664 break;
1665 }
1666 }
1667 for (;i < 4; i++) {
1668 if (imm_h[i] != 0xffffL) {
1669 movk(dst, (u_int32_t)imm_h[i], (i << 4));
1670 }
1671 }
1672 } else {
1673 // use a MOVZ and 3 MOVKs (makes it easier to debug)
1674 movz(dst, (u_int32_t)imm_h[0], 0);
1675 for (i = 1; i < 4; i++) {
1676 movk(dst, (u_int32_t)imm_h[i], (i << 4));
1677 }
1678 }
1679 }
1680 }
1681
1682 void MacroAssembler::mov_immediate32(Register dst, u_int32_t imm32)
1683 {
1684 #ifndef PRODUCT
1685 {
1686 char buffer[64];
1687 snprintf(buffer, sizeof(buffer), "0x%" PRIX32, imm32);
1688 block_comment(buffer);
1689 }
1690 #endif
1691 if (operand_valid_for_logical_immediate(true, imm32)) {
1692 orrw(dst, zr, imm32);
1693 } else {
1694 // we can use MOVZ, MOVN or two calls to MOVK to build up the
1695 // constant
1696 u_int32_t imm_h[2];
1697 imm_h[0] = imm32 & 0xffff;
1698 imm_h[1] = ((imm32 >> 16) & 0xffff);
1699 if (imm_h[0] == 0) {
1700 movzw(dst, imm_h[1], 16);
1701 } else if (imm_h[0] == 0xffff) {
1702 movnw(dst, imm_h[1] ^ 0xffff, 16);
1703 } else if (imm_h[1] == 0) {
1704 movzw(dst, imm_h[0], 0);
1705 } else if (imm_h[1] == 0xffff) {
1706 movnw(dst, imm_h[0] ^ 0xffff, 0);
1707 } else {
1708 // use a MOVZ and MOVK (makes it easier to debug)
1709 movzw(dst, imm_h[0], 0);
1710 movkw(dst, imm_h[1], 16);
1711 }
1712 }
1713 }
1714
1715 // Form an address from base + offset in Rd. Rd may or may
1716 // not actually be used: you must use the Address that is returned.
1717 // It is up to you to ensure that the shift provided matches the size
1718 // of your data.
1719 Address MacroAssembler::form_address(Register Rd, Register base, long byte_offset, int shift) {
1720 if (Address::offset_ok_for_immed(byte_offset, shift))
1721 // It fits; no need for any heroics
1722 return Address(base, byte_offset);
1723
1724 // Don't do anything clever with negative or misaligned offsets
1725 unsigned mask = (1 << shift) - 1;
1726 if (byte_offset < 0 || byte_offset & mask) {
1727 mov(Rd, byte_offset);
1728 add(Rd, base, Rd);
1729 return Address(Rd);
1730 }
1731
1732 // See if we can do this with two 12-bit offsets
1733 {
1734 unsigned long word_offset = byte_offset >> shift;
1735 unsigned long masked_offset = word_offset & 0xfff000;
1736 if (Address::offset_ok_for_immed(word_offset - masked_offset)
1737 && Assembler::operand_valid_for_add_sub_immediate(masked_offset << shift)) {
1738 add(Rd, base, masked_offset << shift);
1739 word_offset -= masked_offset;
1740 return Address(Rd, word_offset << shift);
1741 }
1742 }
1743
1744 // Do it the hard way
1745 mov(Rd, byte_offset);
1746 add(Rd, base, Rd);
1747 return Address(Rd);
1748 }
1749
1750 void MacroAssembler::atomic_incw(Register counter_addr, Register tmp, Register tmp2) {
1751 if (UseLSE) {
1752 mov(tmp, 1);
1753 ldadd(Assembler::word, tmp, zr, counter_addr);
1754 return;
1755 }
1756 Label retry_load;
1757 if ((VM_Version::features() & VM_Version::CPU_STXR_PREFETCH))
1758 prfm(Address(counter_addr), PSTL1STRM);
1759 bind(retry_load);
1760 // flush and load exclusive from the memory location
1761 ldxrw(tmp, counter_addr);
1762 addw(tmp, tmp, 1);
1763 // if we store+flush with no intervening write tmp wil be zero
1764 stxrw(tmp2, tmp, counter_addr);
1765 cbnzw(tmp2, retry_load);
1766 }
1767
1768
1769 int MacroAssembler::corrected_idivl(Register result, Register ra, Register rb,
1770 bool want_remainder, Register scratch)
1771 {
1772 // Full implementation of Java idiv and irem. The function
1773 // returns the (pc) offset of the div instruction - may be needed
1774 // for implicit exceptions.
1775 //
1776 // constraint : ra/rb =/= scratch
1777 // normal case
1778 //
1779 // input : ra: dividend
1780 // rb: divisor
1781 //
1782 // result: either
1783 // quotient (= ra idiv rb)
1784 // remainder (= ra irem rb)
1785
1786 assert(ra != scratch && rb != scratch, "reg cannot be scratch");
1787
1788 int idivl_offset = offset();
1789 if (! want_remainder) {
1790 sdivw(result, ra, rb);
1791 } else {
1792 sdivw(scratch, ra, rb);
1793 Assembler::msubw(result, scratch, rb, ra);
1794 }
1795
1796 return idivl_offset;
1797 }
1798
1799 int MacroAssembler::corrected_idivq(Register result, Register ra, Register rb,
1800 bool want_remainder, Register scratch)
1801 {
1802 // Full implementation of Java ldiv and lrem. The function
1803 // returns the (pc) offset of the div instruction - may be needed
1804 // for implicit exceptions.
1805 //
1806 // constraint : ra/rb =/= scratch
1807 // normal case
1808 //
1809 // input : ra: dividend
1810 // rb: divisor
1811 //
1812 // result: either
1813 // quotient (= ra idiv rb)
1814 // remainder (= ra irem rb)
1815
1816 assert(ra != scratch && rb != scratch, "reg cannot be scratch");
1817
1818 int idivq_offset = offset();
1819 if (! want_remainder) {
1820 sdiv(result, ra, rb);
1821 } else {
1822 sdiv(scratch, ra, rb);
1823 Assembler::msub(result, scratch, rb, ra);
1824 }
1825
1826 return idivq_offset;
1827 }
1828
1829 void MacroAssembler::membar(Membar_mask_bits order_constraint) {
1830 address prev = pc() - NativeMembar::instruction_size;
1831 address last = code()->last_insn();
1832 if (last != NULL && nativeInstruction_at(last)->is_Membar() && prev == last) {
1833 NativeMembar *bar = NativeMembar_at(prev);
1834 // We are merging two memory barrier instructions. On AArch64 we
1835 // can do this simply by ORing them together.
1836 bar->set_kind(bar->get_kind() | order_constraint);
1837 BLOCK_COMMENT("merged membar");
1838 } else {
1839 code()->set_last_insn(pc());
1840 dmb(Assembler::barrier(order_constraint));
1841 }
1842 }
1843
1844 bool MacroAssembler::try_merge_ldst(Register rt, const Address &adr, size_t size_in_bytes, bool is_store) {
1845 if (ldst_can_merge(rt, adr, size_in_bytes, is_store)) {
1846 merge_ldst(rt, adr, size_in_bytes, is_store);
1847 code()->clear_last_insn();
1848 return true;
1849 } else {
1850 assert(size_in_bytes == 8 || size_in_bytes == 4, "only 8 bytes or 4 bytes load/store is supported.");
1851 const unsigned mask = size_in_bytes - 1;
1852 if (adr.getMode() == Address::base_plus_offset &&
1853 (adr.offset() & mask) == 0) { // only supports base_plus_offset.
1854 code()->set_last_insn(pc());
1855 }
1856 return false;
1857 }
1858 }
1859
1860 void MacroAssembler::ldr(Register Rx, const Address &adr) {
1861 // We always try to merge two adjacent loads into one ldp.
1862 if (!try_merge_ldst(Rx, adr, 8, false)) {
1863 Assembler::ldr(Rx, adr);
1864 }
1865 }
1866
1867 void MacroAssembler::ldrw(Register Rw, const Address &adr) {
1868 // We always try to merge two adjacent loads into one ldp.
1869 if (!try_merge_ldst(Rw, adr, 4, false)) {
1870 Assembler::ldrw(Rw, adr);
1871 }
1872 }
1873
1874 void MacroAssembler::str(Register Rx, const Address &adr) {
1875 // We always try to merge two adjacent stores into one stp.
1876 if (!try_merge_ldst(Rx, adr, 8, true)) {
1877 Assembler::str(Rx, adr);
1878 }
1879 }
1880
1881 void MacroAssembler::strw(Register Rw, const Address &adr) {
1882 // We always try to merge two adjacent stores into one stp.
1883 if (!try_merge_ldst(Rw, adr, 4, true)) {
1884 Assembler::strw(Rw, adr);
1885 }
1886 }
1887
1888 // MacroAssembler routines found actually to be needed
1889
1890 void MacroAssembler::push(Register src)
1891 {
1892 str(src, Address(pre(esp, -1 * wordSize)));
1893 }
1894
1895 void MacroAssembler::pop(Register dst)
1896 {
1897 ldr(dst, Address(post(esp, 1 * wordSize)));
1898 }
1899
1900 // Note: load_unsigned_short used to be called load_unsigned_word.
1901 int MacroAssembler::load_unsigned_short(Register dst, Address src) {
1902 int off = offset();
1903 ldrh(dst, src);
1904 return off;
1905 }
1906
1907 int MacroAssembler::load_unsigned_byte(Register dst, Address src) {
1908 int off = offset();
1909 ldrb(dst, src);
1910 return off;
1911 }
1912
1913 int MacroAssembler::load_signed_short(Register dst, Address src) {
1914 int off = offset();
1915 ldrsh(dst, src);
1916 return off;
1917 }
1918
1919 int MacroAssembler::load_signed_byte(Register dst, Address src) {
1920 int off = offset();
1921 ldrsb(dst, src);
1922 return off;
1923 }
1924
1925 int MacroAssembler::load_signed_short32(Register dst, Address src) {
1926 int off = offset();
1927 ldrshw(dst, src);
1928 return off;
1929 }
1930
1931 int MacroAssembler::load_signed_byte32(Register dst, Address src) {
1932 int off = offset();
1933 ldrsbw(dst, src);
1934 return off;
1935 }
1936
1937 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) {
1938 switch (size_in_bytes) {
1939 case 8: ldr(dst, src); break;
1940 case 4: ldrw(dst, src); break;
1941 case 2: is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break;
1942 case 1: is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break;
1943 default: ShouldNotReachHere();
1944 }
1945 }
1946
1947 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2) {
1948 switch (size_in_bytes) {
1949 case 8: str(src, dst); break;
1950 case 4: strw(src, dst); break;
1951 case 2: strh(src, dst); break;
1952 case 1: strb(src, dst); break;
1953 default: ShouldNotReachHere();
1954 }
1955 }
1956
1957 void MacroAssembler::decrementw(Register reg, int value)
1958 {
1959 if (value < 0) { incrementw(reg, -value); return; }
1960 if (value == 0) { return; }
1961 if (value < (1 << 12)) { subw(reg, reg, value); return; }
1962 /* else */ {
1963 guarantee(reg != rscratch2, "invalid dst for register decrement");
1964 movw(rscratch2, (unsigned)value);
1965 subw(reg, reg, rscratch2);
1966 }
1967 }
1968
1969 void MacroAssembler::decrement(Register reg, int value)
1970 {
1971 if (value < 0) { increment(reg, -value); return; }
1972 if (value == 0) { return; }
1973 if (value < (1 << 12)) { sub(reg, reg, value); return; }
1974 /* else */ {
1975 assert(reg != rscratch2, "invalid dst for register decrement");
1976 mov(rscratch2, (unsigned long)value);
1977 sub(reg, reg, rscratch2);
1978 }
1979 }
1980
1981 void MacroAssembler::decrementw(Address dst, int value)
1982 {
1983 assert(!dst.uses(rscratch1), "invalid dst for address decrement");
1984 if (dst.getMode() == Address::literal) {
1985 assert(abs(value) < (1 << 12), "invalid value and address mode combination");
1986 lea(rscratch2, dst);
1987 dst = Address(rscratch2);
1988 }
1989 ldrw(rscratch1, dst);
1990 decrementw(rscratch1, value);
1991 strw(rscratch1, dst);
1992 }
1993
1994 void MacroAssembler::decrement(Address dst, int value)
1995 {
1996 assert(!dst.uses(rscratch1), "invalid address for decrement");
1997 if (dst.getMode() == Address::literal) {
1998 assert(abs(value) < (1 << 12), "invalid value and address mode combination");
1999 lea(rscratch2, dst);
2000 dst = Address(rscratch2);
2001 }
2002 ldr(rscratch1, dst);
2003 decrement(rscratch1, value);
2004 str(rscratch1, dst);
2005 }
2006
2007 void MacroAssembler::incrementw(Register reg, int value)
2008 {
2009 if (value < 0) { decrementw(reg, -value); return; }
2010 if (value == 0) { return; }
2011 if (value < (1 << 12)) { addw(reg, reg, value); return; }
2012 /* else */ {
2013 assert(reg != rscratch2, "invalid dst for register increment");
2014 movw(rscratch2, (unsigned)value);
2015 addw(reg, reg, rscratch2);
2016 }
2017 }
2018
2019 void MacroAssembler::increment(Register reg, int value)
2020 {
2021 if (value < 0) { decrement(reg, -value); return; }
2022 if (value == 0) { return; }
2023 if (value < (1 << 12)) { add(reg, reg, value); return; }
2024 /* else */ {
2025 assert(reg != rscratch2, "invalid dst for register increment");
2026 movw(rscratch2, (unsigned)value);
2027 add(reg, reg, rscratch2);
2028 }
2029 }
2030
2031 void MacroAssembler::incrementw(Address dst, int value)
2032 {
2033 assert(!dst.uses(rscratch1), "invalid dst for address increment");
2034 if (dst.getMode() == Address::literal) {
2035 assert(abs(value) < (1 << 12), "invalid value and address mode combination");
2036 lea(rscratch2, dst);
2037 dst = Address(rscratch2);
2038 }
2039 ldrw(rscratch1, dst);
2040 incrementw(rscratch1, value);
2041 strw(rscratch1, dst);
2042 }
2043
2044 void MacroAssembler::increment(Address dst, int value)
2045 {
2046 assert(!dst.uses(rscratch1), "invalid dst for address increment");
2047 if (dst.getMode() == Address::literal) {
2048 assert(abs(value) < (1 << 12), "invalid value and address mode combination");
2049 lea(rscratch2, dst);
2050 dst = Address(rscratch2);
2051 }
2052 ldr(rscratch1, dst);
2053 increment(rscratch1, value);
2054 str(rscratch1, dst);
2055 }
2056
2057
2058 void MacroAssembler::pusha() {
2059 push(0x7fffffff, sp);
2060 }
2061
2062 void MacroAssembler::popa() {
2063 pop(0x7fffffff, sp);
2064 }
2065
2066 // Push lots of registers in the bit set supplied. Don't push sp.
2067 // Return the number of words pushed
2068 int MacroAssembler::push(unsigned int bitset, Register stack) {
2069 int words_pushed = 0;
2070
2071 // Scan bitset to accumulate register pairs
2072 unsigned char regs[32];
2073 int count = 0;
2074 for (int reg = 0; reg <= 30; reg++) {
2075 if (1 & bitset)
2076 regs[count++] = reg;
2077 bitset >>= 1;
2078 }
2079 regs[count++] = zr->encoding_nocheck();
2080 count &= ~1; // Only push an even nuber of regs
2081
2082 if (count) {
2083 stp(as_Register(regs[0]), as_Register(regs[1]),
2084 Address(pre(stack, -count * wordSize)));
2085 words_pushed += 2;
2086 }
2087 for (int i = 2; i < count; i += 2) {
2088 stp(as_Register(regs[i]), as_Register(regs[i+1]),
2089 Address(stack, i * wordSize));
2090 words_pushed += 2;
2091 }
2092
2093 assert(words_pushed == count, "oops, pushed != count");
2094
2095 return count;
2096 }
2097
2098 int MacroAssembler::pop(unsigned int bitset, Register stack) {
2099 int words_pushed = 0;
2100
2101 // Scan bitset to accumulate register pairs
2102 unsigned char regs[32];
2103 int count = 0;
2104 for (int reg = 0; reg <= 30; reg++) {
2105 if (1 & bitset)
2106 regs[count++] = reg;
2107 bitset >>= 1;
2108 }
2109 regs[count++] = zr->encoding_nocheck();
2110 count &= ~1;
2111
2112 for (int i = 2; i < count; i += 2) {
2113 ldp(as_Register(regs[i]), as_Register(regs[i+1]),
2114 Address(stack, i * wordSize));
2115 words_pushed += 2;
2116 }
2117 if (count) {
2118 ldp(as_Register(regs[0]), as_Register(regs[1]),
2119 Address(post(stack, count * wordSize)));
2120 words_pushed += 2;
2121 }
2122
2123 assert(words_pushed == count, "oops, pushed != count");
2124
2125 return count;
2126 }
2127 #ifdef ASSERT
2128 void MacroAssembler::verify_heapbase(const char* msg) {
2129 #if 0
2130 assert (UseCompressedOops || UseCompressedClassPointers, "should be compressed");
2131 assert (Universe::heap() != NULL, "java heap should be initialized");
2132 if (CheckCompressedOops) {
2133 Label ok;
2134 push(1 << rscratch1->encoding(), sp); // cmpptr trashes rscratch1
2135 cmpptr(rheapbase, ExternalAddress((address)CompressedOops::ptrs_base_addr()));
2136 br(Assembler::EQ, ok);
2137 stop(msg);
2138 bind(ok);
2139 pop(1 << rscratch1->encoding(), sp);
2140 }
2141 #endif
2142 }
2143 #endif
2144
2145 void MacroAssembler::resolve_jobject(Register value, Register thread, Register tmp) {
2146 Label done, not_weak;
2147 cbz(value, done); // Use NULL as-is.
2148
2149 STATIC_ASSERT(JNIHandles::weak_tag_mask == 1u);
2150 tbz(r0, 0, not_weak); // Test for jweak tag.
2151
2152 // Resolve jweak.
2153 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, value,
2154 Address(value, -JNIHandles::weak_tag_value), tmp, thread);
2155 verify_oop(value);
2156 b(done);
2157
2158 bind(not_weak);
2159 // Resolve (untagged) jobject.
2160 access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, 0), tmp, thread);
2161 verify_oop(value);
2162 bind(done);
2163 }
2164
2165 void MacroAssembler::stop(const char* msg) {
2166 address ip = pc();
2167 pusha();
2168 mov(c_rarg0, (address)msg);
2169 mov(c_rarg1, (address)ip);
2170 mov(c_rarg2, sp);
2171 mov(c_rarg3, CAST_FROM_FN_PTR(address, MacroAssembler::debug64));
2172 // call(c_rarg3);
2173 blrt(c_rarg3, 3, 0, 1);
2174 hlt(0);
2175 }
2176
2177 void MacroAssembler::warn(const char* msg) {
2178 pusha();
2179 mov(c_rarg0, (address)msg);
2180 mov(lr, CAST_FROM_FN_PTR(address, warning));
2181 blrt(lr, 1, 0, MacroAssembler::ret_type_void);
2182 popa();
2183 }
2184
2185 void MacroAssembler::unimplemented(const char* what) {
2186 const char* buf = NULL;
2187 {
2188 ResourceMark rm;
2189 stringStream ss;
2190 ss.print("unimplemented: %s", what);
2191 buf = code_string(ss.as_string());
2192 }
2193 stop(buf);
2194 }
2195
2196 // If a constant does not fit in an immediate field, generate some
2197 // number of MOV instructions and then perform the operation.
2198 void MacroAssembler::wrap_add_sub_imm_insn(Register Rd, Register Rn, unsigned imm,
2199 add_sub_imm_insn insn1,
2200 add_sub_reg_insn insn2) {
2201 assert(Rd != zr, "Rd = zr and not setting flags?");
2202 if (operand_valid_for_add_sub_immediate((int)imm)) {
2203 (this->*insn1)(Rd, Rn, imm);
2204 } else {
2205 if (uabs(imm) < (1 << 24)) {
2206 (this->*insn1)(Rd, Rn, imm & -(1 << 12));
2207 (this->*insn1)(Rd, Rd, imm & ((1 << 12)-1));
2208 } else {
2209 assert_different_registers(Rd, Rn);
2210 mov(Rd, (uint64_t)imm);
2211 (this->*insn2)(Rd, Rn, Rd, LSL, 0);
2212 }
2213 }
2214 }
2215
2216 // Seperate vsn which sets the flags. Optimisations are more restricted
2217 // because we must set the flags correctly.
2218 void MacroAssembler::wrap_adds_subs_imm_insn(Register Rd, Register Rn, unsigned imm,
2219 add_sub_imm_insn insn1,
2220 add_sub_reg_insn insn2) {
2221 if (operand_valid_for_add_sub_immediate((int)imm)) {
2222 (this->*insn1)(Rd, Rn, imm);
2223 } else {
2224 assert_different_registers(Rd, Rn);
2225 assert(Rd != zr, "overflow in immediate operand");
2226 mov(Rd, (uint64_t)imm);
2227 (this->*insn2)(Rd, Rn, Rd, LSL, 0);
2228 }
2229 }
2230
2231
2232 void MacroAssembler::add(Register Rd, Register Rn, RegisterOrConstant increment) {
2233 if (increment.is_register()) {
2234 add(Rd, Rn, increment.as_register());
2235 } else {
2236 add(Rd, Rn, increment.as_constant());
2237 }
2238 }
2239
2240 void MacroAssembler::addw(Register Rd, Register Rn, RegisterOrConstant increment) {
2241 if (increment.is_register()) {
2242 addw(Rd, Rn, increment.as_register());
2243 } else {
2244 addw(Rd, Rn, increment.as_constant());
2245 }
2246 }
2247
2248 void MacroAssembler::sub(Register Rd, Register Rn, RegisterOrConstant decrement) {
2249 if (decrement.is_register()) {
2250 sub(Rd, Rn, decrement.as_register());
2251 } else {
2252 sub(Rd, Rn, decrement.as_constant());
2253 }
2254 }
2255
2256 void MacroAssembler::subw(Register Rd, Register Rn, RegisterOrConstant decrement) {
2257 if (decrement.is_register()) {
2258 subw(Rd, Rn, decrement.as_register());
2259 } else {
2260 subw(Rd, Rn, decrement.as_constant());
2261 }
2262 }
2263
2264 void MacroAssembler::reinit_heapbase()
2265 {
2266 if (UseCompressedOops) {
2267 if (Universe::is_fully_initialized()) {
2268 mov(rheapbase, CompressedOops::ptrs_base());
2269 } else {
2270 lea(rheapbase, ExternalAddress((address)CompressedOops::ptrs_base_addr()));
2271 ldr(rheapbase, Address(rheapbase));
2272 }
2273 }
2274 }
2275
2276 // this simulates the behaviour of the x86 cmpxchg instruction using a
2277 // load linked/store conditional pair. we use the acquire/release
2278 // versions of these instructions so that we flush pending writes as
2279 // per Java semantics.
2280
2281 // n.b the x86 version assumes the old value to be compared against is
2282 // in rax and updates rax with the value located in memory if the
2283 // cmpxchg fails. we supply a register for the old value explicitly
2284
2285 // the aarch64 load linked/store conditional instructions do not
2286 // accept an offset. so, unlike x86, we must provide a plain register
2287 // to identify the memory word to be compared/exchanged rather than a
2288 // register+offset Address.
2289
2290 void MacroAssembler::cmpxchgptr(Register oldv, Register newv, Register addr, Register tmp,
2291 Label &succeed, Label *fail) {
2292 // oldv holds comparison value
2293 // newv holds value to write in exchange
2294 // addr identifies memory word to compare against/update
2295 if (UseLSE) {
2296 mov(tmp, oldv);
2297 casal(Assembler::xword, oldv, newv, addr);
2298 cmp(tmp, oldv);
2299 br(Assembler::EQ, succeed);
2300 membar(AnyAny);
2301 } else {
2302 Label retry_load, nope;
2303 if ((VM_Version::features() & VM_Version::CPU_STXR_PREFETCH))
2304 prfm(Address(addr), PSTL1STRM);
2305 bind(retry_load);
2306 // flush and load exclusive from the memory location
2307 // and fail if it is not what we expect
2308 ldaxr(tmp, addr);
2309 cmp(tmp, oldv);
2310 br(Assembler::NE, nope);
2311 // if we store+flush with no intervening write tmp wil be zero
2312 stlxr(tmp, newv, addr);
2313 cbzw(tmp, succeed);
2314 // retry so we only ever return after a load fails to compare
2315 // ensures we don't return a stale value after a failed write.
2316 b(retry_load);
2317 // if the memory word differs we return it in oldv and signal a fail
2318 bind(nope);
2319 membar(AnyAny);
2320 mov(oldv, tmp);
2321 }
2322 if (fail)
2323 b(*fail);
2324 }
2325
2326 void MacroAssembler::cmpxchg_obj_header(Register oldv, Register newv, Register obj, Register tmp,
2327 Label &succeed, Label *fail) {
2328 assert(oopDesc::mark_offset_in_bytes() == 0, "assumption");
2329 cmpxchgptr(oldv, newv, obj, tmp, succeed, fail);
2330 }
2331
2332 void MacroAssembler::cmpxchgw(Register oldv, Register newv, Register addr, Register tmp,
2333 Label &succeed, Label *fail) {
2334 // oldv holds comparison value
2335 // newv holds value to write in exchange
2336 // addr identifies memory word to compare against/update
2337 // tmp returns 0/1 for success/failure
2338 if (UseLSE) {
2339 mov(tmp, oldv);
2340 casal(Assembler::word, oldv, newv, addr);
2341 cmp(tmp, oldv);
2342 br(Assembler::EQ, succeed);
2343 membar(AnyAny);
2344 } else {
2345 Label retry_load, nope;
2346 if ((VM_Version::features() & VM_Version::CPU_STXR_PREFETCH))
2347 prfm(Address(addr), PSTL1STRM);
2348 bind(retry_load);
2349 // flush and load exclusive from the memory location
2350 // and fail if it is not what we expect
2351 ldaxrw(tmp, addr);
2352 cmp(tmp, oldv);
2353 br(Assembler::NE, nope);
2354 // if we store+flush with no intervening write tmp wil be zero
2355 stlxrw(tmp, newv, addr);
2356 cbzw(tmp, succeed);
2357 // retry so we only ever return after a load fails to compare
2358 // ensures we don't return a stale value after a failed write.
2359 b(retry_load);
2360 // if the memory word differs we return it in oldv and signal a fail
2361 bind(nope);
2362 membar(AnyAny);
2363 mov(oldv, tmp);
2364 }
2365 if (fail)
2366 b(*fail);
2367 }
2368
2369 // A generic CAS; success or failure is in the EQ flag. A weak CAS
2370 // doesn't retry and may fail spuriously. If the oldval is wanted,
2371 // Pass a register for the result, otherwise pass noreg.
2372
2373 // Clobbers rscratch1
2374 void MacroAssembler::cmpxchg(Register addr, Register expected,
2375 Register new_val,
2376 enum operand_size size,
2377 bool acquire, bool release,
2378 bool weak,
2379 Register result) {
2380 if (result == noreg) result = rscratch1;
2381 BLOCK_COMMENT("cmpxchg {");
2382 if (UseLSE) {
2383 mov(result, expected);
2384 lse_cas(result, new_val, addr, size, acquire, release, /*not_pair*/ true);
2385 compare_eq(result, expected, size);
2386 } else {
2387 Label retry_load, done;
2388 if ((VM_Version::features() & VM_Version::CPU_STXR_PREFETCH))
2389 prfm(Address(addr), PSTL1STRM);
2390 bind(retry_load);
2391 load_exclusive(result, addr, size, acquire);
2392 compare_eq(result, expected, size);
2393 br(Assembler::NE, done);
2394 store_exclusive(rscratch1, new_val, addr, size, release);
2395 if (weak) {
2396 cmpw(rscratch1, 0u); // If the store fails, return NE to our caller.
2397 } else {
2398 cbnzw(rscratch1, retry_load);
2399 }
2400 bind(done);
2401 }
2402 BLOCK_COMMENT("} cmpxchg");
2403 }
2404
2405 // A generic comparison. Only compares for equality, clobbers rscratch1.
2406 void MacroAssembler::compare_eq(Register rm, Register rn, enum operand_size size) {
2407 if (size == xword) {
2408 cmp(rm, rn);
2409 } else if (size == word) {
2410 cmpw(rm, rn);
2411 } else if (size == halfword) {
2412 eorw(rscratch1, rm, rn);
2413 ands(zr, rscratch1, 0xffff);
2414 } else if (size == byte) {
2415 eorw(rscratch1, rm, rn);
2416 ands(zr, rscratch1, 0xff);
2417 } else {
2418 ShouldNotReachHere();
2419 }
2420 }
2421
2422
2423 static bool different(Register a, RegisterOrConstant b, Register c) {
2424 if (b.is_constant())
2425 return a != c;
2426 else
2427 return a != b.as_register() && a != c && b.as_register() != c;
2428 }
2429
2430 #define ATOMIC_OP(NAME, LDXR, OP, IOP, AOP, STXR, sz) \
2431 void MacroAssembler::atomic_##NAME(Register prev, RegisterOrConstant incr, Register addr) { \
2432 if (UseLSE) { \
2433 prev = prev->is_valid() ? prev : zr; \
2434 if (incr.is_register()) { \
2435 AOP(sz, incr.as_register(), prev, addr); \
2436 } else { \
2437 mov(rscratch2, incr.as_constant()); \
2438 AOP(sz, rscratch2, prev, addr); \
2439 } \
2440 return; \
2441 } \
2442 Register result = rscratch2; \
2443 if (prev->is_valid()) \
2444 result = different(prev, incr, addr) ? prev : rscratch2; \
2445 \
2446 Label retry_load; \
2447 if ((VM_Version::features() & VM_Version::CPU_STXR_PREFETCH)) \
2448 prfm(Address(addr), PSTL1STRM); \
2449 bind(retry_load); \
2450 LDXR(result, addr); \
2451 OP(rscratch1, result, incr); \
2452 STXR(rscratch2, rscratch1, addr); \
2453 cbnzw(rscratch2, retry_load); \
2454 if (prev->is_valid() && prev != result) { \
2455 IOP(prev, rscratch1, incr); \
2456 } \
2457 }
2458
2459 ATOMIC_OP(add, ldxr, add, sub, ldadd, stxr, Assembler::xword)
2460 ATOMIC_OP(addw, ldxrw, addw, subw, ldadd, stxrw, Assembler::word)
2461 ATOMIC_OP(addal, ldaxr, add, sub, ldaddal, stlxr, Assembler::xword)
2462 ATOMIC_OP(addalw, ldaxrw, addw, subw, ldaddal, stlxrw, Assembler::word)
2463
2464 #undef ATOMIC_OP
2465
2466 #define ATOMIC_XCHG(OP, AOP, LDXR, STXR, sz) \
2467 void MacroAssembler::atomic_##OP(Register prev, Register newv, Register addr) { \
2468 if (UseLSE) { \
2469 prev = prev->is_valid() ? prev : zr; \
2470 AOP(sz, newv, prev, addr); \
2471 return; \
2472 } \
2473 Register result = rscratch2; \
2474 if (prev->is_valid()) \
2475 result = different(prev, newv, addr) ? prev : rscratch2; \
2476 \
2477 Label retry_load; \
2478 if ((VM_Version::features() & VM_Version::CPU_STXR_PREFETCH)) \
2479 prfm(Address(addr), PSTL1STRM); \
2480 bind(retry_load); \
2481 LDXR(result, addr); \
2482 STXR(rscratch1, newv, addr); \
2483 cbnzw(rscratch1, retry_load); \
2484 if (prev->is_valid() && prev != result) \
2485 mov(prev, result); \
2486 }
2487
2488 ATOMIC_XCHG(xchg, swp, ldxr, stxr, Assembler::xword)
2489 ATOMIC_XCHG(xchgw, swp, ldxrw, stxrw, Assembler::word)
2490 ATOMIC_XCHG(xchgal, swpal, ldaxr, stlxr, Assembler::xword)
2491 ATOMIC_XCHG(xchgalw, swpal, ldaxrw, stlxrw, Assembler::word)
2492
2493 #undef ATOMIC_XCHG
2494
2495 #ifndef PRODUCT
2496 extern "C" void findpc(intptr_t x);
2497 #endif
2498
2499 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[])
2500 {
2501 // In order to get locks to work, we need to fake a in_VM state
2502 if (ShowMessageBoxOnError ) {
2503 JavaThread* thread = JavaThread::current();
2504 JavaThreadState saved_state = thread->thread_state();
2505 thread->set_thread_state(_thread_in_vm);
2506 #ifndef PRODUCT
2507 if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
2508 ttyLocker ttyl;
2509 BytecodeCounter::print();
2510 }
2511 #endif
2512 if (os::message_box(msg, "Execution stopped, print registers?")) {
2513 ttyLocker ttyl;
2514 tty->print_cr(" pc = 0x%016lx", pc);
2515 #ifndef PRODUCT
2516 tty->cr();
2517 findpc(pc);
2518 tty->cr();
2519 #endif
2520 tty->print_cr(" r0 = 0x%016lx", regs[0]);
2521 tty->print_cr(" r1 = 0x%016lx", regs[1]);
2522 tty->print_cr(" r2 = 0x%016lx", regs[2]);
2523 tty->print_cr(" r3 = 0x%016lx", regs[3]);
2524 tty->print_cr(" r4 = 0x%016lx", regs[4]);
2525 tty->print_cr(" r5 = 0x%016lx", regs[5]);
2526 tty->print_cr(" r6 = 0x%016lx", regs[6]);
2527 tty->print_cr(" r7 = 0x%016lx", regs[7]);
2528 tty->print_cr(" r8 = 0x%016lx", regs[8]);
2529 tty->print_cr(" r9 = 0x%016lx", regs[9]);
2530 tty->print_cr("r10 = 0x%016lx", regs[10]);
2531 tty->print_cr("r11 = 0x%016lx", regs[11]);
2532 tty->print_cr("r12 = 0x%016lx", regs[12]);
2533 tty->print_cr("r13 = 0x%016lx", regs[13]);
2534 tty->print_cr("r14 = 0x%016lx", regs[14]);
2535 tty->print_cr("r15 = 0x%016lx", regs[15]);
2536 tty->print_cr("r16 = 0x%016lx", regs[16]);
2537 tty->print_cr("r17 = 0x%016lx", regs[17]);
2538 tty->print_cr("r18 = 0x%016lx", regs[18]);
2539 tty->print_cr("r19 = 0x%016lx", regs[19]);
2540 tty->print_cr("r20 = 0x%016lx", regs[20]);
2541 tty->print_cr("r21 = 0x%016lx", regs[21]);
2542 tty->print_cr("r22 = 0x%016lx", regs[22]);
2543 tty->print_cr("r23 = 0x%016lx", regs[23]);
2544 tty->print_cr("r24 = 0x%016lx", regs[24]);
2545 tty->print_cr("r25 = 0x%016lx", regs[25]);
2546 tty->print_cr("r26 = 0x%016lx", regs[26]);
2547 tty->print_cr("r27 = 0x%016lx", regs[27]);
2548 tty->print_cr("r28 = 0x%016lx", regs[28]);
2549 tty->print_cr("r30 = 0x%016lx", regs[30]);
2550 tty->print_cr("r31 = 0x%016lx", regs[31]);
2551 BREAKPOINT;
2552 }
2553 ThreadStateTransition::transition(thread, _thread_in_vm, saved_state);
2554 } else {
2555 ttyLocker ttyl;
2556 ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n",
2557 msg);
2558 assert(false, "DEBUG MESSAGE: %s", msg);
2559 }
2560 }
2561
2562 #ifdef BUILTIN_SIM
2563 // routine to generate an x86 prolog for a stub function which
2564 // bootstraps into the generated ARM code which directly follows the
2565 // stub
2566 //
2567 // the argument encodes the number of general and fp registers
2568 // passed by the caller and the callng convention (currently just
2569 // the number of general registers and assumes C argument passing)
2570
2571 extern "C" {
2572 int aarch64_stub_prolog_size();
2573 void aarch64_stub_prolog();
2574 void aarch64_prolog();
2575 }
2576
2577 void MacroAssembler::c_stub_prolog(int gp_arg_count, int fp_arg_count, int ret_type,
2578 address *prolog_ptr)
2579 {
2580 int calltype = (((ret_type & 0x3) << 8) |
2581 ((fp_arg_count & 0xf) << 4) |
2582 (gp_arg_count & 0xf));
2583
2584 // the addresses for the x86 to ARM entry code we need to use
2585 address start = pc();
2586 // printf("start = %lx\n", start);
2587 int byteCount = aarch64_stub_prolog_size();
2588 // printf("byteCount = %x\n", byteCount);
2589 int instructionCount = (byteCount + 3)/ 4;
2590 // printf("instructionCount = %x\n", instructionCount);
2591 for (int i = 0; i < instructionCount; i++) {
2592 nop();
2593 }
2594
2595 memcpy(start, (void*)aarch64_stub_prolog, byteCount);
2596
2597 // write the address of the setup routine and the call format at the
2598 // end of into the copied code
2599 u_int64_t *patch_end = (u_int64_t *)(start + byteCount);
2600 if (prolog_ptr)
2601 patch_end[-2] = (u_int64_t)prolog_ptr;
2602 patch_end[-1] = calltype;
2603 }
2604 #endif
2605
2606 void MacroAssembler::push_call_clobbered_registers() {
2607 int step = 4 * wordSize;
2608 push(RegSet::range(r0, r18) - RegSet::of(rscratch1, rscratch2), sp);
2609 sub(sp, sp, step);
2610 mov(rscratch1, -step);
2611 // Push v0-v7, v16-v31.
2612 for (int i = 31; i>= 4; i -= 4) {
2613 if (i <= v7->encoding() || i >= v16->encoding())
2614 st1(as_FloatRegister(i-3), as_FloatRegister(i-2), as_FloatRegister(i-1),
2615 as_FloatRegister(i), T1D, Address(post(sp, rscratch1)));
2616 }
2617 st1(as_FloatRegister(0), as_FloatRegister(1), as_FloatRegister(2),
2618 as_FloatRegister(3), T1D, Address(sp));
2619 }
2620
2621 void MacroAssembler::pop_call_clobbered_registers() {
2622 for (int i = 0; i < 32; i += 4) {
2623 if (i <= v7->encoding() || i >= v16->encoding())
2624 ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
2625 as_FloatRegister(i+3), T1D, Address(post(sp, 4 * wordSize)));
2626 }
2627
2628 pop(RegSet::range(r0, r18) - RegSet::of(rscratch1, rscratch2), sp);
2629 }
2630
2631 void MacroAssembler::push_CPU_state(bool save_vectors) {
2632 int step = (save_vectors ? 8 : 4) * wordSize;
2633 push(0x3fffffff, sp); // integer registers except lr & sp
2634 mov(rscratch1, -step);
2635 sub(sp, sp, step);
2636 for (int i = 28; i >= 4; i -= 4) {
2637 st1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
2638 as_FloatRegister(i+3), save_vectors ? T2D : T1D, Address(post(sp, rscratch1)));
2639 }
2640 st1(v0, v1, v2, v3, save_vectors ? T2D : T1D, sp);
2641 }
2642
2643 void MacroAssembler::pop_CPU_state(bool restore_vectors) {
2644 int step = (restore_vectors ? 8 : 4) * wordSize;
2645 for (int i = 0; i <= 28; i += 4)
2646 ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
2647 as_FloatRegister(i+3), restore_vectors ? T2D : T1D, Address(post(sp, step)));
2648 pop(0x3fffffff, sp); // integer registers except lr & sp
2649 }
2650
2651 /**
2652 * Helpers for multiply_to_len().
2653 */
2654 void MacroAssembler::add2_with_carry(Register final_dest_hi, Register dest_hi, Register dest_lo,
2655 Register src1, Register src2) {
2656 adds(dest_lo, dest_lo, src1);
2657 adc(dest_hi, dest_hi, zr);
2658 adds(dest_lo, dest_lo, src2);
2659 adc(final_dest_hi, dest_hi, zr);
2660 }
2661
2662 // Generate an address from (r + r1 extend offset). "size" is the
2663 // size of the operand. The result may be in rscratch2.
2664 Address MacroAssembler::offsetted_address(Register r, Register r1,
2665 Address::extend ext, int offset, int size) {
2666 if (offset || (ext.shift() % size != 0)) {
2667 lea(rscratch2, Address(r, r1, ext));
2668 return Address(rscratch2, offset);
2669 } else {
2670 return Address(r, r1, ext);
2671 }
2672 }
2673
2674 Address MacroAssembler::spill_address(int size, int offset, Register tmp)
2675 {
2676 assert(offset >= 0, "spill to negative address?");
2677 // Offset reachable ?
2678 // Not aligned - 9 bits signed offset
2679 // Aligned - 12 bits unsigned offset shifted
2680 Register base = sp;
2681 if ((offset & (size-1)) && offset >= (1<<8)) {
2682 add(tmp, base, offset & ((1<<12)-1));
2683 base = tmp;
2684 offset &= -1u<<12;
2685 }
2686
2687 if (offset >= (1<<12) * size) {
2688 add(tmp, base, offset & (((1<<12)-1)<<12));
2689 base = tmp;
2690 offset &= ~(((1<<12)-1)<<12);
2691 }
2692
2693 return Address(base, offset);
2694 }
2695
2696 // Checks whether offset is aligned.
2697 // Returns true if it is, else false.
2698 bool MacroAssembler::merge_alignment_check(Register base,
2699 size_t size,
2700 long cur_offset,
2701 long prev_offset) const {
2702 if (AvoidUnalignedAccesses) {
2703 if (base == sp) {
2704 // Checks whether low offset if aligned to pair of registers.
2705 long pair_mask = size * 2 - 1;
2706 long offset = prev_offset > cur_offset ? cur_offset : prev_offset;
2707 return (offset & pair_mask) == 0;
2708 } else { // If base is not sp, we can't guarantee the access is aligned.
2709 return false;
2710 }
2711 } else {
2712 long mask = size - 1;
2713 // Load/store pair instruction only supports element size aligned offset.
2714 return (cur_offset & mask) == 0 && (prev_offset & mask) == 0;
2715 }
2716 }
2717
2718 // Checks whether current and previous loads/stores can be merged.
2719 // Returns true if it can be merged, else false.
2720 bool MacroAssembler::ldst_can_merge(Register rt,
2721 const Address &adr,
2722 size_t cur_size_in_bytes,
2723 bool is_store) const {
2724 address prev = pc() - NativeInstruction::instruction_size;
2725 address last = code()->last_insn();
2726
2727 if (last == NULL || !nativeInstruction_at(last)->is_Imm_LdSt()) {
2728 return false;
2729 }
2730
2731 if (adr.getMode() != Address::base_plus_offset || prev != last) {
2732 return false;
2733 }
2734
2735 NativeLdSt* prev_ldst = NativeLdSt_at(prev);
2736 size_t prev_size_in_bytes = prev_ldst->size_in_bytes();
2737
2738 assert(prev_size_in_bytes == 4 || prev_size_in_bytes == 8, "only supports 64/32bit merging.");
2739 assert(cur_size_in_bytes == 4 || cur_size_in_bytes == 8, "only supports 64/32bit merging.");
2740
2741 if (cur_size_in_bytes != prev_size_in_bytes || is_store != prev_ldst->is_store()) {
2742 return false;
2743 }
2744
2745 long max_offset = 63 * prev_size_in_bytes;
2746 long min_offset = -64 * prev_size_in_bytes;
2747
2748 assert(prev_ldst->is_not_pre_post_index(), "pre-index or post-index is not supported to be merged.");
2749
2750 // Only same base can be merged.
2751 if (adr.base() != prev_ldst->base()) {
2752 return false;
2753 }
2754
2755 long cur_offset = adr.offset();
2756 long prev_offset = prev_ldst->offset();
2757 size_t diff = abs(cur_offset - prev_offset);
2758 if (diff != prev_size_in_bytes) {
2759 return false;
2760 }
2761
2762 // Following cases can not be merged:
2763 // ldr x2, [x2, #8]
2764 // ldr x3, [x2, #16]
2765 // or:
2766 // ldr x2, [x3, #8]
2767 // ldr x2, [x3, #16]
2768 // If t1 and t2 is the same in "ldp t1, t2, [xn, #imm]", we'll get SIGILL.
2769 if (!is_store && (adr.base() == prev_ldst->target() || rt == prev_ldst->target())) {
2770 return false;
2771 }
2772
2773 long low_offset = prev_offset > cur_offset ? cur_offset : prev_offset;
2774 // Offset range must be in ldp/stp instruction's range.
2775 if (low_offset > max_offset || low_offset < min_offset) {
2776 return false;
2777 }
2778
2779 if (merge_alignment_check(adr.base(), prev_size_in_bytes, cur_offset, prev_offset)) {
2780 return true;
2781 }
2782
2783 return false;
2784 }
2785
2786 // Merge current load/store with previous load/store into ldp/stp.
2787 void MacroAssembler::merge_ldst(Register rt,
2788 const Address &adr,
2789 size_t cur_size_in_bytes,
2790 bool is_store) {
2791
2792 assert(ldst_can_merge(rt, adr, cur_size_in_bytes, is_store) == true, "cur and prev must be able to be merged.");
2793
2794 Register rt_low, rt_high;
2795 address prev = pc() - NativeInstruction::instruction_size;
2796 NativeLdSt* prev_ldst = NativeLdSt_at(prev);
2797
2798 long offset;
2799
2800 if (adr.offset() < prev_ldst->offset()) {
2801 offset = adr.offset();
2802 rt_low = rt;
2803 rt_high = prev_ldst->target();
2804 } else {
2805 offset = prev_ldst->offset();
2806 rt_low = prev_ldst->target();
2807 rt_high = rt;
2808 }
2809
2810 Address adr_p = Address(prev_ldst->base(), offset);
2811 // Overwrite previous generated binary.
2812 code_section()->set_end(prev);
2813
2814 const int sz = prev_ldst->size_in_bytes();
2815 assert(sz == 8 || sz == 4, "only supports 64/32bit merging.");
2816 if (!is_store) {
2817 BLOCK_COMMENT("merged ldr pair");
2818 if (sz == 8) {
2819 ldp(rt_low, rt_high, adr_p);
2820 } else {
2821 ldpw(rt_low, rt_high, adr_p);
2822 }
2823 } else {
2824 BLOCK_COMMENT("merged str pair");
2825 if (sz == 8) {
2826 stp(rt_low, rt_high, adr_p);
2827 } else {
2828 stpw(rt_low, rt_high, adr_p);
2829 }
2830 }
2831 }
2832
2833 /**
2834 * Multiply 64 bit by 64 bit first loop.
2835 */
2836 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
2837 Register y, Register y_idx, Register z,
2838 Register carry, Register product,
2839 Register idx, Register kdx) {
2840 //
2841 // jlong carry, x[], y[], z[];
2842 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
2843 // huge_128 product = y[idx] * x[xstart] + carry;
2844 // z[kdx] = (jlong)product;
2845 // carry = (jlong)(product >>> 64);
2846 // }
2847 // z[xstart] = carry;
2848 //
2849
2850 Label L_first_loop, L_first_loop_exit;
2851 Label L_one_x, L_one_y, L_multiply;
2852
2853 subsw(xstart, xstart, 1);
2854 br(Assembler::MI, L_one_x);
2855
2856 lea(rscratch1, Address(x, xstart, Address::lsl(LogBytesPerInt)));
2857 ldr(x_xstart, Address(rscratch1));
2858 ror(x_xstart, x_xstart, 32); // convert big-endian to little-endian
2859
2860 bind(L_first_loop);
2861 subsw(idx, idx, 1);
2862 br(Assembler::MI, L_first_loop_exit);
2863 subsw(idx, idx, 1);
2864 br(Assembler::MI, L_one_y);
2865 lea(rscratch1, Address(y, idx, Address::uxtw(LogBytesPerInt)));
2866 ldr(y_idx, Address(rscratch1));
2867 ror(y_idx, y_idx, 32); // convert big-endian to little-endian
2868 bind(L_multiply);
2869
2870 // AArch64 has a multiply-accumulate instruction that we can't use
2871 // here because it has no way to process carries, so we have to use
2872 // separate add and adc instructions. Bah.
2873 umulh(rscratch1, x_xstart, y_idx); // x_xstart * y_idx -> rscratch1:product
2874 mul(product, x_xstart, y_idx);
2875 adds(product, product, carry);
2876 adc(carry, rscratch1, zr); // x_xstart * y_idx + carry -> carry:product
2877
2878 subw(kdx, kdx, 2);
2879 ror(product, product, 32); // back to big-endian
2880 str(product, offsetted_address(z, kdx, Address::uxtw(LogBytesPerInt), 0, BytesPerLong));
2881
2882 b(L_first_loop);
2883
2884 bind(L_one_y);
2885 ldrw(y_idx, Address(y, 0));
2886 b(L_multiply);
2887
2888 bind(L_one_x);
2889 ldrw(x_xstart, Address(x, 0));
2890 b(L_first_loop);
2891
2892 bind(L_first_loop_exit);
2893 }
2894
2895 /**
2896 * Multiply 128 bit by 128. Unrolled inner loop.
2897 *
2898 */
2899 void MacroAssembler::multiply_128_x_128_loop(Register y, Register z,
2900 Register carry, Register carry2,
2901 Register idx, Register jdx,
2902 Register yz_idx1, Register yz_idx2,
2903 Register tmp, Register tmp3, Register tmp4,
2904 Register tmp6, Register product_hi) {
2905
2906 // jlong carry, x[], y[], z[];
2907 // int kdx = ystart+1;
2908 // for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
2909 // huge_128 tmp3 = (y[idx+1] * product_hi) + z[kdx+idx+1] + carry;
2910 // jlong carry2 = (jlong)(tmp3 >>> 64);
2911 // huge_128 tmp4 = (y[idx] * product_hi) + z[kdx+idx] + carry2;
2912 // carry = (jlong)(tmp4 >>> 64);
2913 // z[kdx+idx+1] = (jlong)tmp3;
2914 // z[kdx+idx] = (jlong)tmp4;
2915 // }
2916 // idx += 2;
2917 // if (idx > 0) {
2918 // yz_idx1 = (y[idx] * product_hi) + z[kdx+idx] + carry;
2919 // z[kdx+idx] = (jlong)yz_idx1;
2920 // carry = (jlong)(yz_idx1 >>> 64);
2921 // }
2922 //
2923
2924 Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
2925
2926 lsrw(jdx, idx, 2);
2927
2928 bind(L_third_loop);
2929
2930 subsw(jdx, jdx, 1);
2931 br(Assembler::MI, L_third_loop_exit);
2932 subw(idx, idx, 4);
2933
2934 lea(rscratch1, Address(y, idx, Address::uxtw(LogBytesPerInt)));
2935
2936 ldp(yz_idx2, yz_idx1, Address(rscratch1, 0));
2937
2938 lea(tmp6, Address(z, idx, Address::uxtw(LogBytesPerInt)));
2939
2940 ror(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian
2941 ror(yz_idx2, yz_idx2, 32);
2942
2943 ldp(rscratch2, rscratch1, Address(tmp6, 0));
2944
2945 mul(tmp3, product_hi, yz_idx1); // yz_idx1 * product_hi -> tmp4:tmp3
2946 umulh(tmp4, product_hi, yz_idx1);
2947
2948 ror(rscratch1, rscratch1, 32); // convert big-endian to little-endian
2949 ror(rscratch2, rscratch2, 32);
2950
2951 mul(tmp, product_hi, yz_idx2); // yz_idx2 * product_hi -> carry2:tmp
2952 umulh(carry2, product_hi, yz_idx2);
2953
2954 // propagate sum of both multiplications into carry:tmp4:tmp3
2955 adds(tmp3, tmp3, carry);
2956 adc(tmp4, tmp4, zr);
2957 adds(tmp3, tmp3, rscratch1);
2958 adcs(tmp4, tmp4, tmp);
2959 adc(carry, carry2, zr);
2960 adds(tmp4, tmp4, rscratch2);
2961 adc(carry, carry, zr);
2962
2963 ror(tmp3, tmp3, 32); // convert little-endian to big-endian
2964 ror(tmp4, tmp4, 32);
2965 stp(tmp4, tmp3, Address(tmp6, 0));
2966
2967 b(L_third_loop);
2968 bind (L_third_loop_exit);
2969
2970 andw (idx, idx, 0x3);
2971 cbz(idx, L_post_third_loop_done);
2972
2973 Label L_check_1;
2974 subsw(idx, idx, 2);
2975 br(Assembler::MI, L_check_1);
2976
2977 lea(rscratch1, Address(y, idx, Address::uxtw(LogBytesPerInt)));
2978 ldr(yz_idx1, Address(rscratch1, 0));
2979 ror(yz_idx1, yz_idx1, 32);
2980 mul(tmp3, product_hi, yz_idx1); // yz_idx1 * product_hi -> tmp4:tmp3
2981 umulh(tmp4, product_hi, yz_idx1);
2982 lea(rscratch1, Address(z, idx, Address::uxtw(LogBytesPerInt)));
2983 ldr(yz_idx2, Address(rscratch1, 0));
2984 ror(yz_idx2, yz_idx2, 32);
2985
2986 add2_with_carry(carry, tmp4, tmp3, carry, yz_idx2);
2987
2988 ror(tmp3, tmp3, 32);
2989 str(tmp3, Address(rscratch1, 0));
2990
2991 bind (L_check_1);
2992
2993 andw (idx, idx, 0x1);
2994 subsw(idx, idx, 1);
2995 br(Assembler::MI, L_post_third_loop_done);
2996 ldrw(tmp4, Address(y, idx, Address::uxtw(LogBytesPerInt)));
2997 mul(tmp3, tmp4, product_hi); // tmp4 * product_hi -> carry2:tmp3
2998 umulh(carry2, tmp4, product_hi);
2999 ldrw(tmp4, Address(z, idx, Address::uxtw(LogBytesPerInt)));
3000
3001 add2_with_carry(carry2, tmp3, tmp4, carry);
3002
3003 strw(tmp3, Address(z, idx, Address::uxtw(LogBytesPerInt)));
3004 extr(carry, carry2, tmp3, 32);
3005
3006 bind(L_post_third_loop_done);
3007 }
3008
3009 /**
3010 * Code for BigInteger::multiplyToLen() instrinsic.
3011 *
3012 * r0: x
3013 * r1: xlen
3014 * r2: y
3015 * r3: ylen
3016 * r4: z
3017 * r5: zlen
3018 * r10: tmp1
3019 * r11: tmp2
3020 * r12: tmp3
3021 * r13: tmp4
3022 * r14: tmp5
3023 * r15: tmp6
3024 * r16: tmp7
3025 *
3026 */
3027 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen,
3028 Register z, Register zlen,
3029 Register tmp1, Register tmp2, Register tmp3, Register tmp4,
3030 Register tmp5, Register tmp6, Register product_hi) {
3031
3032 assert_different_registers(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6);
3033
3034 const Register idx = tmp1;
3035 const Register kdx = tmp2;
3036 const Register xstart = tmp3;
3037
3038 const Register y_idx = tmp4;
3039 const Register carry = tmp5;
3040 const Register product = xlen;
3041 const Register x_xstart = zlen; // reuse register
3042
3043 // First Loop.
3044 //
3045 // final static long LONG_MASK = 0xffffffffL;
3046 // int xstart = xlen - 1;
3047 // int ystart = ylen - 1;
3048 // long carry = 0;
3049 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
3050 // long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry;
3051 // z[kdx] = (int)product;
3052 // carry = product >>> 32;
3053 // }
3054 // z[xstart] = (int)carry;
3055 //
3056
3057 movw(idx, ylen); // idx = ylen;
3058 movw(kdx, zlen); // kdx = xlen+ylen;
3059 mov(carry, zr); // carry = 0;
3060
3061 Label L_done;
3062
3063 movw(xstart, xlen);
3064 subsw(xstart, xstart, 1);
3065 br(Assembler::MI, L_done);
3066
3067 multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx);
3068
3069 Label L_second_loop;
3070 cbzw(kdx, L_second_loop);
3071
3072 Label L_carry;
3073 subw(kdx, kdx, 1);
3074 cbzw(kdx, L_carry);
3075
3076 strw(carry, Address(z, kdx, Address::uxtw(LogBytesPerInt)));
3077 lsr(carry, carry, 32);
3078 subw(kdx, kdx, 1);
3079
3080 bind(L_carry);
3081 strw(carry, Address(z, kdx, Address::uxtw(LogBytesPerInt)));
3082
3083 // Second and third (nested) loops.
3084 //
3085 // for (int i = xstart-1; i >= 0; i--) { // Second loop
3086 // carry = 0;
3087 // for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop
3088 // long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) +
3089 // (z[k] & LONG_MASK) + carry;
3090 // z[k] = (int)product;
3091 // carry = product >>> 32;
3092 // }
3093 // z[i] = (int)carry;
3094 // }
3095 //
3096 // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = product_hi
3097
3098 const Register jdx = tmp1;
3099
3100 bind(L_second_loop);
3101 mov(carry, zr); // carry = 0;
3102 movw(jdx, ylen); // j = ystart+1
3103
3104 subsw(xstart, xstart, 1); // i = xstart-1;
3105 br(Assembler::MI, L_done);
3106
3107 str(z, Address(pre(sp, -4 * wordSize)));
3108
3109 Label L_last_x;
3110 lea(z, offsetted_address(z, xstart, Address::uxtw(LogBytesPerInt), 4, BytesPerInt)); // z = z + k - j
3111 subsw(xstart, xstart, 1); // i = xstart-1;
3112 br(Assembler::MI, L_last_x);
3113
3114 lea(rscratch1, Address(x, xstart, Address::uxtw(LogBytesPerInt)));
3115 ldr(product_hi, Address(rscratch1));
3116 ror(product_hi, product_hi, 32); // convert big-endian to little-endian
3117
3118 Label L_third_loop_prologue;
3119 bind(L_third_loop_prologue);
3120
3121 str(ylen, Address(sp, wordSize));
3122 stp(x, xstart, Address(sp, 2 * wordSize));
3123 multiply_128_x_128_loop(y, z, carry, x, jdx, ylen, product,
3124 tmp2, x_xstart, tmp3, tmp4, tmp6, product_hi);
3125 ldp(z, ylen, Address(post(sp, 2 * wordSize)));
3126 ldp(x, xlen, Address(post(sp, 2 * wordSize))); // copy old xstart -> xlen
3127
3128 addw(tmp3, xlen, 1);
3129 strw(carry, Address(z, tmp3, Address::uxtw(LogBytesPerInt)));
3130 subsw(tmp3, tmp3, 1);
3131 br(Assembler::MI, L_done);
3132
3133 lsr(carry, carry, 32);
3134 strw(carry, Address(z, tmp3, Address::uxtw(LogBytesPerInt)));
3135 b(L_second_loop);
3136
3137 // Next infrequent code is moved outside loops.
3138 bind(L_last_x);
3139 ldrw(product_hi, Address(x, 0));
3140 b(L_third_loop_prologue);
3141
3142 bind(L_done);
3143 }
3144
3145 // Code for BigInteger::mulAdd instrinsic
3146 // out = r0
3147 // in = r1
3148 // offset = r2 (already out.length-offset)
3149 // len = r3
3150 // k = r4
3151 //
3152 // pseudo code from java implementation:
3153 // carry = 0;
3154 // offset = out.length-offset - 1;
3155 // for (int j=len-1; j >= 0; j--) {
3156 // product = (in[j] & LONG_MASK) * kLong + (out[offset] & LONG_MASK) + carry;
3157 // out[offset--] = (int)product;
3158 // carry = product >>> 32;
3159 // }
3160 // return (int)carry;
3161 void MacroAssembler::mul_add(Register out, Register in, Register offset,
3162 Register len, Register k) {
3163 Label LOOP, END;
3164 // pre-loop
3165 cmp(len, zr); // cmp, not cbz/cbnz: to use condition twice => less branches
3166 csel(out, zr, out, Assembler::EQ);
3167 br(Assembler::EQ, END);
3168 add(in, in, len, LSL, 2); // in[j+1] address
3169 add(offset, out, offset, LSL, 2); // out[offset + 1] address
3170 mov(out, zr); // used to keep carry now
3171 BIND(LOOP);
3172 ldrw(rscratch1, Address(pre(in, -4)));
3173 madd(rscratch1, rscratch1, k, out);
3174 ldrw(rscratch2, Address(pre(offset, -4)));
3175 add(rscratch1, rscratch1, rscratch2);
3176 strw(rscratch1, Address(offset));
3177 lsr(out, rscratch1, 32);
3178 subs(len, len, 1);
3179 br(Assembler::NE, LOOP);
3180 BIND(END);
3181 }
3182
3183 /**
3184 * Emits code to update CRC-32 with a byte value according to constants in table
3185 *
3186 * @param [in,out]crc Register containing the crc.
3187 * @param [in]val Register containing the byte to fold into the CRC.
3188 * @param [in]table Register containing the table of crc constants.
3189 *
3190 * uint32_t crc;
3191 * val = crc_table[(val ^ crc) & 0xFF];
3192 * crc = val ^ (crc >> 8);
3193 *
3194 */
3195 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) {
3196 eor(val, val, crc);
3197 andr(val, val, 0xff);
3198 ldrw(val, Address(table, val, Address::lsl(2)));
3199 eor(crc, val, crc, Assembler::LSR, 8);
3200 }
3201
3202 /**
3203 * Emits code to update CRC-32 with a 32-bit value according to tables 0 to 3
3204 *
3205 * @param [in,out]crc Register containing the crc.
3206 * @param [in]v Register containing the 32-bit to fold into the CRC.
3207 * @param [in]table0 Register containing table 0 of crc constants.
3208 * @param [in]table1 Register containing table 1 of crc constants.
3209 * @param [in]table2 Register containing table 2 of crc constants.
3210 * @param [in]table3 Register containing table 3 of crc constants.
3211 *
3212 * uint32_t crc;
3213 * v = crc ^ v
3214 * crc = table3[v&0xff]^table2[(v>>8)&0xff]^table1[(v>>16)&0xff]^table0[v>>24]
3215 *
3216 */
3217 void MacroAssembler::update_word_crc32(Register crc, Register v, Register tmp,
3218 Register table0, Register table1, Register table2, Register table3,
3219 bool upper) {
3220 eor(v, crc, v, upper ? LSR:LSL, upper ? 32:0);
3221 uxtb(tmp, v);
3222 ldrw(crc, Address(table3, tmp, Address::lsl(2)));
3223 ubfx(tmp, v, 8, 8);
3224 ldrw(tmp, Address(table2, tmp, Address::lsl(2)));
3225 eor(crc, crc, tmp);
3226 ubfx(tmp, v, 16, 8);
3227 ldrw(tmp, Address(table1, tmp, Address::lsl(2)));
3228 eor(crc, crc, tmp);
3229 ubfx(tmp, v, 24, 8);
3230 ldrw(tmp, Address(table0, tmp, Address::lsl(2)));
3231 eor(crc, crc, tmp);
3232 }
3233
3234 void MacroAssembler::kernel_crc32_using_crc32(Register crc, Register buf,
3235 Register len, Register tmp0, Register tmp1, Register tmp2,
3236 Register tmp3) {
3237 Label CRC_by64_loop, CRC_by4_loop, CRC_by1_loop, CRC_less64, CRC_by64_pre, CRC_by32_loop, CRC_less32, L_exit;
3238 assert_different_registers(crc, buf, len, tmp0, tmp1, tmp2, tmp3);
3239
3240 mvnw(crc, crc);
3241
3242 subs(len, len, 128);
3243 br(Assembler::GE, CRC_by64_pre);
3244 BIND(CRC_less64);
3245 adds(len, len, 128-32);
3246 br(Assembler::GE, CRC_by32_loop);
3247 BIND(CRC_less32);
3248 adds(len, len, 32-4);
3249 br(Assembler::GE, CRC_by4_loop);
3250 adds(len, len, 4);
3251 br(Assembler::GT, CRC_by1_loop);
3252 b(L_exit);
3253
3254 BIND(CRC_by32_loop);
3255 ldp(tmp0, tmp1, Address(post(buf, 16)));
3256 subs(len, len, 32);
3257 crc32x(crc, crc, tmp0);
3258 ldr(tmp2, Address(post(buf, 8)));
3259 crc32x(crc, crc, tmp1);
3260 ldr(tmp3, Address(post(buf, 8)));
3261 crc32x(crc, crc, tmp2);
3262 crc32x(crc, crc, tmp3);
3263 br(Assembler::GE, CRC_by32_loop);
3264 cmn(len, 32);
3265 br(Assembler::NE, CRC_less32);
3266 b(L_exit);
3267
3268 BIND(CRC_by4_loop);
3269 ldrw(tmp0, Address(post(buf, 4)));
3270 subs(len, len, 4);
3271 crc32w(crc, crc, tmp0);
3272 br(Assembler::GE, CRC_by4_loop);
3273 adds(len, len, 4);
3274 br(Assembler::LE, L_exit);
3275 BIND(CRC_by1_loop);
3276 ldrb(tmp0, Address(post(buf, 1)));
3277 subs(len, len, 1);
3278 crc32b(crc, crc, tmp0);
3279 br(Assembler::GT, CRC_by1_loop);
3280 b(L_exit);
3281
3282 BIND(CRC_by64_pre);
3283 sub(buf, buf, 8);
3284 ldp(tmp0, tmp1, Address(buf, 8));
3285 crc32x(crc, crc, tmp0);
3286 ldr(tmp2, Address(buf, 24));
3287 crc32x(crc, crc, tmp1);
3288 ldr(tmp3, Address(buf, 32));
3289 crc32x(crc, crc, tmp2);
3290 ldr(tmp0, Address(buf, 40));
3291 crc32x(crc, crc, tmp3);
3292 ldr(tmp1, Address(buf, 48));
3293 crc32x(crc, crc, tmp0);
3294 ldr(tmp2, Address(buf, 56));
3295 crc32x(crc, crc, tmp1);
3296 ldr(tmp3, Address(pre(buf, 64)));
3297
3298 b(CRC_by64_loop);
3299
3300 align(CodeEntryAlignment);
3301 BIND(CRC_by64_loop);
3302 subs(len, len, 64);
3303 crc32x(crc, crc, tmp2);
3304 ldr(tmp0, Address(buf, 8));
3305 crc32x(crc, crc, tmp3);
3306 ldr(tmp1, Address(buf, 16));
3307 crc32x(crc, crc, tmp0);
3308 ldr(tmp2, Address(buf, 24));
3309 crc32x(crc, crc, tmp1);
3310 ldr(tmp3, Address(buf, 32));
3311 crc32x(crc, crc, tmp2);
3312 ldr(tmp0, Address(buf, 40));
3313 crc32x(crc, crc, tmp3);
3314 ldr(tmp1, Address(buf, 48));
3315 crc32x(crc, crc, tmp0);
3316 ldr(tmp2, Address(buf, 56));
3317 crc32x(crc, crc, tmp1);
3318 ldr(tmp3, Address(pre(buf, 64)));
3319 br(Assembler::GE, CRC_by64_loop);
3320
3321 // post-loop
3322 crc32x(crc, crc, tmp2);
3323 crc32x(crc, crc, tmp3);
3324
3325 sub(len, len, 64);
3326 add(buf, buf, 8);
3327 cmn(len, 128);
3328 br(Assembler::NE, CRC_less64);
3329 BIND(L_exit);
3330 mvnw(crc, crc);
3331 }
3332
3333 /**
3334 * @param crc register containing existing CRC (32-bit)
3335 * @param buf register pointing to input byte buffer (byte*)
3336 * @param len register containing number of bytes
3337 * @param table register that will contain address of CRC table
3338 * @param tmp scratch register
3339 */
3340 void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len,
3341 Register table0, Register table1, Register table2, Register table3,
3342 Register tmp, Register tmp2, Register tmp3) {
3343 Label L_by16, L_by16_loop, L_by4, L_by4_loop, L_by1, L_by1_loop, L_exit;
3344 unsigned long offset;
3345
3346 if (UseCRC32) {
3347 kernel_crc32_using_crc32(crc, buf, len, table0, table1, table2, table3);
3348 return;
3349 }
3350
3351 mvnw(crc, crc);
3352
3353 adrp(table0, ExternalAddress(StubRoutines::crc_table_addr()), offset);
3354 if (offset) add(table0, table0, offset);
3355 add(table1, table0, 1*256*sizeof(juint));
3356 add(table2, table0, 2*256*sizeof(juint));
3357 add(table3, table0, 3*256*sizeof(juint));
3358
3359 if (UseNeon) {
3360 cmp(len, (u1)64);
3361 br(Assembler::LT, L_by16);
3362 eor(v16, T16B, v16, v16);
3363
3364 Label L_fold;
3365
3366 add(tmp, table0, 4*256*sizeof(juint)); // Point at the Neon constants
3367
3368 ld1(v0, v1, T2D, post(buf, 32));
3369 ld1r(v4, T2D, post(tmp, 8));
3370 ld1r(v5, T2D, post(tmp, 8));
3371 ld1r(v6, T2D, post(tmp, 8));
3372 ld1r(v7, T2D, post(tmp, 8));
3373 mov(v16, T4S, 0, crc);
3374
3375 eor(v0, T16B, v0, v16);
3376 sub(len, len, 64);
3377
3378 BIND(L_fold);
3379 pmull(v22, T8H, v0, v5, T8B);
3380 pmull(v20, T8H, v0, v7, T8B);
3381 pmull(v23, T8H, v0, v4, T8B);
3382 pmull(v21, T8H, v0, v6, T8B);
3383
3384 pmull2(v18, T8H, v0, v5, T16B);
3385 pmull2(v16, T8H, v0, v7, T16B);
3386 pmull2(v19, T8H, v0, v4, T16B);
3387 pmull2(v17, T8H, v0, v6, T16B);
3388
3389 uzp1(v24, T8H, v20, v22);
3390 uzp2(v25, T8H, v20, v22);
3391 eor(v20, T16B, v24, v25);
3392
3393 uzp1(v26, T8H, v16, v18);
3394 uzp2(v27, T8H, v16, v18);
3395 eor(v16, T16B, v26, v27);
3396
3397 ushll2(v22, T4S, v20, T8H, 8);
3398 ushll(v20, T4S, v20, T4H, 8);
3399
3400 ushll2(v18, T4S, v16, T8H, 8);
3401 ushll(v16, T4S, v16, T4H, 8);
3402
3403 eor(v22, T16B, v23, v22);
3404 eor(v18, T16B, v19, v18);
3405 eor(v20, T16B, v21, v20);
3406 eor(v16, T16B, v17, v16);
3407
3408 uzp1(v17, T2D, v16, v20);
3409 uzp2(v21, T2D, v16, v20);
3410 eor(v17, T16B, v17, v21);
3411
3412 ushll2(v20, T2D, v17, T4S, 16);
3413 ushll(v16, T2D, v17, T2S, 16);
3414
3415 eor(v20, T16B, v20, v22);
3416 eor(v16, T16B, v16, v18);
3417
3418 uzp1(v17, T2D, v20, v16);
3419 uzp2(v21, T2D, v20, v16);
3420 eor(v28, T16B, v17, v21);
3421
3422 pmull(v22, T8H, v1, v5, T8B);
3423 pmull(v20, T8H, v1, v7, T8B);
3424 pmull(v23, T8H, v1, v4, T8B);
3425 pmull(v21, T8H, v1, v6, T8B);
3426
3427 pmull2(v18, T8H, v1, v5, T16B);
3428 pmull2(v16, T8H, v1, v7, T16B);
3429 pmull2(v19, T8H, v1, v4, T16B);
3430 pmull2(v17, T8H, v1, v6, T16B);
3431
3432 ld1(v0, v1, T2D, post(buf, 32));
3433
3434 uzp1(v24, T8H, v20, v22);
3435 uzp2(v25, T8H, v20, v22);
3436 eor(v20, T16B, v24, v25);
3437
3438 uzp1(v26, T8H, v16, v18);
3439 uzp2(v27, T8H, v16, v18);
3440 eor(v16, T16B, v26, v27);
3441
3442 ushll2(v22, T4S, v20, T8H, 8);
3443 ushll(v20, T4S, v20, T4H, 8);
3444
3445 ushll2(v18, T4S, v16, T8H, 8);
3446 ushll(v16, T4S, v16, T4H, 8);
3447
3448 eor(v22, T16B, v23, v22);
3449 eor(v18, T16B, v19, v18);
3450 eor(v20, T16B, v21, v20);
3451 eor(v16, T16B, v17, v16);
3452
3453 uzp1(v17, T2D, v16, v20);
3454 uzp2(v21, T2D, v16, v20);
3455 eor(v16, T16B, v17, v21);
3456
3457 ushll2(v20, T2D, v16, T4S, 16);
3458 ushll(v16, T2D, v16, T2S, 16);
3459
3460 eor(v20, T16B, v22, v20);
3461 eor(v16, T16B, v16, v18);
3462
3463 uzp1(v17, T2D, v20, v16);
3464 uzp2(v21, T2D, v20, v16);
3465 eor(v20, T16B, v17, v21);
3466
3467 shl(v16, T2D, v28, 1);
3468 shl(v17, T2D, v20, 1);
3469
3470 eor(v0, T16B, v0, v16);
3471 eor(v1, T16B, v1, v17);
3472
3473 subs(len, len, 32);
3474 br(Assembler::GE, L_fold);
3475
3476 mov(crc, 0);
3477 mov(tmp, v0, T1D, 0);
3478 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, false);
3479 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, true);
3480 mov(tmp, v0, T1D, 1);
3481 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, false);
3482 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, true);
3483 mov(tmp, v1, T1D, 0);
3484 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, false);
3485 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, true);
3486 mov(tmp, v1, T1D, 1);
3487 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, false);
3488 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, true);
3489
3490 add(len, len, 32);
3491 }
3492
3493 BIND(L_by16);
3494 subs(len, len, 16);
3495 br(Assembler::GE, L_by16_loop);
3496 adds(len, len, 16-4);
3497 br(Assembler::GE, L_by4_loop);
3498 adds(len, len, 4);
3499 br(Assembler::GT, L_by1_loop);
3500 b(L_exit);
3501
3502 BIND(L_by4_loop);
3503 ldrw(tmp, Address(post(buf, 4)));
3504 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3);
3505 subs(len, len, 4);
3506 br(Assembler::GE, L_by4_loop);
3507 adds(len, len, 4);
3508 br(Assembler::LE, L_exit);
3509 BIND(L_by1_loop);
3510 subs(len, len, 1);
3511 ldrb(tmp, Address(post(buf, 1)));
3512 update_byte_crc32(crc, tmp, table0);
3513 br(Assembler::GT, L_by1_loop);
3514 b(L_exit);
3515
3516 align(CodeEntryAlignment);
3517 BIND(L_by16_loop);
3518 subs(len, len, 16);
3519 ldp(tmp, tmp3, Address(post(buf, 16)));
3520 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, false);
3521 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, true);
3522 update_word_crc32(crc, tmp3, tmp2, table0, table1, table2, table3, false);
3523 update_word_crc32(crc, tmp3, tmp2, table0, table1, table2, table3, true);
3524 br(Assembler::GE, L_by16_loop);
3525 adds(len, len, 16-4);
3526 br(Assembler::GE, L_by4_loop);
3527 adds(len, len, 4);
3528 br(Assembler::GT, L_by1_loop);
3529 BIND(L_exit);
3530 mvnw(crc, crc);
3531 }
3532
3533 void MacroAssembler::kernel_crc32c_using_crc32c(Register crc, Register buf,
3534 Register len, Register tmp0, Register tmp1, Register tmp2,
3535 Register tmp3) {
3536 Label CRC_by64_loop, CRC_by4_loop, CRC_by1_loop, CRC_less64, CRC_by64_pre, CRC_by32_loop, CRC_less32, L_exit;
3537 assert_different_registers(crc, buf, len, tmp0, tmp1, tmp2, tmp3);
3538
3539 subs(len, len, 128);
3540 br(Assembler::GE, CRC_by64_pre);
3541 BIND(CRC_less64);
3542 adds(len, len, 128-32);
3543 br(Assembler::GE, CRC_by32_loop);
3544 BIND(CRC_less32);
3545 adds(len, len, 32-4);
3546 br(Assembler::GE, CRC_by4_loop);
3547 adds(len, len, 4);
3548 br(Assembler::GT, CRC_by1_loop);
3549 b(L_exit);
3550
3551 BIND(CRC_by32_loop);
3552 ldp(tmp0, tmp1, Address(post(buf, 16)));
3553 subs(len, len, 32);
3554 crc32cx(crc, crc, tmp0);
3555 ldr(tmp2, Address(post(buf, 8)));
3556 crc32cx(crc, crc, tmp1);
3557 ldr(tmp3, Address(post(buf, 8)));
3558 crc32cx(crc, crc, tmp2);
3559 crc32cx(crc, crc, tmp3);
3560 br(Assembler::GE, CRC_by32_loop);
3561 cmn(len, 32);
3562 br(Assembler::NE, CRC_less32);
3563 b(L_exit);
3564
3565 BIND(CRC_by4_loop);
3566 ldrw(tmp0, Address(post(buf, 4)));
3567 subs(len, len, 4);
3568 crc32cw(crc, crc, tmp0);
3569 br(Assembler::GE, CRC_by4_loop);
3570 adds(len, len, 4);
3571 br(Assembler::LE, L_exit);
3572 BIND(CRC_by1_loop);
3573 ldrb(tmp0, Address(post(buf, 1)));
3574 subs(len, len, 1);
3575 crc32cb(crc, crc, tmp0);
3576 br(Assembler::GT, CRC_by1_loop);
3577 b(L_exit);
3578
3579 BIND(CRC_by64_pre);
3580 sub(buf, buf, 8);
3581 ldp(tmp0, tmp1, Address(buf, 8));
3582 crc32cx(crc, crc, tmp0);
3583 ldr(tmp2, Address(buf, 24));
3584 crc32cx(crc, crc, tmp1);
3585 ldr(tmp3, Address(buf, 32));
3586 crc32cx(crc, crc, tmp2);
3587 ldr(tmp0, Address(buf, 40));
3588 crc32cx(crc, crc, tmp3);
3589 ldr(tmp1, Address(buf, 48));
3590 crc32cx(crc, crc, tmp0);
3591 ldr(tmp2, Address(buf, 56));
3592 crc32cx(crc, crc, tmp1);
3593 ldr(tmp3, Address(pre(buf, 64)));
3594
3595 b(CRC_by64_loop);
3596
3597 align(CodeEntryAlignment);
3598 BIND(CRC_by64_loop);
3599 subs(len, len, 64);
3600 crc32cx(crc, crc, tmp2);
3601 ldr(tmp0, Address(buf, 8));
3602 crc32cx(crc, crc, tmp3);
3603 ldr(tmp1, Address(buf, 16));
3604 crc32cx(crc, crc, tmp0);
3605 ldr(tmp2, Address(buf, 24));
3606 crc32cx(crc, crc, tmp1);
3607 ldr(tmp3, Address(buf, 32));
3608 crc32cx(crc, crc, tmp2);
3609 ldr(tmp0, Address(buf, 40));
3610 crc32cx(crc, crc, tmp3);
3611 ldr(tmp1, Address(buf, 48));
3612 crc32cx(crc, crc, tmp0);
3613 ldr(tmp2, Address(buf, 56));
3614 crc32cx(crc, crc, tmp1);
3615 ldr(tmp3, Address(pre(buf, 64)));
3616 br(Assembler::GE, CRC_by64_loop);
3617
3618 // post-loop
3619 crc32cx(crc, crc, tmp2);
3620 crc32cx(crc, crc, tmp3);
3621
3622 sub(len, len, 64);
3623 add(buf, buf, 8);
3624 cmn(len, 128);
3625 br(Assembler::NE, CRC_less64);
3626 BIND(L_exit);
3627 }
3628
3629 /**
3630 * @param crc register containing existing CRC (32-bit)
3631 * @param buf register pointing to input byte buffer (byte*)
3632 * @param len register containing number of bytes
3633 * @param table register that will contain address of CRC table
3634 * @param tmp scratch register
3635 */
3636 void MacroAssembler::kernel_crc32c(Register crc, Register buf, Register len,
3637 Register table0, Register table1, Register table2, Register table3,
3638 Register tmp, Register tmp2, Register tmp3) {
3639 kernel_crc32c_using_crc32c(crc, buf, len, table0, table1, table2, table3);
3640 }
3641
3642
3643 SkipIfEqual::SkipIfEqual(
3644 MacroAssembler* masm, const bool* flag_addr, bool value) {
3645 _masm = masm;
3646 unsigned long offset;
3647 _masm->adrp(rscratch1, ExternalAddress((address)flag_addr), offset);
3648 _masm->ldrb(rscratch1, Address(rscratch1, offset));
3649 _masm->cbzw(rscratch1, _label);
3650 }
3651
3652 SkipIfEqual::~SkipIfEqual() {
3653 _masm->bind(_label);
3654 }
3655
3656 void MacroAssembler::addptr(const Address &dst, int32_t src) {
3657 Address adr;
3658 switch(dst.getMode()) {
3659 case Address::base_plus_offset:
3660 // This is the expected mode, although we allow all the other
3661 // forms below.
3662 adr = form_address(rscratch2, dst.base(), dst.offset(), LogBytesPerWord);
3663 break;
3664 default:
3665 lea(rscratch2, dst);
3666 adr = Address(rscratch2);
3667 break;
3668 }
3669 ldr(rscratch1, adr);
3670 add(rscratch1, rscratch1, src);
3671 str(rscratch1, adr);
3672 }
3673
3674 void MacroAssembler::cmpptr(Register src1, Address src2) {
3675 unsigned long offset;
3676 adrp(rscratch1, src2, offset);
3677 ldr(rscratch1, Address(rscratch1, offset));
3678 cmp(src1, rscratch1);
3679 }
3680
3681 void MacroAssembler::cmpoop(Register obj1, Register obj2) {
3682 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3683 bs->obj_equals(this, obj1, obj2);
3684 }
3685
3686 void MacroAssembler::load_klass(Register dst, Register src) {
3687 if (UseCompressedClassPointers) {
3688 ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3689 decode_klass_not_null(dst);
3690 } else {
3691 ldr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3692 }
3693 }
3694
3695 // ((OopHandle)result).resolve();
3696 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) {
3697 // OopHandle::resolve is an indirection.
3698 access_load_at(T_OBJECT, IN_NATIVE, result, Address(result, 0), tmp, noreg);
3699 }
3700
3701 void MacroAssembler::load_mirror(Register dst, Register method, Register tmp) {
3702 const int mirror_offset = in_bytes(Klass::java_mirror_offset());
3703 ldr(dst, Address(rmethod, Method::const_offset()));
3704 ldr(dst, Address(dst, ConstMethod::constants_offset()));
3705 ldr(dst, Address(dst, ConstantPool::pool_holder_offset_in_bytes()));
3706 ldr(dst, Address(dst, mirror_offset));
3707 resolve_oop_handle(dst, tmp);
3708 }
3709
3710 void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp) {
3711 if (UseCompressedClassPointers) {
3712 ldrw(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3713 if (CompressedKlassPointers::base() == NULL) {
3714 cmp(trial_klass, tmp, LSL, CompressedKlassPointers::shift());
3715 return;
3716 } else if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
3717 && CompressedKlassPointers::shift() == 0) {
3718 // Only the bottom 32 bits matter
3719 cmpw(trial_klass, tmp);
3720 return;
3721 }
3722 decode_klass_not_null(tmp);
3723 } else {
3724 ldr(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3725 }
3726 cmp(trial_klass, tmp);
3727 }
3728
3729 void MacroAssembler::load_prototype_header(Register dst, Register src) {
3730 load_klass(dst, src);
3731 ldr(dst, Address(dst, Klass::prototype_header_offset()));
3732 }
3733
3734 void MacroAssembler::store_klass(Register dst, Register src) {
3735 // FIXME: Should this be a store release? concurrent gcs assumes
3736 // klass length is valid if klass field is not null.
3737 if (UseCompressedClassPointers) {
3738 encode_klass_not_null(src);
3739 strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3740 } else {
3741 str(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3742 }
3743 }
3744
3745 void MacroAssembler::store_klass_gap(Register dst, Register src) {
3746 if (UseCompressedClassPointers) {
3747 // Store to klass gap in destination
3748 strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
3749 }
3750 }
3751
3752 // Algorithm must match CompressedOops::encode.
3753 void MacroAssembler::encode_heap_oop(Register d, Register s) {
3754 #ifdef ASSERT
3755 verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
3756 #endif
3757 verify_oop(s, "broken oop in encode_heap_oop");
3758 if (CompressedOops::base() == NULL) {
3759 if (CompressedOops::shift() != 0) {
3760 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
3761 lsr(d, s, LogMinObjAlignmentInBytes);
3762 } else {
3763 mov(d, s);
3764 }
3765 } else {
3766 subs(d, s, rheapbase);
3767 csel(d, d, zr, Assembler::HS);
3768 lsr(d, d, LogMinObjAlignmentInBytes);
3769
3770 /* Old algorithm: is this any worse?
3771 Label nonnull;
3772 cbnz(r, nonnull);
3773 sub(r, r, rheapbase);
3774 bind(nonnull);
3775 lsr(r, r, LogMinObjAlignmentInBytes);
3776 */
3777 }
3778 }
3779
3780 void MacroAssembler::encode_heap_oop_not_null(Register r) {
3781 #ifdef ASSERT
3782 verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?");
3783 if (CheckCompressedOops) {
3784 Label ok;
3785 cbnz(r, ok);
3786 stop("null oop passed to encode_heap_oop_not_null");
3787 bind(ok);
3788 }
3789 #endif
3790 verify_oop(r, "broken oop in encode_heap_oop_not_null");
3791 if (CompressedOops::base() != NULL) {
3792 sub(r, r, rheapbase);
3793 }
3794 if (CompressedOops::shift() != 0) {
3795 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
3796 lsr(r, r, LogMinObjAlignmentInBytes);
3797 }
3798 }
3799
3800 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) {
3801 #ifdef ASSERT
3802 verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?");
3803 if (CheckCompressedOops) {
3804 Label ok;
3805 cbnz(src, ok);
3806 stop("null oop passed to encode_heap_oop_not_null2");
3807 bind(ok);
3808 }
3809 #endif
3810 verify_oop(src, "broken oop in encode_heap_oop_not_null2");
3811
3812 Register data = src;
3813 if (CompressedOops::base() != NULL) {
3814 sub(dst, src, rheapbase);
3815 data = dst;
3816 }
3817 if (CompressedOops::shift() != 0) {
3818 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
3819 lsr(dst, data, LogMinObjAlignmentInBytes);
3820 data = dst;
3821 }
3822 if (data == src)
3823 mov(dst, src);
3824 }
3825
3826 void MacroAssembler::decode_heap_oop(Register d, Register s) {
3827 #ifdef ASSERT
3828 verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?");
3829 #endif
3830 if (CompressedOops::base() == NULL) {
3831 if (CompressedOops::shift() != 0 || d != s) {
3832 lsl(d, s, CompressedOops::shift());
3833 }
3834 } else {
3835 Label done;
3836 if (d != s)
3837 mov(d, s);
3838 cbz(s, done);
3839 add(d, rheapbase, s, Assembler::LSL, LogMinObjAlignmentInBytes);
3840 bind(done);
3841 }
3842 verify_oop(d, "broken oop in decode_heap_oop");
3843 }
3844
3845 void MacroAssembler::decode_heap_oop_not_null(Register r) {
3846 assert (UseCompressedOops, "should only be used for compressed headers");
3847 assert (Universe::heap() != NULL, "java heap should be initialized");
3848 // Cannot assert, unverified entry point counts instructions (see .ad file)
3849 // vtableStubs also counts instructions in pd_code_size_limit.
3850 // Also do not verify_oop as this is called by verify_oop.
3851 if (CompressedOops::shift() != 0) {
3852 assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
3853 if (CompressedOops::base() != NULL) {
3854 add(r, rheapbase, r, Assembler::LSL, LogMinObjAlignmentInBytes);
3855 } else {
3856 add(r, zr, r, Assembler::LSL, LogMinObjAlignmentInBytes);
3857 }
3858 } else {
3859 assert (CompressedOops::base() == NULL, "sanity");
3860 }
3861 }
3862
3863 void MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
3864 assert (UseCompressedOops, "should only be used for compressed headers");
3865 assert (Universe::heap() != NULL, "java heap should be initialized");
3866 // Cannot assert, unverified entry point counts instructions (see .ad file)
3867 // vtableStubs also counts instructions in pd_code_size_limit.
3868 // Also do not verify_oop as this is called by verify_oop.
3869 if (CompressedOops::shift() != 0) {
3870 assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
3871 if (CompressedOops::base() != NULL) {
3872 add(dst, rheapbase, src, Assembler::LSL, LogMinObjAlignmentInBytes);
3873 } else {
3874 add(dst, zr, src, Assembler::LSL, LogMinObjAlignmentInBytes);
3875 }
3876 } else {
3877 assert (CompressedOops::base() == NULL, "sanity");
3878 if (dst != src) {
3879 mov(dst, src);
3880 }
3881 }
3882 }
3883
3884 void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
3885 if (CompressedKlassPointers::base() == NULL) {
3886 if (CompressedKlassPointers::shift() != 0) {
3887 assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
3888 lsr(dst, src, LogKlassAlignmentInBytes);
3889 } else {
3890 if (dst != src) mov(dst, src);
3891 }
3892 return;
3893 }
3894
3895 if (use_XOR_for_compressed_class_base) {
3896 if (CompressedKlassPointers::shift() != 0) {
3897 eor(dst, src, (uint64_t)CompressedKlassPointers::base());
3898 lsr(dst, dst, LogKlassAlignmentInBytes);
3899 } else {
3900 eor(dst, src, (uint64_t)CompressedKlassPointers::base());
3901 }
3902 return;
3903 }
3904
3905 if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
3906 && CompressedKlassPointers::shift() == 0) {
3907 movw(dst, src);
3908 return;
3909 }
3910
3911 #ifdef ASSERT
3912 verify_heapbase("MacroAssembler::encode_klass_not_null2: heap base corrupted?");
3913 #endif
3914
3915 Register rbase = dst;
3916 if (dst == src) rbase = rheapbase;
3917 mov(rbase, (uint64_t)CompressedKlassPointers::base());
3918 sub(dst, src, rbase);
3919 if (CompressedKlassPointers::shift() != 0) {
3920 assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
3921 lsr(dst, dst, LogKlassAlignmentInBytes);
3922 }
3923 if (dst == src) reinit_heapbase();
3924 }
3925
3926 void MacroAssembler::encode_klass_not_null(Register r) {
3927 encode_klass_not_null(r, r);
3928 }
3929
3930 void MacroAssembler::decode_klass_not_null(Register dst, Register src) {
3931 Register rbase = dst;
3932 assert (UseCompressedClassPointers, "should only be used for compressed headers");
3933
3934 if (CompressedKlassPointers::base() == NULL) {
3935 if (CompressedKlassPointers::shift() != 0) {
3936 assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
3937 lsl(dst, src, LogKlassAlignmentInBytes);
3938 } else {
3939 if (dst != src) mov(dst, src);
3940 }
3941 return;
3942 }
3943
3944 if (use_XOR_for_compressed_class_base) {
3945 if (CompressedKlassPointers::shift() != 0) {
3946 lsl(dst, src, LogKlassAlignmentInBytes);
3947 eor(dst, dst, (uint64_t)CompressedKlassPointers::base());
3948 } else {
3949 eor(dst, src, (uint64_t)CompressedKlassPointers::base());
3950 }
3951 return;
3952 }
3953
3954 if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
3955 && CompressedKlassPointers::shift() == 0) {
3956 if (dst != src)
3957 movw(dst, src);
3958 movk(dst, (uint64_t)CompressedKlassPointers::base() >> 32, 32);
3959 return;
3960 }
3961
3962 // Cannot assert, unverified entry point counts instructions (see .ad file)
3963 // vtableStubs also counts instructions in pd_code_size_limit.
3964 // Also do not verify_oop as this is called by verify_oop.
3965 if (dst == src) rbase = rheapbase;
3966 mov(rbase, (uint64_t)CompressedKlassPointers::base());
3967 if (CompressedKlassPointers::shift() != 0) {
3968 assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
3969 add(dst, rbase, src, Assembler::LSL, LogKlassAlignmentInBytes);
3970 } else {
3971 add(dst, rbase, src);
3972 }
3973 if (dst == src) reinit_heapbase();
3974 }
3975
3976 void MacroAssembler::decode_klass_not_null(Register r) {
3977 decode_klass_not_null(r, r);
3978 }
3979
3980 void MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
3981 #ifdef ASSERT
3982 {
3983 ThreadInVMfromUnknown tiv;
3984 assert (UseCompressedOops, "should only be used for compressed oops");
3985 assert (Universe::heap() != NULL, "java heap should be initialized");
3986 assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
3987 assert(Universe::heap()->is_in_reserved(JNIHandles::resolve(obj)), "should be real oop");
3988 }
3989 #endif
3990 int oop_index = oop_recorder()->find_index(obj);
3991 InstructionMark im(this);
3992 RelocationHolder rspec = oop_Relocation::spec(oop_index);
3993 code_section()->relocate(inst_mark(), rspec);
3994 movz(dst, 0xDEAD, 16);
3995 movk(dst, 0xBEEF);
3996 }
3997
3998 void MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
3999 assert (UseCompressedClassPointers, "should only be used for compressed headers");
4000 assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
4001 int index = oop_recorder()->find_index(k);
4002 assert(! Universe::heap()->is_in_reserved(k), "should not be an oop");
4003
4004 InstructionMark im(this);
4005 RelocationHolder rspec = metadata_Relocation::spec(index);
4006 code_section()->relocate(inst_mark(), rspec);
4007 narrowKlass nk = CompressedKlassPointers::encode(k);
4008 movz(dst, (nk >> 16), 16);
4009 movk(dst, nk & 0xffff);
4010 }
4011
4012 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators,
4013 Register dst, Address src,
4014 Register tmp1, Register thread_tmp) {
4015 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
4016 decorators = AccessInternal::decorator_fixup(decorators);
4017 bool as_raw = (decorators & AS_RAW) != 0;
4018 if (as_raw) {
4019 bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
4020 } else {
4021 bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
4022 }
4023 }
4024
4025 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators,
4026 Address dst, Register src,
4027 Register tmp1, Register thread_tmp) {
4028 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
4029 decorators = AccessInternal::decorator_fixup(decorators);
4030 bool as_raw = (decorators & AS_RAW) != 0;
4031 if (as_raw) {
4032 bs->BarrierSetAssembler::store_at(this, decorators, type, dst, src, tmp1, thread_tmp);
4033 } else {
4034 bs->store_at(this, decorators, type, dst, src, tmp1, thread_tmp);
4035 }
4036 }
4037
4038 void MacroAssembler::resolve(DecoratorSet decorators, Register obj) {
4039 // Use stronger ACCESS_WRITE|ACCESS_READ by default.
4040 if ((decorators & (ACCESS_READ | ACCESS_WRITE)) == 0) {
4041 decorators |= ACCESS_READ | ACCESS_WRITE;
4042 }
4043 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
4044 return bs->resolve(this, decorators, obj);
4045 }
4046
4047 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,
4048 Register thread_tmp, DecoratorSet decorators) {
4049 access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp);
4050 }
4051
4052 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1,
4053 Register thread_tmp, DecoratorSet decorators) {
4054 access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, thread_tmp);
4055 }
4056
4057 void MacroAssembler::store_heap_oop(Address dst, Register src, Register tmp1,
4058 Register thread_tmp, DecoratorSet decorators) {
4059 access_store_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp);
4060 }
4061
4062 // Used for storing NULLs.
4063 void MacroAssembler::store_heap_oop_null(Address dst) {
4064 access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg);
4065 }
4066
4067 Address MacroAssembler::allocate_metadata_address(Metadata* obj) {
4068 assert(oop_recorder() != NULL, "this assembler needs a Recorder");
4069 int index = oop_recorder()->allocate_metadata_index(obj);
4070 RelocationHolder rspec = metadata_Relocation::spec(index);
4071 return Address((address)obj, rspec);
4072 }
4073
4074 // Move an oop into a register. immediate is true if we want
4075 // immediate instrcutions, i.e. we are not going to patch this
4076 // instruction while the code is being executed by another thread. In
4077 // that case we can use move immediates rather than the constant pool.
4078 void MacroAssembler::movoop(Register dst, jobject obj, bool immediate) {
4079 int oop_index;
4080 if (obj == NULL) {
4081 oop_index = oop_recorder()->allocate_oop_index(obj);
4082 } else {
4083 #ifdef ASSERT
4084 {
4085 ThreadInVMfromUnknown tiv;
4086 assert(Universe::heap()->is_in_reserved(JNIHandles::resolve(obj)), "should be real oop");
4087 }
4088 #endif
4089 oop_index = oop_recorder()->find_index(obj);
4090 }
4091 RelocationHolder rspec = oop_Relocation::spec(oop_index);
4092 if (! immediate) {
4093 address dummy = address(uintptr_t(pc()) & -wordSize); // A nearby aligned address
4094 ldr_constant(dst, Address(dummy, rspec));
4095 } else
4096 mov(dst, Address((address)obj, rspec));
4097 }
4098
4099 // Move a metadata address into a register.
4100 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
4101 int oop_index;
4102 if (obj == NULL) {
4103 oop_index = oop_recorder()->allocate_metadata_index(obj);
4104 } else {
4105 oop_index = oop_recorder()->find_index(obj);
4106 }
4107 RelocationHolder rspec = metadata_Relocation::spec(oop_index);
4108 mov(dst, Address((address)obj, rspec));
4109 }
4110
4111 Address MacroAssembler::constant_oop_address(jobject obj) {
4112 #ifdef ASSERT
4113 {
4114 ThreadInVMfromUnknown tiv;
4115 assert(oop_recorder() != NULL, "this assembler needs an OopRecorder");
4116 assert(Universe::heap()->is_in_reserved(JNIHandles::resolve(obj)), "not an oop");
4117 }
4118 #endif
4119 int oop_index = oop_recorder()->find_index(obj);
4120 return Address((address)obj, oop_Relocation::spec(oop_index));
4121 }
4122
4123 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
4124 void MacroAssembler::tlab_allocate(Register obj,
4125 Register var_size_in_bytes,
4126 int con_size_in_bytes,
4127 Register t1,
4128 Register t2,
4129 Label& slow_case) {
4130 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
4131 bs->tlab_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
4132 }
4133
4134 // Defines obj, preserves var_size_in_bytes
4135 void MacroAssembler::eden_allocate(Register obj,
4136 Register var_size_in_bytes,
4137 int con_size_in_bytes,
4138 Register t1,
4139 Label& slow_case) {
4140 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
4141 bs->eden_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case);
4142 }
4143
4144 // Zero words; len is in bytes
4145 // Destroys all registers except addr
4146 // len must be a nonzero multiple of wordSize
4147 void MacroAssembler::zero_memory(Register addr, Register len, Register t1) {
4148 assert_different_registers(addr, len, t1, rscratch1, rscratch2);
4149
4150 #ifdef ASSERT
4151 { Label L;
4152 tst(len, BytesPerWord - 1);
4153 br(Assembler::EQ, L);
4154 stop("len is not a multiple of BytesPerWord");
4155 bind(L);
4156 }
4157 #endif
4158
4159 #ifndef PRODUCT
4160 block_comment("zero memory");
4161 #endif
4162
4163 Label loop;
4164 Label entry;
4165
4166 // Algorithm:
4167 //
4168 // scratch1 = cnt & 7;
4169 // cnt -= scratch1;
4170 // p += scratch1;
4171 // switch (scratch1) {
4172 // do {
4173 // cnt -= 8;
4174 // p[-8] = 0;
4175 // case 7:
4176 // p[-7] = 0;
4177 // case 6:
4178 // p[-6] = 0;
4179 // // ...
4180 // case 1:
4181 // p[-1] = 0;
4182 // case 0:
4183 // p += 8;
4184 // } while (cnt);
4185 // }
4186
4187 const int unroll = 8; // Number of str(zr) instructions we'll unroll
4188
4189 lsr(len, len, LogBytesPerWord);
4190 andr(rscratch1, len, unroll - 1); // tmp1 = cnt % unroll
4191 sub(len, len, rscratch1); // cnt -= unroll
4192 // t1 always points to the end of the region we're about to zero
4193 add(t1, addr, rscratch1, Assembler::LSL, LogBytesPerWord);
4194 adr(rscratch2, entry);
4195 sub(rscratch2, rscratch2, rscratch1, Assembler::LSL, 2);
4196 br(rscratch2);
4197 bind(loop);
4198 sub(len, len, unroll);
4199 for (int i = -unroll; i < 0; i++)
4200 Assembler::str(zr, Address(t1, i * wordSize));
4201 bind(entry);
4202 add(t1, t1, unroll * wordSize);
4203 cbnz(len, loop);
4204 }
4205
4206 void MacroAssembler::verify_tlab() {
4207 #ifdef ASSERT
4208 if (UseTLAB && VerifyOops) {
4209 Label next, ok;
4210
4211 stp(rscratch2, rscratch1, Address(pre(sp, -16)));
4212
4213 ldr(rscratch2, Address(rthread, in_bytes(JavaThread::tlab_top_offset())));
4214 ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_start_offset())));
4215 cmp(rscratch2, rscratch1);
4216 br(Assembler::HS, next);
4217 STOP("assert(top >= start)");
4218 should_not_reach_here();
4219
4220 bind(next);
4221 ldr(rscratch2, Address(rthread, in_bytes(JavaThread::tlab_end_offset())));
4222 ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_top_offset())));
4223 cmp(rscratch2, rscratch1);
4224 br(Assembler::HS, ok);
4225 STOP("assert(top <= end)");
4226 should_not_reach_here();
4227
4228 bind(ok);
4229 ldp(rscratch2, rscratch1, Address(post(sp, 16)));
4230 }
4231 #endif
4232 }
4233
4234 // Writes to stack successive pages until offset reached to check for
4235 // stack overflow + shadow pages. This clobbers tmp.
4236 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
4237 assert_different_registers(tmp, size, rscratch1);
4238 mov(tmp, sp);
4239 // Bang stack for total size given plus shadow page size.
4240 // Bang one page at a time because large size can bang beyond yellow and
4241 // red zones.
4242 Label loop;
4243 mov(rscratch1, os::vm_page_size());
4244 bind(loop);
4245 lea(tmp, Address(tmp, -os::vm_page_size()));
4246 subsw(size, size, rscratch1);
4247 str(size, Address(tmp));
4248 br(Assembler::GT, loop);
4249
4250 // Bang down shadow pages too.
4251 // At this point, (tmp-0) is the last address touched, so don't
4252 // touch it again. (It was touched as (tmp-pagesize) but then tmp
4253 // was post-decremented.) Skip this address by starting at i=1, and
4254 // touch a few more pages below. N.B. It is important to touch all
4255 // the way down to and including i=StackShadowPages.
4256 for (int i = 0; i < (int)(JavaThread::stack_shadow_zone_size() / os::vm_page_size()) - 1; i++) {
4257 // this could be any sized move but this is can be a debugging crumb
4258 // so the bigger the better.
4259 lea(tmp, Address(tmp, -os::vm_page_size()));
4260 str(size, Address(tmp));
4261 }
4262 }
4263
4264
4265 // Move the address of the polling page into dest.
4266 void MacroAssembler::get_polling_page(Register dest, address page, relocInfo::relocType rtype) {
4267 if (SafepointMechanism::uses_thread_local_poll()) {
4268 ldr(dest, Address(rthread, Thread::polling_page_offset()));
4269 } else {
4270 unsigned long off;
4271 adrp(dest, Address(page, rtype), off);
4272 assert(off == 0, "polling page must be page aligned");
4273 }
4274 }
4275
4276 // Move the address of the polling page into r, then read the polling
4277 // page.
4278 address MacroAssembler::read_polling_page(Register r, address page, relocInfo::relocType rtype) {
4279 get_polling_page(r, page, rtype);
4280 return read_polling_page(r, rtype);
4281 }
4282
4283 // Read the polling page. The address of the polling page must
4284 // already be in r.
4285 address MacroAssembler::read_polling_page(Register r, relocInfo::relocType rtype) {
4286 InstructionMark im(this);
4287 code_section()->relocate(inst_mark(), rtype);
4288 ldrw(zr, Address(r, 0));
4289 return inst_mark();
4290 }
4291
4292 void MacroAssembler::adrp(Register reg1, const Address &dest, unsigned long &byte_offset) {
4293 relocInfo::relocType rtype = dest.rspec().reloc()->type();
4294 unsigned long low_page = (unsigned long)CodeCache::low_bound() >> 12;
4295 unsigned long high_page = (unsigned long)(CodeCache::high_bound()-1) >> 12;
4296 unsigned long dest_page = (unsigned long)dest.target() >> 12;
4297 long offset_low = dest_page - low_page;
4298 long offset_high = dest_page - high_page;
4299
4300 assert(is_valid_AArch64_address(dest.target()), "bad address");
4301 assert(dest.getMode() == Address::literal, "ADRP must be applied to a literal address");
4302
4303 InstructionMark im(this);
4304 code_section()->relocate(inst_mark(), dest.rspec());
4305 // 8143067: Ensure that the adrp can reach the dest from anywhere within
4306 // the code cache so that if it is relocated we know it will still reach
4307 if (offset_high >= -(1<<20) && offset_low < (1<<20)) {
4308 _adrp(reg1, dest.target());
4309 } else {
4310 unsigned long target = (unsigned long)dest.target();
4311 unsigned long adrp_target
4312 = (target & 0xffffffffUL) | ((unsigned long)pc() & 0xffff00000000UL);
4313
4314 _adrp(reg1, (address)adrp_target);
4315 movk(reg1, target >> 32, 32);
4316 }
4317 byte_offset = (unsigned long)dest.target() & 0xfff;
4318 }
4319
4320 void MacroAssembler::load_byte_map_base(Register reg) {
4321 CardTable::CardValue* byte_map_base =
4322 ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base();
4323
4324 if (is_valid_AArch64_address((address)byte_map_base)) {
4325 // Strictly speaking the byte_map_base isn't an address at all,
4326 // and it might even be negative.
4327 unsigned long offset;
4328 adrp(reg, ExternalAddress((address)byte_map_base), offset);
4329 // We expect offset to be zero with most collectors.
4330 if (offset != 0) {
4331 add(reg, reg, offset);
4332 }
4333 } else {
4334 mov(reg, (uint64_t)byte_map_base);
4335 }
4336 }
4337
4338 void MacroAssembler::build_frame(int framesize) {
4339 assert(framesize > 0, "framesize must be > 0");
4340 if (framesize < ((1 << 9) + 2 * wordSize)) {
4341 sub(sp, sp, framesize);
4342 stp(rfp, lr, Address(sp, framesize - 2 * wordSize));
4343 if (PreserveFramePointer) add(rfp, sp, framesize - 2 * wordSize);
4344 } else {
4345 stp(rfp, lr, Address(pre(sp, -2 * wordSize)));
4346 if (PreserveFramePointer) mov(rfp, sp);
4347 if (framesize < ((1 << 12) + 2 * wordSize))
4348 sub(sp, sp, framesize - 2 * wordSize);
4349 else {
4350 mov(rscratch1, framesize - 2 * wordSize);
4351 sub(sp, sp, rscratch1);
4352 }
4353 }
4354 }
4355
4356 void MacroAssembler::remove_frame(int framesize) {
4357 assert(framesize > 0, "framesize must be > 0");
4358 if (framesize < ((1 << 9) + 2 * wordSize)) {
4359 ldp(rfp, lr, Address(sp, framesize - 2 * wordSize));
4360 add(sp, sp, framesize);
4361 } else {
4362 if (framesize < ((1 << 12) + 2 * wordSize))
4363 add(sp, sp, framesize - 2 * wordSize);
4364 else {
4365 mov(rscratch1, framesize - 2 * wordSize);
4366 add(sp, sp, rscratch1);
4367 }
4368 ldp(rfp, lr, Address(post(sp, 2 * wordSize)));
4369 }
4370 }
4371
4372 #ifdef COMPILER2
4373 typedef void (MacroAssembler::* chr_insn)(Register Rt, const Address &adr);
4374
4375 // Search for str1 in str2 and return index or -1
4376 void MacroAssembler::string_indexof(Register str2, Register str1,
4377 Register cnt2, Register cnt1,
4378 Register tmp1, Register tmp2,
4379 Register tmp3, Register tmp4,
4380 Register tmp5, Register tmp6,
4381 int icnt1, Register result, int ae) {
4382 // NOTE: tmp5, tmp6 can be zr depending on specific method version
4383 Label LINEARSEARCH, LINEARSTUB, LINEAR_MEDIUM, DONE, NOMATCH, MATCH;
4384
4385 Register ch1 = rscratch1;
4386 Register ch2 = rscratch2;
4387 Register cnt1tmp = tmp1;
4388 Register cnt2tmp = tmp2;
4389 Register cnt1_neg = cnt1;
4390 Register cnt2_neg = cnt2;
4391 Register result_tmp = tmp4;
4392
4393 bool isL = ae == StrIntrinsicNode::LL;
4394
4395 bool str1_isL = ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UL;
4396 bool str2_isL = ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::LU;
4397 int str1_chr_shift = str1_isL ? 0:1;
4398 int str2_chr_shift = str2_isL ? 0:1;
4399 int str1_chr_size = str1_isL ? 1:2;
4400 int str2_chr_size = str2_isL ? 1:2;
4401 chr_insn str1_load_1chr = str1_isL ? (chr_insn)&MacroAssembler::ldrb :
4402 (chr_insn)&MacroAssembler::ldrh;
4403 chr_insn str2_load_1chr = str2_isL ? (chr_insn)&MacroAssembler::ldrb :
4404 (chr_insn)&MacroAssembler::ldrh;
4405 chr_insn load_2chr = isL ? (chr_insn)&MacroAssembler::ldrh : (chr_insn)&MacroAssembler::ldrw;
4406 chr_insn load_4chr = isL ? (chr_insn)&MacroAssembler::ldrw : (chr_insn)&MacroAssembler::ldr;
4407
4408 // Note, inline_string_indexOf() generates checks:
4409 // if (substr.count > string.count) return -1;
4410 // if (substr.count == 0) return 0;
4411
4412 // We have two strings, a source string in str2, cnt2 and a pattern string
4413 // in str1, cnt1. Find the 1st occurence of pattern in source or return -1.
4414
4415 // For larger pattern and source we use a simplified Boyer Moore algorithm.
4416 // With a small pattern and source we use linear scan.
4417
4418 if (icnt1 == -1) {
4419 sub(result_tmp, cnt2, cnt1);
4420 cmp(cnt1, (u1)8); // Use Linear Scan if cnt1 < 8 || cnt1 >= 256
4421 br(LT, LINEARSEARCH);
4422 dup(v0, T16B, cnt1); // done in separate FPU pipeline. Almost no penalty
4423 subs(zr, cnt1, 256);
4424 lsr(tmp1, cnt2, 2);
4425 ccmp(cnt1, tmp1, 0b0000, LT); // Source must be 4 * pattern for BM
4426 br(GE, LINEARSTUB);
4427 }
4428
4429 // The Boyer Moore alogorithm is based on the description here:-
4430 //
4431 // http://en.wikipedia.org/wiki/Boyer%E2%80%93Moore_string_search_algorithm
4432 //
4433 // This describes and algorithm with 2 shift rules. The 'Bad Character' rule
4434 // and the 'Good Suffix' rule.
4435 //
4436 // These rules are essentially heuristics for how far we can shift the
4437 // pattern along the search string.
4438 //
4439 // The implementation here uses the 'Bad Character' rule only because of the
4440 // complexity of initialisation for the 'Good Suffix' rule.
4441 //
4442 // This is also known as the Boyer-Moore-Horspool algorithm:-
4443 //
4444 // http://en.wikipedia.org/wiki/Boyer-Moore-Horspool_algorithm
4445 //
4446 // This particular implementation has few java-specific optimizations.
4447 //
4448 // #define ASIZE 256
4449 //
4450 // int bm(unsigned char *x, int m, unsigned char *y, int n) {
4451 // int i, j;
4452 // unsigned c;
4453 // unsigned char bc[ASIZE];
4454 //
4455 // /* Preprocessing */
4456 // for (i = 0; i < ASIZE; ++i)
4457 // bc[i] = m;
4458 // for (i = 0; i < m - 1; ) {
4459 // c = x[i];
4460 // ++i;
4461 // // c < 256 for Latin1 string, so, no need for branch
4462 // #ifdef PATTERN_STRING_IS_LATIN1
4463 // bc[c] = m - i;
4464 // #else
4465 // if (c < ASIZE) bc[c] = m - i;
4466 // #endif
4467 // }
4468 //
4469 // /* Searching */
4470 // j = 0;
4471 // while (j <= n - m) {
4472 // c = y[i+j];
4473 // if (x[m-1] == c)
4474 // for (i = m - 2; i >= 0 && x[i] == y[i + j]; --i);
4475 // if (i < 0) return j;
4476 // // c < 256 for Latin1 string, so, no need for branch
4477 // #ifdef SOURCE_STRING_IS_LATIN1
4478 // // LL case: (c< 256) always true. Remove branch
4479 // j += bc[y[j+m-1]];
4480 // #endif
4481 // #ifndef PATTERN_STRING_IS_UTF
4482 // // UU case: need if (c<ASIZE) check. Skip 1 character if not.
4483 // if (c < ASIZE)
4484 // j += bc[y[j+m-1]];
4485 // else
4486 // j += 1
4487 // #endif
4488 // #ifdef PATTERN_IS_LATIN1_AND_SOURCE_IS_UTF
4489 // // UL case: need if (c<ASIZE) check. Skip <pattern length> if not.
4490 // if (c < ASIZE)
4491 // j += bc[y[j+m-1]];
4492 // else
4493 // j += m
4494 // #endif
4495 // }
4496 // }
4497
4498 if (icnt1 == -1) {
4499 Label BCLOOP, BCSKIP, BMLOOPSTR2, BMLOOPSTR1, BMSKIP, BMADV, BMMATCH,
4500 BMLOOPSTR1_LASTCMP, BMLOOPSTR1_CMP, BMLOOPSTR1_AFTER_LOAD, BM_INIT_LOOP;
4501 Register cnt1end = tmp2;
4502 Register str2end = cnt2;
4503 Register skipch = tmp2;
4504
4505 // str1 length is >=8, so, we can read at least 1 register for cases when
4506 // UTF->Latin1 conversion is not needed(8 LL or 4UU) and half register for
4507 // UL case. We'll re-read last character in inner pre-loop code to have
4508 // single outer pre-loop load
4509 const int firstStep = isL ? 7 : 3;
4510
4511 const int ASIZE = 256;
4512 const int STORED_BYTES = 32; // amount of bytes stored per instruction
4513 sub(sp, sp, ASIZE);
4514 mov(tmp5, ASIZE/STORED_BYTES); // loop iterations
4515 mov(ch1, sp);
4516 BIND(BM_INIT_LOOP);
4517 stpq(v0, v0, Address(post(ch1, STORED_BYTES)));
4518 subs(tmp5, tmp5, 1);
4519 br(GT, BM_INIT_LOOP);
4520
4521 sub(cnt1tmp, cnt1, 1);
4522 mov(tmp5, str2);
4523 add(str2end, str2, result_tmp, LSL, str2_chr_shift);
4524 sub(ch2, cnt1, 1);
4525 mov(tmp3, str1);
4526 BIND(BCLOOP);
4527 (this->*str1_load_1chr)(ch1, Address(post(tmp3, str1_chr_size)));
4528 if (!str1_isL) {
4529 subs(zr, ch1, ASIZE);
4530 br(HS, BCSKIP);
4531 }
4532 strb(ch2, Address(sp, ch1));
4533 BIND(BCSKIP);
4534 subs(ch2, ch2, 1);
4535 br(GT, BCLOOP);
4536
4537 add(tmp6, str1, cnt1, LSL, str1_chr_shift); // address after str1
4538 if (str1_isL == str2_isL) {
4539 // load last 8 bytes (8LL/4UU symbols)
4540 ldr(tmp6, Address(tmp6, -wordSize));
4541 } else {
4542 ldrw(tmp6, Address(tmp6, -wordSize/2)); // load last 4 bytes(4 symbols)
4543 // convert Latin1 to UTF. We'll have to wait until load completed, but
4544 // it's still faster than per-character loads+checks
4545 lsr(tmp3, tmp6, BitsPerByte * (wordSize/2 - str1_chr_size)); // str1[N-1]
4546 ubfx(ch1, tmp6, 8, 8); // str1[N-2]
4547 ubfx(ch2, tmp6, 16, 8); // str1[N-3]
4548 andr(tmp6, tmp6, 0xFF); // str1[N-4]
4549 orr(ch2, ch1, ch2, LSL, 16);
4550 orr(tmp6, tmp6, tmp3, LSL, 48);
4551 orr(tmp6, tmp6, ch2, LSL, 16);
4552 }
4553 BIND(BMLOOPSTR2);
4554 (this->*str2_load_1chr)(skipch, Address(str2, cnt1tmp, Address::lsl(str2_chr_shift)));
4555 sub(cnt1tmp, cnt1tmp, firstStep); // cnt1tmp is positive here, because cnt1 >= 8
4556 if (str1_isL == str2_isL) {
4557 // re-init tmp3. It's for free because it's executed in parallel with
4558 // load above. Alternative is to initialize it before loop, but it'll
4559 // affect performance on in-order systems with 2 or more ld/st pipelines
4560 lsr(tmp3, tmp6, BitsPerByte * (wordSize - str1_chr_size));
4561 }
4562 if (!isL) { // UU/UL case
4563 lsl(ch2, cnt1tmp, 1); // offset in bytes
4564 }
4565 cmp(tmp3, skipch);
4566 br(NE, BMSKIP);
4567 ldr(ch2, Address(str2, isL ? cnt1tmp : ch2));
4568 mov(ch1, tmp6);
4569 if (isL) {
4570 b(BMLOOPSTR1_AFTER_LOAD);
4571 } else {
4572 sub(cnt1tmp, cnt1tmp, 1); // no need to branch for UU/UL case. cnt1 >= 8
4573 b(BMLOOPSTR1_CMP);
4574 }
4575 BIND(BMLOOPSTR1);
4576 (this->*str1_load_1chr)(ch1, Address(str1, cnt1tmp, Address::lsl(str1_chr_shift)));
4577 (this->*str2_load_1chr)(ch2, Address(str2, cnt1tmp, Address::lsl(str2_chr_shift)));
4578 BIND(BMLOOPSTR1_AFTER_LOAD);
4579 subs(cnt1tmp, cnt1tmp, 1);
4580 br(LT, BMLOOPSTR1_LASTCMP);
4581 BIND(BMLOOPSTR1_CMP);
4582 cmp(ch1, ch2);
4583 br(EQ, BMLOOPSTR1);
4584 BIND(BMSKIP);
4585 if (!isL) {
4586 // if we've met UTF symbol while searching Latin1 pattern, then we can
4587 // skip cnt1 symbols
4588 if (str1_isL != str2_isL) {
4589 mov(result_tmp, cnt1);
4590 } else {
4591 mov(result_tmp, 1);
4592 }
4593 subs(zr, skipch, ASIZE);
4594 br(HS, BMADV);
4595 }
4596 ldrb(result_tmp, Address(sp, skipch)); // load skip distance
4597 BIND(BMADV);
4598 sub(cnt1tmp, cnt1, 1);
4599 add(str2, str2, result_tmp, LSL, str2_chr_shift);
4600 cmp(str2, str2end);
4601 br(LE, BMLOOPSTR2);
4602 add(sp, sp, ASIZE);
4603 b(NOMATCH);
4604 BIND(BMLOOPSTR1_LASTCMP);
4605 cmp(ch1, ch2);
4606 br(NE, BMSKIP);
4607 BIND(BMMATCH);
4608 sub(result, str2, tmp5);
4609 if (!str2_isL) lsr(result, result, 1);
4610 add(sp, sp, ASIZE);
4611 b(DONE);
4612
4613 BIND(LINEARSTUB);
4614 cmp(cnt1, (u1)16); // small patterns still should be handled by simple algorithm
4615 br(LT, LINEAR_MEDIUM);
4616 mov(result, zr);
4617 RuntimeAddress stub = NULL;
4618 if (isL) {
4619 stub = RuntimeAddress(StubRoutines::aarch64::string_indexof_linear_ll());
4620 assert(stub.target() != NULL, "string_indexof_linear_ll stub has not been generated");
4621 } else if (str1_isL) {
4622 stub = RuntimeAddress(StubRoutines::aarch64::string_indexof_linear_ul());
4623 assert(stub.target() != NULL, "string_indexof_linear_ul stub has not been generated");
4624 } else {
4625 stub = RuntimeAddress(StubRoutines::aarch64::string_indexof_linear_uu());
4626 assert(stub.target() != NULL, "string_indexof_linear_uu stub has not been generated");
4627 }
4628 trampoline_call(stub);
4629 b(DONE);
4630 }
4631
4632 BIND(LINEARSEARCH);
4633 {
4634 Label DO1, DO2, DO3;
4635
4636 Register str2tmp = tmp2;
4637 Register first = tmp3;
4638
4639 if (icnt1 == -1)
4640 {
4641 Label DOSHORT, FIRST_LOOP, STR2_NEXT, STR1_LOOP, STR1_NEXT;
4642
4643 cmp(cnt1, u1(str1_isL == str2_isL ? 4 : 2));
4644 br(LT, DOSHORT);
4645 BIND(LINEAR_MEDIUM);
4646 (this->*str1_load_1chr)(first, Address(str1));
4647 lea(str1, Address(str1, cnt1, Address::lsl(str1_chr_shift)));
4648 sub(cnt1_neg, zr, cnt1, LSL, str1_chr_shift);
4649 lea(str2, Address(str2, result_tmp, Address::lsl(str2_chr_shift)));
4650 sub(cnt2_neg, zr, result_tmp, LSL, str2_chr_shift);
4651
4652 BIND(FIRST_LOOP);
4653 (this->*str2_load_1chr)(ch2, Address(str2, cnt2_neg));
4654 cmp(first, ch2);
4655 br(EQ, STR1_LOOP);
4656 BIND(STR2_NEXT);
4657 adds(cnt2_neg, cnt2_neg, str2_chr_size);
4658 br(LE, FIRST_LOOP);
4659 b(NOMATCH);
4660
4661 BIND(STR1_LOOP);
4662 adds(cnt1tmp, cnt1_neg, str1_chr_size);
4663 add(cnt2tmp, cnt2_neg, str2_chr_size);
4664 br(GE, MATCH);
4665
4666 BIND(STR1_NEXT);
4667 (this->*str1_load_1chr)(ch1, Address(str1, cnt1tmp));
4668 (this->*str2_load_1chr)(ch2, Address(str2, cnt2tmp));
4669 cmp(ch1, ch2);
4670 br(NE, STR2_NEXT);
4671 adds(cnt1tmp, cnt1tmp, str1_chr_size);
4672 add(cnt2tmp, cnt2tmp, str2_chr_size);
4673 br(LT, STR1_NEXT);
4674 b(MATCH);
4675
4676 BIND(DOSHORT);
4677 if (str1_isL == str2_isL) {
4678 cmp(cnt1, (u1)2);
4679 br(LT, DO1);
4680 br(GT, DO3);
4681 }
4682 }
4683
4684 if (icnt1 == 4) {
4685 Label CH1_LOOP;
4686
4687 (this->*load_4chr)(ch1, str1);
4688 sub(result_tmp, cnt2, 4);
4689 lea(str2, Address(str2, result_tmp, Address::lsl(str2_chr_shift)));
4690 sub(cnt2_neg, zr, result_tmp, LSL, str2_chr_shift);
4691
4692 BIND(CH1_LOOP);
4693 (this->*load_4chr)(ch2, Address(str2, cnt2_neg));
4694 cmp(ch1, ch2);
4695 br(EQ, MATCH);
4696 adds(cnt2_neg, cnt2_neg, str2_chr_size);
4697 br(LE, CH1_LOOP);
4698 b(NOMATCH);
4699 }
4700
4701 if ((icnt1 == -1 && str1_isL == str2_isL) || icnt1 == 2) {
4702 Label CH1_LOOP;
4703
4704 BIND(DO2);
4705 (this->*load_2chr)(ch1, str1);
4706 if (icnt1 == 2) {
4707 sub(result_tmp, cnt2, 2);
4708 }
4709 lea(str2, Address(str2, result_tmp, Address::lsl(str2_chr_shift)));
4710 sub(cnt2_neg, zr, result_tmp, LSL, str2_chr_shift);
4711 BIND(CH1_LOOP);
4712 (this->*load_2chr)(ch2, Address(str2, cnt2_neg));
4713 cmp(ch1, ch2);
4714 br(EQ, MATCH);
4715 adds(cnt2_neg, cnt2_neg, str2_chr_size);
4716 br(LE, CH1_LOOP);
4717 b(NOMATCH);
4718 }
4719
4720 if ((icnt1 == -1 && str1_isL == str2_isL) || icnt1 == 3) {
4721 Label FIRST_LOOP, STR2_NEXT, STR1_LOOP;
4722
4723 BIND(DO3);
4724 (this->*load_2chr)(first, str1);
4725 (this->*str1_load_1chr)(ch1, Address(str1, 2*str1_chr_size));
4726 if (icnt1 == 3) {
4727 sub(result_tmp, cnt2, 3);
4728 }
4729 lea(str2, Address(str2, result_tmp, Address::lsl(str2_chr_shift)));
4730 sub(cnt2_neg, zr, result_tmp, LSL, str2_chr_shift);
4731 BIND(FIRST_LOOP);
4732 (this->*load_2chr)(ch2, Address(str2, cnt2_neg));
4733 cmpw(first, ch2);
4734 br(EQ, STR1_LOOP);
4735 BIND(STR2_NEXT);
4736 adds(cnt2_neg, cnt2_neg, str2_chr_size);
4737 br(LE, FIRST_LOOP);
4738 b(NOMATCH);
4739
4740 BIND(STR1_LOOP);
4741 add(cnt2tmp, cnt2_neg, 2*str2_chr_size);
4742 (this->*str2_load_1chr)(ch2, Address(str2, cnt2tmp));
4743 cmp(ch1, ch2);
4744 br(NE, STR2_NEXT);
4745 b(MATCH);
4746 }
4747
4748 if (icnt1 == -1 || icnt1 == 1) {
4749 Label CH1_LOOP, HAS_ZERO, DO1_SHORT, DO1_LOOP;
4750
4751 BIND(DO1);
4752 (this->*str1_load_1chr)(ch1, str1);
4753 cmp(cnt2, (u1)8);
4754 br(LT, DO1_SHORT);
4755
4756 sub(result_tmp, cnt2, 8/str2_chr_size);
4757 sub(cnt2_neg, zr, result_tmp, LSL, str2_chr_shift);
4758 mov(tmp3, str2_isL ? 0x0101010101010101 : 0x0001000100010001);
4759 lea(str2, Address(str2, result_tmp, Address::lsl(str2_chr_shift)));
4760
4761 if (str2_isL) {
4762 orr(ch1, ch1, ch1, LSL, 8);
4763 }
4764 orr(ch1, ch1, ch1, LSL, 16);
4765 orr(ch1, ch1, ch1, LSL, 32);
4766 BIND(CH1_LOOP);
4767 ldr(ch2, Address(str2, cnt2_neg));
4768 eor(ch2, ch1, ch2);
4769 sub(tmp1, ch2, tmp3);
4770 orr(tmp2, ch2, str2_isL ? 0x7f7f7f7f7f7f7f7f : 0x7fff7fff7fff7fff);
4771 bics(tmp1, tmp1, tmp2);
4772 br(NE, HAS_ZERO);
4773 adds(cnt2_neg, cnt2_neg, 8);
4774 br(LT, CH1_LOOP);
4775
4776 cmp(cnt2_neg, (u1)8);
4777 mov(cnt2_neg, 0);
4778 br(LT, CH1_LOOP);
4779 b(NOMATCH);
4780
4781 BIND(HAS_ZERO);
4782 rev(tmp1, tmp1);
4783 clz(tmp1, tmp1);
4784 add(cnt2_neg, cnt2_neg, tmp1, LSR, 3);
4785 b(MATCH);
4786
4787 BIND(DO1_SHORT);
4788 mov(result_tmp, cnt2);
4789 lea(str2, Address(str2, cnt2, Address::lsl(str2_chr_shift)));
4790 sub(cnt2_neg, zr, cnt2, LSL, str2_chr_shift);
4791 BIND(DO1_LOOP);
4792 (this->*str2_load_1chr)(ch2, Address(str2, cnt2_neg));
4793 cmpw(ch1, ch2);
4794 br(EQ, MATCH);
4795 adds(cnt2_neg, cnt2_neg, str2_chr_size);
4796 br(LT, DO1_LOOP);
4797 }
4798 }
4799 BIND(NOMATCH);
4800 mov(result, -1);
4801 b(DONE);
4802 BIND(MATCH);
4803 add(result, result_tmp, cnt2_neg, ASR, str2_chr_shift);
4804 BIND(DONE);
4805 }
4806
4807 typedef void (MacroAssembler::* chr_insn)(Register Rt, const Address &adr);
4808 typedef void (MacroAssembler::* uxt_insn)(Register Rd, Register Rn);
4809
4810 void MacroAssembler::string_indexof_char(Register str1, Register cnt1,
4811 Register ch, Register result,
4812 Register tmp1, Register tmp2, Register tmp3)
4813 {
4814 Label CH1_LOOP, HAS_ZERO, DO1_SHORT, DO1_LOOP, MATCH, NOMATCH, DONE;
4815 Register cnt1_neg = cnt1;
4816 Register ch1 = rscratch1;
4817 Register result_tmp = rscratch2;
4818
4819 cmp(cnt1, (u1)4);
4820 br(LT, DO1_SHORT);
4821
4822 orr(ch, ch, ch, LSL, 16);
4823 orr(ch, ch, ch, LSL, 32);
4824
4825 sub(cnt1, cnt1, 4);
4826 mov(result_tmp, cnt1);
4827 lea(str1, Address(str1, cnt1, Address::uxtw(1)));
4828 sub(cnt1_neg, zr, cnt1, LSL, 1);
4829
4830 mov(tmp3, 0x0001000100010001);
4831
4832 BIND(CH1_LOOP);
4833 ldr(ch1, Address(str1, cnt1_neg));
4834 eor(ch1, ch, ch1);
4835 sub(tmp1, ch1, tmp3);
4836 orr(tmp2, ch1, 0x7fff7fff7fff7fff);
4837 bics(tmp1, tmp1, tmp2);
4838 br(NE, HAS_ZERO);
4839 adds(cnt1_neg, cnt1_neg, 8);
4840 br(LT, CH1_LOOP);
4841
4842 cmp(cnt1_neg, (u1)8);
4843 mov(cnt1_neg, 0);
4844 br(LT, CH1_LOOP);
4845 b(NOMATCH);
4846
4847 BIND(HAS_ZERO);
4848 rev(tmp1, tmp1);
4849 clz(tmp1, tmp1);
4850 add(cnt1_neg, cnt1_neg, tmp1, LSR, 3);
4851 b(MATCH);
4852
4853 BIND(DO1_SHORT);
4854 mov(result_tmp, cnt1);
4855 lea(str1, Address(str1, cnt1, Address::uxtw(1)));
4856 sub(cnt1_neg, zr, cnt1, LSL, 1);
4857 BIND(DO1_LOOP);
4858 ldrh(ch1, Address(str1, cnt1_neg));
4859 cmpw(ch, ch1);
4860 br(EQ, MATCH);
4861 adds(cnt1_neg, cnt1_neg, 2);
4862 br(LT, DO1_LOOP);
4863 BIND(NOMATCH);
4864 mov(result, -1);
4865 b(DONE);
4866 BIND(MATCH);
4867 add(result, result_tmp, cnt1_neg, ASR, 1);
4868 BIND(DONE);
4869 }
4870
4871 // Compare strings.
4872 void MacroAssembler::string_compare(Register str1, Register str2,
4873 Register cnt1, Register cnt2, Register result, Register tmp1, Register tmp2,
4874 FloatRegister vtmp1, FloatRegister vtmp2, FloatRegister vtmp3, int ae) {
4875 Label DONE, SHORT_LOOP, SHORT_STRING, SHORT_LAST, TAIL, STUB,
4876 DIFFERENCE, NEXT_WORD, SHORT_LOOP_TAIL, SHORT_LAST2, SHORT_LAST_INIT,
4877 SHORT_LOOP_START, TAIL_CHECK;
4878
4879 const u1 STUB_THRESHOLD = 64 + 8;
4880 bool isLL = ae == StrIntrinsicNode::LL;
4881 bool isLU = ae == StrIntrinsicNode::LU;
4882 bool isUL = ae == StrIntrinsicNode::UL;
4883
4884 bool str1_isL = isLL || isLU;
4885 bool str2_isL = isLL || isUL;
4886
4887 int str1_chr_shift = str1_isL ? 0 : 1;
4888 int str2_chr_shift = str2_isL ? 0 : 1;
4889 int str1_chr_size = str1_isL ? 1 : 2;
4890 int str2_chr_size = str2_isL ? 1 : 2;
4891 int minCharsInWord = isLL ? wordSize : wordSize/2;
4892
4893 FloatRegister vtmpZ = vtmp1, vtmp = vtmp2;
4894 chr_insn str1_load_chr = str1_isL ? (chr_insn)&MacroAssembler::ldrb :
4895 (chr_insn)&MacroAssembler::ldrh;
4896 chr_insn str2_load_chr = str2_isL ? (chr_insn)&MacroAssembler::ldrb :
4897 (chr_insn)&MacroAssembler::ldrh;
4898 uxt_insn ext_chr = isLL ? (uxt_insn)&MacroAssembler::uxtbw :
4899 (uxt_insn)&MacroAssembler::uxthw;
4900
4901 BLOCK_COMMENT("string_compare {");
4902
4903 // Bizzarely, the counts are passed in bytes, regardless of whether they
4904 // are L or U strings, however the result is always in characters.
4905 if (!str1_isL) asrw(cnt1, cnt1, 1);
4906 if (!str2_isL) asrw(cnt2, cnt2, 1);
4907
4908 // Compute the minimum of the string lengths and save the difference.
4909 subsw(result, cnt1, cnt2);
4910 cselw(cnt2, cnt1, cnt2, Assembler::LE); // min
4911
4912 // A very short string
4913 cmpw(cnt2, minCharsInWord);
4914 br(Assembler::LE, SHORT_STRING);
4915
4916 // Compare longwords
4917 // load first parts of strings and finish initialization while loading
4918 {
4919 if (str1_isL == str2_isL) { // LL or UU
4920 ldr(tmp1, Address(str1));
4921 cmp(str1, str2);
4922 br(Assembler::EQ, DONE);
4923 ldr(tmp2, Address(str2));
4924 cmp(cnt2, STUB_THRESHOLD);
4925 br(GE, STUB);
4926 subsw(cnt2, cnt2, minCharsInWord);
4927 br(EQ, TAIL_CHECK);
4928 lea(str2, Address(str2, cnt2, Address::uxtw(str2_chr_shift)));
4929 lea(str1, Address(str1, cnt2, Address::uxtw(str1_chr_shift)));
4930 sub(cnt2, zr, cnt2, LSL, str2_chr_shift);
4931 } else if (isLU) {
4932 ldrs(vtmp, Address(str1));
4933 cmp(str1, str2);
4934 br(Assembler::EQ, DONE);
4935 ldr(tmp2, Address(str2));
4936 cmp(cnt2, STUB_THRESHOLD);
4937 br(GE, STUB);
4938 subw(cnt2, cnt2, 4);
4939 eor(vtmpZ, T16B, vtmpZ, vtmpZ);
4940 lea(str1, Address(str1, cnt2, Address::uxtw(str1_chr_shift)));
4941 lea(str2, Address(str2, cnt2, Address::uxtw(str2_chr_shift)));
4942 zip1(vtmp, T8B, vtmp, vtmpZ);
4943 sub(cnt1, zr, cnt2, LSL, str1_chr_shift);
4944 sub(cnt2, zr, cnt2, LSL, str2_chr_shift);
4945 add(cnt1, cnt1, 4);
4946 fmovd(tmp1, vtmp);
4947 } else { // UL case
4948 ldr(tmp1, Address(str1));
4949 cmp(str1, str2);
4950 br(Assembler::EQ, DONE);
4951 ldrs(vtmp, Address(str2));
4952 cmp(cnt2, STUB_THRESHOLD);
4953 br(GE, STUB);
4954 subw(cnt2, cnt2, 4);
4955 lea(str1, Address(str1, cnt2, Address::uxtw(str1_chr_shift)));
4956 eor(vtmpZ, T16B, vtmpZ, vtmpZ);
4957 lea(str2, Address(str2, cnt2, Address::uxtw(str2_chr_shift)));
4958 sub(cnt1, zr, cnt2, LSL, str1_chr_shift);
4959 zip1(vtmp, T8B, vtmp, vtmpZ);
4960 sub(cnt2, zr, cnt2, LSL, str2_chr_shift);
4961 add(cnt1, cnt1, 8);
4962 fmovd(tmp2, vtmp);
4963 }
4964 adds(cnt2, cnt2, isUL ? 4 : 8);
4965 br(GE, TAIL);
4966 eor(rscratch2, tmp1, tmp2);
4967 cbnz(rscratch2, DIFFERENCE);
4968 // main loop
4969 bind(NEXT_WORD);
4970 if (str1_isL == str2_isL) {
4971 ldr(tmp1, Address(str1, cnt2));
4972 ldr(tmp2, Address(str2, cnt2));
4973 adds(cnt2, cnt2, 8);
4974 } else if (isLU) {
4975 ldrs(vtmp, Address(str1, cnt1));
4976 ldr(tmp2, Address(str2, cnt2));
4977 add(cnt1, cnt1, 4);
4978 zip1(vtmp, T8B, vtmp, vtmpZ);
4979 fmovd(tmp1, vtmp);
4980 adds(cnt2, cnt2, 8);
4981 } else { // UL
4982 ldrs(vtmp, Address(str2, cnt2));
4983 ldr(tmp1, Address(str1, cnt1));
4984 zip1(vtmp, T8B, vtmp, vtmpZ);
4985 add(cnt1, cnt1, 8);
4986 fmovd(tmp2, vtmp);
4987 adds(cnt2, cnt2, 4);
4988 }
4989 br(GE, TAIL);
4990
4991 eor(rscratch2, tmp1, tmp2);
4992 cbz(rscratch2, NEXT_WORD);
4993 b(DIFFERENCE);
4994 bind(TAIL);
4995 eor(rscratch2, tmp1, tmp2);
4996 cbnz(rscratch2, DIFFERENCE);
4997 // Last longword. In the case where length == 4 we compare the
4998 // same longword twice, but that's still faster than another
4999 // conditional branch.
5000 if (str1_isL == str2_isL) {
5001 ldr(tmp1, Address(str1));
5002 ldr(tmp2, Address(str2));
5003 } else if (isLU) {
5004 ldrs(vtmp, Address(str1));
5005 ldr(tmp2, Address(str2));
5006 zip1(vtmp, T8B, vtmp, vtmpZ);
5007 fmovd(tmp1, vtmp);
5008 } else { // UL
5009 ldrs(vtmp, Address(str2));
5010 ldr(tmp1, Address(str1));
5011 zip1(vtmp, T8B, vtmp, vtmpZ);
5012 fmovd(tmp2, vtmp);
5013 }
5014 bind(TAIL_CHECK);
5015 eor(rscratch2, tmp1, tmp2);
5016 cbz(rscratch2, DONE);
5017
5018 // Find the first different characters in the longwords and
5019 // compute their difference.
5020 bind(DIFFERENCE);
5021 rev(rscratch2, rscratch2);
5022 clz(rscratch2, rscratch2);
5023 andr(rscratch2, rscratch2, isLL ? -8 : -16);
5024 lsrv(tmp1, tmp1, rscratch2);
5025 (this->*ext_chr)(tmp1, tmp1);
5026 lsrv(tmp2, tmp2, rscratch2);
5027 (this->*ext_chr)(tmp2, tmp2);
5028 subw(result, tmp1, tmp2);
5029 b(DONE);
5030 }
5031
5032 bind(STUB);
5033 RuntimeAddress stub = NULL;
5034 switch(ae) {
5035 case StrIntrinsicNode::LL:
5036 stub = RuntimeAddress(StubRoutines::aarch64::compare_long_string_LL());
5037 break;
5038 case StrIntrinsicNode::UU:
5039 stub = RuntimeAddress(StubRoutines::aarch64::compare_long_string_UU());
5040 break;
5041 case StrIntrinsicNode::LU:
5042 stub = RuntimeAddress(StubRoutines::aarch64::compare_long_string_LU());
5043 break;
5044 case StrIntrinsicNode::UL:
5045 stub = RuntimeAddress(StubRoutines::aarch64::compare_long_string_UL());
5046 break;
5047 default:
5048 ShouldNotReachHere();
5049 }
5050 assert(stub.target() != NULL, "compare_long_string stub has not been generated");
5051 trampoline_call(stub);
5052 b(DONE);
5053
5054 bind(SHORT_STRING);
5055 // Is the minimum length zero?
5056 cbz(cnt2, DONE);
5057 // arrange code to do most branches while loading and loading next characters
5058 // while comparing previous
5059 (this->*str1_load_chr)(tmp1, Address(post(str1, str1_chr_size)));
5060 subs(cnt2, cnt2, 1);
5061 br(EQ, SHORT_LAST_INIT);
5062 (this->*str2_load_chr)(cnt1, Address(post(str2, str2_chr_size)));
5063 b(SHORT_LOOP_START);
5064 bind(SHORT_LOOP);
5065 subs(cnt2, cnt2, 1);
5066 br(EQ, SHORT_LAST);
5067 bind(SHORT_LOOP_START);
5068 (this->*str1_load_chr)(tmp2, Address(post(str1, str1_chr_size)));
5069 (this->*str2_load_chr)(rscratch1, Address(post(str2, str2_chr_size)));
5070 cmp(tmp1, cnt1);
5071 br(NE, SHORT_LOOP_TAIL);
5072 subs(cnt2, cnt2, 1);
5073 br(EQ, SHORT_LAST2);
5074 (this->*str1_load_chr)(tmp1, Address(post(str1, str1_chr_size)));
5075 (this->*str2_load_chr)(cnt1, Address(post(str2, str2_chr_size)));
5076 cmp(tmp2, rscratch1);
5077 br(EQ, SHORT_LOOP);
5078 sub(result, tmp2, rscratch1);
5079 b(DONE);
5080 bind(SHORT_LOOP_TAIL);
5081 sub(result, tmp1, cnt1);
5082 b(DONE);
5083 bind(SHORT_LAST2);
5084 cmp(tmp2, rscratch1);
5085 br(EQ, DONE);
5086 sub(result, tmp2, rscratch1);
5087
5088 b(DONE);
5089 bind(SHORT_LAST_INIT);
5090 (this->*str2_load_chr)(cnt1, Address(post(str2, str2_chr_size)));
5091 bind(SHORT_LAST);
5092 cmp(tmp1, cnt1);
5093 br(EQ, DONE);
5094 sub(result, tmp1, cnt1);
5095
5096 bind(DONE);
5097
5098 BLOCK_COMMENT("} string_compare");
5099 }
5100 #endif // COMPILER2
5101
5102 // This method checks if provided byte array contains byte with highest bit set.
5103 void MacroAssembler::has_negatives(Register ary1, Register len, Register result) {
5104 // Simple and most common case of aligned small array which is not at the
5105 // end of memory page is placed here. All other cases are in stub.
5106 Label LOOP, END, STUB, STUB_LONG, SET_RESULT, DONE;
5107 const uint64_t UPPER_BIT_MASK=0x8080808080808080;
5108 assert_different_registers(ary1, len, result);
5109
5110 cmpw(len, 0);
5111 br(LE, SET_RESULT);
5112 cmpw(len, 4 * wordSize);
5113 br(GE, STUB_LONG); // size > 32 then go to stub
5114
5115 int shift = 64 - exact_log2(os::vm_page_size());
5116 lsl(rscratch1, ary1, shift);
5117 mov(rscratch2, (size_t)(4 * wordSize) << shift);
5118 adds(rscratch2, rscratch1, rscratch2); // At end of page?
5119 br(CS, STUB); // at the end of page then go to stub
5120 subs(len, len, wordSize);
5121 br(LT, END);
5122
5123 BIND(LOOP);
5124 ldr(rscratch1, Address(post(ary1, wordSize)));
5125 tst(rscratch1, UPPER_BIT_MASK);
5126 br(NE, SET_RESULT);
5127 subs(len, len, wordSize);
5128 br(GE, LOOP);
5129 cmpw(len, -wordSize);
5130 br(EQ, SET_RESULT);
5131
5132 BIND(END);
5133 ldr(result, Address(ary1));
5134 sub(len, zr, len, LSL, 3); // LSL 3 is to get bits from bytes
5135 lslv(result, result, len);
5136 tst(result, UPPER_BIT_MASK);
5137 b(SET_RESULT);
5138
5139 BIND(STUB);
5140 RuntimeAddress has_neg = RuntimeAddress(StubRoutines::aarch64::has_negatives());
5141 assert(has_neg.target() != NULL, "has_negatives stub has not been generated");
5142 trampoline_call(has_neg);
5143 b(DONE);
5144
5145 BIND(STUB_LONG);
5146 RuntimeAddress has_neg_long = RuntimeAddress(
5147 StubRoutines::aarch64::has_negatives_long());
5148 assert(has_neg_long.target() != NULL, "has_negatives stub has not been generated");
5149 trampoline_call(has_neg_long);
5150 b(DONE);
5151
5152 BIND(SET_RESULT);
5153 cset(result, NE); // set true or false
5154
5155 BIND(DONE);
5156 }
5157
5158 void MacroAssembler::arrays_equals(Register a1, Register a2, Register tmp3,
5159 Register tmp4, Register tmp5, Register result,
5160 Register cnt1, int elem_size) {
5161 Label DONE, SAME;
5162 Register tmp1 = rscratch1;
5163 Register tmp2 = rscratch2;
5164 Register cnt2 = tmp2; // cnt2 only used in array length compare
5165 int elem_per_word = wordSize/elem_size;
5166 int log_elem_size = exact_log2(elem_size);
5167 int length_offset = arrayOopDesc::length_offset_in_bytes();
5168 int base_offset
5169 = arrayOopDesc::base_offset_in_bytes(elem_size == 2 ? T_CHAR : T_BYTE);
5170 int stubBytesThreshold = 3 * 64 + (UseSIMDForArrayEquals ? 0 : 16);
5171
5172 assert(elem_size == 1 || elem_size == 2, "must be char or byte");
5173 assert_different_registers(a1, a2, result, cnt1, rscratch1, rscratch2);
5174
5175 #ifndef PRODUCT
5176 {
5177 const char kind = (elem_size == 2) ? 'U' : 'L';
5178 char comment[64];
5179 snprintf(comment, sizeof comment, "array_equals%c{", kind);
5180 BLOCK_COMMENT(comment);
5181 }
5182 #endif
5183
5184 // if (a1 == a2)
5185 // return true;
5186 cmpoop(a1, a2); // May have read barriers for a1 and a2.
5187 br(EQ, SAME);
5188
5189 if (UseSimpleArrayEquals) {
5190 Label NEXT_WORD, SHORT, TAIL03, TAIL01, A_MIGHT_BE_NULL, A_IS_NOT_NULL;
5191 // if (a1 == null || a2 == null)
5192 // return false;
5193 // a1 & a2 == 0 means (some-pointer is null) or
5194 // (very-rare-or-even-probably-impossible-pointer-values)
5195 // so, we can save one branch in most cases
5196 tst(a1, a2);
5197 mov(result, false);
5198 br(EQ, A_MIGHT_BE_NULL);
5199 // if (a1.length != a2.length)
5200 // return false;
5201 bind(A_IS_NOT_NULL);
5202 ldrw(cnt1, Address(a1, length_offset));
5203 ldrw(cnt2, Address(a2, length_offset));
5204 eorw(tmp5, cnt1, cnt2);
5205 cbnzw(tmp5, DONE);
5206 lea(a1, Address(a1, base_offset));
5207 lea(a2, Address(a2, base_offset));
5208 // Check for short strings, i.e. smaller than wordSize.
5209 subs(cnt1, cnt1, elem_per_word);
5210 br(Assembler::LT, SHORT);
5211 // Main 8 byte comparison loop.
5212 bind(NEXT_WORD); {
5213 ldr(tmp1, Address(post(a1, wordSize)));
5214 ldr(tmp2, Address(post(a2, wordSize)));
5215 subs(cnt1, cnt1, elem_per_word);
5216 eor(tmp5, tmp1, tmp2);
5217 cbnz(tmp5, DONE);
5218 } br(GT, NEXT_WORD);
5219 // Last longword. In the case where length == 4 we compare the
5220 // same longword twice, but that's still faster than another
5221 // conditional branch.
5222 // cnt1 could be 0, -1, -2, -3, -4 for chars; -4 only happens when
5223 // length == 4.
5224 if (log_elem_size > 0)
5225 lsl(cnt1, cnt1, log_elem_size);
5226 ldr(tmp3, Address(a1, cnt1));
5227 ldr(tmp4, Address(a2, cnt1));
5228 eor(tmp5, tmp3, tmp4);
5229 cbnz(tmp5, DONE);
5230 b(SAME);
5231 bind(A_MIGHT_BE_NULL);
5232 // in case both a1 and a2 are not-null, proceed with loads
5233 cbz(a1, DONE);
5234 cbz(a2, DONE);
5235 b(A_IS_NOT_NULL);
5236 bind(SHORT);
5237
5238 tbz(cnt1, 2 - log_elem_size, TAIL03); // 0-7 bytes left.
5239 {
5240 ldrw(tmp1, Address(post(a1, 4)));
5241 ldrw(tmp2, Address(post(a2, 4)));
5242 eorw(tmp5, tmp1, tmp2);
5243 cbnzw(tmp5, DONE);
5244 }
5245 bind(TAIL03);
5246 tbz(cnt1, 1 - log_elem_size, TAIL01); // 0-3 bytes left.
5247 {
5248 ldrh(tmp3, Address(post(a1, 2)));
5249 ldrh(tmp4, Address(post(a2, 2)));
5250 eorw(tmp5, tmp3, tmp4);
5251 cbnzw(tmp5, DONE);
5252 }
5253 bind(TAIL01);
5254 if (elem_size == 1) { // Only needed when comparing byte arrays.
5255 tbz(cnt1, 0, SAME); // 0-1 bytes left.
5256 {
5257 ldrb(tmp1, a1);
5258 ldrb(tmp2, a2);
5259 eorw(tmp5, tmp1, tmp2);
5260 cbnzw(tmp5, DONE);
5261 }
5262 }
5263 } else {
5264 Label NEXT_DWORD, SHORT, TAIL, TAIL2, STUB, EARLY_OUT,
5265 CSET_EQ, LAST_CHECK;
5266 mov(result, false);
5267 cbz(a1, DONE);
5268 ldrw(cnt1, Address(a1, length_offset));
5269 cbz(a2, DONE);
5270 ldrw(cnt2, Address(a2, length_offset));
5271 // on most CPUs a2 is still "locked"(surprisingly) in ldrw and it's
5272 // faster to perform another branch before comparing a1 and a2
5273 cmp(cnt1, (u1)elem_per_word);
5274 br(LE, SHORT); // short or same
5275 ldr(tmp3, Address(pre(a1, base_offset)));
5276 subs(zr, cnt1, stubBytesThreshold);
5277 br(GE, STUB);
5278 ldr(tmp4, Address(pre(a2, base_offset)));
5279 sub(tmp5, zr, cnt1, LSL, 3 + log_elem_size);
5280 cmp(cnt2, cnt1);
5281 br(NE, DONE);
5282
5283 // Main 16 byte comparison loop with 2 exits
5284 bind(NEXT_DWORD); {
5285 ldr(tmp1, Address(pre(a1, wordSize)));
5286 ldr(tmp2, Address(pre(a2, wordSize)));
5287 subs(cnt1, cnt1, 2 * elem_per_word);
5288 br(LE, TAIL);
5289 eor(tmp4, tmp3, tmp4);
5290 cbnz(tmp4, DONE);
5291 ldr(tmp3, Address(pre(a1, wordSize)));
5292 ldr(tmp4, Address(pre(a2, wordSize)));
5293 cmp(cnt1, (u1)elem_per_word);
5294 br(LE, TAIL2);
5295 cmp(tmp1, tmp2);
5296 } br(EQ, NEXT_DWORD);
5297 b(DONE);
5298
5299 bind(TAIL);
5300 eor(tmp4, tmp3, tmp4);
5301 eor(tmp2, tmp1, tmp2);
5302 lslv(tmp2, tmp2, tmp5);
5303 orr(tmp5, tmp4, tmp2);
5304 cmp(tmp5, zr);
5305 b(CSET_EQ);
5306
5307 bind(TAIL2);
5308 eor(tmp2, tmp1, tmp2);
5309 cbnz(tmp2, DONE);
5310 b(LAST_CHECK);
5311
5312 bind(STUB);
5313 ldr(tmp4, Address(pre(a2, base_offset)));
5314 cmp(cnt2, cnt1);
5315 br(NE, DONE);
5316 if (elem_size == 2) { // convert to byte counter
5317 lsl(cnt1, cnt1, 1);
5318 }
5319 eor(tmp5, tmp3, tmp4);
5320 cbnz(tmp5, DONE);
5321 RuntimeAddress stub = RuntimeAddress(StubRoutines::aarch64::large_array_equals());
5322 assert(stub.target() != NULL, "array_equals_long stub has not been generated");
5323 trampoline_call(stub);
5324 b(DONE);
5325
5326 bind(EARLY_OUT);
5327 // (a1 != null && a2 == null) || (a1 != null && a2 != null && a1 == a2)
5328 // so, if a2 == null => return false(0), else return true, so we can return a2
5329 mov(result, a2);
5330 b(DONE);
5331 bind(SHORT);
5332 cmp(cnt2, cnt1);
5333 br(NE, DONE);
5334 cbz(cnt1, SAME);
5335 sub(tmp5, zr, cnt1, LSL, 3 + log_elem_size);
5336 ldr(tmp3, Address(a1, base_offset));
5337 ldr(tmp4, Address(a2, base_offset));
5338 bind(LAST_CHECK);
5339 eor(tmp4, tmp3, tmp4);
5340 lslv(tmp5, tmp4, tmp5);
5341 cmp(tmp5, zr);
5342 bind(CSET_EQ);
5343 cset(result, EQ);
5344 b(DONE);
5345 }
5346
5347 bind(SAME);
5348 mov(result, true);
5349 // That's it.
5350 bind(DONE);
5351
5352 BLOCK_COMMENT("} array_equals");
5353 }
5354
5355 // Compare Strings
5356
5357 // For Strings we're passed the address of the first characters in a1
5358 // and a2 and the length in cnt1.
5359 // elem_size is the element size in bytes: either 1 or 2.
5360 // There are two implementations. For arrays >= 8 bytes, all
5361 // comparisons (including the final one, which may overlap) are
5362 // performed 8 bytes at a time. For strings < 8 bytes, we compare a
5363 // halfword, then a short, and then a byte.
5364
5365 void MacroAssembler::string_equals(Register a1, Register a2,
5366 Register result, Register cnt1, int elem_size)
5367 {
5368 Label SAME, DONE, SHORT, NEXT_WORD;
5369 Register tmp1 = rscratch1;
5370 Register tmp2 = rscratch2;
5371 Register cnt2 = tmp2; // cnt2 only used in array length compare
5372
5373 assert(elem_size == 1 || elem_size == 2, "must be 2 or 1 byte");
5374 assert_different_registers(a1, a2, result, cnt1, rscratch1, rscratch2);
5375
5376 #ifndef PRODUCT
5377 {
5378 const char kind = (elem_size == 2) ? 'U' : 'L';
5379 char comment[64];
5380 snprintf(comment, sizeof comment, "{string_equals%c", kind);
5381 BLOCK_COMMENT(comment);
5382 }
5383 #endif
5384
5385 mov(result, false);
5386
5387 // Check for short strings, i.e. smaller than wordSize.
5388 subs(cnt1, cnt1, wordSize);
5389 br(Assembler::LT, SHORT);
5390 // Main 8 byte comparison loop.
5391 bind(NEXT_WORD); {
5392 ldr(tmp1, Address(post(a1, wordSize)));
5393 ldr(tmp2, Address(post(a2, wordSize)));
5394 subs(cnt1, cnt1, wordSize);
5395 eor(tmp1, tmp1, tmp2);
5396 cbnz(tmp1, DONE);
5397 } br(GT, NEXT_WORD);
5398 // Last longword. In the case where length == 4 we compare the
5399 // same longword twice, but that's still faster than another
5400 // conditional branch.
5401 // cnt1 could be 0, -1, -2, -3, -4 for chars; -4 only happens when
5402 // length == 4.
5403 ldr(tmp1, Address(a1, cnt1));
5404 ldr(tmp2, Address(a2, cnt1));
5405 eor(tmp2, tmp1, tmp2);
5406 cbnz(tmp2, DONE);
5407 b(SAME);
5408
5409 bind(SHORT);
5410 Label TAIL03, TAIL01;
5411
5412 tbz(cnt1, 2, TAIL03); // 0-7 bytes left.
5413 {
5414 ldrw(tmp1, Address(post(a1, 4)));
5415 ldrw(tmp2, Address(post(a2, 4)));
5416 eorw(tmp1, tmp1, tmp2);
5417 cbnzw(tmp1, DONE);
5418 }
5419 bind(TAIL03);
5420 tbz(cnt1, 1, TAIL01); // 0-3 bytes left.
5421 {
5422 ldrh(tmp1, Address(post(a1, 2)));
5423 ldrh(tmp2, Address(post(a2, 2)));
5424 eorw(tmp1, tmp1, tmp2);
5425 cbnzw(tmp1, DONE);
5426 }
5427 bind(TAIL01);
5428 if (elem_size == 1) { // Only needed when comparing 1-byte elements
5429 tbz(cnt1, 0, SAME); // 0-1 bytes left.
5430 {
5431 ldrb(tmp1, a1);
5432 ldrb(tmp2, a2);
5433 eorw(tmp1, tmp1, tmp2);
5434 cbnzw(tmp1, DONE);
5435 }
5436 }
5437 // Arrays are equal.
5438 bind(SAME);
5439 mov(result, true);
5440
5441 // That's it.
5442 bind(DONE);
5443 BLOCK_COMMENT("} string_equals");
5444 }
5445
5446
5447 // The size of the blocks erased by the zero_blocks stub. We must
5448 // handle anything smaller than this ourselves in zero_words().
5449 const int MacroAssembler::zero_words_block_size = 8;
5450
5451 // zero_words() is used by C2 ClearArray patterns. It is as small as
5452 // possible, handling small word counts locally and delegating
5453 // anything larger to the zero_blocks stub. It is expanded many times
5454 // in compiled code, so it is important to keep it short.
5455
5456 // ptr: Address of a buffer to be zeroed.
5457 // cnt: Count in HeapWords.
5458 //
5459 // ptr, cnt, rscratch1, and rscratch2 are clobbered.
5460 void MacroAssembler::zero_words(Register ptr, Register cnt)
5461 {
5462 assert(is_power_of_2(zero_words_block_size), "adjust this");
5463 assert(ptr == r10 && cnt == r11, "mismatch in register usage");
5464
5465 BLOCK_COMMENT("zero_words {");
5466 cmp(cnt, (u1)zero_words_block_size);
5467 Label around;
5468 br(LO, around);
5469 {
5470 RuntimeAddress zero_blocks = RuntimeAddress(StubRoutines::aarch64::zero_blocks());
5471 assert(zero_blocks.target() != NULL, "zero_blocks stub has not been generated");
5472 if (StubRoutines::aarch64::complete()) {
5473 trampoline_call(zero_blocks);
5474 } else {
5475 bl(zero_blocks);
5476 }
5477 }
5478 bind(around);
5479 for (int i = zero_words_block_size >> 1; i > 1; i >>= 1) {
5480 Label l;
5481 tbz(cnt, exact_log2(i), l);
5482 for (int j = 0; j < i; j += 2) {
5483 stp(zr, zr, post(ptr, 16));
5484 }
5485 bind(l);
5486 }
5487 {
5488 Label l;
5489 tbz(cnt, 0, l);
5490 str(zr, Address(ptr));
5491 bind(l);
5492 }
5493 BLOCK_COMMENT("} zero_words");
5494 }
5495
5496 // base: Address of a buffer to be zeroed, 8 bytes aligned.
5497 // cnt: Immediate count in HeapWords.
5498 #define SmallArraySize (18 * BytesPerLong)
5499 void MacroAssembler::zero_words(Register base, u_int64_t cnt)
5500 {
5501 BLOCK_COMMENT("zero_words {");
5502 int i = cnt & 1; // store any odd word to start
5503 if (i) str(zr, Address(base));
5504
5505 if (cnt <= SmallArraySize / BytesPerLong) {
5506 for (; i < (int)cnt; i += 2)
5507 stp(zr, zr, Address(base, i * wordSize));
5508 } else {
5509 const int unroll = 4; // Number of stp(zr, zr) instructions we'll unroll
5510 int remainder = cnt % (2 * unroll);
5511 for (; i < remainder; i += 2)
5512 stp(zr, zr, Address(base, i * wordSize));
5513
5514 Label loop;
5515 Register cnt_reg = rscratch1;
5516 Register loop_base = rscratch2;
5517 cnt = cnt - remainder;
5518 mov(cnt_reg, cnt);
5519 // adjust base and prebias by -2 * wordSize so we can pre-increment
5520 add(loop_base, base, (remainder - 2) * wordSize);
5521 bind(loop);
5522 sub(cnt_reg, cnt_reg, 2 * unroll);
5523 for (i = 1; i < unroll; i++)
5524 stp(zr, zr, Address(loop_base, 2 * i * wordSize));
5525 stp(zr, zr, Address(pre(loop_base, 2 * unroll * wordSize)));
5526 cbnz(cnt_reg, loop);
5527 }
5528 BLOCK_COMMENT("} zero_words");
5529 }
5530
5531 // Zero blocks of memory by using DC ZVA.
5532 //
5533 // Aligns the base address first sufficently for DC ZVA, then uses
5534 // DC ZVA repeatedly for every full block. cnt is the size to be
5535 // zeroed in HeapWords. Returns the count of words left to be zeroed
5536 // in cnt.
5537 //
5538 // NOTE: This is intended to be used in the zero_blocks() stub. If
5539 // you want to use it elsewhere, note that cnt must be >= 2*zva_length.
5540 void MacroAssembler::zero_dcache_blocks(Register base, Register cnt) {
5541 Register tmp = rscratch1;
5542 Register tmp2 = rscratch2;
5543 int zva_length = VM_Version::zva_length();
5544 Label initial_table_end, loop_zva;
5545 Label fini;
5546
5547 // Base must be 16 byte aligned. If not just return and let caller handle it
5548 tst(base, 0x0f);
5549 br(Assembler::NE, fini);
5550 // Align base with ZVA length.
5551 neg(tmp, base);
5552 andr(tmp, tmp, zva_length - 1);
5553
5554 // tmp: the number of bytes to be filled to align the base with ZVA length.
5555 add(base, base, tmp);
5556 sub(cnt, cnt, tmp, Assembler::ASR, 3);
5557 adr(tmp2, initial_table_end);
5558 sub(tmp2, tmp2, tmp, Assembler::LSR, 2);
5559 br(tmp2);
5560
5561 for (int i = -zva_length + 16; i < 0; i += 16)
5562 stp(zr, zr, Address(base, i));
5563 bind(initial_table_end);
5564
5565 sub(cnt, cnt, zva_length >> 3);
5566 bind(loop_zva);
5567 dc(Assembler::ZVA, base);
5568 subs(cnt, cnt, zva_length >> 3);
5569 add(base, base, zva_length);
5570 br(Assembler::GE, loop_zva);
5571 add(cnt, cnt, zva_length >> 3); // count not zeroed by DC ZVA
5572 bind(fini);
5573 }
5574
5575 // base: Address of a buffer to be filled, 8 bytes aligned.
5576 // cnt: Count in 8-byte unit.
5577 // value: Value to be filled with.
5578 // base will point to the end of the buffer after filling.
5579 void MacroAssembler::fill_words(Register base, Register cnt, Register value)
5580 {
5581 // Algorithm:
5582 //
5583 // scratch1 = cnt & 7;
5584 // cnt -= scratch1;
5585 // p += scratch1;
5586 // switch (scratch1) {
5587 // do {
5588 // cnt -= 8;
5589 // p[-8] = v;
5590 // case 7:
5591 // p[-7] = v;
5592 // case 6:
5593 // p[-6] = v;
5594 // // ...
5595 // case 1:
5596 // p[-1] = v;
5597 // case 0:
5598 // p += 8;
5599 // } while (cnt);
5600 // }
5601
5602 assert_different_registers(base, cnt, value, rscratch1, rscratch2);
5603
5604 Label fini, skip, entry, loop;
5605 const int unroll = 8; // Number of stp instructions we'll unroll
5606
5607 cbz(cnt, fini);
5608 tbz(base, 3, skip);
5609 str(value, Address(post(base, 8)));
5610 sub(cnt, cnt, 1);
5611 bind(skip);
5612
5613 andr(rscratch1, cnt, (unroll-1) * 2);
5614 sub(cnt, cnt, rscratch1);
5615 add(base, base, rscratch1, Assembler::LSL, 3);
5616 adr(rscratch2, entry);
5617 sub(rscratch2, rscratch2, rscratch1, Assembler::LSL, 1);
5618 br(rscratch2);
5619
5620 bind(loop);
5621 add(base, base, unroll * 16);
5622 for (int i = -unroll; i < 0; i++)
5623 stp(value, value, Address(base, i * 16));
5624 bind(entry);
5625 subs(cnt, cnt, unroll * 2);
5626 br(Assembler::GE, loop);
5627
5628 tbz(cnt, 0, fini);
5629 str(value, Address(post(base, 8)));
5630 bind(fini);
5631 }
5632
5633 // Intrinsic for sun/nio/cs/ISO_8859_1$Encoder.implEncodeISOArray and
5634 // java/lang/StringUTF16.compress.
5635 void MacroAssembler::encode_iso_array(Register src, Register dst,
5636 Register len, Register result,
5637 FloatRegister Vtmp1, FloatRegister Vtmp2,
5638 FloatRegister Vtmp3, FloatRegister Vtmp4)
5639 {
5640 Label DONE, SET_RESULT, NEXT_32, NEXT_32_PRFM, LOOP_8, NEXT_8, LOOP_1, NEXT_1,
5641 NEXT_32_START, NEXT_32_PRFM_START;
5642 Register tmp1 = rscratch1, tmp2 = rscratch2;
5643
5644 mov(result, len); // Save initial len
5645
5646 #ifndef BUILTIN_SIM
5647 cmp(len, (u1)8); // handle shortest strings first
5648 br(LT, LOOP_1);
5649 cmp(len, (u1)32);
5650 br(LT, NEXT_8);
5651 // The following code uses the SIMD 'uzp1' and 'uzp2' instructions
5652 // to convert chars to bytes
5653 if (SoftwarePrefetchHintDistance >= 0) {
5654 ld1(Vtmp1, Vtmp2, Vtmp3, Vtmp4, T8H, src);
5655 subs(tmp2, len, SoftwarePrefetchHintDistance/2 + 16);
5656 br(LE, NEXT_32_START);
5657 b(NEXT_32_PRFM_START);
5658 BIND(NEXT_32_PRFM);
5659 ld1(Vtmp1, Vtmp2, Vtmp3, Vtmp4, T8H, src);
5660 BIND(NEXT_32_PRFM_START);
5661 prfm(Address(src, SoftwarePrefetchHintDistance));
5662 orr(v4, T16B, Vtmp1, Vtmp2);
5663 orr(v5, T16B, Vtmp3, Vtmp4);
5664 uzp1(Vtmp1, T16B, Vtmp1, Vtmp2);
5665 uzp1(Vtmp3, T16B, Vtmp3, Vtmp4);
5666 uzp2(v5, T16B, v4, v5); // high bytes
5667 umov(tmp2, v5, D, 1);
5668 fmovd(tmp1, v5);
5669 orr(tmp1, tmp1, tmp2);
5670 cbnz(tmp1, LOOP_8);
5671 stpq(Vtmp1, Vtmp3, dst);
5672 sub(len, len, 32);
5673 add(dst, dst, 32);
5674 add(src, src, 64);
5675 subs(tmp2, len, SoftwarePrefetchHintDistance/2 + 16);
5676 br(GE, NEXT_32_PRFM);
5677 cmp(len, (u1)32);
5678 br(LT, LOOP_8);
5679 BIND(NEXT_32);
5680 ld1(Vtmp1, Vtmp2, Vtmp3, Vtmp4, T8H, src);
5681 BIND(NEXT_32_START);
5682 } else {
5683 BIND(NEXT_32);
5684 ld1(Vtmp1, Vtmp2, Vtmp3, Vtmp4, T8H, src);
5685 }
5686 prfm(Address(src, SoftwarePrefetchHintDistance));
5687 uzp1(v4, T16B, Vtmp1, Vtmp2);
5688 uzp1(v5, T16B, Vtmp3, Vtmp4);
5689 orr(Vtmp1, T16B, Vtmp1, Vtmp2);
5690 orr(Vtmp3, T16B, Vtmp3, Vtmp4);
5691 uzp2(Vtmp1, T16B, Vtmp1, Vtmp3); // high bytes
5692 umov(tmp2, Vtmp1, D, 1);
5693 fmovd(tmp1, Vtmp1);
5694 orr(tmp1, tmp1, tmp2);
5695 cbnz(tmp1, LOOP_8);
5696 stpq(v4, v5, dst);
5697 sub(len, len, 32);
5698 add(dst, dst, 32);
5699 add(src, src, 64);
5700 cmp(len, (u1)32);
5701 br(GE, NEXT_32);
5702 cbz(len, DONE);
5703
5704 BIND(LOOP_8);
5705 cmp(len, (u1)8);
5706 br(LT, LOOP_1);
5707 BIND(NEXT_8);
5708 ld1(Vtmp1, T8H, src);
5709 uzp1(Vtmp2, T16B, Vtmp1, Vtmp1); // low bytes
5710 uzp2(Vtmp3, T16B, Vtmp1, Vtmp1); // high bytes
5711 fmovd(tmp1, Vtmp3);
5712 cbnz(tmp1, NEXT_1);
5713 strd(Vtmp2, dst);
5714
5715 sub(len, len, 8);
5716 add(dst, dst, 8);
5717 add(src, src, 16);
5718 cmp(len, (u1)8);
5719 br(GE, NEXT_8);
5720
5721 BIND(LOOP_1);
5722 #endif
5723 cbz(len, DONE);
5724 BIND(NEXT_1);
5725 ldrh(tmp1, Address(post(src, 2)));
5726 tst(tmp1, 0xff00);
5727 br(NE, SET_RESULT);
5728 strb(tmp1, Address(post(dst, 1)));
5729 subs(len, len, 1);
5730 br(GT, NEXT_1);
5731
5732 BIND(SET_RESULT);
5733 sub(result, result, len); // Return index where we stopped
5734 // Return len == 0 if we processed all
5735 // characters
5736 BIND(DONE);
5737 }
5738
5739
5740 // Inflate byte[] array to char[].
5741 void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len,
5742 FloatRegister vtmp1, FloatRegister vtmp2, FloatRegister vtmp3,
5743 Register tmp4) {
5744 Label big, done, after_init, to_stub;
5745
5746 assert_different_registers(src, dst, len, tmp4, rscratch1);
5747
5748 fmovd(vtmp1, zr);
5749 lsrw(tmp4, len, 3);
5750 bind(after_init);
5751 cbnzw(tmp4, big);
5752 // Short string: less than 8 bytes.
5753 {
5754 Label loop, tiny;
5755
5756 cmpw(len, 4);
5757 br(LT, tiny);
5758 // Use SIMD to do 4 bytes.
5759 ldrs(vtmp2, post(src, 4));
5760 zip1(vtmp3, T8B, vtmp2, vtmp1);
5761 subw(len, len, 4);
5762 strd(vtmp3, post(dst, 8));
5763
5764 cbzw(len, done);
5765
5766 // Do the remaining bytes by steam.
5767 bind(loop);
5768 ldrb(tmp4, post(src, 1));
5769 strh(tmp4, post(dst, 2));
5770 subw(len, len, 1);
5771
5772 bind(tiny);
5773 cbnz(len, loop);
5774
5775 b(done);
5776 }
5777
5778 if (SoftwarePrefetchHintDistance >= 0) {
5779 bind(to_stub);
5780 RuntimeAddress stub = RuntimeAddress(StubRoutines::aarch64::large_byte_array_inflate());
5781 assert(stub.target() != NULL, "large_byte_array_inflate stub has not been generated");
5782 trampoline_call(stub);
5783 b(after_init);
5784 }
5785
5786 // Unpack the bytes 8 at a time.
5787 bind(big);
5788 {
5789 Label loop, around, loop_last, loop_start;
5790
5791 if (SoftwarePrefetchHintDistance >= 0) {
5792 const int large_loop_threshold = (64 + 16)/8;
5793 ldrd(vtmp2, post(src, 8));
5794 andw(len, len, 7);
5795 cmp(tmp4, (u1)large_loop_threshold);
5796 br(GE, to_stub);
5797 b(loop_start);
5798
5799 bind(loop);
5800 ldrd(vtmp2, post(src, 8));
5801 bind(loop_start);
5802 subs(tmp4, tmp4, 1);
5803 br(EQ, loop_last);
5804 zip1(vtmp2, T16B, vtmp2, vtmp1);
5805 ldrd(vtmp3, post(src, 8));
5806 st1(vtmp2, T8H, post(dst, 16));
5807 subs(tmp4, tmp4, 1);
5808 zip1(vtmp3, T16B, vtmp3, vtmp1);
5809 st1(vtmp3, T8H, post(dst, 16));
5810 br(NE, loop);
5811 b(around);
5812 bind(loop_last);
5813 zip1(vtmp2, T16B, vtmp2, vtmp1);
5814 st1(vtmp2, T8H, post(dst, 16));
5815 bind(around);
5816 cbz(len, done);
5817 } else {
5818 andw(len, len, 7);
5819 bind(loop);
5820 ldrd(vtmp2, post(src, 8));
5821 sub(tmp4, tmp4, 1);
5822 zip1(vtmp3, T16B, vtmp2, vtmp1);
5823 st1(vtmp3, T8H, post(dst, 16));
5824 cbnz(tmp4, loop);
5825 }
5826 }
5827
5828 // Do the tail of up to 8 bytes.
5829 add(src, src, len);
5830 ldrd(vtmp3, Address(src, -8));
5831 add(dst, dst, len, ext::uxtw, 1);
5832 zip1(vtmp3, T16B, vtmp3, vtmp1);
5833 strq(vtmp3, Address(dst, -16));
5834
5835 bind(done);
5836 }
5837
5838 // Compress char[] array to byte[].
5839 void MacroAssembler::char_array_compress(Register src, Register dst, Register len,
5840 FloatRegister tmp1Reg, FloatRegister tmp2Reg,
5841 FloatRegister tmp3Reg, FloatRegister tmp4Reg,
5842 Register result) {
5843 encode_iso_array(src, dst, len, result,
5844 tmp1Reg, tmp2Reg, tmp3Reg, tmp4Reg);
5845 cmp(len, zr);
5846 csel(result, result, zr, EQ);
5847 }
5848
5849 // get_thread() can be called anywhere inside generated code so we
5850 // need to save whatever non-callee save context might get clobbered
5851 // by the call to JavaThread::aarch64_get_thread_helper() or, indeed,
5852 // the call setup code.
5853 //
5854 // aarch64_get_thread_helper() clobbers only r0, r1, and flags.
5855 //
5856 void MacroAssembler::get_thread(Register dst) {
5857 RegSet saved_regs = RegSet::range(r0, r1) + lr - dst;
5858 push(saved_regs, sp);
5859
5860 mov(lr, CAST_FROM_FN_PTR(address, JavaThread::aarch64_get_thread_helper));
5861 blrt(lr, 1, 0, 1);
5862 if (dst != c_rarg0) {
5863 mov(dst, c_rarg0);
5864 }
5865
5866 pop(saved_regs, sp);
5867 }