1 /*
2 * Copyright (c) 2013, Red Hat Inc.
3 * Copyright (c) 2003, 2011, Oracle and/or its affiliates.
4 * All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "precompiled.hpp"
28 #include "asm/macroAssembler.hpp"
29 #include "asm/macroAssembler.inline.hpp"
30 #include "interpreter/interpreter.hpp"
31 #include "nativeInst_aarch64.hpp"
32 #include "oops/instanceOop.hpp"
33 #include "oops/method.hpp"
34 #include "oops/objArrayKlass.hpp"
35 #include "oops/oop.inline.hpp"
36 #include "prims/methodHandles.hpp"
37 #include "runtime/frame.inline.hpp"
38 #include "runtime/handles.inline.hpp"
39 #include "runtime/sharedRuntime.hpp"
40 #include "runtime/stubCodeGenerator.hpp"
41 #include "runtime/stubRoutines.hpp"
42 #include "runtime/thread.inline.hpp"
43 #include "utilities/top.hpp"
44 #ifdef COMPILER2
45 #include "opto/runtime.hpp"
46 #endif
47
48 // Declaration and definition of StubGenerator (no .hpp file).
49 // For a more detailed description of the stub routine structure
50 // see the comment in stubRoutines.hpp
51
52 #undef __
53 #define __ _masm->
54 #define TIMES_OOP Address::sxtw(exact_log2(UseCompressedOops ? 4 : 8))
55
56 #ifdef PRODUCT
57 #define BLOCK_COMMENT(str) /* nothing */
58 #else
59 #define BLOCK_COMMENT(str) __ block_comment(str)
60 #endif
61
62 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
63
64 // Stub Code definitions
65
66 class StubGenerator: public StubCodeGenerator {
67 private:
68
69 #ifdef PRODUCT
70 #define inc_counter_np(counter) ((void)0)
71 #else
72 void inc_counter_np_(int& counter) {
73 __ lea(rscratch2, ExternalAddress((address)&counter));
74 __ ldrw(rscratch1, Address(rscratch2));
75 __ addw(rscratch1, rscratch1, 1);
76 __ strw(rscratch1, Address(rscratch2));
77 }
78 #define inc_counter_np(counter) \
79 BLOCK_COMMENT("inc_counter " #counter); \
80 inc_counter_np_(counter);
81 #endif
82
83 // Call stubs are used to call Java from C
84 //
85 // Arguments:
86 // c_rarg0: call wrapper address address
87 // c_rarg1: result address
88 // c_rarg2: result type BasicType
89 // c_rarg3: method Method*
90 // c_rarg4: (interpreter) entry point address
91 // c_rarg5: parameters intptr_t*
92 // c_rarg6: parameter size (in words) int
93 // c_rarg7: thread Thread*
94 //
95 // There is no return from the stub itself as any Java result
96 // is written to result
97 //
98 // we save r30 (lr) as the return PC at the base of the frame and
99 // link r29 (fp) below it as the frame pointer installing sp (r31)
100 // into fp.
101 //
102 // we save r0-r7, which accounts for all the c arguments.
103 //
104 // TODO: strictly do we need to save them all? they are treated as
105 // volatile by C so could we omit saving the ones we are going to
106 // place in global registers (thread? method?) or those we only use
107 // during setup of the Java call?
108 //
109 // we don't need to save r8 which C uses as an indirect result location
110 // return register.
111 //
112 // we don't need to save r9-r15 which both C and Java treat as
113 // volatile
114 //
115 // we don't need to save r16-18 because Java does not use them
116 //
117 // we save r19-r28 which Java uses as scratch registers and C
118 // expects to be callee-save
119 //
120 // we save the bottom 64 bits of each value stored in v8-v15; it is
121 // the responsibility of the caller to preserve larger values.
122 //
123 // so the stub frame looks like this when we enter Java code
124 //
125 // [ return_from_Java ] <--- sp
126 // [ argument word n ]
127 // ...
128 // -27 [ argument word 1 ]
129 // -26 [ saved v15 ] <--- sp_after_call
130 // -25 [ saved v14 ]
131 // -24 [ saved v13 ]
132 // -23 [ saved v12 ]
133 // -22 [ saved v11 ]
134 // -21 [ saved v10 ]
135 // -20 [ saved v9 ]
136 // -19 [ saved v8 ]
137 // -18 [ saved r28 ]
138 // -17 [ saved r27 ]
139 // -16 [ saved r26 ]
140 // -15 [ saved r25 ]
141 // -14 [ saved r24 ]
142 // -13 [ saved r23 ]
143 // -12 [ saved r22 ]
144 // -11 [ saved r21 ]
145 // -10 [ saved r20 ]
146 // -9 [ saved r19 ]
147 // -8 [ call wrapper (r0) ]
148 // -7 [ result (r1) ]
149 // -6 [ result type (r2) ]
150 // -5 [ method (r3) ]
151 // -4 [ entry point (r4) ]
152 // -3 [ parameters (r5) ]
153 // -2 [ parameter size (r6) ]
154 // -1 [ thread (r7) ]
155 // 0 [ saved fp (r29) ] <--- fp == saved sp (r31)
156 // 1 [ saved lr (r30) ]
157
158 // Call stub stack layout word offsets from fp
159 enum call_stub_layout {
160 sp_after_call_off = -26,
161
162 d15_off = -26,
163 d13_off = -24,
164 d11_off = -22,
165 d9_off = -20,
166
167 r28_off = -18,
168 r26_off = -16,
169 r24_off = -14,
170 r22_off = -12,
171 r20_off = -10,
172 call_wrapper_off = -8,
173 result_off = -7,
174 result_type_off = -6,
175 method_off = -5,
176 entry_point_off = -4,
177 parameter_size_off = -2,
178 thread_off = -1,
179 fp_f = 0,
180 retaddr_off = 1,
181 };
182
183 address generate_call_stub(address& return_address) {
184 assert((int)frame::entry_frame_after_call_words == -(int)sp_after_call_off + 1 &&
185 (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off,
186 "adjust this code");
187
188 StubCodeMark mark(this, "StubRoutines", "call_stub");
189 address start = __ pc();
190
191 const Address sp_after_call(rfp, sp_after_call_off * wordSize);
192
193 const Address call_wrapper (rfp, call_wrapper_off * wordSize);
194 const Address result (rfp, result_off * wordSize);
195 const Address result_type (rfp, result_type_off * wordSize);
196 const Address method (rfp, method_off * wordSize);
197 const Address entry_point (rfp, entry_point_off * wordSize);
198 const Address parameter_size(rfp, parameter_size_off * wordSize);
199
200 const Address thread (rfp, thread_off * wordSize);
201
202 const Address d15_save (rfp, d15_off * wordSize);
203 const Address d13_save (rfp, d13_off * wordSize);
204 const Address d11_save (rfp, d11_off * wordSize);
205 const Address d9_save (rfp, d9_off * wordSize);
206
207 const Address r28_save (rfp, r28_off * wordSize);
208 const Address r26_save (rfp, r26_off * wordSize);
209 const Address r24_save (rfp, r24_off * wordSize);
210 const Address r22_save (rfp, r22_off * wordSize);
211 const Address r20_save (rfp, r20_off * wordSize);
212
213 // stub code
214
215 address aarch64_entry = __ pc();
216
217 // set up frame and move sp to end of save area
218 __ enter();
219 __ sub(sp, rfp, -sp_after_call_off * wordSize);
220
221 // save register parameters and Java scratch/global registers
222 // n.b. we save thread even though it gets installed in
223 // rthread because we want to sanity check rthread later
224 __ str(c_rarg7, thread);
225 __ strw(c_rarg6, parameter_size);
226 __ stp(c_rarg4, c_rarg5, entry_point);
227 __ stp(c_rarg2, c_rarg3, result_type);
228 __ stp(c_rarg0, c_rarg1, call_wrapper);
229
230 __ stp(r20, r19, r20_save);
231 __ stp(r22, r21, r22_save);
232 __ stp(r24, r23, r24_save);
233 __ stp(r26, r25, r26_save);
234 __ stp(r28, r27, r28_save);
235
236 __ stpd(v9, v8, d9_save);
237 __ stpd(v11, v10, d11_save);
238 __ stpd(v13, v12, d13_save);
239 __ stpd(v15, v14, d15_save);
240
241 // install Java thread in global register now we have saved
242 // whatever value it held
243 __ mov(rthread, c_rarg7);
244 // And method
245 __ mov(rmethod, c_rarg3);
246
247 // set up the heapbase register
248 __ reinit_heapbase();
249
250 #ifdef ASSERT
251 // make sure we have no pending exceptions
252 {
253 Label L;
254 __ ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
255 __ cmp(rscratch1, (unsigned)NULL_WORD);
256 __ br(Assembler::EQ, L);
257 __ stop("StubRoutines::call_stub: entered with pending exception");
258 __ BIND(L);
259 }
260 #endif
261 // pass parameters if any
262 __ mov(esp, sp);
263 __ sub(rscratch1, sp, c_rarg6, ext::uxtw, LogBytesPerWord); // Move SP out of the way
264 __ andr(sp, rscratch1, -2 * wordSize);
265
266 BLOCK_COMMENT("pass parameters if any");
267 Label parameters_done;
268 // parameter count is still in c_rarg6
269 // and parameter pointer identifying param 1 is in c_rarg5
270 __ cbzw(c_rarg6, parameters_done);
271
272 address loop = __ pc();
273 __ ldr(rscratch1, Address(__ post(c_rarg5, wordSize)));
274 __ subsw(c_rarg6, c_rarg6, 1);
275 __ push(rscratch1);
276 __ br(Assembler::GT, loop);
277
278 __ BIND(parameters_done);
279
280 // call Java entry -- passing methdoOop, and current sp
281 // rmethod: Method*
282 // r13: sender sp
283 BLOCK_COMMENT("call Java function");
284 __ mov(r13, sp);
285 __ blr(c_rarg4);
286
287 // we do this here because the notify will already have been done
288 // if we get to the next instruction via an exception
289 //
290 // n.b. adding this instruction here affects the calculation of
291 // whether or not a routine returns to the call stub (used when
292 // doing stack walks) since the normal test is to check the return
293 // pc against the address saved below. so we may need to allow for
294 // this extra instruction in the check.
295
296 // save current address for use by exception handling code
297
298 return_address = __ pc();
299
300 // store result depending on type (everything that is not
301 // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
302 // n.b. this assumes Java returns an integral result in r0
303 // and a floating result in j_farg0
304 __ ldr(j_rarg2, result);
305 Label is_long, is_float, is_double, exit;
306 __ ldr(j_rarg1, result_type);
307 __ cmp(j_rarg1, T_OBJECT);
308 __ br(Assembler::EQ, is_long);
309 __ cmp(j_rarg1, T_LONG);
310 __ br(Assembler::EQ, is_long);
311 __ cmp(j_rarg1, T_FLOAT);
312 __ br(Assembler::EQ, is_float);
313 __ cmp(j_rarg1, T_DOUBLE);
314 __ br(Assembler::EQ, is_double);
315
316 // handle T_INT case
317 __ strw(r0, Address(j_rarg2));
318
319 __ BIND(exit);
320
321 // pop parameters
322 __ sub(esp, rfp, -sp_after_call_off * wordSize);
323
324 #ifdef ASSERT
325 // verify that threads correspond
326 {
327 Label L, S;
328 __ ldr(rscratch1, thread);
329 __ cmp(rthread, rscratch1);
330 __ br(Assembler::NE, S);
331 __ get_thread(rscratch1);
332 __ cmp(rthread, rscratch1);
333 __ br(Assembler::EQ, L);
334 __ BIND(S);
335 __ stop("StubRoutines::call_stub: threads must correspond");
336 __ BIND(L);
337 }
338 #endif
339
340 // restore callee-save registers
341 __ ldpd(v15, v14, d15_save);
342 __ ldpd(v13, v12, d13_save);
343 __ ldpd(v11, v10, d11_save);
344 __ ldpd(v9, v8, d9_save);
345
346 __ ldp(r28, r27, r28_save);
347 __ ldp(r26, r25, r26_save);
348 __ ldp(r24, r23, r24_save);
349 __ ldp(r22, r21, r22_save);
350 __ ldp(r20, r19, r20_save);
351
352 __ ldp(c_rarg0, c_rarg1, call_wrapper);
353 __ ldrw(c_rarg2, result_type);
354 __ ldr(c_rarg3, method);
355 __ ldp(c_rarg4, c_rarg5, entry_point);
356 __ ldp(c_rarg6, c_rarg7, parameter_size);
357
358 // leave frame and return to caller
359 __ leave();
360 __ ret(lr);
361
362 // handle return types different from T_INT
363
364 __ BIND(is_long);
365 __ str(r0, Address(j_rarg2, 0));
366 __ br(Assembler::AL, exit);
367
368 __ BIND(is_float);
369 __ strs(j_farg0, Address(j_rarg2, 0));
370 __ br(Assembler::AL, exit);
371
372 __ BIND(is_double);
373 __ strd(j_farg0, Address(j_rarg2, 0));
374 __ br(Assembler::AL, exit);
375
376 return start;
377 }
378
379 // Return point for a Java call if there's an exception thrown in
380 // Java code. The exception is caught and transformed into a
381 // pending exception stored in JavaThread that can be tested from
382 // within the VM.
383 //
384 // Note: Usually the parameters are removed by the callee. In case
385 // of an exception crossing an activation frame boundary, that is
386 // not the case if the callee is compiled code => need to setup the
387 // rsp.
388 //
389 // r0: exception oop
390
391 address generate_catch_exception() {
392 StubCodeMark mark(this, "StubRoutines", "catch_exception");
393 address start = __ pc();
394
395 // same as in generate_call_stub():
396 const Address sp_after_call(rfp, sp_after_call_off * wordSize);
397 const Address thread (rfp, thread_off * wordSize);
398
399 #ifdef ASSERT
400 // verify that threads correspond
401 {
402 Label L, S;
403 __ ldr(rscratch1, thread);
404 __ cmp(rthread, rscratch1);
405 __ br(Assembler::NE, S);
406 __ get_thread(rscratch1);
407 __ cmp(rthread, rscratch1);
408 __ br(Assembler::EQ, L);
409 __ bind(S);
410 __ stop("StubRoutines::catch_exception: threads must correspond");
411 __ bind(L);
412 }
413 #endif
414
415 // set pending exception
416 __ verify_oop(r0);
417
418 __ str(r0, Address(rthread, Thread::pending_exception_offset()));
419 __ mov(rscratch1, (address)__FILE__);
420 __ str(rscratch1, Address(rthread, Thread::exception_file_offset()));
421 __ movw(rscratch1, (int)__LINE__);
422 __ strw(rscratch1, Address(rthread, Thread::exception_line_offset()));
423
424 // complete return to VM
425 assert(StubRoutines::_call_stub_return_address != NULL,
426 "_call_stub_return_address must have been generated before");
427 __ b(StubRoutines::_call_stub_return_address);
428
429 return start;
430 }
431
432 // Continuation point for runtime calls returning with a pending
433 // exception. The pending exception check happened in the runtime
434 // or native call stub. The pending exception in Thread is
435 // converted into a Java-level exception.
436 //
437 // Contract with Java-level exception handlers:
438 // r0: exception
439 // r3: throwing pc
440 //
441 // NOTE: At entry of this stub, exception-pc must be in LR !!
442
443 // NOTE: this is always used as a jump target within generated code
444 // so it just needs to be generated code wiht no x86 prolog
445
446 address generate_forward_exception() {
447 StubCodeMark mark(this, "StubRoutines", "forward exception");
448 address start = __ pc();
449
450 // Upon entry, LR points to the return address returning into
451 // Java (interpreted or compiled) code; i.e., the return address
452 // becomes the throwing pc.
453 //
454 // Arguments pushed before the runtime call are still on the stack
455 // but the exception handler will reset the stack pointer ->
456 // ignore them. A potential result in registers can be ignored as
457 // well.
458
459 #ifdef ASSERT
460 // make sure this code is only executed if there is a pending exception
461 {
462 Label L;
463 __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
464 __ cbnz(rscratch1, L);
465 __ stop("StubRoutines::forward exception: no pending exception (1)");
466 __ bind(L);
467 }
468 #endif
469
470 // compute exception handler into r19
471
472 // call the VM to find the handler address associated with the
473 // caller address. pass thread in r0 and caller pc (ret address)
474 // in r1. n.b. the caller pc is in lr, unlike x86 where it is on
475 // the stack.
476 __ mov(c_rarg1, lr);
477 // lr will be trashed by the VM call so we move it to R19
478 // (callee-saved) because we also need to pass it to the handler
479 // returned by this call.
480 __ mov(r19, lr);
481 BLOCK_COMMENT("call exception_handler_for_return_address");
482 __ call_VM_leaf(CAST_FROM_FN_PTR(address,
483 SharedRuntime::exception_handler_for_return_address),
484 rthread, c_rarg1);
485 // we should not really care that lr is no longer the callee
486 // address. we saved the value the handler needs in r19 so we can
487 // just copy it to r3. however, the C2 handler will push its own
488 // frame and then calls into the VM and the VM code asserts that
489 // the PC for the frame above the handler belongs to a compiled
490 // Java method. So, we restore lr here to satisfy that assert.
491 __ mov(lr, r19);
492 // setup r0 & r3 & clear pending exception
493 __ mov(r3, r19);
494 __ mov(r19, r0);
495 __ ldr(r0, Address(rthread, Thread::pending_exception_offset()));
496 __ str(zr, Address(rthread, Thread::pending_exception_offset()));
497
498 #ifdef ASSERT
499 // make sure exception is set
500 {
501 Label L;
502 __ cbnz(r0, L);
503 __ stop("StubRoutines::forward exception: no pending exception (2)");
504 __ bind(L);
505 }
506 #endif
507
508 // continue at exception handler
509 // r0: exception
510 // r3: throwing pc
511 // r19: exception handler
512 __ verify_oop(r0);
513 __ br(r19);
514
515 return start;
516 }
517
518 // Non-destructive plausibility checks for oops
519 //
520 // Arguments:
521 // r0: oop to verify
522 // rscratch1: error message
523 //
524 // Stack after saving c_rarg3:
525 // [tos + 0]: saved c_rarg3
526 // [tos + 1]: saved c_rarg2
527 // [tos + 2]: saved lr
528 // [tos + 3]: saved rscratch2
529 // [tos + 4]: saved r0
530 // [tos + 5]: saved rscratch1
531 address generate_verify_oop() {
532
533 StubCodeMark mark(this, "StubRoutines", "verify_oop");
534 address start = __ pc();
535
536 Label exit, error;
537
538 // save c_rarg2 and c_rarg3
539 __ stp(c_rarg3, c_rarg2, Address(__ pre(sp, -16)));
540
541 // __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
542 __ lea(c_rarg2, ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
543 __ ldr(c_rarg3, Address(c_rarg2));
544 __ add(c_rarg3, c_rarg3, 1);
545 __ str(c_rarg3, Address(c_rarg2));
546
547 // object is in r0
548 // make sure object is 'reasonable'
549 __ cbz(r0, exit); // if obj is NULL it is OK
550
551 // Check if the oop is in the right area of memory
552 __ mov(c_rarg3, (intptr_t) Universe::verify_oop_mask());
553 __ andr(c_rarg2, r0, c_rarg3);
554 __ mov(c_rarg3, (intptr_t) Universe::verify_oop_bits());
555
556 // Compare c_rarg2 and c_rarg3. We don't use a compare
557 // instruction here because the flags register is live.
558 __ eor(c_rarg2, c_rarg2, c_rarg3);
559 __ cbnz(c_rarg2, error);
560
561 // make sure klass is 'reasonable', which is not zero.
562 __ load_klass(r0, r0); // get klass
563 __ cbz(r0, error); // if klass is NULL it is broken
564
565 // return if everything seems ok
566 __ bind(exit);
567
568 __ ldp(c_rarg3, c_rarg2, Address(__ post(sp, 16)));
569 __ ret(lr);
570
571 // handle errors
572 __ bind(error);
573 __ ldp(c_rarg3, c_rarg2, Address(__ post(sp, 16)));
574
575 __ push(RegSet::range(r0, r29), sp);
576 // debug(char* msg, int64_t pc, int64_t regs[])
577 __ mov(c_rarg0, rscratch1); // pass address of error message
578 __ mov(c_rarg1, lr); // pass return address
579 __ mov(c_rarg2, sp); // pass address of regs on stack
580 #ifndef PRODUCT
581 assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
582 #endif
583 BLOCK_COMMENT("call MacroAssembler::debug");
584 __ mov(rscratch1, CAST_FROM_FN_PTR(address, MacroAssembler::debug64));
585 __ blr(rscratch1);
586
587 return start;
588 }
589
590 void array_overlap_test(Label& L_no_overlap, Address::sxtw sf) { __ b(L_no_overlap); }
591
592 // Generate code for an array write pre barrier
593 //
594 // addr - starting address
595 // count - element count
596 // tmp - scratch register
597 //
598 // Destroy no registers except rscratch1 and rscratch2
599 //
600 void gen_write_ref_array_pre_barrier(Register addr, Register count, bool dest_uninitialized) {
601 BarrierSet* bs = Universe::heap()->barrier_set();
602 switch (bs->kind()) {
603 case BarrierSet::G1SATBCT:
604 case BarrierSet::G1SATBCTLogging:
605 // With G1, don't generate the call if we statically know that the target in uninitialized
606 if (!dest_uninitialized) {
607 __ push_call_clobbered_registers();
608 if (count == c_rarg0) {
609 if (addr == c_rarg1) {
610 // exactly backwards!!
611 __ mov(rscratch1, c_rarg0);
612 __ mov(c_rarg0, c_rarg1);
613 __ mov(c_rarg1, rscratch1);
614 } else {
615 __ mov(c_rarg1, count);
616 __ mov(c_rarg0, addr);
617 }
618 } else {
619 __ mov(c_rarg0, addr);
620 __ mov(c_rarg1, count);
621 }
622 __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre), 2);
623 __ pop_call_clobbered_registers();
624 break;
625 case BarrierSet::CardTableModRef:
626 case BarrierSet::CardTableExtension:
627 case BarrierSet::ModRef:
628 break;
629 default:
630 ShouldNotReachHere();
631
632 }
633 }
634 }
635
636 //
637 // Generate code for an array write post barrier
638 //
639 // Input:
640 // start - register containing starting address of destination array
641 // end - register containing ending address of destination array
642 // scratch - scratch register
643 //
644 // The input registers are overwritten.
645 // The ending address is inclusive.
646 void gen_write_ref_array_post_barrier(Register start, Register end, Register scratch) {
647 assert_different_registers(start, end, scratch);
648 Label L_done;
649
650 // "end" is inclusive end pointer == start + (count - 1) * array_element_size
651 // If count == 0, "end" is less than "start" and we need to skip card marking.
652 __ cmp(end, start);
653 __ br(__ LO, L_done);
654
655 BarrierSet* bs = Universe::heap()->barrier_set();
656 switch (bs->kind()) {
657 case BarrierSet::G1SATBCT:
658 case BarrierSet::G1SATBCTLogging:
659
660 {
661 __ push_call_clobbered_registers();
662 // must compute element count unless barrier set interface is changed (other platforms supply count)
663 assert_different_registers(start, end, scratch);
664 __ lea(scratch, Address(end, BytesPerHeapOop));
665 __ sub(scratch, scratch, start); // subtract start to get #bytes
666 __ lsr(scratch, scratch, LogBytesPerHeapOop); // convert to element count
667 __ mov(c_rarg0, start);
668 __ mov(c_rarg1, scratch);
669 __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post), 2);
670 __ pop_call_clobbered_registers();
671 }
672 break;
673 case BarrierSet::CardTableModRef:
674 case BarrierSet::CardTableExtension:
675 {
676 CardTableModRefBS* ct = (CardTableModRefBS*)bs;
677 assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
678
679 Label L_loop;
680
681 __ lsr(start, start, CardTableModRefBS::card_shift);
682 __ lsr(end, end, CardTableModRefBS::card_shift);
683 __ sub(end, end, start); // number of bytes to copy
684
685 const Register count = end; // 'end' register contains bytes count now
686 __ load_byte_map_base(scratch);
687 __ add(start, start, scratch);
688 if (UseConcMarkSweepGC) {
689 __ membar(__ StoreStore);
690 }
691 __ BIND(L_loop);
692 __ strb(zr, Address(start, count));
693 __ subs(count, count, 1);
694 __ br(Assembler::GE, L_loop);
695 }
696 break;
697 default:
698 ShouldNotReachHere();
699
700 }
701 __ bind(L_done);
702 }
703
704 address generate_zero_longs(Register base, Register cnt) {
705 Register tmp = rscratch1;
706 Register tmp2 = rscratch2;
707 int zva_length = VM_Version::zva_length();
708 Label initial_table_end, loop_zva;
709 Label fini;
710
711 __ align(CodeEntryAlignment);
712 StubCodeMark mark(this, "StubRoutines", "zero_longs");
713 address start = __ pc();
714
715 // Base must be 16 byte aligned. If not just return and let caller handle it
716 __ tst(base, 0x0f);
717 __ br(Assembler::NE, fini);
718 // Align base with ZVA length.
719 __ neg(tmp, base);
720 __ andr(tmp, tmp, zva_length - 1);
721
722 // tmp: the number of bytes to be filled to align the base with ZVA length.
723 __ add(base, base, tmp);
724 __ sub(cnt, cnt, tmp, Assembler::ASR, 3);
725 __ adr(tmp2, initial_table_end);
726 __ sub(tmp2, tmp2, tmp, Assembler::LSR, 2);
727 __ br(tmp2);
728
729 for (int i = -zva_length + 16; i < 0; i += 16)
730 __ stp(zr, zr, Address(base, i));
731 __ bind(initial_table_end);
732
733 __ sub(cnt, cnt, zva_length >> 3);
734 __ bind(loop_zva);
735 __ dc(Assembler::ZVA, base);
736 __ subs(cnt, cnt, zva_length >> 3);
737 __ add(base, base, zva_length);
738 __ br(Assembler::GE, loop_zva);
739 __ add(cnt, cnt, zva_length >> 3); // count not zeroed by DC ZVA
740 __ bind(fini);
741 __ ret(lr);
742
743 return start;
744 }
745
746 typedef enum {
747 copy_forwards = 1,
748 copy_backwards = -1
749 } copy_direction;
750
751 // Bulk copy of blocks of 8 words.
752 //
753 // count is a count of words.
754 //
755 // Precondition: count >= 8
756 //
757 // Postconditions:
758 //
759 // The least significant bit of count contains the remaining count
760 // of words to copy. The rest of count is trash.
761 //
762 // s and d are adjusted to point to the remaining words to copy
763 //
764 void generate_copy_longs(Label &start, Register s, Register d, Register count,
765 copy_direction direction) {
766 int unit = wordSize * direction;
767 int bias = (UseSIMDForMemoryOps ? 4:2) * wordSize;
768
769 int offset;
770 const Register t0 = r3, t1 = r4, t2 = r5, t3 = r6,
771 t4 = r7, t5 = r10, t6 = r11, t7 = r12;
772 const Register stride = r13;
773
774 assert_different_registers(rscratch1, t0, t1, t2, t3, t4, t5, t6, t7);
775 assert_different_registers(s, d, count, rscratch1);
776
777 Label again, drain;
778 const char *stub_name;
779 if (direction == copy_forwards)
780 stub_name = "foward_copy_longs";
781 else
782 stub_name = "backward_copy_longs";
783
784 __ align(CodeEntryAlignment);
785
786 StubCodeMark mark(this, "StubRoutines", stub_name);
787
788 __ bind(start);
789
790 Label unaligned_copy_long;
791 if (AvoidUnalignedAccesses) {
792 __ tbnz(d, 3, unaligned_copy_long);
793 }
794
795 if (direction == copy_forwards) {
796 __ sub(s, s, bias);
797 __ sub(d, d, bias);
798 }
799
800 #ifdef ASSERT
801 // Make sure we are never given < 8 words
802 {
803 Label L;
804 __ cmp(count, 8);
805 __ br(Assembler::GE, L);
806 __ stop("genrate_copy_longs called with < 8 words");
807 __ bind(L);
808 }
809 #endif
810
811 // Fill 8 registers
812 if (UseSIMDForMemoryOps) {
813 __ ldpq(v0, v1, Address(s, 4 * unit));
814 __ ldpq(v2, v3, Address(__ pre(s, 8 * unit)));
815 } else {
816 __ ldp(t0, t1, Address(s, 2 * unit));
817 __ ldp(t2, t3, Address(s, 4 * unit));
818 __ ldp(t4, t5, Address(s, 6 * unit));
819 __ ldp(t6, t7, Address(__ pre(s, 8 * unit)));
820 }
821
822 __ subs(count, count, 16);
823 __ br(Assembler::LO, drain);
824
825 int prefetch = PrefetchCopyIntervalInBytes;
826 bool use_stride = false;
827 if (direction == copy_backwards) {
828 use_stride = prefetch > 256;
829 prefetch = -prefetch;
830 if (use_stride) __ mov(stride, prefetch);
831 }
832
833 __ bind(again);
834
835 if (PrefetchCopyIntervalInBytes > 0)
836 __ prfm(use_stride ? Address(s, stride) : Address(s, prefetch), PLDL1KEEP);
837
838 if (UseSIMDForMemoryOps) {
839 __ stpq(v0, v1, Address(d, 4 * unit));
840 __ ldpq(v0, v1, Address(s, 4 * unit));
841 __ stpq(v2, v3, Address(__ pre(d, 8 * unit)));
842 __ ldpq(v2, v3, Address(__ pre(s, 8 * unit)));
843 } else {
844 __ stp(t0, t1, Address(d, 2 * unit));
845 __ ldp(t0, t1, Address(s, 2 * unit));
846 __ stp(t2, t3, Address(d, 4 * unit));
847 __ ldp(t2, t3, Address(s, 4 * unit));
848 __ stp(t4, t5, Address(d, 6 * unit));
849 __ ldp(t4, t5, Address(s, 6 * unit));
850 __ stp(t6, t7, Address(__ pre(d, 8 * unit)));
851 __ ldp(t6, t7, Address(__ pre(s, 8 * unit)));
852 }
853
854 __ subs(count, count, 8);
855 __ br(Assembler::HS, again);
856
857 // Drain
858 __ bind(drain);
859 if (UseSIMDForMemoryOps) {
860 __ stpq(v0, v1, Address(d, 4 * unit));
861 __ stpq(v2, v3, Address(__ pre(d, 8 * unit)));
862 } else {
863 __ stp(t0, t1, Address(d, 2 * unit));
864 __ stp(t2, t3, Address(d, 4 * unit));
865 __ stp(t4, t5, Address(d, 6 * unit));
866 __ stp(t6, t7, Address(__ pre(d, 8 * unit)));
867 }
868
869 {
870 Label L1, L2;
871 __ tbz(count, exact_log2(4), L1);
872 if (UseSIMDForMemoryOps) {
873 __ ldpq(v0, v1, Address(__ pre(s, 4 * unit)));
874 __ stpq(v0, v1, Address(__ pre(d, 4 * unit)));
875 } else {
876 __ ldp(t0, t1, Address(s, 2 * unit));
877 __ ldp(t2, t3, Address(__ pre(s, 4 * unit)));
878 __ stp(t0, t1, Address(d, 2 * unit));
879 __ stp(t2, t3, Address(__ pre(d, 4 * unit)));
880 }
881 __ bind(L1);
882
883 if (direction == copy_forwards) {
884 __ add(s, s, bias);
885 __ add(d, d, bias);
886 }
887
888 __ tbz(count, 1, L2);
889 __ ldp(t0, t1, Address(__ adjust(s, 2 * unit, direction == copy_backwards)));
890 __ stp(t0, t1, Address(__ adjust(d, 2 * unit, direction == copy_backwards)));
891 __ bind(L2);
892 }
893
894 __ ret(lr);
895
896 if (AvoidUnalignedAccesses) {
897 Label drain, again;
898 // Register order for storing. Order is different for backward copy.
899
900 __ bind(unaligned_copy_long);
901
902 // source address is even aligned, target odd aligned
903 //
904 // when forward copying word pairs we read long pairs at offsets
905 // {0, 2, 4, 6} (in long words). when backwards copying we read
906 // long pairs at offsets {-2, -4, -6, -8}. We adjust the source
907 // address by -2 in the forwards case so we can compute the
908 // source offsets for both as {2, 4, 6, 8} * unit where unit = 1
909 // or -1.
910 //
911 // when forward copying we need to store 1 word, 3 pairs and
912 // then 1 word at offsets {0, 1, 3, 5, 7}. Rather thna use a
913 // zero offset We adjust the destination by -1 which means we
914 // have to use offsets { 1, 2, 4, 6, 8} * unit for the stores.
915 //
916 // When backwards copyng we need to store 1 word, 3 pairs and
917 // then 1 word at offsets {-1, -3, -5, -7, -8} i.e. we use
918 // offsets {1, 3, 5, 7, 8} * unit.
919
920 if (direction == copy_forwards) {
921 __ sub(s, s, 16);
922 __ sub(d, d, 8);
923 }
924
925 // Fill 8 registers
926 //
927 // for forwards copy s was offset by -16 from the original input
928 // value of s so the register contents are at these offsets
929 // relative to the 64 bit block addressed by that original input
930 // and so on for each successive 64 byte block when s is updated
931 //
932 // t0 at offset 0, t1 at offset 8
933 // t2 at offset 16, t3 at offset 24
934 // t4 at offset 32, t5 at offset 40
935 // t6 at offset 48, t7 at offset 56
936
937 // for backwards copy s was not offset so the register contents
938 // are at these offsets into the preceding 64 byte block
939 // relative to that original input and so on for each successive
940 // preceding 64 byte block when s is updated. this explains the
941 // slightly counter-intuitive looking pattern of register usage
942 // in the stp instructions for backwards copy.
943 //
944 // t0 at offset -16, t1 at offset -8
945 // t2 at offset -32, t3 at offset -24
946 // t4 at offset -48, t5 at offset -40
947 // t6 at offset -64, t7 at offset -56
948
949 __ ldp(t0, t1, Address(s, 2 * unit));
950 __ ldp(t2, t3, Address(s, 4 * unit));
951 __ ldp(t4, t5, Address(s, 6 * unit));
952 __ ldp(t6, t7, Address(__ pre(s, 8 * unit)));
953
954 __ subs(count, count, 16);
955 __ br(Assembler::LO, drain);
956
957 int prefetch = PrefetchCopyIntervalInBytes;
958 bool use_stride = false;
959 if (direction == copy_backwards) {
960 use_stride = prefetch > 256;
961 prefetch = -prefetch;
962 if (use_stride) __ mov(stride, prefetch);
963 }
964
965 __ bind(again);
966
967 if (PrefetchCopyIntervalInBytes > 0)
968 __ prfm(use_stride ? Address(s, stride) : Address(s, prefetch), PLDL1KEEP);
969
970 if (direction == copy_forwards) {
971 // allowing for the offset of -8 the store instructions place
972 // registers into the target 64 bit block at the following
973 // offsets
974 //
975 // t0 at offset 0
976 // t1 at offset 8, t2 at offset 16
977 // t3 at offset 24, t4 at offset 32
978 // t5 at offset 40, t6 at offset 48
979 // t7 at offset 56
980
981 __ str(t0, Address(d, 1 * unit));
982 __ stp(t1, t2, Address(d, 2 * unit));
983 __ ldp(t0, t1, Address(s, 2 * unit));
984 __ stp(t3, t4, Address(d, 4 * unit));
985 __ ldp(t2, t3, Address(s, 4 * unit));
986 __ stp(t5, t6, Address(d, 6 * unit));
987 __ ldp(t4, t5, Address(s, 6 * unit));
988 __ str(t7, Address(__ pre(d, 8 * unit)));
989 __ ldp(t6, t7, Address(__ pre(s, 8 * unit)));
990 } else {
991 // d was not offset when we started so the registers are
992 // written into the 64 bit block preceding d with the following
993 // offsets
994 //
995 // t1 at offset -8
996 // t3 at offset -24, t0 at offset -16
997 // t5 at offset -48, t2 at offset -32
998 // t7 at offset -56, t4 at offset -48
999 // t6 at offset -64
1000 //
1001 // note that this matches the offsets previously noted for the
1002 // loads
1003
1004 __ str(t1, Address(d, 1 * unit));
1005 __ stp(t3, t0, Address(d, 3 * unit));
1006 __ ldp(t0, t1, Address(s, 2 * unit));
1007 __ stp(t5, t2, Address(d, 5 * unit));
1008 __ ldp(t2, t3, Address(s, 4 * unit));
1009 __ stp(t7, t4, Address(d, 7 * unit));
1010 __ ldp(t4, t5, Address(s, 6 * unit));
1011 __ str(t6, Address(__ pre(d, 8 * unit)));
1012 __ ldp(t6, t7, Address(__ pre(s, 8 * unit)));
1013 }
1014
1015 __ subs(count, count, 8);
1016 __ br(Assembler::HS, again);
1017
1018 // Drain
1019 //
1020 // this uses the same pattern of offsets and register arguments
1021 // as above
1022 __ bind(drain);
1023 if (direction == copy_forwards) {
1024 __ str(t0, Address(d, 1 * unit));
1025 __ stp(t1, t2, Address(d, 2 * unit));
1026 __ stp(t3, t4, Address(d, 4 * unit));
1027 __ stp(t5, t6, Address(d, 6 * unit));
1028 __ str(t7, Address(__ pre(d, 8 * unit)));
1029 } else {
1030 __ str(t1, Address(d, 1 * unit));
1031 __ stp(t3, t0, Address(d, 3 * unit));
1032 __ stp(t5, t2, Address(d, 5 * unit));
1033 __ stp(t7, t4, Address(d, 7 * unit));
1034 __ str(t6, Address(__ pre(d, 8 * unit)));
1035 }
1036 // now we need to copy any remaining part block which may
1037 // include a 4 word block subblock and/or a 2 word subblock.
1038 // bits 2 and 1 in the count are the tell-tale for whetehr we
1039 // have each such subblock
1040 {
1041 Label L1, L2;
1042 __ tbz(count, exact_log2(4), L1);
1043 // this is the same as above but copying only 4 longs hence
1044 // with ony one intervening stp between the str instructions
1045 // but note that the offsets and registers still follow the
1046 // same pattern
1047 __ ldp(t0, t1, Address(s, 2 * unit));
1048 __ ldp(t2, t3, Address(__ pre(s, 4 * unit)));
1049 if (direction == copy_forwards) {
1050 __ str(t0, Address(d, 1 * unit));
1051 __ stp(t1, t2, Address(d, 2 * unit));
1052 __ str(t3, Address(__ pre(d, 4 * unit)));
1053 } else {
1054 __ str(t1, Address(d, 1 * unit));
1055 __ stp(t3, t0, Address(d, 3 * unit));
1056 __ str(t2, Address(__ pre(d, 4 * unit)));
1057 }
1058 __ bind(L1);
1059
1060 __ tbz(count, 1, L2);
1061 // this is the same as above but copying only 2 longs hence
1062 // there is no intervening stp between the str instructions
1063 // but note that the offset and register patterns are still
1064 // the same
1065 __ ldp(t0, t1, Address(__ pre(s, 2 * unit)));
1066 if (direction == copy_forwards) {
1067 __ str(t0, Address(d, 1 * unit));
1068 __ str(t1, Address(__ pre(d, 2 * unit)));
1069 } else {
1070 __ str(t1, Address(d, 1 * unit));
1071 __ str(t0, Address(__ pre(d, 2 * unit)));
1072 }
1073 __ bind(L2);
1074
1075 // for forwards copy we need to re-adjust the offsets we
1076 // applied so that s and d are follow the last words written
1077
1078 if (direction == copy_forwards) {
1079 __ add(s, s, 16);
1080 __ add(d, d, 8);
1081 }
1082
1083 }
1084
1085 __ ret(lr);
1086 }
1087 }
1088
1089 // Small copy: less than 16 bytes.
1090 //
1091 // NB: Ignores all of the bits of count which represent more than 15
1092 // bytes, so a caller doesn't have to mask them.
1093
1094 void copy_memory_small(Register s, Register d, Register count, Register tmp, int step) {
1095 bool is_backwards = step < 0;
1096 size_t granularity = uabs(step);
1097 int direction = is_backwards ? -1 : 1;
1098 int unit = wordSize * direction;
1099
1100 Label Lpair, Lword, Lint, Lshort, Lbyte;
1101
1102 assert(granularity
1103 && granularity <= sizeof (jlong), "Impossible granularity in copy_memory_small");
1104
1105 const Register t0 = r3, t1 = r4, t2 = r5, t3 = r6;
1106
1107 // ??? I don't know if this bit-test-and-branch is the right thing
1108 // to do. It does a lot of jumping, resulting in several
1109 // mispredicted branches. It might make more sense to do this
1110 // with something like Duff's device with a single computed branch.
1111
1112 __ tbz(count, 3 - exact_log2(granularity), Lword);
1113 __ ldr(tmp, Address(__ adjust(s, unit, is_backwards)));
1114 __ str(tmp, Address(__ adjust(d, unit, is_backwards)));
1115 __ bind(Lword);
1116
1117 if (granularity <= sizeof (jint)) {
1118 __ tbz(count, 2 - exact_log2(granularity), Lint);
1119 __ ldrw(tmp, Address(__ adjust(s, sizeof (jint) * direction, is_backwards)));
1120 __ strw(tmp, Address(__ adjust(d, sizeof (jint) * direction, is_backwards)));
1121 __ bind(Lint);
1122 }
1123
1124 if (granularity <= sizeof (jshort)) {
1125 __ tbz(count, 1 - exact_log2(granularity), Lshort);
1126 __ ldrh(tmp, Address(__ adjust(s, sizeof (jshort) * direction, is_backwards)));
1127 __ strh(tmp, Address(__ adjust(d, sizeof (jshort) * direction, is_backwards)));
1128 __ bind(Lshort);
1129 }
1130
1131 if (granularity <= sizeof (jbyte)) {
1132 __ tbz(count, 0, Lbyte);
1133 __ ldrb(tmp, Address(__ adjust(s, sizeof (jbyte) * direction, is_backwards)));
1134 __ strb(tmp, Address(__ adjust(d, sizeof (jbyte) * direction, is_backwards)));
1135 __ bind(Lbyte);
1136 }
1137 }
1138
1139 Label copy_f, copy_b;
1140
1141 // All-singing all-dancing memory copy.
1142 //
1143 // Copy count units of memory from s to d. The size of a unit is
1144 // step, which can be positive or negative depending on the direction
1145 // of copy. If is_aligned is false, we align the source address.
1146 //
1147
1148 void copy_memory(bool is_aligned, Register s, Register d,
1149 Register count, Register tmp, int step) {
1150 copy_direction direction = step < 0 ? copy_backwards : copy_forwards;
1151 bool is_backwards = step < 0;
1152 int granularity = uabs(step);
1153 const Register t0 = r3, t1 = r4;
1154
1155 // <= 96 bytes do inline. Direction doesn't matter because we always
1156 // load all the data before writing anything
1157 Label copy4, copy8, copy16, copy32, copy80, copy128, copy_big, finish;
1158 const Register t2 = r5, t3 = r6, t4 = r7, t5 = r8;
1159 const Register t6 = r9, t7 = r10, t8 = r11, t9 = r12;
1160 const Register send = r17, dend = r18;
1161
1162 if (PrefetchCopyIntervalInBytes > 0)
1163 __ prfm(Address(s, 0), PLDL1KEEP);
1164 __ cmp(count, (UseSIMDForMemoryOps ? 96:80)/granularity);
1165 __ br(Assembler::HI, copy_big);
1166
1167 __ lea(send, Address(s, count, Address::lsl(exact_log2(granularity))));
1168 __ lea(dend, Address(d, count, Address::lsl(exact_log2(granularity))));
1169
1170 __ cmp(count, 16/granularity);
1171 __ br(Assembler::LS, copy16);
1172
1173 __ cmp(count, 64/granularity);
1174 __ br(Assembler::HI, copy80);
1175
1176 __ cmp(count, 32/granularity);
1177 __ br(Assembler::LS, copy32);
1178
1179 // 33..64 bytes
1180 if (UseSIMDForMemoryOps) {
1181 __ ldpq(v0, v1, Address(s, 0));
1182 __ ldpq(v2, v3, Address(send, -32));
1183 __ stpq(v0, v1, Address(d, 0));
1184 __ stpq(v2, v3, Address(dend, -32));
1185 } else {
1186 __ ldp(t0, t1, Address(s, 0));
1187 __ ldp(t2, t3, Address(s, 16));
1188 __ ldp(t4, t5, Address(send, -32));
1189 __ ldp(t6, t7, Address(send, -16));
1190
1191 __ stp(t0, t1, Address(d, 0));
1192 __ stp(t2, t3, Address(d, 16));
1193 __ stp(t4, t5, Address(dend, -32));
1194 __ stp(t6, t7, Address(dend, -16));
1195 }
1196 __ b(finish);
1197
1198 // 17..32 bytes
1199 __ bind(copy32);
1200 __ ldp(t0, t1, Address(s, 0));
1201 __ ldp(t2, t3, Address(send, -16));
1202 __ stp(t0, t1, Address(d, 0));
1203 __ stp(t2, t3, Address(dend, -16));
1204 __ b(finish);
1205
1206 // 65..80/96 bytes
1207 // (96 bytes if SIMD because we do 32 byes per instruction)
1208 __ bind(copy80);
1209 if (UseSIMDForMemoryOps) {
1210 __ ld4(v0, v1, v2, v3, __ T16B, Address(s, 0));
1211 __ ldpq(v4, v5, Address(send, -32));
1212 __ st4(v0, v1, v2, v3, __ T16B, Address(d, 0));
1213 __ stpq(v4, v5, Address(dend, -32));
1214 } else {
1215 __ ldp(t0, t1, Address(s, 0));
1216 __ ldp(t2, t3, Address(s, 16));
1217 __ ldp(t4, t5, Address(s, 32));
1218 __ ldp(t6, t7, Address(s, 48));
1219 __ ldp(t8, t9, Address(send, -16));
1220
1221 __ stp(t0, t1, Address(d, 0));
1222 __ stp(t2, t3, Address(d, 16));
1223 __ stp(t4, t5, Address(d, 32));
1224 __ stp(t6, t7, Address(d, 48));
1225 __ stp(t8, t9, Address(dend, -16));
1226 }
1227 __ b(finish);
1228
1229 // 0..16 bytes
1230 __ bind(copy16);
1231 __ cmp(count, 8/granularity);
1232 __ br(Assembler::LO, copy8);
1233
1234 // 8..16 bytes
1235 __ ldr(t0, Address(s, 0));
1236 __ ldr(t1, Address(send, -8));
1237 __ str(t0, Address(d, 0));
1238 __ str(t1, Address(dend, -8));
1239 __ b(finish);
1240
1241 if (granularity < 8) {
1242 // 4..7 bytes
1243 __ bind(copy8);
1244 __ tbz(count, 2 - exact_log2(granularity), copy4);
1245 __ ldrw(t0, Address(s, 0));
1246 __ ldrw(t1, Address(send, -4));
1247 __ strw(t0, Address(d, 0));
1248 __ strw(t1, Address(dend, -4));
1249 __ b(finish);
1250 if (granularity < 4) {
1251 // 0..3 bytes
1252 __ bind(copy4);
1253 __ cbz(count, finish); // get rid of 0 case
1254 if (granularity == 2) {
1255 __ ldrh(t0, Address(s, 0));
1256 __ strh(t0, Address(d, 0));
1257 } else { // granularity == 1
1258 // Now 1..3 bytes. Handle the 1 and 2 byte case by copying
1259 // the first and last byte.
1260 // Handle the 3 byte case by loading and storing base + count/2
1261 // (count == 1 (s+0)->(d+0), count == 2,3 (s+1) -> (d+1))
1262 // This does means in the 1 byte case we load/store the same
1263 // byte 3 times.
1264 __ lsr(count, count, 1);
1265 __ ldrb(t0, Address(s, 0));
1266 __ ldrb(t1, Address(send, -1));
1267 __ ldrb(t2, Address(s, count));
1268 __ strb(t0, Address(d, 0));
1269 __ strb(t1, Address(dend, -1));
1270 __ strb(t2, Address(d, count));
1271 }
1272 __ b(finish);
1273 }
1274 }
1275
1276 __ bind(copy_big);
1277 if (is_backwards) {
1278 __ lea(s, Address(s, count, Address::lsl(exact_log2(-step))));
1279 __ lea(d, Address(d, count, Address::lsl(exact_log2(-step))));
1280 }
1281
1282 // Now we've got the small case out of the way we can align the
1283 // source address on a 2-word boundary.
1284
1285 Label aligned;
1286
1287 if (is_aligned) {
1288 // We may have to adjust by 1 word to get s 2-word-aligned.
1289 __ tbz(s, exact_log2(wordSize), aligned);
1290 __ ldr(tmp, Address(__ adjust(s, direction * wordSize, is_backwards)));
1291 __ str(tmp, Address(__ adjust(d, direction * wordSize, is_backwards)));
1292 __ sub(count, count, wordSize/granularity);
1293 } else {
1294 if (is_backwards) {
1295 __ andr(rscratch2, s, 2 * wordSize - 1);
1296 } else {
1297 __ neg(rscratch2, s);
1298 __ andr(rscratch2, rscratch2, 2 * wordSize - 1);
1299 }
1300 // rscratch2 is the byte adjustment needed to align s.
1301 __ cbz(rscratch2, aligned);
1302 int shift = exact_log2(granularity);
1303 if (shift) __ lsr(rscratch2, rscratch2, shift);
1304 __ sub(count, count, rscratch2);
1305
1306 #if 0
1307 // ?? This code is only correct for a disjoint copy. It may or
1308 // may not make sense to use it in that case.
1309
1310 // Copy the first pair; s and d may not be aligned.
1311 __ ldp(t0, t1, Address(s, is_backwards ? -2 * wordSize : 0));
1312 __ stp(t0, t1, Address(d, is_backwards ? -2 * wordSize : 0));
1313
1314 // Align s and d, adjust count
1315 if (is_backwards) {
1316 __ sub(s, s, rscratch2);
1317 __ sub(d, d, rscratch2);
1318 } else {
1319 __ add(s, s, rscratch2);
1320 __ add(d, d, rscratch2);
1321 }
1322 #else
1323 copy_memory_small(s, d, rscratch2, rscratch1, step);
1324 #endif
1325 }
1326
1327 __ bind(aligned);
1328
1329 // s is now 2-word-aligned.
1330
1331 // We have a count of units and some trailing bytes. Adjust the
1332 // count and do a bulk copy of words.
1333 __ lsr(rscratch2, count, exact_log2(wordSize/granularity));
1334 if (direction == copy_forwards)
1335 __ bl(copy_f);
1336 else
1337 __ bl(copy_b);
1338
1339 // And the tail.
1340 copy_memory_small(s, d, count, tmp, step);
1341
1342 if (granularity >= 8) __ bind(copy8);
1343 if (granularity >= 4) __ bind(copy4);
1344 __ bind(finish);
1345 }
1346
1347
1348 void clobber_registers() {
1349 #ifdef ASSERT
1350 __ mov(rscratch1, (uint64_t)0xdeadbeef);
1351 __ orr(rscratch1, rscratch1, rscratch1, Assembler::LSL, 32);
1352 for (Register r = r3; r <= r18; r++)
1353 if (r != rscratch1) __ mov(r, rscratch1);
1354 #endif
1355 }
1356
1357 // Scan over array at a for count oops, verifying each one.
1358 // Preserves a and count, clobbers rscratch1 and rscratch2.
1359 void verify_oop_array (size_t size, Register a, Register count, Register temp) {
1360 Label loop, end;
1361 __ mov(rscratch1, a);
1362 __ mov(rscratch2, zr);
1363 __ bind(loop);
1364 __ cmp(rscratch2, count);
1365 __ br(Assembler::HS, end);
1366 if (size == (size_t)wordSize) {
1367 __ ldr(temp, Address(a, rscratch2, Address::lsl(exact_log2(size))));
1368 __ verify_oop(temp);
1369 } else {
1370 __ ldrw(r16, Address(a, rscratch2, Address::lsl(exact_log2(size))));
1371 __ decode_heap_oop(temp); // calls verify_oop
1372 }
1373 __ add(rscratch2, rscratch2, size);
1374 __ b(loop);
1375 __ bind(end);
1376 }
1377
1378 // Arguments:
1379 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1380 // ignored
1381 // is_oop - true => oop array, so generate store check code
1382 // name - stub name string
1383 //
1384 // Inputs:
1385 // c_rarg0 - source array address
1386 // c_rarg1 - destination array address
1387 // c_rarg2 - element count, treated as ssize_t, can be zero
1388 //
1389 // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1390 // the hardware handle it. The two dwords within qwords that span
1391 // cache line boundaries will still be loaded and stored atomicly.
1392 //
1393 // Side Effects:
1394 // disjoint_int_copy_entry is set to the no-overlap entry point
1395 // used by generate_conjoint_int_oop_copy().
1396 //
1397 address generate_disjoint_copy(size_t size, bool aligned, bool is_oop, address *entry,
1398 const char *name, bool dest_uninitialized = false) {
1399 Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
1400 __ align(CodeEntryAlignment);
1401 StubCodeMark mark(this, "StubRoutines", name);
1402 address start = __ pc();
1403 __ enter();
1404
1405 if (entry != NULL) {
1406 *entry = __ pc();
1407 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1408 BLOCK_COMMENT("Entry:");
1409 }
1410
1411 if (is_oop) {
1412 __ push(RegSet::of(d, count), sp);
1413 // no registers are destroyed by this call
1414 gen_write_ref_array_pre_barrier(d, count, dest_uninitialized);
1415 }
1416 copy_memory(aligned, s, d, count, rscratch1, size);
1417 if (is_oop) {
1418 __ pop(RegSet::of(d, count), sp);
1419 if (VerifyOops)
1420 verify_oop_array(size, d, count, r16);
1421 __ sub(count, count, 1); // make an inclusive end pointer
1422 __ lea(count, Address(d, count, Address::lsl(exact_log2(size))));
1423 gen_write_ref_array_post_barrier(d, count, rscratch1);
1424 }
1425 __ leave();
1426 __ mov(r0, zr); // return 0
1427 __ ret(lr);
1428 return start;
1429 }
1430
1431 // Arguments:
1432 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1433 // ignored
1434 // is_oop - true => oop array, so generate store check code
1435 // name - stub name string
1436 //
1437 // Inputs:
1438 // c_rarg0 - source array address
1439 // c_rarg1 - destination array address
1440 // c_rarg2 - element count, treated as ssize_t, can be zero
1441 //
1442 // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1443 // the hardware handle it. The two dwords within qwords that span
1444 // cache line boundaries will still be loaded and stored atomicly.
1445 //
1446 address generate_conjoint_copy(size_t size, bool aligned, bool is_oop, address nooverlap_target,
1447 address *entry, const char *name,
1448 bool dest_uninitialized = false) {
1449 Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
1450
1451 StubCodeMark mark(this, "StubRoutines", name);
1452 address start = __ pc();
1453
1454 __ enter();
1455
1456 if (entry != NULL) {
1457 *entry = __ pc();
1458 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1459 BLOCK_COMMENT("Entry:");
1460 }
1461
1462 // use fwd copy when (d-s) above_equal (count*size)
1463 __ sub(rscratch1, d, s);
1464 __ cmp(rscratch1, count, Assembler::LSL, exact_log2(size));
1465 __ br(Assembler::HS, nooverlap_target);
1466
1467 if (is_oop) {
1468 __ push(RegSet::of(d, count), sp);
1469 // no registers are destroyed by this call
1470 gen_write_ref_array_pre_barrier(d, count, dest_uninitialized);
1471 }
1472 copy_memory(aligned, s, d, count, rscratch1, -size);
1473 if (is_oop) {
1474 __ pop(RegSet::of(d, count), sp);
1475 if (VerifyOops)
1476 verify_oop_array(size, d, count, r16);
1477 __ sub(count, count, 1); // make an inclusive end pointer
1478 __ lea(count, Address(d, count, Address::lsl(exact_log2(size))));
1479 gen_write_ref_array_post_barrier(d, count, rscratch1);
1480 }
1481 __ leave();
1482 __ mov(r0, zr); // return 0
1483 __ ret(lr);
1484 return start;
1485 }
1486
1487 // Arguments:
1488 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1489 // ignored
1490 // name - stub name string
1491 //
1492 // Inputs:
1493 // c_rarg0 - source array address
1494 // c_rarg1 - destination array address
1495 // c_rarg2 - element count, treated as ssize_t, can be zero
1496 //
1497 // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries,
1498 // we let the hardware handle it. The one to eight bytes within words,
1499 // dwords or qwords that span cache line boundaries will still be loaded
1500 // and stored atomically.
1501 //
1502 // Side Effects:
1503 // disjoint_byte_copy_entry is set to the no-overlap entry point //
1504 // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries,
1505 // we let the hardware handle it. The one to eight bytes within words,
1506 // dwords or qwords that span cache line boundaries will still be loaded
1507 // and stored atomically.
1508 //
1509 // Side Effects:
1510 // disjoint_byte_copy_entry is set to the no-overlap entry point
1511 // used by generate_conjoint_byte_copy().
1512 //
1513 address generate_disjoint_byte_copy(bool aligned, address* entry, const char *name) {
1514 const bool not_oop = false;
1515 return generate_disjoint_copy(sizeof (jbyte), aligned, not_oop, entry, name);
1516 }
1517
1518 // Arguments:
1519 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1520 // ignored
1521 // name - stub name string
1522 //
1523 // Inputs:
1524 // c_rarg0 - source array address
1525 // c_rarg1 - destination array address
1526 // c_rarg2 - element count, treated as ssize_t, can be zero
1527 //
1528 // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries,
1529 // we let the hardware handle it. The one to eight bytes within words,
1530 // dwords or qwords that span cache line boundaries will still be loaded
1531 // and stored atomically.
1532 //
1533 address generate_conjoint_byte_copy(bool aligned, address nooverlap_target,
1534 address* entry, const char *name) {
1535 const bool not_oop = false;
1536 return generate_conjoint_copy(sizeof (jbyte), aligned, not_oop, nooverlap_target, entry, name);
1537 }
1538
1539 // Arguments:
1540 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1541 // ignored
1542 // name - stub name string
1543 //
1544 // Inputs:
1545 // c_rarg0 - source array address
1546 // c_rarg1 - destination array address
1547 // c_rarg2 - element count, treated as ssize_t, can be zero
1548 //
1549 // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we
1550 // let the hardware handle it. The two or four words within dwords
1551 // or qwords that span cache line boundaries will still be loaded
1552 // and stored atomically.
1553 //
1554 // Side Effects:
1555 // disjoint_short_copy_entry is set to the no-overlap entry point
1556 // used by generate_conjoint_short_copy().
1557 //
1558 address generate_disjoint_short_copy(bool aligned,
1559 address* entry, const char *name) {
1560 const bool not_oop = false;
1561 return generate_disjoint_copy(sizeof (jshort), aligned, not_oop, entry, name);
1562 }
1563
1564 // Arguments:
1565 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1566 // ignored
1567 // name - stub name string
1568 //
1569 // Inputs:
1570 // c_rarg0 - source array address
1571 // c_rarg1 - destination array address
1572 // c_rarg2 - element count, treated as ssize_t, can be zero
1573 //
1574 // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we
1575 // let the hardware handle it. The two or four words within dwords
1576 // or qwords that span cache line boundaries will still be loaded
1577 // and stored atomically.
1578 //
1579 address generate_conjoint_short_copy(bool aligned, address nooverlap_target,
1580 address *entry, const char *name) {
1581 const bool not_oop = false;
1582 return generate_conjoint_copy(sizeof (jshort), aligned, not_oop, nooverlap_target, entry, name);
1583
1584 }
1585 // Arguments:
1586 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1587 // ignored
1588 // name - stub name string
1589 //
1590 // Inputs:
1591 // c_rarg0 - source array address
1592 // c_rarg1 - destination array address
1593 // c_rarg2 - element count, treated as ssize_t, can be zero
1594 //
1595 // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1596 // the hardware handle it. The two dwords within qwords that span
1597 // cache line boundaries will still be loaded and stored atomicly.
1598 //
1599 // Side Effects:
1600 // disjoint_int_copy_entry is set to the no-overlap entry point
1601 // used by generate_conjoint_int_oop_copy().
1602 //
1603 address generate_disjoint_int_copy(bool aligned, address *entry,
1604 const char *name, bool dest_uninitialized = false) {
1605 const bool not_oop = false;
1606 return generate_disjoint_copy(sizeof (jint), aligned, not_oop, entry, name);
1607 }
1608
1609 // Arguments:
1610 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1611 // ignored
1612 // name - stub name string
1613 //
1614 // Inputs:
1615 // c_rarg0 - source array address
1616 // c_rarg1 - destination array address
1617 // c_rarg2 - element count, treated as ssize_t, can be zero
1618 //
1619 // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1620 // the hardware handle it. The two dwords within qwords that span
1621 // cache line boundaries will still be loaded and stored atomicly.
1622 //
1623 address generate_conjoint_int_copy(bool aligned, address nooverlap_target,
1624 address *entry, const char *name,
1625 bool dest_uninitialized = false) {
1626 const bool not_oop = false;
1627 return generate_conjoint_copy(sizeof (jint), aligned, not_oop, nooverlap_target, entry, name);
1628 }
1629
1630
1631 // Arguments:
1632 // aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes
1633 // ignored
1634 // name - stub name string
1635 //
1636 // Inputs:
1637 // c_rarg0 - source array address
1638 // c_rarg1 - destination array address
1639 // c_rarg2 - element count, treated as size_t, can be zero
1640 //
1641 // Side Effects:
1642 // disjoint_oop_copy_entry or disjoint_long_copy_entry is set to the
1643 // no-overlap entry point used by generate_conjoint_long_oop_copy().
1644 //
1645 address generate_disjoint_long_copy(bool aligned, address *entry,
1646 const char *name, bool dest_uninitialized = false) {
1647 const bool not_oop = false;
1648 return generate_disjoint_copy(sizeof (jlong), aligned, not_oop, entry, name);
1649 }
1650
1651 // Arguments:
1652 // aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes
1653 // ignored
1654 // name - stub name string
1655 //
1656 // Inputs:
1657 // c_rarg0 - source array address
1658 // c_rarg1 - destination array address
1659 // c_rarg2 - element count, treated as size_t, can be zero
1660 //
1661 address generate_conjoint_long_copy(bool aligned,
1662 address nooverlap_target, address *entry,
1663 const char *name, bool dest_uninitialized = false) {
1664 const bool not_oop = false;
1665 return generate_conjoint_copy(sizeof (jlong), aligned, not_oop, nooverlap_target, entry, name);
1666 }
1667
1668 // Arguments:
1669 // aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes
1670 // ignored
1671 // name - stub name string
1672 //
1673 // Inputs:
1674 // c_rarg0 - source array address
1675 // c_rarg1 - destination array address
1676 // c_rarg2 - element count, treated as size_t, can be zero
1677 //
1678 // Side Effects:
1679 // disjoint_oop_copy_entry or disjoint_long_copy_entry is set to the
1680 // no-overlap entry point used by generate_conjoint_long_oop_copy().
1681 //
1682 address generate_disjoint_oop_copy(bool aligned, address *entry,
1683 const char *name, bool dest_uninitialized) {
1684 const bool is_oop = true;
1685 const size_t size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
1686 return generate_disjoint_copy(size, aligned, is_oop, entry, name, dest_uninitialized);
1687 }
1688
1689 // Arguments:
1690 // aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes
1691 // ignored
1692 // name - stub name string
1693 //
1694 // Inputs:
1695 // c_rarg0 - source array address
1696 // c_rarg1 - destination array address
1697 // c_rarg2 - element count, treated as size_t, can be zero
1698 //
1699 address generate_conjoint_oop_copy(bool aligned,
1700 address nooverlap_target, address *entry,
1701 const char *name, bool dest_uninitialized) {
1702 const bool is_oop = true;
1703 const size_t size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
1704 return generate_conjoint_copy(size, aligned, is_oop, nooverlap_target, entry,
1705 name, dest_uninitialized);
1706 }
1707
1708
1709 // Helper for generating a dynamic type check.
1710 // Smashes rscratch1.
1711 void generate_type_check(Register sub_klass,
1712 Register super_check_offset,
1713 Register super_klass,
1714 Label& L_success) {
1715 assert_different_registers(sub_klass, super_check_offset, super_klass);
1716
1717 BLOCK_COMMENT("type_check:");
1718
1719 Label L_miss;
1720
1721 __ check_klass_subtype_fast_path(sub_klass, super_klass, noreg, &L_success, &L_miss, NULL,
1722 super_check_offset);
1723 __ check_klass_subtype_slow_path(sub_klass, super_klass, noreg, noreg, &L_success, NULL);
1724
1725 // Fall through on failure!
1726 __ BIND(L_miss);
1727 }
1728
1729 //
1730 // Generate checkcasting array copy stub
1731 //
1732 // Input:
1733 // c_rarg0 - source array address
1734 // c_rarg1 - destination array address
1735 // c_rarg2 - element count, treated as ssize_t, can be zero
1736 // c_rarg3 - size_t ckoff (super_check_offset)
1737 // c_rarg4 - oop ckval (super_klass)
1738 //
1739 // Output:
1740 // r0 == 0 - success
1741 // r0 == -1^K - failure, where K is partial transfer count
1742 //
1743 address generate_checkcast_copy(const char *name, address *entry,
1744 bool dest_uninitialized = false) {
1745
1746 Label L_load_element, L_store_element, L_do_card_marks, L_done, L_done_pop;
1747
1748 // Input registers (after setup_arg_regs)
1749 const Register from = c_rarg0; // source array address
1750 const Register to = c_rarg1; // destination array address
1751 const Register count = c_rarg2; // elementscount
1752 const Register ckoff = c_rarg3; // super_check_offset
1753 const Register ckval = c_rarg4; // super_klass
1754
1755 // Registers used as temps (r18, r19, r20 are save-on-entry)
1756 const Register count_save = r21; // orig elementscount
1757 const Register start_to = r20; // destination array start address
1758 const Register copied_oop = r18; // actual oop copied
1759 const Register r19_klass = r19; // oop._klass
1760
1761 //---------------------------------------------------------------
1762 // Assembler stub will be used for this call to arraycopy
1763 // if the two arrays are subtypes of Object[] but the
1764 // destination array type is not equal to or a supertype
1765 // of the source type. Each element must be separately
1766 // checked.
1767
1768 assert_different_registers(from, to, count, ckoff, ckval, start_to,
1769 copied_oop, r19_klass, count_save);
1770
1771 __ align(CodeEntryAlignment);
1772 StubCodeMark mark(this, "StubRoutines", name);
1773 address start = __ pc();
1774
1775 __ enter(); // required for proper stackwalking of RuntimeStub frame
1776
1777 #ifdef ASSERT
1778 // caller guarantees that the arrays really are different
1779 // otherwise, we would have to make conjoint checks
1780 { Label L;
1781 array_overlap_test(L, TIMES_OOP);
1782 __ stop("checkcast_copy within a single array");
1783 __ bind(L);
1784 }
1785 #endif //ASSERT
1786
1787 // Caller of this entry point must set up the argument registers.
1788 if (entry != NULL) {
1789 *entry = __ pc();
1790 BLOCK_COMMENT("Entry:");
1791 }
1792
1793 // Empty array: Nothing to do.
1794 __ cbz(count, L_done);
1795
1796 __ push(RegSet::of(r18, r19, r20, r21), sp);
1797
1798 #ifdef ASSERT
1799 BLOCK_COMMENT("assert consistent ckoff/ckval");
1800 // The ckoff and ckval must be mutually consistent,
1801 // even though caller generates both.
1802 { Label L;
1803 int sco_offset = in_bytes(Klass::super_check_offset_offset());
1804 __ ldrw(start_to, Address(ckval, sco_offset));
1805 __ cmpw(ckoff, start_to);
1806 __ br(Assembler::EQ, L);
1807 __ stop("super_check_offset inconsistent");
1808 __ bind(L);
1809 }
1810 #endif //ASSERT
1811
1812 gen_write_ref_array_pre_barrier(to, count, dest_uninitialized);
1813
1814 // save the original count
1815 __ mov(count_save, count);
1816
1817 // Copy from low to high addresses
1818 __ mov(start_to, to); // Save destination array start address
1819 __ b(L_load_element);
1820
1821 // ======== begin loop ========
1822 // (Loop is rotated; its entry is L_load_element.)
1823 // Loop control:
1824 // for (; count != 0; count--) {
1825 // copied_oop = load_heap_oop(from++);
1826 // ... generate_type_check ...;
1827 // store_heap_oop(to++, copied_oop);
1828 // }
1829 __ align(OptoLoopAlignment);
1830
1831 __ BIND(L_store_element);
1832 __ store_heap_oop(__ post(to, UseCompressedOops ? 4 : 8), copied_oop); // store the oop
1833 __ sub(count, count, 1);
1834 __ cbz(count, L_do_card_marks);
1835
1836 // ======== loop entry is here ========
1837 __ BIND(L_load_element);
1838 __ load_heap_oop(copied_oop, __ post(from, UseCompressedOops ? 4 : 8)); // load the oop
1839 __ cbz(copied_oop, L_store_element);
1840
1841 __ load_klass(r19_klass, copied_oop);// query the object klass
1842 generate_type_check(r19_klass, ckoff, ckval, L_store_element);
1843 // ======== end loop ========
1844
1845 // It was a real error; we must depend on the caller to finish the job.
1846 // Register count = remaining oops, count_orig = total oops.
1847 // Emit GC store barriers for the oops we have copied and report
1848 // their number to the caller.
1849
1850 __ subs(count, count_save, count); // K = partially copied oop count
1851 __ eon(count, count, zr); // report (-1^K) to caller
1852 __ br(Assembler::EQ, L_done_pop);
1853
1854 __ BIND(L_do_card_marks);
1855 __ add(to, to, -heapOopSize); // make an inclusive end pointer
1856 gen_write_ref_array_post_barrier(start_to, to, rscratch1);
1857
1858 __ bind(L_done_pop);
1859 __ pop(RegSet::of(r18, r19, r20, r21), sp);
1860 inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr);
1861
1862 __ bind(L_done);
1863 __ mov(r0, count);
1864 __ leave();
1865 __ ret(lr);
1866
1867 return start;
1868 }
1869
1870 // Perform range checks on the proposed arraycopy.
1871 // Kills temp, but nothing else.
1872 // Also, clean the sign bits of src_pos and dst_pos.
1873 void arraycopy_range_checks(Register src, // source array oop (c_rarg0)
1874 Register src_pos, // source position (c_rarg1)
1875 Register dst, // destination array oo (c_rarg2)
1876 Register dst_pos, // destination position (c_rarg3)
1877 Register length,
1878 Register temp,
1879 Label& L_failed) {
1880 BLOCK_COMMENT("arraycopy_range_checks:");
1881
1882 assert_different_registers(rscratch1, temp);
1883
1884 // if (src_pos + length > arrayOop(src)->length()) FAIL;
1885 __ ldrw(rscratch1, Address(src, arrayOopDesc::length_offset_in_bytes()));
1886 __ addw(temp, length, src_pos);
1887 __ cmpw(temp, rscratch1);
1888 __ br(Assembler::HI, L_failed);
1889
1890 // if (dst_pos + length > arrayOop(dst)->length()) FAIL;
1891 __ ldrw(rscratch1, Address(dst, arrayOopDesc::length_offset_in_bytes()));
1892 __ addw(temp, length, dst_pos);
1893 __ cmpw(temp, rscratch1);
1894 __ br(Assembler::HI, L_failed);
1895
1896 // Have to clean up high 32 bits of 'src_pos' and 'dst_pos'.
1897 __ movw(src_pos, src_pos);
1898 __ movw(dst_pos, dst_pos);
1899
1900 BLOCK_COMMENT("arraycopy_range_checks done");
1901 }
1902
1903 // These stubs get called from some dumb test routine.
1904 // I'll write them properly when they're called from
1905 // something that's actually doing something.
1906 static void fake_arraycopy_stub(address src, address dst, int count) {
1907 assert(count == 0, "huh?");
1908 }
1909
1910
1911 //
1912 // Generate stub for array fill. If "aligned" is true, the
1913 // "to" address is assumed to be heapword aligned.
1914 //
1915 // Arguments for generated stub:
1916 // to: c_rarg0
1917 // value: c_rarg1
1918 // count: c_rarg2 treated as signed
1919 //
1920 address generate_fill(BasicType t, bool aligned, const char *name) {
1921 __ align(CodeEntryAlignment);
1922 StubCodeMark mark(this, "StubRoutines", name);
1923 address start = __ pc();
1924
1925 BLOCK_COMMENT("Entry:");
1926
1927 const Register to = c_rarg0; // source array address
1928 const Register value = c_rarg1; // value
1929 const Register count = c_rarg2; // elements count
1930
1931 const Register bz_base = r10; // base for block_zero routine
1932 const Register cnt_words = r11; // temp register
1933
1934 __ enter();
1935
1936 Label L_fill_elements, L_exit1;
1937
1938 int shift = -1;
1939 switch (t) {
1940 case T_BYTE:
1941 shift = 0;
1942 __ cmpw(count, 8 >> shift); // Short arrays (< 8 bytes) fill by element
1943 __ bfi(value, value, 8, 8); // 8 bit -> 16 bit
1944 __ bfi(value, value, 16, 16); // 16 bit -> 32 bit
1945 __ br(Assembler::LO, L_fill_elements);
1946 break;
1947 case T_SHORT:
1948 shift = 1;
1949 __ cmpw(count, 8 >> shift); // Short arrays (< 8 bytes) fill by element
1950 __ bfi(value, value, 16, 16); // 16 bit -> 32 bit
1951 __ br(Assembler::LO, L_fill_elements);
1952 break;
1953 case T_INT:
1954 shift = 2;
1955 __ cmpw(count, 8 >> shift); // Short arrays (< 8 bytes) fill by element
1956 __ br(Assembler::LO, L_fill_elements);
1957 break;
1958 default: ShouldNotReachHere();
1959 }
1960
1961 // Align source address at 8 bytes address boundary.
1962 Label L_skip_align1, L_skip_align2, L_skip_align4;
1963 if (!aligned) {
1964 switch (t) {
1965 case T_BYTE:
1966 // One byte misalignment happens only for byte arrays.
1967 __ tbz(to, 0, L_skip_align1);
1968 __ strb(value, Address(__ post(to, 1)));
1969 __ subw(count, count, 1);
1970 __ bind(L_skip_align1);
1971 // Fallthrough
1972 case T_SHORT:
1973 // Two bytes misalignment happens only for byte and short (char) arrays.
1974 __ tbz(to, 1, L_skip_align2);
1975 __ strh(value, Address(__ post(to, 2)));
1976 __ subw(count, count, 2 >> shift);
1977 __ bind(L_skip_align2);
1978 // Fallthrough
1979 case T_INT:
1980 // Align to 8 bytes, we know we are 4 byte aligned to start.
1981 __ tbz(to, 2, L_skip_align4);
1982 __ strw(value, Address(__ post(to, 4)));
1983 __ subw(count, count, 4 >> shift);
1984 __ bind(L_skip_align4);
1985 break;
1986 default: ShouldNotReachHere();
1987 }
1988 }
1989
1990 //
1991 // Fill large chunks
1992 //
1993 __ lsrw(cnt_words, count, 3 - shift); // number of words
1994 __ bfi(value, value, 32, 32); // 32 bit -> 64 bit
1995 __ subw(count, count, cnt_words, Assembler::LSL, 3 - shift);
1996 if (UseBlockZeroing) {
1997 Label non_block_zeroing, rest;
1998 // count >= BlockZeroingLowLimit && value == 0
1999 __ subs(rscratch1, cnt_words, BlockZeroingLowLimit >> 3);
2000 __ ccmp(value, 0 /* comparing value */, 0 /* NZCV */, Assembler::GE);
2001 __ br(Assembler::NE, non_block_zeroing);
2002 __ mov(bz_base, to);
2003 __ block_zero(bz_base, cnt_words, true);
2004 __ mov(to, bz_base);
2005 __ b(rest);
2006 __ bind(non_block_zeroing);
2007 __ fill_words(to, cnt_words, value);
2008 __ bind(rest);
2009 }
2010 else {
2011 __ fill_words(to, cnt_words, value);
2012 }
2013
2014 // Remaining count is less than 8 bytes. Fill it by a single store.
2015 // Note that the total length is no less than 8 bytes.
2016 if (t == T_BYTE || t == T_SHORT) {
2017 Label L_exit1;
2018 __ cbzw(count, L_exit1);
2019 __ add(to, to, count, Assembler::LSL, shift); // points to the end
2020 __ str(value, Address(to, -8)); // overwrite some elements
2021 __ bind(L_exit1);
2022 __ leave();
2023 __ ret(lr);
2024 }
2025
2026 // Handle copies less than 8 bytes.
2027 Label L_fill_2, L_fill_4, L_exit2;
2028 __ bind(L_fill_elements);
2029 switch (t) {
2030 case T_BYTE:
2031 __ tbz(count, 0, L_fill_2);
2032 __ strb(value, Address(__ post(to, 1)));
2033 __ bind(L_fill_2);
2034 __ tbz(count, 1, L_fill_4);
2035 __ strh(value, Address(__ post(to, 2)));
2036 __ bind(L_fill_4);
2037 __ tbz(count, 2, L_exit2);
2038 __ strw(value, Address(to));
2039 break;
2040 case T_SHORT:
2041 __ tbz(count, 0, L_fill_4);
2042 __ strh(value, Address(__ post(to, 2)));
2043 __ bind(L_fill_4);
2044 __ tbz(count, 1, L_exit2);
2045 __ strw(value, Address(to));
2046 break;
2047 case T_INT:
2048 __ cbzw(count, L_exit2);
2049 __ strw(value, Address(to));
2050 break;
2051 default: ShouldNotReachHere();
2052 }
2053 __ bind(L_exit2);
2054 __ leave();
2055 __ ret(lr);
2056 return start;
2057 }
2058
2059 //
2060 // Generate 'unsafe' array copy stub
2061 // Though just as safe as the other stubs, it takes an unscaled
2062 // size_t argument instead of an element count.
2063 //
2064 // Input:
2065 // c_rarg0 - source array address
2066 // c_rarg1 - destination array address
2067 // c_rarg2 - byte count, treated as ssize_t, can be zero
2068 //
2069 // Examines the alignment of the operands and dispatches
2070 // to a long, int, short, or byte copy loop.
2071 //
2072 address generate_unsafe_copy(const char *name,
2073 address byte_copy_entry,
2074 address short_copy_entry,
2075 address int_copy_entry,
2076 address long_copy_entry) {
2077 Label L_long_aligned, L_int_aligned, L_short_aligned;
2078 Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
2079
2080 __ align(CodeEntryAlignment);
2081 StubCodeMark mark(this, "StubRoutines", name);
2082 address start = __ pc();
2083 __ enter(); // required for proper stackwalking of RuntimeStub frame
2084
2085 // bump this on entry, not on exit:
2086 inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr);
2087
2088 __ orr(rscratch1, s, d);
2089 __ orr(rscratch1, rscratch1, count);
2090
2091 __ andr(rscratch1, rscratch1, BytesPerLong-1);
2092 __ cbz(rscratch1, L_long_aligned);
2093 __ andr(rscratch1, rscratch1, BytesPerInt-1);
2094 __ cbz(rscratch1, L_int_aligned);
2095 __ tbz(rscratch1, 0, L_short_aligned);
2096 __ b(RuntimeAddress(byte_copy_entry));
2097
2098 __ BIND(L_short_aligned);
2099 __ lsr(count, count, LogBytesPerShort); // size => short_count
2100 __ b(RuntimeAddress(short_copy_entry));
2101 __ BIND(L_int_aligned);
2102 __ lsr(count, count, LogBytesPerInt); // size => int_count
2103 __ b(RuntimeAddress(int_copy_entry));
2104 __ BIND(L_long_aligned);
2105 __ lsr(count, count, LogBytesPerLong); // size => long_count
2106 __ b(RuntimeAddress(long_copy_entry));
2107
2108 return start;
2109 }
2110
2111 //
2112 // Generate generic array copy stubs
2113 //
2114 // Input:
2115 // c_rarg0 - src oop
2116 // c_rarg1 - src_pos (32-bits)
2117 // c_rarg2 - dst oop
2118 // c_rarg3 - dst_pos (32-bits)
2119 // c_rarg4 - element count (32-bits)
2120 //
2121 // Output:
2122 // r0 == 0 - success
2123 // r0 == -1^K - failure, where K is partial transfer count
2124 //
2125 address generate_generic_copy(const char *name,
2126 address byte_copy_entry, address short_copy_entry,
2127 address int_copy_entry, address oop_copy_entry,
2128 address long_copy_entry, address checkcast_copy_entry) {
2129
2130 Label L_failed, L_failed_0, L_objArray;
2131 Label L_copy_bytes, L_copy_shorts, L_copy_ints, L_copy_longs;
2132
2133 // Input registers
2134 const Register src = c_rarg0; // source array oop
2135 const Register src_pos = c_rarg1; // source position
2136 const Register dst = c_rarg2; // destination array oop
2137 const Register dst_pos = c_rarg3; // destination position
2138 const Register length = c_rarg4;
2139
2140 __ align(CodeEntryAlignment);
2141
2142 StubCodeMark mark(this, "StubRoutines", name);
2143
2144 address start = __ pc();
2145
2146 __ enter(); // required for proper stackwalking of RuntimeStub frame
2147
2148 // bump this on entry, not on exit:
2149 inc_counter_np(SharedRuntime::_generic_array_copy_ctr);
2150
2151 //-----------------------------------------------------------------------
2152 // Assembler stub will be used for this call to arraycopy
2153 // if the following conditions are met:
2154 //
2155 // (1) src and dst must not be null.
2156 // (2) src_pos must not be negative.
2157 // (3) dst_pos must not be negative.
2158 // (4) length must not be negative.
2159 // (5) src klass and dst klass should be the same and not NULL.
2160 // (6) src and dst should be arrays.
2161 // (7) src_pos + length must not exceed length of src.
2162 // (8) dst_pos + length must not exceed length of dst.
2163 //
2164
2165 // if (src == NULL) return -1;
2166 __ cbz(src, L_failed);
2167
2168 // if (src_pos < 0) return -1;
2169 __ tbnz(src_pos, 31, L_failed); // i.e. sign bit set
2170
2171 // if (dst == NULL) return -1;
2172 __ cbz(dst, L_failed);
2173
2174 // if (dst_pos < 0) return -1;
2175 __ tbnz(dst_pos, 31, L_failed); // i.e. sign bit set
2176
2177 // registers used as temp
2178 const Register scratch_length = r16; // elements count to copy
2179 const Register scratch_src_klass = r17; // array klass
2180 const Register lh = r18; // layout helper
2181
2182 // if (length < 0) return -1;
2183 __ movw(scratch_length, length); // length (elements count, 32-bits value)
2184 __ tbnz(scratch_length, 31, L_failed); // i.e. sign bit set
2185
2186 __ load_klass(scratch_src_klass, src);
2187 #ifdef ASSERT
2188 // assert(src->klass() != NULL);
2189 {
2190 BLOCK_COMMENT("assert klasses not null {");
2191 Label L1, L2;
2192 __ cbnz(scratch_src_klass, L2); // it is broken if klass is NULL
2193 __ bind(L1);
2194 __ stop("broken null klass");
2195 __ bind(L2);
2196 __ load_klass(rscratch1, dst);
2197 __ cbz(rscratch1, L1); // this would be broken also
2198 BLOCK_COMMENT("} assert klasses not null done");
2199 }
2200 #endif
2201
2202 // Load layout helper (32-bits)
2203 //
2204 // |array_tag| | header_size | element_type | |log2_element_size|
2205 // 32 30 24 16 8 2 0
2206 //
2207 // array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
2208 //
2209
2210 const int lh_offset = in_bytes(Klass::layout_helper_offset());
2211
2212 // Handle objArrays completely differently...
2213 const jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2214 __ ldrw(lh, Address(scratch_src_klass, lh_offset));
2215 __ movw(rscratch1, objArray_lh);
2216 __ eorw(rscratch2, lh, rscratch1);
2217 __ cbzw(rscratch2, L_objArray);
2218
2219 // if (src->klass() != dst->klass()) return -1;
2220 __ load_klass(rscratch2, dst);
2221 __ eor(rscratch2, rscratch2, scratch_src_klass);
2222 __ cbnz(rscratch2, L_failed);
2223
2224 // if (!src->is_Array()) return -1;
2225 __ tbz(lh, 31, L_failed); // i.e. (lh >= 0)
2226
2227 // At this point, it is known to be a typeArray (array_tag 0x3).
2228 #ifdef ASSERT
2229 {
2230 BLOCK_COMMENT("assert primitive array {");
2231 Label L;
2232 __ movw(rscratch2, Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift);
2233 __ cmpw(lh, rscratch2);
2234 __ br(Assembler::GE, L);
2235 __ stop("must be a primitive array");
2236 __ bind(L);
2237 BLOCK_COMMENT("} assert primitive array done");
2238 }
2239 #endif
2240
2241 arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
2242 rscratch2, L_failed);
2243
2244 // TypeArrayKlass
2245 //
2246 // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize);
2247 // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize);
2248 //
2249
2250 const Register rscratch1_offset = rscratch1; // array offset
2251 const Register r18_elsize = lh; // element size
2252
2253 __ ubfx(rscratch1_offset, lh, Klass::_lh_header_size_shift,
2254 exact_log2(Klass::_lh_header_size_mask+1)); // array_offset
2255 __ add(src, src, rscratch1_offset); // src array offset
2256 __ add(dst, dst, rscratch1_offset); // dst array offset
2257 BLOCK_COMMENT("choose copy loop based on element size");
2258
2259 // next registers should be set before the jump to corresponding stub
2260 const Register from = c_rarg0; // source array address
2261 const Register to = c_rarg1; // destination array address
2262 const Register count = c_rarg2; // elements count
2263
2264 // 'from', 'to', 'count' registers should be set in such order
2265 // since they are the same as 'src', 'src_pos', 'dst'.
2266
2267 assert(Klass::_lh_log2_element_size_shift == 0, "fix this code");
2268
2269 // The possible values of elsize are 0-3, i.e. exact_log2(element
2270 // size in bytes). We do a simple bitwise binary search.
2271 __ BIND(L_copy_bytes);
2272 __ tbnz(r18_elsize, 1, L_copy_ints);
2273 __ tbnz(r18_elsize, 0, L_copy_shorts);
2274 __ lea(from, Address(src, src_pos));// src_addr
2275 __ lea(to, Address(dst, dst_pos));// dst_addr
2276 __ movw(count, scratch_length); // length
2277 __ b(RuntimeAddress(byte_copy_entry));
2278
2279 __ BIND(L_copy_shorts);
2280 __ lea(from, Address(src, src_pos, Address::lsl(1)));// src_addr
2281 __ lea(to, Address(dst, dst_pos, Address::lsl(1)));// dst_addr
2282 __ movw(count, scratch_length); // length
2283 __ b(RuntimeAddress(short_copy_entry));
2284
2285 __ BIND(L_copy_ints);
2286 __ tbnz(r18_elsize, 0, L_copy_longs);
2287 __ lea(from, Address(src, src_pos, Address::lsl(2)));// src_addr
2288 __ lea(to, Address(dst, dst_pos, Address::lsl(2)));// dst_addr
2289 __ movw(count, scratch_length); // length
2290 __ b(RuntimeAddress(int_copy_entry));
2291
2292 __ BIND(L_copy_longs);
2293 #ifdef ASSERT
2294 {
2295 BLOCK_COMMENT("assert long copy {");
2296 Label L;
2297 __ andw(lh, lh, Klass::_lh_log2_element_size_mask); // lh -> r18_elsize
2298 __ cmpw(r18_elsize, LogBytesPerLong);
2299 __ br(Assembler::EQ, L);
2300 __ stop("must be long copy, but elsize is wrong");
2301 __ bind(L);
2302 BLOCK_COMMENT("} assert long copy done");
2303 }
2304 #endif
2305 __ lea(from, Address(src, src_pos, Address::lsl(3)));// src_addr
2306 __ lea(to, Address(dst, dst_pos, Address::lsl(3)));// dst_addr
2307 __ movw(count, scratch_length); // length
2308 __ b(RuntimeAddress(long_copy_entry));
2309
2310 // ObjArrayKlass
2311 __ BIND(L_objArray);
2312 // live at this point: scratch_src_klass, scratch_length, src[_pos], dst[_pos]
2313
2314 Label L_plain_copy, L_checkcast_copy;
2315 // test array classes for subtyping
2316 __ load_klass(r18, dst);
2317 __ cmp(scratch_src_klass, r18); // usual case is exact equality
2318 __ br(Assembler::NE, L_checkcast_copy);
2319
2320 // Identically typed arrays can be copied without element-wise checks.
2321 arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
2322 rscratch2, L_failed);
2323
2324 __ lea(from, Address(src, src_pos, Address::lsl(LogBytesPerHeapOop)));
2325 __ add(from, from, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
2326 __ lea(to, Address(dst, dst_pos, Address::lsl(LogBytesPerHeapOop)));
2327 __ add(to, to, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
2328 __ movw(count, scratch_length); // length
2329 __ BIND(L_plain_copy);
2330 __ b(RuntimeAddress(oop_copy_entry));
2331
2332 __ BIND(L_checkcast_copy);
2333 // live at this point: scratch_src_klass, scratch_length, r18 (dst_klass)
2334 {
2335 // Before looking at dst.length, make sure dst is also an objArray.
2336 __ ldrw(rscratch1, Address(r18, lh_offset));
2337 __ movw(rscratch2, objArray_lh);
2338 __ eorw(rscratch1, rscratch1, rscratch2);
2339 __ cbnzw(rscratch1, L_failed);
2340
2341 // It is safe to examine both src.length and dst.length.
2342 arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
2343 r18, L_failed);
2344
2345 const Register rscratch2_dst_klass = rscratch2;
2346 __ load_klass(rscratch2_dst_klass, dst); // reload
2347
2348 // Marshal the base address arguments now, freeing registers.
2349 __ lea(from, Address(src, src_pos, Address::lsl(LogBytesPerHeapOop)));
2350 __ add(from, from, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
2351 __ lea(to, Address(dst, dst_pos, Address::lsl(LogBytesPerHeapOop)));
2352 __ add(to, to, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
2353 __ movw(count, length); // length (reloaded)
2354 Register sco_temp = c_rarg3; // this register is free now
2355 assert_different_registers(from, to, count, sco_temp,
2356 rscratch2_dst_klass, scratch_src_klass);
2357 // assert_clean_int(count, sco_temp);
2358
2359 // Generate the type check.
2360 const int sco_offset = in_bytes(Klass::super_check_offset_offset());
2361 __ ldrw(sco_temp, Address(rscratch2_dst_klass, sco_offset));
2362 // assert_clean_int(sco_temp, r18);
2363 generate_type_check(scratch_src_klass, sco_temp, rscratch2_dst_klass, L_plain_copy);
2364
2365 // Fetch destination element klass from the ObjArrayKlass header.
2366 int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
2367 __ ldr(rscratch2_dst_klass, Address(rscratch2_dst_klass, ek_offset));
2368 __ ldrw(sco_temp, Address(rscratch2_dst_klass, sco_offset));
2369
2370 // the checkcast_copy loop needs two extra arguments:
2371 assert(c_rarg3 == sco_temp, "#3 already in place");
2372 // Set up arguments for checkcast_copy_entry.
2373 __ mov(c_rarg4, rscratch2_dst_klass); // dst.klass.element_klass
2374 __ b(RuntimeAddress(checkcast_copy_entry));
2375 }
2376
2377 __ BIND(L_failed);
2378 __ mov(r0, -1);
2379 __ leave(); // required for proper stackwalking of RuntimeStub frame
2380 __ ret(lr);
2381
2382 return start;
2383 }
2384
2385 void generate_arraycopy_stubs() {
2386 address entry;
2387 address entry_jbyte_arraycopy;
2388 address entry_jshort_arraycopy;
2389 address entry_jint_arraycopy;
2390 address entry_oop_arraycopy;
2391 address entry_jlong_arraycopy;
2392 address entry_checkcast_arraycopy;
2393
2394 generate_copy_longs(copy_f, r0, r1, rscratch2, copy_forwards);
2395 generate_copy_longs(copy_b, r0, r1, rscratch2, copy_backwards);
2396
2397 StubRoutines::aarch64::_zero_longs = generate_zero_longs(r10, r11);
2398
2399 //*** jbyte
2400 // Always need aligned and unaligned versions
2401 StubRoutines::_jbyte_disjoint_arraycopy = generate_disjoint_byte_copy(false, &entry,
2402 "jbyte_disjoint_arraycopy");
2403 StubRoutines::_jbyte_arraycopy = generate_conjoint_byte_copy(false, entry,
2404 &entry_jbyte_arraycopy,
2405 "jbyte_arraycopy");
2406 StubRoutines::_arrayof_jbyte_disjoint_arraycopy = generate_disjoint_byte_copy(true, &entry,
2407 "arrayof_jbyte_disjoint_arraycopy");
2408 StubRoutines::_arrayof_jbyte_arraycopy = generate_conjoint_byte_copy(true, entry, NULL,
2409 "arrayof_jbyte_arraycopy");
2410
2411 //*** jshort
2412 // Always need aligned and unaligned versions
2413 StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_short_copy(false, &entry,
2414 "jshort_disjoint_arraycopy");
2415 StubRoutines::_jshort_arraycopy = generate_conjoint_short_copy(false, entry,
2416 &entry_jshort_arraycopy,
2417 "jshort_arraycopy");
2418 StubRoutines::_arrayof_jshort_disjoint_arraycopy = generate_disjoint_short_copy(true, &entry,
2419 "arrayof_jshort_disjoint_arraycopy");
2420 StubRoutines::_arrayof_jshort_arraycopy = generate_conjoint_short_copy(true, entry, NULL,
2421 "arrayof_jshort_arraycopy");
2422
2423 //*** jint
2424 // Aligned versions
2425 StubRoutines::_arrayof_jint_disjoint_arraycopy = generate_disjoint_int_copy(true, &entry,
2426 "arrayof_jint_disjoint_arraycopy");
2427 StubRoutines::_arrayof_jint_arraycopy = generate_conjoint_int_copy(true, entry, &entry_jint_arraycopy,
2428 "arrayof_jint_arraycopy");
2429 // In 64 bit we need both aligned and unaligned versions of jint arraycopy.
2430 // entry_jint_arraycopy always points to the unaligned version
2431 StubRoutines::_jint_disjoint_arraycopy = generate_disjoint_int_copy(false, &entry,
2432 "jint_disjoint_arraycopy");
2433 StubRoutines::_jint_arraycopy = generate_conjoint_int_copy(false, entry,
2434 &entry_jint_arraycopy,
2435 "jint_arraycopy");
2436
2437 //*** jlong
2438 // It is always aligned
2439 StubRoutines::_arrayof_jlong_disjoint_arraycopy = generate_disjoint_long_copy(true, &entry,
2440 "arrayof_jlong_disjoint_arraycopy");
2441 StubRoutines::_arrayof_jlong_arraycopy = generate_conjoint_long_copy(true, entry, &entry_jlong_arraycopy,
2442 "arrayof_jlong_arraycopy");
2443 StubRoutines::_jlong_disjoint_arraycopy = StubRoutines::_arrayof_jlong_disjoint_arraycopy;
2444 StubRoutines::_jlong_arraycopy = StubRoutines::_arrayof_jlong_arraycopy;
2445
2446 //*** oops
2447 {
2448 // With compressed oops we need unaligned versions; notice that
2449 // we overwrite entry_oop_arraycopy.
2450 bool aligned = !UseCompressedOops;
2451
2452 StubRoutines::_arrayof_oop_disjoint_arraycopy
2453 = generate_disjoint_oop_copy(aligned, &entry, "arrayof_oop_disjoint_arraycopy",
2454 /*dest_uninitialized*/false);
2455 StubRoutines::_arrayof_oop_arraycopy
2456 = generate_conjoint_oop_copy(aligned, entry, &entry_oop_arraycopy, "arrayof_oop_arraycopy",
2457 /*dest_uninitialized*/false);
2458 // Aligned versions without pre-barriers
2459 StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit
2460 = generate_disjoint_oop_copy(aligned, &entry, "arrayof_oop_disjoint_arraycopy_uninit",
2461 /*dest_uninitialized*/true);
2462 StubRoutines::_arrayof_oop_arraycopy_uninit
2463 = generate_conjoint_oop_copy(aligned, entry, NULL, "arrayof_oop_arraycopy_uninit",
2464 /*dest_uninitialized*/true);
2465 }
2466
2467 StubRoutines::_oop_disjoint_arraycopy = StubRoutines::_arrayof_oop_disjoint_arraycopy;
2468 StubRoutines::_oop_arraycopy = StubRoutines::_arrayof_oop_arraycopy;
2469 StubRoutines::_oop_disjoint_arraycopy_uninit = StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit;
2470 StubRoutines::_oop_arraycopy_uninit = StubRoutines::_arrayof_oop_arraycopy_uninit;
2471
2472 StubRoutines::_checkcast_arraycopy = generate_checkcast_copy("checkcast_arraycopy", &entry_checkcast_arraycopy);
2473 StubRoutines::_checkcast_arraycopy_uninit = generate_checkcast_copy("checkcast_arraycopy_uninit", NULL,
2474 /*dest_uninitialized*/true);
2475
2476 StubRoutines::_unsafe_arraycopy = generate_unsafe_copy("unsafe_arraycopy",
2477 entry_jbyte_arraycopy,
2478 entry_jshort_arraycopy,
2479 entry_jint_arraycopy,
2480 entry_jlong_arraycopy);
2481
2482 StubRoutines::_generic_arraycopy = generate_generic_copy("generic_arraycopy",
2483 entry_jbyte_arraycopy,
2484 entry_jshort_arraycopy,
2485 entry_jint_arraycopy,
2486 entry_oop_arraycopy,
2487 entry_jlong_arraycopy,
2488 entry_checkcast_arraycopy);
2489
2490 StubRoutines::_jbyte_fill = generate_fill(T_BYTE, false, "jbyte_fill");
2491 StubRoutines::_jshort_fill = generate_fill(T_SHORT, false, "jshort_fill");
2492 StubRoutines::_jint_fill = generate_fill(T_INT, false, "jint_fill");
2493 StubRoutines::_arrayof_jbyte_fill = generate_fill(T_BYTE, true, "arrayof_jbyte_fill");
2494 StubRoutines::_arrayof_jshort_fill = generate_fill(T_SHORT, true, "arrayof_jshort_fill");
2495 StubRoutines::_arrayof_jint_fill = generate_fill(T_INT, true, "arrayof_jint_fill");
2496 }
2497
2498 // Arguments:
2499 //
2500 // Inputs:
2501 // c_rarg0 - source byte array address
2502 // c_rarg1 - destination byte array address
2503 // c_rarg2 - K (key) in little endian int array
2504 //
2505 address generate_aescrypt_encryptBlock() {
2506 __ align(CodeEntryAlignment);
2507 StubCodeMark mark(this, "StubRoutines", "aescrypt_encryptBlock");
2508
2509 Label L_doLast;
2510
2511 const Register from = c_rarg0; // source array address
2512 const Register to = c_rarg1; // destination array address
2513 const Register key = c_rarg2; // key array address
2514 const Register keylen = rscratch1;
2515
2516 address start = __ pc();
2517 __ enter();
2518
2519 __ ldrw(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2520
2521 __ ld1(v0, __ T16B, from); // get 16 bytes of input
2522
2523 __ ld1(v1, v2, v3, v4, __ T16B, __ post(key, 64));
2524 __ rev32(v1, __ T16B, v1);
2525 __ rev32(v2, __ T16B, v2);
2526 __ rev32(v3, __ T16B, v3);
2527 __ rev32(v4, __ T16B, v4);
2528 __ aese(v0, v1);
2529 __ aesmc(v0, v0);
2530 __ aese(v0, v2);
2531 __ aesmc(v0, v0);
2532 __ aese(v0, v3);
2533 __ aesmc(v0, v0);
2534 __ aese(v0, v4);
2535 __ aesmc(v0, v0);
2536
2537 __ ld1(v1, v2, v3, v4, __ T16B, __ post(key, 64));
2538 __ rev32(v1, __ T16B, v1);
2539 __ rev32(v2, __ T16B, v2);
2540 __ rev32(v3, __ T16B, v3);
2541 __ rev32(v4, __ T16B, v4);
2542 __ aese(v0, v1);
2543 __ aesmc(v0, v0);
2544 __ aese(v0, v2);
2545 __ aesmc(v0, v0);
2546 __ aese(v0, v3);
2547 __ aesmc(v0, v0);
2548 __ aese(v0, v4);
2549 __ aesmc(v0, v0);
2550
2551 __ ld1(v1, v2, __ T16B, __ post(key, 32));
2552 __ rev32(v1, __ T16B, v1);
2553 __ rev32(v2, __ T16B, v2);
2554
2555 __ cmpw(keylen, 44);
2556 __ br(Assembler::EQ, L_doLast);
2557
2558 __ aese(v0, v1);
2559 __ aesmc(v0, v0);
2560 __ aese(v0, v2);
2561 __ aesmc(v0, v0);
2562
2563 __ ld1(v1, v2, __ T16B, __ post(key, 32));
2564 __ rev32(v1, __ T16B, v1);
2565 __ rev32(v2, __ T16B, v2);
2566
2567 __ cmpw(keylen, 52);
2568 __ br(Assembler::EQ, L_doLast);
2569
2570 __ aese(v0, v1);
2571 __ aesmc(v0, v0);
2572 __ aese(v0, v2);
2573 __ aesmc(v0, v0);
2574
2575 __ ld1(v1, v2, __ T16B, __ post(key, 32));
2576 __ rev32(v1, __ T16B, v1);
2577 __ rev32(v2, __ T16B, v2);
2578
2579 __ BIND(L_doLast);
2580
2581 __ aese(v0, v1);
2582 __ aesmc(v0, v0);
2583 __ aese(v0, v2);
2584
2585 __ ld1(v1, __ T16B, key);
2586 __ rev32(v1, __ T16B, v1);
2587 __ eor(v0, __ T16B, v0, v1);
2588
2589 __ st1(v0, __ T16B, to);
2590
2591 __ mov(r0, 0);
2592
2593 __ leave();
2594 __ ret(lr);
2595
2596 return start;
2597 }
2598
2599 // Arguments:
2600 //
2601 // Inputs:
2602 // c_rarg0 - source byte array address
2603 // c_rarg1 - destination byte array address
2604 // c_rarg2 - K (key) in little endian int array
2605 //
2606 address generate_aescrypt_decryptBlock() {
2607 assert(UseAES, "need AES instructions and misaligned SSE support");
2608 __ align(CodeEntryAlignment);
2609 StubCodeMark mark(this, "StubRoutines", "aescrypt_decryptBlock");
2610 Label L_doLast;
2611
2612 const Register from = c_rarg0; // source array address
2613 const Register to = c_rarg1; // destination array address
2614 const Register key = c_rarg2; // key array address
2615 const Register keylen = rscratch1;
2616
2617 address start = __ pc();
2618 __ enter(); // required for proper stackwalking of RuntimeStub frame
2619
2620 __ ldrw(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2621
2622 __ ld1(v0, __ T16B, from); // get 16 bytes of input
2623
2624 __ ld1(v5, __ T16B, __ post(key, 16));
2625 __ rev32(v5, __ T16B, v5);
2626
2627 __ ld1(v1, v2, v3, v4, __ T16B, __ post(key, 64));
2628 __ rev32(v1, __ T16B, v1);
2629 __ rev32(v2, __ T16B, v2);
2630 __ rev32(v3, __ T16B, v3);
2631 __ rev32(v4, __ T16B, v4);
2632 __ aesd(v0, v1);
2633 __ aesimc(v0, v0);
2634 __ aesd(v0, v2);
2635 __ aesimc(v0, v0);
2636 __ aesd(v0, v3);
2637 __ aesimc(v0, v0);
2638 __ aesd(v0, v4);
2639 __ aesimc(v0, v0);
2640
2641 __ ld1(v1, v2, v3, v4, __ T16B, __ post(key, 64));
2642 __ rev32(v1, __ T16B, v1);
2643 __ rev32(v2, __ T16B, v2);
2644 __ rev32(v3, __ T16B, v3);
2645 __ rev32(v4, __ T16B, v4);
2646 __ aesd(v0, v1);
2647 __ aesimc(v0, v0);
2648 __ aesd(v0, v2);
2649 __ aesimc(v0, v0);
2650 __ aesd(v0, v3);
2651 __ aesimc(v0, v0);
2652 __ aesd(v0, v4);
2653 __ aesimc(v0, v0);
2654
2655 __ ld1(v1, v2, __ T16B, __ post(key, 32));
2656 __ rev32(v1, __ T16B, v1);
2657 __ rev32(v2, __ T16B, v2);
2658
2659 __ cmpw(keylen, 44);
2660 __ br(Assembler::EQ, L_doLast);
2661
2662 __ aesd(v0, v1);
2663 __ aesimc(v0, v0);
2664 __ aesd(v0, v2);
2665 __ aesimc(v0, v0);
2666
2667 __ ld1(v1, v2, __ T16B, __ post(key, 32));
2668 __ rev32(v1, __ T16B, v1);
2669 __ rev32(v2, __ T16B, v2);
2670
2671 __ cmpw(keylen, 52);
2672 __ br(Assembler::EQ, L_doLast);
2673
2674 __ aesd(v0, v1);
2675 __ aesimc(v0, v0);
2676 __ aesd(v0, v2);
2677 __ aesimc(v0, v0);
2678
2679 __ ld1(v1, v2, __ T16B, __ post(key, 32));
2680 __ rev32(v1, __ T16B, v1);
2681 __ rev32(v2, __ T16B, v2);
2682
2683 __ BIND(L_doLast);
2684
2685 __ aesd(v0, v1);
2686 __ aesimc(v0, v0);
2687 __ aesd(v0, v2);
2688
2689 __ eor(v0, __ T16B, v0, v5);
2690
2691 __ st1(v0, __ T16B, to);
2692
2693 __ mov(r0, 0);
2694
2695 __ leave();
2696 __ ret(lr);
2697
2698 return start;
2699 }
2700
2701 // Arguments:
2702 //
2703 // Inputs:
2704 // c_rarg0 - source byte array address
2705 // c_rarg1 - destination byte array address
2706 // c_rarg2 - K (key) in little endian int array
2707 // c_rarg3 - r vector byte array address
2708 // c_rarg4 - input length
2709 //
2710 // Output:
2711 // x0 - input length
2712 //
2713 address generate_cipherBlockChaining_encryptAESCrypt() {
2714 assert(UseAES, "need AES instructions and misaligned SSE support");
2715 __ align(CodeEntryAlignment);
2716 StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_encryptAESCrypt");
2717
2718 Label L_loadkeys_44, L_loadkeys_52, L_aes_loop, L_rounds_44, L_rounds_52, _L_finish;
2719
2720 const Register from = c_rarg0; // source array address
2721 const Register to = c_rarg1; // destination array address
2722 const Register key = c_rarg2; // key array address
2723 const Register rvec = c_rarg3; // r byte array initialized from initvector array address
2724 // and left with the results of the last encryption block
2725 const Register len_reg = c_rarg4; // src len (must be multiple of blocksize 16)
2726 const Register keylen = rscratch1;
2727
2728 address start = __ pc();
2729
2730 __ enter();
2731
2732 __ subsw(rscratch2, len_reg, zr);
2733 __ br(Assembler::LE, _L_finish);
2734
2735 __ ldrw(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2736
2737 __ ld1(v0, __ T16B, rvec);
2738
2739 __ cmpw(keylen, 52);
2740 __ br(Assembler::CC, L_loadkeys_44);
2741 __ br(Assembler::EQ, L_loadkeys_52);
2742
2743 __ ld1(v17, v18, __ T16B, __ post(key, 32));
2744 __ rev32(v17, __ T16B, v17);
2745 __ rev32(v18, __ T16B, v18);
2746 __ BIND(L_loadkeys_52);
2747 __ ld1(v19, v20, __ T16B, __ post(key, 32));
2748 __ rev32(v19, __ T16B, v19);
2749 __ rev32(v20, __ T16B, v20);
2750 __ BIND(L_loadkeys_44);
2751 __ ld1(v21, v22, v23, v24, __ T16B, __ post(key, 64));
2752 __ rev32(v21, __ T16B, v21);
2753 __ rev32(v22, __ T16B, v22);
2754 __ rev32(v23, __ T16B, v23);
2755 __ rev32(v24, __ T16B, v24);
2756 __ ld1(v25, v26, v27, v28, __ T16B, __ post(key, 64));
2757 __ rev32(v25, __ T16B, v25);
2758 __ rev32(v26, __ T16B, v26);
2759 __ rev32(v27, __ T16B, v27);
2760 __ rev32(v28, __ T16B, v28);
2761 __ ld1(v29, v30, v31, __ T16B, key);
2762 __ rev32(v29, __ T16B, v29);
2763 __ rev32(v30, __ T16B, v30);
2764 __ rev32(v31, __ T16B, v31);
2765
2766 __ BIND(L_aes_loop);
2767 __ ld1(v1, __ T16B, __ post(from, 16));
2768 __ eor(v0, __ T16B, v0, v1);
2769
2770 __ br(Assembler::CC, L_rounds_44);
2771 __ br(Assembler::EQ, L_rounds_52);
2772
2773 __ aese(v0, v17); __ aesmc(v0, v0);
2774 __ aese(v0, v18); __ aesmc(v0, v0);
2775 __ BIND(L_rounds_52);
2776 __ aese(v0, v19); __ aesmc(v0, v0);
2777 __ aese(v0, v20); __ aesmc(v0, v0);
2778 __ BIND(L_rounds_44);
2779 __ aese(v0, v21); __ aesmc(v0, v0);
2780 __ aese(v0, v22); __ aesmc(v0, v0);
2781 __ aese(v0, v23); __ aesmc(v0, v0);
2782 __ aese(v0, v24); __ aesmc(v0, v0);
2783 __ aese(v0, v25); __ aesmc(v0, v0);
2784 __ aese(v0, v26); __ aesmc(v0, v0);
2785 __ aese(v0, v27); __ aesmc(v0, v0);
2786 __ aese(v0, v28); __ aesmc(v0, v0);
2787 __ aese(v0, v29); __ aesmc(v0, v0);
2788 __ aese(v0, v30);
2789 __ eor(v0, __ T16B, v0, v31);
2790
2791 __ st1(v0, __ T16B, __ post(to, 16));
2792
2793 __ subw(len_reg, len_reg, 16);
2794 __ cbnzw(len_reg, L_aes_loop);
2795
2796 __ st1(v0, __ T16B, rvec);
2797
2798 __ BIND(_L_finish);
2799 __ mov(r0, rscratch2);
2800
2801 __ leave();
2802 __ ret(lr);
2803
2804 return start;
2805 }
2806
2807 // Arguments:
2808 //
2809 // Inputs:
2810 // c_rarg0 - source byte array address
2811 // c_rarg1 - destination byte array address
2812 // c_rarg2 - K (key) in little endian int array
2813 // c_rarg3 - r vector byte array address
2814 // c_rarg4 - input length
2815 //
2816 // Output:
2817 // r0 - input length
2818 //
2819 address generate_cipherBlockChaining_decryptAESCrypt() {
2820 assert(UseAES, "need AES instructions and misaligned SSE support");
2821 __ align(CodeEntryAlignment);
2822 StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_decryptAESCrypt");
2823
2824 Label L_loadkeys_44, L_loadkeys_52, L_aes_loop, L_rounds_44, L_rounds_52, _L_finish;
2825
2826 const Register from = c_rarg0; // source array address
2827 const Register to = c_rarg1; // destination array address
2828 const Register key = c_rarg2; // key array address
2829 const Register rvec = c_rarg3; // r byte array initialized from initvector array address
2830 // and left with the results of the last encryption block
2831 const Register len_reg = c_rarg4; // src len (must be multiple of blocksize 16)
2832 const Register keylen = rscratch1;
2833
2834 address start = __ pc();
2835
2836 __ enter();
2837
2838 __ subsw(rscratch2, len_reg, zr);
2839 __ br(Assembler::LE, _L_finish);
2840
2841 __ ldrw(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2842
2843 __ ld1(v2, __ T16B, rvec);
2844
2845 __ ld1(v31, __ T16B, __ post(key, 16));
2846 __ rev32(v31, __ T16B, v31);
2847
2848 __ cmpw(keylen, 52);
2849 __ br(Assembler::CC, L_loadkeys_44);
2850 __ br(Assembler::EQ, L_loadkeys_52);
2851
2852 __ ld1(v17, v18, __ T16B, __ post(key, 32));
2853 __ rev32(v17, __ T16B, v17);
2854 __ rev32(v18, __ T16B, v18);
2855 __ BIND(L_loadkeys_52);
2856 __ ld1(v19, v20, __ T16B, __ post(key, 32));
2857 __ rev32(v19, __ T16B, v19);
2858 __ rev32(v20, __ T16B, v20);
2859 __ BIND(L_loadkeys_44);
2860 __ ld1(v21, v22, v23, v24, __ T16B, __ post(key, 64));
2861 __ rev32(v21, __ T16B, v21);
2862 __ rev32(v22, __ T16B, v22);
2863 __ rev32(v23, __ T16B, v23);
2864 __ rev32(v24, __ T16B, v24);
2865 __ ld1(v25, v26, v27, v28, __ T16B, __ post(key, 64));
2866 __ rev32(v25, __ T16B, v25);
2867 __ rev32(v26, __ T16B, v26);
2868 __ rev32(v27, __ T16B, v27);
2869 __ rev32(v28, __ T16B, v28);
2870 __ ld1(v29, v30, __ T16B, key);
2871 __ rev32(v29, __ T16B, v29);
2872 __ rev32(v30, __ T16B, v30);
2873
2874 __ BIND(L_aes_loop);
2875 __ ld1(v0, __ T16B, __ post(from, 16));
2876 __ orr(v1, __ T16B, v0, v0);
2877
2878 __ br(Assembler::CC, L_rounds_44);
2879 __ br(Assembler::EQ, L_rounds_52);
2880
2881 __ aesd(v0, v17); __ aesimc(v0, v0);
2882 __ aesd(v0, v18); __ aesimc(v0, v0);
2883 __ BIND(L_rounds_52);
2884 __ aesd(v0, v19); __ aesimc(v0, v0);
2885 __ aesd(v0, v20); __ aesimc(v0, v0);
2886 __ BIND(L_rounds_44);
2887 __ aesd(v0, v21); __ aesimc(v0, v0);
2888 __ aesd(v0, v22); __ aesimc(v0, v0);
2889 __ aesd(v0, v23); __ aesimc(v0, v0);
2890 __ aesd(v0, v24); __ aesimc(v0, v0);
2891 __ aesd(v0, v25); __ aesimc(v0, v0);
2892 __ aesd(v0, v26); __ aesimc(v0, v0);
2893 __ aesd(v0, v27); __ aesimc(v0, v0);
2894 __ aesd(v0, v28); __ aesimc(v0, v0);
2895 __ aesd(v0, v29); __ aesimc(v0, v0);
2896 __ aesd(v0, v30);
2897 __ eor(v0, __ T16B, v0, v31);
2898 __ eor(v0, __ T16B, v0, v2);
2899
2900 __ st1(v0, __ T16B, __ post(to, 16));
2901 __ orr(v2, __ T16B, v1, v1);
2902
2903 __ subw(len_reg, len_reg, 16);
2904 __ cbnzw(len_reg, L_aes_loop);
2905
2906 __ st1(v2, __ T16B, rvec);
2907
2908 __ BIND(_L_finish);
2909 __ mov(r0, rscratch2);
2910
2911 __ leave();
2912 __ ret(lr);
2913
2914 return start;
2915 }
2916
2917 // Arguments:
2918 //
2919 // Inputs:
2920 // c_rarg0 - byte[] source+offset
2921 // c_rarg1 - int[] SHA.state
2922 // c_rarg2 - int offset
2923 // c_rarg3 - int limit
2924 //
2925 address generate_sha1_implCompress(bool multi_block, const char *name) {
2926 __ align(CodeEntryAlignment);
2927 StubCodeMark mark(this, "StubRoutines", name);
2928 address start = __ pc();
2929
2930 Register buf = c_rarg0;
2931 Register state = c_rarg1;
2932 Register ofs = c_rarg2;
2933 Register limit = c_rarg3;
2934
2935 Label keys;
2936 Label sha1_loop;
2937
2938 // load the keys into v0..v3
2939 __ adr(rscratch1, keys);
2940 __ ld4r(v0, v1, v2, v3, __ T4S, Address(rscratch1));
2941 // load 5 words state into v6, v7
2942 __ ldrq(v6, Address(state, 0));
2943 __ ldrs(v7, Address(state, 16));
2944
2945
2946 __ BIND(sha1_loop);
2947 // load 64 bytes of data into v16..v19
2948 __ ld1(v16, v17, v18, v19, __ T4S, multi_block ? __ post(buf, 64) : buf);
2949 __ rev32(v16, __ T16B, v16);
2950 __ rev32(v17, __ T16B, v17);
2951 __ rev32(v18, __ T16B, v18);
2952 __ rev32(v19, __ T16B, v19);
2953
2954 // do the sha1
2955 __ addv(v4, __ T4S, v16, v0);
2956 __ orr(v20, __ T16B, v6, v6);
2957
2958 FloatRegister d0 = v16;
2959 FloatRegister d1 = v17;
2960 FloatRegister d2 = v18;
2961 FloatRegister d3 = v19;
2962
2963 for (int round = 0; round < 20; round++) {
2964 FloatRegister tmp1 = (round & 1) ? v4 : v5;
2965 FloatRegister tmp2 = (round & 1) ? v21 : v22;
2966 FloatRegister tmp3 = round ? ((round & 1) ? v22 : v21) : v7;
2967 FloatRegister tmp4 = (round & 1) ? v5 : v4;
2968 FloatRegister key = (round < 4) ? v0 : ((round < 9) ? v1 : ((round < 14) ? v2 : v3));
2969
2970 if (round < 16) __ sha1su0(d0, __ T4S, d1, d2);
2971 if (round < 19) __ addv(tmp1, __ T4S, d1, key);
2972 __ sha1h(tmp2, __ T4S, v20);
2973 if (round < 5)
2974 __ sha1c(v20, __ T4S, tmp3, tmp4);
2975 else if (round < 10 || round >= 15)
2976 __ sha1p(v20, __ T4S, tmp3, tmp4);
2977 else
2978 __ sha1m(v20, __ T4S, tmp3, tmp4);
2979 if (round < 16) __ sha1su1(d0, __ T4S, d3);
2980
2981 tmp1 = d0; d0 = d1; d1 = d2; d2 = d3; d3 = tmp1;
2982 }
2983
2984 __ addv(v7, __ T2S, v7, v21);
2985 __ addv(v6, __ T4S, v6, v20);
2986
2987 if (multi_block) {
2988 __ add(ofs, ofs, 64);
2989 __ cmp(ofs, limit);
2990 __ br(Assembler::LE, sha1_loop);
2991 __ mov(c_rarg0, ofs); // return ofs
2992 }
2993
2994 __ strq(v6, Address(state, 0));
2995 __ strs(v7, Address(state, 16));
2996
2997 __ ret(lr);
2998
2999 __ bind(keys);
3000 __ emit_int32(0x5a827999);
3001 __ emit_int32(0x6ed9eba1);
3002 __ emit_int32(0x8f1bbcdc);
3003 __ emit_int32(0xca62c1d6);
3004
3005 return start;
3006 }
3007
3008
3009 // Arguments:
3010 //
3011 // Inputs:
3012 // c_rarg0 - byte[] source+offset
3013 // c_rarg1 - int[] SHA.state
3014 // c_rarg2 - int offset
3015 // c_rarg3 - int limit
3016 //
3017 address generate_sha256_implCompress(bool multi_block, const char *name) {
3018 static const uint32_t round_consts[64] = {
3019 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
3020 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
3021 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
3022 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
3023 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,
3024 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
3025 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
3026 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
3027 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
3028 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
3029 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
3030 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
3031 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,
3032 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
3033 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
3034 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,
3035 };
3036 __ align(CodeEntryAlignment);
3037 StubCodeMark mark(this, "StubRoutines", name);
3038 address start = __ pc();
3039
3040 Register buf = c_rarg0;
3041 Register state = c_rarg1;
3042 Register ofs = c_rarg2;
3043 Register limit = c_rarg3;
3044
3045 Label sha1_loop;
3046
3047 __ stpd(v8, v9, __ pre(sp, -32));
3048 __ stpd(v10, v11, Address(sp, 16));
3049
3050 // dga == v0
3051 // dgb == v1
3052 // dg0 == v2
3053 // dg1 == v3
3054 // dg2 == v4
3055 // t0 == v6
3056 // t1 == v7
3057
3058 // load 16 keys to v16..v31
3059 __ lea(rscratch1, ExternalAddress((address)round_consts));
3060 __ ld1(v16, v17, v18, v19, __ T4S, __ post(rscratch1, 64));
3061 __ ld1(v20, v21, v22, v23, __ T4S, __ post(rscratch1, 64));
3062 __ ld1(v24, v25, v26, v27, __ T4S, __ post(rscratch1, 64));
3063 __ ld1(v28, v29, v30, v31, __ T4S, rscratch1);
3064
3065 // load 8 words (256 bits) state
3066 __ ldpq(v0, v1, state);
3067
3068 __ BIND(sha1_loop);
3069 // load 64 bytes of data into v8..v11
3070 __ ld1(v8, v9, v10, v11, __ T4S, multi_block ? __ post(buf, 64) : buf);
3071 __ rev32(v8, __ T16B, v8);
3072 __ rev32(v9, __ T16B, v9);
3073 __ rev32(v10, __ T16B, v10);
3074 __ rev32(v11, __ T16B, v11);
3075
3076 __ addv(v6, __ T4S, v8, v16);
3077 __ orr(v2, __ T16B, v0, v0);
3078 __ orr(v3, __ T16B, v1, v1);
3079
3080 FloatRegister d0 = v8;
3081 FloatRegister d1 = v9;
3082 FloatRegister d2 = v10;
3083 FloatRegister d3 = v11;
3084
3085
3086 for (int round = 0; round < 16; round++) {
3087 FloatRegister tmp1 = (round & 1) ? v6 : v7;
3088 FloatRegister tmp2 = (round & 1) ? v7 : v6;
3089 FloatRegister tmp3 = (round & 1) ? v2 : v4;
3090 FloatRegister tmp4 = (round & 1) ? v4 : v2;
3091
3092 if (round < 12) __ sha256su0(d0, __ T4S, d1);
3093 __ orr(v4, __ T16B, v2, v2);
3094 if (round < 15)
3095 __ addv(tmp1, __ T4S, d1, as_FloatRegister(round + 17));
3096 __ sha256h(v2, __ T4S, v3, tmp2);
3097 __ sha256h2(v3, __ T4S, v4, tmp2);
3098 if (round < 12) __ sha256su1(d0, __ T4S, d2, d3);
3099
3100 tmp1 = d0; d0 = d1; d1 = d2; d2 = d3; d3 = tmp1;
3101 }
3102
3103 __ addv(v0, __ T4S, v0, v2);
3104 __ addv(v1, __ T4S, v1, v3);
3105
3106 if (multi_block) {
3107 __ add(ofs, ofs, 64);
3108 __ cmp(ofs, limit);
3109 __ br(Assembler::LE, sha1_loop);
3110 __ mov(c_rarg0, ofs); // return ofs
3111 }
3112
3113 __ ldpd(v10, v11, Address(sp, 16));
3114 __ ldpd(v8, v9, __ post(sp, 32));
3115
3116 __ stpq(v0, v1, state);
3117
3118 __ ret(lr);
3119
3120 return start;
3121 }
3122
3123 // Safefetch stubs.
3124 void generate_safefetch(const char* name, int size, address* entry,
3125 address* fault_pc, address* continuation_pc) {
3126 // safefetch signatures:
3127 // int SafeFetch32(int* adr, int errValue);
3128 // intptr_t SafeFetchN (intptr_t* adr, intptr_t errValue);
3129 //
3130 // arguments:
3131 // c_rarg0 = adr
3132 // c_rarg1 = errValue
3133 //
3134 // result:
3135 // PPC_RET = *adr or errValue
3136
3137 StubCodeMark mark(this, "StubRoutines", name);
3138
3139 // Entry point, pc or function descriptor.
3140 *entry = __ pc();
3141
3142 // Load *adr into c_rarg1, may fault.
3143 *fault_pc = __ pc();
3144 switch (size) {
3145 case 4:
3146 // int32_t
3147 __ ldrw(c_rarg1, Address(c_rarg0, 0));
3148 break;
3149 case 8:
3150 // int64_t
3151 __ ldr(c_rarg1, Address(c_rarg0, 0));
3152 break;
3153 default:
3154 ShouldNotReachHere();
3155 }
3156
3157 // return errValue or *adr
3158 *continuation_pc = __ pc();
3159 __ mov(r0, c_rarg1);
3160 __ ret(lr);
3161 }
3162
3163 /**
3164 * Arguments:
3165 *
3166 * Inputs:
3167 * c_rarg0 - int crc
3168 * c_rarg1 - byte* buf
3169 * c_rarg2 - int length
3170 *
3171 * Output:
3172 * r0 - int crc result
3173 *
3174 * Preserves:
3175 * r13
3176 *
3177 */
3178 address generate_updateBytesCRC32() {
3179 assert(UseCRC32Intrinsics, "what are we doing here?");
3180
3181 __ align(CodeEntryAlignment);
3182 StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32");
3183
3184 address start = __ pc();
3185
3186 const Register crc = c_rarg0; // crc
3187 const Register buf = c_rarg1; // source java byte array address
3188 const Register len = c_rarg2; // length
3189 const Register table0 = c_rarg3; // crc_table address
3190 const Register table1 = c_rarg4;
3191 const Register table2 = c_rarg5;
3192 const Register table3 = c_rarg6;
3193 const Register tmp3 = c_rarg7;
3194
3195 BLOCK_COMMENT("Entry:");
3196 __ enter(); // required for proper stackwalking of RuntimeStub frame
3197
3198 __ kernel_crc32(crc, buf, len,
3199 table0, table1, table2, table3, rscratch1, rscratch2, tmp3);
3200
3201 __ leave(); // required for proper stackwalking of RuntimeStub frame
3202 __ ret(lr);
3203
3204 return start;
3205 }
3206
3207 /**
3208 * Arguments:
3209 *
3210 * Input:
3211 * c_rarg0 - x address
3212 * c_rarg1 - x length
3213 * c_rarg2 - y address
3214 * c_rarg3 - y lenth
3215 * c_rarg4 - z address
3216 * c_rarg5 - z length
3217 */
3218 address generate_multiplyToLen() {
3219 __ align(CodeEntryAlignment);
3220 StubCodeMark mark(this, "StubRoutines", "multiplyToLen");
3221
3222 address start = __ pc();
3223 const Register x = r0;
3224 const Register xlen = r1;
3225 const Register y = r2;
3226 const Register ylen = r3;
3227 const Register z = r4;
3228 const Register zlen = r5;
3229
3230 const Register tmp1 = r10;
3231 const Register tmp2 = r11;
3232 const Register tmp3 = r12;
3233 const Register tmp4 = r13;
3234 const Register tmp5 = r14;
3235 const Register tmp6 = r15;
3236 const Register tmp7 = r16;
3237
3238 BLOCK_COMMENT("Entry:");
3239 __ enter(); // required for proper stackwalking of RuntimeStub frame
3240 __ multiply_to_len(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7);
3241 __ leave(); // required for proper stackwalking of RuntimeStub frame
3242 __ ret(lr);
3243
3244 return start;
3245 }
3246
3247 // Continuation point for throwing of implicit exceptions that are
3248 // not handled in the current activation. Fabricates an exception
3249 // oop and initiates normal exception dispatching in this
3250 // frame. Since we need to preserve callee-saved values (currently
3251 // only for C2, but done for C1 as well) we need a callee-saved oop
3252 // map and therefore have to make these stubs into RuntimeStubs
3253 // rather than BufferBlobs. If the compiler needs all registers to
3254 // be preserved between the fault point and the exception handler
3255 // then it must assume responsibility for that in
3256 // AbstractCompiler::continuation_for_implicit_null_exception or
3257 // continuation_for_implicit_division_by_zero_exception. All other
3258 // implicit exceptions (e.g., NullPointerException or
3259 // AbstractMethodError on entry) are either at call sites or
3260 // otherwise assume that stack unwinding will be initiated, so
3261 // caller saved registers were assumed volatile in the compiler.
3262
3263 #undef __
3264 #define __ masm->
3265
3266 address generate_throw_exception(const char* name,
3267 address runtime_entry,
3268 Register arg1 = noreg,
3269 Register arg2 = noreg) {
3270 // Information about frame layout at time of blocking runtime call.
3271 // Note that we only have to preserve callee-saved registers since
3272 // the compilers are responsible for supplying a continuation point
3273 // if they expect all registers to be preserved.
3274 // n.b. aarch64 asserts that frame::arg_reg_save_area_bytes == 0
3275 enum layout {
3276 rfp_off = 0,
3277 rfp_off2,
3278 return_off,
3279 return_off2,
3280 framesize // inclusive of return address
3281 };
3282
3283 int insts_size = 512;
3284 int locs_size = 64;
3285
3286 CodeBuffer code(name, insts_size, locs_size);
3287 OopMapSet* oop_maps = new OopMapSet();
3288 MacroAssembler* masm = new MacroAssembler(&code);
3289
3290 address start = __ pc();
3291
3292 // This is an inlined and slightly modified version of call_VM
3293 // which has the ability to fetch the return PC out of
3294 // thread-local storage and also sets up last_Java_sp slightly
3295 // differently than the real call_VM
3296
3297 __ enter(); // Save FP and LR before call
3298
3299 assert(is_even(framesize/2), "sp not 16-byte aligned");
3300
3301 // lr and fp are already in place
3302 __ sub(sp, rfp, ((unsigned)framesize-4) << LogBytesPerInt); // prolog
3303
3304 int frame_complete = __ pc() - start;
3305
3306 // Set up last_Java_sp and last_Java_fp
3307 address the_pc = __ pc();
3308 __ set_last_Java_frame(sp, rfp, (address)NULL, rscratch1);
3309
3310 // Call runtime
3311 if (arg1 != noreg) {
3312 assert(arg2 != c_rarg1, "clobbered");
3313 __ mov(c_rarg1, arg1);
3314 }
3315 if (arg2 != noreg) {
3316 __ mov(c_rarg2, arg2);
3317 }
3318 __ mov(c_rarg0, rthread);
3319 BLOCK_COMMENT("call runtime_entry");
3320 __ mov(rscratch1, runtime_entry);
3321 __ blr(rscratch1);
3322
3323 // Generate oop map
3324 OopMap* map = new OopMap(framesize, 0);
3325
3326 oop_maps->add_gc_map(the_pc - start, map);
3327
3328 __ reset_last_Java_frame(true);
3329 __ maybe_isb();
3330
3331 __ leave();
3332
3333 // check for pending exceptions
3334 #ifdef ASSERT
3335 Label L;
3336 __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
3337 __ cbnz(rscratch1, L);
3338 __ should_not_reach_here();
3339 __ bind(L);
3340 #endif // ASSERT
3341 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
3342
3343
3344 // codeBlob framesize is in words (not VMRegImpl::slot_size)
3345 RuntimeStub* stub =
3346 RuntimeStub::new_runtime_stub(name,
3347 &code,
3348 frame_complete,
3349 (framesize >> (LogBytesPerWord - LogBytesPerInt)),
3350 oop_maps, false);
3351 return stub->entry_point();
3352 }
3353
3354 class MontgomeryMultiplyGenerator : public MacroAssembler {
3355
3356 Register Pa_base, Pb_base, Pn_base, Pm_base, inv, Rlen, Ra, Rb, Rm, Rn,
3357 Pa, Pb, Pn, Pm, Rhi_ab, Rlo_ab, Rhi_mn, Rlo_mn, t0, t1, t2, Ri, Rj;
3358
3359 RegSet _toSave;
3360 bool _squaring;
3361
3362 public:
3363 MontgomeryMultiplyGenerator (Assembler *as, bool squaring)
3364 : MacroAssembler(as->code()), _squaring(squaring) {
3365
3366 // Register allocation
3367
3368 Register reg = c_rarg0;
3369 Pa_base = reg; // Argument registers
3370 if (squaring)
3371 Pb_base = Pa_base;
3372 else
3373 Pb_base = ++reg;
3374 Pn_base = ++reg;
3375 Rlen= ++reg;
3376 inv = ++reg;
3377 Pm_base = ++reg;
3378
3379 // Working registers:
3380 Ra = ++reg; // The current digit of a, b, n, and m.
3381 Rb = ++reg;
3382 Rm = ++reg;
3383 Rn = ++reg;
3384
3385 Pa = ++reg; // Pointers to the current/next digit of a, b, n, and m.
3386 Pb = ++reg;
3387 Pm = ++reg;
3388 Pn = ++reg;
3389
3390 t0 = ++reg; // Three registers which form a
3391 t1 = ++reg; // triple-precision accumuator.
3392 t2 = ++reg;
3393
3394 Ri = ++reg; // Inner and outer loop indexes.
3395 Rj = ++reg;
3396
3397 Rhi_ab = ++reg; // Product registers: low and high parts
3398 Rlo_ab = ++reg; // of a*b and m*n.
3399 Rhi_mn = ++reg;
3400 Rlo_mn = ++reg;
3401
3402 // r19 and up are callee-saved.
3403 _toSave = RegSet::range(r19, reg) + Pm_base;
3404 }
3405
3406 private:
3407 void save_regs() {
3408 push(_toSave, sp);
3409 }
3410
3411 void restore_regs() {
3412 pop(_toSave, sp);
3413 }
3414
3415 template <typename T>
3416 void unroll_2(Register count, T block) {
3417 Label loop, end, odd;
3418 tbnz(count, 0, odd);
3419 cbz(count, end);
3420 align(16);
3421 bind(loop);
3422 (this->*block)();
3423 bind(odd);
3424 (this->*block)();
3425 subs(count, count, 2);
3426 br(Assembler::GT, loop);
3427 bind(end);
3428 }
3429
3430 template <typename T>
3431 void unroll_2(Register count, T block, Register d, Register s, Register tmp) {
3432 Label loop, end, odd;
3433 tbnz(count, 0, odd);
3434 cbz(count, end);
3435 align(16);
3436 bind(loop);
3437 (this->*block)(d, s, tmp);
3438 bind(odd);
3439 (this->*block)(d, s, tmp);
3440 subs(count, count, 2);
3441 br(Assembler::GT, loop);
3442 bind(end);
3443 }
3444
3445 void pre1(RegisterOrConstant i) {
3446 block_comment("pre1");
3447 // Pa = Pa_base;
3448 // Pb = Pb_base + i;
3449 // Pm = Pm_base;
3450 // Pn = Pn_base + i;
3451 // Ra = *Pa;
3452 // Rb = *Pb;
3453 // Rm = *Pm;
3454 // Rn = *Pn;
3455 ldr(Ra, Address(Pa_base));
3456 ldr(Rb, Address(Pb_base, i, Address::uxtw(LogBytesPerWord)));
3457 ldr(Rm, Address(Pm_base));
3458 ldr(Rn, Address(Pn_base, i, Address::uxtw(LogBytesPerWord)));
3459 lea(Pa, Address(Pa_base));
3460 lea(Pb, Address(Pb_base, i, Address::uxtw(LogBytesPerWord)));
3461 lea(Pm, Address(Pm_base));
3462 lea(Pn, Address(Pn_base, i, Address::uxtw(LogBytesPerWord)));
3463
3464 // Zero the m*n result.
3465 mov(Rhi_mn, zr);
3466 mov(Rlo_mn, zr);
3467 }
3468
3469 // The core multiply-accumulate step of a Montgomery
3470 // multiplication. The idea is to schedule operations as a
3471 // pipeline so that instructions with long latencies (loads and
3472 // multiplies) have time to complete before their results are
3473 // used. This most benefits in-order implementations of the
3474 // architecture but out-of-order ones also benefit.
3475 void step() {
3476 block_comment("step");
3477 // MACC(Ra, Rb, t0, t1, t2);
3478 // Ra = *++Pa;
3479 // Rb = *--Pb;
3480 umulh(Rhi_ab, Ra, Rb);
3481 mul(Rlo_ab, Ra, Rb);
3482 ldr(Ra, pre(Pa, wordSize));
3483 ldr(Rb, pre(Pb, -wordSize));
3484 acc(Rhi_mn, Rlo_mn, t0, t1, t2); // The pending m*n from the
3485 // previous iteration.
3486 // MACC(Rm, Rn, t0, t1, t2);
3487 // Rm = *++Pm;
3488 // Rn = *--Pn;
3489 umulh(Rhi_mn, Rm, Rn);
3490 mul(Rlo_mn, Rm, Rn);
3491 ldr(Rm, pre(Pm, wordSize));
3492 ldr(Rn, pre(Pn, -wordSize));
3493 acc(Rhi_ab, Rlo_ab, t0, t1, t2);
3494 }
3495
3496 void post1() {
3497 block_comment("post1");
3498
3499 // MACC(Ra, Rb, t0, t1, t2);
3500 // Ra = *++Pa;
3501 // Rb = *--Pb;
3502 umulh(Rhi_ab, Ra, Rb);
3503 mul(Rlo_ab, Ra, Rb);
3504 acc(Rhi_mn, Rlo_mn, t0, t1, t2); // The pending m*n
3505 acc(Rhi_ab, Rlo_ab, t0, t1, t2);
3506
3507 // *Pm = Rm = t0 * inv;
3508 mul(Rm, t0, inv);
3509 str(Rm, Address(Pm));
3510
3511 // MACC(Rm, Rn, t0, t1, t2);
3512 // t0 = t1; t1 = t2; t2 = 0;
3513 umulh(Rhi_mn, Rm, Rn);
3514
3515 #ifndef PRODUCT
3516 // assert(m[i] * n[0] + t0 == 0, "broken Montgomery multiply");
3517 {
3518 mul(Rlo_mn, Rm, Rn);
3519 add(Rlo_mn, t0, Rlo_mn);
3520 Label ok;
3521 cbz(Rlo_mn, ok); {
3522 stop("broken Montgomery multiply");
3523 } bind(ok);
3524 }
3525 #endif
3526 // We have very carefully set things up so that
3527 // m[i]*n[0] + t0 == 0 (mod b), so we don't have to calculate
3528 // the lower half of Rm * Rn because we know the result already:
3529 // it must be -t0. t0 + (-t0) must generate a carry iff
3530 // t0 != 0. So, rather than do a mul and an adds we just set
3531 // the carry flag iff t0 is nonzero.
3532 //
3533 // mul(Rlo_mn, Rm, Rn);
3534 // adds(zr, t0, Rlo_mn);
3535 subs(zr, t0, 1); // Set carry iff t0 is nonzero
3536 adcs(t0, t1, Rhi_mn);
3537 adc(t1, t2, zr);
3538 mov(t2, zr);
3539 }
3540
3541 void pre2(RegisterOrConstant i, RegisterOrConstant len) {
3542 block_comment("pre2");
3543 // Pa = Pa_base + i-len;
3544 // Pb = Pb_base + len;
3545 // Pm = Pm_base + i-len;
3546 // Pn = Pn_base + len;
3547
3548 if (i.is_register()) {
3549 sub(Rj, i.as_register(), len);
3550 } else {
3551 mov(Rj, i.as_constant());
3552 sub(Rj, Rj, len);
3553 }
3554 // Rj == i-len
3555
3556 lea(Pa, Address(Pa_base, Rj, Address::uxtw(LogBytesPerWord)));
3557 lea(Pb, Address(Pb_base, len, Address::uxtw(LogBytesPerWord)));
3558 lea(Pm, Address(Pm_base, Rj, Address::uxtw(LogBytesPerWord)));
3559 lea(Pn, Address(Pn_base, len, Address::uxtw(LogBytesPerWord)));
3560
3561 // Ra = *++Pa;
3562 // Rb = *--Pb;
3563 // Rm = *++Pm;
3564 // Rn = *--Pn;
3565 ldr(Ra, pre(Pa, wordSize));
3566 ldr(Rb, pre(Pb, -wordSize));
3567 ldr(Rm, pre(Pm, wordSize));
3568 ldr(Rn, pre(Pn, -wordSize));
3569
3570 mov(Rhi_mn, zr);
3571 mov(Rlo_mn, zr);
3572 }
3573
3574 void post2(RegisterOrConstant i, RegisterOrConstant len) {
3575 block_comment("post2");
3576 if (i.is_constant()) {
3577 mov(Rj, i.as_constant()-len.as_constant());
3578 } else {
3579 sub(Rj, i.as_register(), len);
3580 }
3581
3582 adds(t0, t0, Rlo_mn); // The pending m*n, low part
3583
3584 // As soon as we know the least significant digit of our result,
3585 // store it.
3586 // Pm_base[i-len] = t0;
3587 str(t0, Address(Pm_base, Rj, Address::uxtw(LogBytesPerWord)));
3588
3589 // t0 = t1; t1 = t2; t2 = 0;
3590 adcs(t0, t1, Rhi_mn); // The pending m*n, high part
3591 adc(t1, t2, zr);
3592 mov(t2, zr);
3593 }
3594
3595 // A carry in t0 after Montgomery multiplication means that we
3596 // should subtract multiples of n from our result in m. We'll
3597 // keep doing that until there is no carry.
3598 void normalize(RegisterOrConstant len) {
3599 block_comment("normalize");
3600 // while (t0)
3601 // t0 = sub(Pm_base, Pn_base, t0, len);
3602 Label loop, post, again;
3603 Register cnt = t1, i = t2; // Re-use registers; we're done with them now
3604 cbz(t0, post); {
3605 bind(again); {
3606 mov(i, zr);
3607 mov(cnt, len);
3608 ldr(Rm, Address(Pm_base, i, Address::uxtw(LogBytesPerWord)));
3609 ldr(Rn, Address(Pn_base, i, Address::uxtw(LogBytesPerWord)));
3610 subs(zr, zr, zr); // set carry flag, i.e. no borrow
3611 align(16);
3612 bind(loop); {
3613 sbcs(Rm, Rm, Rn);
3614 str(Rm, Address(Pm_base, i, Address::uxtw(LogBytesPerWord)));
3615 add(i, i, 1);
3616 ldr(Rm, Address(Pm_base, i, Address::uxtw(LogBytesPerWord)));
3617 ldr(Rn, Address(Pn_base, i, Address::uxtw(LogBytesPerWord)));
3618 sub(cnt, cnt, 1);
3619 } cbnz(cnt, loop);
3620 sbc(t0, t0, zr);
3621 } cbnz(t0, again);
3622 } bind(post);
3623 }
3624
3625 // Move memory at s to d, reversing words.
3626 // Increments d to end of copied memory
3627 // Destroys tmp1, tmp2
3628 // Preserves len
3629 // Leaves s pointing to the address which was in d at start
3630 void reverse(Register d, Register s, Register len, Register tmp1, Register tmp2) {
3631 assert(tmp1 < r19 && tmp2 < r19, "register corruption");
3632
3633 lea(s, Address(s, len, Address::uxtw(LogBytesPerWord)));
3634 mov(tmp1, len);
3635 unroll_2(tmp1, &MontgomeryMultiplyGenerator::reverse1, d, s, tmp2);
3636 sub(s, d, len, ext::uxtw, LogBytesPerWord);
3637 }
3638 // where
3639 void reverse1(Register d, Register s, Register tmp) {
3640 ldr(tmp, pre(s, -wordSize));
3641 ror(tmp, tmp, 32);
3642 str(tmp, post(d, wordSize));
3643 }
3644
3645 void step_squaring() {
3646 // An extra ACC
3647 step();
3648 acc(Rhi_ab, Rlo_ab, t0, t1, t2);
3649 }
3650
3651 void last_squaring(RegisterOrConstant i) {
3652 Label dont;
3653 // if ((i & 1) == 0) {
3654 tbnz(i.as_register(), 0, dont); {
3655 // MACC(Ra, Rb, t0, t1, t2);
3656 // Ra = *++Pa;
3657 // Rb = *--Pb;
3658 umulh(Rhi_ab, Ra, Rb);
3659 mul(Rlo_ab, Ra, Rb);
3660 acc(Rhi_ab, Rlo_ab, t0, t1, t2);
3661 } bind(dont);
3662 }
3663
3664 void extra_step_squaring() {
3665 acc(Rhi_mn, Rlo_mn, t0, t1, t2); // The pending m*n
3666
3667 // MACC(Rm, Rn, t0, t1, t2);
3668 // Rm = *++Pm;
3669 // Rn = *--Pn;
3670 umulh(Rhi_mn, Rm, Rn);
3671 mul(Rlo_mn, Rm, Rn);
3672 ldr(Rm, pre(Pm, wordSize));
3673 ldr(Rn, pre(Pn, -wordSize));
3674 }
3675
3676 void post1_squaring() {
3677 acc(Rhi_mn, Rlo_mn, t0, t1, t2); // The pending m*n
3678
3679 // *Pm = Rm = t0 * inv;
3680 mul(Rm, t0, inv);
3681 str(Rm, Address(Pm));
3682
3683 // MACC(Rm, Rn, t0, t1, t2);
3684 // t0 = t1; t1 = t2; t2 = 0;
3685 umulh(Rhi_mn, Rm, Rn);
3686
3687 #ifndef PRODUCT
3688 // assert(m[i] * n[0] + t0 == 0, "broken Montgomery multiply");
3689 {
3690 mul(Rlo_mn, Rm, Rn);
3691 add(Rlo_mn, t0, Rlo_mn);
3692 Label ok;
3693 cbz(Rlo_mn, ok); {
3694 stop("broken Montgomery multiply");
3695 } bind(ok);
3696 }
3697 #endif
3698 // We have very carefully set things up so that
3699 // m[i]*n[0] + t0 == 0 (mod b), so we don't have to calculate
3700 // the lower half of Rm * Rn because we know the result already:
3701 // it must be -t0. t0 + (-t0) must generate a carry iff
3702 // t0 != 0. So, rather than do a mul and an adds we just set
3703 // the carry flag iff t0 is nonzero.
3704 //
3705 // mul(Rlo_mn, Rm, Rn);
3706 // adds(zr, t0, Rlo_mn);
3707 subs(zr, t0, 1); // Set carry iff t0 is nonzero
3708 adcs(t0, t1, Rhi_mn);
3709 adc(t1, t2, zr);
3710 mov(t2, zr);
3711 }
3712
3713 void acc(Register Rhi, Register Rlo,
3714 Register t0, Register t1, Register t2) {
3715 adds(t0, t0, Rlo);
3716 adcs(t1, t1, Rhi);
3717 adc(t2, t2, zr);
3718 }
3719
3720 public:
3721 /**
3722 * Fast Montgomery multiplication. The derivation of the
3723 * algorithm is in A Cryptographic Library for the Motorola
3724 * DSP56000, Dusse and Kaliski, Proc. EUROCRYPT 90, pp. 230-237.
3725 *
3726 * Arguments:
3727 *
3728 * Inputs for multiplication:
3729 * c_rarg0 - int array elements a
3730 * c_rarg1 - int array elements b
3731 * c_rarg2 - int array elements n (the modulus)
3732 * c_rarg3 - int length
3733 * c_rarg4 - int inv
3734 * c_rarg5 - int array elements m (the result)
3735 *
3736 * Inputs for squaring:
3737 * c_rarg0 - int array elements a
3738 * c_rarg1 - int array elements n (the modulus)
3739 * c_rarg2 - int length
3740 * c_rarg3 - int inv
3741 * c_rarg4 - int array elements m (the result)
3742 *
3743 */
3744 address generate_multiply() {
3745 Label argh, nothing;
3746 bind(argh);
3747 stop("MontgomeryMultiply total_allocation must be <= 8192");
3748
3749 align(CodeEntryAlignment);
3750 address entry = pc();
3751
3752 cbzw(Rlen, nothing);
3753
3754 enter();
3755
3756 // Make room.
3757 cmpw(Rlen, 512);
3758 br(Assembler::HI, argh);
3759 sub(Ra, sp, Rlen, ext::uxtw, exact_log2(4 * sizeof (jint)));
3760 andr(sp, Ra, -2 * wordSize);
3761
3762 lsrw(Rlen, Rlen, 1); // length in longwords = len/2
3763
3764 {
3765 // Copy input args, reversing as we go. We use Ra as a
3766 // temporary variable.
3767 reverse(Ra, Pa_base, Rlen, t0, t1);
3768 if (!_squaring)
3769 reverse(Ra, Pb_base, Rlen, t0, t1);
3770 reverse(Ra, Pn_base, Rlen, t0, t1);
3771 }
3772
3773 // Push all call-saved registers and also Pm_base which we'll need
3774 // at the end.
3775 save_regs();
3776
3777 #ifndef PRODUCT
3778 // assert(inv * n[0] == -1UL, "broken inverse in Montgomery multiply");
3779 {
3780 ldr(Rn, Address(Pn_base, 0));
3781 mul(Rlo_mn, Rn, inv);
3782 cmp(Rlo_mn, -1);
3783 Label ok;
3784 br(EQ, ok); {
3785 stop("broken inverse in Montgomery multiply");
3786 } bind(ok);
3787 }
3788 #endif
3789
3790 mov(Pm_base, Ra);
3791
3792 mov(t0, zr);
3793 mov(t1, zr);
3794 mov(t2, zr);
3795
3796 block_comment("for (int i = 0; i < len; i++) {");
3797 mov(Ri, zr); {
3798 Label loop, end;
3799 cmpw(Ri, Rlen);
3800 br(Assembler::GE, end);
3801
3802 bind(loop);
3803 pre1(Ri);
3804
3805 block_comment(" for (j = i; j; j--) {"); {
3806 movw(Rj, Ri);
3807 unroll_2(Rj, &MontgomeryMultiplyGenerator::step);
3808 } block_comment(" } // j");
3809
3810 post1();
3811 addw(Ri, Ri, 1);
3812 cmpw(Ri, Rlen);
3813 br(Assembler::LT, loop);
3814 bind(end);
3815 block_comment("} // i");
3816 }
3817
3818 block_comment("for (int i = len; i < 2*len; i++) {");
3819 mov(Ri, Rlen); {
3820 Label loop, end;
3821 cmpw(Ri, Rlen, Assembler::LSL, 1);
3822 br(Assembler::GE, end);
3823
3824 bind(loop);
3825 pre2(Ri, Rlen);
3826
3827 block_comment(" for (j = len*2-i-1; j; j--) {"); {
3828 lslw(Rj, Rlen, 1);
3829 subw(Rj, Rj, Ri);
3830 subw(Rj, Rj, 1);
3831 unroll_2(Rj, &MontgomeryMultiplyGenerator::step);
3832 } block_comment(" } // j");
3833
3834 post2(Ri, Rlen);
3835 addw(Ri, Ri, 1);
3836 cmpw(Ri, Rlen, Assembler::LSL, 1);
3837 br(Assembler::LT, loop);
3838 bind(end);
3839 }
3840 block_comment("} // i");
3841
3842 normalize(Rlen);
3843
3844 mov(Ra, Pm_base); // Save Pm_base in Ra
3845 restore_regs(); // Restore caller's Pm_base
3846
3847 // Copy our result into caller's Pm_base
3848 reverse(Pm_base, Ra, Rlen, t0, t1);
3849
3850 leave();
3851 bind(nothing);
3852 ret(lr);
3853
3854 return entry;
3855 }
3856 // In C, approximately:
3857
3858 // void
3859 // montgomery_multiply(unsigned long Pa_base[], unsigned long Pb_base[],
3860 // unsigned long Pn_base[], unsigned long Pm_base[],
3861 // unsigned long inv, int len) {
3862 // unsigned long t0 = 0, t1 = 0, t2 = 0; // Triple-precision accumulator
3863 // unsigned long *Pa, *Pb, *Pn, *Pm;
3864 // unsigned long Ra, Rb, Rn, Rm;
3865
3866 // int i;
3867
3868 // assert(inv * Pn_base[0] == -1UL, "broken inverse in Montgomery multiply");
3869
3870 // for (i = 0; i < len; i++) {
3871 // int j;
3872
3873 // Pa = Pa_base;
3874 // Pb = Pb_base + i;
3875 // Pm = Pm_base;
3876 // Pn = Pn_base + i;
3877
3878 // Ra = *Pa;
3879 // Rb = *Pb;
3880 // Rm = *Pm;
3881 // Rn = *Pn;
3882
3883 // int iters = i;
3884 // for (j = 0; iters--; j++) {
3885 // assert(Ra == Pa_base[j] && Rb == Pb_base[i-j], "must be");
3886 // MACC(Ra, Rb, t0, t1, t2);
3887 // Ra = *++Pa;
3888 // Rb = *--Pb;
3889 // assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
3890 // MACC(Rm, Rn, t0, t1, t2);
3891 // Rm = *++Pm;
3892 // Rn = *--Pn;
3893 // }
3894
3895 // assert(Ra == Pa_base[i] && Rb == Pb_base[0], "must be");
3896 // MACC(Ra, Rb, t0, t1, t2);
3897 // *Pm = Rm = t0 * inv;
3898 // assert(Rm == Pm_base[i] && Rn == Pn_base[0], "must be");
3899 // MACC(Rm, Rn, t0, t1, t2);
3900
3901 // assert(t0 == 0, "broken Montgomery multiply");
3902
3903 // t0 = t1; t1 = t2; t2 = 0;
3904 // }
3905
3906 // for (i = len; i < 2*len; i++) {
3907 // int j;
3908
3909 // Pa = Pa_base + i-len;
3910 // Pb = Pb_base + len;
3911 // Pm = Pm_base + i-len;
3912 // Pn = Pn_base + len;
3913
3914 // Ra = *++Pa;
3915 // Rb = *--Pb;
3916 // Rm = *++Pm;
3917 // Rn = *--Pn;
3918
3919 // int iters = len*2-i-1;
3920 // for (j = i-len+1; iters--; j++) {
3921 // assert(Ra == Pa_base[j] && Rb == Pb_base[i-j], "must be");
3922 // MACC(Ra, Rb, t0, t1, t2);
3923 // Ra = *++Pa;
3924 // Rb = *--Pb;
3925 // assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
3926 // MACC(Rm, Rn, t0, t1, t2);
3927 // Rm = *++Pm;
3928 // Rn = *--Pn;
3929 // }
3930
3931 // Pm_base[i-len] = t0;
3932 // t0 = t1; t1 = t2; t2 = 0;
3933 // }
3934
3935 // while (t0)
3936 // t0 = sub(Pm_base, Pn_base, t0, len);
3937 // }
3938
3939 /**
3940 * Fast Montgomery squaring. This uses asymptotically 25% fewer
3941 * multiplies than Montgomery multiplication so it should be up to
3942 * 25% faster. However, its loop control is more complex and it
3943 * may actually run slower on some machines.
3944 *
3945 * Arguments:
3946 *
3947 * Inputs:
3948 * c_rarg0 - int array elements a
3949 * c_rarg1 - int array elements n (the modulus)
3950 * c_rarg2 - int length
3951 * c_rarg3 - int inv
3952 * c_rarg4 - int array elements m (the result)
3953 *
3954 */
3955 address generate_square() {
3956 Label argh;
3957 bind(argh);
3958 stop("MontgomeryMultiply total_allocation must be <= 8192");
3959
3960 align(CodeEntryAlignment);
3961 address entry = pc();
3962
3963 enter();
3964
3965 // Make room.
3966 cmpw(Rlen, 512);
3967 br(Assembler::HI, argh);
3968 sub(Ra, sp, Rlen, ext::uxtw, exact_log2(4 * sizeof (jint)));
3969 andr(sp, Ra, -2 * wordSize);
3970
3971 lsrw(Rlen, Rlen, 1); // length in longwords = len/2
3972
3973 {
3974 // Copy input args, reversing as we go. We use Ra as a
3975 // temporary variable.
3976 reverse(Ra, Pa_base, Rlen, t0, t1);
3977 reverse(Ra, Pn_base, Rlen, t0, t1);
3978 }
3979
3980 // Push all call-saved registers and also Pm_base which we'll need
3981 // at the end.
3982 save_regs();
3983
3984 mov(Pm_base, Ra);
3985
3986 mov(t0, zr);
3987 mov(t1, zr);
3988 mov(t2, zr);
3989
3990 block_comment("for (int i = 0; i < len; i++) {");
3991 mov(Ri, zr); {
3992 Label loop, end;
3993 bind(loop);
3994 cmp(Ri, Rlen);
3995 br(Assembler::GE, end);
3996
3997 pre1(Ri);
3998
3999 block_comment("for (j = (i+1)/2; j; j--) {"); {
4000 add(Rj, Ri, 1);
4001 lsr(Rj, Rj, 1);
4002 unroll_2(Rj, &MontgomeryMultiplyGenerator::step_squaring);
4003 } block_comment(" } // j");
4004
4005 last_squaring(Ri);
4006
4007 block_comment(" for (j = i/2; j; j--) {"); {
4008 lsr(Rj, Ri, 1);
4009 unroll_2(Rj, &MontgomeryMultiplyGenerator::extra_step_squaring);
4010 } block_comment(" } // j");
4011
4012 post1_squaring();
4013 add(Ri, Ri, 1);
4014 cmp(Ri, Rlen);
4015 br(Assembler::LT, loop);
4016
4017 bind(end);
4018 block_comment("} // i");
4019 }
4020
4021 block_comment("for (int i = len; i < 2*len; i++) {");
4022 mov(Ri, Rlen); {
4023 Label loop, end;
4024 bind(loop);
4025 cmp(Ri, Rlen, Assembler::LSL, 1);
4026 br(Assembler::GE, end);
4027
4028 pre2(Ri, Rlen);
4029
4030 block_comment(" for (j = (2*len-i-1)/2; j; j--) {"); {
4031 lsl(Rj, Rlen, 1);
4032 sub(Rj, Rj, Ri);
4033 sub(Rj, Rj, 1);
4034 lsr(Rj, Rj, 1);
4035 unroll_2(Rj, &MontgomeryMultiplyGenerator::step_squaring);
4036 } block_comment(" } // j");
4037
4038 last_squaring(Ri);
4039
4040 block_comment(" for (j = (2*len-i)/2; j; j--) {"); {
4041 lsl(Rj, Rlen, 1);
4042 sub(Rj, Rj, Ri);
4043 lsr(Rj, Rj, 1);
4044 unroll_2(Rj, &MontgomeryMultiplyGenerator::extra_step_squaring);
4045 } block_comment(" } // j");
4046
4047 post2(Ri, Rlen);
4048 add(Ri, Ri, 1);
4049 cmp(Ri, Rlen, Assembler::LSL, 1);
4050
4051 br(Assembler::LT, loop);
4052 bind(end);
4053 block_comment("} // i");
4054 }
4055
4056 normalize(Rlen);
4057
4058 mov(Ra, Pm_base); // Save Pm_base in Ra
4059 restore_regs(); // Restore caller's Pm_base
4060
4061 // Copy our result into caller's Pm_base
4062 reverse(Pm_base, Ra, Rlen, t0, t1);
4063
4064 leave();
4065 ret(lr);
4066
4067 return entry;
4068 }
4069 // In C, approximately:
4070
4071 // void
4072 // montgomery_square(unsigned long Pa_base[], unsigned long Pn_base[],
4073 // unsigned long Pm_base[], unsigned long inv, int len) {
4074 // unsigned long t0 = 0, t1 = 0, t2 = 0; // Triple-precision accumulator
4075 // unsigned long *Pa, *Pb, *Pn, *Pm;
4076 // unsigned long Ra, Rb, Rn, Rm;
4077
4078 // int i;
4079
4080 // assert(inv * Pn_base[0] == -1UL, "broken inverse in Montgomery multiply");
4081
4082 // for (i = 0; i < len; i++) {
4083 // int j;
4084
4085 // Pa = Pa_base;
4086 // Pb = Pa_base + i;
4087 // Pm = Pm_base;
4088 // Pn = Pn_base + i;
4089
4090 // Ra = *Pa;
4091 // Rb = *Pb;
4092 // Rm = *Pm;
4093 // Rn = *Pn;
4094
4095 // int iters = (i+1)/2;
4096 // for (j = 0; iters--; j++) {
4097 // assert(Ra == Pa_base[j] && Rb == Pa_base[i-j], "must be");
4098 // MACC2(Ra, Rb, t0, t1, t2);
4099 // Ra = *++Pa;
4100 // Rb = *--Pb;
4101 // assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
4102 // MACC(Rm, Rn, t0, t1, t2);
4103 // Rm = *++Pm;
4104 // Rn = *--Pn;
4105 // }
4106 // if ((i & 1) == 0) {
4107 // assert(Ra == Pa_base[j], "must be");
4108 // MACC(Ra, Ra, t0, t1, t2);
4109 // }
4110 // iters = i/2;
4111 // assert(iters == i-j, "must be");
4112 // for (; iters--; j++) {
4113 // assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
4114 // MACC(Rm, Rn, t0, t1, t2);
4115 // Rm = *++Pm;
4116 // Rn = *--Pn;
4117 // }
4118
4119 // *Pm = Rm = t0 * inv;
4120 // assert(Rm == Pm_base[i] && Rn == Pn_base[0], "must be");
4121 // MACC(Rm, Rn, t0, t1, t2);
4122
4123 // assert(t0 == 0, "broken Montgomery multiply");
4124
4125 // t0 = t1; t1 = t2; t2 = 0;
4126 // }
4127
4128 // for (i = len; i < 2*len; i++) {
4129 // int start = i-len+1;
4130 // int end = start + (len - start)/2;
4131 // int j;
4132
4133 // Pa = Pa_base + i-len;
4134 // Pb = Pa_base + len;
4135 // Pm = Pm_base + i-len;
4136 // Pn = Pn_base + len;
4137
4138 // Ra = *++Pa;
4139 // Rb = *--Pb;
4140 // Rm = *++Pm;
4141 // Rn = *--Pn;
4142
4143 // int iters = (2*len-i-1)/2;
4144 // assert(iters == end-start, "must be");
4145 // for (j = start; iters--; j++) {
4146 // assert(Ra == Pa_base[j] && Rb == Pa_base[i-j], "must be");
4147 // MACC2(Ra, Rb, t0, t1, t2);
4148 // Ra = *++Pa;
4149 // Rb = *--Pb;
4150 // assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
4151 // MACC(Rm, Rn, t0, t1, t2);
4152 // Rm = *++Pm;
4153 // Rn = *--Pn;
4154 // }
4155 // if ((i & 1) == 0) {
4156 // assert(Ra == Pa_base[j], "must be");
4157 // MACC(Ra, Ra, t0, t1, t2);
4158 // }
4159 // iters = (2*len-i)/2;
4160 // assert(iters == len-j, "must be");
4161 // for (; iters--; j++) {
4162 // assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
4163 // MACC(Rm, Rn, t0, t1, t2);
4164 // Rm = *++Pm;
4165 // Rn = *--Pn;
4166 // }
4167 // Pm_base[i-len] = t0;
4168 // t0 = t1; t1 = t2; t2 = 0;
4169 // }
4170
4171 // while (t0)
4172 // t0 = sub(Pm_base, Pn_base, t0, len);
4173 // }
4174 };
4175
4176 // Initialization
4177 void generate_initial() {
4178 // Generate initial stubs and initializes the entry points
4179
4180 // entry points that exist in all platforms Note: This is code
4181 // that could be shared among different platforms - however the
4182 // benefit seems to be smaller than the disadvantage of having a
4183 // much more complicated generator structure. See also comment in
4184 // stubRoutines.hpp.
4185
4186 StubRoutines::_forward_exception_entry = generate_forward_exception();
4187
4188 StubRoutines::_call_stub_entry =
4189 generate_call_stub(StubRoutines::_call_stub_return_address);
4190
4191 // is referenced by megamorphic call
4192 StubRoutines::_catch_exception_entry = generate_catch_exception();
4193
4194 // Build this early so it's available for the interpreter.
4195 StubRoutines::_throw_StackOverflowError_entry =
4196 generate_throw_exception("StackOverflowError throw_exception",
4197 CAST_FROM_FN_PTR(address,
4198 SharedRuntime::
4199 throw_StackOverflowError));
4200 if (UseCRC32Intrinsics) {
4201 // set table address before stub generation which use it
4202 StubRoutines::_crc_table_adr = (address)StubRoutines::aarch64::_crc_table;
4203 StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
4204 }
4205 }
4206
4207 void generate_all() {
4208 // support for verify_oop (must happen after universe_init)
4209 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
4210 StubRoutines::_throw_AbstractMethodError_entry =
4211 generate_throw_exception("AbstractMethodError throw_exception",
4212 CAST_FROM_FN_PTR(address,
4213 SharedRuntime::
4214 throw_AbstractMethodError));
4215
4216 StubRoutines::_throw_IncompatibleClassChangeError_entry =
4217 generate_throw_exception("IncompatibleClassChangeError throw_exception",
4218 CAST_FROM_FN_PTR(address,
4219 SharedRuntime::
4220 throw_IncompatibleClassChangeError));
4221
4222 StubRoutines::_throw_NullPointerException_at_call_entry =
4223 generate_throw_exception("NullPointerException at call throw_exception",
4224 CAST_FROM_FN_PTR(address,
4225 SharedRuntime::
4226 throw_NullPointerException_at_call));
4227
4228 // arraycopy stubs used by compilers
4229 generate_arraycopy_stubs();
4230
4231 if (UseMultiplyToLenIntrinsic) {
4232 StubRoutines::_multiplyToLen = generate_multiplyToLen();
4233 }
4234
4235 if (UseMontgomeryMultiplyIntrinsic) {
4236 StubCodeMark mark(this, "StubRoutines", "montgomeryMultiply");
4237 MontgomeryMultiplyGenerator g(_masm, /*squaring*/false);
4238 StubRoutines::_montgomeryMultiply = g.generate_multiply();
4239 }
4240
4241 if (UseMontgomerySquareIntrinsic) {
4242 StubCodeMark mark(this, "StubRoutines", "montgomerySquare");
4243 MontgomeryMultiplyGenerator g(_masm, /*squaring*/true);
4244 // We use generate_multiply() rather than generate_square()
4245 // because it's faster for the sizes of modulus we care about.
4246 StubRoutines::_montgomerySquare = g.generate_multiply();
4247 }
4248
4249 if (UseAESIntrinsics) {
4250 StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
4251 StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
4252 StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
4253 StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt();
4254 }
4255
4256 if (UseSHA1Intrinsics) {
4257 StubRoutines::_sha1_implCompress = generate_sha1_implCompress(false, "sha1_implCompress");
4258 StubRoutines::_sha1_implCompressMB = generate_sha1_implCompress(true, "sha1_implCompressMB");
4259 }
4260 if (UseSHA256Intrinsics) {
4261 StubRoutines::_sha256_implCompress = generate_sha256_implCompress(false, "sha256_implCompress");
4262 StubRoutines::_sha256_implCompressMB = generate_sha256_implCompress(true, "sha256_implCompressMB");
4263 }
4264
4265 // Safefetch stubs.
4266 generate_safefetch("SafeFetch32", sizeof(int), &StubRoutines::_safefetch32_entry,
4267 &StubRoutines::_safefetch32_fault_pc,
4268 &StubRoutines::_safefetch32_continuation_pc);
4269 generate_safefetch("SafeFetchN", sizeof(intptr_t), &StubRoutines::_safefetchN_entry,
4270 &StubRoutines::_safefetchN_fault_pc,
4271 &StubRoutines::_safefetchN_continuation_pc);
4272 }
4273
4274 public:
4275 StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) {
4276 if (all) {
4277 generate_all();
4278 } else {
4279 generate_initial();
4280 }
4281 }
4282 }; // end class declaration
4283
4284 void StubGenerator_generate(CodeBuffer* code, bool all) {
4285 StubGenerator g(code, all);
4286 }