1 /*
2 * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "gc/shared/barrierSet.hpp"
28 #include "gc/shared/barrierSetAssembler.hpp"
29 #include "interpreter/interpreter.hpp"
30 #include "nativeInst_sparc.hpp"
31 #include "oops/instanceOop.hpp"
32 #include "oops/method.hpp"
33 #include "oops/objArrayKlass.hpp"
34 #include "oops/oop.inline.hpp"
35 #include "prims/methodHandles.hpp"
36 #include "runtime/frame.inline.hpp"
37 #include "runtime/handles.inline.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #include "runtime/stubCodeGenerator.hpp"
40 #include "runtime/stubRoutines.hpp"
41 #include "runtime/thread.inline.hpp"
42 #ifdef COMPILER2
43 #include "opto/runtime.hpp"
44 #endif
45
46 // Declaration and definition of StubGenerator (no .hpp file).
47 // For a more detailed description of the stub routine structure
48 // see the comment in stubRoutines.hpp.
49
50 #define __ _masm->
51
52 #ifdef PRODUCT
53 #define BLOCK_COMMENT(str) /* nothing */
54 #else
55 #define BLOCK_COMMENT(str) __ block_comment(str)
56 #endif
57
58 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
59
60 // Note: The register L7 is used as L7_thread_cache, and may not be used
61 // any other way within this module.
62
63 static const Register& Lstub_temp = L2;
64
65 // -------------------------------------------------------------------------------------------------------------------------
66 // Stub Code definitions
67
68 class StubGenerator: public StubCodeGenerator {
69 private:
70
71 #ifdef PRODUCT
72 #define inc_counter_np(a,b,c)
73 #else
74 #define inc_counter_np(counter, t1, t2) \
75 BLOCK_COMMENT("inc_counter " #counter); \
76 __ inc_counter(&counter, t1, t2);
77 #endif
78
79 //----------------------------------------------------------------------------------------------------
80 // Call stubs are used to call Java from C
81
82 address generate_call_stub(address& return_pc) {
83 StubCodeMark mark(this, "StubRoutines", "call_stub");
84 address start = __ pc();
85
86 // Incoming arguments:
87 //
88 // o0 : call wrapper address
89 // o1 : result (address)
90 // o2 : result type
91 // o3 : method
92 // o4 : (interpreter) entry point
93 // o5 : parameters (address)
94 // [sp + 0x5c]: parameter size (in words)
95 // [sp + 0x60]: thread
96 //
97 // +---------------+ <--- sp + 0
98 // | |
99 // . reg save area .
100 // | |
101 // +---------------+ <--- sp + 0x40
102 // | |
103 // . extra 7 slots .
104 // | |
105 // +---------------+ <--- sp + 0x5c
106 // | param. size |
107 // +---------------+ <--- sp + 0x60
108 // | thread |
109 // +---------------+
110 // | |
111
112 // note: if the link argument position changes, adjust
113 // the code in frame::entry_frame_call_wrapper()
114
115 const Argument link = Argument(0, false); // used only for GC
116 const Argument result = Argument(1, false);
117 const Argument result_type = Argument(2, false);
118 const Argument method = Argument(3, false);
119 const Argument entry_point = Argument(4, false);
120 const Argument parameters = Argument(5, false);
121 const Argument parameter_size = Argument(6, false);
122 const Argument thread = Argument(7, false);
123
124 // setup thread register
125 __ ld_ptr(thread.as_address(), G2_thread);
126 __ reinit_heapbase();
127
128 #ifdef ASSERT
129 // make sure we have no pending exceptions
130 { const Register t = G3_scratch;
131 Label L;
132 __ ld_ptr(G2_thread, in_bytes(Thread::pending_exception_offset()), t);
133 __ br_null_short(t, Assembler::pt, L);
134 __ stop("StubRoutines::call_stub: entered with pending exception");
135 __ bind(L);
136 }
137 #endif
138
139 // create activation frame & allocate space for parameters
140 { const Register t = G3_scratch;
141 __ ld_ptr(parameter_size.as_address(), t); // get parameter size (in words)
142 __ add(t, frame::memory_parameter_word_sp_offset, t); // add space for save area (in words)
143 __ round_to(t, WordsPerLong); // make sure it is multiple of 2 (in words)
144 __ sll(t, Interpreter::logStackElementSize, t); // compute number of bytes
145 __ neg(t); // negate so it can be used with save
146 __ save(SP, t, SP); // setup new frame
147 }
148
149 // +---------------+ <--- sp + 0
150 // | |
151 // . reg save area .
152 // | |
153 // +---------------+ <--- sp + 0x40
154 // | |
155 // . extra 7 slots .
156 // | |
157 // +---------------+ <--- sp + 0x5c
158 // | empty slot | (only if parameter size is even)
159 // +---------------+
160 // | |
161 // . parameters .
162 // | |
163 // +---------------+ <--- fp + 0
164 // | |
165 // . reg save area .
166 // | |
167 // +---------------+ <--- fp + 0x40
168 // | |
169 // . extra 7 slots .
170 // | |
171 // +---------------+ <--- fp + 0x5c
172 // | param. size |
173 // +---------------+ <--- fp + 0x60
174 // | thread |
175 // +---------------+
176 // | |
177
178 // pass parameters if any
179 BLOCK_COMMENT("pass parameters if any");
180 { const Register src = parameters.as_in().as_register();
181 const Register dst = Lentry_args;
182 const Register tmp = G3_scratch;
183 const Register cnt = G4_scratch;
184
185 // test if any parameters & setup of Lentry_args
186 Label exit;
187 __ ld_ptr(parameter_size.as_in().as_address(), cnt); // parameter counter
188 __ add( FP, STACK_BIAS, dst );
189 __ cmp_zero_and_br(Assembler::zero, cnt, exit);
190 __ delayed()->sub(dst, BytesPerWord, dst); // setup Lentry_args
191
192 // copy parameters if any
193 Label loop;
194 __ BIND(loop);
195 // Store parameter value
196 __ ld_ptr(src, 0, tmp);
197 __ add(src, BytesPerWord, src);
198 __ st_ptr(tmp, dst, 0);
199 __ deccc(cnt);
200 __ br(Assembler::greater, false, Assembler::pt, loop);
201 __ delayed()->sub(dst, Interpreter::stackElementSize, dst);
202
203 // done
204 __ BIND(exit);
205 }
206
207 // setup parameters, method & call Java function
208 #ifdef ASSERT
209 // layout_activation_impl checks it's notion of saved SP against
210 // this register, so if this changes update it as well.
211 const Register saved_SP = Lscratch;
212 __ mov(SP, saved_SP); // keep track of SP before call
213 #endif
214
215 // setup parameters
216 const Register t = G3_scratch;
217 __ ld_ptr(parameter_size.as_in().as_address(), t); // get parameter size (in words)
218 __ sll(t, Interpreter::logStackElementSize, t); // compute number of bytes
219 __ sub(FP, t, Gargs); // setup parameter pointer
220 __ add( Gargs, STACK_BIAS, Gargs ); // Account for LP64 stack bias
221 __ mov(SP, O5_savedSP);
222
223
224 // do the call
225 //
226 // the following register must be setup:
227 //
228 // G2_thread
229 // G5_method
230 // Gargs
231 BLOCK_COMMENT("call Java function");
232 __ jmpl(entry_point.as_in().as_register(), G0, O7);
233 __ delayed()->mov(method.as_in().as_register(), G5_method); // setup method
234
235 BLOCK_COMMENT("call_stub_return_address:");
236 return_pc = __ pc();
237
238 // The callee, if it wasn't interpreted, can return with SP changed so
239 // we can no longer assert of change of SP.
240
241 // store result depending on type
242 // (everything that is not T_OBJECT, T_LONG, T_FLOAT, or T_DOUBLE
243 // is treated as T_INT)
244 { const Register addr = result .as_in().as_register();
245 const Register type = result_type.as_in().as_register();
246 Label is_long, is_float, is_double, is_object, exit;
247 __ cmp(type, T_OBJECT); __ br(Assembler::equal, false, Assembler::pn, is_object);
248 __ delayed()->cmp(type, T_FLOAT); __ br(Assembler::equal, false, Assembler::pn, is_float);
249 __ delayed()->cmp(type, T_DOUBLE); __ br(Assembler::equal, false, Assembler::pn, is_double);
250 __ delayed()->cmp(type, T_LONG); __ br(Assembler::equal, false, Assembler::pn, is_long);
251 __ delayed()->nop();
252
253 // store int result
254 __ st(O0, addr, G0);
255
256 __ BIND(exit);
257 __ ret();
258 __ delayed()->restore();
259
260 __ BIND(is_object);
261 __ ba(exit);
262 __ delayed()->st_ptr(O0, addr, G0);
263
264 __ BIND(is_float);
265 __ ba(exit);
266 __ delayed()->stf(FloatRegisterImpl::S, F0, addr, G0);
267
268 __ BIND(is_double);
269 __ ba(exit);
270 __ delayed()->stf(FloatRegisterImpl::D, F0, addr, G0);
271
272 __ BIND(is_long);
273 __ ba(exit);
274 __ delayed()->st_long(O0, addr, G0); // store entire long
275 }
276 return start;
277 }
278
279
280 //----------------------------------------------------------------------------------------------------
281 // Return point for a Java call if there's an exception thrown in Java code.
282 // The exception is caught and transformed into a pending exception stored in
283 // JavaThread that can be tested from within the VM.
284 //
285 // Oexception: exception oop
286
287 address generate_catch_exception() {
288 StubCodeMark mark(this, "StubRoutines", "catch_exception");
289
290 address start = __ pc();
291 // verify that thread corresponds
292 __ verify_thread();
293
294 const Register& temp_reg = Gtemp;
295 Address pending_exception_addr (G2_thread, Thread::pending_exception_offset());
296 Address exception_file_offset_addr(G2_thread, Thread::exception_file_offset ());
297 Address exception_line_offset_addr(G2_thread, Thread::exception_line_offset ());
298
299 // set pending exception
300 __ verify_oop(Oexception);
301 __ st_ptr(Oexception, pending_exception_addr);
302 __ set((intptr_t)__FILE__, temp_reg);
303 __ st_ptr(temp_reg, exception_file_offset_addr);
304 __ set((intptr_t)__LINE__, temp_reg);
305 __ st(temp_reg, exception_line_offset_addr);
306
307 // complete return to VM
308 assert(StubRoutines::_call_stub_return_address != NULL, "must have been generated before");
309
310 AddressLiteral stub_ret(StubRoutines::_call_stub_return_address);
311 __ jump_to(stub_ret, temp_reg);
312 __ delayed()->nop();
313
314 return start;
315 }
316
317
318 //----------------------------------------------------------------------------------------------------
319 // Continuation point for runtime calls returning with a pending exception
320 // The pending exception check happened in the runtime or native call stub
321 // The pending exception in Thread is converted into a Java-level exception
322 //
323 // Contract with Java-level exception handler: O0 = exception
324 // O1 = throwing pc
325
326 address generate_forward_exception() {
327 StubCodeMark mark(this, "StubRoutines", "forward_exception");
328 address start = __ pc();
329
330 // Upon entry, O7 has the return address returning into Java
331 // (interpreted or compiled) code; i.e. the return address
332 // becomes the throwing pc.
333
334 const Register& handler_reg = Gtemp;
335
336 Address exception_addr(G2_thread, Thread::pending_exception_offset());
337
338 #ifdef ASSERT
339 // make sure that this code is only executed if there is a pending exception
340 { Label L;
341 __ ld_ptr(exception_addr, Gtemp);
342 __ br_notnull_short(Gtemp, Assembler::pt, L);
343 __ stop("StubRoutines::forward exception: no pending exception (1)");
344 __ bind(L);
345 }
346 #endif
347
348 // compute exception handler into handler_reg
349 __ get_thread();
350 __ ld_ptr(exception_addr, Oexception);
351 __ verify_oop(Oexception);
352 __ save_frame(0); // compensates for compiler weakness
353 __ add(O7->after_save(), frame::pc_return_offset, Lscratch); // save the issuing PC
354 BLOCK_COMMENT("call exception_handler_for_return_address");
355 __ call_VM_leaf(L7_thread_cache, CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), G2_thread, Lscratch);
356 __ mov(O0, handler_reg);
357 __ restore(); // compensates for compiler weakness
358
359 __ ld_ptr(exception_addr, Oexception);
360 __ add(O7, frame::pc_return_offset, Oissuing_pc); // save the issuing PC
361
362 #ifdef ASSERT
363 // make sure exception is set
364 { Label L;
365 __ br_notnull_short(Oexception, Assembler::pt, L);
366 __ stop("StubRoutines::forward exception: no pending exception (2)");
367 __ bind(L);
368 }
369 #endif
370 // jump to exception handler
371 __ jmp(handler_reg, 0);
372 // clear pending exception
373 __ delayed()->st_ptr(G0, exception_addr);
374
375 return start;
376 }
377
378 // Safefetch stubs.
379 void generate_safefetch(const char* name, int size, address* entry,
380 address* fault_pc, address* continuation_pc) {
381 // safefetch signatures:
382 // int SafeFetch32(int* adr, int errValue);
383 // intptr_t SafeFetchN (intptr_t* adr, intptr_t errValue);
384 //
385 // arguments:
386 // o0 = adr
387 // o1 = errValue
388 //
389 // result:
390 // o0 = *adr or errValue
391
392 StubCodeMark mark(this, "StubRoutines", name);
393
394 // Entry point, pc or function descriptor.
395 __ align(CodeEntryAlignment);
396 *entry = __ pc();
397
398 __ mov(O0, G1); // g1 = o0
399 __ mov(O1, O0); // o0 = o1
400 // Load *adr into c_rarg1, may fault.
401 *fault_pc = __ pc();
402 switch (size) {
403 case 4:
404 // int32_t
405 __ ldsw(G1, 0, O0); // o0 = [g1]
406 break;
407 case 8:
408 // int64_t
409 __ ldx(G1, 0, O0); // o0 = [g1]
410 break;
411 default:
412 ShouldNotReachHere();
413 }
414
415 // return errValue or *adr
416 *continuation_pc = __ pc();
417 // By convention with the trap handler we ensure there is a non-CTI
418 // instruction in the trap shadow.
419 __ nop();
420 __ retl();
421 __ delayed()->nop();
422 }
423
424 //------------------------------------------------------------------------------------------------------------------------
425 // Continuation point for throwing of implicit exceptions that are not handled in
426 // the current activation. Fabricates an exception oop and initiates normal
427 // exception dispatching in this frame. Only callee-saved registers are preserved
428 // (through the normal register window / RegisterMap handling).
429 // If the compiler needs all registers to be preserved between the fault
430 // point and the exception handler then it must assume responsibility for that in
431 // AbstractCompiler::continuation_for_implicit_null_exception or
432 // continuation_for_implicit_division_by_zero_exception. All other implicit
433 // exceptions (e.g., NullPointerException or AbstractMethodError on entry) are
434 // either at call sites or otherwise assume that stack unwinding will be initiated,
435 // so caller saved registers were assumed volatile in the compiler.
436
437 // Note that we generate only this stub into a RuntimeStub, because it needs to be
438 // properly traversed and ignored during GC, so we change the meaning of the "__"
439 // macro within this method.
440 #undef __
441 #define __ masm->
442
443 address generate_throw_exception(const char* name, address runtime_entry,
444 Register arg1 = noreg, Register arg2 = noreg) {
445 #ifdef ASSERT
446 int insts_size = VerifyThread ? 1 * K : 600;
447 #else
448 int insts_size = VerifyThread ? 1 * K : 256;
449 #endif /* ASSERT */
450 int locs_size = 32;
451
452 CodeBuffer code(name, insts_size, locs_size);
453 MacroAssembler* masm = new MacroAssembler(&code);
454
455 __ verify_thread();
456
457 // This is an inlined and slightly modified version of call_VM
458 // which has the ability to fetch the return PC out of thread-local storage
459 __ assert_not_delayed();
460
461 // Note that we always push a frame because on the SPARC
462 // architecture, for all of our implicit exception kinds at call
463 // sites, the implicit exception is taken before the callee frame
464 // is pushed.
465 __ save_frame(0);
466
467 int frame_complete = __ offset();
468
469 // Note that we always have a runtime stub frame on the top of stack by this point
470 Register last_java_sp = SP;
471 // 64-bit last_java_sp is biased!
472 __ set_last_Java_frame(last_java_sp, G0);
473 if (VerifyThread) __ mov(G2_thread, O0); // about to be smashed; pass early
474 __ save_thread(noreg);
475 if (arg1 != noreg) {
476 assert(arg2 != O1, "clobbered");
477 __ mov(arg1, O1);
478 }
479 if (arg2 != noreg) {
480 __ mov(arg2, O2);
481 }
482 // do the call
483 BLOCK_COMMENT("call runtime_entry");
484 __ call(runtime_entry, relocInfo::runtime_call_type);
485 if (!VerifyThread)
486 __ delayed()->mov(G2_thread, O0); // pass thread as first argument
487 else
488 __ delayed()->nop(); // (thread already passed)
489 __ restore_thread(noreg);
490 __ reset_last_Java_frame();
491
492 // check for pending exceptions. use Gtemp as scratch register.
493 #ifdef ASSERT
494 Label L;
495
496 Address exception_addr(G2_thread, Thread::pending_exception_offset());
497 Register scratch_reg = Gtemp;
498 __ ld_ptr(exception_addr, scratch_reg);
499 __ br_notnull_short(scratch_reg, Assembler::pt, L);
500 __ should_not_reach_here();
501 __ bind(L);
502 #endif // ASSERT
503 BLOCK_COMMENT("call forward_exception_entry");
504 __ call(StubRoutines::forward_exception_entry(), relocInfo::runtime_call_type);
505 // we use O7 linkage so that forward_exception_entry has the issuing PC
506 __ delayed()->restore();
507
508 RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &code, frame_complete, masm->total_frame_size_in_bytes(0), NULL, false);
509 return stub->entry_point();
510 }
511
512 #undef __
513 #define __ _masm->
514
515
516 // Generate a routine that sets all the registers so we
517 // can tell if the stop routine prints them correctly.
518 address generate_test_stop() {
519 StubCodeMark mark(this, "StubRoutines", "test_stop");
520 address start = __ pc();
521
522 int i;
523
524 __ save_frame(0);
525
526 static jfloat zero = 0.0, one = 1.0;
527
528 // put addr in L0, then load through L0 to F0
529 __ set((intptr_t)&zero, L0); __ ldf( FloatRegisterImpl::S, L0, 0, F0);
530 __ set((intptr_t)&one, L0); __ ldf( FloatRegisterImpl::S, L0, 0, F1); // 1.0 to F1
531
532 // use add to put 2..18 in F2..F18
533 for ( i = 2; i <= 18; ++i ) {
534 __ fadd( FloatRegisterImpl::S, F1, as_FloatRegister(i-1), as_FloatRegister(i));
535 }
536
537 // Now put double 2 in F16, double 18 in F18
538 __ ftof( FloatRegisterImpl::S, FloatRegisterImpl::D, F2, F16 );
539 __ ftof( FloatRegisterImpl::S, FloatRegisterImpl::D, F18, F18 );
540
541 // use add to put 20..32 in F20..F32
542 for (i = 20; i < 32; i += 2) {
543 __ fadd( FloatRegisterImpl::D, F16, as_FloatRegister(i-2), as_FloatRegister(i));
544 }
545
546 // put 0..7 in i's, 8..15 in l's, 16..23 in o's, 24..31 in g's
547 for ( i = 0; i < 8; ++i ) {
548 if (i < 6) {
549 __ set( i, as_iRegister(i));
550 __ set(16 + i, as_oRegister(i));
551 __ set(24 + i, as_gRegister(i));
552 }
553 __ set( 8 + i, as_lRegister(i));
554 }
555
556 __ stop("testing stop");
557
558
559 __ ret();
560 __ delayed()->restore();
561
562 return start;
563 }
564
565
566 address generate_stop_subroutine() {
567 StubCodeMark mark(this, "StubRoutines", "stop_subroutine");
568 address start = __ pc();
569
570 __ stop_subroutine();
571
572 return start;
573 }
574
575 address generate_flush_callers_register_windows() {
576 StubCodeMark mark(this, "StubRoutines", "flush_callers_register_windows");
577 address start = __ pc();
578
579 __ flushw();
580 __ retl(false);
581 __ delayed()->add( FP, STACK_BIAS, O0 );
582 // The returned value must be a stack pointer whose register save area
583 // is flushed, and will stay flushed while the caller executes.
584
585 return start;
586 }
587
588 // Support for jint Atomic::xchg(volatile jint* dest, jint exchange_value).
589 //
590 // Arguments:
591 //
592 // exchange_value: O0
593 // dest: O1
594 //
595 // Results:
596 //
597 // O0: the value previously stored in dest
598 //
599 address generate_atomic_xchg() {
600 StubCodeMark mark(this, "StubRoutines", "atomic_xchg");
601 address start = __ pc();
602
603 if (UseCASForSwap) {
604 // Use CAS instead of swap, just in case the MP hardware
605 // prefers to work with just one kind of synch. instruction.
606 Label retry;
607 __ BIND(retry);
608 __ mov(O0, O3); // scratch copy of exchange value
609 __ ld(O1, 0, O2); // observe the previous value
610 // try to replace O2 with O3
611 __ cas(O1, O2, O3);
612 __ cmp_and_br_short(O2, O3, Assembler::notEqual, Assembler::pn, retry);
613
614 __ retl(false);
615 __ delayed()->mov(O2, O0); // report previous value to caller
616 } else {
617 __ retl(false);
618 __ delayed()->swap(O1, 0, O0);
619 }
620
621 return start;
622 }
623
624
625 // Support for jint Atomic::cmpxchg(volatile jint* dest, jint compare_value, jint exchange_value)
626 //
627 // Arguments:
628 //
629 // exchange_value: O0
630 // dest: O1
631 // compare_value: O2
632 //
633 // Results:
634 //
635 // O0: the value previously stored in dest
636 //
637 address generate_atomic_cmpxchg() {
638 StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg");
639 address start = __ pc();
640
641 // cmpxchg(dest, compare_value, exchange_value)
642 __ cas(O1, O2, O0);
643 __ retl(false);
644 __ delayed()->nop();
645
646 return start;
647 }
648
649 // Support for jlong Atomic::cmpxchg(volatile jlong *dest, jlong compare_value, jlong exchange_value)
650 //
651 // Arguments:
652 //
653 // exchange_value: O1:O0
654 // dest: O2
655 // compare_value: O4:O3
656 //
657 // Results:
658 //
659 // O1:O0: the value previously stored in dest
660 //
661 // Overwrites: G1,G2,G3
662 //
663 address generate_atomic_cmpxchg_long() {
664 StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg_long");
665 address start = __ pc();
666
667 __ sllx(O0, 32, O0);
668 __ srl(O1, 0, O1);
669 __ or3(O0,O1,O0); // O0 holds 64-bit value from compare_value
670 __ sllx(O3, 32, O3);
671 __ srl(O4, 0, O4);
672 __ or3(O3,O4,O3); // O3 holds 64-bit value from exchange_value
673 __ casx(O2, O3, O0);
674 __ srl(O0, 0, O1); // unpacked return value in O1:O0
675 __ retl(false);
676 __ delayed()->srlx(O0, 32, O0);
677
678 return start;
679 }
680
681
682 // Support for jint Atomic::add(volatile jint* dest, jint add_value).
683 //
684 // Arguments:
685 //
686 // add_value: O0 (e.g., +1 or -1)
687 // dest: O1
688 //
689 // Results:
690 //
691 // O0: the new value stored in dest
692 //
693 // Overwrites: O3
694 //
695 address generate_atomic_add() {
696 StubCodeMark mark(this, "StubRoutines", "atomic_add");
697 address start = __ pc();
698 __ BIND(_atomic_add_stub);
699
700 Label(retry);
701 __ BIND(retry);
702
703 __ lduw(O1, 0, O2);
704 __ add(O0, O2, O3);
705 __ cas(O1, O2, O3);
706 __ cmp_and_br_short(O2, O3, Assembler::notEqual, Assembler::pn, retry);
707 __ retl(false);
708 __ delayed()->add(O0, O2, O0); // note that cas made O2==O3
709
710 return start;
711 }
712 Label _atomic_add_stub; // called from other stubs
713
714
715 // Support for uint StubRoutine::Sparc::partial_subtype_check( Klass sub, Klass super );
716 // Arguments :
717 //
718 // ret : O0, returned
719 // icc/xcc: set as O0 (depending on wordSize)
720 // sub : O1, argument, not changed
721 // super: O2, argument, not changed
722 // raddr: O7, blown by call
723 address generate_partial_subtype_check() {
724 __ align(CodeEntryAlignment);
725 StubCodeMark mark(this, "StubRoutines", "partial_subtype_check");
726 address start = __ pc();
727 Label miss;
728
729 __ save_frame(0);
730 Register Rret = I0;
731 Register Rsub = I1;
732 Register Rsuper = I2;
733
734 Register L0_ary_len = L0;
735 Register L1_ary_ptr = L1;
736 Register L2_super = L2;
737 Register L3_index = L3;
738
739 __ check_klass_subtype_slow_path(Rsub, Rsuper,
740 L0, L1, L2, L3,
741 NULL, &miss);
742
743 // Match falls through here.
744 __ addcc(G0,0,Rret); // set Z flags, Z result
745
746 __ ret(); // Result in Rret is zero; flags set to Z
747 __ delayed()->restore();
748
749 __ BIND(miss);
750 __ addcc(G0,1,Rret); // set NZ flags, NZ result
751
752 __ ret(); // Result in Rret is != 0; flags set to NZ
753 __ delayed()->restore();
754
755 return start;
756 }
757
758
759 // Called from MacroAssembler::verify_oop
760 //
761 address generate_verify_oop_subroutine() {
762 StubCodeMark mark(this, "StubRoutines", "verify_oop_stub");
763
764 address start = __ pc();
765
766 __ verify_oop_subroutine();
767
768 return start;
769 }
770
771
772 //
773 // Verify that a register contains clean 32-bits positive value
774 // (high 32-bits are 0) so it could be used in 64-bits shifts (sllx, srax).
775 //
776 // Input:
777 // Rint - 32-bits value
778 // Rtmp - scratch
779 //
780 void assert_clean_int(Register Rint, Register Rtmp) {
781 #if defined(ASSERT)
782 __ signx(Rint, Rtmp);
783 __ cmp(Rint, Rtmp);
784 __ breakpoint_trap(Assembler::notEqual, Assembler::xcc);
785 #endif
786 }
787
788 //
789 // Generate overlap test for array copy stubs
790 //
791 // Input:
792 // O0 - array1
793 // O1 - array2
794 // O2 - element count
795 //
796 // Kills temps: O3, O4
797 //
798 void array_overlap_test(address no_overlap_target, int log2_elem_size) {
799 assert(no_overlap_target != NULL, "must be generated");
800 array_overlap_test(no_overlap_target, NULL, log2_elem_size);
801 }
802 void array_overlap_test(Label& L_no_overlap, int log2_elem_size) {
803 array_overlap_test(NULL, &L_no_overlap, log2_elem_size);
804 }
805 void array_overlap_test(address no_overlap_target, Label* NOLp, int log2_elem_size) {
806 const Register from = O0;
807 const Register to = O1;
808 const Register count = O2;
809 const Register to_from = O3; // to - from
810 const Register byte_count = O4; // count << log2_elem_size
811
812 __ subcc(to, from, to_from);
813 __ sll_ptr(count, log2_elem_size, byte_count);
814 if (NOLp == NULL)
815 __ brx(Assembler::lessEqualUnsigned, false, Assembler::pt, no_overlap_target);
816 else
817 __ brx(Assembler::lessEqualUnsigned, false, Assembler::pt, (*NOLp));
818 __ delayed()->cmp(to_from, byte_count);
819 if (NOLp == NULL)
820 __ brx(Assembler::greaterEqualUnsigned, false, Assembler::pt, no_overlap_target);
821 else
822 __ brx(Assembler::greaterEqualUnsigned, false, Assembler::pt, (*NOLp));
823 __ delayed()->nop();
824 }
825
826
827 //
828 // Generate main code for disjoint arraycopy
829 //
830 typedef void (StubGenerator::*CopyLoopFunc)(Register from, Register to, Register count, int count_dec,
831 Label& L_loop, bool use_prefetch, bool use_bis);
832
833 void disjoint_copy_core(Register from, Register to, Register count, int log2_elem_size,
834 int iter_size, StubGenerator::CopyLoopFunc copy_loop_func) {
835 Label L_copy;
836
837 assert(log2_elem_size <= 3, "the following code should be changed");
838 int count_dec = 16>>log2_elem_size;
839
840 int prefetch_dist = MAX2(ArraycopySrcPrefetchDistance, ArraycopyDstPrefetchDistance);
841 assert(prefetch_dist < 4096, "invalid value");
842 prefetch_dist = (prefetch_dist + (iter_size-1)) & (-iter_size); // round up to one iteration copy size
843 int prefetch_count = (prefetch_dist >> log2_elem_size); // elements count
844
845 if (UseBlockCopy) {
846 Label L_block_copy, L_block_copy_prefetch, L_skip_block_copy;
847
848 // 64 bytes tail + bytes copied in one loop iteration
849 int tail_size = 64 + iter_size;
850 int block_copy_count = (MAX2(tail_size, (int)BlockCopyLowLimit)) >> log2_elem_size;
851 // Use BIS copy only for big arrays since it requires membar.
852 __ set(block_copy_count, O4);
853 __ cmp_and_br_short(count, O4, Assembler::lessUnsigned, Assembler::pt, L_skip_block_copy);
854 // This code is for disjoint source and destination:
855 // to <= from || to >= from+count
856 // but BIS will stomp over 'from' if (to > from-tail_size && to <= from)
857 __ sub(from, to, O4);
858 __ srax(O4, 4, O4); // divide by 16 since following short branch have only 5 bits for imm.
859 __ cmp_and_br_short(O4, (tail_size>>4), Assembler::lessEqualUnsigned, Assembler::pn, L_skip_block_copy);
860
861 __ wrasi(G0, Assembler::ASI_ST_BLKINIT_PRIMARY);
862 // BIS should not be used to copy tail (64 bytes+iter_size)
863 // to avoid zeroing of following values.
864 __ sub(count, (tail_size>>log2_elem_size), count); // count is still positive >= 0
865
866 if (prefetch_count > 0) { // rounded up to one iteration count
867 // Do prefetching only if copy size is bigger
868 // than prefetch distance.
869 __ set(prefetch_count, O4);
870 __ cmp_and_brx_short(count, O4, Assembler::less, Assembler::pt, L_block_copy);
871 __ sub(count, O4, count);
872
873 (this->*copy_loop_func)(from, to, count, count_dec, L_block_copy_prefetch, true, true);
874 __ set(prefetch_count, O4);
875 __ add(count, O4, count);
876
877 } // prefetch_count > 0
878
879 (this->*copy_loop_func)(from, to, count, count_dec, L_block_copy, false, true);
880 __ add(count, (tail_size>>log2_elem_size), count); // restore count
881
882 __ wrasi(G0, Assembler::ASI_PRIMARY_NOFAULT);
883 // BIS needs membar.
884 __ membar(Assembler::StoreLoad);
885 // Copy tail
886 __ ba_short(L_copy);
887
888 __ BIND(L_skip_block_copy);
889 } // UseBlockCopy
890
891 if (prefetch_count > 0) { // rounded up to one iteration count
892 // Do prefetching only if copy size is bigger
893 // than prefetch distance.
894 __ set(prefetch_count, O4);
895 __ cmp_and_brx_short(count, O4, Assembler::lessUnsigned, Assembler::pt, L_copy);
896 __ sub(count, O4, count);
897
898 Label L_copy_prefetch;
899 (this->*copy_loop_func)(from, to, count, count_dec, L_copy_prefetch, true, false);
900 __ set(prefetch_count, O4);
901 __ add(count, O4, count);
902
903 } // prefetch_count > 0
904
905 (this->*copy_loop_func)(from, to, count, count_dec, L_copy, false, false);
906 }
907
908
909
910 //
911 // Helper methods for copy_16_bytes_forward_with_shift()
912 //
913 void copy_16_bytes_shift_loop(Register from, Register to, Register count, int count_dec,
914 Label& L_loop, bool use_prefetch, bool use_bis) {
915
916 const Register left_shift = G1; // left shift bit counter
917 const Register right_shift = G5; // right shift bit counter
918
919 __ align(OptoLoopAlignment);
920 __ BIND(L_loop);
921 if (use_prefetch) {
922 if (ArraycopySrcPrefetchDistance > 0) {
923 __ prefetch(from, ArraycopySrcPrefetchDistance, Assembler::severalReads);
924 }
925 if (ArraycopyDstPrefetchDistance > 0) {
926 __ prefetch(to, ArraycopyDstPrefetchDistance, Assembler::severalWritesAndPossiblyReads);
927 }
928 }
929 __ ldx(from, 0, O4);
930 __ ldx(from, 8, G4);
931 __ inc(to, 16);
932 __ inc(from, 16);
933 __ deccc(count, count_dec); // Can we do next iteration after this one?
934 __ srlx(O4, right_shift, G3);
935 __ bset(G3, O3);
936 __ sllx(O4, left_shift, O4);
937 __ srlx(G4, right_shift, G3);
938 __ bset(G3, O4);
939 if (use_bis) {
940 __ stxa(O3, to, -16);
941 __ stxa(O4, to, -8);
942 } else {
943 __ stx(O3, to, -16);
944 __ stx(O4, to, -8);
945 }
946 __ brx(Assembler::greaterEqual, false, Assembler::pt, L_loop);
947 __ delayed()->sllx(G4, left_shift, O3);
948 }
949
950 // Copy big chunks forward with shift
951 //
952 // Inputs:
953 // from - source arrays
954 // to - destination array aligned to 8-bytes
955 // count - elements count to copy >= the count equivalent to 16 bytes
956 // count_dec - elements count's decrement equivalent to 16 bytes
957 // L_copy_bytes - copy exit label
958 //
959 void copy_16_bytes_forward_with_shift(Register from, Register to,
960 Register count, int log2_elem_size, Label& L_copy_bytes) {
961 Label L_aligned_copy, L_copy_last_bytes;
962 assert(log2_elem_size <= 3, "the following code should be changed");
963 int count_dec = 16>>log2_elem_size;
964
965 // if both arrays have the same alignment mod 8, do 8 bytes aligned copy
966 __ andcc(from, 7, G1); // misaligned bytes
967 __ br(Assembler::zero, false, Assembler::pt, L_aligned_copy);
968 __ delayed()->nop();
969
970 const Register left_shift = G1; // left shift bit counter
971 const Register right_shift = G5; // right shift bit counter
972
973 __ sll(G1, LogBitsPerByte, left_shift);
974 __ mov(64, right_shift);
975 __ sub(right_shift, left_shift, right_shift);
976
977 //
978 // Load 2 aligned 8-bytes chunks and use one from previous iteration
979 // to form 2 aligned 8-bytes chunks to store.
980 //
981 __ dec(count, count_dec); // Pre-decrement 'count'
982 __ andn(from, 7, from); // Align address
983 __ ldx(from, 0, O3);
984 __ inc(from, 8);
985 __ sllx(O3, left_shift, O3);
986
987 disjoint_copy_core(from, to, count, log2_elem_size, 16, &StubGenerator::copy_16_bytes_shift_loop);
988
989 __ inccc(count, count_dec>>1 ); // + 8 bytes
990 __ brx(Assembler::negative, true, Assembler::pn, L_copy_last_bytes);
991 __ delayed()->inc(count, count_dec>>1); // restore 'count'
992
993 // copy 8 bytes, part of them already loaded in O3
994 __ ldx(from, 0, O4);
995 __ inc(to, 8);
996 __ inc(from, 8);
997 __ srlx(O4, right_shift, G3);
998 __ bset(O3, G3);
999 __ stx(G3, to, -8);
1000
1001 __ BIND(L_copy_last_bytes);
1002 __ srl(right_shift, LogBitsPerByte, right_shift); // misaligned bytes
1003 __ br(Assembler::always, false, Assembler::pt, L_copy_bytes);
1004 __ delayed()->sub(from, right_shift, from); // restore address
1005
1006 __ BIND(L_aligned_copy);
1007 }
1008
1009 // Copy big chunks backward with shift
1010 //
1011 // Inputs:
1012 // end_from - source arrays end address
1013 // end_to - destination array end address aligned to 8-bytes
1014 // count - elements count to copy >= the count equivalent to 16 bytes
1015 // count_dec - elements count's decrement equivalent to 16 bytes
1016 // L_aligned_copy - aligned copy exit label
1017 // L_copy_bytes - copy exit label
1018 //
1019 void copy_16_bytes_backward_with_shift(Register end_from, Register end_to,
1020 Register count, int count_dec,
1021 Label& L_aligned_copy, Label& L_copy_bytes) {
1022 Label L_loop, L_copy_last_bytes;
1023
1024 // if both arrays have the same alignment mod 8, do 8 bytes aligned copy
1025 __ andcc(end_from, 7, G1); // misaligned bytes
1026 __ br(Assembler::zero, false, Assembler::pt, L_aligned_copy);
1027 __ delayed()->deccc(count, count_dec); // Pre-decrement 'count'
1028
1029 const Register left_shift = G1; // left shift bit counter
1030 const Register right_shift = G5; // right shift bit counter
1031
1032 __ sll(G1, LogBitsPerByte, left_shift);
1033 __ mov(64, right_shift);
1034 __ sub(right_shift, left_shift, right_shift);
1035
1036 //
1037 // Load 2 aligned 8-bytes chunks and use one from previous iteration
1038 // to form 2 aligned 8-bytes chunks to store.
1039 //
1040 __ andn(end_from, 7, end_from); // Align address
1041 __ ldx(end_from, 0, O3);
1042 __ align(OptoLoopAlignment);
1043 __ BIND(L_loop);
1044 __ ldx(end_from, -8, O4);
1045 __ deccc(count, count_dec); // Can we do next iteration after this one?
1046 __ ldx(end_from, -16, G4);
1047 __ dec(end_to, 16);
1048 __ dec(end_from, 16);
1049 __ srlx(O3, right_shift, O3);
1050 __ sllx(O4, left_shift, G3);
1051 __ bset(G3, O3);
1052 __ stx(O3, end_to, 8);
1053 __ srlx(O4, right_shift, O4);
1054 __ sllx(G4, left_shift, G3);
1055 __ bset(G3, O4);
1056 __ stx(O4, end_to, 0);
1057 __ brx(Assembler::greaterEqual, false, Assembler::pt, L_loop);
1058 __ delayed()->mov(G4, O3);
1059
1060 __ inccc(count, count_dec>>1 ); // + 8 bytes
1061 __ brx(Assembler::negative, true, Assembler::pn, L_copy_last_bytes);
1062 __ delayed()->inc(count, count_dec>>1); // restore 'count'
1063
1064 // copy 8 bytes, part of them already loaded in O3
1065 __ ldx(end_from, -8, O4);
1066 __ dec(end_to, 8);
1067 __ dec(end_from, 8);
1068 __ srlx(O3, right_shift, O3);
1069 __ sllx(O4, left_shift, G3);
1070 __ bset(O3, G3);
1071 __ stx(G3, end_to, 0);
1072
1073 __ BIND(L_copy_last_bytes);
1074 __ srl(left_shift, LogBitsPerByte, left_shift); // misaligned bytes
1075 __ br(Assembler::always, false, Assembler::pt, L_copy_bytes);
1076 __ delayed()->add(end_from, left_shift, end_from); // restore address
1077 }
1078
1079 address generate_unsafecopy_common_error_exit() {
1080 address start_pc = __ pc();
1081 if (UseBlockCopy) {
1082 __ wrasi(G0, Assembler::ASI_PRIMARY_NOFAULT);
1083 __ membar(Assembler::StoreLoad);
1084 }
1085 __ retl();
1086 __ delayed()->mov(G0, O0); // return 0
1087 return start_pc;
1088 }
1089
1090 //
1091 // Generate stub for disjoint byte copy. If "aligned" is true, the
1092 // "from" and "to" addresses are assumed to be heapword aligned.
1093 //
1094 // Arguments for generated stub:
1095 // from: O0
1096 // to: O1
1097 // count: O2 treated as signed
1098 //
1099 address generate_disjoint_byte_copy(bool aligned, address *entry, const char *name) {
1100 __ align(CodeEntryAlignment);
1101 StubCodeMark mark(this, "StubRoutines", name);
1102 address start = __ pc();
1103
1104 Label L_skip_alignment, L_align;
1105 Label L_copy_byte, L_copy_byte_loop, L_exit;
1106
1107 const Register from = O0; // source array address
1108 const Register to = O1; // destination array address
1109 const Register count = O2; // elements count
1110 const Register offset = O5; // offset from start of arrays
1111 // O3, O4, G3, G4 are used as temp registers
1112
1113 assert_clean_int(count, O3); // Make sure 'count' is clean int.
1114
1115 if (entry != NULL) {
1116 *entry = __ pc();
1117 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1118 BLOCK_COMMENT("Entry:");
1119 }
1120
1121 {
1122 // UnsafeCopyMemory page error: continue at UnsafeCopyMemory common_error_exit
1123 UnsafeCopyMemoryMark ucmm(this, !aligned, false);
1124
1125 // for short arrays, just do single element copy
1126 __ cmp(count, 23); // 16 + 7
1127 __ brx(Assembler::less, false, Assembler::pn, L_copy_byte);
1128 __ delayed()->mov(G0, offset);
1129
1130 if (aligned) {
1131 // 'aligned' == true when it is known statically during compilation
1132 // of this arraycopy call site that both 'from' and 'to' addresses
1133 // are HeapWordSize aligned (see LibraryCallKit::basictype2arraycopy()).
1134 //
1135 // Aligned arrays have 4 bytes alignment in 32-bits VM
1136 // and 8 bytes - in 64-bits VM. So we do it only for 32-bits VM
1137 //
1138 } else {
1139 // copy bytes to align 'to' on 8 byte boundary
1140 __ andcc(to, 7, G1); // misaligned bytes
1141 __ br(Assembler::zero, false, Assembler::pt, L_skip_alignment);
1142 __ delayed()->neg(G1);
1143 __ inc(G1, 8); // bytes need to copy to next 8-bytes alignment
1144 __ sub(count, G1, count);
1145 __ BIND(L_align);
1146 __ ldub(from, 0, O3);
1147 __ deccc(G1);
1148 __ inc(from);
1149 __ stb(O3, to, 0);
1150 __ br(Assembler::notZero, false, Assembler::pt, L_align);
1151 __ delayed()->inc(to);
1152 __ BIND(L_skip_alignment);
1153 }
1154 if (!aligned) {
1155 // Copy with shift 16 bytes per iteration if arrays do not have
1156 // the same alignment mod 8, otherwise fall through to the next
1157 // code for aligned copy.
1158 // The compare above (count >= 23) guarantes 'count' >= 16 bytes.
1159 // Also jump over aligned copy after the copy with shift completed.
1160
1161 copy_16_bytes_forward_with_shift(from, to, count, 0, L_copy_byte);
1162 }
1163
1164 // Both array are 8 bytes aligned, copy 16 bytes at a time
1165 __ and3(count, 7, G4); // Save count
1166 __ srl(count, 3, count);
1167 generate_disjoint_long_copy_core(aligned);
1168 __ mov(G4, count); // Restore count
1169
1170 // copy tailing bytes
1171 __ BIND(L_copy_byte);
1172 __ cmp_and_br_short(count, 0, Assembler::equal, Assembler::pt, L_exit);
1173 __ align(OptoLoopAlignment);
1174 __ BIND(L_copy_byte_loop);
1175 __ ldub(from, offset, O3);
1176 __ deccc(count);
1177 __ stb(O3, to, offset);
1178 __ brx(Assembler::notZero, false, Assembler::pt, L_copy_byte_loop);
1179 __ delayed()->inc(offset);
1180 }
1181
1182 __ BIND(L_exit);
1183 // O3, O4 are used as temp registers
1184 inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr, O3, O4);
1185 __ retl();
1186 __ delayed()->mov(G0, O0); // return 0
1187 return start;
1188 }
1189
1190 //
1191 // Generate stub for conjoint byte copy. If "aligned" is true, the
1192 // "from" and "to" addresses are assumed to be heapword aligned.
1193 //
1194 // Arguments for generated stub:
1195 // from: O0
1196 // to: O1
1197 // count: O2 treated as signed
1198 //
1199 address generate_conjoint_byte_copy(bool aligned, address nooverlap_target,
1200 address *entry, const char *name) {
1201 // Do reverse copy.
1202
1203 __ align(CodeEntryAlignment);
1204 StubCodeMark mark(this, "StubRoutines", name);
1205 address start = __ pc();
1206
1207 Label L_skip_alignment, L_align, L_aligned_copy;
1208 Label L_copy_byte, L_copy_byte_loop, L_exit;
1209
1210 const Register from = O0; // source array address
1211 const Register to = O1; // destination array address
1212 const Register count = O2; // elements count
1213 const Register end_from = from; // source array end address
1214 const Register end_to = to; // destination array end address
1215
1216 assert_clean_int(count, O3); // Make sure 'count' is clean int.
1217
1218 if (entry != NULL) {
1219 *entry = __ pc();
1220 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1221 BLOCK_COMMENT("Entry:");
1222 }
1223
1224 array_overlap_test(nooverlap_target, 0);
1225
1226 {
1227 // UnsafeCopyMemory page error: continue at UnsafeCopyMemory common_error_exit
1228 UnsafeCopyMemoryMark ucmm(this, !aligned, false);
1229
1230 __ add(to, count, end_to); // offset after last copied element
1231
1232 // for short arrays, just do single element copy
1233 __ cmp(count, 23); // 16 + 7
1234 __ brx(Assembler::less, false, Assembler::pn, L_copy_byte);
1235 __ delayed()->add(from, count, end_from);
1236
1237 {
1238 // Align end of arrays since they could be not aligned even
1239 // when arrays itself are aligned.
1240
1241 // copy bytes to align 'end_to' on 8 byte boundary
1242 __ andcc(end_to, 7, G1); // misaligned bytes
1243 __ br(Assembler::zero, false, Assembler::pt, L_skip_alignment);
1244 __ delayed()->nop();
1245 __ sub(count, G1, count);
1246 __ BIND(L_align);
1247 __ dec(end_from);
1248 __ dec(end_to);
1249 __ ldub(end_from, 0, O3);
1250 __ deccc(G1);
1251 __ brx(Assembler::notZero, false, Assembler::pt, L_align);
1252 __ delayed()->stb(O3, end_to, 0);
1253 __ BIND(L_skip_alignment);
1254 }
1255 if (aligned) {
1256 // Both arrays are aligned to 8-bytes in 64-bits VM.
1257 // The 'count' is decremented in copy_16_bytes_backward_with_shift()
1258 // in unaligned case.
1259 __ dec(count, 16);
1260 } else {
1261 // Copy with shift 16 bytes per iteration if arrays do not have
1262 // the same alignment mod 8, otherwise jump to the next
1263 // code for aligned copy (and substracting 16 from 'count' before jump).
1264 // The compare above (count >= 11) guarantes 'count' >= 16 bytes.
1265 // Also jump over aligned copy after the copy with shift completed.
1266
1267 copy_16_bytes_backward_with_shift(end_from, end_to, count, 16,
1268 L_aligned_copy, L_copy_byte);
1269 }
1270 // copy 4 elements (16 bytes) at a time
1271 __ align(OptoLoopAlignment);
1272 __ BIND(L_aligned_copy);
1273 __ dec(end_from, 16);
1274 __ ldx(end_from, 8, O3);
1275 __ ldx(end_from, 0, O4);
1276 __ dec(end_to, 16);
1277 __ deccc(count, 16);
1278 __ stx(O3, end_to, 8);
1279 __ brx(Assembler::greaterEqual, false, Assembler::pt, L_aligned_copy);
1280 __ delayed()->stx(O4, end_to, 0);
1281 __ inc(count, 16);
1282
1283 // copy 1 element (2 bytes) at a time
1284 __ BIND(L_copy_byte);
1285 __ cmp_and_br_short(count, 0, Assembler::equal, Assembler::pt, L_exit);
1286 __ align(OptoLoopAlignment);
1287 __ BIND(L_copy_byte_loop);
1288 __ dec(end_from);
1289 __ dec(end_to);
1290 __ ldub(end_from, 0, O4);
1291 __ deccc(count);
1292 __ brx(Assembler::greater, false, Assembler::pt, L_copy_byte_loop);
1293 __ delayed()->stb(O4, end_to, 0);
1294 }
1295
1296 __ BIND(L_exit);
1297 // O3, O4 are used as temp registers
1298 inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr, O3, O4);
1299 __ retl();
1300 __ delayed()->mov(G0, O0); // return 0
1301 return start;
1302 }
1303
1304 //
1305 // Generate stub for disjoint short copy. If "aligned" is true, the
1306 // "from" and "to" addresses are assumed to be heapword aligned.
1307 //
1308 // Arguments for generated stub:
1309 // from: O0
1310 // to: O1
1311 // count: O2 treated as signed
1312 //
1313 address generate_disjoint_short_copy(bool aligned, address *entry, const char * name) {
1314 __ align(CodeEntryAlignment);
1315 StubCodeMark mark(this, "StubRoutines", name);
1316 address start = __ pc();
1317
1318 Label L_skip_alignment, L_skip_alignment2;
1319 Label L_copy_2_bytes, L_copy_2_bytes_loop, L_exit;
1320
1321 const Register from = O0; // source array address
1322 const Register to = O1; // destination array address
1323 const Register count = O2; // elements count
1324 const Register offset = O5; // offset from start of arrays
1325 // O3, O4, G3, G4 are used as temp registers
1326
1327 assert_clean_int(count, O3); // Make sure 'count' is clean int.
1328
1329 if (entry != NULL) {
1330 *entry = __ pc();
1331 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1332 BLOCK_COMMENT("Entry:");
1333 }
1334
1335 {
1336 // UnsafeCopyMemory page error: continue at UnsafeCopyMemory common_error_exit
1337 UnsafeCopyMemoryMark ucmm(this, !aligned, false);
1338 // for short arrays, just do single element copy
1339 __ cmp(count, 11); // 8 + 3 (22 bytes)
1340 __ brx(Assembler::less, false, Assembler::pn, L_copy_2_bytes);
1341 __ delayed()->mov(G0, offset);
1342
1343 if (aligned) {
1344 // 'aligned' == true when it is known statically during compilation
1345 // of this arraycopy call site that both 'from' and 'to' addresses
1346 // are HeapWordSize aligned (see LibraryCallKit::basictype2arraycopy()).
1347 //
1348 // Aligned arrays have 4 bytes alignment in 32-bits VM
1349 // and 8 bytes - in 64-bits VM.
1350 //
1351 } else {
1352 // copy 1 element if necessary to align 'to' on an 4 bytes
1353 __ andcc(to, 3, G0);
1354 __ br(Assembler::zero, false, Assembler::pt, L_skip_alignment);
1355 __ delayed()->lduh(from, 0, O3);
1356 __ inc(from, 2);
1357 __ inc(to, 2);
1358 __ dec(count);
1359 __ sth(O3, to, -2);
1360 __ BIND(L_skip_alignment);
1361
1362 // copy 2 elements to align 'to' on an 8 byte boundary
1363 __ andcc(to, 7, G0);
1364 __ br(Assembler::zero, false, Assembler::pn, L_skip_alignment2);
1365 __ delayed()->lduh(from, 0, O3);
1366 __ dec(count, 2);
1367 __ lduh(from, 2, O4);
1368 __ inc(from, 4);
1369 __ inc(to, 4);
1370 __ sth(O3, to, -4);
1371 __ sth(O4, to, -2);
1372 __ BIND(L_skip_alignment2);
1373 }
1374 if (!aligned) {
1375 // Copy with shift 16 bytes per iteration if arrays do not have
1376 // the same alignment mod 8, otherwise fall through to the next
1377 // code for aligned copy.
1378 // The compare above (count >= 11) guarantes 'count' >= 16 bytes.
1379 // Also jump over aligned copy after the copy with shift completed.
1380
1381 copy_16_bytes_forward_with_shift(from, to, count, 1, L_copy_2_bytes);
1382 }
1383
1384 // Both array are 8 bytes aligned, copy 16 bytes at a time
1385 __ and3(count, 3, G4); // Save
1386 __ srl(count, 2, count);
1387 generate_disjoint_long_copy_core(aligned);
1388 __ mov(G4, count); // restore
1389
1390 // copy 1 element at a time
1391 __ BIND(L_copy_2_bytes);
1392 __ cmp_and_br_short(count, 0, Assembler::equal, Assembler::pt, L_exit);
1393 __ align(OptoLoopAlignment);
1394 __ BIND(L_copy_2_bytes_loop);
1395 __ lduh(from, offset, O3);
1396 __ deccc(count);
1397 __ sth(O3, to, offset);
1398 __ brx(Assembler::notZero, false, Assembler::pt, L_copy_2_bytes_loop);
1399 __ delayed()->inc(offset, 2);
1400 }
1401
1402 __ BIND(L_exit);
1403 // O3, O4 are used as temp registers
1404 inc_counter_np(SharedRuntime::_jshort_array_copy_ctr, O3, O4);
1405 __ retl();
1406 __ delayed()->mov(G0, O0); // return 0
1407 return start;
1408 }
1409
1410 //
1411 // Generate stub for disjoint short fill. If "aligned" is true, the
1412 // "to" address is assumed to be heapword aligned.
1413 //
1414 // Arguments for generated stub:
1415 // to: O0
1416 // value: O1
1417 // count: O2 treated as signed
1418 //
1419 address generate_fill(BasicType t, bool aligned, const char* name) {
1420 __ align(CodeEntryAlignment);
1421 StubCodeMark mark(this, "StubRoutines", name);
1422 address start = __ pc();
1423
1424 const Register to = O0; // source array address
1425 const Register value = O1; // fill value
1426 const Register count = O2; // elements count
1427 // O3 is used as a temp register
1428
1429 assert_clean_int(count, O3); // Make sure 'count' is clean int.
1430
1431 Label L_exit, L_skip_align1, L_skip_align2, L_fill_byte;
1432 Label L_fill_2_bytes, L_fill_elements, L_fill_32_bytes;
1433
1434 int shift = -1;
1435 switch (t) {
1436 case T_BYTE:
1437 shift = 2;
1438 break;
1439 case T_SHORT:
1440 shift = 1;
1441 break;
1442 case T_INT:
1443 shift = 0;
1444 break;
1445 default: ShouldNotReachHere();
1446 }
1447
1448 BLOCK_COMMENT("Entry:");
1449
1450 if (t == T_BYTE) {
1451 // Zero extend value
1452 __ and3(value, 0xff, value);
1453 __ sllx(value, 8, O3);
1454 __ or3(value, O3, value);
1455 }
1456 if (t == T_SHORT) {
1457 // Zero extend value
1458 __ sllx(value, 48, value);
1459 __ srlx(value, 48, value);
1460 }
1461 if (t == T_BYTE || t == T_SHORT) {
1462 __ sllx(value, 16, O3);
1463 __ or3(value, O3, value);
1464 }
1465
1466 __ cmp(count, 2<<shift); // Short arrays (< 8 bytes) fill by element
1467 __ brx(Assembler::lessUnsigned, false, Assembler::pn, L_fill_elements); // use unsigned cmp
1468 __ delayed()->andcc(count, 1, G0);
1469
1470 if (!aligned && (t == T_BYTE || t == T_SHORT)) {
1471 // align source address at 4 bytes address boundary
1472 if (t == T_BYTE) {
1473 // One byte misalignment happens only for byte arrays
1474 __ andcc(to, 1, G0);
1475 __ br(Assembler::zero, false, Assembler::pt, L_skip_align1);
1476 __ delayed()->nop();
1477 __ stb(value, to, 0);
1478 __ inc(to, 1);
1479 __ dec(count, 1);
1480 __ BIND(L_skip_align1);
1481 }
1482 // Two bytes misalignment happens only for byte and short (char) arrays
1483 __ andcc(to, 2, G0);
1484 __ br(Assembler::zero, false, Assembler::pt, L_skip_align2);
1485 __ delayed()->nop();
1486 __ sth(value, to, 0);
1487 __ inc(to, 2);
1488 __ dec(count, 1 << (shift - 1));
1489 __ BIND(L_skip_align2);
1490 }
1491 if (!aligned) {
1492 // align to 8 bytes, we know we are 4 byte aligned to start
1493 __ andcc(to, 7, G0);
1494 __ br(Assembler::zero, false, Assembler::pt, L_fill_32_bytes);
1495 __ delayed()->nop();
1496 __ stw(value, to, 0);
1497 __ inc(to, 4);
1498 __ dec(count, 1 << shift);
1499 __ BIND(L_fill_32_bytes);
1500 }
1501
1502 if (t == T_INT) {
1503 // Zero extend value
1504 __ srl(value, 0, value);
1505 }
1506 if (t == T_BYTE || t == T_SHORT || t == T_INT) {
1507 __ sllx(value, 32, O3);
1508 __ or3(value, O3, value);
1509 }
1510
1511 Label L_check_fill_8_bytes;
1512 // Fill 32-byte chunks
1513 __ subcc(count, 8 << shift, count);
1514 __ brx(Assembler::less, false, Assembler::pt, L_check_fill_8_bytes);
1515 __ delayed()->nop();
1516
1517 Label L_fill_32_bytes_loop, L_fill_4_bytes;
1518 __ align(16);
1519 __ BIND(L_fill_32_bytes_loop);
1520
1521 __ stx(value, to, 0);
1522 __ stx(value, to, 8);
1523 __ stx(value, to, 16);
1524 __ stx(value, to, 24);
1525
1526 __ subcc(count, 8 << shift, count);
1527 __ brx(Assembler::greaterEqual, false, Assembler::pt, L_fill_32_bytes_loop);
1528 __ delayed()->add(to, 32, to);
1529
1530 __ BIND(L_check_fill_8_bytes);
1531 __ addcc(count, 8 << shift, count);
1532 __ brx(Assembler::zero, false, Assembler::pn, L_exit);
1533 __ delayed()->subcc(count, 1 << (shift + 1), count);
1534 __ brx(Assembler::less, false, Assembler::pn, L_fill_4_bytes);
1535 __ delayed()->andcc(count, 1<<shift, G0);
1536
1537 //
1538 // length is too short, just fill 8 bytes at a time
1539 //
1540 Label L_fill_8_bytes_loop;
1541 __ BIND(L_fill_8_bytes_loop);
1542 __ stx(value, to, 0);
1543 __ subcc(count, 1 << (shift + 1), count);
1544 __ brx(Assembler::greaterEqual, false, Assembler::pn, L_fill_8_bytes_loop);
1545 __ delayed()->add(to, 8, to);
1546
1547 // fill trailing 4 bytes
1548 __ andcc(count, 1<<shift, G0); // in delay slot of branches
1549 if (t == T_INT) {
1550 __ BIND(L_fill_elements);
1551 }
1552 __ BIND(L_fill_4_bytes);
1553 __ brx(Assembler::zero, false, Assembler::pt, L_fill_2_bytes);
1554 if (t == T_BYTE || t == T_SHORT) {
1555 __ delayed()->andcc(count, 1<<(shift-1), G0);
1556 } else {
1557 __ delayed()->nop();
1558 }
1559 __ stw(value, to, 0);
1560 if (t == T_BYTE || t == T_SHORT) {
1561 __ inc(to, 4);
1562 // fill trailing 2 bytes
1563 __ andcc(count, 1<<(shift-1), G0); // in delay slot of branches
1564 __ BIND(L_fill_2_bytes);
1565 __ brx(Assembler::zero, false, Assembler::pt, L_fill_byte);
1566 __ delayed()->andcc(count, 1, count);
1567 __ sth(value, to, 0);
1568 if (t == T_BYTE) {
1569 __ inc(to, 2);
1570 // fill trailing byte
1571 __ andcc(count, 1, count); // in delay slot of branches
1572 __ BIND(L_fill_byte);
1573 __ brx(Assembler::zero, false, Assembler::pt, L_exit);
1574 __ delayed()->nop();
1575 __ stb(value, to, 0);
1576 } else {
1577 __ BIND(L_fill_byte);
1578 }
1579 } else {
1580 __ BIND(L_fill_2_bytes);
1581 }
1582 __ BIND(L_exit);
1583 __ retl();
1584 __ delayed()->nop();
1585
1586 // Handle copies less than 8 bytes. Int is handled elsewhere.
1587 if (t == T_BYTE) {
1588 __ BIND(L_fill_elements);
1589 Label L_fill_2, L_fill_4;
1590 // in delay slot __ andcc(count, 1, G0);
1591 __ brx(Assembler::zero, false, Assembler::pt, L_fill_2);
1592 __ delayed()->andcc(count, 2, G0);
1593 __ stb(value, to, 0);
1594 __ inc(to, 1);
1595 __ BIND(L_fill_2);
1596 __ brx(Assembler::zero, false, Assembler::pt, L_fill_4);
1597 __ delayed()->andcc(count, 4, G0);
1598 __ stb(value, to, 0);
1599 __ stb(value, to, 1);
1600 __ inc(to, 2);
1601 __ BIND(L_fill_4);
1602 __ brx(Assembler::zero, false, Assembler::pt, L_exit);
1603 __ delayed()->nop();
1604 __ stb(value, to, 0);
1605 __ stb(value, to, 1);
1606 __ stb(value, to, 2);
1607 __ retl();
1608 __ delayed()->stb(value, to, 3);
1609 }
1610
1611 if (t == T_SHORT) {
1612 Label L_fill_2;
1613 __ BIND(L_fill_elements);
1614 // in delay slot __ andcc(count, 1, G0);
1615 __ brx(Assembler::zero, false, Assembler::pt, L_fill_2);
1616 __ delayed()->andcc(count, 2, G0);
1617 __ sth(value, to, 0);
1618 __ inc(to, 2);
1619 __ BIND(L_fill_2);
1620 __ brx(Assembler::zero, false, Assembler::pt, L_exit);
1621 __ delayed()->nop();
1622 __ sth(value, to, 0);
1623 __ retl();
1624 __ delayed()->sth(value, to, 2);
1625 }
1626 return start;
1627 }
1628
1629 //
1630 // Generate stub for conjoint short copy. If "aligned" is true, the
1631 // "from" and "to" addresses are assumed to be heapword aligned.
1632 //
1633 // Arguments for generated stub:
1634 // from: O0
1635 // to: O1
1636 // count: O2 treated as signed
1637 //
1638 address generate_conjoint_short_copy(bool aligned, address nooverlap_target,
1639 address *entry, const char *name) {
1640 // Do reverse copy.
1641
1642 __ align(CodeEntryAlignment);
1643 StubCodeMark mark(this, "StubRoutines", name);
1644 address start = __ pc();
1645
1646 Label L_skip_alignment, L_skip_alignment2, L_aligned_copy;
1647 Label L_copy_2_bytes, L_copy_2_bytes_loop, L_exit;
1648
1649 const Register from = O0; // source array address
1650 const Register to = O1; // destination array address
1651 const Register count = O2; // elements count
1652 const Register end_from = from; // source array end address
1653 const Register end_to = to; // destination array end address
1654
1655 const Register byte_count = O3; // bytes count to copy
1656
1657 assert_clean_int(count, O3); // Make sure 'count' is clean int.
1658
1659 if (entry != NULL) {
1660 *entry = __ pc();
1661 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1662 BLOCK_COMMENT("Entry:");
1663 }
1664
1665 array_overlap_test(nooverlap_target, 1);
1666
1667 {
1668 // UnsafeCopyMemory page error: continue at UnsafeCopyMemory common_error_exit
1669 UnsafeCopyMemoryMark ucmm(this, !aligned, false);
1670
1671 __ sllx(count, LogBytesPerShort, byte_count);
1672 __ add(to, byte_count, end_to); // offset after last copied element
1673
1674 // for short arrays, just do single element copy
1675 __ cmp(count, 11); // 8 + 3 (22 bytes)
1676 __ brx(Assembler::less, false, Assembler::pn, L_copy_2_bytes);
1677 __ delayed()->add(from, byte_count, end_from);
1678
1679 {
1680 // Align end of arrays since they could be not aligned even
1681 // when arrays itself are aligned.
1682
1683 // copy 1 element if necessary to align 'end_to' on an 4 bytes
1684 __ andcc(end_to, 3, G0);
1685 __ br(Assembler::zero, false, Assembler::pt, L_skip_alignment);
1686 __ delayed()->lduh(end_from, -2, O3);
1687 __ dec(end_from, 2);
1688 __ dec(end_to, 2);
1689 __ dec(count);
1690 __ sth(O3, end_to, 0);
1691 __ BIND(L_skip_alignment);
1692
1693 // copy 2 elements to align 'end_to' on an 8 byte boundary
1694 __ andcc(end_to, 7, G0);
1695 __ br(Assembler::zero, false, Assembler::pn, L_skip_alignment2);
1696 __ delayed()->lduh(end_from, -2, O3);
1697 __ dec(count, 2);
1698 __ lduh(end_from, -4, O4);
1699 __ dec(end_from, 4);
1700 __ dec(end_to, 4);
1701 __ sth(O3, end_to, 2);
1702 __ sth(O4, end_to, 0);
1703 __ BIND(L_skip_alignment2);
1704 }
1705 if (aligned) {
1706 // Both arrays are aligned to 8-bytes in 64-bits VM.
1707 // The 'count' is decremented in copy_16_bytes_backward_with_shift()
1708 // in unaligned case.
1709 __ dec(count, 8);
1710 } else {
1711 // Copy with shift 16 bytes per iteration if arrays do not have
1712 // the same alignment mod 8, otherwise jump to the next
1713 // code for aligned copy (and substracting 8 from 'count' before jump).
1714 // The compare above (count >= 11) guarantes 'count' >= 16 bytes.
1715 // Also jump over aligned copy after the copy with shift completed.
1716
1717 copy_16_bytes_backward_with_shift(end_from, end_to, count, 8,
1718 L_aligned_copy, L_copy_2_bytes);
1719 }
1720 // copy 4 elements (16 bytes) at a time
1721 __ align(OptoLoopAlignment);
1722 __ BIND(L_aligned_copy);
1723 __ dec(end_from, 16);
1724 __ ldx(end_from, 8, O3);
1725 __ ldx(end_from, 0, O4);
1726 __ dec(end_to, 16);
1727 __ deccc(count, 8);
1728 __ stx(O3, end_to, 8);
1729 __ brx(Assembler::greaterEqual, false, Assembler::pt, L_aligned_copy);
1730 __ delayed()->stx(O4, end_to, 0);
1731 __ inc(count, 8);
1732
1733 // copy 1 element (2 bytes) at a time
1734 __ BIND(L_copy_2_bytes);
1735 __ cmp_and_br_short(count, 0, Assembler::equal, Assembler::pt, L_exit);
1736 __ BIND(L_copy_2_bytes_loop);
1737 __ dec(end_from, 2);
1738 __ dec(end_to, 2);
1739 __ lduh(end_from, 0, O4);
1740 __ deccc(count);
1741 __ brx(Assembler::greater, false, Assembler::pt, L_copy_2_bytes_loop);
1742 __ delayed()->sth(O4, end_to, 0);
1743 }
1744 __ BIND(L_exit);
1745 // O3, O4 are used as temp registers
1746 inc_counter_np(SharedRuntime::_jshort_array_copy_ctr, O3, O4);
1747 __ retl();
1748 __ delayed()->mov(G0, O0); // return 0
1749 return start;
1750 }
1751
1752 //
1753 // Helper methods for generate_disjoint_int_copy_core()
1754 //
1755 void copy_16_bytes_loop(Register from, Register to, Register count, int count_dec,
1756 Label& L_loop, bool use_prefetch, bool use_bis) {
1757
1758 __ align(OptoLoopAlignment);
1759 __ BIND(L_loop);
1760 if (use_prefetch) {
1761 if (ArraycopySrcPrefetchDistance > 0) {
1762 __ prefetch(from, ArraycopySrcPrefetchDistance, Assembler::severalReads);
1763 }
1764 if (ArraycopyDstPrefetchDistance > 0) {
1765 __ prefetch(to, ArraycopyDstPrefetchDistance, Assembler::severalWritesAndPossiblyReads);
1766 }
1767 }
1768 __ ldx(from, 4, O4);
1769 __ ldx(from, 12, G4);
1770 __ inc(to, 16);
1771 __ inc(from, 16);
1772 __ deccc(count, 4); // Can we do next iteration after this one?
1773
1774 __ srlx(O4, 32, G3);
1775 __ bset(G3, O3);
1776 __ sllx(O4, 32, O4);
1777 __ srlx(G4, 32, G3);
1778 __ bset(G3, O4);
1779 if (use_bis) {
1780 __ stxa(O3, to, -16);
1781 __ stxa(O4, to, -8);
1782 } else {
1783 __ stx(O3, to, -16);
1784 __ stx(O4, to, -8);
1785 }
1786 __ brx(Assembler::greaterEqual, false, Assembler::pt, L_loop);
1787 __ delayed()->sllx(G4, 32, O3);
1788
1789 }
1790
1791 //
1792 // Generate core code for disjoint int copy (and oop copy on 32-bit).
1793 // If "aligned" is true, the "from" and "to" addresses are assumed
1794 // to be heapword aligned.
1795 //
1796 // Arguments:
1797 // from: O0
1798 // to: O1
1799 // count: O2 treated as signed
1800 //
1801 void generate_disjoint_int_copy_core(bool aligned) {
1802
1803 Label L_skip_alignment, L_aligned_copy;
1804 Label L_copy_4_bytes, L_copy_4_bytes_loop, L_exit;
1805
1806 const Register from = O0; // source array address
1807 const Register to = O1; // destination array address
1808 const Register count = O2; // elements count
1809 const Register offset = O5; // offset from start of arrays
1810 // O3, O4, G3, G4 are used as temp registers
1811
1812 // 'aligned' == true when it is known statically during compilation
1813 // of this arraycopy call site that both 'from' and 'to' addresses
1814 // are HeapWordSize aligned (see LibraryCallKit::basictype2arraycopy()).
1815 //
1816 // Aligned arrays have 4 bytes alignment in 32-bits VM
1817 // and 8 bytes - in 64-bits VM.
1818 //
1819 if (!aligned) {
1820 // The next check could be put under 'ifndef' since the code in
1821 // generate_disjoint_long_copy_core() has own checks and set 'offset'.
1822
1823 // for short arrays, just do single element copy
1824 __ cmp(count, 5); // 4 + 1 (20 bytes)
1825 __ brx(Assembler::lessEqual, false, Assembler::pn, L_copy_4_bytes);
1826 __ delayed()->mov(G0, offset);
1827
1828 // copy 1 element to align 'to' on an 8 byte boundary
1829 __ andcc(to, 7, G0);
1830 __ br(Assembler::zero, false, Assembler::pt, L_skip_alignment);
1831 __ delayed()->ld(from, 0, O3);
1832 __ inc(from, 4);
1833 __ inc(to, 4);
1834 __ dec(count);
1835 __ st(O3, to, -4);
1836 __ BIND(L_skip_alignment);
1837
1838 // if arrays have same alignment mod 8, do 4 elements copy
1839 __ andcc(from, 7, G0);
1840 __ br(Assembler::zero, false, Assembler::pt, L_aligned_copy);
1841 __ delayed()->ld(from, 0, O3);
1842
1843 //
1844 // Load 2 aligned 8-bytes chunks and use one from previous iteration
1845 // to form 2 aligned 8-bytes chunks to store.
1846 //
1847 // copy_16_bytes_forward_with_shift() is not used here since this
1848 // code is more optimal.
1849
1850 // copy with shift 4 elements (16 bytes) at a time
1851 __ dec(count, 4); // The cmp at the beginning guaranty count >= 4
1852 __ sllx(O3, 32, O3);
1853
1854 disjoint_copy_core(from, to, count, 2, 16, &StubGenerator::copy_16_bytes_loop);
1855
1856 __ br(Assembler::always, false, Assembler::pt, L_copy_4_bytes);
1857 __ delayed()->inc(count, 4); // restore 'count'
1858
1859 __ BIND(L_aligned_copy);
1860 } // !aligned
1861
1862 // copy 4 elements (16 bytes) at a time
1863 __ and3(count, 1, G4); // Save
1864 __ srl(count, 1, count);
1865 generate_disjoint_long_copy_core(aligned);
1866 __ mov(G4, count); // Restore
1867
1868 // copy 1 element at a time
1869 __ BIND(L_copy_4_bytes);
1870 __ cmp_and_br_short(count, 0, Assembler::equal, Assembler::pt, L_exit);
1871 __ BIND(L_copy_4_bytes_loop);
1872 __ ld(from, offset, O3);
1873 __ deccc(count);
1874 __ st(O3, to, offset);
1875 __ brx(Assembler::notZero, false, Assembler::pt, L_copy_4_bytes_loop);
1876 __ delayed()->inc(offset, 4);
1877 __ BIND(L_exit);
1878 }
1879
1880 //
1881 // Generate stub for disjoint int copy. If "aligned" is true, the
1882 // "from" and "to" addresses are assumed to be heapword aligned.
1883 //
1884 // Arguments for generated stub:
1885 // from: O0
1886 // to: O1
1887 // count: O2 treated as signed
1888 //
1889 address generate_disjoint_int_copy(bool aligned, address *entry, const char *name) {
1890 __ align(CodeEntryAlignment);
1891 StubCodeMark mark(this, "StubRoutines", name);
1892 address start = __ pc();
1893
1894 const Register count = O2;
1895 assert_clean_int(count, O3); // Make sure 'count' is clean int.
1896
1897 if (entry != NULL) {
1898 *entry = __ pc();
1899 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1900 BLOCK_COMMENT("Entry:");
1901 }
1902 {
1903 // UnsafeCopyMemory page error: continue at UnsafeCopyMemory common_error_exit
1904 UnsafeCopyMemoryMark ucmm(this, !aligned, false);
1905 generate_disjoint_int_copy_core(aligned);
1906 }
1907 // O3, O4 are used as temp registers
1908 inc_counter_np(SharedRuntime::_jint_array_copy_ctr, O3, O4);
1909 __ retl();
1910 __ delayed()->mov(G0, O0); // return 0
1911 return start;
1912 }
1913
1914 //
1915 // Generate core code for conjoint int copy (and oop copy on 32-bit).
1916 // If "aligned" is true, the "from" and "to" addresses are assumed
1917 // to be heapword aligned.
1918 //
1919 // Arguments:
1920 // from: O0
1921 // to: O1
1922 // count: O2 treated as signed
1923 //
1924 void generate_conjoint_int_copy_core(bool aligned) {
1925 // Do reverse copy.
1926
1927 Label L_skip_alignment, L_aligned_copy;
1928 Label L_copy_16_bytes, L_copy_4_bytes, L_copy_4_bytes_loop, L_exit;
1929
1930 const Register from = O0; // source array address
1931 const Register to = O1; // destination array address
1932 const Register count = O2; // elements count
1933 const Register end_from = from; // source array end address
1934 const Register end_to = to; // destination array end address
1935 // O3, O4, O5, G3 are used as temp registers
1936
1937 const Register byte_count = O3; // bytes count to copy
1938
1939 __ sllx(count, LogBytesPerInt, byte_count);
1940 __ add(to, byte_count, end_to); // offset after last copied element
1941
1942 __ cmp(count, 5); // for short arrays, just do single element copy
1943 __ brx(Assembler::lessEqual, false, Assembler::pn, L_copy_4_bytes);
1944 __ delayed()->add(from, byte_count, end_from);
1945
1946 // copy 1 element to align 'to' on an 8 byte boundary
1947 __ andcc(end_to, 7, G0);
1948 __ br(Assembler::zero, false, Assembler::pt, L_skip_alignment);
1949 __ delayed()->nop();
1950 __ dec(count);
1951 __ dec(end_from, 4);
1952 __ dec(end_to, 4);
1953 __ ld(end_from, 0, O4);
1954 __ st(O4, end_to, 0);
1955 __ BIND(L_skip_alignment);
1956
1957 // Check if 'end_from' and 'end_to' has the same alignment.
1958 __ andcc(end_from, 7, G0);
1959 __ br(Assembler::zero, false, Assembler::pt, L_aligned_copy);
1960 __ delayed()->dec(count, 4); // The cmp at the start guaranty cnt >= 4
1961
1962 // copy with shift 4 elements (16 bytes) at a time
1963 //
1964 // Load 2 aligned 8-bytes chunks and use one from previous iteration
1965 // to form 2 aligned 8-bytes chunks to store.
1966 //
1967 __ ldx(end_from, -4, O3);
1968 __ align(OptoLoopAlignment);
1969 __ BIND(L_copy_16_bytes);
1970 __ ldx(end_from, -12, O4);
1971 __ deccc(count, 4);
1972 __ ldx(end_from, -20, O5);
1973 __ dec(end_to, 16);
1974 __ dec(end_from, 16);
1975 __ srlx(O3, 32, O3);
1976 __ sllx(O4, 32, G3);
1977 __ bset(G3, O3);
1978 __ stx(O3, end_to, 8);
1979 __ srlx(O4, 32, O4);
1980 __ sllx(O5, 32, G3);
1981 __ bset(O4, G3);
1982 __ stx(G3, end_to, 0);
1983 __ brx(Assembler::greaterEqual, false, Assembler::pt, L_copy_16_bytes);
1984 __ delayed()->mov(O5, O3);
1985
1986 __ br(Assembler::always, false, Assembler::pt, L_copy_4_bytes);
1987 __ delayed()->inc(count, 4);
1988
1989 // copy 4 elements (16 bytes) at a time
1990 __ align(OptoLoopAlignment);
1991 __ BIND(L_aligned_copy);
1992 __ dec(end_from, 16);
1993 __ ldx(end_from, 8, O3);
1994 __ ldx(end_from, 0, O4);
1995 __ dec(end_to, 16);
1996 __ deccc(count, 4);
1997 __ stx(O3, end_to, 8);
1998 __ brx(Assembler::greaterEqual, false, Assembler::pt, L_aligned_copy);
1999 __ delayed()->stx(O4, end_to, 0);
2000 __ inc(count, 4);
2001
2002 // copy 1 element (4 bytes) at a time
2003 __ BIND(L_copy_4_bytes);
2004 __ cmp_and_br_short(count, 0, Assembler::equal, Assembler::pt, L_exit);
2005 __ BIND(L_copy_4_bytes_loop);
2006 __ dec(end_from, 4);
2007 __ dec(end_to, 4);
2008 __ ld(end_from, 0, O4);
2009 __ deccc(count);
2010 __ brx(Assembler::greater, false, Assembler::pt, L_copy_4_bytes_loop);
2011 __ delayed()->st(O4, end_to, 0);
2012 __ BIND(L_exit);
2013 }
2014
2015 //
2016 // Generate stub for conjoint int copy. If "aligned" is true, the
2017 // "from" and "to" addresses are assumed to be heapword aligned.
2018 //
2019 // Arguments for generated stub:
2020 // from: O0
2021 // to: O1
2022 // count: O2 treated as signed
2023 //
2024 address generate_conjoint_int_copy(bool aligned, address nooverlap_target,
2025 address *entry, const char *name) {
2026 __ align(CodeEntryAlignment);
2027 StubCodeMark mark(this, "StubRoutines", name);
2028 address start = __ pc();
2029
2030 assert_clean_int(O2, O3); // Make sure 'count' is clean int.
2031
2032 if (entry != NULL) {
2033 *entry = __ pc();
2034 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2035 BLOCK_COMMENT("Entry:");
2036 }
2037
2038 array_overlap_test(nooverlap_target, 2);
2039 {
2040 // UnsafeCopyMemory page error: continue at UnsafeCopyMemory common_error_exit
2041 UnsafeCopyMemoryMark ucmm(this, !aligned, false);
2042 generate_conjoint_int_copy_core(aligned);
2043 }
2044 // O3, O4 are used as temp registers
2045 inc_counter_np(SharedRuntime::_jint_array_copy_ctr, O3, O4);
2046 __ retl();
2047 __ delayed()->mov(G0, O0); // return 0
2048 return start;
2049 }
2050
2051 //
2052 // Helper methods for generate_disjoint_long_copy_core()
2053 //
2054 void copy_64_bytes_loop(Register from, Register to, Register count, int count_dec,
2055 Label& L_loop, bool use_prefetch, bool use_bis) {
2056 __ align(OptoLoopAlignment);
2057 __ BIND(L_loop);
2058 for (int off = 0; off < 64; off += 16) {
2059 if (use_prefetch && (off & 31) == 0) {
2060 if (ArraycopySrcPrefetchDistance > 0) {
2061 __ prefetch(from, ArraycopySrcPrefetchDistance+off, Assembler::severalReads);
2062 }
2063 if (ArraycopyDstPrefetchDistance > 0) {
2064 __ prefetch(to, ArraycopyDstPrefetchDistance+off, Assembler::severalWritesAndPossiblyReads);
2065 }
2066 }
2067 __ ldx(from, off+0, O4);
2068 __ ldx(from, off+8, O5);
2069 if (use_bis) {
2070 __ stxa(O4, to, off+0);
2071 __ stxa(O5, to, off+8);
2072 } else {
2073 __ stx(O4, to, off+0);
2074 __ stx(O5, to, off+8);
2075 }
2076 }
2077 __ deccc(count, 8);
2078 __ inc(from, 64);
2079 __ brx(Assembler::greaterEqual, false, Assembler::pt, L_loop);
2080 __ delayed()->inc(to, 64);
2081 }
2082
2083 //
2084 // Generate core code for disjoint long copy (and oop copy on 64-bit).
2085 // "aligned" is ignored, because we must make the stronger
2086 // assumption that both addresses are always 64-bit aligned.
2087 //
2088 // Arguments:
2089 // from: O0
2090 // to: O1
2091 // count: O2 treated as signed
2092 //
2093 // count -= 2;
2094 // if ( count >= 0 ) { // >= 2 elements
2095 // if ( count > 6) { // >= 8 elements
2096 // count -= 6; // original count - 8
2097 // do {
2098 // copy_8_elements;
2099 // count -= 8;
2100 // } while ( count >= 0 );
2101 // count += 6;
2102 // }
2103 // if ( count >= 0 ) { // >= 2 elements
2104 // do {
2105 // copy_2_elements;
2106 // } while ( (count=count-2) >= 0 );
2107 // }
2108 // }
2109 // count += 2;
2110 // if ( count != 0 ) { // 1 element left
2111 // copy_1_element;
2112 // }
2113 //
2114 void generate_disjoint_long_copy_core(bool aligned) {
2115 Label L_copy_8_bytes, L_copy_16_bytes, L_exit;
2116 const Register from = O0; // source array address
2117 const Register to = O1; // destination array address
2118 const Register count = O2; // elements count
2119 const Register offset0 = O4; // element offset
2120 const Register offset8 = O5; // next element offset
2121
2122 __ deccc(count, 2);
2123 __ mov(G0, offset0); // offset from start of arrays (0)
2124 __ brx(Assembler::negative, false, Assembler::pn, L_copy_8_bytes );
2125 __ delayed()->add(offset0, 8, offset8);
2126
2127 // Copy by 64 bytes chunks
2128
2129 const Register from64 = O3; // source address
2130 const Register to64 = G3; // destination address
2131 __ subcc(count, 6, O3);
2132 __ brx(Assembler::negative, false, Assembler::pt, L_copy_16_bytes );
2133 __ delayed()->mov(to, to64);
2134 // Now we can use O4(offset0), O5(offset8) as temps
2135 __ mov(O3, count);
2136 // count >= 0 (original count - 8)
2137 __ mov(from, from64);
2138
2139 disjoint_copy_core(from64, to64, count, 3, 64, &StubGenerator::copy_64_bytes_loop);
2140
2141 // Restore O4(offset0), O5(offset8)
2142 __ sub(from64, from, offset0);
2143 __ inccc(count, 6); // restore count
2144 __ brx(Assembler::negative, false, Assembler::pn, L_copy_8_bytes );
2145 __ delayed()->add(offset0, 8, offset8);
2146
2147 // Copy by 16 bytes chunks
2148 __ align(OptoLoopAlignment);
2149 __ BIND(L_copy_16_bytes);
2150 __ ldx(from, offset0, O3);
2151 __ ldx(from, offset8, G3);
2152 __ deccc(count, 2);
2153 __ stx(O3, to, offset0);
2154 __ inc(offset0, 16);
2155 __ stx(G3, to, offset8);
2156 __ brx(Assembler::greaterEqual, false, Assembler::pt, L_copy_16_bytes);
2157 __ delayed()->inc(offset8, 16);
2158
2159 // Copy last 8 bytes
2160 __ BIND(L_copy_8_bytes);
2161 __ inccc(count, 2);
2162 __ brx(Assembler::zero, true, Assembler::pn, L_exit );
2163 __ delayed()->mov(offset0, offset8); // Set O5 used by other stubs
2164 __ ldx(from, offset0, O3);
2165 __ stx(O3, to, offset0);
2166 __ BIND(L_exit);
2167 }
2168
2169 //
2170 // Generate stub for disjoint long copy.
2171 // "aligned" is ignored, because we must make the stronger
2172 // assumption that both addresses are always 64-bit aligned.
2173 //
2174 // Arguments for generated stub:
2175 // from: O0
2176 // to: O1
2177 // count: O2 treated as signed
2178 //
2179 address generate_disjoint_long_copy(bool aligned, address *entry, const char *name) {
2180 __ align(CodeEntryAlignment);
2181 StubCodeMark mark(this, "StubRoutines", name);
2182 address start = __ pc();
2183
2184 assert_clean_int(O2, O3); // Make sure 'count' is clean int.
2185
2186 if (entry != NULL) {
2187 *entry = __ pc();
2188 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2189 BLOCK_COMMENT("Entry:");
2190 }
2191
2192 {
2193 // UnsafeCopyMemory page error: continue at UnsafeCopyMemory common_error_exit
2194 UnsafeCopyMemoryMark ucmm(this, true, false);
2195 generate_disjoint_long_copy_core(aligned);
2196 }
2197 // O3, O4 are used as temp registers
2198 inc_counter_np(SharedRuntime::_jlong_array_copy_ctr, O3, O4);
2199 __ retl();
2200 __ delayed()->mov(G0, O0); // return 0
2201 return start;
2202 }
2203
2204 //
2205 // Generate core code for conjoint long copy (and oop copy on 64-bit).
2206 // "aligned" is ignored, because we must make the stronger
2207 // assumption that both addresses are always 64-bit aligned.
2208 //
2209 // Arguments:
2210 // from: O0
2211 // to: O1
2212 // count: O2 treated as signed
2213 //
2214 void generate_conjoint_long_copy_core(bool aligned) {
2215 // Do reverse copy.
2216 Label L_copy_8_bytes, L_copy_16_bytes, L_exit;
2217 const Register from = O0; // source array address
2218 const Register to = O1; // destination array address
2219 const Register count = O2; // elements count
2220 const Register offset8 = O4; // element offset
2221 const Register offset0 = O5; // previous element offset
2222
2223 __ subcc(count, 1, count);
2224 __ brx(Assembler::lessEqual, false, Assembler::pn, L_copy_8_bytes );
2225 __ delayed()->sllx(count, LogBytesPerLong, offset8);
2226 __ sub(offset8, 8, offset0);
2227 __ align(OptoLoopAlignment);
2228 __ BIND(L_copy_16_bytes);
2229 __ ldx(from, offset8, O2);
2230 __ ldx(from, offset0, O3);
2231 __ stx(O2, to, offset8);
2232 __ deccc(offset8, 16); // use offset8 as counter
2233 __ stx(O3, to, offset0);
2234 __ brx(Assembler::greater, false, Assembler::pt, L_copy_16_bytes);
2235 __ delayed()->dec(offset0, 16);
2236
2237 __ BIND(L_copy_8_bytes);
2238 __ brx(Assembler::negative, false, Assembler::pn, L_exit );
2239 __ delayed()->nop();
2240 __ ldx(from, 0, O3);
2241 __ stx(O3, to, 0);
2242 __ BIND(L_exit);
2243 }
2244
2245 // Generate stub for conjoint long copy.
2246 // "aligned" is ignored, because we must make the stronger
2247 // assumption that both addresses are always 64-bit aligned.
2248 //
2249 // Arguments for generated stub:
2250 // from: O0
2251 // to: O1
2252 // count: O2 treated as signed
2253 //
2254 address generate_conjoint_long_copy(bool aligned, address nooverlap_target,
2255 address *entry, const char *name) {
2256 __ align(CodeEntryAlignment);
2257 StubCodeMark mark(this, "StubRoutines", name);
2258 address start = __ pc();
2259
2260 assert(aligned, "Should always be aligned");
2261
2262 assert_clean_int(O2, O3); // Make sure 'count' is clean int.
2263
2264 if (entry != NULL) {
2265 *entry = __ pc();
2266 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2267 BLOCK_COMMENT("Entry:");
2268 }
2269
2270 array_overlap_test(nooverlap_target, 3);
2271 {
2272 // UnsafeCopyMemory page error: continue at UnsafeCopyMemory common_error_exit
2273 UnsafeCopyMemoryMark ucmm(this, true, false);
2274 generate_conjoint_long_copy_core(aligned);
2275 }
2276 // O3, O4 are used as temp registers
2277 inc_counter_np(SharedRuntime::_jlong_array_copy_ctr, O3, O4);
2278 __ retl();
2279 __ delayed()->mov(G0, O0); // return 0
2280 return start;
2281 }
2282
2283 // Generate stub for disjoint oop copy. If "aligned" is true, the
2284 // "from" and "to" addresses are assumed to be heapword aligned.
2285 //
2286 // Arguments for generated stub:
2287 // from: O0
2288 // to: O1
2289 // count: O2 treated as signed
2290 //
2291 address generate_disjoint_oop_copy(bool aligned, address *entry, const char *name,
2292 bool dest_uninitialized = false) {
2293
2294 const Register from = O0; // source array address
2295 const Register to = O1; // destination array address
2296 const Register count = O2; // elements count
2297
2298 __ align(CodeEntryAlignment);
2299 StubCodeMark mark(this, "StubRoutines", name);
2300 address start = __ pc();
2301
2302 assert_clean_int(count, O3); // Make sure 'count' is clean int.
2303
2304 if (entry != NULL) {
2305 *entry = __ pc();
2306 // caller can pass a 64-bit byte count here
2307 BLOCK_COMMENT("Entry:");
2308 }
2309
2310 DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT;
2311 if (dest_uninitialized) {
2312 decorators |= IS_DEST_UNINITIALIZED;
2313 }
2314 if (aligned) {
2315 decorators |= ARRAYCOPY_ALIGNED;
2316 }
2317
2318 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
2319 bs->arraycopy_prologue(_masm, decorators, T_OBJECT, from, to, count);
2320
2321 assert_clean_int(count, O3); // Make sure 'count' is clean int.
2322 if (UseCompressedOops) {
2323 generate_disjoint_int_copy_core(aligned);
2324 } else {
2325 generate_disjoint_long_copy_core(aligned);
2326 }
2327
2328 bs->arraycopy_epilogue(_masm, decorators, T_OBJECT, from, to, count);
2329
2330 // O3, O4 are used as temp registers
2331 inc_counter_np(SharedRuntime::_oop_array_copy_ctr, O3, O4);
2332 __ retl();
2333 __ delayed()->mov(G0, O0); // return 0
2334 return start;
2335 }
2336
2337 // Generate stub for conjoint oop copy. If "aligned" is true, the
2338 // "from" and "to" addresses are assumed to be heapword aligned.
2339 //
2340 // Arguments for generated stub:
2341 // from: O0
2342 // to: O1
2343 // count: O2 treated as signed
2344 //
2345 address generate_conjoint_oop_copy(bool aligned, address nooverlap_target,
2346 address *entry, const char *name,
2347 bool dest_uninitialized = false) {
2348
2349 const Register from = O0; // source array address
2350 const Register to = O1; // destination array address
2351 const Register count = O2; // elements count
2352
2353 __ align(CodeEntryAlignment);
2354 StubCodeMark mark(this, "StubRoutines", name);
2355 address start = __ pc();
2356
2357 assert_clean_int(count, O3); // Make sure 'count' is clean int.
2358
2359 if (entry != NULL) {
2360 *entry = __ pc();
2361 // caller can pass a 64-bit byte count here
2362 BLOCK_COMMENT("Entry:");
2363 }
2364
2365 array_overlap_test(nooverlap_target, LogBytesPerHeapOop);
2366
2367 DecoratorSet decorators = IN_HEAP | IS_ARRAY;
2368 if (dest_uninitialized) {
2369 decorators |= IS_DEST_UNINITIALIZED;
2370 }
2371 if (aligned) {
2372 decorators |= ARRAYCOPY_ALIGNED;
2373 }
2374
2375 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
2376 bs->arraycopy_prologue(_masm, decorators, T_OBJECT, from, to, count);
2377
2378 if (UseCompressedOops) {
2379 generate_conjoint_int_copy_core(aligned);
2380 } else {
2381 generate_conjoint_long_copy_core(aligned);
2382 }
2383
2384 bs->arraycopy_epilogue(_masm, decorators, T_OBJECT, from, to, count);
2385
2386 // O3, O4 are used as temp registers
2387 inc_counter_np(SharedRuntime::_oop_array_copy_ctr, O3, O4);
2388 __ retl();
2389 __ delayed()->mov(G0, O0); // return 0
2390 return start;
2391 }
2392
2393
2394 // Helper for generating a dynamic type check.
2395 // Smashes only the given temp registers.
2396 void generate_type_check(Register sub_klass,
2397 Register super_check_offset,
2398 Register super_klass,
2399 Register temp,
2400 Label& L_success) {
2401 assert_different_registers(sub_klass, super_check_offset, super_klass, temp);
2402
2403 BLOCK_COMMENT("type_check:");
2404
2405 Label L_miss, L_pop_to_miss;
2406
2407 assert_clean_int(super_check_offset, temp);
2408
2409 __ check_klass_subtype_fast_path(sub_klass, super_klass, temp, noreg,
2410 &L_success, &L_miss, NULL,
2411 super_check_offset);
2412
2413 BLOCK_COMMENT("type_check_slow_path:");
2414 __ save_frame(0);
2415 __ check_klass_subtype_slow_path(sub_klass->after_save(),
2416 super_klass->after_save(),
2417 L0, L1, L2, L4,
2418 NULL, &L_pop_to_miss);
2419 __ ba(L_success);
2420 __ delayed()->restore();
2421
2422 __ bind(L_pop_to_miss);
2423 __ restore();
2424
2425 // Fall through on failure!
2426 __ BIND(L_miss);
2427 }
2428
2429
2430 // Generate stub for checked oop copy.
2431 //
2432 // Arguments for generated stub:
2433 // from: O0
2434 // to: O1
2435 // count: O2 treated as signed
2436 // ckoff: O3 (super_check_offset)
2437 // ckval: O4 (super_klass)
2438 // ret: O0 zero for success; (-1^K) where K is partial transfer count
2439 //
2440 address generate_checkcast_copy(const char *name, address *entry, bool dest_uninitialized = false) {
2441
2442 const Register O0_from = O0; // source array address
2443 const Register O1_to = O1; // destination array address
2444 const Register O2_count = O2; // elements count
2445 const Register O3_ckoff = O3; // super_check_offset
2446 const Register O4_ckval = O4; // super_klass
2447
2448 const Register O5_offset = O5; // loop var, with stride wordSize
2449 const Register G1_remain = G1; // loop var, with stride -1
2450 const Register G3_oop = G3; // actual oop copied
2451 const Register G4_klass = G4; // oop._klass
2452 const Register G5_super = G5; // oop._klass._primary_supers[ckval]
2453
2454 __ align(CodeEntryAlignment);
2455 StubCodeMark mark(this, "StubRoutines", name);
2456 address start = __ pc();
2457
2458 #ifdef ASSERT
2459 // We sometimes save a frame (see generate_type_check below).
2460 // If this will cause trouble, let's fail now instead of later.
2461 __ save_frame(0);
2462 __ restore();
2463 #endif
2464
2465 assert_clean_int(O2_count, G1); // Make sure 'count' is clean int.
2466
2467 #ifdef ASSERT
2468 // caller guarantees that the arrays really are different
2469 // otherwise, we would have to make conjoint checks
2470 { Label L;
2471 __ mov(O3, G1); // spill: overlap test smashes O3
2472 __ mov(O4, G4); // spill: overlap test smashes O4
2473 array_overlap_test(L, LogBytesPerHeapOop);
2474 __ stop("checkcast_copy within a single array");
2475 __ bind(L);
2476 __ mov(G1, O3);
2477 __ mov(G4, O4);
2478 }
2479 #endif //ASSERT
2480
2481 if (entry != NULL) {
2482 *entry = __ pc();
2483 // caller can pass a 64-bit byte count here (from generic stub)
2484 BLOCK_COMMENT("Entry:");
2485 }
2486
2487 DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_CHECKCAST;
2488 if (dest_uninitialized) {
2489 decorators |= IS_DEST_UNINITIALIZED;
2490 }
2491
2492 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
2493 bs->arraycopy_prologue(_masm, decorators, T_OBJECT, O0_from, O1_to, O2_count);
2494
2495 Label load_element, store_element, do_epilogue, fail, done;
2496 __ addcc(O2_count, 0, G1_remain); // initialize loop index, and test it
2497 __ brx(Assembler::notZero, false, Assembler::pt, load_element);
2498 __ delayed()->mov(G0, O5_offset); // offset from start of arrays
2499
2500 // Empty array: Nothing to do.
2501 inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr, O3, O4);
2502 __ retl();
2503 __ delayed()->set(0, O0); // return 0 on (trivial) success
2504
2505 // ======== begin loop ========
2506 // (Loop is rotated; its entry is load_element.)
2507 // Loop variables:
2508 // (O5 = 0; ; O5 += wordSize) --- offset from src, dest arrays
2509 // (O2 = len; O2 != 0; O2--) --- number of oops *remaining*
2510 // G3, G4, G5 --- current oop, oop.klass, oop.klass.super
2511 __ align(OptoLoopAlignment);
2512
2513 __ BIND(store_element);
2514 __ deccc(G1_remain); // decrement the count
2515 __ store_heap_oop(G3_oop, O1_to, O5_offset, noreg, AS_RAW); // store the oop
2516 __ inc(O5_offset, heapOopSize); // step to next offset
2517 __ brx(Assembler::zero, true, Assembler::pt, do_epilogue);
2518 __ delayed()->set(0, O0); // return -1 on success
2519
2520 // ======== loop entry is here ========
2521 __ BIND(load_element);
2522 __ load_heap_oop(O0_from, O5_offset, G3_oop, noreg, AS_RAW); // load the oop
2523 __ br_null_short(G3_oop, Assembler::pt, store_element);
2524
2525 __ load_klass(G3_oop, G4_klass); // query the object klass
2526
2527 generate_type_check(G4_klass, O3_ckoff, O4_ckval, G5_super,
2528 // branch to this on success:
2529 store_element);
2530 // ======== end loop ========
2531
2532 // It was a real error; we must depend on the caller to finish the job.
2533 // Register G1 has number of *remaining* oops, O2 number of *total* oops.
2534 // Emit GC store barriers for the oops we have copied (O2 minus G1),
2535 // and report their number to the caller.
2536 __ BIND(fail);
2537 __ subcc(O2_count, G1_remain, O2_count);
2538 __ brx(Assembler::zero, false, Assembler::pt, done);
2539 __ delayed()->not1(O2_count, O0); // report (-1^K) to caller
2540
2541 __ BIND(do_epilogue);
2542 bs->arraycopy_epilogue(_masm, decorators, T_OBJECT, O0_from, O1_to, O2_count);
2543
2544 __ BIND(done);
2545 inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr, O3, O4);
2546 __ retl();
2547 __ delayed()->nop(); // return value in 00
2548
2549 return start;
2550 }
2551
2552
2553 // Generate 'unsafe' array copy stub
2554 // Though just as safe as the other stubs, it takes an unscaled
2555 // size_t argument instead of an element count.
2556 //
2557 // Arguments for generated stub:
2558 // from: O0
2559 // to: O1
2560 // count: O2 byte count, treated as ssize_t, can be zero
2561 //
2562 // Examines the alignment of the operands and dispatches
2563 // to a long, int, short, or byte copy loop.
2564 //
2565 address generate_unsafe_copy(const char* name,
2566 address byte_copy_entry,
2567 address short_copy_entry,
2568 address int_copy_entry,
2569 address long_copy_entry) {
2570
2571 const Register O0_from = O0; // source array address
2572 const Register O1_to = O1; // destination array address
2573 const Register O2_count = O2; // elements count
2574
2575 const Register G1_bits = G1; // test copy of low bits
2576
2577 __ align(CodeEntryAlignment);
2578 StubCodeMark mark(this, "StubRoutines", name);
2579 address start = __ pc();
2580
2581 // bump this on entry, not on exit:
2582 inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr, G1, G3);
2583
2584 __ or3(O0_from, O1_to, G1_bits);
2585 __ or3(O2_count, G1_bits, G1_bits);
2586
2587 __ btst(BytesPerLong-1, G1_bits);
2588 __ br(Assembler::zero, true, Assembler::pt,
2589 long_copy_entry, relocInfo::runtime_call_type);
2590 // scale the count on the way out:
2591 __ delayed()->srax(O2_count, LogBytesPerLong, O2_count);
2592
2593 __ btst(BytesPerInt-1, G1_bits);
2594 __ br(Assembler::zero, true, Assembler::pt,
2595 int_copy_entry, relocInfo::runtime_call_type);
2596 // scale the count on the way out:
2597 __ delayed()->srax(O2_count, LogBytesPerInt, O2_count);
2598
2599 __ btst(BytesPerShort-1, G1_bits);
2600 __ br(Assembler::zero, true, Assembler::pt,
2601 short_copy_entry, relocInfo::runtime_call_type);
2602 // scale the count on the way out:
2603 __ delayed()->srax(O2_count, LogBytesPerShort, O2_count);
2604
2605 __ br(Assembler::always, false, Assembler::pt,
2606 byte_copy_entry, relocInfo::runtime_call_type);
2607 __ delayed()->nop();
2608
2609 return start;
2610 }
2611
2612
2613 // Perform range checks on the proposed arraycopy.
2614 // Kills the two temps, but nothing else.
2615 // Also, clean the sign bits of src_pos and dst_pos.
2616 void arraycopy_range_checks(Register src, // source array oop (O0)
2617 Register src_pos, // source position (O1)
2618 Register dst, // destination array oo (O2)
2619 Register dst_pos, // destination position (O3)
2620 Register length, // length of copy (O4)
2621 Register temp1, Register temp2,
2622 Label& L_failed) {
2623 BLOCK_COMMENT("arraycopy_range_checks:");
2624
2625 // if (src_pos + length > arrayOop(src)->length() ) FAIL;
2626
2627 const Register array_length = temp1; // scratch
2628 const Register end_pos = temp2; // scratch
2629
2630 // Note: This next instruction may be in the delay slot of a branch:
2631 __ add(length, src_pos, end_pos); // src_pos + length
2632 __ lduw(src, arrayOopDesc::length_offset_in_bytes(), array_length);
2633 __ cmp(end_pos, array_length);
2634 __ br(Assembler::greater, false, Assembler::pn, L_failed);
2635
2636 // if (dst_pos + length > arrayOop(dst)->length() ) FAIL;
2637 __ delayed()->add(length, dst_pos, end_pos); // dst_pos + length
2638 __ lduw(dst, arrayOopDesc::length_offset_in_bytes(), array_length);
2639 __ cmp(end_pos, array_length);
2640 __ br(Assembler::greater, false, Assembler::pn, L_failed);
2641
2642 // Have to clean up high 32-bits of 'src_pos' and 'dst_pos'.
2643 // Move with sign extension can be used since they are positive.
2644 __ delayed()->signx(src_pos, src_pos);
2645 __ signx(dst_pos, dst_pos);
2646
2647 BLOCK_COMMENT("arraycopy_range_checks done");
2648 }
2649
2650
2651 //
2652 // Generate generic array copy stubs
2653 //
2654 // Input:
2655 // O0 - src oop
2656 // O1 - src_pos
2657 // O2 - dst oop
2658 // O3 - dst_pos
2659 // O4 - element count
2660 //
2661 // Output:
2662 // O0 == 0 - success
2663 // O0 == -1 - need to call System.arraycopy
2664 //
2665 address generate_generic_copy(const char *name,
2666 address entry_jbyte_arraycopy,
2667 address entry_jshort_arraycopy,
2668 address entry_jint_arraycopy,
2669 address entry_oop_arraycopy,
2670 address entry_jlong_arraycopy,
2671 address entry_checkcast_arraycopy) {
2672 Label L_failed, L_objArray;
2673
2674 // Input registers
2675 const Register src = O0; // source array oop
2676 const Register src_pos = O1; // source position
2677 const Register dst = O2; // destination array oop
2678 const Register dst_pos = O3; // destination position
2679 const Register length = O4; // elements count
2680
2681 // registers used as temp
2682 const Register G3_src_klass = G3; // source array klass
2683 const Register G4_dst_klass = G4; // destination array klass
2684 const Register G5_lh = G5; // layout handler
2685 const Register O5_temp = O5;
2686
2687 __ align(CodeEntryAlignment);
2688 StubCodeMark mark(this, "StubRoutines", name);
2689 address start = __ pc();
2690
2691 // bump this on entry, not on exit:
2692 inc_counter_np(SharedRuntime::_generic_array_copy_ctr, G1, G3);
2693
2694 // In principle, the int arguments could be dirty.
2695 //assert_clean_int(src_pos, G1);
2696 //assert_clean_int(dst_pos, G1);
2697 //assert_clean_int(length, G1);
2698
2699 //-----------------------------------------------------------------------
2700 // Assembler stubs will be used for this call to arraycopy
2701 // if the following conditions are met:
2702 //
2703 // (1) src and dst must not be null.
2704 // (2) src_pos must not be negative.
2705 // (3) dst_pos must not be negative.
2706 // (4) length must not be negative.
2707 // (5) src klass and dst klass should be the same and not NULL.
2708 // (6) src and dst should be arrays.
2709 // (7) src_pos + length must not exceed length of src.
2710 // (8) dst_pos + length must not exceed length of dst.
2711 BLOCK_COMMENT("arraycopy initial argument checks");
2712
2713 // if (src == NULL) return -1;
2714 __ br_null(src, false, Assembler::pn, L_failed);
2715
2716 // if (src_pos < 0) return -1;
2717 __ delayed()->tst(src_pos);
2718 __ br(Assembler::negative, false, Assembler::pn, L_failed);
2719 __ delayed()->nop();
2720
2721 // if (dst == NULL) return -1;
2722 __ br_null(dst, false, Assembler::pn, L_failed);
2723
2724 // if (dst_pos < 0) return -1;
2725 __ delayed()->tst(dst_pos);
2726 __ br(Assembler::negative, false, Assembler::pn, L_failed);
2727
2728 // if (length < 0) return -1;
2729 __ delayed()->tst(length);
2730 __ br(Assembler::negative, false, Assembler::pn, L_failed);
2731
2732 BLOCK_COMMENT("arraycopy argument klass checks");
2733 // get src->klass()
2734 if (UseCompressedClassPointers) {
2735 __ delayed()->nop(); // ??? not good
2736 __ load_klass(src, G3_src_klass);
2737 } else {
2738 __ delayed()->ld_ptr(src, oopDesc::klass_offset_in_bytes(), G3_src_klass);
2739 }
2740
2741 #ifdef ASSERT
2742 // assert(src->klass() != NULL);
2743 BLOCK_COMMENT("assert klasses not null");
2744 { Label L_a, L_b;
2745 __ br_notnull_short(G3_src_klass, Assembler::pt, L_b); // it is broken if klass is NULL
2746 __ bind(L_a);
2747 __ stop("broken null klass");
2748 __ bind(L_b);
2749 __ load_klass(dst, G4_dst_klass);
2750 __ br_null(G4_dst_klass, false, Assembler::pn, L_a); // this would be broken also
2751 __ delayed()->mov(G0, G4_dst_klass); // scribble the temp
2752 BLOCK_COMMENT("assert done");
2753 }
2754 #endif
2755
2756 // Load layout helper
2757 //
2758 // |array_tag| | header_size | element_type | |log2_element_size|
2759 // 32 30 24 16 8 2 0
2760 //
2761 // array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
2762 //
2763
2764 int lh_offset = in_bytes(Klass::layout_helper_offset());
2765
2766 // Load 32-bits signed value. Use br() instruction with it to check icc.
2767 __ lduw(G3_src_klass, lh_offset, G5_lh);
2768
2769 if (UseCompressedClassPointers) {
2770 __ load_klass(dst, G4_dst_klass);
2771 }
2772 // Handle objArrays completely differently...
2773 juint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2774 __ set(objArray_lh, O5_temp);
2775 __ cmp(G5_lh, O5_temp);
2776 __ br(Assembler::equal, false, Assembler::pt, L_objArray);
2777 if (UseCompressedClassPointers) {
2778 __ delayed()->nop();
2779 } else {
2780 __ delayed()->ld_ptr(dst, oopDesc::klass_offset_in_bytes(), G4_dst_klass);
2781 }
2782
2783 // if (src->klass() != dst->klass()) return -1;
2784 __ cmp_and_brx_short(G3_src_klass, G4_dst_klass, Assembler::notEqual, Assembler::pn, L_failed);
2785
2786 // if (!src->is_Array()) return -1;
2787 __ cmp(G5_lh, Klass::_lh_neutral_value); // < 0
2788 __ br(Assembler::greaterEqual, false, Assembler::pn, L_failed);
2789
2790 // At this point, it is known to be a typeArray (array_tag 0x3).
2791 #ifdef ASSERT
2792 __ delayed()->nop();
2793 { Label L;
2794 jint lh_prim_tag_in_place = (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift);
2795 __ set(lh_prim_tag_in_place, O5_temp);
2796 __ cmp(G5_lh, O5_temp);
2797 __ br(Assembler::greaterEqual, false, Assembler::pt, L);
2798 __ delayed()->nop();
2799 __ stop("must be a primitive array");
2800 __ bind(L);
2801 }
2802 #else
2803 __ delayed(); // match next insn to prev branch
2804 #endif
2805
2806 arraycopy_range_checks(src, src_pos, dst, dst_pos, length,
2807 O5_temp, G4_dst_klass, L_failed);
2808
2809 // TypeArrayKlass
2810 //
2811 // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize);
2812 // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize);
2813 //
2814
2815 const Register G4_offset = G4_dst_klass; // array offset
2816 const Register G3_elsize = G3_src_klass; // log2 element size
2817
2818 __ srl(G5_lh, Klass::_lh_header_size_shift, G4_offset);
2819 __ and3(G4_offset, Klass::_lh_header_size_mask, G4_offset); // array_offset
2820 __ add(src, G4_offset, src); // src array offset
2821 __ add(dst, G4_offset, dst); // dst array offset
2822 __ and3(G5_lh, Klass::_lh_log2_element_size_mask, G3_elsize); // log2 element size
2823
2824 // next registers should be set before the jump to corresponding stub
2825 const Register from = O0; // source array address
2826 const Register to = O1; // destination array address
2827 const Register count = O2; // elements count
2828
2829 // 'from', 'to', 'count' registers should be set in this order
2830 // since they are the same as 'src', 'src_pos', 'dst'.
2831
2832 BLOCK_COMMENT("scale indexes to element size");
2833 __ sll_ptr(src_pos, G3_elsize, src_pos);
2834 __ sll_ptr(dst_pos, G3_elsize, dst_pos);
2835 __ add(src, src_pos, from); // src_addr
2836 __ add(dst, dst_pos, to); // dst_addr
2837
2838 BLOCK_COMMENT("choose copy loop based on element size");
2839 __ cmp(G3_elsize, 0);
2840 __ br(Assembler::equal, true, Assembler::pt, entry_jbyte_arraycopy);
2841 __ delayed()->signx(length, count); // length
2842
2843 __ cmp(G3_elsize, LogBytesPerShort);
2844 __ br(Assembler::equal, true, Assembler::pt, entry_jshort_arraycopy);
2845 __ delayed()->signx(length, count); // length
2846
2847 __ cmp(G3_elsize, LogBytesPerInt);
2848 __ br(Assembler::equal, true, Assembler::pt, entry_jint_arraycopy);
2849 __ delayed()->signx(length, count); // length
2850 #ifdef ASSERT
2851 { Label L;
2852 __ cmp_and_br_short(G3_elsize, LogBytesPerLong, Assembler::equal, Assembler::pt, L);
2853 __ stop("must be long copy, but elsize is wrong");
2854 __ bind(L);
2855 }
2856 #endif
2857 __ br(Assembler::always, false, Assembler::pt, entry_jlong_arraycopy);
2858 __ delayed()->signx(length, count); // length
2859
2860 // ObjArrayKlass
2861 __ BIND(L_objArray);
2862 // live at this point: G3_src_klass, G4_dst_klass, src[_pos], dst[_pos], length
2863
2864 Label L_plain_copy, L_checkcast_copy;
2865 // test array classes for subtyping
2866 __ cmp(G3_src_klass, G4_dst_klass); // usual case is exact equality
2867 __ brx(Assembler::notEqual, true, Assembler::pn, L_checkcast_copy);
2868 __ delayed()->lduw(G4_dst_klass, lh_offset, O5_temp); // hoisted from below
2869
2870 // Identically typed arrays can be copied without element-wise checks.
2871 arraycopy_range_checks(src, src_pos, dst, dst_pos, length,
2872 O5_temp, G5_lh, L_failed);
2873
2874 __ add(src, arrayOopDesc::base_offset_in_bytes(T_OBJECT), src); //src offset
2875 __ add(dst, arrayOopDesc::base_offset_in_bytes(T_OBJECT), dst); //dst offset
2876 __ sll_ptr(src_pos, LogBytesPerHeapOop, src_pos);
2877 __ sll_ptr(dst_pos, LogBytesPerHeapOop, dst_pos);
2878 __ add(src, src_pos, from); // src_addr
2879 __ add(dst, dst_pos, to); // dst_addr
2880 __ BIND(L_plain_copy);
2881 __ br(Assembler::always, false, Assembler::pt, entry_oop_arraycopy);
2882 __ delayed()->signx(length, count); // length
2883
2884 __ BIND(L_checkcast_copy);
2885 // live at this point: G3_src_klass, G4_dst_klass
2886 {
2887 // Before looking at dst.length, make sure dst is also an objArray.
2888 // lduw(G4_dst_klass, lh_offset, O5_temp); // hoisted to delay slot
2889 __ cmp(G5_lh, O5_temp);
2890 __ br(Assembler::notEqual, false, Assembler::pn, L_failed);
2891
2892 // It is safe to examine both src.length and dst.length.
2893 __ delayed(); // match next insn to prev branch
2894 arraycopy_range_checks(src, src_pos, dst, dst_pos, length,
2895 O5_temp, G5_lh, L_failed);
2896
2897 // Marshal the base address arguments now, freeing registers.
2898 __ add(src, arrayOopDesc::base_offset_in_bytes(T_OBJECT), src); //src offset
2899 __ add(dst, arrayOopDesc::base_offset_in_bytes(T_OBJECT), dst); //dst offset
2900 __ sll_ptr(src_pos, LogBytesPerHeapOop, src_pos);
2901 __ sll_ptr(dst_pos, LogBytesPerHeapOop, dst_pos);
2902 __ add(src, src_pos, from); // src_addr
2903 __ add(dst, dst_pos, to); // dst_addr
2904 __ signx(length, count); // length (reloaded)
2905
2906 Register sco_temp = O3; // this register is free now
2907 assert_different_registers(from, to, count, sco_temp,
2908 G4_dst_klass, G3_src_klass);
2909
2910 // Generate the type check.
2911 int sco_offset = in_bytes(Klass::super_check_offset_offset());
2912 __ lduw(G4_dst_klass, sco_offset, sco_temp);
2913 generate_type_check(G3_src_klass, sco_temp, G4_dst_klass,
2914 O5_temp, L_plain_copy);
2915
2916 // Fetch destination element klass from the ObjArrayKlass header.
2917 int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
2918
2919 // the checkcast_copy loop needs two extra arguments:
2920 __ ld_ptr(G4_dst_klass, ek_offset, O4); // dest elem klass
2921 // lduw(O4, sco_offset, O3); // sco of elem klass
2922
2923 __ br(Assembler::always, false, Assembler::pt, entry_checkcast_arraycopy);
2924 __ delayed()->lduw(O4, sco_offset, O3);
2925 }
2926
2927 __ BIND(L_failed);
2928 __ retl();
2929 __ delayed()->sub(G0, 1, O0); // return -1
2930 return start;
2931 }
2932
2933 //
2934 // Generate stub for heap zeroing.
2935 // "to" address is aligned to jlong (8 bytes).
2936 //
2937 // Arguments for generated stub:
2938 // to: O0
2939 // count: O1 treated as signed (count of HeapWord)
2940 // count could be 0
2941 //
2942 address generate_zero_aligned_words(const char* name) {
2943 __ align(CodeEntryAlignment);
2944 StubCodeMark mark(this, "StubRoutines", name);
2945 address start = __ pc();
2946
2947 const Register to = O0; // source array address
2948 const Register count = O1; // HeapWords count
2949 const Register temp = O2; // scratch
2950
2951 Label Ldone;
2952 __ sllx(count, LogHeapWordSize, count); // to bytes count
2953 // Use BIS for zeroing
2954 __ bis_zeroing(to, count, temp, Ldone);
2955 __ bind(Ldone);
2956 __ retl();
2957 __ delayed()->nop();
2958 return start;
2959 }
2960
2961 void generate_arraycopy_stubs() {
2962 address entry;
2963 address entry_jbyte_arraycopy;
2964 address entry_jshort_arraycopy;
2965 address entry_jint_arraycopy;
2966 address entry_oop_arraycopy;
2967 address entry_jlong_arraycopy;
2968 address entry_checkcast_arraycopy;
2969
2970 address ucm_common_error_exit = generate_unsafecopy_common_error_exit();
2971 UnsafeCopyMemory::set_common_exit_stub_pc(ucm_common_error_exit);
2972
2973 //*** jbyte
2974 // Always need aligned and unaligned versions
2975 StubRoutines::_jbyte_disjoint_arraycopy = generate_disjoint_byte_copy(false, &entry,
2976 "jbyte_disjoint_arraycopy");
2977 StubRoutines::_jbyte_arraycopy = generate_conjoint_byte_copy(false, entry,
2978 &entry_jbyte_arraycopy,
2979 "jbyte_arraycopy");
2980 StubRoutines::_arrayof_jbyte_disjoint_arraycopy = generate_disjoint_byte_copy(true, &entry,
2981 "arrayof_jbyte_disjoint_arraycopy");
2982 StubRoutines::_arrayof_jbyte_arraycopy = generate_conjoint_byte_copy(true, entry, NULL,
2983 "arrayof_jbyte_arraycopy");
2984
2985 //*** jshort
2986 // Always need aligned and unaligned versions
2987 StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_short_copy(false, &entry,
2988 "jshort_disjoint_arraycopy");
2989 StubRoutines::_jshort_arraycopy = generate_conjoint_short_copy(false, entry,
2990 &entry_jshort_arraycopy,
2991 "jshort_arraycopy");
2992 StubRoutines::_arrayof_jshort_disjoint_arraycopy = generate_disjoint_short_copy(true, &entry,
2993 "arrayof_jshort_disjoint_arraycopy");
2994 StubRoutines::_arrayof_jshort_arraycopy = generate_conjoint_short_copy(true, entry, NULL,
2995 "arrayof_jshort_arraycopy");
2996
2997 //*** jint
2998 // Aligned versions
2999 StubRoutines::_arrayof_jint_disjoint_arraycopy = generate_disjoint_int_copy(true, &entry,
3000 "arrayof_jint_disjoint_arraycopy");
3001 StubRoutines::_arrayof_jint_arraycopy = generate_conjoint_int_copy(true, entry, &entry_jint_arraycopy,
3002 "arrayof_jint_arraycopy");
3003 // In 64 bit we need both aligned and unaligned versions of jint arraycopy.
3004 // entry_jint_arraycopy always points to the unaligned version (notice that we overwrite it).
3005 StubRoutines::_jint_disjoint_arraycopy = generate_disjoint_int_copy(false, &entry,
3006 "jint_disjoint_arraycopy");
3007 StubRoutines::_jint_arraycopy = generate_conjoint_int_copy(false, entry,
3008 &entry_jint_arraycopy,
3009 "jint_arraycopy");
3010
3011 //*** jlong
3012 // It is always aligned
3013 StubRoutines::_arrayof_jlong_disjoint_arraycopy = generate_disjoint_long_copy(true, &entry,
3014 "arrayof_jlong_disjoint_arraycopy");
3015 StubRoutines::_arrayof_jlong_arraycopy = generate_conjoint_long_copy(true, entry, &entry_jlong_arraycopy,
3016 "arrayof_jlong_arraycopy");
3017 StubRoutines::_jlong_disjoint_arraycopy = StubRoutines::_arrayof_jlong_disjoint_arraycopy;
3018 StubRoutines::_jlong_arraycopy = StubRoutines::_arrayof_jlong_arraycopy;
3019
3020
3021 //*** oops
3022 // Aligned versions
3023 StubRoutines::_arrayof_oop_disjoint_arraycopy = generate_disjoint_oop_copy(true, &entry,
3024 "arrayof_oop_disjoint_arraycopy");
3025 StubRoutines::_arrayof_oop_arraycopy = generate_conjoint_oop_copy(true, entry, &entry_oop_arraycopy,
3026 "arrayof_oop_arraycopy");
3027 // Aligned versions without pre-barriers
3028 StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit = generate_disjoint_oop_copy(true, &entry,
3029 "arrayof_oop_disjoint_arraycopy_uninit",
3030 /*dest_uninitialized*/true);
3031 StubRoutines::_arrayof_oop_arraycopy_uninit = generate_conjoint_oop_copy(true, entry, NULL,
3032 "arrayof_oop_arraycopy_uninit",
3033 /*dest_uninitialized*/true);
3034 if (UseCompressedOops) {
3035 // With compressed oops we need unaligned versions, notice that we overwrite entry_oop_arraycopy.
3036 StubRoutines::_oop_disjoint_arraycopy = generate_disjoint_oop_copy(false, &entry,
3037 "oop_disjoint_arraycopy");
3038 StubRoutines::_oop_arraycopy = generate_conjoint_oop_copy(false, entry, &entry_oop_arraycopy,
3039 "oop_arraycopy");
3040 // Unaligned versions without pre-barriers
3041 StubRoutines::_oop_disjoint_arraycopy_uninit = generate_disjoint_oop_copy(false, &entry,
3042 "oop_disjoint_arraycopy_uninit",
3043 /*dest_uninitialized*/true);
3044 StubRoutines::_oop_arraycopy_uninit = generate_conjoint_oop_copy(false, entry, NULL,
3045 "oop_arraycopy_uninit",
3046 /*dest_uninitialized*/true);
3047 } else {
3048 // oop arraycopy is always aligned on 32bit and 64bit without compressed oops
3049 StubRoutines::_oop_disjoint_arraycopy = StubRoutines::_arrayof_oop_disjoint_arraycopy;
3050 StubRoutines::_oop_arraycopy = StubRoutines::_arrayof_oop_arraycopy;
3051 StubRoutines::_oop_disjoint_arraycopy_uninit = StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit;
3052 StubRoutines::_oop_arraycopy_uninit = StubRoutines::_arrayof_oop_arraycopy_uninit;
3053 }
3054
3055 StubRoutines::_checkcast_arraycopy = generate_checkcast_copy("checkcast_arraycopy", &entry_checkcast_arraycopy);
3056 StubRoutines::_checkcast_arraycopy_uninit = generate_checkcast_copy("checkcast_arraycopy_uninit", NULL,
3057 /*dest_uninitialized*/true);
3058
3059 StubRoutines::_unsafe_arraycopy = generate_unsafe_copy("unsafe_arraycopy",
3060 entry_jbyte_arraycopy,
3061 entry_jshort_arraycopy,
3062 entry_jint_arraycopy,
3063 entry_jlong_arraycopy);
3064 StubRoutines::_generic_arraycopy = generate_generic_copy("generic_arraycopy",
3065 entry_jbyte_arraycopy,
3066 entry_jshort_arraycopy,
3067 entry_jint_arraycopy,
3068 entry_oop_arraycopy,
3069 entry_jlong_arraycopy,
3070 entry_checkcast_arraycopy);
3071
3072 StubRoutines::_jbyte_fill = generate_fill(T_BYTE, false, "jbyte_fill");
3073 StubRoutines::_jshort_fill = generate_fill(T_SHORT, false, "jshort_fill");
3074 StubRoutines::_jint_fill = generate_fill(T_INT, false, "jint_fill");
3075 StubRoutines::_arrayof_jbyte_fill = generate_fill(T_BYTE, true, "arrayof_jbyte_fill");
3076 StubRoutines::_arrayof_jshort_fill = generate_fill(T_SHORT, true, "arrayof_jshort_fill");
3077 StubRoutines::_arrayof_jint_fill = generate_fill(T_INT, true, "arrayof_jint_fill");
3078
3079 if (UseBlockZeroing) {
3080 StubRoutines::_zero_aligned_words = generate_zero_aligned_words("zero_aligned_words");
3081 }
3082 }
3083
3084 address generate_aescrypt_encryptBlock() {
3085 // required since we read expanded key 'int' array starting first element without alignment considerations
3086 assert((arrayOopDesc::base_offset_in_bytes(T_INT) & 7) == 0,
3087 "the following code assumes that first element of an int array is aligned to 8 bytes");
3088 __ align(CodeEntryAlignment);
3089 StubCodeMark mark(this, "StubRoutines", "aescrypt_encryptBlock");
3090 Label L_load_misaligned_input, L_load_expanded_key, L_doLast128bit, L_storeOutput, L_store_misaligned_output;
3091 address start = __ pc();
3092 Register from = O0; // source byte array
3093 Register to = O1; // destination byte array
3094 Register key = O2; // expanded key array
3095 const Register keylen = O4; //reg for storing expanded key array length
3096
3097 // read expanded key length
3098 __ ldsw(Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)), keylen, 0);
3099
3100 // Method to address arbitrary alignment for load instructions:
3101 // Check last 3 bits of 'from' address to see if it is aligned to 8-byte boundary
3102 // If zero/aligned then continue with double FP load instructions
3103 // If not zero/mis-aligned then alignaddr will set GSR.align with number of bytes to skip during faligndata
3104 // alignaddr will also convert arbitrary aligned 'from' address to nearest 8-byte aligned address
3105 // load 3 * 8-byte components (to read 16 bytes input) in 3 different FP regs starting at this aligned address
3106 // faligndata will then extract (based on GSR.align value) the appropriate 8 bytes from the 2 source regs
3107
3108 // check for 8-byte alignment since source byte array may have an arbitrary alignment if offset mod 8 is non-zero
3109 __ andcc(from, 7, G0);
3110 __ br(Assembler::notZero, true, Assembler::pn, L_load_misaligned_input);
3111 __ delayed()->alignaddr(from, G0, from);
3112
3113 // aligned case: load input into F54-F56
3114 __ ldf(FloatRegisterImpl::D, from, 0, F54);
3115 __ ldf(FloatRegisterImpl::D, from, 8, F56);
3116 __ ba_short(L_load_expanded_key);
3117
3118 __ BIND(L_load_misaligned_input);
3119 __ ldf(FloatRegisterImpl::D, from, 0, F54);
3120 __ ldf(FloatRegisterImpl::D, from, 8, F56);
3121 __ ldf(FloatRegisterImpl::D, from, 16, F58);
3122 __ faligndata(F54, F56, F54);
3123 __ faligndata(F56, F58, F56);
3124
3125 __ BIND(L_load_expanded_key);
3126 // Since we load expanded key buffers starting first element, 8-byte alignment is guaranteed
3127 for ( int i = 0; i <= 38; i += 2 ) {
3128 __ ldf(FloatRegisterImpl::D, key, i*4, as_FloatRegister(i));
3129 }
3130
3131 // perform cipher transformation
3132 __ fxor(FloatRegisterImpl::D, F0, F54, F54);
3133 __ fxor(FloatRegisterImpl::D, F2, F56, F56);
3134 // rounds 1 through 8
3135 for ( int i = 4; i <= 28; i += 8 ) {
3136 __ aes_eround01(as_FloatRegister(i), F54, F56, F58);
3137 __ aes_eround23(as_FloatRegister(i+2), F54, F56, F60);
3138 __ aes_eround01(as_FloatRegister(i+4), F58, F60, F54);
3139 __ aes_eround23(as_FloatRegister(i+6), F58, F60, F56);
3140 }
3141 __ aes_eround01(F36, F54, F56, F58); //round 9
3142 __ aes_eround23(F38, F54, F56, F60);
3143
3144 // 128-bit original key size
3145 __ cmp_and_brx_short(keylen, 44, Assembler::equal, Assembler::pt, L_doLast128bit);
3146
3147 for ( int i = 40; i <= 50; i += 2 ) {
3148 __ ldf(FloatRegisterImpl::D, key, i*4, as_FloatRegister(i) );
3149 }
3150 __ aes_eround01(F40, F58, F60, F54); //round 10
3151 __ aes_eround23(F42, F58, F60, F56);
3152 __ aes_eround01(F44, F54, F56, F58); //round 11
3153 __ aes_eround23(F46, F54, F56, F60);
3154
3155 // 192-bit original key size
3156 __ cmp_and_brx_short(keylen, 52, Assembler::equal, Assembler::pt, L_storeOutput);
3157
3158 __ ldf(FloatRegisterImpl::D, key, 208, F52);
3159 __ aes_eround01(F48, F58, F60, F54); //round 12
3160 __ aes_eround23(F50, F58, F60, F56);
3161 __ ldf(FloatRegisterImpl::D, key, 216, F46);
3162 __ ldf(FloatRegisterImpl::D, key, 224, F48);
3163 __ ldf(FloatRegisterImpl::D, key, 232, F50);
3164 __ aes_eround01(F52, F54, F56, F58); //round 13
3165 __ aes_eround23(F46, F54, F56, F60);
3166 __ ba_short(L_storeOutput);
3167
3168 __ BIND(L_doLast128bit);
3169 __ ldf(FloatRegisterImpl::D, key, 160, F48);
3170 __ ldf(FloatRegisterImpl::D, key, 168, F50);
3171
3172 __ BIND(L_storeOutput);
3173 // perform last round of encryption common for all key sizes
3174 __ aes_eround01_l(F48, F58, F60, F54); //last round
3175 __ aes_eround23_l(F50, F58, F60, F56);
3176
3177 // Method to address arbitrary alignment for store instructions:
3178 // Check last 3 bits of 'dest' address to see if it is aligned to 8-byte boundary
3179 // If zero/aligned then continue with double FP store instructions
3180 // If not zero/mis-aligned then edge8n will generate edge mask in result reg (O3 in below case)
3181 // Example: If dest address is 0x07 and nearest 8-byte aligned address is 0x00 then edge mask will be 00000001
3182 // Compute (8-n) where n is # of bytes skipped by partial store(stpartialf) inst from edge mask, n=7 in this case
3183 // We get the value of n from the andcc that checks 'dest' alignment. n is available in O5 in below case.
3184 // Set GSR.align to (8-n) using alignaddr
3185 // Circular byte shift store values by n places so that the original bytes are at correct position for stpartialf
3186 // Set the arbitrarily aligned 'dest' address to nearest 8-byte aligned address
3187 // Store (partial) the original first (8-n) bytes starting at the original 'dest' address
3188 // Negate the edge mask so that the subsequent stpartialf can store the original (8-n-1)th through 8th bytes at appropriate address
3189 // We need to execute this process for both the 8-byte result values
3190
3191 // check for 8-byte alignment since dest byte array may have arbitrary alignment if offset mod 8 is non-zero
3192 __ andcc(to, 7, O5);
3193 __ br(Assembler::notZero, true, Assembler::pn, L_store_misaligned_output);
3194 __ delayed()->edge8n(to, G0, O3);
3195
3196 // aligned case: store output into the destination array
3197 __ stf(FloatRegisterImpl::D, F54, to, 0);
3198 __ retl();
3199 __ delayed()->stf(FloatRegisterImpl::D, F56, to, 8);
3200
3201 __ BIND(L_store_misaligned_output);
3202 __ add(to, 8, O4);
3203 __ mov(8, O2);
3204 __ sub(O2, O5, O2);
3205 __ alignaddr(O2, G0, O2);
3206 __ faligndata(F54, F54, F54);
3207 __ faligndata(F56, F56, F56);
3208 __ and3(to, -8, to);
3209 __ and3(O4, -8, O4);
3210 __ stpartialf(to, O3, F54, Assembler::ASI_PST8_PRIMARY);
3211 __ stpartialf(O4, O3, F56, Assembler::ASI_PST8_PRIMARY);
3212 __ add(to, 8, to);
3213 __ add(O4, 8, O4);
3214 __ orn(G0, O3, O3);
3215 __ stpartialf(to, O3, F54, Assembler::ASI_PST8_PRIMARY);
3216 __ retl();
3217 __ delayed()->stpartialf(O4, O3, F56, Assembler::ASI_PST8_PRIMARY);
3218
3219 return start;
3220 }
3221
3222 address generate_aescrypt_decryptBlock() {
3223 assert((arrayOopDesc::base_offset_in_bytes(T_INT) & 7) == 0,
3224 "the following code assumes that first element of an int array is aligned to 8 bytes");
3225 // required since we read original key 'byte' array as well in the decryption stubs
3226 assert((arrayOopDesc::base_offset_in_bytes(T_BYTE) & 7) == 0,
3227 "the following code assumes that first element of a byte array is aligned to 8 bytes");
3228 __ align(CodeEntryAlignment);
3229 StubCodeMark mark(this, "StubRoutines", "aescrypt_decryptBlock");
3230 address start = __ pc();
3231 Label L_load_misaligned_input, L_load_original_key, L_expand192bit, L_expand256bit, L_reload_misaligned_input;
3232 Label L_256bit_transform, L_common_transform, L_store_misaligned_output;
3233 Register from = O0; // source byte array
3234 Register to = O1; // destination byte array
3235 Register key = O2; // expanded key array
3236 Register original_key = O3; // original key array only required during decryption
3237 const Register keylen = O4; // reg for storing expanded key array length
3238
3239 // read expanded key array length
3240 __ ldsw(Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)), keylen, 0);
3241
3242 // save 'from' since we may need to recheck alignment in case of 256-bit decryption
3243 __ mov(from, G1);
3244
3245 // check for 8-byte alignment since source byte array may have an arbitrary alignment if offset mod 8 is non-zero
3246 __ andcc(from, 7, G0);
3247 __ br(Assembler::notZero, true, Assembler::pn, L_load_misaligned_input);
3248 __ delayed()->alignaddr(from, G0, from);
3249
3250 // aligned case: load input into F52-F54
3251 __ ldf(FloatRegisterImpl::D, from, 0, F52);
3252 __ ldf(FloatRegisterImpl::D, from, 8, F54);
3253 __ ba_short(L_load_original_key);
3254
3255 __ BIND(L_load_misaligned_input);
3256 __ ldf(FloatRegisterImpl::D, from, 0, F52);
3257 __ ldf(FloatRegisterImpl::D, from, 8, F54);
3258 __ ldf(FloatRegisterImpl::D, from, 16, F56);
3259 __ faligndata(F52, F54, F52);
3260 __ faligndata(F54, F56, F54);
3261
3262 __ BIND(L_load_original_key);
3263 // load original key from SunJCE expanded decryption key
3264 // Since we load original key buffer starting first element, 8-byte alignment is guaranteed
3265 for ( int i = 0; i <= 3; i++ ) {
3266 __ ldf(FloatRegisterImpl::S, original_key, i*4, as_FloatRegister(i));
3267 }
3268
3269 // 256-bit original key size
3270 __ cmp_and_brx_short(keylen, 60, Assembler::equal, Assembler::pn, L_expand256bit);
3271
3272 // 192-bit original key size
3273 __ cmp_and_brx_short(keylen, 52, Assembler::equal, Assembler::pn, L_expand192bit);
3274
3275 // 128-bit original key size
3276 // perform key expansion since SunJCE decryption-key expansion is not compatible with SPARC crypto instructions
3277 for ( int i = 0; i <= 36; i += 4 ) {
3278 __ aes_kexpand1(as_FloatRegister(i), as_FloatRegister(i+2), i/4, as_FloatRegister(i+4));
3279 __ aes_kexpand2(as_FloatRegister(i+2), as_FloatRegister(i+4), as_FloatRegister(i+6));
3280 }
3281
3282 // perform 128-bit key specific inverse cipher transformation
3283 __ fxor(FloatRegisterImpl::D, F42, F54, F54);
3284 __ fxor(FloatRegisterImpl::D, F40, F52, F52);
3285 __ ba_short(L_common_transform);
3286
3287 __ BIND(L_expand192bit);
3288
3289 // start loading rest of the 192-bit key
3290 __ ldf(FloatRegisterImpl::S, original_key, 16, F4);
3291 __ ldf(FloatRegisterImpl::S, original_key, 20, F5);
3292
3293 // perform key expansion since SunJCE decryption-key expansion is not compatible with SPARC crypto instructions
3294 for ( int i = 0; i <= 36; i += 6 ) {
3295 __ aes_kexpand1(as_FloatRegister(i), as_FloatRegister(i+4), i/6, as_FloatRegister(i+6));
3296 __ aes_kexpand2(as_FloatRegister(i+2), as_FloatRegister(i+6), as_FloatRegister(i+8));
3297 __ aes_kexpand2(as_FloatRegister(i+4), as_FloatRegister(i+8), as_FloatRegister(i+10));
3298 }
3299 __ aes_kexpand1(F42, F46, 7, F48);
3300 __ aes_kexpand2(F44, F48, F50);
3301
3302 // perform 192-bit key specific inverse cipher transformation
3303 __ fxor(FloatRegisterImpl::D, F50, F54, F54);
3304 __ fxor(FloatRegisterImpl::D, F48, F52, F52);
3305 __ aes_dround23(F46, F52, F54, F58);
3306 __ aes_dround01(F44, F52, F54, F56);
3307 __ aes_dround23(F42, F56, F58, F54);
3308 __ aes_dround01(F40, F56, F58, F52);
3309 __ ba_short(L_common_transform);
3310
3311 __ BIND(L_expand256bit);
3312
3313 // load rest of the 256-bit key
3314 for ( int i = 4; i <= 7; i++ ) {
3315 __ ldf(FloatRegisterImpl::S, original_key, i*4, as_FloatRegister(i));
3316 }
3317
3318 // perform key expansion since SunJCE decryption-key expansion is not compatible with SPARC crypto instructions
3319 for ( int i = 0; i <= 40; i += 8 ) {
3320 __ aes_kexpand1(as_FloatRegister(i), as_FloatRegister(i+6), i/8, as_FloatRegister(i+8));
3321 __ aes_kexpand2(as_FloatRegister(i+2), as_FloatRegister(i+8), as_FloatRegister(i+10));
3322 __ aes_kexpand0(as_FloatRegister(i+4), as_FloatRegister(i+10), as_FloatRegister(i+12));
3323 __ aes_kexpand2(as_FloatRegister(i+6), as_FloatRegister(i+12), as_FloatRegister(i+14));
3324 }
3325 __ aes_kexpand1(F48, F54, 6, F56);
3326 __ aes_kexpand2(F50, F56, F58);
3327
3328 for ( int i = 0; i <= 6; i += 2 ) {
3329 __ fsrc2(FloatRegisterImpl::D, as_FloatRegister(58-i), as_FloatRegister(i));
3330 }
3331
3332 // reload original 'from' address
3333 __ mov(G1, from);
3334
3335 // re-check 8-byte alignment
3336 __ andcc(from, 7, G0);
3337 __ br(Assembler::notZero, true, Assembler::pn, L_reload_misaligned_input);
3338 __ delayed()->alignaddr(from, G0, from);
3339
3340 // aligned case: load input into F52-F54
3341 __ ldf(FloatRegisterImpl::D, from, 0, F52);
3342 __ ldf(FloatRegisterImpl::D, from, 8, F54);
3343 __ ba_short(L_256bit_transform);
3344
3345 __ BIND(L_reload_misaligned_input);
3346 __ ldf(FloatRegisterImpl::D, from, 0, F52);
3347 __ ldf(FloatRegisterImpl::D, from, 8, F54);
3348 __ ldf(FloatRegisterImpl::D, from, 16, F56);
3349 __ faligndata(F52, F54, F52);
3350 __ faligndata(F54, F56, F54);
3351
3352 // perform 256-bit key specific inverse cipher transformation
3353 __ BIND(L_256bit_transform);
3354 __ fxor(FloatRegisterImpl::D, F0, F54, F54);
3355 __ fxor(FloatRegisterImpl::D, F2, F52, F52);
3356 __ aes_dround23(F4, F52, F54, F58);
3357 __ aes_dround01(F6, F52, F54, F56);
3358 __ aes_dround23(F50, F56, F58, F54);
3359 __ aes_dround01(F48, F56, F58, F52);
3360 __ aes_dround23(F46, F52, F54, F58);
3361 __ aes_dround01(F44, F52, F54, F56);
3362 __ aes_dround23(F42, F56, F58, F54);
3363 __ aes_dround01(F40, F56, F58, F52);
3364
3365 for ( int i = 0; i <= 7; i++ ) {
3366 __ ldf(FloatRegisterImpl::S, original_key, i*4, as_FloatRegister(i));
3367 }
3368
3369 // perform inverse cipher transformations common for all key sizes
3370 __ BIND(L_common_transform);
3371 for ( int i = 38; i >= 6; i -= 8 ) {
3372 __ aes_dround23(as_FloatRegister(i), F52, F54, F58);
3373 __ aes_dround01(as_FloatRegister(i-2), F52, F54, F56);
3374 if ( i != 6) {
3375 __ aes_dround23(as_FloatRegister(i-4), F56, F58, F54);
3376 __ aes_dround01(as_FloatRegister(i-6), F56, F58, F52);
3377 } else {
3378 __ aes_dround23_l(as_FloatRegister(i-4), F56, F58, F54);
3379 __ aes_dround01_l(as_FloatRegister(i-6), F56, F58, F52);
3380 }
3381 }
3382
3383 // check for 8-byte alignment since dest byte array may have arbitrary alignment if offset mod 8 is non-zero
3384 __ andcc(to, 7, O5);
3385 __ br(Assembler::notZero, true, Assembler::pn, L_store_misaligned_output);
3386 __ delayed()->edge8n(to, G0, O3);
3387
3388 // aligned case: store output into the destination array
3389 __ stf(FloatRegisterImpl::D, F52, to, 0);
3390 __ retl();
3391 __ delayed()->stf(FloatRegisterImpl::D, F54, to, 8);
3392
3393 __ BIND(L_store_misaligned_output);
3394 __ add(to, 8, O4);
3395 __ mov(8, O2);
3396 __ sub(O2, O5, O2);
3397 __ alignaddr(O2, G0, O2);
3398 __ faligndata(F52, F52, F52);
3399 __ faligndata(F54, F54, F54);
3400 __ and3(to, -8, to);
3401 __ and3(O4, -8, O4);
3402 __ stpartialf(to, O3, F52, Assembler::ASI_PST8_PRIMARY);
3403 __ stpartialf(O4, O3, F54, Assembler::ASI_PST8_PRIMARY);
3404 __ add(to, 8, to);
3405 __ add(O4, 8, O4);
3406 __ orn(G0, O3, O3);
3407 __ stpartialf(to, O3, F52, Assembler::ASI_PST8_PRIMARY);
3408 __ retl();
3409 __ delayed()->stpartialf(O4, O3, F54, Assembler::ASI_PST8_PRIMARY);
3410
3411 return start;
3412 }
3413
3414 address generate_cipherBlockChaining_encryptAESCrypt() {
3415 assert((arrayOopDesc::base_offset_in_bytes(T_INT) & 7) == 0,
3416 "the following code assumes that first element of an int array is aligned to 8 bytes");
3417 assert((arrayOopDesc::base_offset_in_bytes(T_BYTE) & 7) == 0,
3418 "the following code assumes that first element of a byte array is aligned to 8 bytes");
3419 __ align(CodeEntryAlignment);
3420 StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_encryptAESCrypt");
3421 Label L_cbcenc128, L_load_misaligned_input_128bit, L_128bit_transform, L_store_misaligned_output_128bit;
3422 Label L_check_loop_end_128bit, L_cbcenc192, L_load_misaligned_input_192bit, L_192bit_transform;
3423 Label L_store_misaligned_output_192bit, L_check_loop_end_192bit, L_cbcenc256, L_load_misaligned_input_256bit;
3424 Label L_256bit_transform, L_store_misaligned_output_256bit, L_check_loop_end_256bit;
3425 address start = __ pc();
3426 Register from = I0; // source byte array
3427 Register to = I1; // destination byte array
3428 Register key = I2; // expanded key array
3429 Register rvec = I3; // init vector
3430 const Register len_reg = I4; // cipher length
3431 const Register keylen = I5; // reg for storing expanded key array length
3432
3433 __ save_frame(0);
3434 // save cipher len to return in the end
3435 __ mov(len_reg, L0);
3436
3437 // read expanded key length
3438 __ ldsw(Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)), keylen, 0);
3439
3440 // load initial vector, 8-byte alignment is guranteed
3441 __ ldf(FloatRegisterImpl::D, rvec, 0, F60);
3442 __ ldf(FloatRegisterImpl::D, rvec, 8, F62);
3443 // load key, 8-byte alignment is guranteed
3444 __ ldx(key,0,G1);
3445 __ ldx(key,8,G5);
3446
3447 // start loading expanded key, 8-byte alignment is guranteed
3448 for ( int i = 0, j = 16; i <= 38; i += 2, j += 8 ) {
3449 __ ldf(FloatRegisterImpl::D, key, j, as_FloatRegister(i));
3450 }
3451
3452 // 128-bit original key size
3453 __ cmp_and_brx_short(keylen, 44, Assembler::equal, Assembler::pt, L_cbcenc128);
3454
3455 for ( int i = 40, j = 176; i <= 46; i += 2, j += 8 ) {
3456 __ ldf(FloatRegisterImpl::D, key, j, as_FloatRegister(i));
3457 }
3458
3459 // 192-bit original key size
3460 __ cmp_and_brx_short(keylen, 52, Assembler::equal, Assembler::pt, L_cbcenc192);
3461
3462 for ( int i = 48, j = 208; i <= 54; i += 2, j += 8 ) {
3463 __ ldf(FloatRegisterImpl::D, key, j, as_FloatRegister(i));
3464 }
3465
3466 // 256-bit original key size
3467 __ ba_short(L_cbcenc256);
3468
3469 __ align(OptoLoopAlignment);
3470 __ BIND(L_cbcenc128);
3471 // check for 8-byte alignment since source byte array may have an arbitrary alignment if offset mod 8 is non-zero
3472 __ andcc(from, 7, G0);
3473 __ br(Assembler::notZero, true, Assembler::pn, L_load_misaligned_input_128bit);
3474 __ delayed()->mov(from, L1); // save original 'from' address before alignaddr
3475
3476 // aligned case: load input into G3 and G4
3477 __ ldx(from,0,G3);
3478 __ ldx(from,8,G4);
3479 __ ba_short(L_128bit_transform);
3480
3481 __ BIND(L_load_misaligned_input_128bit);
3482 // can clobber F48, F50 and F52 as they are not used in 128 and 192-bit key encryption
3483 __ alignaddr(from, G0, from);
3484 __ ldf(FloatRegisterImpl::D, from, 0, F48);
3485 __ ldf(FloatRegisterImpl::D, from, 8, F50);
3486 __ ldf(FloatRegisterImpl::D, from, 16, F52);
3487 __ faligndata(F48, F50, F48);
3488 __ faligndata(F50, F52, F50);
3489 __ movdtox(F48, G3);
3490 __ movdtox(F50, G4);
3491 __ mov(L1, from);
3492
3493 __ BIND(L_128bit_transform);
3494 __ xor3(G1,G3,G3);
3495 __ xor3(G5,G4,G4);
3496 __ movxtod(G3,F56);
3497 __ movxtod(G4,F58);
3498 __ fxor(FloatRegisterImpl::D, F60, F56, F60);
3499 __ fxor(FloatRegisterImpl::D, F62, F58, F62);
3500
3501 // TEN_EROUNDS
3502 for ( int i = 0; i <= 32; i += 8 ) {
3503 __ aes_eround01(as_FloatRegister(i), F60, F62, F56);
3504 __ aes_eround23(as_FloatRegister(i+2), F60, F62, F58);
3505 if (i != 32 ) {
3506 __ aes_eround01(as_FloatRegister(i+4), F56, F58, F60);
3507 __ aes_eround23(as_FloatRegister(i+6), F56, F58, F62);
3508 } else {
3509 __ aes_eround01_l(as_FloatRegister(i+4), F56, F58, F60);
3510 __ aes_eround23_l(as_FloatRegister(i+6), F56, F58, F62);
3511 }
3512 }
3513
3514 // check for 8-byte alignment since dest byte array may have arbitrary alignment if offset mod 8 is non-zero
3515 __ andcc(to, 7, L1);
3516 __ br(Assembler::notZero, true, Assembler::pn, L_store_misaligned_output_128bit);
3517 __ delayed()->edge8n(to, G0, L2);
3518
3519 // aligned case: store output into the destination array
3520 __ stf(FloatRegisterImpl::D, F60, to, 0);
3521 __ stf(FloatRegisterImpl::D, F62, to, 8);
3522 __ ba_short(L_check_loop_end_128bit);
3523
3524 __ BIND(L_store_misaligned_output_128bit);
3525 __ add(to, 8, L3);
3526 __ mov(8, L4);
3527 __ sub(L4, L1, L4);
3528 __ alignaddr(L4, G0, L4);
3529 // save cipher text before circular right shift
3530 // as it needs to be stored as iv for next block (see code before next retl)
3531 __ movdtox(F60, L6);
3532 __ movdtox(F62, L7);
3533 __ faligndata(F60, F60, F60);
3534 __ faligndata(F62, F62, F62);
3535 __ mov(to, L5);
3536 __ and3(to, -8, to);
3537 __ and3(L3, -8, L3);
3538 __ stpartialf(to, L2, F60, Assembler::ASI_PST8_PRIMARY);
3539 __ stpartialf(L3, L2, F62, Assembler::ASI_PST8_PRIMARY);
3540 __ add(to, 8, to);
3541 __ add(L3, 8, L3);
3542 __ orn(G0, L2, L2);
3543 __ stpartialf(to, L2, F60, Assembler::ASI_PST8_PRIMARY);
3544 __ stpartialf(L3, L2, F62, Assembler::ASI_PST8_PRIMARY);
3545 __ mov(L5, to);
3546 __ movxtod(L6, F60);
3547 __ movxtod(L7, F62);
3548
3549 __ BIND(L_check_loop_end_128bit);
3550 __ add(from, 16, from);
3551 __ add(to, 16, to);
3552 __ subcc(len_reg, 16, len_reg);
3553 __ br(Assembler::notEqual, false, Assembler::pt, L_cbcenc128);
3554 __ delayed()->nop();
3555 // re-init intial vector for next block, 8-byte alignment is guaranteed
3556 __ stf(FloatRegisterImpl::D, F60, rvec, 0);
3557 __ stf(FloatRegisterImpl::D, F62, rvec, 8);
3558 __ mov(L0, I0);
3559 __ ret();
3560 __ delayed()->restore();
3561
3562 __ align(OptoLoopAlignment);
3563 __ BIND(L_cbcenc192);
3564 // check for 8-byte alignment since source byte array may have an arbitrary alignment if offset mod 8 is non-zero
3565 __ andcc(from, 7, G0);
3566 __ br(Assembler::notZero, true, Assembler::pn, L_load_misaligned_input_192bit);
3567 __ delayed()->mov(from, L1); // save original 'from' address before alignaddr
3568
3569 // aligned case: load input into G3 and G4
3570 __ ldx(from,0,G3);
3571 __ ldx(from,8,G4);
3572 __ ba_short(L_192bit_transform);
3573
3574 __ BIND(L_load_misaligned_input_192bit);
3575 // can clobber F48, F50 and F52 as they are not used in 128 and 192-bit key encryption
3576 __ alignaddr(from, G0, from);
3577 __ ldf(FloatRegisterImpl::D, from, 0, F48);
3578 __ ldf(FloatRegisterImpl::D, from, 8, F50);
3579 __ ldf(FloatRegisterImpl::D, from, 16, F52);
3580 __ faligndata(F48, F50, F48);
3581 __ faligndata(F50, F52, F50);
3582 __ movdtox(F48, G3);
3583 __ movdtox(F50, G4);
3584 __ mov(L1, from);
3585
3586 __ BIND(L_192bit_transform);
3587 __ xor3(G1,G3,G3);
3588 __ xor3(G5,G4,G4);
3589 __ movxtod(G3,F56);
3590 __ movxtod(G4,F58);
3591 __ fxor(FloatRegisterImpl::D, F60, F56, F60);
3592 __ fxor(FloatRegisterImpl::D, F62, F58, F62);
3593
3594 // TWELEVE_EROUNDS
3595 for ( int i = 0; i <= 40; i += 8 ) {
3596 __ aes_eround01(as_FloatRegister(i), F60, F62, F56);
3597 __ aes_eround23(as_FloatRegister(i+2), F60, F62, F58);
3598 if (i != 40 ) {
3599 __ aes_eround01(as_FloatRegister(i+4), F56, F58, F60);
3600 __ aes_eround23(as_FloatRegister(i+6), F56, F58, F62);
3601 } else {
3602 __ aes_eround01_l(as_FloatRegister(i+4), F56, F58, F60);
3603 __ aes_eround23_l(as_FloatRegister(i+6), F56, F58, F62);
3604 }
3605 }
3606
3607 // check for 8-byte alignment since dest byte array may have arbitrary alignment if offset mod 8 is non-zero
3608 __ andcc(to, 7, L1);
3609 __ br(Assembler::notZero, true, Assembler::pn, L_store_misaligned_output_192bit);
3610 __ delayed()->edge8n(to, G0, L2);
3611
3612 // aligned case: store output into the destination array
3613 __ stf(FloatRegisterImpl::D, F60, to, 0);
3614 __ stf(FloatRegisterImpl::D, F62, to, 8);
3615 __ ba_short(L_check_loop_end_192bit);
3616
3617 __ BIND(L_store_misaligned_output_192bit);
3618 __ add(to, 8, L3);
3619 __ mov(8, L4);
3620 __ sub(L4, L1, L4);
3621 __ alignaddr(L4, G0, L4);
3622 __ movdtox(F60, L6);
3623 __ movdtox(F62, L7);
3624 __ faligndata(F60, F60, F60);
3625 __ faligndata(F62, F62, F62);
3626 __ mov(to, L5);
3627 __ and3(to, -8, to);
3628 __ and3(L3, -8, L3);
3629 __ stpartialf(to, L2, F60, Assembler::ASI_PST8_PRIMARY);
3630 __ stpartialf(L3, L2, F62, Assembler::ASI_PST8_PRIMARY);
3631 __ add(to, 8, to);
3632 __ add(L3, 8, L3);
3633 __ orn(G0, L2, L2);
3634 __ stpartialf(to, L2, F60, Assembler::ASI_PST8_PRIMARY);
3635 __ stpartialf(L3, L2, F62, Assembler::ASI_PST8_PRIMARY);
3636 __ mov(L5, to);
3637 __ movxtod(L6, F60);
3638 __ movxtod(L7, F62);
3639
3640 __ BIND(L_check_loop_end_192bit);
3641 __ add(from, 16, from);
3642 __ subcc(len_reg, 16, len_reg);
3643 __ add(to, 16, to);
3644 __ br(Assembler::notEqual, false, Assembler::pt, L_cbcenc192);
3645 __ delayed()->nop();
3646 // re-init intial vector for next block, 8-byte alignment is guaranteed
3647 __ stf(FloatRegisterImpl::D, F60, rvec, 0);
3648 __ stf(FloatRegisterImpl::D, F62, rvec, 8);
3649 __ mov(L0, I0);
3650 __ ret();
3651 __ delayed()->restore();
3652
3653 __ align(OptoLoopAlignment);
3654 __ BIND(L_cbcenc256);
3655 // check for 8-byte alignment since source byte array may have an arbitrary alignment if offset mod 8 is non-zero
3656 __ andcc(from, 7, G0);
3657 __ br(Assembler::notZero, true, Assembler::pn, L_load_misaligned_input_256bit);
3658 __ delayed()->mov(from, L1); // save original 'from' address before alignaddr
3659
3660 // aligned case: load input into G3 and G4
3661 __ ldx(from,0,G3);
3662 __ ldx(from,8,G4);
3663 __ ba_short(L_256bit_transform);
3664
3665 __ BIND(L_load_misaligned_input_256bit);
3666 // cannot clobber F48, F50 and F52. F56, F58 can be used though
3667 __ alignaddr(from, G0, from);
3668 __ movdtox(F60, L2); // save F60 before overwriting
3669 __ ldf(FloatRegisterImpl::D, from, 0, F56);
3670 __ ldf(FloatRegisterImpl::D, from, 8, F58);
3671 __ ldf(FloatRegisterImpl::D, from, 16, F60);
3672 __ faligndata(F56, F58, F56);
3673 __ faligndata(F58, F60, F58);
3674 __ movdtox(F56, G3);
3675 __ movdtox(F58, G4);
3676 __ mov(L1, from);
3677 __ movxtod(L2, F60);
3678
3679 __ BIND(L_256bit_transform);
3680 __ xor3(G1,G3,G3);
3681 __ xor3(G5,G4,G4);
3682 __ movxtod(G3,F56);
3683 __ movxtod(G4,F58);
3684 __ fxor(FloatRegisterImpl::D, F60, F56, F60);
3685 __ fxor(FloatRegisterImpl::D, F62, F58, F62);
3686
3687 // FOURTEEN_EROUNDS
3688 for ( int i = 0; i <= 48; i += 8 ) {
3689 __ aes_eround01(as_FloatRegister(i), F60, F62, F56);
3690 __ aes_eround23(as_FloatRegister(i+2), F60, F62, F58);
3691 if (i != 48 ) {
3692 __ aes_eround01(as_FloatRegister(i+4), F56, F58, F60);
3693 __ aes_eround23(as_FloatRegister(i+6), F56, F58, F62);
3694 } else {
3695 __ aes_eround01_l(as_FloatRegister(i+4), F56, F58, F60);
3696 __ aes_eround23_l(as_FloatRegister(i+6), F56, F58, F62);
3697 }
3698 }
3699
3700 // check for 8-byte alignment since dest byte array may have arbitrary alignment if offset mod 8 is non-zero
3701 __ andcc(to, 7, L1);
3702 __ br(Assembler::notZero, true, Assembler::pn, L_store_misaligned_output_256bit);
3703 __ delayed()->edge8n(to, G0, L2);
3704
3705 // aligned case: store output into the destination array
3706 __ stf(FloatRegisterImpl::D, F60, to, 0);
3707 __ stf(FloatRegisterImpl::D, F62, to, 8);
3708 __ ba_short(L_check_loop_end_256bit);
3709
3710 __ BIND(L_store_misaligned_output_256bit);
3711 __ add(to, 8, L3);
3712 __ mov(8, L4);
3713 __ sub(L4, L1, L4);
3714 __ alignaddr(L4, G0, L4);
3715 __ movdtox(F60, L6);
3716 __ movdtox(F62, L7);
3717 __ faligndata(F60, F60, F60);
3718 __ faligndata(F62, F62, F62);
3719 __ mov(to, L5);
3720 __ and3(to, -8, to);
3721 __ and3(L3, -8, L3);
3722 __ stpartialf(to, L2, F60, Assembler::ASI_PST8_PRIMARY);
3723 __ stpartialf(L3, L2, F62, Assembler::ASI_PST8_PRIMARY);
3724 __ add(to, 8, to);
3725 __ add(L3, 8, L3);
3726 __ orn(G0, L2, L2);
3727 __ stpartialf(to, L2, F60, Assembler::ASI_PST8_PRIMARY);
3728 __ stpartialf(L3, L2, F62, Assembler::ASI_PST8_PRIMARY);
3729 __ mov(L5, to);
3730 __ movxtod(L6, F60);
3731 __ movxtod(L7, F62);
3732
3733 __ BIND(L_check_loop_end_256bit);
3734 __ add(from, 16, from);
3735 __ subcc(len_reg, 16, len_reg);
3736 __ add(to, 16, to);
3737 __ br(Assembler::notEqual, false, Assembler::pt, L_cbcenc256);
3738 __ delayed()->nop();
3739 // re-init intial vector for next block, 8-byte alignment is guaranteed
3740 __ stf(FloatRegisterImpl::D, F60, rvec, 0);
3741 __ stf(FloatRegisterImpl::D, F62, rvec, 8);
3742 __ mov(L0, I0);
3743 __ ret();
3744 __ delayed()->restore();
3745
3746 return start;
3747 }
3748
3749 address generate_cipherBlockChaining_decryptAESCrypt_Parallel() {
3750 assert((arrayOopDesc::base_offset_in_bytes(T_INT) & 7) == 0,
3751 "the following code assumes that first element of an int array is aligned to 8 bytes");
3752 assert((arrayOopDesc::base_offset_in_bytes(T_BYTE) & 7) == 0,
3753 "the following code assumes that first element of a byte array is aligned to 8 bytes");
3754 __ align(CodeEntryAlignment);
3755 StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_decryptAESCrypt");
3756 Label L_cbcdec_end, L_expand192bit, L_expand256bit, L_dec_first_block_start;
3757 Label L_dec_first_block128, L_dec_first_block192, L_dec_next2_blocks128, L_dec_next2_blocks192, L_dec_next2_blocks256;
3758 Label L_load_misaligned_input_first_block, L_transform_first_block, L_load_misaligned_next2_blocks128, L_transform_next2_blocks128;
3759 Label L_load_misaligned_next2_blocks192, L_transform_next2_blocks192, L_load_misaligned_next2_blocks256, L_transform_next2_blocks256;
3760 Label L_store_misaligned_output_first_block, L_check_decrypt_end, L_store_misaligned_output_next2_blocks128;
3761 Label L_check_decrypt_loop_end128, L_store_misaligned_output_next2_blocks192, L_check_decrypt_loop_end192;
3762 Label L_store_misaligned_output_next2_blocks256, L_check_decrypt_loop_end256;
3763 address start = __ pc();
3764 Register from = I0; // source byte array
3765 Register to = I1; // destination byte array
3766 Register key = I2; // expanded key array
3767 Register rvec = I3; // init vector
3768 const Register len_reg = I4; // cipher length
3769 const Register original_key = I5; // original key array only required during decryption
3770 const Register keylen = L6; // reg for storing expanded key array length
3771
3772 __ save_frame(0); //args are read from I* registers since we save the frame in the beginning
3773 // save cipher len to return in the end
3774 __ mov(len_reg, L7);
3775
3776 // load original key from SunJCE expanded decryption key
3777 // Since we load original key buffer starting first element, 8-byte alignment is guaranteed
3778 for ( int i = 0; i <= 3; i++ ) {
3779 __ ldf(FloatRegisterImpl::S, original_key, i*4, as_FloatRegister(i));
3780 }
3781
3782 // load initial vector, 8-byte alignment is guaranteed
3783 __ ldx(rvec,0,L0);
3784 __ ldx(rvec,8,L1);
3785
3786 // read expanded key array length
3787 __ ldsw(Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)), keylen, 0);
3788
3789 // 256-bit original key size
3790 __ cmp_and_brx_short(keylen, 60, Assembler::equal, Assembler::pn, L_expand256bit);
3791
3792 // 192-bit original key size
3793 __ cmp_and_brx_short(keylen, 52, Assembler::equal, Assembler::pn, L_expand192bit);
3794
3795 // 128-bit original key size
3796 // perform key expansion since SunJCE decryption-key expansion is not compatible with SPARC crypto instructions
3797 for ( int i = 0; i <= 36; i += 4 ) {
3798 __ aes_kexpand1(as_FloatRegister(i), as_FloatRegister(i+2), i/4, as_FloatRegister(i+4));
3799 __ aes_kexpand2(as_FloatRegister(i+2), as_FloatRegister(i+4), as_FloatRegister(i+6));
3800 }
3801
3802 // load expanded key[last-1] and key[last] elements
3803 __ movdtox(F40,L2);
3804 __ movdtox(F42,L3);
3805
3806 __ and3(len_reg, 16, L4);
3807 __ br_null_short(L4, Assembler::pt, L_dec_next2_blocks128);
3808 __ nop();
3809
3810 __ ba_short(L_dec_first_block_start);
3811
3812 __ BIND(L_expand192bit);
3813 // load rest of the 192-bit key
3814 __ ldf(FloatRegisterImpl::S, original_key, 16, F4);
3815 __ ldf(FloatRegisterImpl::S, original_key, 20, F5);
3816
3817 // perform key expansion since SunJCE decryption-key expansion is not compatible with SPARC crypto instructions
3818 for ( int i = 0; i <= 36; i += 6 ) {
3819 __ aes_kexpand1(as_FloatRegister(i), as_FloatRegister(i+4), i/6, as_FloatRegister(i+6));
3820 __ aes_kexpand2(as_FloatRegister(i+2), as_FloatRegister(i+6), as_FloatRegister(i+8));
3821 __ aes_kexpand2(as_FloatRegister(i+4), as_FloatRegister(i+8), as_FloatRegister(i+10));
3822 }
3823 __ aes_kexpand1(F42, F46, 7, F48);
3824 __ aes_kexpand2(F44, F48, F50);
3825
3826 // load expanded key[last-1] and key[last] elements
3827 __ movdtox(F48,L2);
3828 __ movdtox(F50,L3);
3829
3830 __ and3(len_reg, 16, L4);
3831 __ br_null_short(L4, Assembler::pt, L_dec_next2_blocks192);
3832 __ nop();
3833
3834 __ ba_short(L_dec_first_block_start);
3835
3836 __ BIND(L_expand256bit);
3837 // load rest of the 256-bit key
3838 for ( int i = 4; i <= 7; i++ ) {
3839 __ ldf(FloatRegisterImpl::S, original_key, i*4, as_FloatRegister(i));
3840 }
3841
3842 // perform key expansion since SunJCE decryption-key expansion is not compatible with SPARC crypto instructions
3843 for ( int i = 0; i <= 40; i += 8 ) {
3844 __ aes_kexpand1(as_FloatRegister(i), as_FloatRegister(i+6), i/8, as_FloatRegister(i+8));
3845 __ aes_kexpand2(as_FloatRegister(i+2), as_FloatRegister(i+8), as_FloatRegister(i+10));
3846 __ aes_kexpand0(as_FloatRegister(i+4), as_FloatRegister(i+10), as_FloatRegister(i+12));
3847 __ aes_kexpand2(as_FloatRegister(i+6), as_FloatRegister(i+12), as_FloatRegister(i+14));
3848 }
3849 __ aes_kexpand1(F48, F54, 6, F56);
3850 __ aes_kexpand2(F50, F56, F58);
3851
3852 // load expanded key[last-1] and key[last] elements
3853 __ movdtox(F56,L2);
3854 __ movdtox(F58,L3);
3855
3856 __ and3(len_reg, 16, L4);
3857 __ br_null_short(L4, Assembler::pt, L_dec_next2_blocks256);
3858
3859 __ BIND(L_dec_first_block_start);
3860 // check for 8-byte alignment since source byte array may have an arbitrary alignment if offset mod 8 is non-zero
3861 __ andcc(from, 7, G0);
3862 __ br(Assembler::notZero, true, Assembler::pn, L_load_misaligned_input_first_block);
3863 __ delayed()->mov(from, G1); // save original 'from' address before alignaddr
3864
3865 // aligned case: load input into L4 and L5
3866 __ ldx(from,0,L4);
3867 __ ldx(from,8,L5);
3868 __ ba_short(L_transform_first_block);
3869
3870 __ BIND(L_load_misaligned_input_first_block);
3871 __ alignaddr(from, G0, from);
3872 // F58, F60, F62 can be clobbered
3873 __ ldf(FloatRegisterImpl::D, from, 0, F58);
3874 __ ldf(FloatRegisterImpl::D, from, 8, F60);
3875 __ ldf(FloatRegisterImpl::D, from, 16, F62);
3876 __ faligndata(F58, F60, F58);
3877 __ faligndata(F60, F62, F60);
3878 __ movdtox(F58, L4);
3879 __ movdtox(F60, L5);
3880 __ mov(G1, from);
3881
3882 __ BIND(L_transform_first_block);
3883 __ xor3(L2,L4,G1);
3884 __ movxtod(G1,F60);
3885 __ xor3(L3,L5,G1);
3886 __ movxtod(G1,F62);
3887
3888 // 128-bit original key size
3889 __ cmp_and_brx_short(keylen, 44, Assembler::equal, Assembler::pn, L_dec_first_block128);
3890
3891 // 192-bit original key size
3892 __ cmp_and_brx_short(keylen, 52, Assembler::equal, Assembler::pn, L_dec_first_block192);
3893
3894 __ aes_dround23(F54, F60, F62, F58);
3895 __ aes_dround01(F52, F60, F62, F56);
3896 __ aes_dround23(F50, F56, F58, F62);
3897 __ aes_dround01(F48, F56, F58, F60);
3898
3899 __ BIND(L_dec_first_block192);
3900 __ aes_dround23(F46, F60, F62, F58);
3901 __ aes_dround01(F44, F60, F62, F56);
3902 __ aes_dround23(F42, F56, F58, F62);
3903 __ aes_dround01(F40, F56, F58, F60);
3904
3905 __ BIND(L_dec_first_block128);
3906 for ( int i = 38; i >= 6; i -= 8 ) {
3907 __ aes_dround23(as_FloatRegister(i), F60, F62, F58);
3908 __ aes_dround01(as_FloatRegister(i-2), F60, F62, F56);
3909 if ( i != 6) {
3910 __ aes_dround23(as_FloatRegister(i-4), F56, F58, F62);
3911 __ aes_dround01(as_FloatRegister(i-6), F56, F58, F60);
3912 } else {
3913 __ aes_dround23_l(as_FloatRegister(i-4), F56, F58, F62);
3914 __ aes_dround01_l(as_FloatRegister(i-6), F56, F58, F60);
3915 }
3916 }
3917
3918 __ movxtod(L0,F56);
3919 __ movxtod(L1,F58);
3920 __ mov(L4,L0);
3921 __ mov(L5,L1);
3922 __ fxor(FloatRegisterImpl::D, F56, F60, F60);
3923 __ fxor(FloatRegisterImpl::D, F58, F62, F62);
3924
3925 // check for 8-byte alignment since dest byte array may have arbitrary alignment if offset mod 8 is non-zero
3926 __ andcc(to, 7, G1);
3927 __ br(Assembler::notZero, true, Assembler::pn, L_store_misaligned_output_first_block);
3928 __ delayed()->edge8n(to, G0, G2);
3929
3930 // aligned case: store output into the destination array
3931 __ stf(FloatRegisterImpl::D, F60, to, 0);
3932 __ stf(FloatRegisterImpl::D, F62, to, 8);
3933 __ ba_short(L_check_decrypt_end);
3934
3935 __ BIND(L_store_misaligned_output_first_block);
3936 __ add(to, 8, G3);
3937 __ mov(8, G4);
3938 __ sub(G4, G1, G4);
3939 __ alignaddr(G4, G0, G4);
3940 __ faligndata(F60, F60, F60);
3941 __ faligndata(F62, F62, F62);
3942 __ mov(to, G1);
3943 __ and3(to, -8, to);
3944 __ and3(G3, -8, G3);
3945 __ stpartialf(to, G2, F60, Assembler::ASI_PST8_PRIMARY);
3946 __ stpartialf(G3, G2, F62, Assembler::ASI_PST8_PRIMARY);
3947 __ add(to, 8, to);
3948 __ add(G3, 8, G3);
3949 __ orn(G0, G2, G2);
3950 __ stpartialf(to, G2, F60, Assembler::ASI_PST8_PRIMARY);
3951 __ stpartialf(G3, G2, F62, Assembler::ASI_PST8_PRIMARY);
3952 __ mov(G1, to);
3953
3954 __ BIND(L_check_decrypt_end);
3955 __ add(from, 16, from);
3956 __ add(to, 16, to);
3957 __ subcc(len_reg, 16, len_reg);
3958 __ br(Assembler::equal, false, Assembler::pt, L_cbcdec_end);
3959 __ delayed()->nop();
3960
3961 // 256-bit original key size
3962 __ cmp_and_brx_short(keylen, 60, Assembler::equal, Assembler::pn, L_dec_next2_blocks256);
3963
3964 // 192-bit original key size
3965 __ cmp_and_brx_short(keylen, 52, Assembler::equal, Assembler::pn, L_dec_next2_blocks192);
3966
3967 __ align(OptoLoopAlignment);
3968 __ BIND(L_dec_next2_blocks128);
3969 __ nop();
3970
3971 // check for 8-byte alignment since source byte array may have an arbitrary alignment if offset mod 8 is non-zero
3972 __ andcc(from, 7, G0);
3973 __ br(Assembler::notZero, true, Assembler::pn, L_load_misaligned_next2_blocks128);
3974 __ delayed()->mov(from, G1); // save original 'from' address before alignaddr
3975
3976 // aligned case: load input into G4, G5, L4 and L5
3977 __ ldx(from,0,G4);
3978 __ ldx(from,8,G5);
3979 __ ldx(from,16,L4);
3980 __ ldx(from,24,L5);
3981 __ ba_short(L_transform_next2_blocks128);
3982
3983 __ BIND(L_load_misaligned_next2_blocks128);
3984 __ alignaddr(from, G0, from);
3985 // F40, F42, F58, F60, F62 can be clobbered
3986 __ ldf(FloatRegisterImpl::D, from, 0, F40);
3987 __ ldf(FloatRegisterImpl::D, from, 8, F42);
3988 __ ldf(FloatRegisterImpl::D, from, 16, F60);
3989 __ ldf(FloatRegisterImpl::D, from, 24, F62);
3990 __ ldf(FloatRegisterImpl::D, from, 32, F58);
3991 __ faligndata(F40, F42, F40);
3992 __ faligndata(F42, F60, F42);
3993 __ faligndata(F60, F62, F60);
3994 __ faligndata(F62, F58, F62);
3995 __ movdtox(F40, G4);
3996 __ movdtox(F42, G5);
3997 __ movdtox(F60, L4);
3998 __ movdtox(F62, L5);
3999 __ mov(G1, from);
4000
4001 __ BIND(L_transform_next2_blocks128);
4002 // F40:F42 used for first 16-bytes
4003 __ xor3(L2,G4,G1);
4004 __ movxtod(G1,F40);
4005 __ xor3(L3,G5,G1);
4006 __ movxtod(G1,F42);
4007
4008 // F60:F62 used for next 16-bytes
4009 __ xor3(L2,L4,G1);
4010 __ movxtod(G1,F60);
4011 __ xor3(L3,L5,G1);
4012 __ movxtod(G1,F62);
4013
4014 for ( int i = 38; i >= 6; i -= 8 ) {
4015 __ aes_dround23(as_FloatRegister(i), F40, F42, F44);
4016 __ aes_dround01(as_FloatRegister(i-2), F40, F42, F46);
4017 __ aes_dround23(as_FloatRegister(i), F60, F62, F58);
4018 __ aes_dround01(as_FloatRegister(i-2), F60, F62, F56);
4019 if (i != 6 ) {
4020 __ aes_dround23(as_FloatRegister(i-4), F46, F44, F42);
4021 __ aes_dround01(as_FloatRegister(i-6), F46, F44, F40);
4022 __ aes_dround23(as_FloatRegister(i-4), F56, F58, F62);
4023 __ aes_dround01(as_FloatRegister(i-6), F56, F58, F60);
4024 } else {
4025 __ aes_dround23_l(as_FloatRegister(i-4), F46, F44, F42);
4026 __ aes_dround01_l(as_FloatRegister(i-6), F46, F44, F40);
4027 __ aes_dround23_l(as_FloatRegister(i-4), F56, F58, F62);
4028 __ aes_dround01_l(as_FloatRegister(i-6), F56, F58, F60);
4029 }
4030 }
4031
4032 __ movxtod(L0,F46);
4033 __ movxtod(L1,F44);
4034 __ fxor(FloatRegisterImpl::D, F46, F40, F40);
4035 __ fxor(FloatRegisterImpl::D, F44, F42, F42);
4036
4037 __ movxtod(G4,F56);
4038 __ movxtod(G5,F58);
4039 __ mov(L4,L0);
4040 __ mov(L5,L1);
4041 __ fxor(FloatRegisterImpl::D, F56, F60, F60);
4042 __ fxor(FloatRegisterImpl::D, F58, F62, F62);
4043
4044 // For mis-aligned store of 32 bytes of result we can do:
4045 // Circular right-shift all 4 FP registers so that 'head' and 'tail'
4046 // parts that need to be stored starting at mis-aligned address are in a FP reg
4047 // the other 3 FP regs can thus be stored using regular store
4048 // we then use the edge + partial-store mechanism to store the 'head' and 'tail' parts
4049
4050 // check for 8-byte alignment since dest byte array may have arbitrary alignment if offset mod 8 is non-zero
4051 __ andcc(to, 7, G1);
4052 __ br(Assembler::notZero, true, Assembler::pn, L_store_misaligned_output_next2_blocks128);
4053 __ delayed()->edge8n(to, G0, G2);
4054
4055 // aligned case: store output into the destination array
4056 __ stf(FloatRegisterImpl::D, F40, to, 0);
4057 __ stf(FloatRegisterImpl::D, F42, to, 8);
4058 __ stf(FloatRegisterImpl::D, F60, to, 16);
4059 __ stf(FloatRegisterImpl::D, F62, to, 24);
4060 __ ba_short(L_check_decrypt_loop_end128);
4061
4062 __ BIND(L_store_misaligned_output_next2_blocks128);
4063 __ mov(8, G4);
4064 __ sub(G4, G1, G4);
4065 __ alignaddr(G4, G0, G4);
4066 __ faligndata(F40, F42, F56); // F56 can be clobbered
4067 __ faligndata(F42, F60, F42);
4068 __ faligndata(F60, F62, F60);
4069 __ faligndata(F62, F40, F40);
4070 __ mov(to, G1);
4071 __ and3(to, -8, to);
4072 __ stpartialf(to, G2, F40, Assembler::ASI_PST8_PRIMARY);
4073 __ stf(FloatRegisterImpl::D, F56, to, 8);
4074 __ stf(FloatRegisterImpl::D, F42, to, 16);
4075 __ stf(FloatRegisterImpl::D, F60, to, 24);
4076 __ add(to, 32, to);
4077 __ orn(G0, G2, G2);
4078 __ stpartialf(to, G2, F40, Assembler::ASI_PST8_PRIMARY);
4079 __ mov(G1, to);
4080
4081 __ BIND(L_check_decrypt_loop_end128);
4082 __ add(from, 32, from);
4083 __ add(to, 32, to);
4084 __ subcc(len_reg, 32, len_reg);
4085 __ br(Assembler::notEqual, false, Assembler::pt, L_dec_next2_blocks128);
4086 __ delayed()->nop();
4087 __ ba_short(L_cbcdec_end);
4088
4089 __ align(OptoLoopAlignment);
4090 __ BIND(L_dec_next2_blocks192);
4091 __ nop();
4092
4093 // check for 8-byte alignment since source byte array may have an arbitrary alignment if offset mod 8 is non-zero
4094 __ andcc(from, 7, G0);
4095 __ br(Assembler::notZero, true, Assembler::pn, L_load_misaligned_next2_blocks192);
4096 __ delayed()->mov(from, G1); // save original 'from' address before alignaddr
4097
4098 // aligned case: load input into G4, G5, L4 and L5
4099 __ ldx(from,0,G4);
4100 __ ldx(from,8,G5);
4101 __ ldx(from,16,L4);
4102 __ ldx(from,24,L5);
4103 __ ba_short(L_transform_next2_blocks192);
4104
4105 __ BIND(L_load_misaligned_next2_blocks192);
4106 __ alignaddr(from, G0, from);
4107 // F48, F50, F52, F60, F62 can be clobbered
4108 __ ldf(FloatRegisterImpl::D, from, 0, F48);
4109 __ ldf(FloatRegisterImpl::D, from, 8, F50);
4110 __ ldf(FloatRegisterImpl::D, from, 16, F60);
4111 __ ldf(FloatRegisterImpl::D, from, 24, F62);
4112 __ ldf(FloatRegisterImpl::D, from, 32, F52);
4113 __ faligndata(F48, F50, F48);
4114 __ faligndata(F50, F60, F50);
4115 __ faligndata(F60, F62, F60);
4116 __ faligndata(F62, F52, F62);
4117 __ movdtox(F48, G4);
4118 __ movdtox(F50, G5);
4119 __ movdtox(F60, L4);
4120 __ movdtox(F62, L5);
4121 __ mov(G1, from);
4122
4123 __ BIND(L_transform_next2_blocks192);
4124 // F48:F50 used for first 16-bytes
4125 __ xor3(L2,G4,G1);
4126 __ movxtod(G1,F48);
4127 __ xor3(L3,G5,G1);
4128 __ movxtod(G1,F50);
4129
4130 // F60:F62 used for next 16-bytes
4131 __ xor3(L2,L4,G1);
4132 __ movxtod(G1,F60);
4133 __ xor3(L3,L5,G1);
4134 __ movxtod(G1,F62);
4135
4136 for ( int i = 46; i >= 6; i -= 8 ) {
4137 __ aes_dround23(as_FloatRegister(i), F48, F50, F52);
4138 __ aes_dround01(as_FloatRegister(i-2), F48, F50, F54);
4139 __ aes_dround23(as_FloatRegister(i), F60, F62, F58);
4140 __ aes_dround01(as_FloatRegister(i-2), F60, F62, F56);
4141 if (i != 6 ) {
4142 __ aes_dround23(as_FloatRegister(i-4), F54, F52, F50);
4143 __ aes_dround01(as_FloatRegister(i-6), F54, F52, F48);
4144 __ aes_dround23(as_FloatRegister(i-4), F56, F58, F62);
4145 __ aes_dround01(as_FloatRegister(i-6), F56, F58, F60);
4146 } else {
4147 __ aes_dround23_l(as_FloatRegister(i-4), F54, F52, F50);
4148 __ aes_dround01_l(as_FloatRegister(i-6), F54, F52, F48);
4149 __ aes_dround23_l(as_FloatRegister(i-4), F56, F58, F62);
4150 __ aes_dround01_l(as_FloatRegister(i-6), F56, F58, F60);
4151 }
4152 }
4153
4154 __ movxtod(L0,F54);
4155 __ movxtod(L1,F52);
4156 __ fxor(FloatRegisterImpl::D, F54, F48, F48);
4157 __ fxor(FloatRegisterImpl::D, F52, F50, F50);
4158
4159 __ movxtod(G4,F56);
4160 __ movxtod(G5,F58);
4161 __ mov(L4,L0);
4162 __ mov(L5,L1);
4163 __ fxor(FloatRegisterImpl::D, F56, F60, F60);
4164 __ fxor(FloatRegisterImpl::D, F58, F62, F62);
4165
4166 // check for 8-byte alignment since dest byte array may have arbitrary alignment if offset mod 8 is non-zero
4167 __ andcc(to, 7, G1);
4168 __ br(Assembler::notZero, true, Assembler::pn, L_store_misaligned_output_next2_blocks192);
4169 __ delayed()->edge8n(to, G0, G2);
4170
4171 // aligned case: store output into the destination array
4172 __ stf(FloatRegisterImpl::D, F48, to, 0);
4173 __ stf(FloatRegisterImpl::D, F50, to, 8);
4174 __ stf(FloatRegisterImpl::D, F60, to, 16);
4175 __ stf(FloatRegisterImpl::D, F62, to, 24);
4176 __ ba_short(L_check_decrypt_loop_end192);
4177
4178 __ BIND(L_store_misaligned_output_next2_blocks192);
4179 __ mov(8, G4);
4180 __ sub(G4, G1, G4);
4181 __ alignaddr(G4, G0, G4);
4182 __ faligndata(F48, F50, F56); // F56 can be clobbered
4183 __ faligndata(F50, F60, F50);
4184 __ faligndata(F60, F62, F60);
4185 __ faligndata(F62, F48, F48);
4186 __ mov(to, G1);
4187 __ and3(to, -8, to);
4188 __ stpartialf(to, G2, F48, Assembler::ASI_PST8_PRIMARY);
4189 __ stf(FloatRegisterImpl::D, F56, to, 8);
4190 __ stf(FloatRegisterImpl::D, F50, to, 16);
4191 __ stf(FloatRegisterImpl::D, F60, to, 24);
4192 __ add(to, 32, to);
4193 __ orn(G0, G2, G2);
4194 __ stpartialf(to, G2, F48, Assembler::ASI_PST8_PRIMARY);
4195 __ mov(G1, to);
4196
4197 __ BIND(L_check_decrypt_loop_end192);
4198 __ add(from, 32, from);
4199 __ add(to, 32, to);
4200 __ subcc(len_reg, 32, len_reg);
4201 __ br(Assembler::notEqual, false, Assembler::pt, L_dec_next2_blocks192);
4202 __ delayed()->nop();
4203 __ ba_short(L_cbcdec_end);
4204
4205 __ align(OptoLoopAlignment);
4206 __ BIND(L_dec_next2_blocks256);
4207 __ nop();
4208
4209 // check for 8-byte alignment since source byte array may have an arbitrary alignment if offset mod 8 is non-zero
4210 __ andcc(from, 7, G0);
4211 __ br(Assembler::notZero, true, Assembler::pn, L_load_misaligned_next2_blocks256);
4212 __ delayed()->mov(from, G1); // save original 'from' address before alignaddr
4213
4214 // aligned case: load input into G4, G5, L4 and L5
4215 __ ldx(from,0,G4);
4216 __ ldx(from,8,G5);
4217 __ ldx(from,16,L4);
4218 __ ldx(from,24,L5);
4219 __ ba_short(L_transform_next2_blocks256);
4220
4221 __ BIND(L_load_misaligned_next2_blocks256);
4222 __ alignaddr(from, G0, from);
4223 // F0, F2, F4, F60, F62 can be clobbered
4224 __ ldf(FloatRegisterImpl::D, from, 0, F0);
4225 __ ldf(FloatRegisterImpl::D, from, 8, F2);
4226 __ ldf(FloatRegisterImpl::D, from, 16, F60);
4227 __ ldf(FloatRegisterImpl::D, from, 24, F62);
4228 __ ldf(FloatRegisterImpl::D, from, 32, F4);
4229 __ faligndata(F0, F2, F0);
4230 __ faligndata(F2, F60, F2);
4231 __ faligndata(F60, F62, F60);
4232 __ faligndata(F62, F4, F62);
4233 __ movdtox(F0, G4);
4234 __ movdtox(F2, G5);
4235 __ movdtox(F60, L4);
4236 __ movdtox(F62, L5);
4237 __ mov(G1, from);
4238
4239 __ BIND(L_transform_next2_blocks256);
4240 // F0:F2 used for first 16-bytes
4241 __ xor3(L2,G4,G1);
4242 __ movxtod(G1,F0);
4243 __ xor3(L3,G5,G1);
4244 __ movxtod(G1,F2);
4245
4246 // F60:F62 used for next 16-bytes
4247 __ xor3(L2,L4,G1);
4248 __ movxtod(G1,F60);
4249 __ xor3(L3,L5,G1);
4250 __ movxtod(G1,F62);
4251
4252 __ aes_dround23(F54, F0, F2, F4);
4253 __ aes_dround01(F52, F0, F2, F6);
4254 __ aes_dround23(F54, F60, F62, F58);
4255 __ aes_dround01(F52, F60, F62, F56);
4256 __ aes_dround23(F50, F6, F4, F2);
4257 __ aes_dround01(F48, F6, F4, F0);
4258 __ aes_dround23(F50, F56, F58, F62);
4259 __ aes_dround01(F48, F56, F58, F60);
4260 // save F48:F54 in temp registers
4261 __ movdtox(F54,G2);
4262 __ movdtox(F52,G3);
4263 __ movdtox(F50,L6);
4264 __ movdtox(F48,G1);
4265 for ( int i = 46; i >= 14; i -= 8 ) {
4266 __ aes_dround23(as_FloatRegister(i), F0, F2, F4);
4267 __ aes_dround01(as_FloatRegister(i-2), F0, F2, F6);
4268 __ aes_dround23(as_FloatRegister(i), F60, F62, F58);
4269 __ aes_dround01(as_FloatRegister(i-2), F60, F62, F56);
4270 __ aes_dround23(as_FloatRegister(i-4), F6, F4, F2);
4271 __ aes_dround01(as_FloatRegister(i-6), F6, F4, F0);
4272 __ aes_dround23(as_FloatRegister(i-4), F56, F58, F62);
4273 __ aes_dround01(as_FloatRegister(i-6), F56, F58, F60);
4274 }
4275 // init F48:F54 with F0:F6 values (original key)
4276 __ ldf(FloatRegisterImpl::D, original_key, 0, F48);
4277 __ ldf(FloatRegisterImpl::D, original_key, 8, F50);
4278 __ ldf(FloatRegisterImpl::D, original_key, 16, F52);
4279 __ ldf(FloatRegisterImpl::D, original_key, 24, F54);
4280 __ aes_dround23(F54, F0, F2, F4);
4281 __ aes_dround01(F52, F0, F2, F6);
4282 __ aes_dround23(F54, F60, F62, F58);
4283 __ aes_dround01(F52, F60, F62, F56);
4284 __ aes_dround23_l(F50, F6, F4, F2);
4285 __ aes_dround01_l(F48, F6, F4, F0);
4286 __ aes_dround23_l(F50, F56, F58, F62);
4287 __ aes_dround01_l(F48, F56, F58, F60);
4288 // re-init F48:F54 with their original values
4289 __ movxtod(G2,F54);
4290 __ movxtod(G3,F52);
4291 __ movxtod(L6,F50);
4292 __ movxtod(G1,F48);
4293
4294 __ movxtod(L0,F6);
4295 __ movxtod(L1,F4);
4296 __ fxor(FloatRegisterImpl::D, F6, F0, F0);
4297 __ fxor(FloatRegisterImpl::D, F4, F2, F2);
4298
4299 __ movxtod(G4,F56);
4300 __ movxtod(G5,F58);
4301 __ mov(L4,L0);
4302 __ mov(L5,L1);
4303 __ fxor(FloatRegisterImpl::D, F56, F60, F60);
4304 __ fxor(FloatRegisterImpl::D, F58, F62, F62);
4305
4306 // check for 8-byte alignment since dest byte array may have arbitrary alignment if offset mod 8 is non-zero
4307 __ andcc(to, 7, G1);
4308 __ br(Assembler::notZero, true, Assembler::pn, L_store_misaligned_output_next2_blocks256);
4309 __ delayed()->edge8n(to, G0, G2);
4310
4311 // aligned case: store output into the destination array
4312 __ stf(FloatRegisterImpl::D, F0, to, 0);
4313 __ stf(FloatRegisterImpl::D, F2, to, 8);
4314 __ stf(FloatRegisterImpl::D, F60, to, 16);
4315 __ stf(FloatRegisterImpl::D, F62, to, 24);
4316 __ ba_short(L_check_decrypt_loop_end256);
4317
4318 __ BIND(L_store_misaligned_output_next2_blocks256);
4319 __ mov(8, G4);
4320 __ sub(G4, G1, G4);
4321 __ alignaddr(G4, G0, G4);
4322 __ faligndata(F0, F2, F56); // F56 can be clobbered
4323 __ faligndata(F2, F60, F2);
4324 __ faligndata(F60, F62, F60);
4325 __ faligndata(F62, F0, F0);
4326 __ mov(to, G1);
4327 __ and3(to, -8, to);
4328 __ stpartialf(to, G2, F0, Assembler::ASI_PST8_PRIMARY);
4329 __ stf(FloatRegisterImpl::D, F56, to, 8);
4330 __ stf(FloatRegisterImpl::D, F2, to, 16);
4331 __ stf(FloatRegisterImpl::D, F60, to, 24);
4332 __ add(to, 32, to);
4333 __ orn(G0, G2, G2);
4334 __ stpartialf(to, G2, F0, Assembler::ASI_PST8_PRIMARY);
4335 __ mov(G1, to);
4336
4337 __ BIND(L_check_decrypt_loop_end256);
4338 __ add(from, 32, from);
4339 __ add(to, 32, to);
4340 __ subcc(len_reg, 32, len_reg);
4341 __ br(Assembler::notEqual, false, Assembler::pt, L_dec_next2_blocks256);
4342 __ delayed()->nop();
4343
4344 __ BIND(L_cbcdec_end);
4345 // re-init intial vector for next block, 8-byte alignment is guaranteed
4346 __ stx(L0, rvec, 0);
4347 __ stx(L1, rvec, 8);
4348 __ mov(L7, I0);
4349 __ ret();
4350 __ delayed()->restore();
4351
4352 return start;
4353 }
4354
4355 address generate_sha1_implCompress(bool multi_block, const char *name) {
4356 __ align(CodeEntryAlignment);
4357 StubCodeMark mark(this, "StubRoutines", name);
4358 address start = __ pc();
4359
4360 Label L_sha1_loop, L_sha1_unaligned_input, L_sha1_unaligned_input_loop;
4361 int i;
4362
4363 Register buf = O0; // byte[] source+offset
4364 Register state = O1; // int[] SHA.state
4365 Register ofs = O2; // int offset
4366 Register limit = O3; // int limit
4367
4368 // load state into F0-F4
4369 for (i = 0; i < 5; i++) {
4370 __ ldf(FloatRegisterImpl::S, state, i*4, as_FloatRegister(i));
4371 }
4372
4373 __ andcc(buf, 7, G0);
4374 __ br(Assembler::notZero, false, Assembler::pn, L_sha1_unaligned_input);
4375 __ delayed()->nop();
4376
4377 __ BIND(L_sha1_loop);
4378 // load buf into F8-F22
4379 for (i = 0; i < 8; i++) {
4380 __ ldf(FloatRegisterImpl::D, buf, i*8, as_FloatRegister(i*2 + 8));
4381 }
4382 __ sha1();
4383 if (multi_block) {
4384 __ add(ofs, 64, ofs);
4385 __ add(buf, 64, buf);
4386 __ cmp_and_brx_short(ofs, limit, Assembler::lessEqual, Assembler::pt, L_sha1_loop);
4387 __ mov(ofs, O0); // to be returned
4388 }
4389
4390 // store F0-F4 into state and return
4391 for (i = 0; i < 4; i++) {
4392 __ stf(FloatRegisterImpl::S, as_FloatRegister(i), state, i*4);
4393 }
4394 __ retl();
4395 __ delayed()->stf(FloatRegisterImpl::S, F4, state, 0x10);
4396
4397 __ BIND(L_sha1_unaligned_input);
4398 __ alignaddr(buf, G0, buf);
4399
4400 __ BIND(L_sha1_unaligned_input_loop);
4401 // load buf into F8-F22
4402 for (i = 0; i < 9; i++) {
4403 __ ldf(FloatRegisterImpl::D, buf, i*8, as_FloatRegister(i*2 + 8));
4404 }
4405 for (i = 0; i < 8; i++) {
4406 __ faligndata(as_FloatRegister(i*2 + 8), as_FloatRegister(i*2 + 10), as_FloatRegister(i*2 + 8));
4407 }
4408 __ sha1();
4409 if (multi_block) {
4410 __ add(ofs, 64, ofs);
4411 __ add(buf, 64, buf);
4412 __ cmp_and_brx_short(ofs, limit, Assembler::lessEqual, Assembler::pt, L_sha1_unaligned_input_loop);
4413 __ mov(ofs, O0); // to be returned
4414 }
4415
4416 // store F0-F4 into state and return
4417 for (i = 0; i < 4; i++) {
4418 __ stf(FloatRegisterImpl::S, as_FloatRegister(i), state, i*4);
4419 }
4420 __ retl();
4421 __ delayed()->stf(FloatRegisterImpl::S, F4, state, 0x10);
4422
4423 return start;
4424 }
4425
4426 address generate_sha256_implCompress(bool multi_block, const char *name) {
4427 __ align(CodeEntryAlignment);
4428 StubCodeMark mark(this, "StubRoutines", name);
4429 address start = __ pc();
4430
4431 Label L_sha256_loop, L_sha256_unaligned_input, L_sha256_unaligned_input_loop;
4432 int i;
4433
4434 Register buf = O0; // byte[] source+offset
4435 Register state = O1; // int[] SHA2.state
4436 Register ofs = O2; // int offset
4437 Register limit = O3; // int limit
4438
4439 // load state into F0-F7
4440 for (i = 0; i < 8; i++) {
4441 __ ldf(FloatRegisterImpl::S, state, i*4, as_FloatRegister(i));
4442 }
4443
4444 __ andcc(buf, 7, G0);
4445 __ br(Assembler::notZero, false, Assembler::pn, L_sha256_unaligned_input);
4446 __ delayed()->nop();
4447
4448 __ BIND(L_sha256_loop);
4449 // load buf into F8-F22
4450 for (i = 0; i < 8; i++) {
4451 __ ldf(FloatRegisterImpl::D, buf, i*8, as_FloatRegister(i*2 + 8));
4452 }
4453 __ sha256();
4454 if (multi_block) {
4455 __ add(ofs, 64, ofs);
4456 __ add(buf, 64, buf);
4457 __ cmp_and_brx_short(ofs, limit, Assembler::lessEqual, Assembler::pt, L_sha256_loop);
4458 __ mov(ofs, O0); // to be returned
4459 }
4460
4461 // store F0-F7 into state and return
4462 for (i = 0; i < 7; i++) {
4463 __ stf(FloatRegisterImpl::S, as_FloatRegister(i), state, i*4);
4464 }
4465 __ retl();
4466 __ delayed()->stf(FloatRegisterImpl::S, F7, state, 0x1c);
4467
4468 __ BIND(L_sha256_unaligned_input);
4469 __ alignaddr(buf, G0, buf);
4470
4471 __ BIND(L_sha256_unaligned_input_loop);
4472 // load buf into F8-F22
4473 for (i = 0; i < 9; i++) {
4474 __ ldf(FloatRegisterImpl::D, buf, i*8, as_FloatRegister(i*2 + 8));
4475 }
4476 for (i = 0; i < 8; i++) {
4477 __ faligndata(as_FloatRegister(i*2 + 8), as_FloatRegister(i*2 + 10), as_FloatRegister(i*2 + 8));
4478 }
4479 __ sha256();
4480 if (multi_block) {
4481 __ add(ofs, 64, ofs);
4482 __ add(buf, 64, buf);
4483 __ cmp_and_brx_short(ofs, limit, Assembler::lessEqual, Assembler::pt, L_sha256_unaligned_input_loop);
4484 __ mov(ofs, O0); // to be returned
4485 }
4486
4487 // store F0-F7 into state and return
4488 for (i = 0; i < 7; i++) {
4489 __ stf(FloatRegisterImpl::S, as_FloatRegister(i), state, i*4);
4490 }
4491 __ retl();
4492 __ delayed()->stf(FloatRegisterImpl::S, F7, state, 0x1c);
4493
4494 return start;
4495 }
4496
4497 address generate_sha512_implCompress(bool multi_block, const char *name) {
4498 __ align(CodeEntryAlignment);
4499 StubCodeMark mark(this, "StubRoutines", name);
4500 address start = __ pc();
4501
4502 Label L_sha512_loop, L_sha512_unaligned_input, L_sha512_unaligned_input_loop;
4503 int i;
4504
4505 Register buf = O0; // byte[] source+offset
4506 Register state = O1; // long[] SHA5.state
4507 Register ofs = O2; // int offset
4508 Register limit = O3; // int limit
4509
4510 // load state into F0-F14
4511 for (i = 0; i < 8; i++) {
4512 __ ldf(FloatRegisterImpl::D, state, i*8, as_FloatRegister(i*2));
4513 }
4514
4515 __ andcc(buf, 7, G0);
4516 __ br(Assembler::notZero, false, Assembler::pn, L_sha512_unaligned_input);
4517 __ delayed()->nop();
4518
4519 __ BIND(L_sha512_loop);
4520 // load buf into F16-F46
4521 for (i = 0; i < 16; i++) {
4522 __ ldf(FloatRegisterImpl::D, buf, i*8, as_FloatRegister(i*2 + 16));
4523 }
4524 __ sha512();
4525 if (multi_block) {
4526 __ add(ofs, 128, ofs);
4527 __ add(buf, 128, buf);
4528 __ cmp_and_brx_short(ofs, limit, Assembler::lessEqual, Assembler::pt, L_sha512_loop);
4529 __ mov(ofs, O0); // to be returned
4530 }
4531
4532 // store F0-F14 into state and return
4533 for (i = 0; i < 7; i++) {
4534 __ stf(FloatRegisterImpl::D, as_FloatRegister(i*2), state, i*8);
4535 }
4536 __ retl();
4537 __ delayed()->stf(FloatRegisterImpl::D, F14, state, 0x38);
4538
4539 __ BIND(L_sha512_unaligned_input);
4540 __ alignaddr(buf, G0, buf);
4541
4542 __ BIND(L_sha512_unaligned_input_loop);
4543 // load buf into F16-F46
4544 for (i = 0; i < 17; i++) {
4545 __ ldf(FloatRegisterImpl::D, buf, i*8, as_FloatRegister(i*2 + 16));
4546 }
4547 for (i = 0; i < 16; i++) {
4548 __ faligndata(as_FloatRegister(i*2 + 16), as_FloatRegister(i*2 + 18), as_FloatRegister(i*2 + 16));
4549 }
4550 __ sha512();
4551 if (multi_block) {
4552 __ add(ofs, 128, ofs);
4553 __ add(buf, 128, buf);
4554 __ cmp_and_brx_short(ofs, limit, Assembler::lessEqual, Assembler::pt, L_sha512_unaligned_input_loop);
4555 __ mov(ofs, O0); // to be returned
4556 }
4557
4558 // store F0-F14 into state and return
4559 for (i = 0; i < 7; i++) {
4560 __ stf(FloatRegisterImpl::D, as_FloatRegister(i*2), state, i*8);
4561 }
4562 __ retl();
4563 __ delayed()->stf(FloatRegisterImpl::D, F14, state, 0x38);
4564
4565 return start;
4566 }
4567
4568 /* Single and multi-block ghash operations */
4569 address generate_ghash_processBlocks() {
4570 __ align(CodeEntryAlignment);
4571 Label L_ghash_loop, L_aligned, L_main;
4572 StubCodeMark mark(this, "StubRoutines", "ghash_processBlocks");
4573 address start = __ pc();
4574
4575 Register state = I0;
4576 Register subkeyH = I1;
4577 Register data = I2;
4578 Register len = I3;
4579
4580 __ save_frame(0);
4581
4582 __ ldx(state, 0, O0);
4583 __ ldx(state, 8, O1);
4584
4585 // Loop label for multiblock operations
4586 __ BIND(L_ghash_loop);
4587
4588 // Check if 'data' is unaligned
4589 __ andcc(data, 7, G1);
4590 __ br(Assembler::zero, false, Assembler::pt, L_aligned);
4591 __ delayed()->nop();
4592
4593 Register left_shift = L1;
4594 Register right_shift = L2;
4595 Register data_ptr = L3;
4596
4597 // Get left and right shift values in bits
4598 __ sll(G1, LogBitsPerByte, left_shift);
4599 __ mov(64, right_shift);
4600 __ sub(right_shift, left_shift, right_shift);
4601
4602 // Align to read 'data'
4603 __ sub(data, G1, data_ptr);
4604
4605 // Load first 8 bytes of 'data'
4606 __ ldx(data_ptr, 0, O4);
4607 __ sllx(O4, left_shift, O4);
4608 __ ldx(data_ptr, 8, O5);
4609 __ srlx(O5, right_shift, G4);
4610 __ bset(G4, O4);
4611
4612 // Load second 8 bytes of 'data'
4613 __ sllx(O5, left_shift, O5);
4614 __ ldx(data_ptr, 16, G4);
4615 __ srlx(G4, right_shift, G4);
4616 __ ba(L_main);
4617 __ delayed()->bset(G4, O5);
4618
4619 // If 'data' is aligned, load normally
4620 __ BIND(L_aligned);
4621 __ ldx(data, 0, O4);
4622 __ ldx(data, 8, O5);
4623
4624 __ BIND(L_main);
4625 __ ldx(subkeyH, 0, O2);
4626 __ ldx(subkeyH, 8, O3);
4627
4628 __ xor3(O0, O4, O0);
4629 __ xor3(O1, O5, O1);
4630
4631 __ xmulxhi(O0, O3, G3);
4632 __ xmulx(O0, O2, O5);
4633 __ xmulxhi(O1, O2, G4);
4634 __ xmulxhi(O1, O3, G5);
4635 __ xmulx(O0, O3, G1);
4636 __ xmulx(O1, O3, G2);
4637 __ xmulx(O1, O2, O3);
4638 __ xmulxhi(O0, O2, O4);
4639
4640 __ mov(0xE1, O0);
4641 __ sllx(O0, 56, O0);
4642
4643 __ xor3(O5, G3, O5);
4644 __ xor3(O5, G4, O5);
4645 __ xor3(G5, G1, G1);
4646 __ xor3(G1, O3, G1);
4647 __ srlx(G2, 63, O1);
4648 __ srlx(G1, 63, G3);
4649 __ sllx(G2, 63, O3);
4650 __ sllx(G2, 58, O2);
4651 __ xor3(O3, O2, O2);
4652
4653 __ sllx(G1, 1, G1);
4654 __ or3(G1, O1, G1);
4655
4656 __ xor3(G1, O2, G1);
4657
4658 __ sllx(G2, 1, G2);
4659
4660 __ xmulxhi(G1, O0, O1);
4661 __ xmulx(G1, O0, O2);
4662 __ xmulxhi(G2, O0, O3);
4663 __ xmulx(G2, O0, G1);
4664
4665 __ xor3(O4, O1, O4);
4666 __ xor3(O5, O2, O5);
4667 __ xor3(O5, O3, O5);
4668
4669 __ sllx(O4, 1, O2);
4670 __ srlx(O5, 63, O3);
4671
4672 __ or3(O2, O3, O0);
4673
4674 __ sllx(O5, 1, O1);
4675 __ srlx(G1, 63, O2);
4676 __ or3(O1, O2, O1);
4677 __ xor3(O1, G3, O1);
4678
4679 __ deccc(len);
4680 __ br(Assembler::notZero, true, Assembler::pt, L_ghash_loop);
4681 __ delayed()->add(data, 16, data);
4682
4683 __ stx(O0, I0, 0);
4684 __ stx(O1, I0, 8);
4685
4686 __ ret();
4687 __ delayed()->restore();
4688
4689 return start;
4690 }
4691
4692 /**
4693 * Arguments:
4694 *
4695 * Inputs:
4696 * O0 - int crc
4697 * O1 - byte* buf
4698 * O2 - int len
4699 * O3 - int* table
4700 *
4701 * Output:
4702 * O0 - int crc result
4703 */
4704 address generate_updateBytesCRC32C() {
4705 assert(UseCRC32CIntrinsics, "need CRC32C instruction");
4706
4707 __ align(CodeEntryAlignment);
4708 StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32C");
4709 address start = __ pc();
4710
4711 const Register crc = O0; // crc
4712 const Register buf = O1; // source java byte array address
4713 const Register len = O2; // number of bytes
4714 const Register table = O3; // byteTable
4715
4716 __ kernel_crc32c(crc, buf, len, table);
4717
4718 __ retl();
4719 __ delayed()->nop();
4720
4721 return start;
4722 }
4723
4724 #define ADLER32_NUM_TEMPS 16
4725
4726 /**
4727 * Arguments:
4728 *
4729 * Inputs:
4730 * O0 - int adler
4731 * O1 - byte* buff
4732 * O2 - int len
4733 *
4734 * Output:
4735 * O0 - int adler result
4736 */
4737 address generate_updateBytesAdler32() {
4738 __ align(CodeEntryAlignment);
4739 StubCodeMark mark(this, "StubRoutines", "updateBytesAdler32");
4740 address start = __ pc();
4741
4742 Label L_cleanup_loop, L_cleanup_loop_check;
4743 Label L_main_loop_check, L_main_loop, L_inner_loop, L_inner_loop_check;
4744 Label L_nmax_check_done;
4745
4746 // Aliases
4747 Register s1 = O0;
4748 Register s2 = O3;
4749 Register buff = O1;
4750 Register len = O2;
4751 Register temp[ADLER32_NUM_TEMPS] = {L0, L1, L2, L3, L4, L5, L6, L7, I0, I1, I2, I3, I4, I5, G3, I7};
4752
4753 // Max number of bytes we can process before having to take the mod
4754 // 0x15B0 is 5552 in decimal, the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1
4755 unsigned long NMAX = 0x15B0;
4756
4757 // Zero-out the upper bits of len
4758 __ clruwu(len);
4759
4760 // Create the mask 0xFFFF
4761 __ set64(0x00FFFF, O4, O5); // O5 is the temp register
4762
4763 // s1 is initialized to the lower 16 bits of adler
4764 // s2 is initialized to the upper 16 bits of adler
4765 __ srlx(O0, 16, O5); // adler >> 16
4766 __ and3(O0, O4, s1); // s1 = (adler & 0xFFFF)
4767 __ and3(O5, O4, s2); // s2 = ((adler >> 16) & 0xFFFF)
4768
4769 // The pipelined loop needs at least 16 elements for 1 iteration
4770 // It does check this, but it is more effective to skip to the cleanup loop
4771 // Setup the constant for cutoff checking
4772 __ mov(15, O4);
4773
4774 // Check if we are above the cutoff, if not go to the cleanup loop immediately
4775 __ cmp_and_br_short(len, O4, Assembler::lessEqualUnsigned, Assembler::pt, L_cleanup_loop_check);
4776
4777 // Free up some registers for our use
4778 for (int i = 0; i < ADLER32_NUM_TEMPS; i++) {
4779 __ movxtod(temp[i], as_FloatRegister(2*i));
4780 }
4781
4782 // Loop maintenance stuff is done at the end of the loop, so skip to there
4783 __ ba_short(L_main_loop_check);
4784
4785 __ BIND(L_main_loop);
4786
4787 // Prologue for inner loop
4788 __ ldub(buff, 0, L0);
4789 __ dec(O5);
4790
4791 for (int i = 1; i < 8; i++) {
4792 __ ldub(buff, i, temp[i]);
4793 }
4794
4795 __ inc(buff, 8);
4796
4797 // Inner loop processes 16 elements at a time, might never execute if only 16 elements
4798 // to be processed by the outter loop
4799 __ ba_short(L_inner_loop_check);
4800
4801 __ BIND(L_inner_loop);
4802
4803 for (int i = 0; i < 8; i++) {
4804 __ ldub(buff, (2*i), temp[(8+(2*i)) % ADLER32_NUM_TEMPS]);
4805 __ add(s1, temp[i], s1);
4806 __ ldub(buff, (2*i)+1, temp[(8+(2*i)+1) % ADLER32_NUM_TEMPS]);
4807 __ add(s2, s1, s2);
4808 }
4809
4810 // Original temp 0-7 used and new loads to temp 0-7 issued
4811 // temp 8-15 ready to be consumed
4812 __ add(s1, I0, s1);
4813 __ dec(O5);
4814 __ add(s2, s1, s2);
4815 __ add(s1, I1, s1);
4816 __ inc(buff, 16);
4817 __ add(s2, s1, s2);
4818
4819 for (int i = 0; i < 6; i++) {
4820 __ add(s1, temp[10+i], s1);
4821 __ add(s2, s1, s2);
4822 }
4823
4824 __ BIND(L_inner_loop_check);
4825 __ nop();
4826 __ cmp_and_br_short(O5, 0, Assembler::notEqual, Assembler::pt, L_inner_loop);
4827
4828 // Epilogue
4829 for (int i = 0; i < 4; i++) {
4830 __ ldub(buff, (2*i), temp[8+(2*i)]);
4831 __ add(s1, temp[i], s1);
4832 __ ldub(buff, (2*i)+1, temp[8+(2*i)+1]);
4833 __ add(s2, s1, s2);
4834 }
4835
4836 __ add(s1, temp[4], s1);
4837 __ inc(buff, 8);
4838
4839 for (int i = 0; i < 11; i++) {
4840 __ add(s2, s1, s2);
4841 __ add(s1, temp[5+i], s1);
4842 }
4843
4844 __ add(s2, s1, s2);
4845
4846 // Take the mod for s1 and s2
4847 __ set64(0xFFF1, L0, L1);
4848 __ udivx(s1, L0, L1);
4849 __ udivx(s2, L0, L2);
4850 __ mulx(L0, L1, L1);
4851 __ mulx(L0, L2, L2);
4852 __ sub(s1, L1, s1);
4853 __ sub(s2, L2, s2);
4854
4855 // Make sure there is something left to process
4856 __ BIND(L_main_loop_check);
4857 __ set64(NMAX, L0, L1);
4858 // k = len < NMAX ? len : NMAX
4859 __ cmp_and_br_short(len, L0, Assembler::greaterEqualUnsigned, Assembler::pt, L_nmax_check_done);
4860 __ andn(len, 0x0F, L0); // only loop a multiple of 16 times
4861 __ BIND(L_nmax_check_done);
4862 __ mov(L0, O5);
4863 __ sub(len, L0, len); // len -= k
4864
4865 __ srlx(O5, 4, O5); // multiplies of 16
4866 __ cmp_and_br_short(O5, 0, Assembler::notEqual, Assembler::pt, L_main_loop);
4867
4868 // Restore anything we used, take the mod one last time, combine and return
4869 // Restore any registers we saved
4870 for (int i = 0; i < ADLER32_NUM_TEMPS; i++) {
4871 __ movdtox(as_FloatRegister(2*i), temp[i]);
4872 }
4873
4874 // There might be nothing left to process
4875 __ ba_short(L_cleanup_loop_check);
4876
4877 __ BIND(L_cleanup_loop);
4878 __ ldub(buff, 0, O4); // load single byte form buffer
4879 __ inc(buff); // buff++
4880 __ add(s1, O4, s1); // s1 += *buff++;
4881 __ dec(len); // len--
4882 __ add(s1, s2, s2); // s2 += s1;
4883 __ BIND(L_cleanup_loop_check);
4884 __ nop();
4885 __ cmp_and_br_short(len, 0, Assembler::notEqual, Assembler::pt, L_cleanup_loop);
4886
4887 // Take the mod one last time
4888 __ set64(0xFFF1, O1, O2);
4889 __ udivx(s1, O1, O2);
4890 __ udivx(s2, O1, O5);
4891 __ mulx(O1, O2, O2);
4892 __ mulx(O1, O5, O5);
4893 __ sub(s1, O2, s1);
4894 __ sub(s2, O5, s2);
4895
4896 // Combine lower bits and higher bits
4897 __ sllx(s2, 16, s2); // s2 = s2 << 16
4898 __ or3(s1, s2, s1); // adler = s2 | s1
4899 // Final return value is in O0
4900 __ retl();
4901 __ delayed()->nop();
4902
4903 return start;
4904 }
4905
4906 /**
4907 * Arguments:
4908 *
4909 * Inputs:
4910 * O0 - int crc
4911 * O1 - byte* buf
4912 * O2 - int len
4913 * O3 - int* table
4914 *
4915 * Output:
4916 * O0 - int crc result
4917 */
4918 address generate_updateBytesCRC32() {
4919 assert(UseCRC32Intrinsics, "need VIS3 instructions");
4920
4921 __ align(CodeEntryAlignment);
4922 StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32");
4923 address start = __ pc();
4924
4925 const Register crc = O0; // crc
4926 const Register buf = O1; // source java byte array address
4927 const Register len = O2; // length
4928 const Register table = O3; // crc_table address (reuse register)
4929
4930 __ kernel_crc32(crc, buf, len, table);
4931
4932 __ retl();
4933 __ delayed()->nop();
4934
4935 return start;
4936 }
4937
4938 /**
4939 * Arguments:
4940 *
4941 * Inputs:
4942 * I0 - int* x-addr
4943 * I1 - int x-len
4944 * I2 - int* y-addr
4945 * I3 - int y-len
4946 * I4 - int* z-addr (output vector)
4947 * I5 - int z-len
4948 */
4949 address generate_multiplyToLen() {
4950 assert(UseMultiplyToLenIntrinsic, "need VIS3 instructions");
4951
4952 __ align(CodeEntryAlignment);
4953 StubCodeMark mark(this, "StubRoutines", "multiplyToLen");
4954 address start = __ pc();
4955
4956 __ save_frame(0);
4957
4958 const Register xptr = I0; // input address
4959 const Register xlen = I1; // ...and length in 32b-words
4960 const Register yptr = I2; //
4961 const Register ylen = I3; //
4962 const Register zptr = I4; // output address
4963 const Register zlen = I5; // ...and length in 32b-words
4964
4965 /* The minimal "limb" representation suggest that odd length vectors are as
4966 * likely as even length dittos. This in turn suggests that we need to cope
4967 * with odd/even length arrays and data not aligned properly for 64-bit read
4968 * and write operations. We thus use a number of different kernels:
4969 *
4970 * if (is_even(x.len) && is_even(y.len))
4971 * if (is_align64(x) && is_align64(y) && is_align64(z))
4972 * if (x.len == y.len && 16 <= x.len && x.len <= 64)
4973 * memv_mult_mpmul(...)
4974 * else
4975 * memv_mult_64x64(...)
4976 * else
4977 * memv_mult_64x64u(...)
4978 * else
4979 * memv_mult_32x32(...)
4980 *
4981 * Here we assume VIS3 support (for 'umulxhi', 'addxc' and 'addxccc').
4982 * In case CBCOND instructions are supported, we will use 'cxbX'. If the
4983 * MPMUL instruction is supported, we will generate a kernel using 'mpmul'
4984 * (for vectors with proper characteristics).
4985 */
4986 const Register tmp0 = L0;
4987 const Register tmp1 = L1;
4988
4989 Label L_mult_32x32;
4990 Label L_mult_64x64u;
4991 Label L_mult_64x64;
4992 Label L_exit;
4993
4994 if_both_even(xlen, ylen, tmp0, false, L_mult_32x32);
4995 if_all3_aligned(xptr, yptr, zptr, tmp1, 64, false, L_mult_64x64u);
4996
4997 if (UseMPMUL) {
4998 if_eq(xlen, ylen, false, L_mult_64x64);
4999 if_in_rng(xlen, 16, 64, tmp0, tmp1, false, L_mult_64x64);
5000
5001 // 1. Multiply naturally aligned 64b-datums using a generic 'mpmul' kernel,
5002 // operating on equal length vectors of size [16..64].
5003 gen_mult_mpmul(xlen, xptr, yptr, zptr, L_exit);
5004 }
5005
5006 // 2. Multiply naturally aligned 64-bit datums (64x64).
5007 __ bind(L_mult_64x64);
5008 gen_mult_64x64(xptr, xlen, yptr, ylen, zptr, zlen, L_exit);
5009
5010 // 3. Multiply unaligned 64-bit datums (64x64).
5011 __ bind(L_mult_64x64u);
5012 gen_mult_64x64_unaligned(xptr, xlen, yptr, ylen, zptr, zlen, L_exit);
5013
5014 // 4. Multiply naturally aligned 32-bit datums (32x32).
5015 __ bind(L_mult_32x32);
5016 gen_mult_32x32(xptr, xlen, yptr, ylen, zptr, zlen, L_exit);
5017
5018 __ bind(L_exit);
5019 __ ret();
5020 __ delayed()->restore();
5021
5022 return start;
5023 }
5024
5025 // Additional help functions used by multiplyToLen generation.
5026
5027 void if_both_even(Register r1, Register r2, Register tmp, bool iseven, Label &L)
5028 {
5029 __ or3(r1, r2, tmp);
5030 __ andcc(tmp, 0x1, tmp);
5031 __ br_icc_zero(iseven, Assembler::pn, L);
5032 }
5033
5034 void if_all3_aligned(Register r1, Register r2, Register r3,
5035 Register tmp, uint align, bool isalign, Label &L)
5036 {
5037 __ or3(r1, r2, tmp);
5038 __ or3(r3, tmp, tmp);
5039 __ andcc(tmp, (align - 1), tmp);
5040 __ br_icc_zero(isalign, Assembler::pn, L);
5041 }
5042
5043 void if_eq(Register x, Register y, bool iseq, Label &L)
5044 {
5045 Assembler::Condition cf = (iseq ? Assembler::equal : Assembler::notEqual);
5046 __ cmp_and_br_short(x, y, cf, Assembler::pt, L);
5047 }
5048
5049 void if_in_rng(Register x, int lb, int ub, Register t1, Register t2, bool inrng, Label &L)
5050 {
5051 assert(Assembler::is_simm13(lb), "Small ints only!");
5052 assert(Assembler::is_simm13(ub), "Small ints only!");
5053 // Compute (x - lb) * (ub - x) >= 0
5054 // NOTE: With the local use of this routine, we rely on small integers to
5055 // guarantee that we do not overflow in the multiplication.
5056 __ add(G0, ub, t2);
5057 __ sub(x, lb, t1);
5058 __ sub(t2, x, t2);
5059 __ mulx(t1, t2, t1);
5060 Assembler::Condition cf = (inrng ? Assembler::greaterEqual : Assembler::less);
5061 __ cmp_and_br_short(t1, G0, cf, Assembler::pt, L);
5062 }
5063
5064 void ldd_entry(Register base, Register offs, FloatRegister dest)
5065 {
5066 __ ldd(base, offs, dest);
5067 __ inc(offs, 8);
5068 }
5069
5070 void ldx_entry(Register base, Register offs, Register dest)
5071 {
5072 __ ldx(base, offs, dest);
5073 __ inc(offs, 8);
5074 }
5075
5076 void mpmul_entry(int m, Label &next)
5077 {
5078 __ mpmul(m);
5079 __ cbcond(Assembler::equal, Assembler::icc, G0, G0, next);
5080 }
5081
5082 void stx_entry(Label &L, Register r1, Register r2, Register base, Register offs)
5083 {
5084 __ bind(L);
5085 __ stx(r1, base, offs);
5086 __ inc(offs, 8);
5087 __ stx(r2, base, offs);
5088 __ inc(offs, 8);
5089 }
5090
5091 void offs_entry(Label &Lbl0, Label &Lbl1)
5092 {
5093 assert(Lbl0.is_bound(), "must be");
5094 assert(Lbl1.is_bound(), "must be");
5095
5096 int offset = Lbl0.loc_pos() - Lbl1.loc_pos();
5097
5098 __ emit_data(offset);
5099 }
5100
5101 /* Generate the actual multiplication kernels for BigInteger vectors:
5102 *
5103 * 1. gen_mult_mpmul(...)
5104 *
5105 * 2. gen_mult_64x64(...)
5106 *
5107 * 3. gen_mult_64x64_unaligned(...)
5108 *
5109 * 4. gen_mult_32x32(...)
5110 */
5111 void gen_mult_mpmul(Register len, Register xptr, Register yptr, Register zptr,
5112 Label &L_exit)
5113 {
5114 const Register zero = G0;
5115 const Register gxp = G1; // Need to use global registers across RWs.
5116 const Register gyp = G2;
5117 const Register gzp = G3;
5118 const Register disp = G4;
5119 const Register offs = G5;
5120
5121 __ mov(xptr, gxp);
5122 __ mov(yptr, gyp);
5123 __ mov(zptr, gzp);
5124
5125 /* Compute jump vector entry:
5126 *
5127 * 1. mpmul input size (0..31) x 64b
5128 * 2. vector input size in 32b limbs (even number)
5129 * 3. branch entries in reverse order (31..0), using two
5130 * instructions per entry (2 * 4 bytes).
5131 *
5132 * displacement = byte_offset(bra_offset(len))
5133 * = byte_offset((64 - len)/2)
5134 * = 8 * (64 - len)/2
5135 * = 4 * (64 - len)
5136 */
5137 Register temp = I5; // Alright to use input regs. in first batch.
5138
5139 __ sub(zero, len, temp);
5140 __ add(temp, 64, temp);
5141 __ sllx(temp, 2, disp); // disp := (64 - len) << 2
5142
5143 // Dispatch relative current PC, into instruction table below.
5144 __ rdpc(temp);
5145 __ add(temp, 16, temp);
5146 __ jmp(temp, disp);
5147 __ delayed()->clr(offs);
5148
5149 ldd_entry(gxp, offs, F22);
5150 ldd_entry(gxp, offs, F20);
5151 ldd_entry(gxp, offs, F18);
5152 ldd_entry(gxp, offs, F16);
5153 ldd_entry(gxp, offs, F14);
5154 ldd_entry(gxp, offs, F12);
5155 ldd_entry(gxp, offs, F10);
5156 ldd_entry(gxp, offs, F8);
5157 ldd_entry(gxp, offs, F6);
5158 ldd_entry(gxp, offs, F4);
5159 ldx_entry(gxp, offs, I5);
5160 ldx_entry(gxp, offs, I4);
5161 ldx_entry(gxp, offs, I3);
5162 ldx_entry(gxp, offs, I2);
5163 ldx_entry(gxp, offs, I1);
5164 ldx_entry(gxp, offs, I0);
5165 ldx_entry(gxp, offs, L7);
5166 ldx_entry(gxp, offs, L6);
5167 ldx_entry(gxp, offs, L5);
5168 ldx_entry(gxp, offs, L4);
5169 ldx_entry(gxp, offs, L3);
5170 ldx_entry(gxp, offs, L2);
5171 ldx_entry(gxp, offs, L1);
5172 ldx_entry(gxp, offs, L0);
5173 ldd_entry(gxp, offs, F2);
5174 ldd_entry(gxp, offs, F0);
5175 ldx_entry(gxp, offs, O5);
5176 ldx_entry(gxp, offs, O4);
5177 ldx_entry(gxp, offs, O3);
5178 ldx_entry(gxp, offs, O2);
5179 ldx_entry(gxp, offs, O1);
5180 ldx_entry(gxp, offs, O0);
5181
5182 __ save(SP, -176, SP);
5183
5184 const Register addr = gxp; // Alright to reuse 'gxp'.
5185
5186 // Dispatch relative current PC, into instruction table below.
5187 __ rdpc(addr);
5188 __ add(addr, 16, addr);
5189 __ jmp(addr, disp);
5190 __ delayed()->clr(offs);
5191
5192 ldd_entry(gyp, offs, F58);
5193 ldd_entry(gyp, offs, F56);
5194 ldd_entry(gyp, offs, F54);
5195 ldd_entry(gyp, offs, F52);
5196 ldd_entry(gyp, offs, F50);
5197 ldd_entry(gyp, offs, F48);
5198 ldd_entry(gyp, offs, F46);
5199 ldd_entry(gyp, offs, F44);
5200 ldd_entry(gyp, offs, F42);
5201 ldd_entry(gyp, offs, F40);
5202 ldd_entry(gyp, offs, F38);
5203 ldd_entry(gyp, offs, F36);
5204 ldd_entry(gyp, offs, F34);
5205 ldd_entry(gyp, offs, F32);
5206 ldd_entry(gyp, offs, F30);
5207 ldd_entry(gyp, offs, F28);
5208 ldd_entry(gyp, offs, F26);
5209 ldd_entry(gyp, offs, F24);
5210 ldx_entry(gyp, offs, O5);
5211 ldx_entry(gyp, offs, O4);
5212 ldx_entry(gyp, offs, O3);
5213 ldx_entry(gyp, offs, O2);
5214 ldx_entry(gyp, offs, O1);
5215 ldx_entry(gyp, offs, O0);
5216 ldx_entry(gyp, offs, L7);
5217 ldx_entry(gyp, offs, L6);
5218 ldx_entry(gyp, offs, L5);
5219 ldx_entry(gyp, offs, L4);
5220 ldx_entry(gyp, offs, L3);
5221 ldx_entry(gyp, offs, L2);
5222 ldx_entry(gyp, offs, L1);
5223 ldx_entry(gyp, offs, L0);
5224
5225 __ save(SP, -176, SP);
5226 __ save(SP, -176, SP);
5227 __ save(SP, -176, SP);
5228 __ save(SP, -176, SP);
5229 __ save(SP, -176, SP);
5230
5231 Label L_mpmul_restore_4, L_mpmul_restore_3, L_mpmul_restore_2;
5232 Label L_mpmul_restore_1, L_mpmul_restore_0;
5233
5234 // Dispatch relative current PC, into instruction table below.
5235 __ rdpc(addr);
5236 __ add(addr, 16, addr);
5237 __ jmp(addr, disp);
5238 __ delayed()->clr(offs);
5239
5240 mpmul_entry(31, L_mpmul_restore_0);
5241 mpmul_entry(30, L_mpmul_restore_0);
5242 mpmul_entry(29, L_mpmul_restore_0);
5243 mpmul_entry(28, L_mpmul_restore_0);
5244 mpmul_entry(27, L_mpmul_restore_1);
5245 mpmul_entry(26, L_mpmul_restore_1);
5246 mpmul_entry(25, L_mpmul_restore_1);
5247 mpmul_entry(24, L_mpmul_restore_1);
5248 mpmul_entry(23, L_mpmul_restore_1);
5249 mpmul_entry(22, L_mpmul_restore_1);
5250 mpmul_entry(21, L_mpmul_restore_1);
5251 mpmul_entry(20, L_mpmul_restore_2);
5252 mpmul_entry(19, L_mpmul_restore_2);
5253 mpmul_entry(18, L_mpmul_restore_2);
5254 mpmul_entry(17, L_mpmul_restore_2);
5255 mpmul_entry(16, L_mpmul_restore_2);
5256 mpmul_entry(15, L_mpmul_restore_2);
5257 mpmul_entry(14, L_mpmul_restore_2);
5258 mpmul_entry(13, L_mpmul_restore_3);
5259 mpmul_entry(12, L_mpmul_restore_3);
5260 mpmul_entry(11, L_mpmul_restore_3);
5261 mpmul_entry(10, L_mpmul_restore_3);
5262 mpmul_entry( 9, L_mpmul_restore_3);
5263 mpmul_entry( 8, L_mpmul_restore_3);
5264 mpmul_entry( 7, L_mpmul_restore_3);
5265 mpmul_entry( 6, L_mpmul_restore_4);
5266 mpmul_entry( 5, L_mpmul_restore_4);
5267 mpmul_entry( 4, L_mpmul_restore_4);
5268 mpmul_entry( 3, L_mpmul_restore_4);
5269 mpmul_entry( 2, L_mpmul_restore_4);
5270 mpmul_entry( 1, L_mpmul_restore_4);
5271 mpmul_entry( 0, L_mpmul_restore_4);
5272
5273 Label L_z31, L_z30, L_z29, L_z28, L_z27, L_z26, L_z25, L_z24;
5274 Label L_z23, L_z22, L_z21, L_z20, L_z19, L_z18, L_z17, L_z16;
5275 Label L_z15, L_z14, L_z13, L_z12, L_z11, L_z10, L_z09, L_z08;
5276 Label L_z07, L_z06, L_z05, L_z04, L_z03, L_z02, L_z01, L_z00;
5277
5278 Label L_zst_base; // Store sequence base address.
5279 __ bind(L_zst_base);
5280
5281 stx_entry(L_z31, L7, L6, gzp, offs);
5282 stx_entry(L_z30, L5, L4, gzp, offs);
5283 stx_entry(L_z29, L3, L2, gzp, offs);
5284 stx_entry(L_z28, L1, L0, gzp, offs);
5285 __ restore();
5286 stx_entry(L_z27, O5, O4, gzp, offs);
5287 stx_entry(L_z26, O3, O2, gzp, offs);
5288 stx_entry(L_z25, O1, O0, gzp, offs);
5289 stx_entry(L_z24, L7, L6, gzp, offs);
5290 stx_entry(L_z23, L5, L4, gzp, offs);
5291 stx_entry(L_z22, L3, L2, gzp, offs);
5292 stx_entry(L_z21, L1, L0, gzp, offs);
5293 __ restore();
5294 stx_entry(L_z20, O5, O4, gzp, offs);
5295 stx_entry(L_z19, O3, O2, gzp, offs);
5296 stx_entry(L_z18, O1, O0, gzp, offs);
5297 stx_entry(L_z17, L7, L6, gzp, offs);
5298 stx_entry(L_z16, L5, L4, gzp, offs);
5299 stx_entry(L_z15, L3, L2, gzp, offs);
5300 stx_entry(L_z14, L1, L0, gzp, offs);
5301 __ restore();
5302 stx_entry(L_z13, O5, O4, gzp, offs);
5303 stx_entry(L_z12, O3, O2, gzp, offs);
5304 stx_entry(L_z11, O1, O0, gzp, offs);
5305 stx_entry(L_z10, L7, L6, gzp, offs);
5306 stx_entry(L_z09, L5, L4, gzp, offs);
5307 stx_entry(L_z08, L3, L2, gzp, offs);
5308 stx_entry(L_z07, L1, L0, gzp, offs);
5309 __ restore();
5310 stx_entry(L_z06, O5, O4, gzp, offs);
5311 stx_entry(L_z05, O3, O2, gzp, offs);
5312 stx_entry(L_z04, O1, O0, gzp, offs);
5313 stx_entry(L_z03, L7, L6, gzp, offs);
5314 stx_entry(L_z02, L5, L4, gzp, offs);
5315 stx_entry(L_z01, L3, L2, gzp, offs);
5316 stx_entry(L_z00, L1, L0, gzp, offs);
5317
5318 __ restore();
5319 __ restore();
5320 // Exit out of 'mpmul' routine, back to multiplyToLen.
5321 __ ba_short(L_exit);
5322
5323 Label L_zst_offs;
5324 __ bind(L_zst_offs);
5325
5326 offs_entry(L_z31, L_zst_base); // index 31: 2048x2048
5327 offs_entry(L_z30, L_zst_base);
5328 offs_entry(L_z29, L_zst_base);
5329 offs_entry(L_z28, L_zst_base);
5330 offs_entry(L_z27, L_zst_base);
5331 offs_entry(L_z26, L_zst_base);
5332 offs_entry(L_z25, L_zst_base);
5333 offs_entry(L_z24, L_zst_base);
5334 offs_entry(L_z23, L_zst_base);
5335 offs_entry(L_z22, L_zst_base);
5336 offs_entry(L_z21, L_zst_base);
5337 offs_entry(L_z20, L_zst_base);
5338 offs_entry(L_z19, L_zst_base);
5339 offs_entry(L_z18, L_zst_base);
5340 offs_entry(L_z17, L_zst_base);
5341 offs_entry(L_z16, L_zst_base);
5342 offs_entry(L_z15, L_zst_base);
5343 offs_entry(L_z14, L_zst_base);
5344 offs_entry(L_z13, L_zst_base);
5345 offs_entry(L_z12, L_zst_base);
5346 offs_entry(L_z11, L_zst_base);
5347 offs_entry(L_z10, L_zst_base);
5348 offs_entry(L_z09, L_zst_base);
5349 offs_entry(L_z08, L_zst_base);
5350 offs_entry(L_z07, L_zst_base);
5351 offs_entry(L_z06, L_zst_base);
5352 offs_entry(L_z05, L_zst_base);
5353 offs_entry(L_z04, L_zst_base);
5354 offs_entry(L_z03, L_zst_base);
5355 offs_entry(L_z02, L_zst_base);
5356 offs_entry(L_z01, L_zst_base);
5357 offs_entry(L_z00, L_zst_base); // index 0: 64x64
5358
5359 __ bind(L_mpmul_restore_4);
5360 __ restore();
5361 __ bind(L_mpmul_restore_3);
5362 __ restore();
5363 __ bind(L_mpmul_restore_2);
5364 __ restore();
5365 __ bind(L_mpmul_restore_1);
5366 __ restore();
5367 __ bind(L_mpmul_restore_0);
5368
5369 // Dispatch via offset vector entry, into z-store sequence.
5370 Label L_zst_rdpc;
5371 __ bind(L_zst_rdpc);
5372
5373 assert(L_zst_base.is_bound(), "must be");
5374 assert(L_zst_offs.is_bound(), "must be");
5375 assert(L_zst_rdpc.is_bound(), "must be");
5376
5377 int dbase = L_zst_rdpc.loc_pos() - L_zst_base.loc_pos();
5378 int doffs = L_zst_rdpc.loc_pos() - L_zst_offs.loc_pos();
5379
5380 temp = gyp; // Alright to reuse 'gyp'.
5381
5382 __ rdpc(addr);
5383 __ sub(addr, doffs, temp);
5384 __ srlx(disp, 1, disp);
5385 __ lduw(temp, disp, offs);
5386 __ sub(addr, dbase, temp);
5387 __ jmp(temp, offs);
5388 __ delayed()->clr(offs);
5389 }
5390
5391 void gen_mult_64x64(Register xp, Register xn,
5392 Register yp, Register yn,
5393 Register zp, Register zn, Label &L_exit)
5394 {
5395 // Assuming that a stack frame has already been created, i.e. local and
5396 // output registers are available for immediate use.
5397
5398 const Register ri = L0; // Outer loop index, xv[i]
5399 const Register rj = L1; // Inner loop index, yv[j]
5400 const Register rk = L2; // Output loop index, zv[k]
5401 const Register rx = L4; // x-vector datum [i]
5402 const Register ry = L5; // y-vector datum [j]
5403 const Register rz = L6; // z-vector datum [k]
5404 const Register rc = L7; // carry over (to z-vector datum [k-1])
5405
5406 const Register lop = O0; // lo-64b product
5407 const Register hip = O1; // hi-64b product
5408
5409 const Register zero = G0;
5410
5411 Label L_loop_i, L_exit_loop_i;
5412 Label L_loop_j;
5413 Label L_loop_i2, L_exit_loop_i2;
5414
5415 __ srlx(xn, 1, xn); // index for u32 to u64 ditto
5416 __ srlx(yn, 1, yn); // index for u32 to u64 ditto
5417 __ srlx(zn, 1, zn); // index for u32 to u64 ditto
5418 __ dec(xn); // Adjust [0..(N/2)-1]
5419 __ dec(yn);
5420 __ dec(zn);
5421 __ clr(rc); // u64 c = 0
5422 __ sllx(xn, 3, ri); // int i = xn (byte offset i = 8*xn)
5423 __ sllx(yn, 3, rj); // int j = yn (byte offset i = 8*xn)
5424 __ sllx(zn, 3, rk); // int k = zn (byte offset k = 8*zn)
5425 __ ldx(yp, rj, ry); // u64 y = yp[yn]
5426
5427 // for (int i = xn; i >= 0; i--)
5428 __ bind(L_loop_i);
5429
5430 __ cmp_and_br_short(ri, 0, // i >= 0
5431 Assembler::less, Assembler::pn, L_exit_loop_i);
5432 __ ldx(xp, ri, rx); // x = xp[i]
5433 __ mulx(rx, ry, lop); // lo-64b-part of result 64x64
5434 __ umulxhi(rx, ry, hip); // hi-64b-part of result 64x64
5435 __ addcc(rc, lop, lop); // Accumulate lower order bits (producing carry)
5436 __ addxc(hip, zero, rc); // carry over to next datum [k-1]
5437 __ stx(lop, zp, rk); // z[k] = lop
5438 __ dec(rk, 8); // k--
5439 __ dec(ri, 8); // i--
5440 __ ba_short(L_loop_i);
5441
5442 __ bind(L_exit_loop_i);
5443 __ stx(rc, zp, rk); // z[k] = c
5444
5445 // for (int j = yn - 1; j >= 0; j--)
5446 __ sllx(yn, 3, rj); // int j = yn - 1 (byte offset j = 8*yn)
5447 __ dec(rj, 8);
5448
5449 __ bind(L_loop_j);
5450
5451 __ cmp_and_br_short(rj, 0, // j >= 0
5452 Assembler::less, Assembler::pn, L_exit);
5453 __ clr(rc); // u64 c = 0
5454 __ ldx(yp, rj, ry); // u64 y = yp[j]
5455
5456 // for (int i = xn, k = --zn; i >= 0; i--)
5457 __ dec(zn); // --zn
5458 __ sllx(xn, 3, ri); // int i = xn (byte offset i = 8*xn)
5459 __ sllx(zn, 3, rk); // int k = zn (byte offset k = 8*zn)
5460
5461 __ bind(L_loop_i2);
5462
5463 __ cmp_and_br_short(ri, 0, // i >= 0
5464 Assembler::less, Assembler::pn, L_exit_loop_i2);
5465 __ ldx(xp, ri, rx); // x = xp[i]
5466 __ ldx(zp, rk, rz); // z = zp[k], accumulator
5467 __ mulx(rx, ry, lop); // lo-64b-part of result 64x64
5468 __ umulxhi(rx, ry, hip); // hi-64b-part of result 64x64
5469 __ addcc(rz, rc, rz); // Accumulate lower order bits,
5470 __ addxc(hip, zero, rc); // Accumulate higher order bits to carry
5471 __ addcc(rz, lop, rz); // z += lo(p) + c
5472 __ addxc(rc, zero, rc);
5473 __ stx(rz, zp, rk); // zp[k] = z
5474 __ dec(rk, 8); // k--
5475 __ dec(ri, 8); // i--
5476 __ ba_short(L_loop_i2);
5477
5478 __ bind(L_exit_loop_i2);
5479 __ stx(rc, zp, rk); // z[k] = c
5480 __ dec(rj, 8); // j--
5481 __ ba_short(L_loop_j);
5482 }
5483
5484 void gen_mult_64x64_unaligned(Register xp, Register xn,
5485 Register yp, Register yn,
5486 Register zp, Register zn, Label &L_exit)
5487 {
5488 // Assuming that a stack frame has already been created, i.e. local and
5489 // output registers are available for use.
5490
5491 const Register xpc = L0; // Outer loop cursor, xp[i]
5492 const Register ypc = L1; // Inner loop cursor, yp[j]
5493 const Register zpc = L2; // Output loop cursor, zp[k]
5494 const Register rx = L4; // x-vector datum [i]
5495 const Register ry = L5; // y-vector datum [j]
5496 const Register rz = L6; // z-vector datum [k]
5497 const Register rc = L7; // carry over (to z-vector datum [k-1])
5498 const Register rt = O2;
5499
5500 const Register lop = O0; // lo-64b product
5501 const Register hip = O1; // hi-64b product
5502
5503 const Register zero = G0;
5504
5505 Label L_loop_i, L_exit_loop_i;
5506 Label L_loop_j;
5507 Label L_loop_i2, L_exit_loop_i2;
5508
5509 __ srlx(xn, 1, xn); // index for u32 to u64 ditto
5510 __ srlx(yn, 1, yn); // index for u32 to u64 ditto
5511 __ srlx(zn, 1, zn); // index for u32 to u64 ditto
5512 __ dec(xn); // Adjust [0..(N/2)-1]
5513 __ dec(yn);
5514 __ dec(zn);
5515 __ clr(rc); // u64 c = 0
5516 __ sllx(xn, 3, xpc); // u32* xpc = &xp[xn] (byte offset 8*xn)
5517 __ add(xp, xpc, xpc);
5518 __ sllx(yn, 3, ypc); // u32* ypc = &yp[yn] (byte offset 8*yn)
5519 __ add(yp, ypc, ypc);
5520 __ sllx(zn, 3, zpc); // u32* zpc = &zp[zn] (byte offset 8*zn)
5521 __ add(zp, zpc, zpc);
5522 __ lduw(ypc, 0, rt); // u64 y = yp[yn]
5523 __ lduw(ypc, 4, ry); // ...
5524 __ sllx(rt, 32, rt);
5525 __ or3(rt, ry, ry);
5526
5527 // for (int i = xn; i >= 0; i--)
5528 __ bind(L_loop_i);
5529
5530 __ cmp_and_brx_short(xpc, xp,// i >= 0
5531 Assembler::lessUnsigned, Assembler::pn, L_exit_loop_i);
5532 __ lduw(xpc, 0, rt); // u64 x = xp[i]
5533 __ lduw(xpc, 4, rx); // ...
5534 __ sllx(rt, 32, rt);
5535 __ or3(rt, rx, rx);
5536 __ mulx(rx, ry, lop); // lo-64b-part of result 64x64
5537 __ umulxhi(rx, ry, hip); // hi-64b-part of result 64x64
5538 __ addcc(rc, lop, lop); // Accumulate lower order bits (producing carry)
5539 __ addxc(hip, zero, rc); // carry over to next datum [k-1]
5540 __ srlx(lop, 32, rt);
5541 __ stw(rt, zpc, 0); // z[k] = lop
5542 __ stw(lop, zpc, 4); // ...
5543 __ dec(zpc, 8); // k-- (zpc--)
5544 __ dec(xpc, 8); // i-- (xpc--)
5545 __ ba_short(L_loop_i);
5546
5547 __ bind(L_exit_loop_i);
5548 __ srlx(rc, 32, rt);
5549 __ stw(rt, zpc, 0); // z[k] = c
5550 __ stw(rc, zpc, 4);
5551
5552 // for (int j = yn - 1; j >= 0; j--)
5553 __ sllx(yn, 3, ypc); // u32* ypc = &yp[yn] (byte offset 8*yn)
5554 __ add(yp, ypc, ypc);
5555 __ dec(ypc, 8); // yn - 1 (ypc--)
5556
5557 __ bind(L_loop_j);
5558
5559 __ cmp_and_brx_short(ypc, yp,// j >= 0
5560 Assembler::lessUnsigned, Assembler::pn, L_exit);
5561 __ clr(rc); // u64 c = 0
5562 __ lduw(ypc, 0, rt); // u64 y = yp[j] (= *ypc)
5563 __ lduw(ypc, 4, ry); // ...
5564 __ sllx(rt, 32, rt);
5565 __ or3(rt, ry, ry);
5566
5567 // for (int i = xn, k = --zn; i >= 0; i--)
5568 __ sllx(xn, 3, xpc); // u32* xpc = &xp[xn] (byte offset 8*xn)
5569 __ add(xp, xpc, xpc);
5570 __ dec(zn); // --zn
5571 __ sllx(zn, 3, zpc); // u32* zpc = &zp[zn] (byte offset 8*zn)
5572 __ add(zp, zpc, zpc);
5573
5574 __ bind(L_loop_i2);
5575
5576 __ cmp_and_brx_short(xpc, xp,// i >= 0
5577 Assembler::lessUnsigned, Assembler::pn, L_exit_loop_i2);
5578 __ lduw(xpc, 0, rt); // u64 x = xp[i] (= *xpc)
5579 __ lduw(xpc, 4, rx); // ...
5580 __ sllx(rt, 32, rt);
5581 __ or3(rt, rx, rx);
5582
5583 __ lduw(zpc, 0, rt); // u64 z = zp[k] (= *zpc)
5584 __ lduw(zpc, 4, rz); // ...
5585 __ sllx(rt, 32, rt);
5586 __ or3(rt, rz, rz);
5587
5588 __ mulx(rx, ry, lop); // lo-64b-part of result 64x64
5589 __ umulxhi(rx, ry, hip); // hi-64b-part of result 64x64
5590 __ addcc(rz, rc, rz); // Accumulate lower order bits...
5591 __ addxc(hip, zero, rc); // Accumulate higher order bits to carry
5592 __ addcc(rz, lop, rz); // ... z += lo(p) + c
5593 __ addxccc(rc, zero, rc);
5594 __ srlx(rz, 32, rt);
5595 __ stw(rt, zpc, 0); // zp[k] = z (*zpc = z)
5596 __ stw(rz, zpc, 4);
5597 __ dec(zpc, 8); // k-- (zpc--)
5598 __ dec(xpc, 8); // i-- (xpc--)
5599 __ ba_short(L_loop_i2);
5600
5601 __ bind(L_exit_loop_i2);
5602 __ srlx(rc, 32, rt);
5603 __ stw(rt, zpc, 0); // z[k] = c
5604 __ stw(rc, zpc, 4);
5605 __ dec(ypc, 8); // j-- (ypc--)
5606 __ ba_short(L_loop_j);
5607 }
5608
5609 void gen_mult_32x32(Register xp, Register xn,
5610 Register yp, Register yn,
5611 Register zp, Register zn, Label &L_exit)
5612 {
5613 // Assuming that a stack frame has already been created, i.e. local and
5614 // output registers are available for use.
5615
5616 const Register ri = L0; // Outer loop index, xv[i]
5617 const Register rj = L1; // Inner loop index, yv[j]
5618 const Register rk = L2; // Output loop index, zv[k]
5619 const Register rx = L4; // x-vector datum [i]
5620 const Register ry = L5; // y-vector datum [j]
5621 const Register rz = L6; // z-vector datum [k]
5622 const Register rc = L7; // carry over (to z-vector datum [k-1])
5623
5624 const Register p64 = O0; // 64b product
5625 const Register z65 = O1; // carry+64b accumulator
5626 const Register c65 = O2; // carry at bit 65
5627 const Register c33 = O2; // carry at bit 33 (after shift)
5628
5629 const Register zero = G0;
5630
5631 Label L_loop_i, L_exit_loop_i;
5632 Label L_loop_j;
5633 Label L_loop_i2, L_exit_loop_i2;
5634
5635 __ dec(xn); // Adjust [0..N-1]
5636 __ dec(yn);
5637 __ dec(zn);
5638 __ clr(rc); // u32 c = 0
5639 __ sllx(xn, 2, ri); // int i = xn (byte offset i = 4*xn)
5640 __ sllx(yn, 2, rj); // int j = yn (byte offset i = 4*xn)
5641 __ sllx(zn, 2, rk); // int k = zn (byte offset k = 4*zn)
5642 __ lduw(yp, rj, ry); // u32 y = yp[yn]
5643
5644 // for (int i = xn; i >= 0; i--)
5645 __ bind(L_loop_i);
5646
5647 __ cmp_and_br_short(ri, 0, // i >= 0
5648 Assembler::less, Assembler::pn, L_exit_loop_i);
5649 __ lduw(xp, ri, rx); // x = xp[i]
5650 __ mulx(rx, ry, p64); // 64b result of 32x32
5651 __ addcc(rc, p64, z65); // Accumulate to 65 bits (producing carry)
5652 __ addxc(zero, zero, c65); // Materialise carry (in bit 65) into lsb,
5653 __ sllx(c65, 32, c33); // and shift into bit 33
5654 __ srlx(z65, 32, rc); // carry = c33 | hi(z65) >> 32
5655 __ add(c33, rc, rc); // carry over to next datum [k-1]
5656 __ stw(z65, zp, rk); // z[k] = lo(z65)
5657 __ dec(rk, 4); // k--
5658 __ dec(ri, 4); // i--
5659 __ ba_short(L_loop_i);
5660
5661 __ bind(L_exit_loop_i);
5662 __ stw(rc, zp, rk); // z[k] = c
5663
5664 // for (int j = yn - 1; j >= 0; j--)
5665 __ sllx(yn, 2, rj); // int j = yn - 1 (byte offset j = 4*yn)
5666 __ dec(rj, 4);
5667
5668 __ bind(L_loop_j);
5669
5670 __ cmp_and_br_short(rj, 0, // j >= 0
5671 Assembler::less, Assembler::pn, L_exit);
5672 __ clr(rc); // u32 c = 0
5673 __ lduw(yp, rj, ry); // u32 y = yp[j]
5674
5675 // for (int i = xn, k = --zn; i >= 0; i--)
5676 __ dec(zn); // --zn
5677 __ sllx(xn, 2, ri); // int i = xn (byte offset i = 4*xn)
5678 __ sllx(zn, 2, rk); // int k = zn (byte offset k = 4*zn)
5679
5680 __ bind(L_loop_i2);
5681
5682 __ cmp_and_br_short(ri, 0, // i >= 0
5683 Assembler::less, Assembler::pn, L_exit_loop_i2);
5684 __ lduw(xp, ri, rx); // x = xp[i]
5685 __ lduw(zp, rk, rz); // z = zp[k], accumulator
5686 __ mulx(rx, ry, p64); // 64b result of 32x32
5687 __ add(rz, rc, rz); // Accumulate lower order bits,
5688 __ addcc(rz, p64, z65); // z += lo(p64) + c
5689 __ addxc(zero, zero, c65); // Materialise carry (in bit 65) into lsb,
5690 __ sllx(c65, 32, c33); // and shift into bit 33
5691 __ srlx(z65, 32, rc); // carry = c33 | hi(z65) >> 32
5692 __ add(c33, rc, rc); // carry over to next datum [k-1]
5693 __ stw(z65, zp, rk); // zp[k] = lo(z65)
5694 __ dec(rk, 4); // k--
5695 __ dec(ri, 4); // i--
5696 __ ba_short(L_loop_i2);
5697
5698 __ bind(L_exit_loop_i2);
5699 __ stw(rc, zp, rk); // z[k] = c
5700 __ dec(rj, 4); // j--
5701 __ ba_short(L_loop_j);
5702 }
5703
5704
5705 void generate_initial() {
5706 // Generates all stubs and initializes the entry points
5707
5708 //------------------------------------------------------------------------------------------------------------------------
5709 // entry points that exist in all platforms
5710 // Note: This is code that could be shared among different platforms - however the benefit seems to be smaller than
5711 // the disadvantage of having a much more complicated generator structure. See also comment in stubRoutines.hpp.
5712 StubRoutines::_forward_exception_entry = generate_forward_exception();
5713
5714 StubRoutines::_call_stub_entry = generate_call_stub(StubRoutines::_call_stub_return_address);
5715 StubRoutines::_catch_exception_entry = generate_catch_exception();
5716
5717 //------------------------------------------------------------------------------------------------------------------------
5718 // entry points that are platform specific
5719 StubRoutines::Sparc::_test_stop_entry = generate_test_stop();
5720
5721 StubRoutines::Sparc::_stop_subroutine_entry = generate_stop_subroutine();
5722 StubRoutines::Sparc::_flush_callers_register_windows_entry = generate_flush_callers_register_windows();
5723
5724 // Build this early so it's available for the interpreter.
5725 StubRoutines::_throw_StackOverflowError_entry =
5726 generate_throw_exception("StackOverflowError throw_exception",
5727 CAST_FROM_FN_PTR(address, SharedRuntime::throw_StackOverflowError));
5728 StubRoutines::_throw_delayed_StackOverflowError_entry =
5729 generate_throw_exception("delayed StackOverflowError throw_exception",
5730 CAST_FROM_FN_PTR(address, SharedRuntime::throw_delayed_StackOverflowError));
5731
5732 if (UseCRC32Intrinsics) {
5733 // set table address before stub generation which use it
5734 StubRoutines::_crc_table_adr = (address)StubRoutines::Sparc::_crc_table;
5735 StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
5736 }
5737
5738 if (UseCRC32CIntrinsics) {
5739 // set table address before stub generation which use it
5740 StubRoutines::_crc32c_table_addr = (address)StubRoutines::Sparc::_crc32c_table;
5741 StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C();
5742 }
5743 }
5744
5745
5746 void generate_all() {
5747 // Generates all stubs and initializes the entry points
5748
5749 // Generate partial_subtype_check first here since its code depends on
5750 // UseZeroBaseCompressedOops which is defined after heap initialization.
5751 StubRoutines::Sparc::_partial_subtype_check = generate_partial_subtype_check();
5752 // These entry points require SharedInfo::stack0 to be set up in non-core builds
5753 StubRoutines::_throw_AbstractMethodError_entry = generate_throw_exception("AbstractMethodError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_AbstractMethodError));
5754 StubRoutines::_throw_IncompatibleClassChangeError_entry= generate_throw_exception("IncompatibleClassChangeError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_IncompatibleClassChangeError));
5755 StubRoutines::_throw_NullPointerException_at_call_entry= generate_throw_exception("NullPointerException at call throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_NullPointerException_at_call));
5756
5757 // support for verify_oop (must happen after universe_init)
5758 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop_subroutine();
5759
5760 // arraycopy stubs used by compilers
5761 generate_arraycopy_stubs();
5762
5763 // Don't initialize the platform math functions since sparc
5764 // doesn't have intrinsics for these operations.
5765
5766 // Safefetch stubs.
5767 generate_safefetch("SafeFetch32", sizeof(int), &StubRoutines::_safefetch32_entry,
5768 &StubRoutines::_safefetch32_fault_pc,
5769 &StubRoutines::_safefetch32_continuation_pc);
5770 generate_safefetch("SafeFetchN", sizeof(intptr_t), &StubRoutines::_safefetchN_entry,
5771 &StubRoutines::_safefetchN_fault_pc,
5772 &StubRoutines::_safefetchN_continuation_pc);
5773
5774 // generate AES intrinsics code
5775 if (UseAESIntrinsics) {
5776 StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
5777 StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
5778 StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
5779 StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt_Parallel();
5780 }
5781 // generate GHASH intrinsics code
5782 if (UseGHASHIntrinsics) {
5783 StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks();
5784 }
5785
5786 // generate SHA1/SHA256/SHA512 intrinsics code
5787 if (UseSHA1Intrinsics) {
5788 StubRoutines::_sha1_implCompress = generate_sha1_implCompress(false, "sha1_implCompress");
5789 StubRoutines::_sha1_implCompressMB = generate_sha1_implCompress(true, "sha1_implCompressMB");
5790 }
5791 if (UseSHA256Intrinsics) {
5792 StubRoutines::_sha256_implCompress = generate_sha256_implCompress(false, "sha256_implCompress");
5793 StubRoutines::_sha256_implCompressMB = generate_sha256_implCompress(true, "sha256_implCompressMB");
5794 }
5795 if (UseSHA512Intrinsics) {
5796 StubRoutines::_sha512_implCompress = generate_sha512_implCompress(false, "sha512_implCompress");
5797 StubRoutines::_sha512_implCompressMB = generate_sha512_implCompress(true, "sha512_implCompressMB");
5798 }
5799 // generate Adler32 intrinsics code
5800 if (UseAdler32Intrinsics) {
5801 StubRoutines::_updateBytesAdler32 = generate_updateBytesAdler32();
5802 }
5803
5804 #ifdef COMPILER2
5805 // Intrinsics supported by C2 only:
5806 if (UseMultiplyToLenIntrinsic) {
5807 StubRoutines::_multiplyToLen = generate_multiplyToLen();
5808 }
5809 #endif // COMPILER2
5810 }
5811
5812 public:
5813 StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) {
5814 // replace the standard masm with a special one:
5815 _masm = new MacroAssembler(code);
5816
5817 _stub_count = !all ? 0x100 : 0x200;
5818 if (all) {
5819 generate_all();
5820 } else {
5821 generate_initial();
5822 }
5823
5824 // make sure this stub is available for all local calls
5825 if (_atomic_add_stub.is_unbound()) {
5826 // generate a second time, if necessary
5827 (void) generate_atomic_add();
5828 }
5829 }
5830
5831
5832 private:
5833 int _stub_count;
5834 void stub_prolog(StubCodeDesc* cdesc) {
5835 # ifdef ASSERT
5836 // put extra information in the stub code, to make it more readable
5837 // Write the high part of the address
5838 // [RGV] Check if there is a dependency on the size of this prolog
5839 __ emit_data((intptr_t)cdesc >> 32, relocInfo::none);
5840 __ emit_data((intptr_t)cdesc, relocInfo::none);
5841 __ emit_data(++_stub_count, relocInfo::none);
5842 # endif
5843 align(true);
5844 }
5845
5846 void align(bool at_header = false) {
5847 // %%%%% move this constant somewhere else
5848 // UltraSPARC cache line size is 8 instructions:
5849 const unsigned int icache_line_size = 32;
5850 const unsigned int icache_half_line_size = 16;
5851
5852 if (at_header) {
5853 while ((intptr_t)(__ pc()) % icache_line_size != 0) {
5854 __ emit_data(0, relocInfo::none);
5855 }
5856 } else {
5857 while ((intptr_t)(__ pc()) % icache_half_line_size != 0) {
5858 __ nop();
5859 }
5860 }
5861 }
5862
5863 }; // end class declaration
5864
5865 #define UCM_TABLE_MAX_ENTRIES 8
5866 void StubGenerator_generate(CodeBuffer* code, bool all) {
5867 if (UnsafeCopyMemory::_table == NULL) {
5868 UnsafeCopyMemory::create_table(UCM_TABLE_MAX_ENTRIES);
5869 }
5870 StubGenerator g(code, all);
5871 }