9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "asm/macroAssembler.hpp"
28 #include "asm/macroAssembler.inline.hpp"
29 #include "gc/shared/cardTable.hpp"
30 #include "gc/shared/cardTableModRefBS.hpp"
31 #include "interpreter/interpreter.hpp"
32 #include "nativeInst_aarch64.hpp"
33 #include "oops/instanceOop.hpp"
34 #include "oops/method.hpp"
35 #include "oops/objArrayKlass.hpp"
36 #include "oops/oop.inline.hpp"
37 #include "prims/methodHandles.hpp"
38 #include "runtime/frame.inline.hpp"
39 #include "runtime/handles.inline.hpp"
40 #include "runtime/sharedRuntime.hpp"
41 #include "runtime/stubCodeGenerator.hpp"
42 #include "runtime/stubRoutines.hpp"
43 #include "runtime/thread.inline.hpp"
44 #include "utilities/align.hpp"
45 #ifdef COMPILER2
46 #include "opto/runtime.hpp"
47 #endif
48
603 __ bind(error);
604 __ ldp(c_rarg3, c_rarg2, Address(__ post(sp, 16)));
605
606 __ push(RegSet::range(r0, r29), sp);
607 // debug(char* msg, int64_t pc, int64_t regs[])
608 __ mov(c_rarg0, rscratch1); // pass address of error message
609 __ mov(c_rarg1, lr); // pass return address
610 __ mov(c_rarg2, sp); // pass address of regs on stack
611 #ifndef PRODUCT
612 assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
613 #endif
614 BLOCK_COMMENT("call MacroAssembler::debug");
615 __ mov(rscratch1, CAST_FROM_FN_PTR(address, MacroAssembler::debug64));
616 __ blrt(rscratch1, 3, 0, 1);
617
618 return start;
619 }
620
621 void array_overlap_test(Label& L_no_overlap, Address::sxtw sf) { __ b(L_no_overlap); }
622
623 // Generate code for an array write pre barrier
624 //
625 // addr - starting address
626 // count - element count
627 // tmp - scratch register
628 // saved_regs - registers to be saved before calling static_write_ref_array_pre
629 //
630 // Callers must specify which registers to preserve in saved_regs.
631 // Clobbers: r0-r18, v0-v7, v16-v31, except saved_regs.
632 //
633 void gen_write_ref_array_pre_barrier(Register addr, Register count, bool dest_uninitialized, RegSet saved_regs) {
634 BarrierSet* bs = Universe::heap()->barrier_set();
635 switch (bs->kind()) {
636 case BarrierSet::G1BarrierSet:
637 // With G1, don't generate the call if we statically know that the target in uninitialized
638 if (!dest_uninitialized) {
639 __ push(saved_regs, sp);
640 if (count == c_rarg0) {
641 if (addr == c_rarg1) {
642 // exactly backwards!!
643 __ mov(rscratch1, c_rarg0);
644 __ mov(c_rarg0, c_rarg1);
645 __ mov(c_rarg1, rscratch1);
646 } else {
647 __ mov(c_rarg1, count);
648 __ mov(c_rarg0, addr);
649 }
650 } else {
651 __ mov(c_rarg0, addr);
652 __ mov(c_rarg1, count);
653 }
654 __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre), 2);
655 __ pop(saved_regs, sp);
656 break;
657 case BarrierSet::CardTableModRef:
658 break;
659 default:
660 ShouldNotReachHere();
661
662 }
663 }
664 }
665
666 //
667 // Generate code for an array write post barrier
668 //
669 // Input:
670 // start - register containing starting address of destination array
671 // end - register containing ending address of destination array
672 // scratch - scratch register
673 // saved_regs - registers to be saved before calling static_write_ref_array_post
674 //
675 // The input registers are overwritten.
676 // The ending address is inclusive.
677 // Callers must specify which registers to preserve in saved_regs.
678 // Clobbers: r0-r18, v0-v7, v16-v31, except saved_regs.
679 void gen_write_ref_array_post_barrier(Register start, Register end, Register scratch, RegSet saved_regs) {
680 assert_different_registers(start, end, scratch);
681 BarrierSet* bs = Universe::heap()->barrier_set();
682 switch (bs->kind()) {
683 case BarrierSet::G1BarrierSet:
684
685 {
686 __ push(saved_regs, sp);
687 // must compute element count unless barrier set interface is changed (other platforms supply count)
688 assert_different_registers(start, end, scratch);
689 __ lea(scratch, Address(end, BytesPerHeapOop));
690 __ sub(scratch, scratch, start); // subtract start to get #bytes
691 __ lsr(scratch, scratch, LogBytesPerHeapOop); // convert to element count
692 __ mov(c_rarg0, start);
693 __ mov(c_rarg1, scratch);
694 __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post), 2);
695 __ pop(saved_regs, sp);
696 }
697 break;
698 case BarrierSet::CardTableModRef:
699 {
700 CardTableModRefBS* ctbs = barrier_set_cast<CardTableModRefBS>(bs);
701 CardTable* ct = ctbs->card_table();
702 assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "adjust this code");
703
704 Label L_loop;
705
706 __ lsr(start, start, CardTable::card_shift);
707 __ lsr(end, end, CardTable::card_shift);
708 __ sub(end, end, start); // number of bytes to copy
709
710 const Register count = end; // 'end' register contains bytes count now
711 __ load_byte_map_base(scratch);
712 __ add(start, start, scratch);
713 if (UseConcMarkSweepGC) {
714 __ membar(__ StoreStore);
715 }
716 __ BIND(L_loop);
717 __ strb(zr, Address(start, count));
718 __ subs(count, count, 1);
719 __ br(Assembler::GE, L_loop);
720 }
721 break;
722 default:
723 ShouldNotReachHere();
724
725 }
726 }
727
728 // The inner part of zero_words(). This is the bulk operation,
729 // zeroing words in blocks, possibly using DC ZVA to do it. The
730 // caller is responsible for zeroing the last few words.
731 //
732 // Inputs:
733 // r10: the HeapWord-aligned base address of an array to zero.
734 // r11: the count in HeapWords, r11 > 0.
735 //
736 // Returns r10 and r11, adjusted for the caller to clear.
737 // r10: the base address of the tail of words left to clear.
738 // r11: the number of words in the tail.
739 // r11 < MacroAssembler::zero_words_block_size.
740
741 address generate_zero_blocks() {
742 Label store_pair, loop_store_pair, done;
743 Label base_aligned;
744
745 Register base = r10, cnt = r11;
746
747 __ align(CodeEntryAlignment);
1439 //
1440 // Side Effects:
1441 // disjoint_int_copy_entry is set to the no-overlap entry point
1442 // used by generate_conjoint_int_oop_copy().
1443 //
1444 address generate_disjoint_copy(size_t size, bool aligned, bool is_oop, address *entry,
1445 const char *name, bool dest_uninitialized = false) {
1446 Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
1447 RegSet saved_reg = RegSet::of(s, d, count);
1448 __ align(CodeEntryAlignment);
1449 StubCodeMark mark(this, "StubRoutines", name);
1450 address start = __ pc();
1451 __ enter();
1452
1453 if (entry != NULL) {
1454 *entry = __ pc();
1455 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1456 BLOCK_COMMENT("Entry:");
1457 }
1458
1459 if (is_oop) {
1460 gen_write_ref_array_pre_barrier(d, count, dest_uninitialized, saved_reg);
1461 // save regs before copy_memory
1462 __ push(RegSet::of(d, count), sp);
1463 }
1464 copy_memory(aligned, s, d, count, rscratch1, size);
1465 if (is_oop) {
1466 __ pop(RegSet::of(d, count), sp);
1467 if (VerifyOops)
1468 verify_oop_array(size, d, count, r16);
1469 __ sub(count, count, 1); // make an inclusive end pointer
1470 __ lea(count, Address(d, count, Address::lsl(exact_log2(size))));
1471 gen_write_ref_array_post_barrier(d, count, rscratch1, RegSet());
1472 }
1473 __ leave();
1474 __ mov(r0, zr); // return 0
1475 __ ret(lr);
1476 #ifdef BUILTIN_SIM
1477 {
1478 AArch64Simulator *sim = AArch64Simulator::get_current(UseSimulatorCache, DisableBCCheck);
1479 sim->notifyCompile(const_cast<char*>(name), start);
1480 }
1481 #endif
1482 return start;
1483 }
1484
1485 // Arguments:
1486 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1487 // ignored
1488 // is_oop - true => oop array, so generate store check code
1489 // name - stub name string
1490 //
1491 // Inputs:
1492 // c_rarg0 - source array address
1500 address generate_conjoint_copy(size_t size, bool aligned, bool is_oop, address nooverlap_target,
1501 address *entry, const char *name,
1502 bool dest_uninitialized = false) {
1503 Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
1504 RegSet saved_regs = RegSet::of(s, d, count);
1505 StubCodeMark mark(this, "StubRoutines", name);
1506 address start = __ pc();
1507 __ enter();
1508
1509 if (entry != NULL) {
1510 *entry = __ pc();
1511 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1512 BLOCK_COMMENT("Entry:");
1513 }
1514
1515 // use fwd copy when (d-s) above_equal (count*size)
1516 __ sub(rscratch1, d, s);
1517 __ cmp(rscratch1, count, Assembler::LSL, exact_log2(size));
1518 __ br(Assembler::HS, nooverlap_target);
1519
1520 if (is_oop) {
1521 gen_write_ref_array_pre_barrier(d, count, dest_uninitialized, saved_regs);
1522 // save regs before copy_memory
1523 __ push(RegSet::of(d, count), sp);
1524 }
1525 copy_memory(aligned, s, d, count, rscratch1, -size);
1526 if (is_oop) {
1527 __ pop(RegSet::of(d, count), sp);
1528 if (VerifyOops)
1529 verify_oop_array(size, d, count, r16);
1530 __ sub(count, count, 1); // make an inclusive end pointer
1531 __ lea(count, Address(d, count, Address::lsl(exact_log2(size))));
1532 gen_write_ref_array_post_barrier(d, count, rscratch1, RegSet());
1533 }
1534 __ leave();
1535 __ mov(r0, zr); // return 0
1536 __ ret(lr);
1537 #ifdef BUILTIN_SIM
1538 {
1539 AArch64Simulator *sim = AArch64Simulator::get_current(UseSimulatorCache, DisableBCCheck);
1540 sim->notifyCompile(const_cast<char*>(name), start);
1541 }
1542 #endif
1543 return start;
1544 }
1545
1546 // Arguments:
1547 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1548 // ignored
1549 // name - stub name string
1550 //
1551 // Inputs:
1552 // c_rarg0 - source array address
1553 // c_rarg1 - destination array address
1854
1855 // Empty array: Nothing to do.
1856 __ cbz(count, L_done);
1857
1858 __ push(RegSet::of(r18, r19, r20, r21), sp);
1859
1860 #ifdef ASSERT
1861 BLOCK_COMMENT("assert consistent ckoff/ckval");
1862 // The ckoff and ckval must be mutually consistent,
1863 // even though caller generates both.
1864 { Label L;
1865 int sco_offset = in_bytes(Klass::super_check_offset_offset());
1866 __ ldrw(start_to, Address(ckval, sco_offset));
1867 __ cmpw(ckoff, start_to);
1868 __ br(Assembler::EQ, L);
1869 __ stop("super_check_offset inconsistent");
1870 __ bind(L);
1871 }
1872 #endif //ASSERT
1873
1874 gen_write_ref_array_pre_barrier(to, count, dest_uninitialized, wb_pre_saved_regs);
1875
1876 // save the original count
1877 __ mov(count_save, count);
1878
1879 // Copy from low to high addresses
1880 __ mov(start_to, to); // Save destination array start address
1881 __ b(L_load_element);
1882
1883 // ======== begin loop ========
1884 // (Loop is rotated; its entry is L_load_element.)
1885 // Loop control:
1886 // for (; count != 0; count--) {
1887 // copied_oop = load_heap_oop(from++);
1888 // ... generate_type_check ...;
1889 // store_heap_oop(to++, copied_oop);
1890 // }
1891 __ align(OptoLoopAlignment);
1892
1893 __ BIND(L_store_element);
1894 __ store_heap_oop(__ post(to, UseCompressedOops ? 4 : 8), copied_oop); // store the oop
1898 // ======== loop entry is here ========
1899 __ BIND(L_load_element);
1900 __ load_heap_oop(copied_oop, __ post(from, UseCompressedOops ? 4 : 8)); // load the oop
1901 __ cbz(copied_oop, L_store_element);
1902
1903 __ load_klass(r19_klass, copied_oop);// query the object klass
1904 generate_type_check(r19_klass, ckoff, ckval, L_store_element);
1905 // ======== end loop ========
1906
1907 // It was a real error; we must depend on the caller to finish the job.
1908 // Register count = remaining oops, count_orig = total oops.
1909 // Emit GC store barriers for the oops we have copied and report
1910 // their number to the caller.
1911
1912 __ subs(count, count_save, count); // K = partially copied oop count
1913 __ eon(count, count, zr); // report (-1^K) to caller
1914 __ br(Assembler::EQ, L_done_pop);
1915
1916 __ BIND(L_do_card_marks);
1917 __ add(to, to, -heapOopSize); // make an inclusive end pointer
1918 gen_write_ref_array_post_barrier(start_to, to, rscratch1, wb_post_saved_regs);
1919
1920 __ bind(L_done_pop);
1921 __ pop(RegSet::of(r18, r19, r20, r21), sp);
1922 inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr);
1923
1924 __ bind(L_done);
1925 __ mov(r0, count);
1926 __ leave();
1927 __ ret(lr);
1928
1929 return start;
1930 }
1931
1932 // Perform range checks on the proposed arraycopy.
1933 // Kills temp, but nothing else.
1934 // Also, clean the sign bits of src_pos and dst_pos.
1935 void arraycopy_range_checks(Register src, // source array oop (c_rarg0)
1936 Register src_pos, // source position (c_rarg1)
1937 Register dst, // destination array oo (c_rarg2)
1938 Register dst_pos, // destination position (c_rarg3)
|
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "asm/macroAssembler.hpp"
28 #include "asm/macroAssembler.inline.hpp"
29 #include "gc/shared/barrierSetCodeGen.hpp"
30 #include "gc/shared/cardTable.hpp"
31 #include "gc/shared/cardTableModRefBS.hpp"
32 #include "interpreter/interpreter.hpp"
33 #include "nativeInst_aarch64.hpp"
34 #include "oops/instanceOop.hpp"
35 #include "oops/method.hpp"
36 #include "oops/objArrayKlass.hpp"
37 #include "oops/oop.inline.hpp"
38 #include "prims/methodHandles.hpp"
39 #include "runtime/frame.inline.hpp"
40 #include "runtime/handles.inline.hpp"
41 #include "runtime/sharedRuntime.hpp"
42 #include "runtime/stubCodeGenerator.hpp"
43 #include "runtime/stubRoutines.hpp"
44 #include "runtime/thread.inline.hpp"
45 #include "utilities/align.hpp"
46 #ifdef COMPILER2
47 #include "opto/runtime.hpp"
48 #endif
49
604 __ bind(error);
605 __ ldp(c_rarg3, c_rarg2, Address(__ post(sp, 16)));
606
607 __ push(RegSet::range(r0, r29), sp);
608 // debug(char* msg, int64_t pc, int64_t regs[])
609 __ mov(c_rarg0, rscratch1); // pass address of error message
610 __ mov(c_rarg1, lr); // pass return address
611 __ mov(c_rarg2, sp); // pass address of regs on stack
612 #ifndef PRODUCT
613 assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
614 #endif
615 BLOCK_COMMENT("call MacroAssembler::debug");
616 __ mov(rscratch1, CAST_FROM_FN_PTR(address, MacroAssembler::debug64));
617 __ blrt(rscratch1, 3, 0, 1);
618
619 return start;
620 }
621
622 void array_overlap_test(Label& L_no_overlap, Address::sxtw sf) { __ b(L_no_overlap); }
623
624 // The inner part of zero_words(). This is the bulk operation,
625 // zeroing words in blocks, possibly using DC ZVA to do it. The
626 // caller is responsible for zeroing the last few words.
627 //
628 // Inputs:
629 // r10: the HeapWord-aligned base address of an array to zero.
630 // r11: the count in HeapWords, r11 > 0.
631 //
632 // Returns r10 and r11, adjusted for the caller to clear.
633 // r10: the base address of the tail of words left to clear.
634 // r11: the number of words in the tail.
635 // r11 < MacroAssembler::zero_words_block_size.
636
637 address generate_zero_blocks() {
638 Label store_pair, loop_store_pair, done;
639 Label base_aligned;
640
641 Register base = r10, cnt = r11;
642
643 __ align(CodeEntryAlignment);
1335 //
1336 // Side Effects:
1337 // disjoint_int_copy_entry is set to the no-overlap entry point
1338 // used by generate_conjoint_int_oop_copy().
1339 //
1340 address generate_disjoint_copy(size_t size, bool aligned, bool is_oop, address *entry,
1341 const char *name, bool dest_uninitialized = false) {
1342 Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
1343 RegSet saved_reg = RegSet::of(s, d, count);
1344 __ align(CodeEntryAlignment);
1345 StubCodeMark mark(this, "StubRoutines", name);
1346 address start = __ pc();
1347 __ enter();
1348
1349 if (entry != NULL) {
1350 *entry = __ pc();
1351 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1352 BLOCK_COMMENT("Entry:");
1353 }
1354
1355 BarrierSetCodeGen *bs = Universe::heap()->barrier_set()->code_gen();
1356 DecoratorSet decorators = ARRAYCOPY_DISJOINT;
1357 if (dest_uninitialized) {
1358 decorators |= DEST_NOT_INITIALIZED;
1359 }
1360 if (aligned) {
1361 decorators |= ARRAYCOPY_ALIGNED;
1362 }
1363
1364 bs->arraycopy_prologue(_masm, decorators, is_oop, d, count, saved_reg);
1365
1366 if (is_oop) {
1367 // save regs before copy_memory
1368 __ push(RegSet::of(d, count), sp);
1369 }
1370 copy_memory(aligned, s, d, count, rscratch1, size);
1371
1372 if (is_oop) {
1373 __ pop(RegSet::of(d, count), sp);
1374 if (VerifyOops)
1375 verify_oop_array(size, d, count, r16);
1376 __ sub(count, count, 1); // make an inclusive end pointer
1377 __ lea(count, Address(d, count, Address::lsl(exact_log2(size))));
1378 }
1379
1380 bs->arraycopy_epilogue(_masm, decorators, is_oop, d, count, rscratch1, RegSet());
1381
1382 __ leave();
1383 __ mov(r0, zr); // return 0
1384 __ ret(lr);
1385 #ifdef BUILTIN_SIM
1386 {
1387 AArch64Simulator *sim = AArch64Simulator::get_current(UseSimulatorCache, DisableBCCheck);
1388 sim->notifyCompile(const_cast<char*>(name), start);
1389 }
1390 #endif
1391 return start;
1392 }
1393
1394 // Arguments:
1395 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1396 // ignored
1397 // is_oop - true => oop array, so generate store check code
1398 // name - stub name string
1399 //
1400 // Inputs:
1401 // c_rarg0 - source array address
1409 address generate_conjoint_copy(size_t size, bool aligned, bool is_oop, address nooverlap_target,
1410 address *entry, const char *name,
1411 bool dest_uninitialized = false) {
1412 Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
1413 RegSet saved_regs = RegSet::of(s, d, count);
1414 StubCodeMark mark(this, "StubRoutines", name);
1415 address start = __ pc();
1416 __ enter();
1417
1418 if (entry != NULL) {
1419 *entry = __ pc();
1420 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1421 BLOCK_COMMENT("Entry:");
1422 }
1423
1424 // use fwd copy when (d-s) above_equal (count*size)
1425 __ sub(rscratch1, d, s);
1426 __ cmp(rscratch1, count, Assembler::LSL, exact_log2(size));
1427 __ br(Assembler::HS, nooverlap_target);
1428
1429 BarrierSetCodeGen *bs = Universe::heap()->barrier_set()->code_gen();
1430 DecoratorSet decorators = DECORATOR_DEFAULT;
1431 if (dest_uninitialized) {
1432 decorators |= DEST_NOT_INITIALIZED;
1433 }
1434 if (aligned) {
1435 decorators |= ARRAYCOPY_ALIGNED;
1436 }
1437 bs->arraycopy_prologue(_masm, decorators, is_oop, d, count, saved_regs);
1438
1439 if (is_oop) {
1440 // save regs before copy_memory
1441 __ push(RegSet::of(d, count), sp);
1442 }
1443 copy_memory(aligned, s, d, count, rscratch1, -size);
1444 if (is_oop) {
1445 __ pop(RegSet::of(d, count), sp);
1446 if (VerifyOops)
1447 verify_oop_array(size, d, count, r16);
1448 __ sub(count, count, 1); // make an inclusive end pointer
1449 __ lea(count, Address(d, count, Address::lsl(exact_log2(size))));
1450 }
1451 bs->arraycopy_epilogue(_masm, decorators, is_oop, d, count, rscratch1, RegSet());
1452 __ leave();
1453 __ mov(r0, zr); // return 0
1454 __ ret(lr);
1455 #ifdef BUILTIN_SIM
1456 {
1457 AArch64Simulator *sim = AArch64Simulator::get_current(UseSimulatorCache, DisableBCCheck);
1458 sim->notifyCompile(const_cast<char*>(name), start);
1459 }
1460 #endif
1461 return start;
1462 }
1463
1464 // Arguments:
1465 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1466 // ignored
1467 // name - stub name string
1468 //
1469 // Inputs:
1470 // c_rarg0 - source array address
1471 // c_rarg1 - destination array address
1772
1773 // Empty array: Nothing to do.
1774 __ cbz(count, L_done);
1775
1776 __ push(RegSet::of(r18, r19, r20, r21), sp);
1777
1778 #ifdef ASSERT
1779 BLOCK_COMMENT("assert consistent ckoff/ckval");
1780 // The ckoff and ckval must be mutually consistent,
1781 // even though caller generates both.
1782 { Label L;
1783 int sco_offset = in_bytes(Klass::super_check_offset_offset());
1784 __ ldrw(start_to, Address(ckval, sco_offset));
1785 __ cmpw(ckoff, start_to);
1786 __ br(Assembler::EQ, L);
1787 __ stop("super_check_offset inconsistent");
1788 __ bind(L);
1789 }
1790 #endif //ASSERT
1791
1792 BarrierSetCodeGen *bs = Universe::heap()->barrier_set()->code_gen();
1793 DecoratorSet decorators = ARRAYCOPY_CONTRAVARIANT | ARRAYCOPY_DISJOINT;
1794 bool is_oop = true;
1795 if (dest_uninitialized) {
1796 decorators |= DEST_NOT_INITIALIZED;
1797 }
1798
1799 bs->arraycopy_prologue(_masm, decorators, is_oop, to, count, wb_pre_saved_regs);
1800
1801 // save the original count
1802 __ mov(count_save, count);
1803
1804 // Copy from low to high addresses
1805 __ mov(start_to, to); // Save destination array start address
1806 __ b(L_load_element);
1807
1808 // ======== begin loop ========
1809 // (Loop is rotated; its entry is L_load_element.)
1810 // Loop control:
1811 // for (; count != 0; count--) {
1812 // copied_oop = load_heap_oop(from++);
1813 // ... generate_type_check ...;
1814 // store_heap_oop(to++, copied_oop);
1815 // }
1816 __ align(OptoLoopAlignment);
1817
1818 __ BIND(L_store_element);
1819 __ store_heap_oop(__ post(to, UseCompressedOops ? 4 : 8), copied_oop); // store the oop
1823 // ======== loop entry is here ========
1824 __ BIND(L_load_element);
1825 __ load_heap_oop(copied_oop, __ post(from, UseCompressedOops ? 4 : 8)); // load the oop
1826 __ cbz(copied_oop, L_store_element);
1827
1828 __ load_klass(r19_klass, copied_oop);// query the object klass
1829 generate_type_check(r19_klass, ckoff, ckval, L_store_element);
1830 // ======== end loop ========
1831
1832 // It was a real error; we must depend on the caller to finish the job.
1833 // Register count = remaining oops, count_orig = total oops.
1834 // Emit GC store barriers for the oops we have copied and report
1835 // their number to the caller.
1836
1837 __ subs(count, count_save, count); // K = partially copied oop count
1838 __ eon(count, count, zr); // report (-1^K) to caller
1839 __ br(Assembler::EQ, L_done_pop);
1840
1841 __ BIND(L_do_card_marks);
1842 __ add(to, to, -heapOopSize); // make an inclusive end pointer
1843 bs->arraycopy_epilogue(_masm, decorators, is_oop, start_to, to, rscratch1, wb_post_saved_regs);
1844
1845 __ bind(L_done_pop);
1846 __ pop(RegSet::of(r18, r19, r20, r21), sp);
1847 inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr);
1848
1849 __ bind(L_done);
1850 __ mov(r0, count);
1851 __ leave();
1852 __ ret(lr);
1853
1854 return start;
1855 }
1856
1857 // Perform range checks on the proposed arraycopy.
1858 // Kills temp, but nothing else.
1859 // Also, clean the sign bits of src_pos and dst_pos.
1860 void arraycopy_range_checks(Register src, // source array oop (c_rarg0)
1861 Register src_pos, // source position (c_rarg1)
1862 Register dst, // destination array oo (c_rarg2)
1863 Register dst_pos, // destination position (c_rarg3)
|