2634 // Push v0-v7, v16-v31.
2635 for (int i = 31; i>= 4; i -= 4) {
2636 if (i <= v7->encoding() || i >= v16->encoding())
2637 st1(as_FloatRegister(i-3), as_FloatRegister(i-2), as_FloatRegister(i-1),
2638 as_FloatRegister(i), T1D, Address(post(sp, rscratch1)));
2639 }
2640 st1(as_FloatRegister(0), as_FloatRegister(1), as_FloatRegister(2),
2641 as_FloatRegister(3), T1D, Address(sp));
2642 }
2643
2644 void MacroAssembler::pop_call_clobbered_registers_except(RegSet exclude) {
2645 for (int i = 0; i < 32; i += 4) {
2646 if (i <= v7->encoding() || i >= v16->encoding())
2647 ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
2648 as_FloatRegister(i+3), T1D, Address(post(sp, 4 * wordSize)));
2649 }
2650
2651 pop(RegSet::range(r0, r18) - RegSet::of(rscratch1, rscratch2) - exclude, sp);
2652 }
2653
2654 void MacroAssembler::push_CPU_state(bool save_vectors) {
2655 int step = (save_vectors ? 8 : 4) * wordSize;
2656 push(0x3fffffff, sp); // integer registers except lr & sp
2657 mov(rscratch1, -step);
2658 sub(sp, sp, step);
2659 for (int i = 28; i >= 4; i -= 4) {
2660 st1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
2661 as_FloatRegister(i+3), save_vectors ? T2D : T1D, Address(post(sp, rscratch1)));
2662 }
2663 st1(v0, v1, v2, v3, save_vectors ? T2D : T1D, sp);
2664 }
2665
2666 void MacroAssembler::pop_CPU_state(bool restore_vectors) {
2667 int step = (restore_vectors ? 8 : 4) * wordSize;
2668 for (int i = 0; i <= 28; i += 4)
2669 ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
2670 as_FloatRegister(i+3), restore_vectors ? T2D : T1D, Address(post(sp, step)));
2671 pop(0x3fffffff, sp); // integer registers except lr & sp
2672 }
2673
2674 /**
2675 * Helpers for multiply_to_len().
2676 */
2677 void MacroAssembler::add2_with_carry(Register final_dest_hi, Register dest_hi, Register dest_lo,
2678 Register src1, Register src2) {
2679 adds(dest_lo, dest_lo, src1);
2680 adc(dest_hi, dest_hi, zr);
2681 adds(dest_lo, dest_lo, src2);
2682 adc(final_dest_hi, dest_hi, zr);
2683 }
2684
2685 // Generate an address from (r + r1 extend offset). "size" is the
2686 // size of the operand. The result may be in rscratch2.
2687 Address MacroAssembler::offsetted_address(Register r, Register r1,
2688 Address::extend ext, int offset, int size) {
2689 if (offset || (ext.shift() % size != 0)) {
2690 lea(rscratch2, Address(r, r1, ext));
2699 assert(offset >= 0, "spill to negative address?");
2700 // Offset reachable ?
2701 // Not aligned - 9 bits signed offset
2702 // Aligned - 12 bits unsigned offset shifted
2703 Register base = sp;
2704 if ((offset & (size-1)) && offset >= (1<<8)) {
2705 add(tmp, base, offset & ((1<<12)-1));
2706 base = tmp;
2707 offset &= -1u<<12;
2708 }
2709
2710 if (offset >= (1<<12) * size) {
2711 add(tmp, base, offset & (((1<<12)-1)<<12));
2712 base = tmp;
2713 offset &= ~(((1<<12)-1)<<12);
2714 }
2715
2716 return Address(base, offset);
2717 }
2718
2719 // Checks whether offset is aligned.
2720 // Returns true if it is, else false.
2721 bool MacroAssembler::merge_alignment_check(Register base,
2722 size_t size,
2723 int64_t cur_offset,
2724 int64_t prev_offset) const {
2725 if (AvoidUnalignedAccesses) {
2726 if (base == sp) {
2727 // Checks whether low offset if aligned to pair of registers.
2728 int64_t pair_mask = size * 2 - 1;
2729 int64_t offset = prev_offset > cur_offset ? cur_offset : prev_offset;
2730 return (offset & pair_mask) == 0;
2731 } else { // If base is not sp, we can't guarantee the access is aligned.
2732 return false;
2733 }
2734 } else {
2735 int64_t mask = size - 1;
2736 // Load/store pair instruction only supports element size aligned offset.
2737 return (cur_offset & mask) == 0 && (prev_offset & mask) == 0;
2738 }
5219 }
5220 }
5221
5222 void MacroAssembler::cache_wbsync(bool is_pre) {
5223 // we only need a barrier post sync
5224 if (!is_pre) {
5225 membar(Assembler::AnyAny);
5226 }
5227 }
5228
5229 void MacroAssembler::verify_sve_vector_length() {
5230 Label verify_ok;
5231 assert(UseSVE > 0, "should only be used for SVE");
5232 movw(rscratch1, zr);
5233 sve_inc(rscratch1, B);
5234 subsw(zr, rscratch1, VM_Version::get_initial_sve_vector_length());
5235 br(EQ, verify_ok);
5236 stop("Error: SVE vector length has changed since jvm startup");
5237 bind(verify_ok);
5238 }
|
2634 // Push v0-v7, v16-v31.
2635 for (int i = 31; i>= 4; i -= 4) {
2636 if (i <= v7->encoding() || i >= v16->encoding())
2637 st1(as_FloatRegister(i-3), as_FloatRegister(i-2), as_FloatRegister(i-1),
2638 as_FloatRegister(i), T1D, Address(post(sp, rscratch1)));
2639 }
2640 st1(as_FloatRegister(0), as_FloatRegister(1), as_FloatRegister(2),
2641 as_FloatRegister(3), T1D, Address(sp));
2642 }
2643
2644 void MacroAssembler::pop_call_clobbered_registers_except(RegSet exclude) {
2645 for (int i = 0; i < 32; i += 4) {
2646 if (i <= v7->encoding() || i >= v16->encoding())
2647 ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
2648 as_FloatRegister(i+3), T1D, Address(post(sp, 4 * wordSize)));
2649 }
2650
2651 pop(RegSet::range(r0, r18) - RegSet::of(rscratch1, rscratch2) - exclude, sp);
2652 }
2653
2654 void MacroAssembler::push_CPU_state(bool save_vectors, bool use_sve,
2655 int sve_vector_size_in_bytes) {
2656 push(0x3fffffff, sp); // integer registers except lr & sp
2657 if (save_vectors && use_sve) {
2658 assert(sve_vector_size_in_bytes >= 16, "illegal scalable vector size");
2659 sub(sp, sp, sve_vector_size_in_bytes * FloatRegisterImpl::number_of_registers);
2660 for (int i = 0; i < FloatRegisterImpl::number_of_registers; i++) {
2661 sve_str(as_FloatRegister(i), Address(sp, i));
2662 }
2663 } else {
2664 int step = (save_vectors ? 8 : 4) * wordSize;
2665 mov(rscratch1, -step);
2666 sub(sp, sp, step);
2667 for (int i = 28; i >= 4; i -= 4) {
2668 st1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
2669 as_FloatRegister(i+3), save_vectors ? T2D : T1D, Address(post(sp, rscratch1)));
2670 }
2671 st1(v0, v1, v2, v3, save_vectors ? T2D : T1D, sp);
2672 }
2673 }
2674
2675 void MacroAssembler::pop_CPU_state(bool restore_vectors, bool use_sve,
2676 int sve_vector_size_in_bytes) {
2677 if (restore_vectors && use_sve) {
2678 assert(sve_vector_size_in_bytes >= 16, "illegal scalable vector size");
2679 for (int i = FloatRegisterImpl::number_of_registers - 1; i >= 0; i--) {
2680 sve_ldr(as_FloatRegister(i), Address(sp, i));
2681 }
2682 add(sp, sp, sve_vector_size_in_bytes * FloatRegisterImpl::number_of_registers);
2683 } else {
2684 int step = (restore_vectors ? 8 : 4) * wordSize;
2685 for (int i = 0; i <= 28; i += 4)
2686 ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
2687 as_FloatRegister(i+3), restore_vectors ? T2D : T1D, Address(post(sp, step)));
2688 }
2689 pop(0x3fffffff, sp); // integer registers except lr & sp
2690 }
2691
2692 /**
2693 * Helpers for multiply_to_len().
2694 */
2695 void MacroAssembler::add2_with_carry(Register final_dest_hi, Register dest_hi, Register dest_lo,
2696 Register src1, Register src2) {
2697 adds(dest_lo, dest_lo, src1);
2698 adc(dest_hi, dest_hi, zr);
2699 adds(dest_lo, dest_lo, src2);
2700 adc(final_dest_hi, dest_hi, zr);
2701 }
2702
2703 // Generate an address from (r + r1 extend offset). "size" is the
2704 // size of the operand. The result may be in rscratch2.
2705 Address MacroAssembler::offsetted_address(Register r, Register r1,
2706 Address::extend ext, int offset, int size) {
2707 if (offset || (ext.shift() % size != 0)) {
2708 lea(rscratch2, Address(r, r1, ext));
2717 assert(offset >= 0, "spill to negative address?");
2718 // Offset reachable ?
2719 // Not aligned - 9 bits signed offset
2720 // Aligned - 12 bits unsigned offset shifted
2721 Register base = sp;
2722 if ((offset & (size-1)) && offset >= (1<<8)) {
2723 add(tmp, base, offset & ((1<<12)-1));
2724 base = tmp;
2725 offset &= -1u<<12;
2726 }
2727
2728 if (offset >= (1<<12) * size) {
2729 add(tmp, base, offset & (((1<<12)-1)<<12));
2730 base = tmp;
2731 offset &= ~(((1<<12)-1)<<12);
2732 }
2733
2734 return Address(base, offset);
2735 }
2736
2737 Address MacroAssembler::sve_spill_address(int sve_reg_size_in_bytes, int offset, Register tmp) {
2738 assert(offset >= 0, "spill to negative address?");
2739
2740 Register base = sp;
2741
2742 // An immediate offset in the range 0 to 255 which is multiplied
2743 // by the current vector or predicate register size in bytes.
2744 if (offset % sve_reg_size_in_bytes == 0 && offset < ((1<<8)*sve_reg_size_in_bytes)) {
2745 return Address(base, offset / sve_reg_size_in_bytes);
2746 }
2747
2748 add(tmp, base, offset);
2749 return Address(tmp);
2750 }
2751
2752 // Checks whether offset is aligned.
2753 // Returns true if it is, else false.
2754 bool MacroAssembler::merge_alignment_check(Register base,
2755 size_t size,
2756 int64_t cur_offset,
2757 int64_t prev_offset) const {
2758 if (AvoidUnalignedAccesses) {
2759 if (base == sp) {
2760 // Checks whether low offset if aligned to pair of registers.
2761 int64_t pair_mask = size * 2 - 1;
2762 int64_t offset = prev_offset > cur_offset ? cur_offset : prev_offset;
2763 return (offset & pair_mask) == 0;
2764 } else { // If base is not sp, we can't guarantee the access is aligned.
2765 return false;
2766 }
2767 } else {
2768 int64_t mask = size - 1;
2769 // Load/store pair instruction only supports element size aligned offset.
2770 return (cur_offset & mask) == 0 && (prev_offset & mask) == 0;
2771 }
5252 }
5253 }
5254
5255 void MacroAssembler::cache_wbsync(bool is_pre) {
5256 // we only need a barrier post sync
5257 if (!is_pre) {
5258 membar(Assembler::AnyAny);
5259 }
5260 }
5261
5262 void MacroAssembler::verify_sve_vector_length() {
5263 Label verify_ok;
5264 assert(UseSVE > 0, "should only be used for SVE");
5265 movw(rscratch1, zr);
5266 sve_inc(rscratch1, B);
5267 subsw(zr, rscratch1, VM_Version::get_initial_sve_vector_length());
5268 br(EQ, verify_ok);
5269 stop("Error: SVE vector length has changed since jvm startup");
5270 bind(verify_ok);
5271 }
5272
5273 void MacroAssembler::verify_ptrue() {
5274 Label verify_ok;
5275 assert(UseSVE > 0, "should only be used for SVE");
5276 sve_cntp(rscratch1, B, ptrue, ptrue); // get true elements count.
5277 sve_dec(rscratch1, B);
5278 cbz(rscratch1, verify_ok);
5279 stop("Error: the preserved predicate register (p7) elements are not all true");
5280 bind(verify_ok);
5281 }
|