src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

*** 2649,27 ***
    }
  
    pop(RegSet::range(r0, r18) - RegSet::of(rscratch1, rscratch2) - exclude, sp);
  }
  
! void MacroAssembler::push_CPU_state(bool save_vectors) {
!   int step = (save_vectors ? 8 : 4) * wordSize;
    push(0x3fffffff, sp);         // integer registers except lr & sp
!   mov(rscratch1, -step);
!   sub(sp, sp, step);
!   for (int i = 28; i >= 4; i -= 4) {
!     st1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
!         as_FloatRegister(i+3), save_vectors ? T2D : T1D, Address(post(sp, rscratch1)));
    }
-   st1(v0, v1, v2, v3, save_vectors ? T2D : T1D, sp);
  }
  
! void MacroAssembler::pop_CPU_state(bool restore_vectors) {
!   int step = (restore_vectors ? 8 : 4) * wordSize;
!   for (int i = 0; i <= 28; i += 4)
!     ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
!         as_FloatRegister(i+3), restore_vectors ? T2D : T1D, Address(post(sp, step)));
    pop(0x3fffffff, sp);         // integer registers except lr & sp
  }
  
  /**
   * Helpers for multiply_to_len().
--- 2649,45 ---
    }
  
    pop(RegSet::range(r0, r18) - RegSet::of(rscratch1, rscratch2) - exclude, sp);
  }
  
! void MacroAssembler::push_CPU_state(bool save_vectors, bool use_sve,
!                                     int sve_vector_size_in_bytes) {
    push(0x3fffffff, sp);         // integer registers except lr & sp
!   if (save_vectors && use_sve) {
!     assert(sve_vector_size_in_bytes >= 16, "illegal scalable vector size");
!     sub(sp, sp, sve_vector_size_in_bytes * FloatRegisterImpl::number_of_registers);
!     for (int i = 0; i < FloatRegisterImpl::number_of_registers; i++) {
!       sve_str(as_FloatRegister(i), Address(sp, i));
+     }
+   } else {
+     int step = (save_vectors ? 8 : 4) * wordSize;
+     mov(rscratch1, -step);
+     sub(sp, sp, step);
+     for (int i = 28; i >= 4; i -= 4) {
+       st1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
+           as_FloatRegister(i+3), save_vectors ? T2D : T1D, Address(post(sp, rscratch1)));
+     }
+     st1(v0, v1, v2, v3, save_vectors ? T2D : T1D, sp);
    }
  }
  
! void MacroAssembler::pop_CPU_state(bool restore_vectors, bool use_sve,
!                                    int sve_vector_size_in_bytes) {
!   if (restore_vectors && use_sve) {
!     assert(sve_vector_size_in_bytes >= 16, "illegal scalable vector size");
!     for (int i = FloatRegisterImpl::number_of_registers - 1; i >= 0; i--) {
+       sve_ldr(as_FloatRegister(i), Address(sp, i));
+     }
+     add(sp, sp, sve_vector_size_in_bytes * FloatRegisterImpl::number_of_registers);
+   } else {
+     int step = (restore_vectors ? 8 : 4) * wordSize;
+     for (int i = 0; i <= 28; i += 4)
+       ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
+           as_FloatRegister(i+3), restore_vectors ? T2D : T1D, Address(post(sp, step)));
+   }
    pop(0x3fffffff, sp);         // integer registers except lr & sp
  }
  
  /**
   * Helpers for multiply_to_len().

*** 2714,10 ***
--- 2732,25 ---
    }
  
    return Address(base, offset);
  }
  
+ Address MacroAssembler::sve_spill_address(int sve_reg_size_in_bytes, int offset, Register tmp) {
+   assert(offset >= 0, "spill to negative address?");
+ 
+   Register base = sp;
+ 
+   // An immediate offset in the range 0 to 255 which is multiplied
+   // by the current vector or predicate register size in bytes.
+   if (offset % sve_reg_size_in_bytes == 0 && offset < ((1<<8)*sve_reg_size_in_bytes)) {
+     return Address(base, offset / sve_reg_size_in_bytes);
+   }
+ 
+   add(tmp, base, offset);
+   return Address(tmp);
+ }
+ 
  // Checks whether offset is aligned.
  // Returns true if it is, else false.
  bool MacroAssembler::merge_alignment_check(Register base,
                                             size_t size,
                                             int64_t cur_offset,

*** 5234,5 ***
--- 5267,15 ---
    subsw(zr, rscratch1, VM_Version::get_initial_sve_vector_length());
    br(EQ, verify_ok);
    stop("Error: SVE vector length has changed since jvm startup");
    bind(verify_ok);
  }
+ 
+ void MacroAssembler::verify_ptrue() {
+   Label verify_ok;
+   assert(UseSVE > 0, "should only be used for SVE");
+   sve_cntp(rscratch1, B, ptrue, ptrue); // get true elements count.
+   sve_dec(rscratch1, B);
+   cbz(rscratch1, verify_ok);
+   stop("Error: the preserved predicate register (p7) elements are not all true");
+   bind(verify_ok);
+ }