src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

@@ -2649,27 +2649,45 @@
    }
  
    pop(RegSet::range(r0, r18) - RegSet::of(rscratch1, rscratch2) - exclude, sp);
  }
  
- void MacroAssembler::push_CPU_state(bool save_vectors) {
-   int step = (save_vectors ? 8 : 4) * wordSize;
+ void MacroAssembler::push_CPU_state(bool save_vectors, bool use_sve,
+                                     int sve_vector_size_in_bytes) {
    push(0x3fffffff, sp);         // integer registers except lr & sp
-   mov(rscratch1, -step);
-   sub(sp, sp, step);
-   for (int i = 28; i >= 4; i -= 4) {
-     st1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
-         as_FloatRegister(i+3), save_vectors ? T2D : T1D, Address(post(sp, rscratch1)));
+   if (save_vectors && use_sve) {
+     assert(sve_vector_size_in_bytes >= 16, "illegal scalable vector size");
+     sub(sp, sp, sve_vector_size_in_bytes * FloatRegisterImpl::number_of_registers);
+     for (int i = 0; i < FloatRegisterImpl::number_of_registers; i++) {
+       sve_str(as_FloatRegister(i), Address(sp, i));
+     }
+   } else {
+     int step = (save_vectors ? 8 : 4) * wordSize;
+     mov(rscratch1, -step);
+     sub(sp, sp, step);
+     for (int i = 28; i >= 4; i -= 4) {
+       st1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
+           as_FloatRegister(i+3), save_vectors ? T2D : T1D, Address(post(sp, rscratch1)));
+     }
+     st1(v0, v1, v2, v3, save_vectors ? T2D : T1D, sp);
    }
-   st1(v0, v1, v2, v3, save_vectors ? T2D : T1D, sp);
  }
  
- void MacroAssembler::pop_CPU_state(bool restore_vectors) {
-   int step = (restore_vectors ? 8 : 4) * wordSize;
-   for (int i = 0; i <= 28; i += 4)
-     ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
-         as_FloatRegister(i+3), restore_vectors ? T2D : T1D, Address(post(sp, step)));
+ void MacroAssembler::pop_CPU_state(bool restore_vectors, bool use_sve,
+                                    int sve_vector_size_in_bytes) {
+   if (restore_vectors && use_sve) {
+     assert(sve_vector_size_in_bytes >= 16, "illegal scalable vector size");
+     for (int i = FloatRegisterImpl::number_of_registers - 1; i >= 0; i--) {
+       sve_ldr(as_FloatRegister(i), Address(sp, i));
+     }
+     add(sp, sp, sve_vector_size_in_bytes * FloatRegisterImpl::number_of_registers);
+   } else {
+     int step = (restore_vectors ? 8 : 4) * wordSize;
+     for (int i = 0; i <= 28; i += 4)
+       ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
+           as_FloatRegister(i+3), restore_vectors ? T2D : T1D, Address(post(sp, step)));
+   }
    pop(0x3fffffff, sp);         // integer registers except lr & sp
  }
  
  /**
   * Helpers for multiply_to_len().

@@ -2714,10 +2732,25 @@
    }
  
    return Address(base, offset);
  }
  
+ Address MacroAssembler::sve_spill_address(int sve_reg_size_in_bytes, int offset, Register tmp) {
+   assert(offset >= 0, "spill to negative address?");
+ 
+   Register base = sp;
+ 
+   // An immediate offset in the range 0 to 255 which is multiplied
+   // by the current vector or predicate register size in bytes.
+   if (offset % sve_reg_size_in_bytes == 0 && offset < ((1<<8)*sve_reg_size_in_bytes)) {
+     return Address(base, offset / sve_reg_size_in_bytes);
+   }
+ 
+   add(tmp, base, offset);
+   return Address(tmp);
+ }
+ 
  // Checks whether offset is aligned.
  // Returns true if it is, else false.
  bool MacroAssembler::merge_alignment_check(Register base,
                                             size_t size,
                                             int64_t cur_offset,

@@ -5234,5 +5267,15 @@
    subsw(zr, rscratch1, VM_Version::get_initial_sve_vector_length());
    br(EQ, verify_ok);
    stop("Error: SVE vector length has changed since jvm startup");
    bind(verify_ok);
  }
+ 
+ void MacroAssembler::verify_ptrue() {
+   Label verify_ok;
+   assert(UseSVE > 0, "should only be used for SVE");
+   sve_cntp(rscratch1, B, ptrue, ptrue); // get true elements count.
+   sve_dec(rscratch1, B);
+   cbz(rscratch1, verify_ok);
+   stop("Error: the preserved predicate register (p7) elements are not all true");
+   bind(verify_ok);
+ }