src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

rev 60615 : 8231441: Initial SVE backend support
Reviewed-by: adinn, pli
Contributed-by: joshua.zhu@arm.com, yang.zhang@arm.com, ningsheng.jian@arm.com

*** 2649,2675 ****
    }
  
    pop(RegSet::range(r0, r18) - RegSet::of(rscratch1, rscratch2) - exclude, sp);
  }
  
! void MacroAssembler::push_CPU_state(bool save_vectors) {
!   int step = (save_vectors ? 8 : 4) * wordSize;
    push(0x3fffffff, sp);         // integer registers except lr & sp
    mov(rscratch1, -step);
    sub(sp, sp, step);
    for (int i = 28; i >= 4; i -= 4) {
      st1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
          as_FloatRegister(i+3), save_vectors ? T2D : T1D, Address(post(sp, rscratch1)));
    }
    st1(v0, v1, v2, v3, save_vectors ? T2D : T1D, sp);
  }
  
! void MacroAssembler::pop_CPU_state(bool restore_vectors) {
    int step = (restore_vectors ? 8 : 4) * wordSize;
    for (int i = 0; i <= 28; i += 4)
      ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
          as_FloatRegister(i+3), restore_vectors ? T2D : T1D, Address(post(sp, step)));
    pop(0x3fffffff, sp);         // integer registers except lr & sp
  }
  
  /**
   * Helpers for multiply_to_len().
--- 2649,2693 ----
    }
  
    pop(RegSet::range(r0, r18) - RegSet::of(rscratch1, rscratch2) - exclude, sp);
  }
  
! void MacroAssembler::push_CPU_state(bool save_vectors, bool use_sve,
!                                     int sve_vector_size_in_bytes) {
    push(0x3fffffff, sp);         // integer registers except lr & sp
+   if (save_vectors && use_sve) {
+     assert(sve_vector_size_in_bytes >= 16, "illegal scalable vector size");
+     sub(sp, sp, sve_vector_size_in_bytes * FloatRegisterImpl::number_of_registers);
+     for (int i = 0; i < FloatRegisterImpl::number_of_registers; i++) {
+       sve_str(as_FloatRegister(i), Address(sp, i));
+     }
+   } else {
+     int step = (save_vectors ? 8 : 4) * wordSize;
      mov(rscratch1, -step);
      sub(sp, sp, step);
      for (int i = 28; i >= 4; i -= 4) {
        st1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
            as_FloatRegister(i+3), save_vectors ? T2D : T1D, Address(post(sp, rscratch1)));
      }
      st1(v0, v1, v2, v3, save_vectors ? T2D : T1D, sp);
+   }
  }
  
! void MacroAssembler::pop_CPU_state(bool restore_vectors, bool use_sve,
!                                    int sve_vector_size_in_bytes) {
!   if (restore_vectors && use_sve) {
!     assert(sve_vector_size_in_bytes >= 16, "illegal scalable vector size");
!     for (int i = FloatRegisterImpl::number_of_registers - 1; i >= 0; i--) {
!       sve_ldr(as_FloatRegister(i), Address(sp, i));
!     }
!     add(sp, sp, sve_vector_size_in_bytes * FloatRegisterImpl::number_of_registers);
!   } else {
      int step = (restore_vectors ? 8 : 4) * wordSize;
      for (int i = 0; i <= 28; i += 4)
        ld1(as_FloatRegister(i), as_FloatRegister(i+1), as_FloatRegister(i+2),
            as_FloatRegister(i+3), restore_vectors ? T2D : T1D, Address(post(sp, step)));
+   }
    pop(0x3fffffff, sp);         // integer registers except lr & sp
  }
  
  /**
   * Helpers for multiply_to_len().
*** 2714,2723 ****
--- 2732,2756 ----
    }
  
    return Address(base, offset);
  }
  
+ Address MacroAssembler::sve_spill_address(int sve_reg_size_in_bytes, int offset, Register tmp) {
+   assert(offset >= 0, "spill to negative address?");
+ 
+   Register base = sp;
+ 
+   // An immediate offset in the range 0 to 255 which is multiplied
+   // by the current vector or predicate register size in bytes.
+   if (offset % sve_reg_size_in_bytes == 0 && offset < ((1<<8)*sve_reg_size_in_bytes)) {
+     return Address(base, offset / sve_reg_size_in_bytes);
+   }
+ 
+   add(tmp, base, offset);
+   return Address(tmp);
+ }
+ 
  // Checks whether offset is aligned.
  // Returns true if it is, else false.
  bool MacroAssembler::merge_alignment_check(Register base,
                                             size_t size,
                                             int64_t cur_offset,
*** 5223,5227 ****
--- 5256,5281 ----
    // we only need a barrier post sync
    if (!is_pre) {
      membar(Assembler::AnyAny);
    }
  }
+ 
+ void MacroAssembler::verify_sve_vector_length() {
+   Label verify_ok;
+   assert(UseSVE > 0, "should only be used for SVE");
+   movw(rscratch1, zr);
+   sve_inc(rscratch1, B);
+   subsw(zr, rscratch1, VM_Version::get_initial_sve_vector_length());
+   br(EQ, verify_ok);
+   stop("Error: SVE vector length has changed since jvm startup");
+   bind(verify_ok);
+ }
+ 
+ void MacroAssembler::verify_ptrue() {
+   Label verify_ok;
+   assert(UseSVE > 0, "should only be used for SVE");
+   sve_cntp(rscratch1, B, ptrue, ptrue); // get true elements count.
+   sve_dec(rscratch1, B);
+   cbz(rscratch1, verify_ok);
+   stop("Error: the preserved predicate register (p7) elements are not all true");
+   bind(verify_ok);
+ }