956 __ jcc(Assembler::negative, L); // NaN -> 0
957 __ testq(c_rarg0, c_rarg0); // signed ? min_jlong : max_jlong
958 __ mov64(c_rarg2, 0x8000000000000000);
959 __ mov64(rax, 0x7fffffffffffffff);
960 __ cmovq(Assembler::positive, c_rarg2, rax);
961
962 __ bind(L);
963 __ movq(inout, c_rarg2);
964
965 __ pop(c_rarg0);
966 __ pop(c_rarg1);
967 __ pop(c_rarg2);
968 __ pop(c_rarg3);
969 __ pop(rax);
970
971 __ ret(0);
972
973 return start;
974 }
975
976 address generate_fp_mask(const char *stub_name, int64_t mask) {
977 __ align(CodeEntryAlignment);
978 StubCodeMark mark(this, "StubRoutines", stub_name);
979 address start = __ pc();
980
981 __ emit_data64( mask, relocInfo::none );
982 __ emit_data64( mask, relocInfo::none );
983
984 return start;
985 }
986
987 address generate_vector_mask(const char *stub_name, int64_t mask) {
988 __ align(CodeEntryAlignment);
989 StubCodeMark mark(this, "StubRoutines", stub_name);
990 address start = __ pc();
991
992 __ emit_data64(mask, relocInfo::none);
993 __ emit_data64(mask, relocInfo::none);
994 __ emit_data64(mask, relocInfo::none);
995 __ emit_data64(mask, relocInfo::none);
1001 return start;
1002 }
1003
1004 address generate_vector_byte_perm_mask(const char *stub_name) {
1005 __ align(CodeEntryAlignment);
1006 StubCodeMark mark(this, "StubRoutines", stub_name);
1007 address start = __ pc();
1008
1009 __ emit_data64(0x0000000000000001, relocInfo::none);
1010 __ emit_data64(0x0000000000000003, relocInfo::none);
1011 __ emit_data64(0x0000000000000005, relocInfo::none);
1012 __ emit_data64(0x0000000000000007, relocInfo::none);
1013 __ emit_data64(0x0000000000000000, relocInfo::none);
1014 __ emit_data64(0x0000000000000002, relocInfo::none);
1015 __ emit_data64(0x0000000000000004, relocInfo::none);
1016 __ emit_data64(0x0000000000000006, relocInfo::none);
1017
1018 return start;
1019 }
1020
1021 // Non-destructive plausibility checks for oops
1022 //
1023 // Arguments:
1024 // all args on stack!
1025 //
1026 // Stack after saving c_rarg3:
1027 // [tos + 0]: saved c_rarg3
1028 // [tos + 1]: saved c_rarg2
1029 // [tos + 2]: saved r12 (several TemplateTable methods use it)
1030 // [tos + 3]: saved flags
1031 // [tos + 4]: return address
1032 // * [tos + 5]: error message (char*)
1033 // * [tos + 6]: object to verify (oop)
1034 // * [tos + 7]: saved rax - saved by caller and bashed
1035 // * [tos + 8]: saved r10 (rscratch1) - saved by caller
1036 // * = popped on exit
1037 address generate_verify_oop() {
1038 StubCodeMark mark(this, "StubRoutines", "verify_oop");
1039 address start = __ pc();
1040
6429 SharedRuntime::
6430 throw_AbstractMethodError));
6431
6432 StubRoutines::_throw_IncompatibleClassChangeError_entry =
6433 generate_throw_exception("IncompatibleClassChangeError throw_exception",
6434 CAST_FROM_FN_PTR(address,
6435 SharedRuntime::
6436 throw_IncompatibleClassChangeError));
6437
6438 StubRoutines::_throw_NullPointerException_at_call_entry =
6439 generate_throw_exception("NullPointerException at call throw_exception",
6440 CAST_FROM_FN_PTR(address,
6441 SharedRuntime::
6442 throw_NullPointerException_at_call));
6443
6444 // entry points that are platform specific
6445 StubRoutines::x86::_vector_float_sign_mask = generate_vector_mask("vector_float_sign_mask", 0x7FFFFFFF7FFFFFFF);
6446 StubRoutines::x86::_vector_float_sign_flip = generate_vector_mask("vector_float_sign_flip", 0x8000000080000000);
6447 StubRoutines::x86::_vector_double_sign_mask = generate_vector_mask("vector_double_sign_mask", 0x7FFFFFFFFFFFFFFF);
6448 StubRoutines::x86::_vector_double_sign_flip = generate_vector_mask("vector_double_sign_flip", 0x8000000000000000);
6449 StubRoutines::x86::_vector_short_to_byte_mask = generate_vector_mask("vector_short_to_byte_mask", 0x00ff00ff00ff00ff);
6450 StubRoutines::x86::_vector_byte_perm_mask = generate_vector_byte_perm_mask("vector_byte_perm_mask");
6451 StubRoutines::x86::_vector_long_sign_mask = generate_vector_mask("vector_long_sign_mask", 0x8000000000000000);
6452
6453 // support for verify_oop (must happen after universe_init)
6454 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
6455
6456 // data cache line writeback
6457 StubRoutines::_data_cache_writeback = generate_data_cache_writeback();
6458 StubRoutines::_data_cache_writeback_sync = generate_data_cache_writeback_sync();
6459
6460 // arraycopy stubs used by compilers
6461 generate_arraycopy_stubs();
6462
6463 // don't bother generating these AES intrinsic stubs unless global flag is set
6464 if (UseAESIntrinsics) {
6465 StubRoutines::x86::_key_shuffle_mask_addr = generate_key_shuffle_mask(); // needed by the others
6466 StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
6467 StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
6468 StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
6469 if (VM_Version::supports_avx512_vaes() && VM_Version::supports_avx512vl() && VM_Version::supports_avx512dq() ) {
6470 StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptVectorAESCrypt();
6471 StubRoutines::_electronicCodeBook_encryptAESCrypt = generate_electronicCodeBook_encryptAESCrypt();
|
956 __ jcc(Assembler::negative, L); // NaN -> 0
957 __ testq(c_rarg0, c_rarg0); // signed ? min_jlong : max_jlong
958 __ mov64(c_rarg2, 0x8000000000000000);
959 __ mov64(rax, 0x7fffffffffffffff);
960 __ cmovq(Assembler::positive, c_rarg2, rax);
961
962 __ bind(L);
963 __ movq(inout, c_rarg2);
964
965 __ pop(c_rarg0);
966 __ pop(c_rarg1);
967 __ pop(c_rarg2);
968 __ pop(c_rarg3);
969 __ pop(rax);
970
971 __ ret(0);
972
973 return start;
974 }
975
976 address generate_iota_indices(const char *stub_name) {
977 __ align(CodeEntryAlignment);
978 StubCodeMark mark(this, "StubRoutines", stub_name);
979 address start = __ pc();
980 __ emit_data64(0x0706050403020100, relocInfo::none);
981 __ emit_data64(0x0F0E0D0C0B0A0908, relocInfo::none);
982 __ emit_data64(0x1716151413121110, relocInfo::none);
983 __ emit_data64(0x1F1E1D1C1B1A1918, relocInfo::none);
984 __ emit_data64(0x2726252423222120, relocInfo::none);
985 __ emit_data64(0x2F2E2D2C2B2A2928, relocInfo::none);
986 __ emit_data64(0x3736353433323130, relocInfo::none);
987 __ emit_data64(0x3F3E3D3C3B3A3938, relocInfo::none);
988 return start;
989 }
990
991 address generate_fp_mask(const char *stub_name, int64_t mask) {
992 __ align(CodeEntryAlignment);
993 StubCodeMark mark(this, "StubRoutines", stub_name);
994 address start = __ pc();
995
996 __ emit_data64( mask, relocInfo::none );
997 __ emit_data64( mask, relocInfo::none );
998
999 return start;
1000 }
1001
1002 address generate_vector_mask(const char *stub_name, int64_t mask) {
1003 __ align(CodeEntryAlignment);
1004 StubCodeMark mark(this, "StubRoutines", stub_name);
1005 address start = __ pc();
1006
1007 __ emit_data64(mask, relocInfo::none);
1008 __ emit_data64(mask, relocInfo::none);
1009 __ emit_data64(mask, relocInfo::none);
1010 __ emit_data64(mask, relocInfo::none);
1016 return start;
1017 }
1018
1019 address generate_vector_byte_perm_mask(const char *stub_name) {
1020 __ align(CodeEntryAlignment);
1021 StubCodeMark mark(this, "StubRoutines", stub_name);
1022 address start = __ pc();
1023
1024 __ emit_data64(0x0000000000000001, relocInfo::none);
1025 __ emit_data64(0x0000000000000003, relocInfo::none);
1026 __ emit_data64(0x0000000000000005, relocInfo::none);
1027 __ emit_data64(0x0000000000000007, relocInfo::none);
1028 __ emit_data64(0x0000000000000000, relocInfo::none);
1029 __ emit_data64(0x0000000000000002, relocInfo::none);
1030 __ emit_data64(0x0000000000000004, relocInfo::none);
1031 __ emit_data64(0x0000000000000006, relocInfo::none);
1032
1033 return start;
1034 }
1035
1036 address generate_vector_fp_mask(const char *stub_name, int64_t mask) {
1037 __ align(CodeEntryAlignment);
1038 StubCodeMark mark(this, "StubRoutines", stub_name);
1039 address start = __ pc();
1040
1041 __ emit_data64(mask, relocInfo::none);
1042 __ emit_data64(mask, relocInfo::none);
1043 __ emit_data64(mask, relocInfo::none);
1044 __ emit_data64(mask, relocInfo::none);
1045 __ emit_data64(mask, relocInfo::none);
1046 __ emit_data64(mask, relocInfo::none);
1047 __ emit_data64(mask, relocInfo::none);
1048 __ emit_data64(mask, relocInfo::none);
1049
1050 return start;
1051 }
1052
1053 address generate_vector_custom_i32(const char *stub_name, Assembler::AvxVectorLen len,
1054 int32_t val0, int32_t val1, int32_t val2, int32_t val3,
1055 int32_t val4 = 0, int32_t val5 = 0, int32_t val6 = 0, int32_t val7 = 0,
1056 int32_t val8 = 0, int32_t val9 = 0, int32_t val10 = 0, int32_t val11 = 0,
1057 int32_t val12 = 0, int32_t val13 = 0, int32_t val14 = 0, int32_t val15 = 0) {
1058 __ align(CodeEntryAlignment);
1059 StubCodeMark mark(this, "StubRoutines", stub_name);
1060 address start = __ pc();
1061
1062 assert(len != Assembler::AVX_NoVec, "vector len must be specified");
1063 __ emit_data(val0, relocInfo::none, 0);
1064 __ emit_data(val1, relocInfo::none, 0);
1065 __ emit_data(val2, relocInfo::none, 0);
1066 __ emit_data(val3, relocInfo::none, 0);
1067 if (len >= Assembler::AVX_256bit) {
1068 __ emit_data(val4, relocInfo::none, 0);
1069 __ emit_data(val5, relocInfo::none, 0);
1070 __ emit_data(val6, relocInfo::none, 0);
1071 __ emit_data(val7, relocInfo::none, 0);
1072 if (len >= Assembler::AVX_512bit) {
1073 __ emit_data(val8, relocInfo::none, 0);
1074 __ emit_data(val9, relocInfo::none, 0);
1075 __ emit_data(val10, relocInfo::none, 0);
1076 __ emit_data(val11, relocInfo::none, 0);
1077 __ emit_data(val12, relocInfo::none, 0);
1078 __ emit_data(val13, relocInfo::none, 0);
1079 __ emit_data(val14, relocInfo::none, 0);
1080 __ emit_data(val15, relocInfo::none, 0);
1081 }
1082 }
1083
1084 return start;
1085 }
1086
1087 // Non-destructive plausibility checks for oops
1088 //
1089 // Arguments:
1090 // all args on stack!
1091 //
1092 // Stack after saving c_rarg3:
1093 // [tos + 0]: saved c_rarg3
1094 // [tos + 1]: saved c_rarg2
1095 // [tos + 2]: saved r12 (several TemplateTable methods use it)
1096 // [tos + 3]: saved flags
1097 // [tos + 4]: return address
1098 // * [tos + 5]: error message (char*)
1099 // * [tos + 6]: object to verify (oop)
1100 // * [tos + 7]: saved rax - saved by caller and bashed
1101 // * [tos + 8]: saved r10 (rscratch1) - saved by caller
1102 // * = popped on exit
1103 address generate_verify_oop() {
1104 StubCodeMark mark(this, "StubRoutines", "verify_oop");
1105 address start = __ pc();
1106
6495 SharedRuntime::
6496 throw_AbstractMethodError));
6497
6498 StubRoutines::_throw_IncompatibleClassChangeError_entry =
6499 generate_throw_exception("IncompatibleClassChangeError throw_exception",
6500 CAST_FROM_FN_PTR(address,
6501 SharedRuntime::
6502 throw_IncompatibleClassChangeError));
6503
6504 StubRoutines::_throw_NullPointerException_at_call_entry =
6505 generate_throw_exception("NullPointerException at call throw_exception",
6506 CAST_FROM_FN_PTR(address,
6507 SharedRuntime::
6508 throw_NullPointerException_at_call));
6509
6510 // entry points that are platform specific
6511 StubRoutines::x86::_vector_float_sign_mask = generate_vector_mask("vector_float_sign_mask", 0x7FFFFFFF7FFFFFFF);
6512 StubRoutines::x86::_vector_float_sign_flip = generate_vector_mask("vector_float_sign_flip", 0x8000000080000000);
6513 StubRoutines::x86::_vector_double_sign_mask = generate_vector_mask("vector_double_sign_mask", 0x7FFFFFFFFFFFFFFF);
6514 StubRoutines::x86::_vector_double_sign_flip = generate_vector_mask("vector_double_sign_flip", 0x8000000000000000);
6515 StubRoutines::x86::_vector_all_bits_set = generate_vector_mask("vector_all_bits_set", 0xFFFFFFFFFFFFFFFF);
6516 StubRoutines::x86::_vector_short_to_byte_mask = generate_vector_mask("vector_short_to_byte_mask", 0x00ff00ff00ff00ff);
6517 StubRoutines::x86::_vector_byte_perm_mask = generate_vector_byte_perm_mask("vector_byte_perm_mask");
6518 StubRoutines::x86::_vector_int_to_byte_mask = generate_vector_mask("vector_int_to_byte_mask", 0x000000ff000000ff);
6519 StubRoutines::x86::_vector_int_to_short_mask = generate_vector_mask("vector_int_to_short_mask", 0x0000ffff0000ffff);
6520 StubRoutines::x86::_vector_32_bit_mask = generate_vector_custom_i32("vector_32_bit_mask", Assembler::AVX_512bit,
6521 0xFFFFFFFF, 0, 0, 0);
6522 StubRoutines::x86::_vector_64_bit_mask = generate_vector_custom_i32("vector_64_bit_mask", Assembler::AVX_512bit,
6523 0xFFFFFFFF, 0xFFFFFFFF, 0, 0);
6524 StubRoutines::x86::_vector_int_shuffle_mask = generate_vector_mask("vector_int_shuffle_mask", 0x0302010003020100);
6525 StubRoutines::x86::_vector_short_shuffle_mask = generate_vector_mask("vector_short_shuffle_mask", 0x0100010001000100);
6526 StubRoutines::x86::_vector_long_shuffle_mask = generate_vector_mask("vector_long_shuffle_mask", 0x0000000100000000);
6527 StubRoutines::x86::_vector_long_sign_mask = generate_vector_mask("vector_long_sign_mask", 0x8000000000000000);
6528 StubRoutines::x86::_vector_iota_indices = generate_iota_indices("iota_indices");
6529
6530 // support for verify_oop (must happen after universe_init)
6531 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
6532
6533 // data cache line writeback
6534 StubRoutines::_data_cache_writeback = generate_data_cache_writeback();
6535 StubRoutines::_data_cache_writeback_sync = generate_data_cache_writeback_sync();
6536
6537 // arraycopy stubs used by compilers
6538 generate_arraycopy_stubs();
6539
6540 // don't bother generating these AES intrinsic stubs unless global flag is set
6541 if (UseAESIntrinsics) {
6542 StubRoutines::x86::_key_shuffle_mask_addr = generate_key_shuffle_mask(); // needed by the others
6543 StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
6544 StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
6545 StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
6546 if (VM_Version::supports_avx512_vaes() && VM_Version::supports_avx512vl() && VM_Version::supports_avx512dq() ) {
6547 StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptVectorAESCrypt();
6548 StubRoutines::_electronicCodeBook_encryptAESCrypt = generate_electronicCodeBook_encryptAESCrypt();
|