956 __ jcc(Assembler::negative, L); // NaN -> 0
957 __ testq(c_rarg0, c_rarg0); // signed ? min_jlong : max_jlong
958 __ mov64(c_rarg2, 0x8000000000000000);
959 __ mov64(rax, 0x7fffffffffffffff);
960 __ cmovq(Assembler::positive, c_rarg2, rax);
961
962 __ bind(L);
963 __ movq(inout, c_rarg2);
964
965 __ pop(c_rarg0);
966 __ pop(c_rarg1);
967 __ pop(c_rarg2);
968 __ pop(c_rarg3);
969 __ pop(rax);
970
971 __ ret(0);
972
973 return start;
974 }
975
976 address generate_fp_mask(const char *stub_name, int64_t mask) {
977 __ align(CodeEntryAlignment);
978 StubCodeMark mark(this, "StubRoutines", stub_name);
979 address start = __ pc();
980
981 __ emit_data64( mask, relocInfo::none );
982 __ emit_data64( mask, relocInfo::none );
983
984 return start;
985 }
986
987 address generate_vector_mask(const char *stub_name, int64_t mask) {
988 __ align(CodeEntryAlignment);
989 StubCodeMark mark(this, "StubRoutines", stub_name);
990 address start = __ pc();
991
992 __ emit_data64(mask, relocInfo::none);
993 __ emit_data64(mask, relocInfo::none);
994 __ emit_data64(mask, relocInfo::none);
995 __ emit_data64(mask, relocInfo::none);
1001 return start;
1002 }
1003
1004 address generate_vector_byte_perm_mask(const char *stub_name) {
1005 __ align(CodeEntryAlignment);
1006 StubCodeMark mark(this, "StubRoutines", stub_name);
1007 address start = __ pc();
1008
1009 __ emit_data64(0x0000000000000001, relocInfo::none);
1010 __ emit_data64(0x0000000000000003, relocInfo::none);
1011 __ emit_data64(0x0000000000000005, relocInfo::none);
1012 __ emit_data64(0x0000000000000007, relocInfo::none);
1013 __ emit_data64(0x0000000000000000, relocInfo::none);
1014 __ emit_data64(0x0000000000000002, relocInfo::none);
1015 __ emit_data64(0x0000000000000004, relocInfo::none);
1016 __ emit_data64(0x0000000000000006, relocInfo::none);
1017
1018 return start;
1019 }
1020
1021 // Non-destructive plausibility checks for oops
1022 //
1023 // Arguments:
1024 // all args on stack!
1025 //
1026 // Stack after saving c_rarg3:
1027 // [tos + 0]: saved c_rarg3
1028 // [tos + 1]: saved c_rarg2
1029 // [tos + 2]: saved r12 (several TemplateTable methods use it)
1030 // [tos + 3]: saved flags
1031 // [tos + 4]: return address
1032 // * [tos + 5]: error message (char*)
1033 // * [tos + 6]: object to verify (oop)
1034 // * [tos + 7]: saved rax - saved by caller and bashed
1035 // * [tos + 8]: saved r10 (rscratch1) - saved by caller
1036 // * = popped on exit
1037 address generate_verify_oop() {
1038 StubCodeMark mark(this, "StubRoutines", "verify_oop");
1039 address start = __ pc();
1040
6443 SharedRuntime::
6444 throw_AbstractMethodError));
6445
6446 StubRoutines::_throw_IncompatibleClassChangeError_entry =
6447 generate_throw_exception("IncompatibleClassChangeError throw_exception",
6448 CAST_FROM_FN_PTR(address,
6449 SharedRuntime::
6450 throw_IncompatibleClassChangeError));
6451
6452 StubRoutines::_throw_NullPointerException_at_call_entry =
6453 generate_throw_exception("NullPointerException at call throw_exception",
6454 CAST_FROM_FN_PTR(address,
6455 SharedRuntime::
6456 throw_NullPointerException_at_call));
6457
6458 // entry points that are platform specific
6459 StubRoutines::x86::_vector_float_sign_mask = generate_vector_mask("vector_float_sign_mask", 0x7FFFFFFF7FFFFFFF);
6460 StubRoutines::x86::_vector_float_sign_flip = generate_vector_mask("vector_float_sign_flip", 0x8000000080000000);
6461 StubRoutines::x86::_vector_double_sign_mask = generate_vector_mask("vector_double_sign_mask", 0x7FFFFFFFFFFFFFFF);
6462 StubRoutines::x86::_vector_double_sign_flip = generate_vector_mask("vector_double_sign_flip", 0x8000000000000000);
6463 StubRoutines::x86::_vector_short_to_byte_mask = generate_vector_mask("vector_short_to_byte_mask", 0x00ff00ff00ff00ff);
6464 StubRoutines::x86::_vector_byte_perm_mask = generate_vector_byte_perm_mask("vector_byte_perm_mask");
6465 StubRoutines::x86::_vector_long_sign_mask = generate_vector_mask("vector_long_sign_mask", 0x8000000000000000);
6466
6467 // support for verify_oop (must happen after universe_init)
6468 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
6469
6470 // data cache line writeback
6471 StubRoutines::_data_cache_writeback = generate_data_cache_writeback();
6472 StubRoutines::_data_cache_writeback_sync = generate_data_cache_writeback_sync();
6473
6474 // arraycopy stubs used by compilers
6475 generate_arraycopy_stubs();
6476
6477 // don't bother generating these AES intrinsic stubs unless global flag is set
6478 if (UseAESIntrinsics) {
6479 StubRoutines::x86::_key_shuffle_mask_addr = generate_key_shuffle_mask(); // needed by the others
6480 StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
6481 StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
6482 StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
6483 if (VM_Version::supports_avx512_vaes() && VM_Version::supports_avx512vl() && VM_Version::supports_avx512dq() ) {
6484 StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptVectorAESCrypt();
6485 StubRoutines::_electronicCodeBook_encryptAESCrypt = generate_electronicCodeBook_encryptAESCrypt();
|
956 __ jcc(Assembler::negative, L); // NaN -> 0
957 __ testq(c_rarg0, c_rarg0); // signed ? min_jlong : max_jlong
958 __ mov64(c_rarg2, 0x8000000000000000);
959 __ mov64(rax, 0x7fffffffffffffff);
960 __ cmovq(Assembler::positive, c_rarg2, rax);
961
962 __ bind(L);
963 __ movq(inout, c_rarg2);
964
965 __ pop(c_rarg0);
966 __ pop(c_rarg1);
967 __ pop(c_rarg2);
968 __ pop(c_rarg3);
969 __ pop(rax);
970
971 __ ret(0);
972
973 return start;
974 }
975
976 address generate_iota_indices(const char *stub_name) {
977 __ align(CodeEntryAlignment);
978 StubCodeMark mark(this, "StubRoutines", stub_name);
979 address start = __ pc();
980 __ emit_data64(0x0706050403020100, relocInfo::none);
981 __ emit_data64(0x0F0E0D0C0B0A0908, relocInfo::none);
982 __ emit_data64(0x1716151413121110, relocInfo::none);
983 __ emit_data64(0x1F1E1D1C1B1A1918, relocInfo::none);
984 __ emit_data64(0x2726252423222120, relocInfo::none);
985 __ emit_data64(0x2F2E2D2C2B2A2928, relocInfo::none);
986 __ emit_data64(0x3736353433323130, relocInfo::none);
987 __ emit_data64(0x3F3E3D3C3B3A3938, relocInfo::none);
988 return start;
989 }
990
991 address generate_fp_mask(const char *stub_name, int64_t mask) {
992 __ align(CodeEntryAlignment);
993 StubCodeMark mark(this, "StubRoutines", stub_name);
994 address start = __ pc();
995
996 __ emit_data64( mask, relocInfo::none );
997 __ emit_data64( mask, relocInfo::none );
998
999 return start;
1000 }
1001
1002 address generate_vector_mask(const char *stub_name, int64_t mask) {
1003 __ align(CodeEntryAlignment);
1004 StubCodeMark mark(this, "StubRoutines", stub_name);
1005 address start = __ pc();
1006
1007 __ emit_data64(mask, relocInfo::none);
1008 __ emit_data64(mask, relocInfo::none);
1009 __ emit_data64(mask, relocInfo::none);
1010 __ emit_data64(mask, relocInfo::none);
1016 return start;
1017 }
1018
1019 address generate_vector_byte_perm_mask(const char *stub_name) {
1020 __ align(CodeEntryAlignment);
1021 StubCodeMark mark(this, "StubRoutines", stub_name);
1022 address start = __ pc();
1023
1024 __ emit_data64(0x0000000000000001, relocInfo::none);
1025 __ emit_data64(0x0000000000000003, relocInfo::none);
1026 __ emit_data64(0x0000000000000005, relocInfo::none);
1027 __ emit_data64(0x0000000000000007, relocInfo::none);
1028 __ emit_data64(0x0000000000000000, relocInfo::none);
1029 __ emit_data64(0x0000000000000002, relocInfo::none);
1030 __ emit_data64(0x0000000000000004, relocInfo::none);
1031 __ emit_data64(0x0000000000000006, relocInfo::none);
1032
1033 return start;
1034 }
1035
1036 address generate_vector_fp_mask(const char *stub_name, int64_t mask) {
1037 __ align(CodeEntryAlignment);
1038 StubCodeMark mark(this, "StubRoutines", stub_name);
1039 address start = __ pc();
1040
1041 __ emit_data64(mask, relocInfo::none);
1042 __ emit_data64(mask, relocInfo::none);
1043 __ emit_data64(mask, relocInfo::none);
1044 __ emit_data64(mask, relocInfo::none);
1045 __ emit_data64(mask, relocInfo::none);
1046 __ emit_data64(mask, relocInfo::none);
1047 __ emit_data64(mask, relocInfo::none);
1048 __ emit_data64(mask, relocInfo::none);
1049
1050 return start;
1051 }
1052
1053 address generate_vector_custom_i32(const char *stub_name, Assembler::AvxVectorLen len,
1054 int32_t val0, int32_t val1, int32_t val2, int32_t val3,
1055 int32_t val4 = 0, int32_t val5 = 0, int32_t val6 = 0, int32_t val7 = 0,
1056 int32_t val8 = 0, int32_t val9 = 0, int32_t val10 = 0, int32_t val11 = 0,
1057 int32_t val12 = 0, int32_t val13 = 0, int32_t val14 = 0, int32_t val15 = 0) {
1058 __ align(CodeEntryAlignment);
1059 StubCodeMark mark(this, "StubRoutines", stub_name);
1060 address start = __ pc();
1061
1062 assert(len != Assembler::AVX_NoVec, "vector len must be specified");
1063 __ emit_data(val0, relocInfo::none, 0);
1064 __ emit_data(val1, relocInfo::none, 0);
1065 __ emit_data(val2, relocInfo::none, 0);
1066 __ emit_data(val3, relocInfo::none, 0);
1067 if (len >= Assembler::AVX_256bit) {
1068 __ emit_data(val4, relocInfo::none, 0);
1069 __ emit_data(val5, relocInfo::none, 0);
1070 __ emit_data(val6, relocInfo::none, 0);
1071 __ emit_data(val7, relocInfo::none, 0);
1072 if (len >= Assembler::AVX_512bit) {
1073 __ emit_data(val8, relocInfo::none, 0);
1074 __ emit_data(val9, relocInfo::none, 0);
1075 __ emit_data(val10, relocInfo::none, 0);
1076 __ emit_data(val11, relocInfo::none, 0);
1077 __ emit_data(val12, relocInfo::none, 0);
1078 __ emit_data(val13, relocInfo::none, 0);
1079 __ emit_data(val14, relocInfo::none, 0);
1080 __ emit_data(val15, relocInfo::none, 0);
1081 }
1082 }
1083
1084 return start;
1085 }
1086
1087 // Non-destructive plausibility checks for oops
1088 //
1089 // Arguments:
1090 // all args on stack!
1091 //
1092 // Stack after saving c_rarg3:
1093 // [tos + 0]: saved c_rarg3
1094 // [tos + 1]: saved c_rarg2
1095 // [tos + 2]: saved r12 (several TemplateTable methods use it)
1096 // [tos + 3]: saved flags
1097 // [tos + 4]: return address
1098 // * [tos + 5]: error message (char*)
1099 // * [tos + 6]: object to verify (oop)
1100 // * [tos + 7]: saved rax - saved by caller and bashed
1101 // * [tos + 8]: saved r10 (rscratch1) - saved by caller
1102 // * = popped on exit
1103 address generate_verify_oop() {
1104 StubCodeMark mark(this, "StubRoutines", "verify_oop");
1105 address start = __ pc();
1106
6509 SharedRuntime::
6510 throw_AbstractMethodError));
6511
6512 StubRoutines::_throw_IncompatibleClassChangeError_entry =
6513 generate_throw_exception("IncompatibleClassChangeError throw_exception",
6514 CAST_FROM_FN_PTR(address,
6515 SharedRuntime::
6516 throw_IncompatibleClassChangeError));
6517
6518 StubRoutines::_throw_NullPointerException_at_call_entry =
6519 generate_throw_exception("NullPointerException at call throw_exception",
6520 CAST_FROM_FN_PTR(address,
6521 SharedRuntime::
6522 throw_NullPointerException_at_call));
6523
6524 // entry points that are platform specific
6525 StubRoutines::x86::_vector_float_sign_mask = generate_vector_mask("vector_float_sign_mask", 0x7FFFFFFF7FFFFFFF);
6526 StubRoutines::x86::_vector_float_sign_flip = generate_vector_mask("vector_float_sign_flip", 0x8000000080000000);
6527 StubRoutines::x86::_vector_double_sign_mask = generate_vector_mask("vector_double_sign_mask", 0x7FFFFFFFFFFFFFFF);
6528 StubRoutines::x86::_vector_double_sign_flip = generate_vector_mask("vector_double_sign_flip", 0x8000000000000000);
6529 StubRoutines::x86::_vector_all_bits_set = generate_vector_mask("vector_all_bits_set", 0xFFFFFFFFFFFFFFFF);
6530 StubRoutines::x86::_vector_short_to_byte_mask = generate_vector_mask("vector_short_to_byte_mask", 0x00ff00ff00ff00ff);
6531 StubRoutines::x86::_vector_byte_perm_mask = generate_vector_byte_perm_mask("vector_byte_perm_mask");
6532 StubRoutines::x86::_vector_int_to_byte_mask = generate_vector_mask("vector_int_to_byte_mask", 0x000000ff000000ff);
6533 StubRoutines::x86::_vector_int_to_short_mask = generate_vector_mask("vector_int_to_short_mask", 0x0000ffff0000ffff);
6534 StubRoutines::x86::_vector_32_bit_mask = generate_vector_custom_i32("vector_32_bit_mask", Assembler::AVX_512bit,
6535 0xFFFFFFFF, 0, 0, 0);
6536 StubRoutines::x86::_vector_64_bit_mask = generate_vector_custom_i32("vector_64_bit_mask", Assembler::AVX_512bit,
6537 0xFFFFFFFF, 0xFFFFFFFF, 0, 0);
6538 StubRoutines::x86::_vector_int_shuffle_mask = generate_vector_mask("vector_int_shuffle_mask", 0x0302010003020100);
6539 StubRoutines::x86::_vector_short_shuffle_mask = generate_vector_mask("vector_short_shuffle_mask", 0x0100010001000100);
6540 StubRoutines::x86::_vector_long_shuffle_mask = generate_vector_mask("vector_long_shuffle_mask", 0x0000000100000000);
6541 StubRoutines::x86::_vector_long_sign_mask = generate_vector_mask("vector_long_sign_mask", 0x8000000000000000);
6542 StubRoutines::x86::_vector_iota_indices = generate_iota_indices("iota_indices");
6543
6544 // support for verify_oop (must happen after universe_init)
6545 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
6546
6547 // data cache line writeback
6548 StubRoutines::_data_cache_writeback = generate_data_cache_writeback();
6549 StubRoutines::_data_cache_writeback_sync = generate_data_cache_writeback_sync();
6550
6551 // arraycopy stubs used by compilers
6552 generate_arraycopy_stubs();
6553
6554 // don't bother generating these AES intrinsic stubs unless global flag is set
6555 if (UseAESIntrinsics) {
6556 StubRoutines::x86::_key_shuffle_mask_addr = generate_key_shuffle_mask(); // needed by the others
6557 StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
6558 StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
6559 StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
6560 if (VM_Version::supports_avx512_vaes() && VM_Version::supports_avx512vl() && VM_Version::supports_avx512dq() ) {
6561 StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptVectorAESCrypt();
6562 StubRoutines::_electronicCodeBook_encryptAESCrypt = generate_electronicCodeBook_encryptAESCrypt();
|