602 __ addptr(rsp, wordSize * 2);
603
604 __ ret(0);
605
606 return start;
607 }
608 //---------------------------------------------------------------------------------------------------
609
610 address generate_vector_mask(const char *stub_name, int32_t mask) {
611 __ align(CodeEntryAlignment);
612 StubCodeMark mark(this, "StubRoutines", stub_name);
613 address start = __ pc();
614
615 for (int i = 0; i < 16; i++) {
616 __ emit_data(mask, relocInfo::none, 0);
617 }
618
619 return start;
620 }
621
622 address generate_vector_mask_long_double(const char *stub_name, int32_t maskhi, int32_t masklo) {
623 __ align(CodeEntryAlignment);
624 StubCodeMark mark(this, "StubRoutines", stub_name);
625 address start = __ pc();
626
627 for (int i = 0; i < 8; i++) {
628 __ emit_data(masklo, relocInfo::none, 0);
629 __ emit_data(maskhi, relocInfo::none, 0);
630 }
631
632 return start;
633 }
634
635 //----------------------------------------------------------------------------------------------------
636
637 address generate_vector_byte_perm_mask(const char *stub_name) {
638 __ align(CodeEntryAlignment);
639 StubCodeMark mark(this, "StubRoutines", stub_name);
640 address start = __ pc();
641
642 __ emit_data(0x00000001, relocInfo::none, 0);
643 __ emit_data(0x00000000, relocInfo::none, 0);
644 __ emit_data(0x00000003, relocInfo::none, 0);
645 __ emit_data(0x00000000, relocInfo::none, 0);
646 __ emit_data(0x00000005, relocInfo::none, 0);
647 __ emit_data(0x00000000, relocInfo::none, 0);
648 __ emit_data(0x00000007, relocInfo::none, 0);
649 __ emit_data(0x00000000, relocInfo::none, 0);
650 __ emit_data(0x00000000, relocInfo::none, 0);
651 __ emit_data(0x00000000, relocInfo::none, 0);
652 __ emit_data(0x00000002, relocInfo::none, 0);
653 __ emit_data(0x00000000, relocInfo::none, 0);
654 __ emit_data(0x00000004, relocInfo::none, 0);
655 __ emit_data(0x00000000, relocInfo::none, 0);
656 __ emit_data(0x00000006, relocInfo::none, 0);
657 __ emit_data(0x00000000, relocInfo::none, 0);
658
659 return start;
660 }
661
662 //----------------------------------------------------------------------------------------------------
663 // Non-destructive plausibility checks for oops
664
665 address generate_verify_oop() {
666 StubCodeMark mark(this, "StubRoutines", "verify_oop");
667 address start = __ pc();
668
669 // Incoming arguments on stack after saving rax,:
670 //
671 // [tos ]: saved rdx
672 // [tos + 1]: saved EFLAGS
673 // [tos + 2]: return address
674 // [tos + 3]: char* error message
675 // [tos + 4]: oop object to verify
676 // [tos + 5]: saved rax, - saved by caller and bashed
677
678 Label exit, error;
679 __ pushf();
680 __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
681 __ push(rdx); // save rdx
3880 StubRoutines::_safefetchN_continuation_pc = StubRoutines::_safefetch32_continuation_pc;
3881 }
3882
3883 void generate_all() {
3884 // Generates all stubs and initializes the entry points
3885
3886 // These entry points require SharedInfo::stack0 to be set up in non-core builds
3887 // and need to be relocatable, so they each fabricate a RuntimeStub internally.
3888 StubRoutines::_throw_AbstractMethodError_entry = generate_throw_exception("AbstractMethodError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_AbstractMethodError));
3889 StubRoutines::_throw_IncompatibleClassChangeError_entry= generate_throw_exception("IncompatibleClassChangeError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_IncompatibleClassChangeError));
3890 StubRoutines::_throw_NullPointerException_at_call_entry= generate_throw_exception("NullPointerException at call throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_NullPointerException_at_call));
3891
3892 //------------------------------------------------------------------------------------------------------------------------
3893 // entry points that are platform specific
3894
3895 StubRoutines::x86::_vector_float_sign_mask = generate_vector_mask("vector_float_sign_mask", 0x7FFFFFFF);
3896 StubRoutines::x86::_vector_float_sign_flip = generate_vector_mask("vector_float_sign_flip", 0x80000000);
3897 StubRoutines::x86::_vector_double_sign_mask = generate_vector_mask_long_double("vector_double_sign_mask", 0x7FFFFFFF, 0xFFFFFFFF);
3898 StubRoutines::x86::_vector_double_sign_flip = generate_vector_mask_long_double("vector_double_sign_flip", 0x80000000, 0x00000000);
3899 StubRoutines::x86::_vector_short_to_byte_mask = generate_vector_mask("vector_short_to_byte_mask", 0x00ff00ff);
3900 StubRoutines::x86::_vector_byte_perm_mask = generate_vector_byte_perm_mask("vector_byte_perm_mask");
3901 StubRoutines::x86::_vector_long_sign_mask = generate_vector_mask_long_double("vector_long_sign_mask", 0x80000000, 0x00000000);
3902
3903 // support for verify_oop (must happen after universe_init)
3904 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
3905
3906 // arraycopy stubs used by compilers
3907 generate_arraycopy_stubs();
3908
3909 // don't bother generating these AES intrinsic stubs unless global flag is set
3910 if (UseAESIntrinsics) {
3911 StubRoutines::x86::_key_shuffle_mask_addr = generate_key_shuffle_mask(); // might be needed by the others
3912
3913 StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
3914 StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
3915 StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
3916 StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt_Parallel();
3917 }
3918
3919 if (UseAESCTRIntrinsics) {
3920 StubRoutines::x86::_counter_shuffle_mask_addr = generate_counter_shuffle_mask();
3921 StubRoutines::_counterMode_AESCrypt = generate_counterMode_AESCrypt_Parallel();
|
602 __ addptr(rsp, wordSize * 2);
603
604 __ ret(0);
605
606 return start;
607 }
608 //---------------------------------------------------------------------------------------------------
609
610 address generate_vector_mask(const char *stub_name, int32_t mask) {
611 __ align(CodeEntryAlignment);
612 StubCodeMark mark(this, "StubRoutines", stub_name);
613 address start = __ pc();
614
615 for (int i = 0; i < 16; i++) {
616 __ emit_data(mask, relocInfo::none, 0);
617 }
618
619 return start;
620 }
621
622 address generate_iota_indices(const char *stub_name) {
623 __ align(CodeEntryAlignment);
624 StubCodeMark mark(this, "StubRoutines", stub_name);
625 address start = __ pc();
626 __ emit_data(0x03020100, relocInfo::none, 0);
627 __ emit_data(0x07060504, relocInfo::none, 0);
628 __ emit_data(0x0B0A0908, relocInfo::none, 0);
629 __ emit_data(0x0F0E0D0C, relocInfo::none, 0);
630 __ emit_data(0x13121110, relocInfo::none, 0);
631 __ emit_data(0x17161514, relocInfo::none, 0);
632 __ emit_data(0x1B1A1918, relocInfo::none, 0);
633 __ emit_data(0x1F1E1D1C, relocInfo::none, 0);
634 __ emit_data(0x23222120, relocInfo::none, 0);
635 __ emit_data(0x27262524, relocInfo::none, 0);
636 __ emit_data(0x2B2A2928, relocInfo::none, 0);
637 __ emit_data(0x2F2E2D2C, relocInfo::none, 0);
638 __ emit_data(0x33323130, relocInfo::none, 0);
639 __ emit_data(0x37363534, relocInfo::none, 0);
640 __ emit_data(0x3B3A3938, relocInfo::none, 0);
641 __ emit_data(0x3F3E3D3C, relocInfo::none, 0);
642 return start;
643 }
644
645 address generate_vector_mask_long_double(const char *stub_name, int32_t maskhi, int32_t masklo) {
646 __ align(CodeEntryAlignment);
647 StubCodeMark mark(this, "StubRoutines", stub_name);
648 address start = __ pc();
649
650 for (int i = 0; i < 8; i++) {
651 __ emit_data(masklo, relocInfo::none, 0);
652 __ emit_data(maskhi, relocInfo::none, 0);
653 }
654
655 return start;
656 }
657
658 //----------------------------------------------------------------------------------------------------
659
660 address generate_vector_byte_perm_mask(const char *stub_name) {
661 __ align(CodeEntryAlignment);
662 StubCodeMark mark(this, "StubRoutines", stub_name);
663 address start = __ pc();
664
665 __ emit_data(0x00000001, relocInfo::none, 0);
666 __ emit_data(0x00000000, relocInfo::none, 0);
667 __ emit_data(0x00000003, relocInfo::none, 0);
668 __ emit_data(0x00000000, relocInfo::none, 0);
669 __ emit_data(0x00000005, relocInfo::none, 0);
670 __ emit_data(0x00000000, relocInfo::none, 0);
671 __ emit_data(0x00000007, relocInfo::none, 0);
672 __ emit_data(0x00000000, relocInfo::none, 0);
673 __ emit_data(0x00000000, relocInfo::none, 0);
674 __ emit_data(0x00000000, relocInfo::none, 0);
675 __ emit_data(0x00000002, relocInfo::none, 0);
676 __ emit_data(0x00000000, relocInfo::none, 0);
677 __ emit_data(0x00000004, relocInfo::none, 0);
678 __ emit_data(0x00000000, relocInfo::none, 0);
679 __ emit_data(0x00000006, relocInfo::none, 0);
680 __ emit_data(0x00000000, relocInfo::none, 0);
681
682 return start;
683 }
684
685 address generate_vector_custom_i32(const char *stub_name, Assembler::AvxVectorLen len,
686 int32_t val0, int32_t val1, int32_t val2, int32_t val3,
687 int32_t val4 = 0, int32_t val5 = 0, int32_t val6 = 0, int32_t val7 = 0,
688 int32_t val8 = 0, int32_t val9 = 0, int32_t val10 = 0, int32_t val11 = 0,
689 int32_t val12 = 0, int32_t val13 = 0, int32_t val14 = 0, int32_t val15 = 0) {
690 __ align(CodeEntryAlignment);
691 StubCodeMark mark(this, "StubRoutines", stub_name);
692 address start = __ pc();
693
694 assert(len != Assembler::AVX_NoVec, "vector len must be specified");
695 __ emit_data(val0, relocInfo::none, 0);
696 __ emit_data(val1, relocInfo::none, 0);
697 __ emit_data(val2, relocInfo::none, 0);
698 __ emit_data(val3, relocInfo::none, 0);
699 if (len >= Assembler::AVX_256bit) {
700 __ emit_data(val4, relocInfo::none, 0);
701 __ emit_data(val5, relocInfo::none, 0);
702 __ emit_data(val6, relocInfo::none, 0);
703 __ emit_data(val7, relocInfo::none, 0);
704 if (len >= Assembler::AVX_512bit) {
705 __ emit_data(val8, relocInfo::none, 0);
706 __ emit_data(val9, relocInfo::none, 0);
707 __ emit_data(val10, relocInfo::none, 0);
708 __ emit_data(val11, relocInfo::none, 0);
709 __ emit_data(val12, relocInfo::none, 0);
710 __ emit_data(val13, relocInfo::none, 0);
711 __ emit_data(val14, relocInfo::none, 0);
712 __ emit_data(val15, relocInfo::none, 0);
713 }
714 }
715
716 return start;
717 }
718
719 //----------------------------------------------------------------------------------------------------
720 // Non-destructive plausibility checks for oops
721
722 address generate_verify_oop() {
723 StubCodeMark mark(this, "StubRoutines", "verify_oop");
724 address start = __ pc();
725
726 // Incoming arguments on stack after saving rax,:
727 //
728 // [tos ]: saved rdx
729 // [tos + 1]: saved EFLAGS
730 // [tos + 2]: return address
731 // [tos + 3]: char* error message
732 // [tos + 4]: oop object to verify
733 // [tos + 5]: saved rax, - saved by caller and bashed
734
735 Label exit, error;
736 __ pushf();
737 __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
738 __ push(rdx); // save rdx
3937 StubRoutines::_safefetchN_continuation_pc = StubRoutines::_safefetch32_continuation_pc;
3938 }
3939
3940 void generate_all() {
3941 // Generates all stubs and initializes the entry points
3942
3943 // These entry points require SharedInfo::stack0 to be set up in non-core builds
3944 // and need to be relocatable, so they each fabricate a RuntimeStub internally.
3945 StubRoutines::_throw_AbstractMethodError_entry = generate_throw_exception("AbstractMethodError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_AbstractMethodError));
3946 StubRoutines::_throw_IncompatibleClassChangeError_entry= generate_throw_exception("IncompatibleClassChangeError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_IncompatibleClassChangeError));
3947 StubRoutines::_throw_NullPointerException_at_call_entry= generate_throw_exception("NullPointerException at call throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_NullPointerException_at_call));
3948
3949 //------------------------------------------------------------------------------------------------------------------------
3950 // entry points that are platform specific
3951
3952 StubRoutines::x86::_vector_float_sign_mask = generate_vector_mask("vector_float_sign_mask", 0x7FFFFFFF);
3953 StubRoutines::x86::_vector_float_sign_flip = generate_vector_mask("vector_float_sign_flip", 0x80000000);
3954 StubRoutines::x86::_vector_double_sign_mask = generate_vector_mask_long_double("vector_double_sign_mask", 0x7FFFFFFF, 0xFFFFFFFF);
3955 StubRoutines::x86::_vector_double_sign_flip = generate_vector_mask_long_double("vector_double_sign_flip", 0x80000000, 0x00000000);
3956 StubRoutines::x86::_vector_short_to_byte_mask = generate_vector_mask("vector_short_to_byte_mask", 0x00ff00ff);
3957 StubRoutines::x86::_vector_int_to_byte_mask = generate_vector_mask("vector_int_to_byte_mask", 0x000000ff);
3958 StubRoutines::x86::_vector_int_to_short_mask = generate_vector_mask("vector_int_to_short_mask", 0x0000ffff);
3959 StubRoutines::x86::_vector_32_bit_mask = generate_vector_custom_i32("vector_32_bit_mask", Assembler::AVX_512bit,
3960 0xFFFFFFFF, 0, 0, 0);
3961 StubRoutines::x86::_vector_64_bit_mask = generate_vector_custom_i32("vector_64_bit_mask", Assembler::AVX_512bit,
3962 0xFFFFFFFF, 0xFFFFFFFF, 0, 0);
3963 StubRoutines::x86::_vector_int_shuffle_mask = generate_vector_mask("vector_int_shuffle_mask", 0x03020100);
3964 StubRoutines::x86::_vector_short_shuffle_mask = generate_vector_mask("vector_short_shuffle_mask", 0x01000100);
3965 StubRoutines::x86::_vector_long_shuffle_mask = generate_vector_mask_long_double("vector_long_shuffle_mask", 0x00000001, 0x0);
3966 StubRoutines::x86::_vector_byte_perm_mask = generate_vector_byte_perm_mask("vector_byte_perm_mask");
3967 StubRoutines::x86::_vector_long_sign_mask = generate_vector_mask_long_double("vector_long_sign_mask", 0x80000000, 0x00000000);
3968 StubRoutines::x86::_vector_all_bits_set = generate_vector_mask("vector_all_bits_set", 0xFFFFFFFF);
3969 StubRoutines::x86::_vector_iota_indices = generate_iota_indices("iota_indices");
3970
3971 // support for verify_oop (must happen after universe_init)
3972 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
3973
3974 // arraycopy stubs used by compilers
3975 generate_arraycopy_stubs();
3976
3977 // don't bother generating these AES intrinsic stubs unless global flag is set
3978 if (UseAESIntrinsics) {
3979 StubRoutines::x86::_key_shuffle_mask_addr = generate_key_shuffle_mask(); // might be needed by the others
3980
3981 StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
3982 StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
3983 StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
3984 StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt_Parallel();
3985 }
3986
3987 if (UseAESCTRIntrinsics) {
3988 StubRoutines::x86::_counter_shuffle_mask_addr = generate_counter_shuffle_mask();
3989 StubRoutines::_counterMode_AESCrypt = generate_counterMode_AESCrypt_Parallel();
|