24
25 #include "precompiled.hpp"
26 #include "asm/macroAssembler.hpp"
27 #include "asm/macroAssembler.inline.hpp"
28 #include "ci/ciUtilities.hpp"
29 #include "gc/shared/barrierSet.hpp"
30 #include "gc/shared/barrierSetAssembler.hpp"
31 #include "interpreter/interpreter.hpp"
32 #include "nativeInst_x86.hpp"
33 #include "oops/instanceOop.hpp"
34 #include "oops/method.hpp"
35 #include "oops/objArrayKlass.hpp"
36 #include "oops/oop.inline.hpp"
37 #include "prims/methodHandles.hpp"
38 #include "runtime/frame.inline.hpp"
39 #include "runtime/handles.inline.hpp"
40 #include "runtime/sharedRuntime.hpp"
41 #include "runtime/stubCodeGenerator.hpp"
42 #include "runtime/stubRoutines.hpp"
43 #include "runtime/thread.inline.hpp"
44 #include "utilities/macros.hpp"
45 #ifdef COMPILER2
46 #include "opto/runtime.hpp"
47 #endif
48 #if INCLUDE_SHENANDOAHGC
49 #include "gc/shenandoah/brooksPointer.hpp"
50 #include "gc/shenandoah/shenandoahHeap.hpp"
51 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
52 #include "gc/shenandoah/shenandoahRuntime.hpp"
53 #endif
54 #if INCLUDE_ZGC
55 #include "gc/z/zThreadLocalData.hpp"
56 #endif
57
58 // Declaration and definition of StubGenerator (no .hpp file).
59 // For a more detailed description of the stub routine structure
60 // see the comment in stubRoutines.hpp
61
62 #define __ _masm->
63 #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8)
64 #define a__ ((Assembler*)_masm)->
65
66 #ifdef PRODUCT
67 #define BLOCK_COMMENT(str) /* nothing */
68 #else
69 #define BLOCK_COMMENT(str) __ block_comment(str)
70 #endif
71
72 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
73 const int MXCSR_MASK = 0xFFC0; // Mask out any pending exceptions
791 __ stmxcsr(mxcsr_save);
792 __ movl(rax, mxcsr_save);
793 __ andl(rax, MXCSR_MASK); // Only check control and mask bits
794 __ cmp32(rax, mxcsr_std);
795 __ jcc(Assembler::equal, ok_ret);
796
797 __ warn("MXCSR changed by native JNI code, use -XX:+RestoreMXCSROnJNICall");
798
799 __ ldmxcsr(mxcsr_std);
800
801 __ bind(ok_ret);
802 __ addptr(rsp, wordSize);
803 __ pop(rax);
804 }
805
806 __ ret(0);
807
808 return start;
809 }
810
811 #if INCLUDE_SHENANDOAHGC
812 address generate_shenandoah_wb(bool c_abi, bool do_cset_test) {
813 StubCodeMark mark(this, "StubRoutines", "shenandoah_wb");
814 address start = __ pc();
815
816 Label not_done;
817
818 // We use RDI, which also serves as argument register for slow call.
819 // RAX always holds the src object ptr, except after the slow call and
820 // the cmpxchg, then it holds the result.
821 // R8 and RCX are used as temporary registers.
822 if (!c_abi) {
823 __ push(rdi);
824 __ push(r8);
825 }
826
827 // Check for object beeing in the collection set.
828 // TODO: Can we use only 1 register here?
829 // The source object arrives here in rax.
830 // live: rax
831 // live: rdi
832 if (!c_abi) {
833 __ mov(rdi, rax);
834 } else {
835 if (rax != c_rarg0) {
836 __ mov(rax, c_rarg0);
837 }
838 }
839 if (do_cset_test) {
840 __ shrptr(rdi, ShenandoahHeapRegion::region_size_bytes_shift_jint());
841 // live: r8
842 __ movptr(r8, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
843 __ movbool(r8, Address(r8, rdi, Address::times_1));
844 // unlive: rdi
845 __ testbool(r8);
846 // unlive: r8
847 __ jccb(Assembler::notZero, not_done);
848
849 if (!c_abi) {
850 __ pop(r8);
851 __ pop(rdi);
852 }
853 __ ret(0);
854
855 __ bind(not_done);
856 }
857
858 if (!c_abi) {
859 __ push(rcx);
860 }
861
862 if (!c_abi) {
863 __ push(rdx);
864 __ push(rdi);
865 __ push(rsi);
866 __ push(r8);
867 __ push(r9);
868 __ push(r10);
869 __ push(r11);
870 __ push(r12);
871 __ push(r13);
872 __ push(r14);
873 __ push(r15);
874 }
875 __ save_vector_registers();
876 __ movptr(rdi, rax);
877 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT), rdi);
878 __ restore_vector_registers();
879 if (!c_abi) {
880 __ pop(r15);
881 __ pop(r14);
882 __ pop(r13);
883 __ pop(r12);
884 __ pop(r11);
885 __ pop(r10);
886 __ pop(r9);
887 __ pop(r8);
888 __ pop(rsi);
889 __ pop(rdi);
890 __ pop(rdx);
891
892 __ pop(rcx);
893 __ pop(r8);
894 __ pop(rdi);
895 }
896 __ ret(0);
897
898 return start;
899 }
900 #endif
901
902 address generate_f2i_fixup() {
903 StubCodeMark mark(this, "StubRoutines", "f2i_fixup");
904 Address inout(rsp, 5 * wordSize); // return address + 4 saves
905
906 address start = __ pc();
907
908 Label L;
909
910 __ push(rax);
911 __ push(c_rarg3);
912 __ push(c_rarg2);
913 __ push(c_rarg1);
914
915 __ movl(rax, 0x7f800000);
916 __ xorl(c_rarg3, c_rarg3);
917 __ movl(c_rarg2, inout);
918 __ movl(c_rarg1, c_rarg2);
919 __ andl(c_rarg1, 0x7fffffff);
920 __ cmpl(rax, c_rarg1); // NaN? -> 0
921 __ jcc(Assembler::negative, L);
5137 // fabricate a RuntimeStub internally.
5138 StubRoutines::_throw_AbstractMethodError_entry =
5139 generate_throw_exception("AbstractMethodError throw_exception",
5140 CAST_FROM_FN_PTR(address,
5141 SharedRuntime::
5142 throw_AbstractMethodError));
5143
5144 StubRoutines::_throw_IncompatibleClassChangeError_entry =
5145 generate_throw_exception("IncompatibleClassChangeError throw_exception",
5146 CAST_FROM_FN_PTR(address,
5147 SharedRuntime::
5148 throw_IncompatibleClassChangeError));
5149
5150 StubRoutines::_throw_NullPointerException_at_call_entry =
5151 generate_throw_exception("NullPointerException at call throw_exception",
5152 CAST_FROM_FN_PTR(address,
5153 SharedRuntime::
5154 throw_NullPointerException_at_call));
5155
5156 // entry points that are platform specific
5157 #if INCLUDE_SHENANDOAHGC
5158 if (UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier)) {
5159 StubRoutines::x86::_shenandoah_wb = generate_shenandoah_wb(false, true);
5160 StubRoutines::_shenandoah_wb_C = generate_shenandoah_wb(true, !ShenandoahWriteBarrierCsetTestInIR);
5161 }
5162 #endif
5163
5164 StubRoutines::x86::_f2i_fixup = generate_f2i_fixup();
5165 StubRoutines::x86::_f2l_fixup = generate_f2l_fixup();
5166 StubRoutines::x86::_d2i_fixup = generate_d2i_fixup();
5167 StubRoutines::x86::_d2l_fixup = generate_d2l_fixup();
5168
5169 StubRoutines::x86::_float_sign_mask = generate_fp_mask("float_sign_mask", 0x7FFFFFFF7FFFFFFF);
5170 StubRoutines::x86::_float_sign_flip = generate_fp_mask("float_sign_flip", 0x8000000080000000);
5171 StubRoutines::x86::_double_sign_mask = generate_fp_mask("double_sign_mask", 0x7FFFFFFFFFFFFFFF);
5172 StubRoutines::x86::_double_sign_flip = generate_fp_mask("double_sign_flip", 0x8000000000000000);
5173
5174 // support for verify_oop (must happen after universe_init)
5175 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
5176
5177 // arraycopy stubs used by compilers
5178 generate_arraycopy_stubs();
5179
5180 // don't bother generating these AES intrinsic stubs unless global flag is set
5181 if (UseAESIntrinsics) {
5182 StubRoutines::x86::_key_shuffle_mask_addr = generate_key_shuffle_mask(); // needed by the others
5183 StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
|
24
25 #include "precompiled.hpp"
26 #include "asm/macroAssembler.hpp"
27 #include "asm/macroAssembler.inline.hpp"
28 #include "ci/ciUtilities.hpp"
29 #include "gc/shared/barrierSet.hpp"
30 #include "gc/shared/barrierSetAssembler.hpp"
31 #include "interpreter/interpreter.hpp"
32 #include "nativeInst_x86.hpp"
33 #include "oops/instanceOop.hpp"
34 #include "oops/method.hpp"
35 #include "oops/objArrayKlass.hpp"
36 #include "oops/oop.inline.hpp"
37 #include "prims/methodHandles.hpp"
38 #include "runtime/frame.inline.hpp"
39 #include "runtime/handles.inline.hpp"
40 #include "runtime/sharedRuntime.hpp"
41 #include "runtime/stubCodeGenerator.hpp"
42 #include "runtime/stubRoutines.hpp"
43 #include "runtime/thread.inline.hpp"
44 #ifdef COMPILER2
45 #include "opto/runtime.hpp"
46 #endif
47 #if INCLUDE_ZGC
48 #include "gc/z/zThreadLocalData.hpp"
49 #endif
50
51 // Declaration and definition of StubGenerator (no .hpp file).
52 // For a more detailed description of the stub routine structure
53 // see the comment in stubRoutines.hpp
54
55 #define __ _masm->
56 #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8)
57 #define a__ ((Assembler*)_masm)->
58
59 #ifdef PRODUCT
60 #define BLOCK_COMMENT(str) /* nothing */
61 #else
62 #define BLOCK_COMMENT(str) __ block_comment(str)
63 #endif
64
65 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
66 const int MXCSR_MASK = 0xFFC0; // Mask out any pending exceptions
784 __ stmxcsr(mxcsr_save);
785 __ movl(rax, mxcsr_save);
786 __ andl(rax, MXCSR_MASK); // Only check control and mask bits
787 __ cmp32(rax, mxcsr_std);
788 __ jcc(Assembler::equal, ok_ret);
789
790 __ warn("MXCSR changed by native JNI code, use -XX:+RestoreMXCSROnJNICall");
791
792 __ ldmxcsr(mxcsr_std);
793
794 __ bind(ok_ret);
795 __ addptr(rsp, wordSize);
796 __ pop(rax);
797 }
798
799 __ ret(0);
800
801 return start;
802 }
803
804 address generate_f2i_fixup() {
805 StubCodeMark mark(this, "StubRoutines", "f2i_fixup");
806 Address inout(rsp, 5 * wordSize); // return address + 4 saves
807
808 address start = __ pc();
809
810 Label L;
811
812 __ push(rax);
813 __ push(c_rarg3);
814 __ push(c_rarg2);
815 __ push(c_rarg1);
816
817 __ movl(rax, 0x7f800000);
818 __ xorl(c_rarg3, c_rarg3);
819 __ movl(c_rarg2, inout);
820 __ movl(c_rarg1, c_rarg2);
821 __ andl(c_rarg1, 0x7fffffff);
822 __ cmpl(rax, c_rarg1); // NaN? -> 0
823 __ jcc(Assembler::negative, L);
5039 // fabricate a RuntimeStub internally.
5040 StubRoutines::_throw_AbstractMethodError_entry =
5041 generate_throw_exception("AbstractMethodError throw_exception",
5042 CAST_FROM_FN_PTR(address,
5043 SharedRuntime::
5044 throw_AbstractMethodError));
5045
5046 StubRoutines::_throw_IncompatibleClassChangeError_entry =
5047 generate_throw_exception("IncompatibleClassChangeError throw_exception",
5048 CAST_FROM_FN_PTR(address,
5049 SharedRuntime::
5050 throw_IncompatibleClassChangeError));
5051
5052 StubRoutines::_throw_NullPointerException_at_call_entry =
5053 generate_throw_exception("NullPointerException at call throw_exception",
5054 CAST_FROM_FN_PTR(address,
5055 SharedRuntime::
5056 throw_NullPointerException_at_call));
5057
5058 // entry points that are platform specific
5059 StubRoutines::x86::_f2i_fixup = generate_f2i_fixup();
5060 StubRoutines::x86::_f2l_fixup = generate_f2l_fixup();
5061 StubRoutines::x86::_d2i_fixup = generate_d2i_fixup();
5062 StubRoutines::x86::_d2l_fixup = generate_d2l_fixup();
5063
5064 StubRoutines::x86::_float_sign_mask = generate_fp_mask("float_sign_mask", 0x7FFFFFFF7FFFFFFF);
5065 StubRoutines::x86::_float_sign_flip = generate_fp_mask("float_sign_flip", 0x8000000080000000);
5066 StubRoutines::x86::_double_sign_mask = generate_fp_mask("double_sign_mask", 0x7FFFFFFFFFFFFFFF);
5067 StubRoutines::x86::_double_sign_flip = generate_fp_mask("double_sign_flip", 0x8000000000000000);
5068
5069 // support for verify_oop (must happen after universe_init)
5070 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
5071
5072 // arraycopy stubs used by compilers
5073 generate_arraycopy_stubs();
5074
5075 // don't bother generating these AES intrinsic stubs unless global flag is set
5076 if (UseAESIntrinsics) {
5077 StubRoutines::x86::_key_shuffle_mask_addr = generate_key_shuffle_mask(); // needed by the others
5078 StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
|