332 movl(as_Address(dst), src);
333 }
334
335 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
336 movl(dst, as_Address(src));
337 }
338
339 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
340 void MacroAssembler::movptr(Address dst, intptr_t src) {
341 movl(dst, src);
342 }
343
344
345 void MacroAssembler::pop_callee_saved_registers() {
346 pop(rcx);
347 pop(rdx);
348 pop(rdi);
349 pop(rsi);
350 }
351
352 void MacroAssembler::pop_fTOS() {
353 fld_d(Address(rsp, 0));
354 addl(rsp, 2 * wordSize);
355 }
356
357 void MacroAssembler::push_callee_saved_registers() {
358 push(rsi);
359 push(rdi);
360 push(rdx);
361 push(rcx);
362 }
363
364 void MacroAssembler::push_fTOS() {
365 subl(rsp, 2 * wordSize);
366 fstp_d(Address(rsp, 0));
367 }
368
369
370 void MacroAssembler::pushoop(jobject obj) {
371 push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate());
372 }
373
374 void MacroAssembler::pushklass(Metadata* obj) {
375 push_literal32((int32_t)obj, metadata_Relocation::spec_for_immediate());
376 }
377
378 void MacroAssembler::pushptr(AddressLiteral src) {
379 if (src.is_lval()) {
380 push_literal32((int32_t)src.target(), src.rspec());
381 } else {
382 pushl(as_Address(src));
383 }
384 }
385
386 void MacroAssembler::set_word_if_not_zero(Register dst) {
387 xorl(dst, dst);
388 set_byte_if_not_zero(dst);
389 }
2718 }
2719
2720 void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src) {
2721 if (reachable(src)) {
2722 Assembler::divsd(dst, as_Address(src));
2723 } else {
2724 lea(rscratch1, src);
2725 Assembler::divsd(dst, Address(rscratch1, 0));
2726 }
2727 }
2728
2729 void MacroAssembler::divss(XMMRegister dst, AddressLiteral src) {
2730 if (reachable(src)) {
2731 Assembler::divss(dst, as_Address(src));
2732 } else {
2733 lea(rscratch1, src);
2734 Assembler::divss(dst, Address(rscratch1, 0));
2735 }
2736 }
2737
2738 // !defined(COMPILER2) is because of stupid core builds
2739 #if !defined(_LP64) || defined(COMPILER1) || !defined(COMPILER2) || INCLUDE_JVMCI
2740 void MacroAssembler::empty_FPU_stack() {
2741 if (VM_Version::supports_mmx()) {
2742 emms();
2743 } else {
2744 for (int i = 8; i-- > 0; ) ffree(i);
2745 }
2746 }
2747 #endif // !LP64 || C1 || !C2 || INCLUDE_JVMCI
2748
2749
2750 void MacroAssembler::enter() {
2751 push(rbp);
2752 mov(rbp, rsp);
2753 }
2754
2755 // A 5 byte nop that is safe for patching (see patch_verified_entry)
2756 void MacroAssembler::fat_nop() {
2757 if (UseAddressNop) {
2758 addr_nop_5();
2759 } else {
2760 emit_int8(0x26); // es:
2761 emit_int8(0x2e); // cs:
2762 emit_int8(0x64); // fs:
2763 emit_int8(0x65); // gs:
2764 emit_int8((unsigned char)0x90);
2765 }
2766 }
2767
2768 void MacroAssembler::fcmp(Register tmp) {
2769 fcmp(tmp, 1, true, true);
2770 }
2771
2772 void MacroAssembler::fcmp(Register tmp, int index, bool pop_left, bool pop_right) {
2773 assert(!pop_right || pop_left, "usage error");
2774 if (VM_Version::supports_cmov()) {
2775 assert(tmp == noreg, "unneeded temp");
2776 if (pop_left) {
2777 fucomip(index);
2778 } else {
2779 fucomi(index);
2780 }
2781 if (pop_right) {
2782 fpop();
2783 }
2784 } else {
2785 assert(tmp != noreg, "need temp");
2786 if (pop_left) {
2787 if (pop_right) {
2829 }
2830 bind(L);
2831 }
2832
2833 void MacroAssembler::fld_d(AddressLiteral src) {
2834 fld_d(as_Address(src));
2835 }
2836
2837 void MacroAssembler::fld_s(AddressLiteral src) {
2838 fld_s(as_Address(src));
2839 }
2840
2841 void MacroAssembler::fld_x(AddressLiteral src) {
2842 Assembler::fld_x(as_Address(src));
2843 }
2844
2845 void MacroAssembler::fldcw(AddressLiteral src) {
2846 Assembler::fldcw(as_Address(src));
2847 }
2848
2849 void MacroAssembler::mulpd(XMMRegister dst, AddressLiteral src) {
2850 if (reachable(src)) {
2851 Assembler::mulpd(dst, as_Address(src));
2852 } else {
2853 lea(rscratch1, src);
2854 Assembler::mulpd(dst, Address(rscratch1, 0));
2855 }
2856 }
2857
2858 void MacroAssembler::increase_precision() {
2859 subptr(rsp, BytesPerWord);
2860 fnstcw(Address(rsp, 0));
2861 movl(rax, Address(rsp, 0));
2862 orl(rax, 0x300);
2863 push(rax);
2864 fldcw(Address(rsp, 0));
2865 pop(rax);
2866 }
2867
2868 void MacroAssembler::restore_precision() {
2869 fldcw(Address(rsp, 0));
2870 addptr(rsp, BytesPerWord);
2871 }
2872
2873 void MacroAssembler::fpop() {
2874 ffree();
2875 fincstp();
2876 }
2877
2878 void MacroAssembler::load_float(Address src) {
2879 if (UseSSE >= 1) {
2880 movflt(xmm0, src);
2881 } else {
2882 LP64_ONLY(ShouldNotReachHere());
2883 NOT_LP64(fld_s(src));
2884 }
2885 }
2886
2887 void MacroAssembler::store_float(Address dst) {
2888 if (UseSSE >= 1) {
2889 movflt(dst, xmm0);
2890 } else {
2891 LP64_ONLY(ShouldNotReachHere());
2892 NOT_LP64(fstp_s(dst));
2893 }
2894 }
2895
2896 void MacroAssembler::load_double(Address src) {
2897 if (UseSSE >= 2) {
2898 movdbl(xmm0, src);
2899 } else {
2900 LP64_ONLY(ShouldNotReachHere());
2901 NOT_LP64(fld_d(src));
2902 }
2903 }
2904
2905 void MacroAssembler::store_double(Address dst) {
2906 if (UseSSE >= 2) {
2907 movdbl(dst, xmm0);
2908 } else {
2909 LP64_ONLY(ShouldNotReachHere());
2910 NOT_LP64(fstp_d(dst));
2911 }
2912 }
2913
2914 void MacroAssembler::fremr(Register tmp) {
2915 save_rax(tmp);
2916 { Label L;
2917 bind(L);
2918 fprem();
2919 fwait(); fnstsw_ax();
2920 #ifdef _LP64
2921 testl(rax, 0x400);
2922 jcc(Assembler::notEqual, L);
2923 #else
2924 sahf();
2925 jcc(Assembler::parity, L);
2926 #endif // _LP64
2927 }
2928 restore_rax(tmp);
2929 // Result is in ST0.
2930 // Note: fxch & fpop to get rid of ST1
2931 // (otherwise FPU stack could overflow eventually)
2932 fxch(1);
2933 fpop();
2934 }
2935
2936 // dst = c = a * b + c
2937 void MacroAssembler::fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
2938 Assembler::vfmadd231sd(c, a, b);
2939 if (dst != c) {
2940 movdbl(dst, c);
2941 }
2942 }
2943
2944 // dst = c = a * b + c
2945 void MacroAssembler::fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
2946 Assembler::vfmadd231ss(c, a, b);
2947 if (dst != c) {
2948 movflt(dst, c);
2949 }
2950 }
2951
2952 // dst = c = a * b + c
2953 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
2954 Assembler::vfmadd231pd(c, a, b, vector_len);
2955 if (dst != c) {
5081 printf("--------------------------------------------------\n");
5082 }
5083
5084 };
5085
5086
5087 static void _print_CPU_state(CPU_State* state) {
5088 state->print();
5089 };
5090
5091
5092 void MacroAssembler::print_CPU_state() {
5093 push_CPU_state();
5094 push(rsp); // pass CPU state
5095 call(RuntimeAddress(CAST_FROM_FN_PTR(address, _print_CPU_state)));
5096 addptr(rsp, wordSize); // discard argument
5097 pop_CPU_state();
5098 }
5099
5100
5101 static bool _verify_FPU(int stack_depth, char* s, CPU_State* state) {
5102 static int counter = 0;
5103 FPU_State* fs = &state->_fpu_state;
5104 counter++;
5105 // For leaf calls, only verify that the top few elements remain empty.
5106 // We only need 1 empty at the top for C2 code.
5107 if( stack_depth < 0 ) {
5108 if( fs->tag_for_st(7) != 3 ) {
5109 printf("FPR7 not empty\n");
5110 state->print();
5111 assert(false, "error");
5112 return false;
5113 }
5114 return true; // All other stack states do not matter
5115 }
5116
5117 assert((fs->_control_word._value & 0xffff) == StubRoutines::_fpu_cntrl_wrd_std,
5118 "bad FPU control word");
5119
5120 // compute stack depth
5137 // too many elements on the stack
5138 printf("%s: <= %d stack elements expected but found %d\n", s, -stack_depth, d);
5139 state->print();
5140 assert(false, "error");
5141 return false;
5142 }
5143 } else {
5144 // expected stack depth is stack_depth
5145 if (d != stack_depth) {
5146 // wrong stack depth
5147 printf("%s: %d stack elements expected but found %d\n", s, stack_depth, d);
5148 state->print();
5149 assert(false, "error");
5150 return false;
5151 }
5152 }
5153 // everything is cool
5154 return true;
5155 }
5156
5157
5158 void MacroAssembler::verify_FPU(int stack_depth, const char* s) {
5159 if (!VerifyFPU) return;
5160 push_CPU_state();
5161 push(rsp); // pass CPU state
5162 ExternalAddress msg((address) s);
5163 // pass message string s
5164 pushptr(msg.addr());
5165 push(stack_depth); // pass stack depth
5166 call(RuntimeAddress(CAST_FROM_FN_PTR(address, _verify_FPU)));
5167 addptr(rsp, 3 * wordSize); // discard arguments
5168 // check for error
5169 { Label L;
5170 testl(rax, rax);
5171 jcc(Assembler::notZero, L);
5172 int3(); // break if error condition
5173 bind(L);
5174 }
5175 pop_CPU_state();
5176 }
5177
5178 void MacroAssembler::restore_cpu_control_state_after_jni() {
5179 // Either restore the MXCSR register after returning from the JNI Call
5180 // or verify that it wasn't changed (with -Xcheck:jni flag).
5181 if (VM_Version::supports_sse()) {
5182 if (RestoreMXCSROnJNICalls) {
5183 ldmxcsr(ExternalAddress(StubRoutines::addr_mxcsr_std()));
5184 } else if (CheckJNICalls) {
5185 call(RuntimeAddress(StubRoutines::x86::verify_mxcsr_entry()));
5186 }
5187 }
5188 // Clear upper bits of YMM registers to avoid SSE <-> AVX transition penalty.
5189 vzeroupper();
5190 // Reset k1 to 0xffff.
5191
5192 #ifdef COMPILER2
5193 if (PostLoopMultiversioning && VM_Version::supports_evex()) {
5194 push(rcx);
5195 movl(rcx, 0xffff);
5196 kmovwl(k1, rcx);
9871 bind(below_threshold);
9872 }
9873
9874 testl(len, len);
9875 jccb(Assembler::zero, done);
9876 lea(src, Address(src, len, Address::times_1));
9877 lea(dst, Address(dst, len, Address::times_2));
9878 negptr(len);
9879
9880 // inflate 1 char per iter
9881 bind(copy_chars_loop);
9882 load_unsigned_byte(tmp2, Address(src, len, Address::times_1)); // load byte char
9883 movw(Address(dst, len, Address::times_2), tmp2); // inflate byte char to word
9884 increment(len);
9885 jcc(Assembler::notZero, copy_chars_loop);
9886
9887 bind(done);
9888 }
9889
9890 #ifdef _LP64
9891 void MacroAssembler::cache_wb(Address line)
9892 {
9893 // 64 bit cpus always support clflush
9894 assert(VM_Version::supports_clflush(), "clflush should be available");
9895 bool optimized = VM_Version::supports_clflushopt();
9896 bool no_evict = VM_Version::supports_clwb();
9897
9898 // prefer clwb (writeback without evict) otherwise
9899 // prefer clflushopt (potentially parallel writeback with evict)
9900 // otherwise fallback on clflush (serial writeback with evict)
9901
9902 if (optimized) {
9903 if (no_evict) {
9904 clwb(line);
9905 } else {
9906 clflushopt(line);
9907 }
9908 } else {
9909 // no need for fence when using CLFLUSH
9910 clflush(line);
9983 #endif
9984
9985 MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0);
9986
9987 #ifdef _LP64
9988 pop(r11);
9989 pop(r10);
9990 pop(r9);
9991 pop(r8);
9992 #endif
9993 pop(rcx);
9994 pop(rdx);
9995 LP64_ONLY(pop(rsi);)
9996 LP64_ONLY(pop(rdi);)
9997 if (thread != rax) {
9998 mov(thread, rax);
9999 pop(rax);
10000 }
10001 }
10002
10003 #endif
|
332 movl(as_Address(dst), src);
333 }
334
335 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
336 movl(dst, as_Address(src));
337 }
338
339 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
340 void MacroAssembler::movptr(Address dst, intptr_t src) {
341 movl(dst, src);
342 }
343
344
345 void MacroAssembler::pop_callee_saved_registers() {
346 pop(rcx);
347 pop(rdx);
348 pop(rdi);
349 pop(rsi);
350 }
351
352 void MacroAssembler::push_callee_saved_registers() {
353 push(rsi);
354 push(rdi);
355 push(rdx);
356 push(rcx);
357 }
358
359 void MacroAssembler::pushoop(jobject obj) {
360 push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate());
361 }
362
363 void MacroAssembler::pushklass(Metadata* obj) {
364 push_literal32((int32_t)obj, metadata_Relocation::spec_for_immediate());
365 }
366
367 void MacroAssembler::pushptr(AddressLiteral src) {
368 if (src.is_lval()) {
369 push_literal32((int32_t)src.target(), src.rspec());
370 } else {
371 pushl(as_Address(src));
372 }
373 }
374
375 void MacroAssembler::set_word_if_not_zero(Register dst) {
376 xorl(dst, dst);
377 set_byte_if_not_zero(dst);
378 }
2707 }
2708
2709 void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src) {
2710 if (reachable(src)) {
2711 Assembler::divsd(dst, as_Address(src));
2712 } else {
2713 lea(rscratch1, src);
2714 Assembler::divsd(dst, Address(rscratch1, 0));
2715 }
2716 }
2717
2718 void MacroAssembler::divss(XMMRegister dst, AddressLiteral src) {
2719 if (reachable(src)) {
2720 Assembler::divss(dst, as_Address(src));
2721 } else {
2722 lea(rscratch1, src);
2723 Assembler::divss(dst, Address(rscratch1, 0));
2724 }
2725 }
2726
2727 #ifndef _LP64
2728 void MacroAssembler::empty_FPU_stack() {
2729 if (VM_Version::supports_mmx()) {
2730 emms();
2731 } else {
2732 for (int i = 8; i-- > 0; ) ffree(i);
2733 }
2734 }
2735 #endif // !LP64
2736
2737
2738 void MacroAssembler::enter() {
2739 push(rbp);
2740 mov(rbp, rsp);
2741 }
2742
2743 // A 5 byte nop that is safe for patching (see patch_verified_entry)
2744 void MacroAssembler::fat_nop() {
2745 if (UseAddressNop) {
2746 addr_nop_5();
2747 } else {
2748 emit_int8(0x26); // es:
2749 emit_int8(0x2e); // cs:
2750 emit_int8(0x64); // fs:
2751 emit_int8(0x65); // gs:
2752 emit_int8((unsigned char)0x90);
2753 }
2754 }
2755
2756 #if !defined(_LP64)
2757 void MacroAssembler::fcmp(Register tmp) {
2758 fcmp(tmp, 1, true, true);
2759 }
2760
2761 void MacroAssembler::fcmp(Register tmp, int index, bool pop_left, bool pop_right) {
2762 assert(!pop_right || pop_left, "usage error");
2763 if (VM_Version::supports_cmov()) {
2764 assert(tmp == noreg, "unneeded temp");
2765 if (pop_left) {
2766 fucomip(index);
2767 } else {
2768 fucomi(index);
2769 }
2770 if (pop_right) {
2771 fpop();
2772 }
2773 } else {
2774 assert(tmp != noreg, "need temp");
2775 if (pop_left) {
2776 if (pop_right) {
2818 }
2819 bind(L);
2820 }
2821
2822 void MacroAssembler::fld_d(AddressLiteral src) {
2823 fld_d(as_Address(src));
2824 }
2825
2826 void MacroAssembler::fld_s(AddressLiteral src) {
2827 fld_s(as_Address(src));
2828 }
2829
2830 void MacroAssembler::fld_x(AddressLiteral src) {
2831 Assembler::fld_x(as_Address(src));
2832 }
2833
2834 void MacroAssembler::fldcw(AddressLiteral src) {
2835 Assembler::fldcw(as_Address(src));
2836 }
2837
2838 void MacroAssembler::fpop() {
2839 ffree();
2840 fincstp();
2841 }
2842
2843 void MacroAssembler::fremr(Register tmp) {
2844 save_rax(tmp);
2845 { Label L;
2846 bind(L);
2847 fprem();
2848 fwait(); fnstsw_ax();
2849 sahf();
2850 jcc(Assembler::parity, L);
2851 }
2852 restore_rax(tmp);
2853 // Result is in ST0.
2854 // Note: fxch & fpop to get rid of ST1
2855 // (otherwise FPU stack could overflow eventually)
2856 fxch(1);
2857 fpop();
2858 }
2859 #endif // !LP64
2860
2861 void MacroAssembler::mulpd(XMMRegister dst, AddressLiteral src) {
2862 if (reachable(src)) {
2863 Assembler::mulpd(dst, as_Address(src));
2864 } else {
2865 lea(rscratch1, src);
2866 Assembler::mulpd(dst, Address(rscratch1, 0));
2867 }
2868 }
2869
2870 void MacroAssembler::load_float(Address src) {
2871 if (UseSSE >= 1) {
2872 movflt(xmm0, src);
2873 } else {
2874 LP64_ONLY(ShouldNotReachHere());
2875 NOT_LP64(fld_s(src));
2876 }
2877 }
2878
2879 void MacroAssembler::store_float(Address dst) {
2880 if (UseSSE >= 1) {
2881 movflt(dst, xmm0);
2882 } else {
2883 LP64_ONLY(ShouldNotReachHere());
2884 NOT_LP64(fstp_s(dst));
2885 }
2886 }
2887
2888 void MacroAssembler::load_double(Address src) {
2889 if (UseSSE >= 2) {
2890 movdbl(xmm0, src);
2891 } else {
2892 LP64_ONLY(ShouldNotReachHere());
2893 NOT_LP64(fld_d(src));
2894 }
2895 }
2896
2897 void MacroAssembler::store_double(Address dst) {
2898 if (UseSSE >= 2) {
2899 movdbl(dst, xmm0);
2900 } else {
2901 LP64_ONLY(ShouldNotReachHere());
2902 NOT_LP64(fstp_d(dst));
2903 }
2904 }
2905
2906 // dst = c = a * b + c
2907 void MacroAssembler::fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
2908 Assembler::vfmadd231sd(c, a, b);
2909 if (dst != c) {
2910 movdbl(dst, c);
2911 }
2912 }
2913
2914 // dst = c = a * b + c
2915 void MacroAssembler::fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
2916 Assembler::vfmadd231ss(c, a, b);
2917 if (dst != c) {
2918 movflt(dst, c);
2919 }
2920 }
2921
2922 // dst = c = a * b + c
2923 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
2924 Assembler::vfmadd231pd(c, a, b, vector_len);
2925 if (dst != c) {
5051 printf("--------------------------------------------------\n");
5052 }
5053
5054 };
5055
5056
5057 static void _print_CPU_state(CPU_State* state) {
5058 state->print();
5059 };
5060
5061
5062 void MacroAssembler::print_CPU_state() {
5063 push_CPU_state();
5064 push(rsp); // pass CPU state
5065 call(RuntimeAddress(CAST_FROM_FN_PTR(address, _print_CPU_state)));
5066 addptr(rsp, wordSize); // discard argument
5067 pop_CPU_state();
5068 }
5069
5070
5071 #ifndef _LP64
5072 static bool _verify_FPU(int stack_depth, char* s, CPU_State* state) {
5073 static int counter = 0;
5074 FPU_State* fs = &state->_fpu_state;
5075 counter++;
5076 // For leaf calls, only verify that the top few elements remain empty.
5077 // We only need 1 empty at the top for C2 code.
5078 if( stack_depth < 0 ) {
5079 if( fs->tag_for_st(7) != 3 ) {
5080 printf("FPR7 not empty\n");
5081 state->print();
5082 assert(false, "error");
5083 return false;
5084 }
5085 return true; // All other stack states do not matter
5086 }
5087
5088 assert((fs->_control_word._value & 0xffff) == StubRoutines::_fpu_cntrl_wrd_std,
5089 "bad FPU control word");
5090
5091 // compute stack depth
5108 // too many elements on the stack
5109 printf("%s: <= %d stack elements expected but found %d\n", s, -stack_depth, d);
5110 state->print();
5111 assert(false, "error");
5112 return false;
5113 }
5114 } else {
5115 // expected stack depth is stack_depth
5116 if (d != stack_depth) {
5117 // wrong stack depth
5118 printf("%s: %d stack elements expected but found %d\n", s, stack_depth, d);
5119 state->print();
5120 assert(false, "error");
5121 return false;
5122 }
5123 }
5124 // everything is cool
5125 return true;
5126 }
5127
5128 void MacroAssembler::verify_FPU(int stack_depth, const char* s) {
5129 if (!VerifyFPU) return;
5130 push_CPU_state();
5131 push(rsp); // pass CPU state
5132 ExternalAddress msg((address) s);
5133 // pass message string s
5134 pushptr(msg.addr());
5135 push(stack_depth); // pass stack depth
5136 call(RuntimeAddress(CAST_FROM_FN_PTR(address, _verify_FPU)));
5137 addptr(rsp, 3 * wordSize); // discard arguments
5138 // check for error
5139 { Label L;
5140 testl(rax, rax);
5141 jcc(Assembler::notZero, L);
5142 int3(); // break if error condition
5143 bind(L);
5144 }
5145 pop_CPU_state();
5146 }
5147 #endif // _LP64
5148
5149 void MacroAssembler::restore_cpu_control_state_after_jni() {
5150 // Either restore the MXCSR register after returning from the JNI Call
5151 // or verify that it wasn't changed (with -Xcheck:jni flag).
5152 if (VM_Version::supports_sse()) {
5153 if (RestoreMXCSROnJNICalls) {
5154 ldmxcsr(ExternalAddress(StubRoutines::addr_mxcsr_std()));
5155 } else if (CheckJNICalls) {
5156 call(RuntimeAddress(StubRoutines::x86::verify_mxcsr_entry()));
5157 }
5158 }
5159 // Clear upper bits of YMM registers to avoid SSE <-> AVX transition penalty.
5160 vzeroupper();
5161 // Reset k1 to 0xffff.
5162
5163 #ifdef COMPILER2
5164 if (PostLoopMultiversioning && VM_Version::supports_evex()) {
5165 push(rcx);
5166 movl(rcx, 0xffff);
5167 kmovwl(k1, rcx);
9842 bind(below_threshold);
9843 }
9844
9845 testl(len, len);
9846 jccb(Assembler::zero, done);
9847 lea(src, Address(src, len, Address::times_1));
9848 lea(dst, Address(dst, len, Address::times_2));
9849 negptr(len);
9850
9851 // inflate 1 char per iter
9852 bind(copy_chars_loop);
9853 load_unsigned_byte(tmp2, Address(src, len, Address::times_1)); // load byte char
9854 movw(Address(dst, len, Address::times_2), tmp2); // inflate byte char to word
9855 increment(len);
9856 jcc(Assembler::notZero, copy_chars_loop);
9857
9858 bind(done);
9859 }
9860
9861 #ifdef _LP64
9862 void MacroAssembler::convert_f2i(Register dst, XMMRegister src) {
9863 Label done;
9864 cvttss2sil(dst, src);
9865 // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
9866 cmpl(dst, 0x80000000); // float_sign_flip
9867 jccb(Assembler::notEqual, done);
9868 subptr(rsp, 8);
9869 movflt(Address(rsp, 0), src);
9870 call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
9871 pop(dst);
9872 bind(done);
9873 }
9874
9875 void MacroAssembler::convert_d2i(Register dst, XMMRegister src) {
9876 Label done;
9877 cvttsd2sil(dst, src);
9878 // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
9879 cmpl(dst, 0x80000000); // float_sign_flip
9880 jccb(Assembler::notEqual, done);
9881 subptr(rsp, 8);
9882 movdbl(Address(rsp, 0), src);
9883 call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
9884 pop(dst);
9885 bind(done);
9886 }
9887
9888 void MacroAssembler::convert_f2l(Register dst, XMMRegister src) {
9889 Label done;
9890 cvttss2siq(dst, src);
9891 cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
9892 jccb(Assembler::notEqual, done);
9893 subptr(rsp, 8);
9894 movflt(Address(rsp, 0), src);
9895 call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
9896 pop(dst);
9897 bind(done);
9898 }
9899
9900 void MacroAssembler::convert_d2l(Register dst, XMMRegister src) {
9901 Label done;
9902 cvttsd2siq(dst, src);
9903 cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
9904 jccb(Assembler::notEqual, done);
9905 subptr(rsp, 8);
9906 movdbl(Address(rsp, 0), src);
9907 call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
9908 pop(dst);
9909 bind(done);
9910 }
9911
9912 void MacroAssembler::cache_wb(Address line)
9913 {
9914 // 64 bit cpus always support clflush
9915 assert(VM_Version::supports_clflush(), "clflush should be available");
9916 bool optimized = VM_Version::supports_clflushopt();
9917 bool no_evict = VM_Version::supports_clwb();
9918
9919 // prefer clwb (writeback without evict) otherwise
9920 // prefer clflushopt (potentially parallel writeback with evict)
9921 // otherwise fallback on clflush (serial writeback with evict)
9922
9923 if (optimized) {
9924 if (no_evict) {
9925 clwb(line);
9926 } else {
9927 clflushopt(line);
9928 }
9929 } else {
9930 // no need for fence when using CLFLUSH
9931 clflush(line);
10004 #endif
10005
10006 MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0);
10007
10008 #ifdef _LP64
10009 pop(r11);
10010 pop(r10);
10011 pop(r9);
10012 pop(r8);
10013 #endif
10014 pop(rcx);
10015 pop(rdx);
10016 LP64_ONLY(pop(rsi);)
10017 LP64_ONLY(pop(rdi);)
10018 if (thread != rax) {
10019 mov(thread, rax);
10020 pop(rax);
10021 }
10022 }
10023
10024 #endif // !WIN32 || _LP64
|