628 case vmIntrinsics::_getBooleanOpaque: return inline_unsafe_access(!is_store, T_BOOLEAN, Opaque, false);
629 case vmIntrinsics::_getByteOpaque: return inline_unsafe_access(!is_store, T_BYTE, Opaque, false);
630 case vmIntrinsics::_getShortOpaque: return inline_unsafe_access(!is_store, T_SHORT, Opaque, false);
631 case vmIntrinsics::_getCharOpaque: return inline_unsafe_access(!is_store, T_CHAR, Opaque, false);
632 case vmIntrinsics::_getIntOpaque: return inline_unsafe_access(!is_store, T_INT, Opaque, false);
633 case vmIntrinsics::_getLongOpaque: return inline_unsafe_access(!is_store, T_LONG, Opaque, false);
634 case vmIntrinsics::_getFloatOpaque: return inline_unsafe_access(!is_store, T_FLOAT, Opaque, false);
635 case vmIntrinsics::_getDoubleOpaque: return inline_unsafe_access(!is_store, T_DOUBLE, Opaque, false);
636
637 case vmIntrinsics::_putObjectOpaque: return inline_unsafe_access( is_store, T_OBJECT, Opaque, false);
638 case vmIntrinsics::_putBooleanOpaque: return inline_unsafe_access( is_store, T_BOOLEAN, Opaque, false);
639 case vmIntrinsics::_putByteOpaque: return inline_unsafe_access( is_store, T_BYTE, Opaque, false);
640 case vmIntrinsics::_putShortOpaque: return inline_unsafe_access( is_store, T_SHORT, Opaque, false);
641 case vmIntrinsics::_putCharOpaque: return inline_unsafe_access( is_store, T_CHAR, Opaque, false);
642 case vmIntrinsics::_putIntOpaque: return inline_unsafe_access( is_store, T_INT, Opaque, false);
643 case vmIntrinsics::_putLongOpaque: return inline_unsafe_access( is_store, T_LONG, Opaque, false);
644 case vmIntrinsics::_putFloatOpaque: return inline_unsafe_access( is_store, T_FLOAT, Opaque, false);
645 case vmIntrinsics::_putDoubleOpaque: return inline_unsafe_access( is_store, T_DOUBLE, Opaque, false);
646
647 case vmIntrinsics::_compareAndSwapObject: return inline_unsafe_load_store(T_OBJECT, LS_cmp_swap, Volatile);
648 case vmIntrinsics::_compareAndSwapInt: return inline_unsafe_load_store(T_INT, LS_cmp_swap, Volatile);
649 case vmIntrinsics::_compareAndSwapLong: return inline_unsafe_load_store(T_LONG, LS_cmp_swap, Volatile);
650
651 case vmIntrinsics::_weakCompareAndSwapObject: return inline_unsafe_load_store(T_OBJECT, LS_cmp_swap_weak, Relaxed);
652 case vmIntrinsics::_weakCompareAndSwapObjectAcquire: return inline_unsafe_load_store(T_OBJECT, LS_cmp_swap_weak, Acquire);
653 case vmIntrinsics::_weakCompareAndSwapObjectRelease: return inline_unsafe_load_store(T_OBJECT, LS_cmp_swap_weak, Release);
654 case vmIntrinsics::_weakCompareAndSwapObjectVolatile: return inline_unsafe_load_store(T_OBJECT, LS_cmp_swap_weak, Volatile);
655 case vmIntrinsics::_weakCompareAndSwapInt: return inline_unsafe_load_store(T_INT, LS_cmp_swap_weak, Relaxed);
656 case vmIntrinsics::_weakCompareAndSwapIntAcquire: return inline_unsafe_load_store(T_INT, LS_cmp_swap_weak, Acquire);
657 case vmIntrinsics::_weakCompareAndSwapIntRelease: return inline_unsafe_load_store(T_INT, LS_cmp_swap_weak, Release);
658 case vmIntrinsics::_weakCompareAndSwapIntVolatile: return inline_unsafe_load_store(T_INT, LS_cmp_swap_weak, Volatile);
659 case vmIntrinsics::_weakCompareAndSwapLong: return inline_unsafe_load_store(T_LONG, LS_cmp_swap_weak, Relaxed);
660 case vmIntrinsics::_weakCompareAndSwapLongAcquire: return inline_unsafe_load_store(T_LONG, LS_cmp_swap_weak, Acquire);
661 case vmIntrinsics::_weakCompareAndSwapLongRelease: return inline_unsafe_load_store(T_LONG, LS_cmp_swap_weak, Release);
662 case vmIntrinsics::_weakCompareAndSwapLongVolatile: return inline_unsafe_load_store(T_LONG, LS_cmp_swap_weak, Volatile);
663
664 case vmIntrinsics::_compareAndExchangeObjectVolatile: return inline_unsafe_load_store(T_OBJECT, LS_cmp_exchange, Volatile);
665 case vmIntrinsics::_compareAndExchangeObjectAcquire: return inline_unsafe_load_store(T_OBJECT, LS_cmp_exchange, Acquire);
666 case vmIntrinsics::_compareAndExchangeObjectRelease: return inline_unsafe_load_store(T_OBJECT, LS_cmp_exchange, Release);
667 case vmIntrinsics::_compareAndExchangeIntVolatile: return inline_unsafe_load_store(T_INT, LS_cmp_exchange, Volatile);
668 case vmIntrinsics::_compareAndExchangeIntAcquire: return inline_unsafe_load_store(T_INT, LS_cmp_exchange, Acquire);
669 case vmIntrinsics::_compareAndExchangeIntRelease: return inline_unsafe_load_store(T_INT, LS_cmp_exchange, Release);
670 case vmIntrinsics::_compareAndExchangeLongVolatile: return inline_unsafe_load_store(T_LONG, LS_cmp_exchange, Volatile);
671 case vmIntrinsics::_compareAndExchangeLongAcquire: return inline_unsafe_load_store(T_LONG, LS_cmp_exchange, Acquire);
672 case vmIntrinsics::_compareAndExchangeLongRelease: return inline_unsafe_load_store(T_LONG, LS_cmp_exchange, Release);
673
674 case vmIntrinsics::_getAndAddInt: return inline_unsafe_load_store(T_INT, LS_get_add, Volatile);
675 case vmIntrinsics::_getAndAddLong: return inline_unsafe_load_store(T_LONG, LS_get_add, Volatile);
676 case vmIntrinsics::_getAndSetInt: return inline_unsafe_load_store(T_INT, LS_get_set, Volatile);
677 case vmIntrinsics::_getAndSetLong: return inline_unsafe_load_store(T_LONG, LS_get_set, Volatile);
678 case vmIntrinsics::_getAndSetObject: return inline_unsafe_load_store(T_OBJECT, LS_get_set, Volatile);
679
680 case vmIntrinsics::_loadFence:
681 case vmIntrinsics::_storeFence:
682 case vmIntrinsics::_fullFence: return inline_unsafe_fence(intrinsic_id());
683
684 case vmIntrinsics::_onSpinWait: return inline_onspinwait();
685
686 case vmIntrinsics::_currentThread: return inline_native_currentThread();
687 case vmIntrinsics::_isInterrupted: return inline_native_isInterrupted();
688
689 #ifdef TRACE_HAVE_INTRINSICS
690 case vmIntrinsics::_counterTime: return inline_native_time_funcs(CAST_FROM_FN_PTR(address, TRACE_TIME_METHOD), "counterTime");
691 #endif
692 case vmIntrinsics::_currentTimeMillis: return inline_native_time_funcs(CAST_FROM_FN_PTR(address, os::javaTimeMillis), "currentTimeMillis");
693 case vmIntrinsics::_nanoTime: return inline_native_time_funcs(CAST_FROM_FN_PTR(address, os::javaTimeNanos), "nanoTime");
694 case vmIntrinsics::_allocateInstance: return inline_unsafe_allocate();
695 case vmIntrinsics::_copyMemory: return inline_unsafe_copyMemory();
2832 insert_mem_bar(Op_MemBarRelease);
2833 }
2834 break;
2835 default:
2836 ShouldNotReachHere();
2837 }
2838 insert_mem_bar(Op_MemBarCPUOrder);
2839
2840 // Figure out the memory ordering.
2841 MemNode::MemOrd mo = access_kind_to_memord(access_kind);
2842
2843 // 4984716: MemBars must be inserted before this
2844 // memory node in order to avoid a false
2845 // dependency which will confuse the scheduler.
2846 Node *mem = memory(alias_idx);
2847
2848 // For now, we handle only those cases that actually exist: ints,
2849 // longs, and Object. Adding others should be straightforward.
2850 Node* load_store = NULL;
2851 switch(type) {
2852 case T_INT:
2853 switch(kind) {
2854 case LS_get_add:
2855 load_store = _gvn.transform(new GetAndAddINode(control(), mem, adr, newval, adr_type));
2856 break;
2857 case LS_get_set:
2858 load_store = _gvn.transform(new GetAndSetINode(control(), mem, adr, newval, adr_type));
2859 break;
2860 case LS_cmp_swap_weak:
2861 load_store = _gvn.transform(new WeakCompareAndSwapINode(control(), mem, adr, newval, oldval, mo));
2862 break;
2863 case LS_cmp_swap:
2864 load_store = _gvn.transform(new CompareAndSwapINode(control(), mem, adr, newval, oldval, mo));
2865 break;
2866 case LS_cmp_exchange:
2867 load_store = _gvn.transform(new CompareAndExchangeINode(control(), mem, adr, newval, oldval, adr_type, mo));
2868 break;
2869 default:
2870 ShouldNotReachHere();
2871 }
|
628 case vmIntrinsics::_getBooleanOpaque: return inline_unsafe_access(!is_store, T_BOOLEAN, Opaque, false);
629 case vmIntrinsics::_getByteOpaque: return inline_unsafe_access(!is_store, T_BYTE, Opaque, false);
630 case vmIntrinsics::_getShortOpaque: return inline_unsafe_access(!is_store, T_SHORT, Opaque, false);
631 case vmIntrinsics::_getCharOpaque: return inline_unsafe_access(!is_store, T_CHAR, Opaque, false);
632 case vmIntrinsics::_getIntOpaque: return inline_unsafe_access(!is_store, T_INT, Opaque, false);
633 case vmIntrinsics::_getLongOpaque: return inline_unsafe_access(!is_store, T_LONG, Opaque, false);
634 case vmIntrinsics::_getFloatOpaque: return inline_unsafe_access(!is_store, T_FLOAT, Opaque, false);
635 case vmIntrinsics::_getDoubleOpaque: return inline_unsafe_access(!is_store, T_DOUBLE, Opaque, false);
636
637 case vmIntrinsics::_putObjectOpaque: return inline_unsafe_access( is_store, T_OBJECT, Opaque, false);
638 case vmIntrinsics::_putBooleanOpaque: return inline_unsafe_access( is_store, T_BOOLEAN, Opaque, false);
639 case vmIntrinsics::_putByteOpaque: return inline_unsafe_access( is_store, T_BYTE, Opaque, false);
640 case vmIntrinsics::_putShortOpaque: return inline_unsafe_access( is_store, T_SHORT, Opaque, false);
641 case vmIntrinsics::_putCharOpaque: return inline_unsafe_access( is_store, T_CHAR, Opaque, false);
642 case vmIntrinsics::_putIntOpaque: return inline_unsafe_access( is_store, T_INT, Opaque, false);
643 case vmIntrinsics::_putLongOpaque: return inline_unsafe_access( is_store, T_LONG, Opaque, false);
644 case vmIntrinsics::_putFloatOpaque: return inline_unsafe_access( is_store, T_FLOAT, Opaque, false);
645 case vmIntrinsics::_putDoubleOpaque: return inline_unsafe_access( is_store, T_DOUBLE, Opaque, false);
646
647 case vmIntrinsics::_compareAndSwapObject: return inline_unsafe_load_store(T_OBJECT, LS_cmp_swap, Volatile);
648 case vmIntrinsics::_compareAndSwapByte: return inline_unsafe_load_store(T_BYTE, LS_cmp_swap, Volatile);
649 case vmIntrinsics::_compareAndSwapShort: return inline_unsafe_load_store(T_SHORT, LS_cmp_swap, Volatile);
650 case vmIntrinsics::_compareAndSwapInt: return inline_unsafe_load_store(T_INT, LS_cmp_swap, Volatile);
651 case vmIntrinsics::_compareAndSwapLong: return inline_unsafe_load_store(T_LONG, LS_cmp_swap, Volatile);
652
653 case vmIntrinsics::_weakCompareAndSwapObject: return inline_unsafe_load_store(T_OBJECT, LS_cmp_swap_weak, Relaxed);
654 case vmIntrinsics::_weakCompareAndSwapObjectAcquire: return inline_unsafe_load_store(T_OBJECT, LS_cmp_swap_weak, Acquire);
655 case vmIntrinsics::_weakCompareAndSwapObjectRelease: return inline_unsafe_load_store(T_OBJECT, LS_cmp_swap_weak, Release);
656 case vmIntrinsics::_weakCompareAndSwapObjectVolatile: return inline_unsafe_load_store(T_OBJECT, LS_cmp_swap_weak, Volatile);
657 case vmIntrinsics::_weakCompareAndSwapByte: return inline_unsafe_load_store(T_BYTE, LS_cmp_swap_weak, Relaxed);
658 case vmIntrinsics::_weakCompareAndSwapByteAcquire: return inline_unsafe_load_store(T_BYTE, LS_cmp_swap_weak, Acquire);
659 case vmIntrinsics::_weakCompareAndSwapByteRelease: return inline_unsafe_load_store(T_BYTE, LS_cmp_swap_weak, Release);
660 case vmIntrinsics::_weakCompareAndSwapByteVolatile: return inline_unsafe_load_store(T_BYTE, LS_cmp_swap_weak, Volatile);
661 case vmIntrinsics::_weakCompareAndSwapShort: return inline_unsafe_load_store(T_SHORT, LS_cmp_swap_weak, Relaxed);
662 case vmIntrinsics::_weakCompareAndSwapShortAcquire: return inline_unsafe_load_store(T_SHORT, LS_cmp_swap_weak, Acquire);
663 case vmIntrinsics::_weakCompareAndSwapShortRelease: return inline_unsafe_load_store(T_SHORT, LS_cmp_swap_weak, Release);
664 case vmIntrinsics::_weakCompareAndSwapShortVolatile: return inline_unsafe_load_store(T_SHORT, LS_cmp_swap_weak, Volatile);
665 case vmIntrinsics::_weakCompareAndSwapInt: return inline_unsafe_load_store(T_INT, LS_cmp_swap_weak, Relaxed);
666 case vmIntrinsics::_weakCompareAndSwapIntAcquire: return inline_unsafe_load_store(T_INT, LS_cmp_swap_weak, Acquire);
667 case vmIntrinsics::_weakCompareAndSwapIntRelease: return inline_unsafe_load_store(T_INT, LS_cmp_swap_weak, Release);
668 case vmIntrinsics::_weakCompareAndSwapIntVolatile: return inline_unsafe_load_store(T_INT, LS_cmp_swap_weak, Volatile);
669 case vmIntrinsics::_weakCompareAndSwapLong: return inline_unsafe_load_store(T_LONG, LS_cmp_swap_weak, Relaxed);
670 case vmIntrinsics::_weakCompareAndSwapLongAcquire: return inline_unsafe_load_store(T_LONG, LS_cmp_swap_weak, Acquire);
671 case vmIntrinsics::_weakCompareAndSwapLongRelease: return inline_unsafe_load_store(T_LONG, LS_cmp_swap_weak, Release);
672 case vmIntrinsics::_weakCompareAndSwapLongVolatile: return inline_unsafe_load_store(T_LONG, LS_cmp_swap_weak, Volatile);
673
674 case vmIntrinsics::_compareAndExchangeObjectVolatile: return inline_unsafe_load_store(T_OBJECT, LS_cmp_exchange, Volatile);
675 case vmIntrinsics::_compareAndExchangeObjectAcquire: return inline_unsafe_load_store(T_OBJECT, LS_cmp_exchange, Acquire);
676 case vmIntrinsics::_compareAndExchangeObjectRelease: return inline_unsafe_load_store(T_OBJECT, LS_cmp_exchange, Release);
677 case vmIntrinsics::_compareAndExchangeByteVolatile: return inline_unsafe_load_store(T_BYTE, LS_cmp_exchange, Volatile);
678 case vmIntrinsics::_compareAndExchangeByteAcquire: return inline_unsafe_load_store(T_BYTE, LS_cmp_exchange, Acquire);
679 case vmIntrinsics::_compareAndExchangeByteRelease: return inline_unsafe_load_store(T_BYTE, LS_cmp_exchange, Release);
680 case vmIntrinsics::_compareAndExchangeShortVolatile: return inline_unsafe_load_store(T_SHORT, LS_cmp_exchange, Volatile);
681 case vmIntrinsics::_compareAndExchangeShortAcquire: return inline_unsafe_load_store(T_SHORT, LS_cmp_exchange, Acquire);
682 case vmIntrinsics::_compareAndExchangeShortRelease: return inline_unsafe_load_store(T_SHORT, LS_cmp_exchange, Release);
683 case vmIntrinsics::_compareAndExchangeIntVolatile: return inline_unsafe_load_store(T_INT, LS_cmp_exchange, Volatile);
684 case vmIntrinsics::_compareAndExchangeIntAcquire: return inline_unsafe_load_store(T_INT, LS_cmp_exchange, Acquire);
685 case vmIntrinsics::_compareAndExchangeIntRelease: return inline_unsafe_load_store(T_INT, LS_cmp_exchange, Release);
686 case vmIntrinsics::_compareAndExchangeLongVolatile: return inline_unsafe_load_store(T_LONG, LS_cmp_exchange, Volatile);
687 case vmIntrinsics::_compareAndExchangeLongAcquire: return inline_unsafe_load_store(T_LONG, LS_cmp_exchange, Acquire);
688 case vmIntrinsics::_compareAndExchangeLongRelease: return inline_unsafe_load_store(T_LONG, LS_cmp_exchange, Release);
689
690 case vmIntrinsics::_getAndAddByte: return inline_unsafe_load_store(T_BYTE, LS_get_add, Volatile);
691 case vmIntrinsics::_getAndAddShort: return inline_unsafe_load_store(T_SHORT, LS_get_add, Volatile);
692 case vmIntrinsics::_getAndAddInt: return inline_unsafe_load_store(T_INT, LS_get_add, Volatile);
693 case vmIntrinsics::_getAndAddLong: return inline_unsafe_load_store(T_LONG, LS_get_add, Volatile);
694
695 case vmIntrinsics::_getAndSetByte: return inline_unsafe_load_store(T_BYTE, LS_get_set, Volatile);
696 case vmIntrinsics::_getAndSetShort: return inline_unsafe_load_store(T_SHORT, LS_get_set, Volatile);
697 case vmIntrinsics::_getAndSetInt: return inline_unsafe_load_store(T_INT, LS_get_set, Volatile);
698 case vmIntrinsics::_getAndSetLong: return inline_unsafe_load_store(T_LONG, LS_get_set, Volatile);
699 case vmIntrinsics::_getAndSetObject: return inline_unsafe_load_store(T_OBJECT, LS_get_set, Volatile);
700
701 case vmIntrinsics::_loadFence:
702 case vmIntrinsics::_storeFence:
703 case vmIntrinsics::_fullFence: return inline_unsafe_fence(intrinsic_id());
704
705 case vmIntrinsics::_onSpinWait: return inline_onspinwait();
706
707 case vmIntrinsics::_currentThread: return inline_native_currentThread();
708 case vmIntrinsics::_isInterrupted: return inline_native_isInterrupted();
709
710 #ifdef TRACE_HAVE_INTRINSICS
711 case vmIntrinsics::_counterTime: return inline_native_time_funcs(CAST_FROM_FN_PTR(address, TRACE_TIME_METHOD), "counterTime");
712 #endif
713 case vmIntrinsics::_currentTimeMillis: return inline_native_time_funcs(CAST_FROM_FN_PTR(address, os::javaTimeMillis), "currentTimeMillis");
714 case vmIntrinsics::_nanoTime: return inline_native_time_funcs(CAST_FROM_FN_PTR(address, os::javaTimeNanos), "nanoTime");
715 case vmIntrinsics::_allocateInstance: return inline_unsafe_allocate();
716 case vmIntrinsics::_copyMemory: return inline_unsafe_copyMemory();
2853 insert_mem_bar(Op_MemBarRelease);
2854 }
2855 break;
2856 default:
2857 ShouldNotReachHere();
2858 }
2859 insert_mem_bar(Op_MemBarCPUOrder);
2860
2861 // Figure out the memory ordering.
2862 MemNode::MemOrd mo = access_kind_to_memord(access_kind);
2863
2864 // 4984716: MemBars must be inserted before this
2865 // memory node in order to avoid a false
2866 // dependency which will confuse the scheduler.
2867 Node *mem = memory(alias_idx);
2868
2869 // For now, we handle only those cases that actually exist: ints,
2870 // longs, and Object. Adding others should be straightforward.
2871 Node* load_store = NULL;
2872 switch(type) {
2873 case T_BYTE:
2874 switch(kind) {
2875 case LS_get_add:
2876 load_store = _gvn.transform(new GetAndAddBNode(control(), mem, adr, newval, adr_type));
2877 break;
2878 case LS_get_set:
2879 load_store = _gvn.transform(new GetAndSetBNode(control(), mem, adr, newval, adr_type));
2880 break;
2881 case LS_cmp_swap_weak:
2882 load_store = _gvn.transform(new WeakCompareAndSwapBNode(control(), mem, adr, newval, oldval, mo));
2883 break;
2884 case LS_cmp_swap:
2885 load_store = _gvn.transform(new CompareAndSwapBNode(control(), mem, adr, newval, oldval, mo));
2886 break;
2887 case LS_cmp_exchange:
2888 load_store = _gvn.transform(new CompareAndExchangeBNode(control(), mem, adr, newval, oldval, adr_type, mo));
2889 break;
2890 default:
2891 ShouldNotReachHere();
2892 }
2893 break;
2894 case T_SHORT:
2895 switch(kind) {
2896 case LS_get_add:
2897 load_store = _gvn.transform(new GetAndAddSNode(control(), mem, adr, newval, adr_type));
2898 break;
2899 case LS_get_set:
2900 load_store = _gvn.transform(new GetAndSetSNode(control(), mem, adr, newval, adr_type));
2901 break;
2902 case LS_cmp_swap_weak:
2903 load_store = _gvn.transform(new WeakCompareAndSwapSNode(control(), mem, adr, newval, oldval, mo));
2904 break;
2905 case LS_cmp_swap:
2906 load_store = _gvn.transform(new CompareAndSwapSNode(control(), mem, adr, newval, oldval, mo));
2907 break;
2908 case LS_cmp_exchange:
2909 load_store = _gvn.transform(new CompareAndExchangeSNode(control(), mem, adr, newval, oldval, adr_type, mo));
2910 break;
2911 default:
2912 ShouldNotReachHere();
2913 }
2914 break;
2915 case T_INT:
2916 switch(kind) {
2917 case LS_get_add:
2918 load_store = _gvn.transform(new GetAndAddINode(control(), mem, adr, newval, adr_type));
2919 break;
2920 case LS_get_set:
2921 load_store = _gvn.transform(new GetAndSetINode(control(), mem, adr, newval, adr_type));
2922 break;
2923 case LS_cmp_swap_weak:
2924 load_store = _gvn.transform(new WeakCompareAndSwapINode(control(), mem, adr, newval, oldval, mo));
2925 break;
2926 case LS_cmp_swap:
2927 load_store = _gvn.transform(new CompareAndSwapINode(control(), mem, adr, newval, oldval, mo));
2928 break;
2929 case LS_cmp_exchange:
2930 load_store = _gvn.transform(new CompareAndExchangeINode(control(), mem, adr, newval, oldval, adr_type, mo));
2931 break;
2932 default:
2933 ShouldNotReachHere();
2934 }
|