735 // itself and from the retry loop.
736 __ bind(done);
737 if (!exchange) {
738 assert(res != NULL, "need result register");
739 #ifdef _LP64
740 __ setb(Assembler::equal, res);
741 __ movzbl(res, res);
742 #else
743 // Need something else to clean the result, because some registers
744 // do not have byte encoding that movzbl wants. Cannot do the xor first,
745 // because it modifies the flags.
746 Label res_non_zero;
747 __ movptr(res, 1);
748 __ jcc(Assembler::equal, res_non_zero, true);
749 __ xorptr(res, res);
750 __ bind(res_non_zero);
751 #endif
752 }
753 }
754
755 // Generate cset check. If obj is not in cset, branch to done label, otherwise fall through
756 // obj: Register holding the oop, preserved
757 // tmp1, tmp2: temp registers, trashed
758 void ShenandoahBarrierSetAssembler::gen_cset_check(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& done) {
759 // Check for object being in the collection set.
760 // TODO: Can we use only 1 register here?
761 // The source object arrives here in rax.
762 // live: rax
763 // live: tmp1
764 __ mov(tmp1, obj);
765 __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
766 // live: tmp2
767 __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
768 __ movbool(tmp2, Address(tmp2, tmp1, Address::times_1));
769 // unlive: tmp1
770 __ testbool(tmp2);
771 // unlive: tmp2
772 __ jcc(Assembler::zero, done);
773 }
774
775 // Generate check if object is resolved. Branch to resolved label, if not. Otherwise return resolved
776 // object in obj register.
777 // obj: object, resolved object on normal return
778 // tmp: temp register, trashed
779 void ShenandoahBarrierSetAssembler::gen_resolved_check(MacroAssembler* masm, Register obj, Register tmp, Label& not_resolved) {
780 __ movptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
781 // Test if both lowest bits are set. We trick it by negating the bits
782 // then test for both bits clear.
783 __ notptr(tmp);
784 __ testb(tmp, markOopDesc::marked_value);
785 __ jccb(Assembler::notZero, not_resolved);
786 // Clear both lower bits. It's still inverted, so set them, and then invert back.
787 __ orptr(tmp, markOopDesc::marked_value);
788 __ notptr(tmp);
789 // At this point, tmp2 contains the decoded forwarding pointer.
790 __ mov(obj, tmp);
791 }
792
793 #undef __
794
795 #ifdef COMPILER1
796
797 #define __ ce->masm()->
798
799 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
800 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
801 // At this point we know that marking is in progress.
802 // If do_load() is true then we have to emit the
803 // load of the previous value; otherwise it has already
804 // been loaded into _pre_val.
805
806 __ bind(*stub->entry());
807 assert(stub->pre_val()->is_register(), "Precondition.");
808
809 Register pre_val_reg = stub->pre_val()->as_register();
810
811 if (stub->do_load()) {
812 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/);
824 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
825 __ bind(*stub->entry());
826
827 Register obj = stub->obj()->as_register();
828 Register res = stub->result()->as_register();
829 Register tmp1 = stub->tmp1()->as_register();
830 Register tmp2 = stub->tmp2()->as_register();
831
832 Label slow_path;
833
834 assert(res == rax, "result must arrive in rax");
835
836 if (res != obj) {
837 __ mov(res, obj);
838 }
839
840 // Check for null.
841 __ testptr(res, res);
842 __ jcc(Assembler::zero, *stub->continuation());
843
844 gen_cset_check(ce->masm(), res, tmp1, tmp2, *stub->continuation());
845 gen_resolved_check(ce->masm(), rax, tmp1, slow_path);
846
847 __ jmp(*stub->continuation());
848
849 __ bind(slow_path);
850 ce->store_parameter(res, 0);
851 __ call(RuntimeAddress(bs->load_reference_barrier_rt_code_blob()->code_begin()));
852
853 __ jmp(*stub->continuation());
854 }
855
856 #undef __
857
858 #define __ sasm->
859
860 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
861 __ prologue("shenandoah_pre_barrier", false);
862 // arg0 : previous value of memory
863
864 __ push(rax);
865 __ push(rdx);
936
937 #define __ cgen->assembler()->
938
939 address ShenandoahBarrierSetAssembler::generate_shenandoah_lrb(StubCodeGenerator* cgen) {
940 __ align(CodeEntryAlignment);
941 StubCodeMark mark(cgen, "StubRoutines", "shenandoah_lrb");
942 address start = __ pc();
943
944 Label resolve_oop, slow_path, done;
945
946 // We use RDI, which also serves as argument register for slow call.
947 // RAX always holds the src object ptr, except after the slow call,
948 // then it holds the result. R8/RBX is used as temporary register.
949
950 Register tmp1 = rdi;
951 Register tmp2 = LP64_ONLY(r8) NOT_LP64(rbx);
952
953 __ push(tmp1);
954 __ push(tmp2);
955
956 gen_cset_check(cgen->assembler(), rax, tmp1, tmp2, done);
957
958 __ bind(resolve_oop);
959
960 gen_resolved_check(cgen->assembler(), rax, tmp2, slow_path);
961
962 __ bind(done);
963 __ pop(tmp2);
964 __ pop(tmp1);
965 __ ret(0);
966
967 __ bind(slow_path);
968
969 __ push(rcx);
970 __ push(rdx);
971 __ push(rdi);
972 __ push(rsi);
973 #ifdef _LP64
974 __ push(r8);
975 __ push(r9);
976 __ push(r10);
977 __ push(r11);
978 __ push(r12);
979 __ push(r13);
980 __ push(r14);
|
735 // itself and from the retry loop.
736 __ bind(done);
737 if (!exchange) {
738 assert(res != NULL, "need result register");
739 #ifdef _LP64
740 __ setb(Assembler::equal, res);
741 __ movzbl(res, res);
742 #else
743 // Need something else to clean the result, because some registers
744 // do not have byte encoding that movzbl wants. Cannot do the xor first,
745 // because it modifies the flags.
746 Label res_non_zero;
747 __ movptr(res, 1);
748 __ jcc(Assembler::equal, res_non_zero, true);
749 __ xorptr(res, res);
750 __ bind(res_non_zero);
751 #endif
752 }
753 }
754
755 #undef __
756
757 #ifdef COMPILER1
758
759 #define __ ce->masm()->
760
761 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
762 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
763 // At this point we know that marking is in progress.
764 // If do_load() is true then we have to emit the
765 // load of the previous value; otherwise it has already
766 // been loaded into _pre_val.
767
768 __ bind(*stub->entry());
769 assert(stub->pre_val()->is_register(), "Precondition.");
770
771 Register pre_val_reg = stub->pre_val()->as_register();
772
773 if (stub->do_load()) {
774 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/);
786 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
787 __ bind(*stub->entry());
788
789 Register obj = stub->obj()->as_register();
790 Register res = stub->result()->as_register();
791 Register tmp1 = stub->tmp1()->as_register();
792 Register tmp2 = stub->tmp2()->as_register();
793
794 Label slow_path;
795
796 assert(res == rax, "result must arrive in rax");
797
798 if (res != obj) {
799 __ mov(res, obj);
800 }
801
802 // Check for null.
803 __ testptr(res, res);
804 __ jcc(Assembler::zero, *stub->continuation());
805
806 // Check for object being in the collection set.
807 __ mov(tmp1, res);
808 __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
809 __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
810 __ movbool(tmp2, Address(tmp2, tmp1, Address::times_1));
811 __ testbool(tmp2);
812 __ jcc(Assembler::zero, *stub->continuation());
813
814 // Test if object is resolved.
815 __ movptr(tmp1, Address(res, oopDesc::mark_offset_in_bytes()));
816 // Test if both lowest bits are set. We trick it by negating the bits
817 // then test for both bits clear.
818 __ notptr(tmp1);
819 __ testb(tmp1, markOopDesc::marked_value);
820 __ jccb(Assembler::notZero, slow_path);
821 // Clear both lower bits. It's still inverted, so set them, and then invert back.
822 __ orptr(tmp1, markOopDesc::marked_value);
823 __ notptr(tmp1);
824 // At this point, tmp1 contains the decoded forwarding pointer.
825 __ mov(res, tmp1);
826
827 __ jmp(*stub->continuation());
828
829 __ bind(slow_path);
830 ce->store_parameter(res, 0);
831 __ call(RuntimeAddress(bs->load_reference_barrier_rt_code_blob()->code_begin()));
832
833 __ jmp(*stub->continuation());
834 }
835
836 #undef __
837
838 #define __ sasm->
839
840 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
841 __ prologue("shenandoah_pre_barrier", false);
842 // arg0 : previous value of memory
843
844 __ push(rax);
845 __ push(rdx);
916
917 #define __ cgen->assembler()->
918
919 address ShenandoahBarrierSetAssembler::generate_shenandoah_lrb(StubCodeGenerator* cgen) {
920 __ align(CodeEntryAlignment);
921 StubCodeMark mark(cgen, "StubRoutines", "shenandoah_lrb");
922 address start = __ pc();
923
924 Label resolve_oop, slow_path, done;
925
926 // We use RDI, which also serves as argument register for slow call.
927 // RAX always holds the src object ptr, except after the slow call,
928 // then it holds the result. R8/RBX is used as temporary register.
929
930 Register tmp1 = rdi;
931 Register tmp2 = LP64_ONLY(r8) NOT_LP64(rbx);
932
933 __ push(tmp1);
934 __ push(tmp2);
935
936 // Check for object being in the collection set.
937 __ mov(tmp1, rax);
938 __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
939 __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
940 __ movbool(tmp2, Address(tmp2, tmp1, Address::times_1));
941 __ testbool(tmp2);
942 __ jccb(Assembler::notZero, resolve_oop);
943 __ pop(tmp2);
944 __ pop(tmp1);
945 __ ret(0);
946
947 // Test if object is already resolved.
948 __ bind(resolve_oop);
949 __ movptr(tmp2, Address(rax, oopDesc::mark_offset_in_bytes()));
950 // Test if both lowest bits are set. We trick it by negating the bits
951 // then test for both bits clear.
952 __ notptr(tmp2);
953 __ testb(tmp2, markOopDesc::marked_value);
954 __ jccb(Assembler::notZero, slow_path);
955 // Clear both lower bits. It's still inverted, so set them, and then invert back.
956 __ orptr(tmp2, markOopDesc::marked_value);
957 __ notptr(tmp2);
958 // At this point, tmp2 contains the decoded forwarding pointer.
959 __ mov(rax, tmp2);
960
961 __ bind(done);
962 __ pop(tmp2);
963 __ pop(tmp1);
964 __ ret(0);
965
966 __ bind(slow_path);
967
968 __ push(rcx);
969 __ push(rdx);
970 __ push(rdi);
971 __ push(rsi);
972 #ifdef _LP64
973 __ push(r8);
974 __ push(r9);
975 __ push(r10);
976 __ push(r11);
977 __ push(r12);
978 __ push(r13);
979 __ push(r14);
|