# HG changeset patch # User rbackman # Date 1347433865 -7200 # Node ID 369a0ff144189e3ef12ae913bbc53d074eabb56c # Parent aa7b717af722394fcb2da5ab771416ef9a3e38c5 tagged: initial patch diff --git a/series b/series --- a/series +++ b/series @@ -8,6 +8,8 @@ meth.proj.patch #-/meth #+projects anonk.proj.patch #-/anonk #+projects +tagged.patch #+tagged #-/tagged #+36d1d483d5d6 + # Keep these separate, for debugging and review: tagu.patch #+tagu #-/tagu #+36d1d483d5d6 #-buildable inti.patch #+inti #-/inti #+d1605aabd0a1 #+jdk7-b30 #-buildable diff --git a/tagged.patch b/tagged.patch new file mode 100644 --- /dev/null +++ b/tagged.patch @@ -0,0 +1,2389 @@ +# HG changeset patch +# User rbackman +# Date 1346130953 -7200 +# Node ID a25ea835125789c5ac10b6044ba30d4bc2177de7 +# Parent 5af51c8822075c8da449dd7ba0e2cc9c84658e5f +Tagged arrays for HotSpot/OpenJDK + +diff --git a/src/cpu/x86/vm/assembler_x86.cpp b/src/cpu/x86/vm/assembler_x86.cpp +--- a/src/cpu/x86/vm/assembler_x86.cpp ++++ b/src/cpu/x86/vm/assembler_x86.cpp +@@ -1631,6 +1631,22 @@ + emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_66); + } + ++void Assembler::movdqa(XMMRegister dst, Address src) { ++ NOT_LP64(assert(VM_Version::supports_sse2(), "")); ++ InstructionMark im(this); ++ simd_prefix(dst, src, VEX_SIMD_66); ++ emit_byte(0x6F); ++ emit_operand(dst, src); ++} ++ ++void Assembler::movdqa(Address dst, XMMRegister src) { ++ NOT_LP64(assert(VM_Version::supports_sse2(), "")); ++ InstructionMark im(this); ++ simd_prefix(dst, src, VEX_SIMD_66); ++ emit_byte(0x7F); ++ emit_operand(src, dst); ++} ++ + void Assembler::movdqu(XMMRegister dst, Address src) { + NOT_LP64(assert(VM_Version::supports_sse2(), "")); + emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_F3); +diff --git a/src/cpu/x86/vm/assembler_x86.hpp b/src/cpu/x86/vm/assembler_x86.hpp +--- a/src/cpu/x86/vm/assembler_x86.hpp ++++ b/src/cpu/x86/vm/assembler_x86.hpp +@@ -1271,6 +1271,8 @@ + + // Move Aligned Double Quadword + void movdqa(XMMRegister dst, XMMRegister src); ++ void movdqa(XMMRegister dst, Address src); ++ void movdqa(Address dst, XMMRegister src); + + // Move Unaligned Double Quadword + void movdqu(Address dst, XMMRegister src); +diff --git a/src/cpu/x86/vm/stubGenerator_x86_64.cpp b/src/cpu/x86/vm/stubGenerator_x86_64.cpp +--- a/src/cpu/x86/vm/stubGenerator_x86_64.cpp ++++ b/src/cpu/x86/vm/stubGenerator_x86_64.cpp +@@ -25,6 +25,7 @@ + #include "precompiled.hpp" + #include "asm/assembler.hpp" + #include "assembler_x86.inline.hpp" ++#include "classfile/javaClasses.hpp" + #include "interpreter/interpreter.hpp" + #include "nativeInst_x86.hpp" + #include "oops/instanceOop.hpp" +@@ -2763,6 +2764,450 @@ + return start; + } + ++ address generate_tagged_copy_forward(address* entry) { ++ __ align(CodeEntryAlignment); ++ StubCodeMark mark(this, "StubRoutines", "taggedCopyForward"); ++ address start = __ pc(); ++ ++ Label L_copy1, L_copy_end, L_copy_aligned_16, L_unaligned, L_copy_single, L_copy_unaligned_16, L_leave; ++ ++ // rdi [src], rsi [srcpos], rdx [dst], rcx [dstpos], r8 [len] ++ __ enter(); ++ ++ const Register src = rdi; ++ const Register src_start = rsi; ++ ++ const Register dst = rdx; ++ const Register dst_start = rcx; ++ ++ const Register count = r8; ++ ++ const Register sptr = r10; ++ const Register dptr = rax; ++ ++ const Register to = r9; ++ ++ if (entry != NULL) { ++ *entry = __ pc(); ++ BLOCK_COMMENT("Entry:"); ++ } ++ ++ __ testl(count, count); ++ __ jcc(Assembler::zero, L_leave); ++ ++ size_t slot_offset = sun_misc_taggedarray_TaggedArrayNativeImpl::slot_offset(); ++ __ lea(sptr, Address(src, src_start, Address::times_8, slot_offset)); ++ __ lea(dptr, Address(dst, dst_start, Address::times_8, slot_offset)); ++ ++ __ movptr(dst_start, dptr); ++ ++ gen_write_ref_array_pre_barrier(dst_start, count, false); ++ ++ __ testl(dptr, 0x0f); // 16 bit alignment ++ __ jcc(Assembler::notZero, L_unaligned); ++ __ testl(sptr, 0x0f); // 16 bit alignment ++ __ jcc(Assembler::notZero, L_unaligned); ++ ++ // dst & src are aligned on 16 bit ++ // check size ++ ++ __ align(OptoLoopAlignment); ++ __ BIND(L_copy_aligned_16); ++ __ cmpl(count, 8); ++ __ jcc(Assembler::less, L_unaligned); ++ ++ // copy ++ ++ __ movdqa(xmm0, Address(sptr, 0 * 16)); ++ __ movdqa(xmm1, Address(sptr, 1 * 16)); ++ __ movdqa(xmm2, Address(sptr, 2 * 16)); ++ __ movdqa(xmm3, Address(sptr, 3 * 16)); ++ __ movdqa(xmm4, Address(sptr, 4 * 16)); ++ __ movdqa(xmm5, Address(sptr, 5 * 16)); ++ __ movdqa(xmm6, Address(sptr, 6 * 16)); ++ __ movdqa(xmm7, Address(sptr, 7 * 16)); ++ ++ __ movdqa(Address(dptr, 0 * 16), xmm0); ++ __ movdqa(Address(dptr, 1 * 16), xmm1); ++ __ movdqa(Address(dptr, 2 * 16), xmm2); ++ __ movdqa(Address(dptr, 3 * 16), xmm3); ++ __ movdqa(Address(dptr, 4 * 16), xmm4); ++ __ movdqa(Address(dptr, 5 * 16), xmm5); ++ __ movdqa(Address(dptr, 6 * 16), xmm6); ++ __ movdqa(Address(dptr, 7 * 16), xmm7); ++ ++ __ addptr(sptr, 16 * 8); ++ __ addptr(dptr, 16 * 8); ++ __ subl(count, 16); ++ __ jccb(Assembler::notZero, L_copy_aligned_16); ++ __ jmp(L_copy_end); ++ ++ ++ __ align(OptoLoopAlignment); ++ __ BIND(L_unaligned); ++ __ BIND(L_copy_unaligned_16); ++ __ cmpl(count, 16); ++ __ jcc(Assembler::less, L_copy_single); ++ ++ // copy ++ __ movdqu(xmm0, Address(sptr, 0 * 16)); ++ __ movdqu(xmm1, Address(sptr, 1 * 16)); ++ __ movdqu(xmm2, Address(sptr, 2 * 16)); ++ __ movdqu(xmm3, Address(sptr, 3 * 16)); ++ __ movdqu(xmm4, Address(sptr, 4 * 16)); ++ __ movdqu(xmm5, Address(sptr, 5 * 16)); ++ __ movdqu(xmm6, Address(sptr, 6 * 16)); ++ __ movdqu(xmm7, Address(sptr, 7 * 16)); ++ ++ __ movdqu(Address(dptr, 0 * 16), xmm0); ++ __ movdqu(Address(dptr, 1 * 16), xmm1); ++ __ movdqu(Address(dptr, 2 * 16), xmm2); ++ __ movdqu(Address(dptr, 3 * 16), xmm3); ++ __ movdqu(Address(dptr, 4 * 16), xmm4); ++ __ movdqu(Address(dptr, 5 * 16), xmm5); ++ __ movdqu(Address(dptr, 6 * 16), xmm6); ++ __ movdqu(Address(dptr, 7 * 16), xmm7); ++ ++ __ addptr(sptr, 16 * 8); ++ __ addptr(dptr, 16 * 8); ++ __ subl(count, 16); ++ __ jccb(Assembler::notZero, L_copy_unaligned_16); ++ __ jmpb(L_copy_end); ++ ++ __ BIND(L_copy_single); ++ ++ __ cmpl(count, 0); ++ __ jccb(Assembler::lessEqual, L_copy_end); ++ ++ __ align(OptoLoopAlignment); ++ __ BIND(L_copy1); ++ ++ __ movq(to, Address(sptr, 0)); ++ __ movq(Address(dptr, 0), to); ++ __ addptr(sptr, 8); ++ __ addptr(dptr, 8); ++ __ subl(count, 1); ++ __ jccb(Assembler::notZero, L_copy1); ++ ++ __ BIND(L_copy_end); ++ ++ __ subptr(dptr, 8); ++ ++ gen_write_ref_array_post_barrier(dst_start, dptr, rscratch1); ++ inc_counter_np(SharedRuntime::_tagged_copy_ctr); // Update counter after rscratch1 is free ++ ++ __ BIND(L_leave); ++ __ leave(); // required for proper stackwalking of RuntimeStub frame ++ __ ret(0); ++ return start; ++ } ++ ++ ++ address generate_tagged_copy_backward(address* entry) { ++ __ align(CodeEntryAlignment); ++ StubCodeMark mark(this, "StubRoutines", "taggedCopyBackward"); ++ address start = __ pc(); ++ ++ Label L_copy1, L_copy_end, L_copy_aligned_16, L_unaligned, L_copy_single, L_copy_unaligned_16, L_leave; ++ ++ // rdi [src], rsi [srcpos], rdx [dst], rcx [dstpos], r8 [len] ++ __ enter(); ++ ++ const Register src = rdi; ++ ++ const Register src_start = rsi; ++ ++ const Register dst = rdx; ++ const Register dst_start = rcx; ++ const Register dst_end = src_start; ++ ++ const Register count = r8; ++ ++ const Register sptr = rax; ++ const Register dptr = r10; ++ ++ const Register to = r9; ++ ++ if (entry != NULL) { ++ *entry = __ pc(); ++ BLOCK_COMMENT("Entry:"); ++ } ++ ++ __ testl(count, count); ++ __ jcc(Assembler::zero, L_leave); ++ ++ size_t slot_offset = sun_misc_taggedarray_TaggedArrayNativeImpl::slot_offset(); ++ __ lea(sptr, Address(src, src_start, Address::times_8, slot_offset)); ++ __ lea(dptr, Address(dst, dst_start, Address::times_8, slot_offset)); ++ ++ __ movptr(dst_start, dptr); ++ ++ __ lea(sptr, Address(sptr, count, Address::times_1, -1)); ++ __ lea(dptr, Address(dptr, count, Address::times_1, -1)); ++ ++ ++ gen_write_ref_array_pre_barrier(dst_start, count, false); ++ ++ __ movptr (dst_end, dptr); // last element now ++ ++ __ testl(dptr, 0x0f); // 16 bit alignment ++ __ jcc(Assembler::notZero, L_unaligned); ++ __ testl(sptr, 0x0f); // 16 bit alignment ++ __ jcc(Assembler::notZero, L_unaligned); ++ ++ // dst & src are aligned on 16 bit ++ // check size ++ ++ __ align(OptoLoopAlignment); ++ __ BIND(L_copy_aligned_16); ++ __ cmpl(count, 8); ++ __ jcc(Assembler::less, L_unaligned); ++ ++ // copy ++ ++ __ movdqa(xmm0, Address(sptr, 0 * 16)); ++ __ movdqa(xmm1, Address(sptr, -1 * 16)); ++ __ movdqa(xmm2, Address(sptr, -2 * 16)); ++ __ movdqa(xmm3, Address(sptr, -3 * 16)); ++ __ movdqa(xmm4, Address(sptr, -4 * 16)); ++ __ movdqa(xmm5, Address(sptr, -5 * 16)); ++ __ movdqa(xmm6, Address(sptr, -6 * 16)); ++ __ movdqa(xmm7, Address(sptr, -7 * 16)); ++ ++ __ movdqa(Address(dptr, 0 * 16), xmm0); ++ __ movdqa(Address(dptr, -1 * 16), xmm1); ++ __ movdqa(Address(dptr, -2 * 16), xmm2); ++ __ movdqa(Address(dptr, -3 * 16), xmm3); ++ __ movdqa(Address(dptr, -4 * 16), xmm4); ++ __ movdqa(Address(dptr, -5 * 16), xmm5); ++ __ movdqa(Address(dptr, -6 * 16), xmm6); ++ __ movdqa(Address(dptr, -7 * 16), xmm7); ++ ++ __ subptr(sptr, 16 * 8); ++ __ subptr(dptr, 16 * 8); ++ __ subl(count, 16); ++ __ jccb(Assembler::notZero, L_copy_aligned_16); ++ __ jmp(L_copy_end); ++ ++ __ BIND(L_unaligned); ++ ++ __ align(OptoLoopAlignment); ++ __ BIND(L_copy_unaligned_16); ++ __ cmpl(count, 16); ++ __ jcc(Assembler::less, L_copy_single); ++ ++ // copy ++ __ movdqu(xmm0, Address(sptr, 0 * 16)); ++ __ movdqu(xmm1, Address(sptr, -1 * 16)); ++ __ movdqu(xmm2, Address(sptr, -2 * 16)); ++ __ movdqu(xmm3, Address(sptr, -3 * 16)); ++ __ movdqu(xmm4, Address(sptr, -4 * 16)); ++ __ movdqu(xmm5, Address(sptr, -5 * 16)); ++ __ movdqu(xmm6, Address(sptr, -6 * 16)); ++ __ movdqu(xmm7, Address(sptr, -7 * 16)); ++ ++ __ movdqu(Address(dptr, 0 * 16), xmm0); ++ __ movdqu(Address(dptr, -1 * 16), xmm1); ++ __ movdqu(Address(dptr, -2 * 16), xmm2); ++ __ movdqu(Address(dptr, -3 * 16), xmm3); ++ __ movdqu(Address(dptr, -4 * 16), xmm4); ++ __ movdqu(Address(dptr, -5 * 16), xmm5); ++ __ movdqu(Address(dptr, -6 * 16), xmm6); ++ __ movdqu(Address(dptr, -7 * 16), xmm7); ++ ++ __ subptr(sptr, 16 * 8); ++ __ subptr(dptr, 16 * 8); ++ __ subl(count, 16); ++ __ jccb(Assembler::notZero, L_copy_unaligned_16); ++ __ jmpb(L_copy_end); ++ ++ __ BIND(L_copy_single); ++ ++ __ cmpl(count, 0); ++ __ jccb(Assembler::lessEqual, L_copy_end); ++ ++ __ align(OptoLoopAlignment); ++ __ BIND(L_copy1); ++ ++ __ movq(to, Address(sptr, 0)); ++ __ movq(Address(dptr, 0), to); ++ __ subptr(sptr, 8); ++ __ subptr(dptr, 8); ++ __ subl(count, 1); ++ __ jccb(Assembler::notZero, L_copy1); ++ ++ __ BIND(L_copy_end); ++ ++ gen_write_ref_array_post_barrier(dst_start, dst_end, rscratch1); ++ inc_counter_np(SharedRuntime::_tagged_copy_ctr); // Update counter after rscratch1 is free ++ ++ __ BIND(L_leave); ++ __ leave(); // required for proper stackwalking of RuntimeStub frame ++ __ ret(0); ++ return start; ++ } ++ ++ address generate_tagged_fill(address* entry) { ++ __ align(CodeEntryAlignment); ++ StubCodeMark mark(this, "StubRoutines", "taggedFill"); ++ address start = __ pc(); ++ ++ Label L_set8, L_set4, L_set1, L_end; ++ Label L_aligned_loop16, L_aligned_loop8, L_aligned_loop4, L_leave; ++ ++ __ enter(); ++ ++ if (entry != NULL) { ++ *entry = __ pc(); ++ BLOCK_COMMENT("Entry:"); ++ } ++ ++ __ subptr(rsp, 16); // still aligned ++ ++ const Register dst = rdi; ++ const Register dst_start = dst; ++ const Register from = rsi; ++ const Register to = rdx; ++ const Register value = rcx; ++ ++ if (UseCompressedOops) { ++ __ encode_heap_oop_not_null(value); // Changes flags. ++ __ shlptr(value, 32); // put in the upper 4 bytes ++ } ++ ++ const Register count = r8; ++ ++ __ movl(count, to); ++ __ subl(count, from); ++ ++ __ testl(count, count); ++ __ jcc(Assembler::zero, L_leave); ++ ++ size_t slot_offset = sun_misc_taggedarray_TaggedArrayNativeImpl::slot_offset(); ++ const Register dptr = rax; ++ __ lea(dptr, Address(dst, from, Address::times_8, slot_offset)); ++ __ movptr(dst_start, dptr); ++ ++ gen_write_ref_array_pre_barrier(dst_start, count, false); ++ ++ __ testl(dptr, 0x0f); // 16 bit alignment ++ __ jcc(Assembler::notZero, L_set8); ++ ++ // aligned: ++ __ align(OptoLoopAlignment); ++ __ movptr(Address(rsp, 0), value); ++ __ movptr(Address(rsp, 8), value); ++ __ movdqa(xmm0, Address(rsp, 0)); ++ ++ __ BIND(L_aligned_loop16); ++ __ cmpl(count, 16); ++ __ jcc(Assembler::less, L_aligned_loop8); ++ ++ __ movdqa(Address(dptr, 0 * 8), xmm0); ++ __ movdqa(Address(dptr, 2 * 8), xmm0); ++ __ movdqa(Address(dptr, 4 * 8), xmm0); ++ __ movdqa(Address(dptr, 6 * 8), xmm0); ++ __ movdqa(Address(dptr, 8 * 8), xmm0); ++ __ movdqa(Address(dptr, 10 * 8), xmm0); ++ __ movdqa(Address(dptr, 12 * 8), xmm0); ++ __ movdqa(Address(dptr, 14 * 8), xmm0); ++ ++ __ addptr(dptr, 16 * 8); ++ __ subl(count, 16); ++ __ jccb(Assembler::notZero, L_aligned_loop16); ++ __ jmp(L_end); ++ ++ __ BIND(L_aligned_loop8); ++ __ cmpl(count, 8); ++ __ jccb(Assembler::less, L_aligned_loop4); ++ ++ __ movdqa(Address(dptr, 0 * 8), xmm0); ++ __ movdqa(Address(dptr, 2 * 8), xmm0); ++ __ movdqa(Address(dptr, 4 * 8), xmm0); ++ __ movdqa(Address(dptr, 6 * 8), xmm0); ++ ++ __ addptr(dptr, 8 * 8); ++ __ subl(count, 8); ++ __ jccb(Assembler::notZero, L_aligned_loop8); ++ __ jmp(L_end); ++ ++ __ BIND(L_aligned_loop4); ++ __ cmpl(count, 4); ++ __ jcc(Assembler::less, L_set1); ++ ++ __ movdqa(Address(dptr, 0 * 8), xmm0); ++ __ movdqa(Address(dptr, 2 * 8), xmm0); ++ ++ __ addptr(dptr, 4 * 8); ++ __ subl(count, 4); ++ __ jccb(Assembler::notZero, L_aligned_loop4); ++ __ jmp(L_end); ++ ++ __ align(OptoLoopAlignment); ++ __ BIND(L_set8); ++ __ cmpl(count, 8); ++ __ jccb(Assembler::less, L_set4); ++ ++ __ movq(Address(dptr, 0 * 8), value); ++ __ movq(Address(dptr, 1 * 8), value); ++ __ movq(Address(dptr, 2 * 8), value); ++ __ movq(Address(dptr, 3 * 8), value); ++ ++ __ movq(Address(dptr, 4 * 8), value); ++ __ movq(Address(dptr, 5 * 8), value); ++ __ movq(Address(dptr, 6 * 8), value); ++ __ movq(Address(dptr, 7 * 8), value); ++ ++ __ addptr(dptr, 8 * 8); ++ __ subl(count, 8); ++ __ jccb(Assembler::notZero, L_set8); ++ __ jmpb(L_end); ++ ++ __ align(OptoLoopAlignment); ++ __ BIND(L_set4); ++ __ cmpl(count, 4); ++ __ jccb(Assembler::less, L_set1); ++ ++ __ movq(Address(dptr, 0 * 8), value); ++ __ movq(Address(dptr, 1 * 8), value); ++ __ movq(Address(dptr, 2 * 8), value); ++ __ movq(Address(dptr, 3 * 8), value); ++ ++ __ addptr(dptr, 4 * 8); ++ __ subl(count, 4); ++ __ jccb(Assembler::notZero, L_set4); ++ __ jmpb(L_end); ++ ++ __ align(OptoLoopAlignment); ++ ++ __ BIND(L_set1); ++ ++ __ movq(Address(dptr, 0), value); ++ __ addptr(dptr, 8); ++ __ subl(count, 1); ++ __ jccb(Assembler::notZero, L_set1); ++ ++ __ BIND(L_end); ++ ++ __ subptr(dptr, 8); ++ ++ gen_write_ref_array_post_barrier(dst_start, dptr, rscratch1); ++ inc_counter_np(SharedRuntime::_tagged_fill_ctr); // Update counter after rscratch1 is free ++ ++ __ BIND(L_leave); ++ ++ __ leave(); ++ __ ret(0); ++ ++ return start; ++ } ++ ++ void generate_tagged_stubs() { ++ StubRoutines::_tagged_fill = generate_tagged_fill(NULL); ++ StubRoutines::_tagged_copy_forward = generate_tagged_copy_forward(NULL); ++ StubRoutines::_tagged_copy_backward = generate_tagged_copy_backward(NULL); ++ } ++ + void generate_arraycopy_stubs() { + address entry; + address entry_jbyte_arraycopy; +@@ -3150,6 +3595,8 @@ + + // arraycopy stubs used by compilers + generate_arraycopy_stubs(); ++ ++ generate_tagged_stubs(); + + generate_math_stubs(); + } +diff --git a/src/share/vm/c1/c1_GraphBuilder.cpp b/src/share/vm/c1/c1_GraphBuilder.cpp +--- a/src/share/vm/c1/c1_GraphBuilder.cpp ++++ b/src/share/vm/c1/c1_GraphBuilder.cpp +@@ -3250,6 +3250,12 @@ + if (!InlineArrayCopy) return false; + break; + ++ case vmIntrinsics::_TaggedArrays_doesVMImplement: ++ if (!NativeTaggedArrays) return false; ++ preserves_state = true; ++ cantrap = false; ++ break; ++ + #ifdef TRACE_HAVE_INTRINSICS + case vmIntrinsics::_classID: + case vmIntrinsics::_threadID: +diff --git a/src/share/vm/c1/c1_LIRGenerator.cpp b/src/share/vm/c1/c1_LIRGenerator.cpp +--- a/src/share/vm/c1/c1_LIRGenerator.cpp ++++ b/src/share/vm/c1/c1_LIRGenerator.cpp +@@ -2885,6 +2885,10 @@ + __ move(reg, result); + } + ++void LIRGenerator::do_TaggedArrays_doesVMImplement(Intrinsic* x) { ++ __ move(LIR_OprFact::intConst(0x1), rlock_result(x)); ++} ++ + #ifdef TRACE_HAVE_INTRINSICS + void LIRGenerator::do_ThreadIDIntrinsic(Intrinsic* x) { + LIR_Opr thread = getThreadPointer(); +@@ -2931,6 +2935,7 @@ + break; + } + ++ case vmIntrinsics::_TaggedArrays_doesVMImplement: do_TaggedArrays_doesVMImplement(x); break; + #ifdef TRACE_HAVE_INTRINSICS + case vmIntrinsics::_threadID: do_ThreadIDIntrinsic(x); break; + case vmIntrinsics::_classID: do_ClassIDIntrinsic(x); break; +diff --git a/src/share/vm/c1/c1_LIRGenerator.hpp b/src/share/vm/c1/c1_LIRGenerator.hpp +--- a/src/share/vm/c1/c1_LIRGenerator.hpp ++++ b/src/share/vm/c1/c1_LIRGenerator.hpp +@@ -431,6 +431,7 @@ + void do_ThreadIDIntrinsic(Intrinsic* x); + void do_ClassIDIntrinsic(Intrinsic* x); + #endif ++ void do_TaggedArrays_doesVMImplement(Intrinsic* x); + + public: + Compilation* compilation() const { return _compilation; } +diff --git a/src/share/vm/classfile/classFileParser.cpp b/src/share/vm/classfile/classFileParser.cpp +--- a/src/share/vm/classfile/classFileParser.cpp ++++ b/src/share/vm/classfile/classFileParser.cpp +@@ -3352,6 +3352,7 @@ + } + #endif + bool compact_fields = CompactFields; ++ bool slow_path = false; + int allocation_style = FieldsAllocationStyle; + if( allocation_style < 0 || allocation_style > 2 ) { // Out of range? + assert(false, "0 <= FieldsAllocationStyle <= 2"); +@@ -3377,7 +3378,11 @@ + class_name == vmSymbols::java_lang_Byte() || + class_name == vmSymbols::java_lang_Short() || + class_name == vmSymbols::java_lang_Integer() || +- class_name == vmSymbols::java_lang_Long())) { ++ class_name == vmSymbols::java_lang_Long() || ++ class_name == vmSymbols::sun_misc_taggedarray_TaggedArrayNativeImpl())) { ++ if (class_name == vmSymbols::sun_misc_taggedarray_TaggedArrayNativeImpl()) { ++ slow_path = true; ++ } + allocation_style = 0; // Allocate oops first + compact_fields = false; // Don't compact fields + } +@@ -3612,7 +3617,7 @@ + + // Fill in information already parsed + this_klass->set_should_verify_class(verify); +- jint lh = Klass::instance_layout_helper(instance_size, false); ++ jint lh = Klass::instance_layout_helper(instance_size, slow_path); + this_klass->set_layout_helper(lh); + assert(this_klass->oop_is_instance(), "layout is correct"); + assert(this_klass->size_helper() == instance_size, "correct size_helper"); +diff --git a/src/share/vm/classfile/javaClasses.cpp b/src/share/vm/classfile/javaClasses.cpp +--- a/src/share/vm/classfile/javaClasses.cpp ++++ b/src/share/vm/classfile/javaClasses.cpp +@@ -36,6 +36,7 @@ + #include "memory/universe.inline.hpp" + #include "oops/fieldStreams.hpp" + #include "oops/instanceKlass.hpp" ++#include "oops/instanceTagKlass.hpp" + #include "oops/instanceMirrorKlass.hpp" + #include "oops/klass.hpp" + #include "oops/klassOop.hpp" +@@ -580,7 +581,61 @@ + } + } + +- ++int sun_misc_taggedarray_TaggedArrayNativeImpl::slots_count(oop bag) { ++ jint ival = bag->int_field(_length_offset); ++ return ival; ++} ++ ++oop sun_misc_taggedarray_TaggedArrayNativeImpl::get_slot_ref(oop bag, jint slot) { ++ assert(_data_offset != 0, "must be set"); ++ if (UseCompressedOops) { ++ return bag->obj_field(_data_offset + BytesPerInt + BytesPerLong * slot); ++ } else { ++ return bag->obj_field(_data_offset + BytesPerLong * slot); ++ } ++} ++ ++void sun_misc_taggedarray_TaggedArrayNativeImpl::set_slot_ref(oop bag, jint slot, oop value) { ++ assert(_data_offset != 0, "must be set"); ++ if (UseCompressedOops) { ++ // write compressed oops to the high 4 bytes ++ bag->int_field_put(_data_offset + BytesPerLong * slot, 0); // overwrite previous tag ++ bag->obj_field_put(_data_offset + BytesPerInt + BytesPerLong * slot, value); ++ } else { ++ bag->obj_field_put(_data_offset + BytesPerLong * slot, value); ++ } ++} ++ ++jboolean sun_misc_taggedarray_TaggedArrayNativeImpl::is_slot_tagged(oop bag, jint slot) { ++ assert(_data_offset != 0, "must be set"); ++ jlong value = bag->long_field(_data_offset + BytesPerLong * slot); ++ return (jboolean) (value & 0x1); ++} ++ ++void sun_misc_taggedarray_TaggedArrayNativeImpl::set_slot_value(oop bag, jint slot, jlong value) { ++ assert(_data_offset != 0, "must be set"); ++ assert((value & 0x1) == 0x1, "tagged bit must be set"); ++ bag->long_field_put(_data_offset + BytesPerLong * slot, value); ++} ++ ++jlong sun_misc_taggedarray_TaggedArrayNativeImpl::get_slot_value(oop bag, jint slot) { ++ assert(_data_offset != 0, "must be set"); ++ return bag->long_field(_data_offset + BytesPerLong * slot); ++} ++ ++void sun_misc_taggedarray_TaggedArrayNativeImpl::copy_slot(oop dst, oop src, jint dstpos, jint srcpos) { ++ if (is_slot_tagged(src, srcpos)) { ++ set_slot_value(dst, dstpos, get_slot_value(src, srcpos)); ++ } else { ++ oop tmp = get_slot_ref(src, srcpos); ++ set_slot_ref(dst, dstpos, tmp); ++ } ++} ++ ++void sun_misc_taggedarray_TaggedArrayNativeImpl::set_length(oop java_class, jint length) { ++ assert(_length_offset != 0, "must be set"); ++ java_class->int_field_put(_length_offset, length); ++} + + int java_lang_Class::oop_size(oop java_class) { + assert(_oop_size_offset != 0, "must be set"); +@@ -749,6 +804,19 @@ + return mirror; + } + ++bool sun_misc_taggedarray_TaggedArrayNativeImpl::offsets_computed = false; ++ ++void sun_misc_taggedarray_TaggedArrayNativeImpl::compute_offsets() { ++ assert(!offsets_computed, "offsets should be initialized only once"); ++ offsets_computed = true; ++ ++ klassOop k = SystemDictionary::TaggedArrayNativeImpl_klass(); ++ if (k) { ++ compute_offset(_length_offset, k, vmSymbols::length_name(), vmSymbols::int_signature()); ++ _data_offset = instanceTagKlass::cast(SystemDictionary::TaggedArrayNativeImpl_klass())->size_helper() << LogHeapWordSize; ++ } ++} ++ + bool java_lang_Class::offsets_computed = false; + int java_lang_Class::classRedefinedCount_offset = -1; + +@@ -2777,6 +2845,8 @@ + int java_lang_Class::_resolved_constructor_offset; + int java_lang_Class::_oop_size_offset; + int java_lang_Class::_static_oop_field_count_offset; ++int sun_misc_taggedarray_TaggedArrayNativeImpl::_length_offset; ++int sun_misc_taggedarray_TaggedArrayNativeImpl::_data_offset; + int java_lang_Throwable::backtrace_offset; + int java_lang_Throwable::detailMessage_offset; + int java_lang_Throwable::cause_offset; +@@ -2993,6 +3063,8 @@ + sun_reflect_ConstantPool::compute_offsets(); + sun_reflect_UnsafeStaticFieldAccessorImpl::compute_offsets(); + } ++ sun_misc_taggedarray_TaggedArrayNativeImpl::compute_offsets(); ++ + + // generated interpreter code wants to know about the offsets we just computed: + AbstractAssembler::update_delayed_values(); +diff --git a/src/share/vm/classfile/javaClasses.hpp b/src/share/vm/classfile/javaClasses.hpp +--- a/src/share/vm/classfile/javaClasses.hpp ++++ b/src/share/vm/classfile/javaClasses.hpp +@@ -201,6 +201,30 @@ + friend class JavaClasses; + }; + ++#define TAG_INJECTED_FIELDS(macro) \ ++ macro(sun_misc_taggedarray_TaggedArrayNativeImpl, data, int_signature, false) ++/* data must be last */ ++ ++class sun_misc_taggedarray_TaggedArrayNativeImpl : public AllStatic { ++ friend class VMStructs; ++ private: ++ static int _length_offset; ++ static int _data_offset; ++ static bool offsets_computed; ++ public: ++ static int length_offset() { return _length_offset; } ++ static void compute_offsets(); ++ static void set_slot_value(oop java_class, jint slot, jlong value); ++ static jlong get_slot_value(oop java_class, jint slot); ++ static void set_length(oop java_class, jint length); ++ static int slot_size_modifier() { return BytesPerLong / BytesPerWord; } ++ static int slot_offset() { return _data_offset; } ++ static oop get_slot_ref(oop java_class, jint slot); ++ static void set_slot_ref(oop java_class, jint slot, oop value); ++ static jboolean is_slot_tagged(oop bag, jint slot); ++ static int slots_count(oop bag); ++ static void copy_slot(oop dst, oop src, jint dstpos, jint srcpos); ++}; + + // Interface to java.lang.Class objects + +@@ -1285,7 +1309,8 @@ + + #define ALL_INJECTED_FIELDS(macro) \ + CLASS_INJECTED_FIELDS(macro) \ +- MEMBERNAME_INJECTED_FIELDS(macro) ++ MEMBERNAME_INJECTED_FIELDS(macro) \ ++ TAG_INJECTED_FIELDS(macro) + + // Interface to hard-coded offset checking + +diff --git a/src/share/vm/classfile/systemDictionary.hpp b/src/share/vm/classfile/systemDictionary.hpp +--- a/src/share/vm/classfile/systemDictionary.hpp ++++ b/src/share/vm/classfile/systemDictionary.hpp +@@ -169,6 +169,7 @@ + template(DownloadManager_klass, sun_jkernel_DownloadManager, Opt_Kernel) \ + \ + template(PostVMInitHook_klass, sun_misc_PostVMInitHook, Opt) \ ++ template(TaggedArrayNativeImpl_klass, sun_misc_taggedarray_TaggedArrayNativeImpl, Opt) \ + \ + /* Preload boxing klasses */ \ + template(Boolean_klass, java_lang_Boolean, Pre) \ +diff --git a/src/share/vm/classfile/vmSymbols.hpp b/src/share/vm/classfile/vmSymbols.hpp +--- a/src/share/vm/classfile/vmSymbols.hpp ++++ b/src/share/vm/classfile/vmSymbols.hpp +@@ -110,6 +110,9 @@ + template(sun_jkernel_DownloadManager, "sun/jkernel/DownloadManager") \ + template(getBootClassPathEntryForClass_name, "getBootClassPathEntryForClass") \ + template(sun_misc_PostVMInitHook, "sun/misc/PostVMInitHook") \ ++ template(sun_misc_taggedarray_TaggedArrayNativeVMImpl, "sun/misc/taggedarray/TaggedArrayNativeVMImpl") \ ++ template(sun_misc_taggedarray_TaggedArrays, "sun/misc/taggedarray/TaggedArrays") \ ++ template(sun_misc_taggedarray_TaggedArrayNativeImpl,"sun/misc/taggedarray/TaggedArrayNativeImpl") \ + \ + /* Java runtime version access */ \ + template(sun_misc_Version, "sun/misc/Version") \ +@@ -372,6 +375,8 @@ + template(resolved_constructor_name, "resolved_constructor") \ + template(array_klass_name, "array_klass") \ + template(oop_size_name, "oop_size") \ ++ template(data_name, "data") \ ++ template(length_name, "length") \ + template(static_oop_field_count_name, "static_oop_field_count") \ + \ + /* non-intrinsic name/signature pairs: */ \ +@@ -393,6 +398,7 @@ + template(void_double_signature, "()D") \ + template(int_void_signature, "(I)V") \ + template(int_int_signature, "(I)I") \ ++ template(int_long_signature, "(I)J") \ + template(char_char_signature, "(C)C") \ + template(short_short_signature, "(S)S") \ + template(int_bool_signature, "(I)Z") \ +@@ -471,6 +477,14 @@ + template(int_StringBuffer_signature, "(I)Ljava/lang/StringBuffer;") \ + template(char_StringBuffer_signature, "(C)Ljava/lang/StringBuffer;") \ + template(int_String_signature, "(I)Ljava/lang/String;") \ ++ template(Tagged_index_bool_signature, "(Lsun/misc/taggedarray/TaggedArrayNativeImpl;I)Z") \ ++ template(Tagged_index_long_signature, "(Lsun/misc/taggedarray/TaggedArrayNativeImpl;I)J") \ ++ template(Tagged_index_Object_signature, "(Lsun/misc/taggedarray/TaggedArrayNativeImpl;I)Ljava/lang/Object;") \ ++ template(Tagged_index_long_void_signature, "(Lsun/misc/taggedarray/TaggedArrayNativeImpl;IJ)V") \ ++ template(Tagged_index_Object_void_signature, "(Lsun/misc/taggedarray/TaggedArrayNativeImpl;ILjava/lang/Object;)V") \ ++ template(Tagged_fillObject_signature, "(Lsun/misc/taggedarray/TaggedArrayNativeImpl;IILjava/lang/Object;)V") \ ++ template(Tagged_copy_signature, "(Lsun/misc/taggedarray/TaggedArrayNativeImpl;ILsun/misc/taggedarray/TaggedArrayNativeImpl;II)V") \ ++ template(int_Tagged_signature, "(I)Lsun/misc/taggedarray/TaggedArray;") \ + /* signature symbols needed by intrinsics */ \ + VM_INTRINSICS_DO(VM_INTRINSIC_IGNORE, VM_SYMBOL_IGNORE, VM_SYMBOL_IGNORE, template, VM_ALIAS_IGNORE) \ + \ +@@ -719,6 +733,28 @@ + /* java/lang/ref/Reference */ \ + do_intrinsic(_Reference_get, java_lang_ref_Reference, get_name, void_object_signature, F_R) \ + \ ++ do_intrinsic(_TaggedArrays_doesVMImplement, sun_misc_taggedarray_TaggedArrays, doesVMImplement_name, void_boolean_signature, F_S) \ ++ do_name(doesVMImplement_name, "doesVMImplement") \ ++ do_intrinsic(_VMTaggedArray_isReference, sun_misc_taggedarray_TaggedArrayNativeVMImpl, isReference_name, Tagged_index_bool_signature, F_SN) \ ++ do_name(isReference_name, "isReference") \ ++ do_intrinsic(_VMTaggedArray_getValue, sun_misc_taggedarray_TaggedArrayNativeVMImpl, getValue_name, Tagged_index_long_signature, F_SN) \ ++ do_name(getValue_name, "getValue") \ ++ do_intrinsic(_VMTaggedArray_getReference, sun_misc_taggedarray_TaggedArrayNativeVMImpl, getReference_name, Tagged_index_Object_signature, F_SN) \ ++ do_name(getReference_name, "getReference") \ ++ do_intrinsic(_VMTaggedArray_setValue, sun_misc_taggedarray_TaggedArrayNativeVMImpl, setValue_name, Tagged_index_long_void_signature, F_SN) \ ++ do_name(setValue_name, "setValue") \ ++ do_intrinsic(_VMTaggedArray_setReference, sun_misc_taggedarray_TaggedArrayNativeVMImpl, setReference_name, Tagged_index_Object_void_signature, F_SN) \ ++ do_name(setReference_name, "setReference") \ ++ do_intrinsic(_VMTaggedArray_fillObject, sun_misc_taggedarray_TaggedArrayNativeVMImpl, fillObject_name, Tagged_fillObject_signature, F_SN) \ ++ do_name(fillObject_name, "fillObject") \ ++ do_intrinsic(_VMTaggedArray_copyFrontToBack, sun_misc_taggedarray_TaggedArrayNativeVMImpl, copyFrontToBack_name, Tagged_copy_signature, F_S) \ ++ do_name(copyFrontToBack_name, "copyFrontToBack") \ ++ do_intrinsic(_VMTaggedArray_copyBackToFront, sun_misc_taggedarray_TaggedArrayNativeVMImpl, copyBackToFront_name, Tagged_copy_signature, F_S) \ ++ do_name(copyBackToFront_name, "copyBackToFront") \ ++ do_intrinsic(_VMTaggedArray_allocate, sun_misc_taggedarray_TaggedArrayNativeVMImpl, allocate_name, int_Tagged_signature, F_SN) \ ++ do_name(allocate_name, "allocate") \ ++ \ ++ \ + /* support for sun.misc.Unsafe */ \ + do_class(sun_misc_Unsafe, "sun/misc/Unsafe") \ + \ +diff --git a/src/share/vm/interpreter/rewriter.cpp b/src/share/vm/interpreter/rewriter.cpp +--- a/src/share/vm/interpreter/rewriter.cpp ++++ b/src/share/vm/interpreter/rewriter.cpp +@@ -277,6 +277,18 @@ + const address code_base = method->code_base(); + const int code_length = method->code_size(); + ++ // HACK HACK ++ if (NativeTaggedArrays) { ++ if (_klass->name() == vmSymbols::sun_misc_taggedarray_TaggedArrays()) { ++ if (method->intrinsic_id() == vmIntrinsics::_TaggedArrays_doesVMImplement) { ++ c = (Bytecodes::Code)(*code_base); ++ if (c == Bytecodes::_iconst_0) { ++ *code_base = Bytecodes::_iconst_1; ++ } ++ } ++ } ++ } ++ + int bc_length; + for (int bci = 0; bci < code_length; bci += bc_length) { + address bcp = code_base + bci; +diff --git a/src/share/vm/memory/universe.cpp b/src/share/vm/memory/universe.cpp +--- a/src/share/vm/memory/universe.cpp ++++ b/src/share/vm/memory/universe.cpp +@@ -54,6 +54,7 @@ + #include "oops/instanceMirrorKlass.hpp" + #include "oops/instanceKlassKlass.hpp" + #include "oops/instanceRefKlass.hpp" ++#include "oops/instanceTagKlass.hpp" + #include "oops/klassKlass.hpp" + #include "oops/klassOop.hpp" + #include "oops/methodDataKlass.hpp" +@@ -536,6 +537,7 @@ + { objArrayKlass o; add_vtable(list, &n, &o, count); } + { methodDataKlass o; add_vtable(list, &n, &o, count); } + { compiledICHolderKlass o; add_vtable(list, &n, &o, count); } ++ { instanceTagKlass o; add_vtable(list, &n, &o, count); } + #ifndef PRODUCT + // In non-product builds CHeapObj is derived from AllocatedObj, + // so symbols in CDS archive should have their vtable pointer patched. +diff --git a/src/share/vm/oops/instanceKlass.cpp b/src/share/vm/oops/instanceKlass.cpp +--- a/src/share/vm/oops/instanceKlass.cpp ++++ b/src/share/vm/oops/instanceKlass.cpp +@@ -2426,6 +2426,8 @@ + } else if (as_klassOop() == SystemDictionary::MethodType_klass()) { + st->print(" = "); + java_lang_invoke_MethodType::print_signature(obj, st); ++ } else if (as_klassOop() == SystemDictionary::TaggedArrayNativeImpl_klass()) { ++ st->print(" = [unimplemented]"); + } else if (java_lang_boxing_object::is_instance(obj)) { + st->print(" = "); + java_lang_boxing_object::print(obj, st); +diff --git a/src/share/vm/oops/instanceKlassKlass.cpp b/src/share/vm/oops/instanceKlassKlass.cpp +--- a/src/share/vm/oops/instanceKlassKlass.cpp ++++ b/src/share/vm/oops/instanceKlassKlass.cpp +@@ -32,6 +32,7 @@ + #include "oops/constantPoolOop.hpp" + #include "oops/instanceKlass.hpp" + #include "oops/instanceMirrorKlass.hpp" ++#include "oops/instanceTagKlass.hpp" + #include "oops/instanceKlassKlass.hpp" + #include "oops/instanceRefKlass.hpp" + #include "oops/objArrayKlassKlass.hpp" +@@ -379,14 +380,18 @@ + KlassHandle h_this_klass(THREAD, as_klassOop()); + KlassHandle k; + if (rt == REF_NONE) { +- if (name != vmSymbols::java_lang_Class()) { ++ if (name == vmSymbols::java_lang_Class()) { ++ // Class ++ instanceMirrorKlass o; ++ k = base_create_klass(h_this_klass, size, o.vtbl_value(), CHECK_NULL); ++ } else if (name == vmSymbols::sun_misc_taggedarray_TaggedArrayNativeImpl()) { ++ // sun_misc_taggedarray_TaggedArrayNativeImpl ++ instanceTagKlass o; ++ k = base_create_klass(h_this_klass, size, o.vtbl_value(), CHECK_NULL); ++ } else { + // regular klass + instanceKlass o; + k = base_create_klass(h_this_klass, size, o.vtbl_value(), CHECK_NULL); +- } else { +- // Class +- instanceMirrorKlass o; +- k = base_create_klass(h_this_klass, size, o.vtbl_value(), CHECK_NULL); + } + } else { + // reference klass +diff --git a/src/share/vm/oops/instanceTagKlass.cpp b/src/share/vm/oops/instanceTagKlass.cpp +new file mode 100644 +--- /dev/null ++++ b/src/share/vm/oops/instanceTagKlass.cpp +@@ -0,0 +1,360 @@ ++/* ++ * Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved. ++ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. ++ * ++ * This code is free software; you can redistribute it and/or modify it ++ * under the terms of the GNU General Public License version 2 only, as ++ * published by the Free Software Foundation. ++ * ++ * This code is distributed in the hope that it will be useful, but WITHOUT ++ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ++ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License ++ * version 2 for more details (a copy is included in the LICENSE file that ++ * accompanied this code). ++ * ++ * You should have received a copy of the GNU General Public License version ++ * 2 along with this work; if not, write to the Free Software Foundation, ++ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. ++ * ++ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA ++ * or visit www.oracle.com if you need additional information or have any ++ * questions. ++ * ++ */ ++ ++#include "precompiled.hpp" ++#include "classfile/javaClasses.hpp" ++#include "classfile/systemDictionary.hpp" ++#include "gc_implementation/shared/markSweep.inline.hpp" ++#include "gc_interface/collectedHeap.inline.hpp" ++#include "memory/genOopClosures.inline.hpp" ++#include "memory/oopFactory.hpp" ++#include "memory/permGen.hpp" ++#include "oops/instanceKlass.hpp" ++#include "oops/instanceTagKlass.hpp" ++#include "oops/instanceOop.hpp" ++#include "oops/oop.inline.hpp" ++#include "oops/symbol.hpp" ++#include "runtime/handles.inline.hpp" ++#ifndef SERIALGC ++#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" ++#include "gc_implementation/g1/g1OopClosures.inline.hpp" ++#include "gc_implementation/g1/g1RemSet.inline.hpp" ++#include "gc_implementation/g1/heapRegionSeq.inline.hpp" ++#include "gc_implementation/parNew/parOopClosures.inline.hpp" ++#include "gc_implementation/parallelScavenge/psPromotionManager.inline.hpp" ++#include "gc_implementation/parallelScavenge/psScavenge.inline.hpp" ++#include "oops/oop.pcgc.inline.hpp" ++#endif ++ ++int instanceTagKlass::_offset_of_static_fields = 0; ++ ++#ifdef ASSERT ++template void assert_is_in(T *p) { ++ T heap_oop = oopDesc::load_heap_oop(p); ++ if (!oopDesc::is_null(heap_oop)) { ++ oop o = oopDesc::decode_heap_oop_not_null(heap_oop); ++ assert(Universe::heap()->is_in(o), "should be in heap"); ++ } ++} ++template void assert_is_in_closed_subset(T *p) { ++ T heap_oop = oopDesc::load_heap_oop(p); ++ if (!oopDesc::is_null(heap_oop)) { ++ oop o = oopDesc::decode_heap_oop_not_null(heap_oop); ++ assert(Universe::heap()->is_in_closed_subset(o), "should be in closed"); ++ } ++} ++template void assert_is_in_reserved(T *p) { ++ T heap_oop = oopDesc::load_heap_oop(p); ++ if (!oopDesc::is_null(heap_oop)) { ++ oop o = oopDesc::decode_heap_oop_not_null(heap_oop); ++ assert(Universe::heap()->is_in_reserved(o), "should be in reserved"); ++ } ++} ++template void assert_nothing(T *p) {} ++ ++#else ++template void assert_is_in(T *p) {} ++template void assert_is_in_closed_subset(T *p) {} ++template void assert_is_in_reserved(T *p) {} ++template void assert_nothing(T *p) {} ++#endif // ASSERT ++ ++#define NON_ZERO_OOP(v) (*(u8 *) v) != 0x0 ++#define IS_TAGGED(v) ((*(u8 *) v) & 0x1) == 0x0 ++ ++#define InstanceTagKlass_SPECIALIZED_OOP_ITERATE_COMPRESSED( \ ++ T, start_p, count, do_oop, \ ++ assert_fn) \ ++{ \ ++ T* v = (T*)(start_p); \ ++ void* const end = ((u8 *) v) + (count); \ ++ while (v < end) { \ ++ if (NON_ZERO_OOP(v) && IS_TAGGED(v)) { \ ++ T* p = (T*) (((T*) v) + 1); \ ++ (assert_fn)(p); \ ++ do_oop; \ ++ } else { \ ++ } \ ++ v += 2; \ ++ } \ ++} ++ ++ ++#define InstanceTagKlass_SPECIALIZED_OOP_ITERATE( \ ++ T, start_p, count, do_oop, \ ++ assert_fn) \ ++{ \ ++ T* p = (T*)(start_p); \ ++ T* const end = p + (count); \ ++ while (p < end) { \ ++ if (NON_ZERO_OOP(p) && IS_TAGGED(p)) { \ ++ (assert_fn)(p); \ ++ do_oop; \ ++ } else { \ ++ } \ ++ ++p; \ ++ } \ ++} ++ ++#define InstanceTagKlass_SPECIALIZED_BOUNDED_OOP_ITERATE_COMPRESSED( \ ++ T, start_p, count, low, high, \ ++ do_oop, assert_fn) \ ++{ \ ++ T* const l = (T*)(low); \ ++ T* const h = (T*)(high); \ ++ assert(mask_bits((intptr_t)l, sizeof(T)-1) == 0 && \ ++ mask_bits((intptr_t)h, sizeof(T)-1) == 0, \ ++ "bounded region must be properly aligned"); \ ++ T* v = (T*)(start_p); \ ++ void* end = ((u8 *) v) + (count); \ ++ if (v < l) v = l; \ ++ if (end > h) end = h; \ ++ while (v < end) { \ ++ if (NON_ZERO_OOP(v) && IS_TAGGED(v)) { \ ++ T* p = (T*) (((T*) v) + 1); \ ++ (assert_fn)(p); \ ++ do_oop; \ ++ } else { \ ++ } \ ++ v += 2; \ ++ } \ ++} ++ ++ ++#define InstanceTagKlass_SPECIALIZED_BOUNDED_OOP_ITERATE( \ ++ T, start_p, count, low, high, \ ++ do_oop, assert_fn) \ ++{ \ ++ T* const l = (T*)(low); \ ++ T* const h = (T*)(high); \ ++ assert(mask_bits((intptr_t)l, sizeof(T)-1) == 0 && \ ++ mask_bits((intptr_t)h, sizeof(T)-1) == 0, \ ++ "bounded region must be properly aligned"); \ ++ T* p = (T*)(start_p); \ ++ T* end = p + (count); \ ++ if (p < l) p = l; \ ++ if (end > h) end = h; \ ++ while (p < end) { \ ++ if (NON_ZERO_OOP(p) && IS_TAGGED(p)) { \ ++ (assert_fn)(p); \ ++ do_oop; \ ++ } else { \ ++ } \ ++ ++p; \ ++ } \ ++} ++ ++ ++#define InstanceTagKlass_OOP_ITERATE(start_p, count, \ ++ do_oop, assert_fn) \ ++{ \ ++ if (UseCompressedOops) { \ ++ InstanceTagKlass_SPECIALIZED_OOP_ITERATE_COMPRESSED(narrowOop, \ ++ start_p, count, \ ++ do_oop, assert_fn) \ ++ } else { \ ++ InstanceTagKlass_SPECIALIZED_OOP_ITERATE(oop, \ ++ start_p, count, \ ++ do_oop, assert_fn) \ ++ } \ ++} ++ ++// The following macros call specialized macros, passing either oop or ++// narrowOop as the specialization type. These test the UseCompressedOops ++// flag. ++#define InstanceTagKlass_BOUNDED_OOP_ITERATE(start_p, count, low, high, \ ++ do_oop, assert_fn) \ ++{ \ ++ if (UseCompressedOops) { \ ++ InstanceTagKlass_SPECIALIZED_BOUNDED_OOP_ITERATE_COMPRESSED(narrowOop, \ ++ start_p, count, \ ++ low, high, \ ++ do_oop, assert_fn) \ ++ } else { \ ++ InstanceTagKlass_SPECIALIZED_BOUNDED_OOP_ITERATE(oop, \ ++ start_p, count, \ ++ low, high, \ ++ do_oop, assert_fn) \ ++ } \ ++} ++ ++ ++void instanceTagKlass::oop_follow_contents(oop obj) { ++ instanceKlass::oop_follow_contents(obj); ++ InstanceTagKlass_OOP_ITERATE( \ ++ start_of_slots(obj), sun_misc_taggedarray_TaggedArrayNativeImpl::slots_count(obj), \ ++ MarkSweep::mark_and_push(p), \ ++ assert_is_in_closed_subset) ++} ++ ++#ifndef SERIALGC ++void instanceTagKlass::oop_follow_contents(ParCompactionManager* cm, ++ oop obj) { ++ instanceKlass::oop_follow_contents(cm, obj); ++ InstanceTagKlass_OOP_ITERATE( \ ++ start_of_slots(obj), sun_misc_taggedarray_TaggedArrayNativeImpl::slots_count(obj), \ ++ PSParallelCompact::mark_and_push(cm, p), \ ++ assert_is_in) ++} ++#endif // SERIALGC ++ ++int instanceTagKlass::oop_adjust_pointers(oop obj) { ++ int size = oop_size(obj); ++ instanceKlass::oop_adjust_pointers(obj); ++ InstanceTagKlass_OOP_ITERATE( \ ++ start_of_slots(obj), sun_misc_taggedarray_TaggedArrayNativeImpl::slots_count(obj), \ ++ MarkSweep::adjust_pointer(p), \ ++ assert_nothing) ++ return size; ++} ++ ++#define InstanceTagKlass_SPECIALIZED_OOP_ITERATE_DEFN(T, nv_suffix) \ ++ InstanceTagKlass_OOP_ITERATE( \ ++ start_of_slots(obj), sun_misc_taggedarray_TaggedArrayNativeImpl::slots_count(obj), \ ++ (closure)->do_oop##nv_suffix(p), \ ++ assert_is_in_closed_subset) \ ++ return oop_size(obj); \ ++ ++#define InstanceTagKlass_BOUNDED_SPECIALIZED_OOP_ITERATE(T, nv_suffix, mr) \ ++ InstanceTagKlass_BOUNDED_OOP_ITERATE( \ ++ start_of_slots(obj), sun_misc_taggedarray_TaggedArrayNativeImpl::slots_count(obj), \ ++ mr.start(), mr.end(), \ ++ (closure)->do_oop##nv_suffix(p), \ ++ assert_is_in_closed_subset) \ ++ return oop_size(obj); \ ++ ++ ++// Macro to define instanceTagKlass::oop_oop_iterate for virtual/nonvirtual for ++// all closures. Macros calling macros above for each oop size. ++ ++#define InstanceTagKlass_OOP_OOP_ITERATE_DEFN(OopClosureType, nv_suffix) \ ++ \ ++int instanceTagKlass:: \ ++oop_oop_iterate##nv_suffix(oop obj, OopClosureType* closure) { \ ++ /* Get size before changing pointers */ \ ++ SpecializationStats::record_iterate_call##nv_suffix(SpecializationStats::irk); \ ++ \ ++ instanceKlass::oop_oop_iterate##nv_suffix(obj, closure); \ ++ \ ++ if (UseCompressedOops) { \ ++ InstanceTagKlass_SPECIALIZED_OOP_ITERATE_DEFN(narrowOop, nv_suffix); \ ++ } else { \ ++ InstanceTagKlass_SPECIALIZED_OOP_ITERATE_DEFN(oop, nv_suffix); \ ++ } \ ++} ++ ++#ifndef SERIALGC ++#define InstanceTagKlass_OOP_OOP_ITERATE_BACKWARDS_DEFN(OopClosureType, nv_suffix) \ ++ \ ++int instanceTagKlass:: \ ++oop_oop_iterate_backwards##nv_suffix(oop obj, OopClosureType* closure) { \ ++ /* Get size before changing pointers */ \ ++ SpecializationStats::record_iterate_call##nv_suffix(SpecializationStats::irk); \ ++ \ ++ instanceKlass::oop_oop_iterate_backwards##nv_suffix(obj, closure); \ ++ \ ++ if (UseCompressedOops) { \ ++ InstanceTagKlass_SPECIALIZED_OOP_ITERATE_DEFN(narrowOop, nv_suffix); \ ++ } else { \ ++ InstanceTagKlass_SPECIALIZED_OOP_ITERATE_DEFN(oop, nv_suffix); \ ++ } \ ++} ++#endif // !SERIALGC ++ ++ ++#define InstanceTagKlass_OOP_OOP_ITERATE_DEFN_m(OopClosureType, nv_suffix) \ ++ \ ++int instanceTagKlass:: \ ++oop_oop_iterate##nv_suffix##_m(oop obj, \ ++ OopClosureType* closure, \ ++ MemRegion mr) { \ ++ SpecializationStats::record_iterate_call##nv_suffix(SpecializationStats::irk); \ ++ \ ++ instanceKlass::oop_oop_iterate##nv_suffix##_m(obj, closure, mr); \ ++ if (UseCompressedOops) { \ ++ InstanceTagKlass_BOUNDED_SPECIALIZED_OOP_ITERATE(narrowOop, nv_suffix, mr); \ ++ } else { \ ++ InstanceTagKlass_BOUNDED_SPECIALIZED_OOP_ITERATE(oop, nv_suffix, mr); \ ++ } \ ++} ++ ++ALL_OOP_OOP_ITERATE_CLOSURES_1(InstanceTagKlass_OOP_OOP_ITERATE_DEFN) ++ALL_OOP_OOP_ITERATE_CLOSURES_2(InstanceTagKlass_OOP_OOP_ITERATE_DEFN) ++#ifndef SERIALGC ++ALL_OOP_OOP_ITERATE_CLOSURES_1(InstanceTagKlass_OOP_OOP_ITERATE_BACKWARDS_DEFN) ++ALL_OOP_OOP_ITERATE_CLOSURES_2(InstanceTagKlass_OOP_OOP_ITERATE_BACKWARDS_DEFN) ++#endif // SERIALGC ++ALL_OOP_OOP_ITERATE_CLOSURES_1(InstanceTagKlass_OOP_OOP_ITERATE_DEFN_m) ++ALL_OOP_OOP_ITERATE_CLOSURES_2(InstanceTagKlass_OOP_OOP_ITERATE_DEFN_m) ++ ++#ifndef SERIALGC ++void instanceTagKlass::oop_push_contents(PSPromotionManager* pm, oop obj) { ++ instanceKlass::oop_push_contents(pm, obj); ++ InstanceTagKlass_OOP_ITERATE( \ ++ start_of_slots(obj), sun_misc_taggedarray_TaggedArrayNativeImpl::slots_count(obj),\ ++ if (PSScavenge::should_scavenge(p)) { \ ++ pm->claim_or_forward_depth(p); \ ++ }, \ ++ assert_nothing ) ++} ++ ++int instanceTagKlass::oop_update_pointers(ParCompactionManager* cm, oop obj) { ++ instanceKlass::oop_update_pointers(cm, obj); ++ InstanceTagKlass_OOP_ITERATE( \ ++ start_of_slots(obj), sun_misc_taggedarray_TaggedArrayNativeImpl::slots_count(obj),\ ++ PSParallelCompact::adjust_pointer(p), \ ++ assert_nothing) ++ return oop_size(obj); ++} ++#endif // SERIALGC ++ ++int instanceTagKlass::instance_size(int slots) const { ++ return size_helper() + sun_misc_taggedarray_TaggedArrayNativeImpl::slot_size_modifier() * slots; //8 + 8 * slots; ++} ++ ++instanceOop instanceTagKlass::allocate_instance(int slots, TRAPS) { ++ // Query before forming handle. ++ int size = instance_size(slots); ++ KlassHandle h_k(THREAD, as_klassOop()); ++ instanceOop i = (instanceOop) CollectedHeap::obj_allocate(h_k, size, CHECK_NULL); ++ return i; ++} ++ ++int instanceTagKlass::oop_size(oop obj) const { ++ int size = instance_size(sun_misc_taggedarray_TaggedArrayNativeImpl::slot_size_modifier() * sun_misc_taggedarray_TaggedArrayNativeImpl::slots_count(obj)); ++ return size; ++} ++ ++int instanceTagKlass::klass_oop_size() const { ++ int size = object_size(); ++ return size; ++} ++ ++int instanceTagKlass::compute_static_oop_field_count(oop obj) { ++ klassOop k = java_lang_Class::as_klassOop(obj); ++ if (k != NULL && k->klass_part()->oop_is_instance()) { ++ return instanceKlass::cast(k)->static_oop_field_count(); ++ } ++ return 0; ++} +diff --git a/src/share/vm/oops/instanceTagKlass.hpp b/src/share/vm/oops/instanceTagKlass.hpp +new file mode 100644 +--- /dev/null ++++ b/src/share/vm/oops/instanceTagKlass.hpp +@@ -0,0 +1,112 @@ ++/* ++ * Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved. ++ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. ++ * ++ * This code is free software; you can redistribute it and/or modify it ++ * under the terms of the GNU General Public License version 2 only, as ++ * published by the Free Software Foundation. ++ * ++ * This code is distributed in the hope that it will be useful, but WITHOUT ++ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ++ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License ++ * version 2 for more details (a copy is included in the LICENSE file that ++ * accompanied this code). ++ * ++ * You should have received a copy of the GNU General Public License version ++ * 2 along with this work; if not, write to the Free Software Foundation, ++ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. ++ * ++ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA ++ * or visit www.oracle.com if you need additional information or have any ++ * questions. ++ * ++ */ ++ ++#ifndef SHARE_VM_OOPS_INSTANCETAGKLASS_HPP ++#define SHARE_VM_OOPS_INSTANCETAGKLASS_HPP ++ ++#include "classfile/javaClasses.hpp" ++#include "oops/instanceKlass.hpp" ++ ++// An instanceTagKlass is a specialized instanceKlass for ++// java.lang.Class instances. These instances are special because ++// they contain the static fields of the class in addition to the ++// normal fields of Class. This means they are variable sized ++// instances and need special logic for computing their size and for ++// iteration of their oops. ++ ++ ++class instanceTagKlass: public instanceKlass { ++ friend class VMStructs; ++ ++ private: ++ static int _offset_of_static_fields; ++ ++ public: ++ // Type testing ++ bool oop_is_instanceTag() const { return true; } ++ ++ // Casting from klassOop ++ static instanceTagKlass* cast(klassOop k) { ++ assert(k->klass_part()->oop_is_instanceTag(), "cast to instanceTagKlass"); ++ return (instanceTagKlass*) k->klass_part(); ++ } ++ ++ // Returns the size of the instance including the extra static fields. ++ virtual int oop_size(oop obj) const; ++ int klass_oop_size() const; ++ ++ // Static field offset is an offset into the Heap, should be converted by ++ // based on UseCompressedOop for traversal ++ static HeapWord* start_of_slots(oop obj) { ++ return (HeapWord*)((intptr_t)obj + sun_misc_taggedarray_TaggedArrayNativeImpl::slot_offset()); ++ } ++ ++ static int data_offset() { ++ return sun_misc_taggedarray_TaggedArrayNativeImpl::slot_offset(); ++ } ++ ++ static int offset_of_static_fields() { ++ return _offset_of_static_fields; ++ } ++ ++ int compute_static_oop_field_count(oop obj); ++ ++ // Given a Klass return the size of the instance ++ int instance_size(int slots) const; ++ ++ // allocation ++ DEFINE_ALLOCATE_PERMANENT(instanceTagKlass); ++ instanceOop allocate_instance(int slots, TRAPS); ++ ++ // Garbage collection ++ int oop_adjust_pointers(oop obj); ++ void oop_follow_contents(oop obj); ++ ++ // Parallel Scavenge and Parallel Old ++ PARALLEL_GC_DECLS ++ ++ int oop_oop_iterate(oop obj, OopClosure* blk) { ++ return oop_oop_iterate_v(obj, blk); ++ } ++ int oop_oop_iterate_m(oop obj, OopClosure* blk, MemRegion mr) { ++ return oop_oop_iterate_v_m(obj, blk, mr); ++ } ++ ++#define InstanceTagKlass_OOP_OOP_ITERATE_DECL(OopClosureType, nv_suffix) \ ++ int oop_oop_iterate##nv_suffix(oop obj, OopClosureType* blk); \ ++ int oop_oop_iterate##nv_suffix##_m(oop obj, OopClosureType* blk, MemRegion mr); ++ ++ ALL_OOP_OOP_ITERATE_CLOSURES_1(InstanceTagKlass_OOP_OOP_ITERATE_DECL) ++ ALL_OOP_OOP_ITERATE_CLOSURES_2(InstanceTagKlass_OOP_OOP_ITERATE_DECL) ++ ++#ifndef SERIALGC ++#define InstanceTagKlass_OOP_OOP_ITERATE_BACKWARDS_DECL(OopClosureType, nv_suffix) \ ++ int oop_oop_iterate_backwards##nv_suffix(oop obj, OopClosureType* blk); ++ ++ ALL_OOP_OOP_ITERATE_CLOSURES_1(InstanceTagKlass_OOP_OOP_ITERATE_BACKWARDS_DECL) ++ ALL_OOP_OOP_ITERATE_CLOSURES_2(InstanceTagKlass_OOP_OOP_ITERATE_BACKWARDS_DECL) ++#endif // !SERIALGC ++}; ++ ++#endif // SHARE_VM_OOPS_INSTANCEMIRRORKLASS_HPP +diff --git a/src/share/vm/oops/klass.hpp b/src/share/vm/oops/klass.hpp +--- a/src/share/vm/oops/klass.hpp ++++ b/src/share/vm/oops/klass.hpp +@@ -597,6 +597,7 @@ + virtual bool oop_is_typeArrayKlass() const { return false; } + virtual bool oop_is_compiledICHolder() const { return false; } + virtual bool oop_is_instanceKlass() const { return false; } ++ virtual bool oop_is_instanceTag() const { return false; } + + bool oop_is_javaArray_slow() const { + return oop_is_objArray_slow() || oop_is_typeArray_slow(); +diff --git a/src/share/vm/opto/c2_globals.hpp b/src/share/vm/opto/c2_globals.hpp +--- a/src/share/vm/opto/c2_globals.hpp ++++ b/src/share/vm/opto/c2_globals.hpp +@@ -484,6 +484,12 @@ + product(bool, OptimizeStringConcat, true, \ + "Optimize the construction of Strings by StringBuilder") \ + \ ++ product(bool, TaggedArrayIntrinsics, false, \ ++ "Use intrinsics for the methods in TaggedArray") \ ++ \ ++ product(bool, NativeTaggedArrays, false, \ ++ "Support NativeTaggedArrays") \ ++ \ + notproduct(bool, PrintOptimizeStringConcat, false, \ + "Print information about transformations performed on Strings") \ + \ +diff --git a/src/share/vm/opto/callnode.cpp b/src/share/vm/opto/callnode.cpp +--- a/src/share/vm/opto/callnode.cpp ++++ b/src/share/vm/opto/callnode.cpp +@@ -1118,7 +1118,8 @@ + + AllocateNode::AllocateNode(Compile* C, const TypeFunc *atype, + Node *ctrl, Node *mem, Node *abio, +- Node *size, Node *klass_node, Node *initial_test) ++ Node *size, Node *klass_node, Node *initial_test, ++ Node *slots) + : CallNode(atype, NULL, TypeRawPtr::BOTTOM) + { + init_class_id(Class_Allocate); +@@ -1135,6 +1136,10 @@ + init_req( KlassNode , klass_node); + init_req( InitialTest , initial_test); + init_req( ALength , topnode); ++ if (slots == NULL) { ++ slots = topnode; ++ } ++ init_req( Slots , slots); + C->add_macro_node(this); + } + +diff --git a/src/share/vm/opto/callnode.hpp b/src/share/vm/opto/callnode.hpp +--- a/src/share/vm/opto/callnode.hpp ++++ b/src/share/vm/opto/callnode.hpp +@@ -730,6 +730,7 @@ + KlassNode, // type (maybe dynamic) of the obj. + InitialTest, // slow-path test (may be constant) + ALength, // array length (or TOP if none) ++ Slots, // number of slots for taggedarrays + ParmLimit + }; + +@@ -739,6 +740,7 @@ + fields[KlassNode] = TypeInstPtr::NOTNULL; + fields[InitialTest] = TypeInt::BOOL; + fields[ALength] = TypeInt::INT; // length (can be a bad length) ++ fields[Slots] = TypeInt::INT; + + const TypeTuple *domain = TypeTuple::make(ParmLimit, fields); + +@@ -755,7 +757,7 @@ + + virtual uint size_of() const; // Size is bigger + AllocateNode(Compile* C, const TypeFunc *atype, Node *ctrl, Node *mem, Node *abio, +- Node *size, Node *klass_node, Node *initial_test); ++ Node *size, Node *klass_node, Node *initial_test, Node* slots = NULL); + // Expansion modifies the JVMState, so we need to clone it + virtual void clone_jvms() { + set_jvms(jvms()->clone_deep(Compile::current())); +diff --git a/src/share/vm/opto/escape.cpp b/src/share/vm/opto/escape.cpp +--- a/src/share/vm/opto/escape.cpp ++++ b/src/share/vm/opto/escape.cpp +@@ -870,7 +870,9 @@ + if (!(is_arraycopy || + call->as_CallLeaf()->_name != NULL && + (strcmp(call->as_CallLeaf()->_name, "g1_wb_pre") == 0 || +- strcmp(call->as_CallLeaf()->_name, "g1_wb_post") == 0 )) ++ strcmp(call->as_CallLeaf()->_name, "g1_wb_post") == 0 || ++ strcmp(call->as_CallLeaf()->_name, "tagged_copy") == 0 || ++ strcmp(call->as_CallLeaf()->_name, "tagged_fill") == 0)) + ) { + call->dump(); + assert(false, "EA: unexpected CallLeaf"); +diff --git a/src/share/vm/opto/graphKit.cpp b/src/share/vm/opto/graphKit.cpp +--- a/src/share/vm/opto/graphKit.cpp ++++ b/src/share/vm/opto/graphKit.cpp +@@ -3044,6 +3044,92 @@ + return javaoop; + } + ++Node* GraphKit::new_tagged(Node* klass_node, ++ Node* extra_slow_test, ++ Node* elements, ++ Node* *return_size_val) { ++ // Compute size in doublewords ++ // The size is always an integral number of doublewords, represented ++ // as a positive bytewise size stored in the klass's layout_helper. ++ // The layout_helper also encodes (in a low bit) the need for a slow path. ++ jint layout_con = Klass::_lh_neutral_value; ++ Node* layout_val = get_layout_helper(klass_node, layout_con); ++ int layout_is_con = (layout_val == NULL); ++ ++ if (extra_slow_test == NULL) extra_slow_test = intcon(0); ++ // Generate the initial go-slow test. It's either ALWAYS (return a ++ // Node for 1) or NEVER (return a NULL) or perhaps (in the reflective ++ // case) a computed value derived from the layout_helper. ++ Node* initial_slow_test = NULL; ++ if (layout_is_con) { ++ assert(!StressReflectiveCode, "stress mode does not use these paths"); ++ bool must_go_slow = false; //Klass::layout_helper_needs_slow_path(layout_con); ++ initial_slow_test = must_go_slow? intcon(1): extra_slow_test; ++ ++ } else { // reflective case ++ // This reflective path is used by Unsafe.allocateInstance. ++ // (It may be stress-tested by specifying StressReflectiveCode.) ++ // Basically, we want to get into the VM is there's an illegal argument. ++ Node* bit = intcon(Klass::_lh_instance_slow_path_bit); ++ initial_slow_test = _gvn.transform( new (C, 3) AndINode(layout_val, bit) ); ++ if (extra_slow_test != intcon(0)) { ++ initial_slow_test = _gvn.transform( new (C, 3) OrINode(initial_slow_test, extra_slow_test) ); ++ } ++ // (Macro-expander will further convert this to a Bool, if necessary.) ++ } ++ ++ // Find the size in bytes. This is easy; it's the layout_helper. ++ // The size value must be valid even if the slow path is taken. ++ Node* size = NULL; ++ assert(layout_is_con, "Layout should be con"); ++ size = MakeConX(Klass::layout_helper_size_in_bytes(layout_con)); ++ ++ int modifier = sun_misc_taggedarray_TaggedArrayNativeImpl::slot_size_modifier(); ++ ++ if (modifier == 1) { ++ //size = _gvn.transform( new(C,3) AddXNode(elements, size)); ++ Node* i = _gvn.transform( new(C,3) LShiftXNode(ConvI2X(elements), intcon(3) )); // * 8 ++ size = _gvn.transform( new(C,3) AddXNode(i, size)); ++ } else if (modifier == 2) { ++ Node* i = _gvn.transform( new(C,3) LShiftXNode(ConvI2X(elements), intcon(1) )); // * 2 ++ size = _gvn.transform( new(C,3) AddXNode(i, size)); ++ } else { ++ ShouldNotReachHere(); ++ } ++ ++ Node* mask = MakeConX(~ (intptr_t)right_n_bits(LogBytesPerLong)); ++ size = _gvn.transform( new (C, 3) AndXNode(size, mask) ); ++ ++ if (return_size_val != NULL) { ++ (*return_size_val) = size; ++ } ++ ++ // This is a precise notnull oop of the klass. ++ // (Actually, it need not be precise if this is a reflective allocation.) ++ // It's what we cast the result to. ++ const TypeKlassPtr* tklass = _gvn.type(klass_node)->isa_klassptr(); ++ if (!tklass) tklass = TypeKlassPtr::OBJECT; ++ const TypeOopPtr* oop_type = tklass->as_instance_type(); ++ ++ // Now generate allocation code ++ ++ // The entire memory state is needed for slow path of the allocation ++ // since GC and deoptimization can happened. ++ Node *mem = reset_memory(); ++ set_all_memory(mem); // Create new memory state ++ ++ AllocateNode* alloc ++ = new (C, AllocateNode::ParmLimit) ++ AllocateNode(C, AllocateNode::alloc_type(), ++ control(), mem, i_o(), ++ size, klass_node, ++ initial_slow_test, ++ elements); ++ ++ return set_output_for_allocation(alloc, oop_type); ++} ++ ++ + //---------------------------new_instance-------------------------------------- + // This routine takes a klass_node which may be constant (for a static type) + // or may be non-constant (for reflective code). It will work equally well +diff --git a/src/share/vm/opto/graphKit.hpp b/src/share/vm/opto/graphKit.hpp +--- a/src/share/vm/opto/graphKit.hpp ++++ b/src/share/vm/opto/graphKit.hpp +@@ -776,6 +776,11 @@ + Node* set_output_for_allocation(AllocateNode* alloc, + const TypeOopPtr* oop_type); + Node* get_layout_helper(Node* klass_node, jint& constant_value); ++ ++ Node* new_tagged(Node* klass_node, ++ Node* extra_slow_test, ++ Node* elements, ++ Node* *return_size_val); + Node* new_instance(Node* klass_node, + Node* slow_test = NULL, + Node* *return_size_val = NULL); +diff --git a/src/share/vm/opto/library_call.cpp b/src/share/vm/opto/library_call.cpp +--- a/src/share/vm/opto/library_call.cpp ++++ b/src/share/vm/opto/library_call.cpp +@@ -28,6 +28,7 @@ + #include "compiler/compileBroker.hpp" + #include "compiler/compileLog.hpp" + #include "oops/objArrayKlass.hpp" ++#include "oops/instanceTagKlass.hpp" + #include "opto/addnode.hpp" + #include "opto/callGenerator.hpp" + #include "opto/cfgnode.hpp" +@@ -177,6 +178,17 @@ + bool inline_unsafe_allocate(); + bool inline_unsafe_copyMemory(); + bool inline_native_currentThread(); ++ Node* inline_taggedarray_slot(Node* tagged, Node* slot, bool upperword); ++ bool inline_taggedarray_doesvmimplement(); ++ bool inline_taggedarray_isReference(); ++ bool inline_taggedarray_setValue(); ++ bool inline_taggedarray_setReference(); ++ bool inline_taggedarray_getValue(); ++ bool inline_taggedarray_getReference(); ++ bool inline_taggedarray_allocate(); ++ bool inline_taggedarray_fillObject(); ++ bool inline_taggedarray_copyFrontToBack(); ++ bool inline_taggedarray_copyBackToFront(); + #ifdef TRACE_HAVE_INTRINSICS + bool inline_native_classID(); + bool inline_native_threadID(); +@@ -369,6 +381,19 @@ + // across safepoint since GC can change it value. + break; + ++ case vmIntrinsics::_VMTaggedArray_isReference: ++ case vmIntrinsics::_VMTaggedArray_getValue: ++ case vmIntrinsics::_VMTaggedArray_setValue: ++ case vmIntrinsics::_VMTaggedArray_setReference: ++ case vmIntrinsics::_VMTaggedArray_getReference: ++ case vmIntrinsics::_VMTaggedArray_allocate: ++ case vmIntrinsics::_VMTaggedArray_copyFrontToBack: ++ case vmIntrinsics::_VMTaggedArray_copyBackToFront: ++ case vmIntrinsics::_VMTaggedArray_fillObject: ++ case vmIntrinsics::_TaggedArrays_doesVMImplement: ++ if (!NativeTaggedArrays || !TaggedArrayIntrinsics) return NULL; ++ break; ++ + default: + assert(id <= vmIntrinsics::LAST_COMPILER_INLINE, "caller responsibility"); + assert(id != vmIntrinsics::_Object_init && id != vmIntrinsics::_invoke, "enum out of order?"); +@@ -635,6 +660,26 @@ + + case vmIntrinsics::_currentThread: + return inline_native_currentThread(); ++ case vmIntrinsics::_TaggedArrays_doesVMImplement: ++ return inline_taggedarray_doesvmimplement(); ++ case vmIntrinsics::_VMTaggedArray_isReference: ++ return inline_taggedarray_isReference(); ++ case vmIntrinsics::_VMTaggedArray_getValue: ++ return inline_taggedarray_getValue(); ++ case vmIntrinsics::_VMTaggedArray_setValue: ++ return inline_taggedarray_setValue(); ++ case vmIntrinsics::_VMTaggedArray_setReference: ++ return inline_taggedarray_setReference(); ++ case vmIntrinsics::_VMTaggedArray_getReference: ++ return inline_taggedarray_getReference(); ++ case vmIntrinsics::_VMTaggedArray_allocate: ++ return inline_taggedarray_allocate(); ++ case vmIntrinsics::_VMTaggedArray_fillObject: ++ return inline_taggedarray_fillObject(); ++ case vmIntrinsics::_VMTaggedArray_copyFrontToBack: ++ return inline_taggedarray_copyFrontToBack(); ++ case vmIntrinsics::_VMTaggedArray_copyBackToFront: ++ return inline_taggedarray_copyBackToFront(); + case vmIntrinsics::_isInterrupted: + return inline_native_isInterrupted(); + +@@ -2986,6 +3031,141 @@ + return true; + } + ++bool LibraryCallKit::inline_taggedarray_doesvmimplement() { ++ push(intcon(NativeTaggedArrays ? 1 : 0)); ++ return true; ++} ++ ++Node* LibraryCallKit::inline_taggedarray_slot(Node* tagged, Node* slot, bool upperword) { ++ C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe". ++ int slot_offset = sun_misc_taggedarray_TaggedArrayNativeImpl::slot_offset(); ++ ++ Node* sv = _gvn.transform( new(C,3) LShiftXNode(ConvI2L(slot), intcon(3) )); ++ if (upperword) { ++ sv = _gvn.transform( new(C,3) AddLNode(sv, longcon(BytesPerInt + slot_offset))); ++ } else { ++ sv = _gvn.transform( new(C,3) AddLNode(sv, longcon(slot_offset))); ++ } ++ return make_unsafe_address(tagged, sv); ++} ++ ++bool LibraryCallKit::inline_taggedarray_isReference() { ++ Node* addr = inline_taggedarray_slot(argument(0), argument(1), false); ++ Node* result = ConvL2I(make_load(control(), addr, TypeLong::LONG, T_LONG)); ++ Node* bits = intcon(0x01l); ++ Node* andi = _gvn.transform(new (C, 3) AndINode(result, bits)); ++ Node* xori = _gvn.transform(new (C, 3) XorINode(andi, bits)); ++ push(xori); ++ return true; ++} ++ ++bool LibraryCallKit::inline_taggedarray_getValue() { ++ Node* addr = inline_taggedarray_slot(argument(0), argument(1), false); ++ Node* result = make_load(control(), addr, TypeLong::LONG, T_LONG); ++ push_pair(result); ++ return true; ++} ++ ++bool LibraryCallKit::inline_taggedarray_setValue() { ++ Node* addr = inline_taggedarray_slot(argument(0), argument(1), false); ++ Node* v = argument(2); ++ store_to_memory(control(), addr, v, T_LONG, TypeRawPtr::BOTTOM); ++ insert_mem_bar(Op_MemBarCPUOrder); ++ return true; ++} ++ ++bool LibraryCallKit::inline_taggedarray_setReference() { ++ Node* addr = inline_taggedarray_slot(argument(0), argument(1), UseCompressedOops); ++ Node* v = argument(2); ++ if (UseCompressedOops) { ++ // would be nice to have a better way of clearing the bit ++ // maybe introduce a StoreTaggedP or something that writes 8 bytes ++ store_to_memory(control(), inline_taggedarray_slot(argument(0), argument(1), false), longcon(0), T_LONG, TypeRawPtr::BOTTOM); ++ insert_mem_bar(Op_MemBarCPUOrder); ++ } ++ store_oop_to_unknown(control(), argument(0), addr, TypeRawPtr::BOTTOM, v, T_OBJECT); ++ insert_mem_bar(Op_MemBarCPUOrder); ++ return true; ++} ++ ++bool LibraryCallKit::inline_taggedarray_getReference() { ++ Node* addr = inline_taggedarray_slot(argument(0), argument(1), UseCompressedOops); ++ Node* result = make_load(control(), addr, TypeInstPtr::BOTTOM, T_OBJECT); ++ push(result); ++ return true; ++} ++ ++bool LibraryCallKit::inline_taggedarray_copyBackToFront() { ++ Node* src = argument(0); ++ Node* srcpos = argument(1); ++ Node* dst = argument(2); ++ Node* dstpos = argument(3); ++ Node* len = argument(4); ++ ++ make_runtime_call(RC_LEAF | RC_NO_FP, ++ OptoRuntime::tagged_copy_Type(), ++ StubRoutines::tagged_copy_backward(), ++ "tagged_copy", ++ TypeRawPtr::BOTTOM, ++ src, srcpos, dst, dstpos, len); ++ ++ insert_mem_bar(Op_MemBarCPUOrder); ++ return true; ++} ++ ++bool LibraryCallKit::inline_taggedarray_copyFrontToBack() { ++ Node* src = argument(0); ++ Node* srcpos = argument(1); ++ Node* dst = argument(2); ++ Node* dstpos = argument(3); ++ Node* len = argument(4); ++ ++ make_runtime_call(RC_LEAF | RC_NO_FP, ++ OptoRuntime::tagged_copy_Type(), ++ StubRoutines::tagged_copy_forward(), ++ "tagged_copy", ++ TypeRawPtr::BOTTOM, ++ src, srcpos, dst, dstpos, len); ++ ++ insert_mem_bar(Op_MemBarCPUOrder); ++ return true; ++} ++ ++bool LibraryCallKit::inline_taggedarray_fillObject() { ++ Node* ary = argument(0); ++ Node* from = argument(1); ++ Node* to = argument(2); ++ Node* value = argument(3); ++ ++ make_runtime_call(RC_LEAF|RC_NO_FP, ++ OptoRuntime::tagged_fillObject_Type(), ++ StubRoutines::tagged_fill(), ++ "tagged_fill", ++ TypeRawPtr::BOTTOM, ++ ary, from, to, value); ++ ++ insert_mem_bar(Op_MemBarCPUOrder); ++ return true; ++} ++ ++bool LibraryCallKit::inline_taggedarray_allocate() { ++ ciKlass* ck2 = env()->TaggedArrayNativeImpl_klass(); ++ // arg0 = elements ++ Node* size = argument(0); ++ Node* klass_node = makecon(TypeKlassPtr::make(ck2)); ++ assert(klass_node != NULL, "Expected Non-NULL"); ++ ++ Node* obj_size = NULL; ++ Node* obj = new_tagged(klass_node, NULL, size, &obj_size); ++ int offset = sun_misc_taggedarray_TaggedArrayNativeImpl::length_offset(); ++ Node* adr = basic_plus_adr(obj, obj, offset); ++ const TypeKlassPtr* tklass = _gvn.type(klass_node)->isa_klassptr(); ++ store_to_memory(control(), adr, argument(0), T_INT, tklass); ++ insert_mem_bar(Op_MemBarCPUOrder); ++ ++ push(obj); ++} ++ + //------------------------inline_native_currentThread------------------ + bool LibraryCallKit::inline_native_currentThread() { + Node* junk = NULL; +diff --git a/src/share/vm/opto/macro.cpp b/src/share/vm/opto/macro.cpp +--- a/src/share/vm/opto/macro.cpp ++++ b/src/share/vm/opto/macro.cpp +@@ -1084,7 +1084,8 @@ + AllocateNode* alloc, // allocation node to be expanded + Node* length, // array length for an array allocation + const TypeFunc* slow_call_type, // Type of slow call +- address slow_call_address // Address of slow call ++ address slow_call_address, // Address of slow call ++ Node* slots + ) + { + +@@ -1413,6 +1414,8 @@ + call->init_req(TypeFunc::Parms+0, klass_node); + if (length != NULL) { + call->init_req(TypeFunc::Parms+1, length); ++ } else if (slots != NULL) { ++ call->init_req(TypeFunc::Parms+1, slots); + } + + // Copy debug information and adjust JVMState information, then replace +@@ -1768,9 +1771,22 @@ + + + void PhaseMacroExpand::expand_allocate(AllocateNode *alloc) { +- expand_allocate_common(alloc, NULL, +- OptoRuntime::new_instance_Type(), +- OptoRuntime::new_instance_Java()); ++ Node *topnode = C->top(); ++ Node* slots = alloc->in(AllocateNode::Slots); ++ if (slots != topnode) { ++ Node* slots = alloc->in(AllocateNode::Slots); ++ expand_allocate_common(alloc, NULL, ++ OptoRuntime::new_instance_TaggedType(), ++ OptoRuntime::new_instance_Tagged(), ++ slots); ++ } else { ++ Node* klass_node = alloc->in(AllocateNode::KlassNode); ++ ciKlass* k = _igvn.type(klass_node)->is_klassptr()->klass(); ++ assert(k != C->env()->TaggedArrayNativeImpl_klass(), "Should not use new_instance_Java for TaggedArray"); ++ expand_allocate_common(alloc, NULL, ++ OptoRuntime::new_instance_Type(), ++ OptoRuntime::new_instance_Java()); ++ } + } + + void PhaseMacroExpand::expand_allocate_array(AllocateArrayNode *alloc) { +diff --git a/src/share/vm/opto/macro.hpp b/src/share/vm/opto/macro.hpp +--- a/src/share/vm/opto/macro.hpp ++++ b/src/share/vm/opto/macro.hpp +@@ -82,7 +82,8 @@ + void expand_allocate_common(AllocateNode* alloc, + Node* length, + const TypeFunc* slow_call_type, +- address slow_call_address); ++ address slow_call_address, ++ Node* size = NULL); + Node *value_from_mem(Node *mem, BasicType ft, const Type *ftype, const TypeOopPtr *adr_t, Node *alloc); + Node *value_from_mem_phi(Node *mem, BasicType ft, const Type *ftype, const TypeOopPtr *adr_t, Node *alloc, Node_Stack *value_phis, int level); + +diff --git a/src/share/vm/opto/runtime.cpp b/src/share/vm/opto/runtime.cpp +--- a/src/share/vm/opto/runtime.cpp ++++ b/src/share/vm/opto/runtime.cpp +@@ -44,6 +44,7 @@ + #include "memory/gcLocker.inline.hpp" + #include "memory/oopFactory.hpp" + #include "oops/objArrayKlass.hpp" ++#include "oops/instanceTagKlass.hpp" + #include "oops/oop.inline.hpp" + #include "opto/addnode.hpp" + #include "opto/callnode.hpp" +@@ -101,6 +102,7 @@ + + // Compiled code entry points + address OptoRuntime::_new_instance_Java = NULL; ++address OptoRuntime::_new_instance_Tagged = NULL; + address OptoRuntime::_new_array_Java = NULL; + address OptoRuntime::_new_array_nozero_Java = NULL; + address OptoRuntime::_multianewarray2_Java = NULL; +@@ -151,6 +153,7 @@ + // variable/name type-function-gen , runtime method ,fncy_jp, tls,save_args,retpc + // ------------------------------------------------------------------------------------------------------------------------------- + gen(env, _new_instance_Java , new_instance_Type , new_instance_C , 0 , true , false, false); ++ gen(env, _new_instance_Tagged , new_instance_TaggedType , new_tagged_C , 0 , true , false, false); + gen(env, _new_array_Java , new_array_Type , new_array_C , 0 , true , false, false); + gen(env, _new_array_nozero_Java , new_array_Type , new_array_nozero_C , 0 , true , false, false); + gen(env, _multianewarray2_Java , multianewarray2_Type , multianewarray2_C , 0 , true , false, false); +@@ -227,6 +230,53 @@ + thread->set_vm_result(new_obj); + } + ++// tagged object allocation ++JRT_BLOCK_ENTRY(void, OptoRuntime::new_tagged_C(klassOopDesc* klass, int slots, JavaThread* thread)) ++ JRT_BLOCK; ++#ifndef PRODUCT ++ SharedRuntime::_new_instance_ctr++; // new instance requires GC ++#endif ++ assert(check_compiled_frame(thread), "incorrect caller"); ++ assert(klass == SystemDictionary::TaggedArrayNativeImpl_klass(), "Must be tagged array"); ++ ++ // These checks are cheap to make and support reflective allocation. ++ int lh = Klass::cast(klass)->layout_helper(); ++ if (Klass::layout_helper_needs_slow_path(lh) ++ || !instanceKlass::cast(klass)->is_initialized()) { ++ KlassHandle kh(THREAD, klass); ++ kh->check_valid_for_instantiation(false, THREAD); ++ if (!HAS_PENDING_EXCEPTION) { ++ instanceKlass::cast(kh())->initialize(THREAD); ++ } ++ if (!HAS_PENDING_EXCEPTION) { ++ klass = kh(); ++ } else { ++ klass = NULL; ++ } ++ } ++ ++ if (klass != NULL) { ++ // Scavenge and allocate an instance. ++ ++ oop result = instanceTagKlass::cast(klass)->allocate_instance(slots, THREAD); ++ thread->set_vm_result(result); ++ ++ // Pass oops back through thread local storage. Our apparent type to Java ++ // is that we return an oop, but we can block on exit from this routine and ++ // a GC can trash the oop in C's return register. The generated stub will ++ // fetch the oop from TLS after any possible GC. ++ } ++ ++ deoptimize_caller_frame(thread, HAS_PENDING_EXCEPTION); ++ JRT_BLOCK_END; ++ ++ if (GraphKit::use_ReduceInitialCardMarks()) { ++ // inform GC that we won't do card marks for initializing writes. ++ new_store_pre_barrier(thread); ++ } ++JRT_END ++ ++ + // object allocation + JRT_BLOCK_ENTRY(void, OptoRuntime::new_instance_C(klassOopDesc* klass, JavaThread* thread)) + JRT_BLOCK; +@@ -253,6 +303,8 @@ + + if (klass != NULL) { + // Scavenge and allocate an instance. ++ ++ assert(klass != SystemDictionary::TaggedArrayNativeImpl_klass(), "Should not be tagged array"); + oop result = instanceKlass::cast(klass)->allocate_instance(THREAD); + thread->set_vm_result(result); + +@@ -443,6 +495,21 @@ + thread->set_vm_result(obj); + JRT_END + ++const TypeFunc *OptoRuntime::new_instance_TaggedType() { ++ // create input type (domain) ++ const Type **fields = TypeTuple::fields(2); ++ fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // Klass to be allocated ++ fields[TypeFunc::Parms+1] = TypeInt::INT; // array size ++ const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields); ++ ++ // create result type (range) ++ fields = TypeTuple::fields(1); ++ fields[TypeFunc::Parms+0] = TypeRawPtr::NOTNULL; // Returned oop ++ ++ const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields); ++ ++ return TypeFunc::make(domain, range); ++} + + const TypeFunc *OptoRuntime::new_instance_Type() { + // create input type (domain) +@@ -631,6 +698,42 @@ + return TypeFunc::make(domain,range); + } + ++const TypeFunc* OptoRuntime::tagged_fillObject_Type() { ++ // dst, from, to, value ++ const Type** fields = TypeTuple::fields(4); ++ fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; ++ fields[TypeFunc::Parms+1] = TypeInt::INT; ++ fields[TypeFunc::Parms+2] = TypeInt::INT; ++ fields[TypeFunc::Parms+3] = TypeInstPtr::NOTNULL; ++ ++ const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+4,fields); ++ ++ fields = TypeTuple::fields(1); ++ fields[TypeFunc::Parms+0] = NULL; // void ++ const TypeTuple *range = TypeTuple::make(TypeFunc::Parms, fields); ++ ++ return TypeFunc::make(domain, range); ++} ++ ++const TypeFunc* OptoRuntime::tagged_copy_Type() { ++ // dst, from, to, value ++ const Type** fields = TypeTuple::fields(5); ++ fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; ++ fields[TypeFunc::Parms+1] = TypeInt::INT; ++ fields[TypeFunc::Parms+2] = TypeInstPtr::NOTNULL; ++ fields[TypeFunc::Parms+3] = TypeInt::INT; ++ fields[TypeFunc::Parms+4] = TypeInt::INT; ++ ++ const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+5,fields); ++ ++ fields = TypeTuple::fields(1); ++ fields[TypeFunc::Parms+0] = NULL; // void ++ const TypeTuple *range = TypeTuple::make(TypeFunc::Parms, fields); ++ ++ return TypeFunc::make(domain, range); ++} ++ ++ + const TypeFunc* OptoRuntime::flush_windows_Type() { + // create input type (domain) + const Type** fields = TypeTuple::fields(1); +diff --git a/src/share/vm/opto/runtime.hpp b/src/share/vm/opto/runtime.hpp +--- a/src/share/vm/opto/runtime.hpp ++++ b/src/share/vm/opto/runtime.hpp +@@ -113,6 +113,7 @@ + + // References to generated stubs + static address _new_instance_Java; ++ static address _new_instance_Tagged; + static address _new_array_Java; + static address _new_array_nozero_Java; + static address _multianewarray2_Java; +@@ -141,6 +142,8 @@ + + // Allocate storage for a Java instance. + static void new_instance_C(klassOopDesc* instance_klass, JavaThread *thread); ++ // Allocate storage for a Tagged Java instance. ++ static void new_tagged_C(klassOopDesc* instance_klass, int slots, JavaThread *thread); + + // Allocate storage for a objArray or typeArray + static void new_array_C(klassOopDesc* array_klass, int len, JavaThread *thread); +@@ -210,6 +213,7 @@ + + // access to runtime stubs entry points for java code + static address new_instance_Java() { return _new_instance_Java; } ++ static address new_instance_Tagged() { return _new_instance_Tagged; } + static address new_array_Java() { return _new_array_Java; } + static address new_array_nozero_Java() { return _new_array_nozero_Java; } + static address multianewarray2_Java() { return _multianewarray2_Java; } +@@ -249,6 +253,7 @@ + // Type functions + // ====================================================== + ++ static const TypeFunc* new_instance_TaggedType(); // tagged object allocation (slow case) + static const TypeFunc* new_instance_Type(); // object allocation (slow case) + static const TypeFunc* new_array_Type (); // [a]newarray (slow case) + static const TypeFunc* multianewarray_Type(int ndim); // multianewarray +@@ -271,6 +276,8 @@ + static const TypeFunc* void_long_Type(); + + static const TypeFunc* flush_windows_Type(); ++ static const TypeFunc* tagged_fillObject_Type(); ++ static const TypeFunc* tagged_copy_Type(); + + // arraycopy routine types + static const TypeFunc* fast_arraycopy_Type(); // bit-blasters +diff --git a/src/share/vm/opto/type.cpp b/src/share/vm/opto/type.cpp +--- a/src/share/vm/opto/type.cpp ++++ b/src/share/vm/opto/type.cpp +@@ -33,6 +33,7 @@ + #include "memory/resourceArea.hpp" + #include "oops/instanceKlass.hpp" + #include "oops/instanceMirrorKlass.hpp" ++#include "oops/instanceTagKlass.hpp" + #include "oops/klassKlass.hpp" + #include "oops/objArrayKlass.hpp" + #include "oops/typeArrayKlass.hpp" +@@ -2380,6 +2381,8 @@ + _is_ptr_to_narrowoop = true; + } else if (_offset == oopDesc::klass_offset_in_bytes()) { + _is_ptr_to_narrowoop = true; ++ } else if (klass() == ciEnv::current()->TaggedArrayNativeImpl_klass()) { ++ _is_ptr_to_narrowoop = true; + } else if (this->isa_aryptr()) { + _is_ptr_to_narrowoop = (klass()->is_obj_array_klass() && + _offset != arrayOopDesc::length_offset_in_bytes()); +diff --git a/src/share/vm/prims/nativeLookup.cpp b/src/share/vm/prims/nativeLookup.cpp +--- a/src/share/vm/prims/nativeLookup.cpp ++++ b/src/share/vm/prims/nativeLookup.cpp +@@ -122,6 +122,7 @@ + void JNICALL JVM_RegisterMethodHandleMethods(JNIEnv *env, jclass unsafecls); + void JNICALL JVM_RegisterPerfMethods(JNIEnv *env, jclass perfclass); + void JNICALL JVM_RegisterWhiteBoxMethods(JNIEnv *env, jclass wbclass); ++ void JNICALL JVM_RegisterTagMethods(JNIEnv *env, jclass tagclass); + } + + #define CC (char*) /* cast a literal from (const char*) */ +@@ -136,6 +137,7 @@ + { CC"Java_java_lang_invoke_MethodHandleNatives_registerNatives", NULL, FN_PTR(JVM_RegisterMethodHandleMethods) }, + { CC"Java_sun_misc_Perf_registerNatives", NULL, FN_PTR(JVM_RegisterPerfMethods) }, + { CC"Java_sun_hotspot_WhiteBox_registerNatives", NULL, FN_PTR(JVM_RegisterWhiteBoxMethods) }, ++ { CC"Java_sun_misc_taggedarray_TaggedArrayNativeVMImpl_registerNatives", NULL, FN_PTR(JVM_RegisterTagMethods) }, + }; + + static address lookup_special_native(char* jni_name) { +diff --git a/src/share/vm/prims/tagged.cpp b/src/share/vm/prims/tagged.cpp +new file mode 100644 +--- /dev/null ++++ b/src/share/vm/prims/tagged.cpp +@@ -0,0 +1,138 @@ ++/* ++ * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved. ++ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. ++ * ++ * This code is free software; you can redistribute it and/or modify it ++ * under the terms of the GNU General Public License version 2 only, as ++ * published by the Free Software Foundation. ++ * ++ * This code is distributed in the hope that it will be useful, but WITHOUT ++ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ++ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License ++ * version 2 for more details (a copy is included in the LICENSE file that ++ * accompanied this code). ++ * ++ * You should have received a copy of the GNU General Public License version ++ * 2 along with this work; if not, write to the Free Software Foundation, ++ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. ++ * ++ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA ++ * or visit www.oracle.com if you need additional information or have any ++ * questions. ++ * ++ */ ++ ++#include "precompiled.hpp" ++ ++#include "memory/universe.hpp" ++#include "oops/oop.inline.hpp" ++#include "oops/instanceTagKlass.hpp" ++#include "classfile/javaClasses.hpp" ++#include "runtime/javaCalls.hpp" ++ ++#include "classfile/symbolTable.hpp" ++ ++#include "prims/tagged.hpp" ++#include "runtime/interfaceSupport.hpp" ++#include "runtime/os.hpp" ++#include "utilities/debug.hpp" ++ ++TAG_ENTRY(jboolean, Tagged_isReference(JNIEnv* env, jclass clz, jobject o, jint slot)) ++ jboolean v = !sun_misc_taggedarray_TaggedArrayNativeImpl::is_slot_tagged(JNIHandles::resolve(o), slot); ++ return v; ++TAG_END ++ ++TAG_ENTRY(jlong, Tagged_getValue(JNIEnv* env, jclass clz, jobject o, jint slot)) ++ return (jlong) sun_misc_taggedarray_TaggedArrayNativeImpl::get_slot_value(JNIHandles::resolve(o), slot); ++TAG_END ++ ++TAG_ENTRY(jobject, Tagged_getReference(JNIEnv* env, jclass clz, jobject o, jint slot)) ++ return JNIHandles::make_local(env, sun_misc_taggedarray_TaggedArrayNativeImpl::get_slot_ref(JNIHandles::resolve(o), slot)); ++TAG_END ++ ++TAG_ENTRY(void, Tagged_setValue(JNIEnv* env, jclass clz, jobject o, jint slot, jlong value)) ++ sun_misc_taggedarray_TaggedArrayNativeImpl::set_slot_value(JNIHandles::resolve(o), slot, value); ++TAG_END ++ ++TAG_ENTRY(void, Tagged_fillValue(JNIEnv* env, jclass clz, jobject o, jint from, jint to, jlong value)) ++ oop obj = JNIHandles::resolve(o); ++ for (; from < to; ++from) { ++ sun_misc_taggedarray_TaggedArrayNativeImpl::set_slot_value(obj, from, value); ++ } ++TAG_END ++ ++TAG_ENTRY(void, Tagged_copy(JNIEnv* env, jclass clz, jobject src, jint srcpos, jobject dst, jint dstpos, jint length)) ++ oop osrc = JNIHandles::resolve(src); ++ oop odst = JNIHandles::resolve(dst); ++ for (jint i = 0; i < length; ++i, ++srcpos, ++dstpos) { ++ sun_misc_taggedarray_TaggedArrayNativeImpl::copy_slot(odst, osrc, dstpos, srcpos); ++ } ++TAG_END ++ ++TAG_ENTRY(void, Tagged_fillObject(JNIEnv* env, jclass clz, jobject o, jint from, jint to, jobject value)) ++ oop obj = JNIHandles::resolve(o); ++ oop objval = JNIHandles::resolve(value); ++ for (; from < to; ++from) { ++ sun_misc_taggedarray_TaggedArrayNativeImpl::set_slot_ref(obj, from, objval); ++ } ++TAG_END ++ ++TAG_ENTRY(void, Tagged_setReference(JNIEnv* env, jclass clz, jobject o, jint slot, jobject value)) ++ sun_misc_taggedarray_TaggedArrayNativeImpl::set_slot_ref(JNIHandles::resolve(o), slot, JNIHandles::resolve(value)); ++TAG_END ++ ++jobject TaggedFactory_allocate(JNIEnv* env, int slots, TRAPS) { ++ instanceTagKlass::cast(SystemDictionary::TaggedArrayNativeImpl_klass())->initialize(CHECK_NULL); ++ oop taggedoop = instanceTagKlass::cast(SystemDictionary::TaggedArrayNativeImpl_klass())->allocate_instance(slots, CHECK_NULL); ++ Handle h(taggedoop); ++ ++ klassOop k = SystemDictionary::resolve_or_fail(vmSymbols::sun_misc_taggedarray_TaggedArrayNativeImpl(), true, CHECK_NULL); ++ instanceKlassHandle klass (THREAD, k); ++ ++ JavaValue result(T_VOID); ++ JavaCallArguments jca; ++ jca.push_oop(h); ++ jca.push_int(slots); ++ ++ JavaCalls::call_special(&result, klass, ++ vmSymbols::object_initializer_name(), ++ vmSymbols::int_void_signature(), ++ &jca, ++ CHECK_NULL); ++ ++ if (taggedoop) { ++ return JNIHandles::make_local(env, taggedoop); ++ } ++ return NULL; ++} ++ ++TAG_ENTRY(jobject, Tagged_allocate(JNIEnv* env, jclass clz, jint length)) ++ return TaggedFactory_allocate(env, length, CHECK_NULL); ++TAG_END ++ ++#define CC (char*) ++ ++static JNINativeMethod methods[] = { ++ {CC"isReference", CC"(Lsun/misc/taggedarray/TaggedArrayNativeImpl;I)Z", (void*)&Tagged_isReference }, ++ {CC"setValue", CC"(Lsun/misc/taggedarray/TaggedArrayNativeImpl;IJ)V", (void*)&Tagged_setValue }, ++ {CC"setReference", CC"(Lsun/misc/taggedarray/TaggedArrayNativeImpl;ILjava/lang/Object;)V", (void*)&Tagged_setReference }, ++ {CC"getValue", CC"(Lsun/misc/taggedarray/TaggedArrayNativeImpl;I)J", (void*)&Tagged_getValue }, ++ {CC"getReference", CC"(Lsun/misc/taggedarray/TaggedArrayNativeImpl;I)Ljava/lang/Object;", (void*)&Tagged_getReference }, ++ {CC"fillValue", CC"(Lsun/misc/taggedarray/TaggedArrayNativeImpl;IIJ)V", (void*)&Tagged_fillValue }, ++ {CC"fillObject", CC"(Lsun/misc/taggedarray/TaggedArrayNativeImpl;IILjava/lang/Object;)V", (void*)&Tagged_fillObject }, ++ {CC"allocate", CC"(I)Lsun/misc/taggedarray/TaggedArray;", (void*)&Tagged_allocate }, ++}; ++ ++#undef CC ++ ++JVM_ENTRY(void, JVM_RegisterTagMethods(JNIEnv* env, jclass tagclass)) ++ { ++ // Make sure that the class is loaded by the null classloader ++ instanceKlassHandle ikh = instanceKlassHandle(JNIHandles::resolve(tagclass)->klass()); ++ Handle loader(ikh->class_loader()); ++ if (loader.is_null()) { ++ ThreadToNativeFromVM ttnfv(thread); // can't be in VM when we call JNI ++ jint result = env->RegisterNatives(tagclass, methods, sizeof(methods)/sizeof(methods[0])); ++ } ++ } ++JVM_END +diff --git a/src/share/vm/prims/tagged.hpp b/src/share/vm/prims/tagged.hpp +new file mode 100644 +--- /dev/null ++++ b/src/share/vm/prims/tagged.hpp +@@ -0,0 +1,39 @@ ++/* ++ * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved. ++ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. ++ * ++ * This code is free software; you can redistribute it and/or modify it ++ * under the terms of the GNU General Public License version 2 only, as ++ * published by the Free Software Foundation. ++ * ++ * This code is distributed in the hope that it will be useful, but WITHOUT ++ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ++ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License ++ * version 2 for more details (a copy is included in the LICENSE file that ++ * accompanied this code). ++ * ++ * You should have received a copy of the GNU General Public License version ++ * 2 along with this work; if not, write to the Free Software Foundation, ++ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. ++ * ++ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA ++ * or visit www.oracle.com if you need additional information or have any ++ * questions. ++ * ++ */ ++ ++#ifndef SHARE_VM_PRIMS_TAGGED_HPP ++#define SHARE_VM_PRIMS_TAGGED_HPP ++ ++#include "prims/jni.h" ++ ++#include "memory/allocation.hpp" ++#include "oops/oopsHierarchy.hpp" ++ ++// Entry macro to transition from JNI to VM state. ++ ++#define TAG_ENTRY(result_type, header) JNI_ENTRY(result_type, header) ++#define TAG_END JNI_END ++#define TAG_METHOD_DECLARE extern "C" jobjectArray JNICALL ++ ++#endif // SHARE_VM_PRIMS_WHITEBOX_HPP +diff --git a/src/share/vm/runtime/sharedRuntime.cpp b/src/share/vm/runtime/sharedRuntime.cpp +--- a/src/share/vm/runtime/sharedRuntime.cpp ++++ b/src/share/vm/runtime/sharedRuntime.cpp +@@ -172,6 +172,8 @@ + int SharedRuntime::_slow_array_copy_ctr=0; + int SharedRuntime::_find_handler_ctr=0; + int SharedRuntime::_rethrow_ctr=0; ++int SharedRuntime::_tagged_fill_ctr=0; ++int SharedRuntime::_tagged_copy_ctr=0; + + int SharedRuntime::_ICmiss_index = 0; + int SharedRuntime::_ICmiss_count[SharedRuntime::maxICmiss_count]; +@@ -1870,6 +1872,8 @@ + tty->print_cr("%5d unresolved static call site", _resolve_static_ctr ); + tty->print_cr("%5d unresolved virtual call site", _resolve_virtual_ctr ); + tty->print_cr("%5d unresolved opt virtual call site", _resolve_opt_virtual_ctr ); ++ tty->print_cr("%5d tagged copies", _tagged_copy_ctr ); ++ tty->print_cr("%5d tagged fills", _tagged_fill_ctr ); + + if( _mon_enter_stub_ctr ) tty->print_cr("%5d monitor enter stub", _mon_enter_stub_ctr ); + if( _mon_exit_stub_ctr ) tty->print_cr("%5d monitor exit stub", _mon_exit_stub_ctr ); +diff --git a/src/share/vm/runtime/sharedRuntime.hpp b/src/share/vm/runtime/sharedRuntime.hpp +--- a/src/share/vm/runtime/sharedRuntime.hpp ++++ b/src/share/vm/runtime/sharedRuntime.hpp +@@ -495,6 +495,8 @@ + static int _implicit_null_throws; + static int _implicit_div0_throws; + ++ static int _tagged_fill_ctr; ++ static int _tagged_copy_ctr; + static int _jbyte_array_copy_ctr; // Slow-path byte array copy + static int _jshort_array_copy_ctr; // Slow-path short array copy + static int _jint_array_copy_ctr; // Slow-path int array copy +diff --git a/src/share/vm/runtime/stubRoutines.cpp b/src/share/vm/runtime/stubRoutines.cpp +--- a/src/share/vm/runtime/stubRoutines.cpp ++++ b/src/share/vm/runtime/stubRoutines.cpp +@@ -112,6 +112,9 @@ + address StubRoutines::_unsafe_arraycopy = NULL; + address StubRoutines::_generic_arraycopy = NULL; + ++address StubRoutines::_tagged_fill = NULL; ++address StubRoutines::_tagged_copy_forward = NULL; ++address StubRoutines::_tagged_copy_backward = NULL; + + address StubRoutines::_jbyte_fill; + address StubRoutines::_jshort_fill; +diff --git a/src/share/vm/runtime/stubRoutines.hpp b/src/share/vm/runtime/stubRoutines.hpp +--- a/src/share/vm/runtime/stubRoutines.hpp ++++ b/src/share/vm/runtime/stubRoutines.hpp +@@ -169,6 +169,10 @@ + static address _jlong_disjoint_arraycopy; + static address _oop_disjoint_arraycopy, _oop_disjoint_arraycopy_uninit; + ++ static address _tagged_fill; ++ static address _tagged_copy_forward; ++ static address _tagged_copy_backward; ++ + // arraycopy operands aligned on zero'th element boundary + // These are identical to the ones aligned aligned on an + // element type boundary, except that they assume that both +@@ -323,6 +327,10 @@ + static address unsafe_arraycopy() { return _unsafe_arraycopy; } + static address generic_arraycopy() { return _generic_arraycopy; } + ++ static address tagged_fill() { return _tagged_fill; } ++ static address tagged_copy_forward() { return _tagged_copy_forward; } ++ static address tagged_copy_backward() { return _tagged_copy_backward; } ++ + static address jbyte_fill() { return _jbyte_fill; } + static address jshort_fill() { return _jshort_fill; } + static address jint_fill() { return _jint_fill; }