14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/assembler.hpp"
27 #include "asm/assembler.inline.hpp"
28 #include "asm/macroAssembler.hpp"
29 #include "ci/ciEnv.hpp"
30 #include "code/nativeInst.hpp"
31 #include "compiler/disassembler.hpp"
32 #include "gc/shared/barrierSet.hpp"
33 #include "gc/shared/cardTable.hpp"
34 #include "gc/shared/cardTableBarrierSet.hpp"
35 #include "gc/shared/collectedHeap.inline.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "memory/resourceArea.hpp"
38 #include "oops/klass.inline.hpp"
39 #include "prims/methodHandles.hpp"
40 #include "runtime/biasedLocking.hpp"
41 #include "runtime/interfaceSupport.inline.hpp"
42 #include "runtime/objectMonitor.hpp"
43 #include "runtime/os.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "runtime/stubRoutines.hpp"
46 #include "utilities/macros.hpp"
47 #if INCLUDE_ALL_GCS
48 #include "gc/g1/g1BarrierSet.hpp"
49 #include "gc/g1/g1CardTable.hpp"
50 #include "gc/g1/g1ThreadLocalData.hpp"
51 #include "gc/g1/heapRegion.hpp"
52 #endif
53
2109 void MacroAssembler::biased_locking_exit(Register obj_reg, Register tmp_reg, Label& done) {
2110 assert(UseBiasedLocking, "why call this otherwise?");
2111
2112 // Check for biased locking unlock case, which is a no-op
2113 // Note: we do not have to check the thread ID for two reasons.
2114 // First, the interpreter checks for IllegalMonitorStateException at
2115 // a higher level. Second, if the bias was revoked while we held the
2116 // lock, the object could not be rebiased toward another thread, so
2117 // the bias bit would be clear.
2118 ldr(tmp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
2119
2120 andr(tmp_reg, tmp_reg, (uintx)markOopDesc::biased_lock_mask_in_place);
2121 cmp(tmp_reg, markOopDesc::biased_lock_pattern);
2122 b(done, eq);
2123 }
2124
2125
2126 void MacroAssembler::resolve_jobject(Register value,
2127 Register tmp1,
2128 Register tmp2) {
2129 assert_different_registers(value, tmp1, tmp2);
2130 Label done, not_weak;
2131 cbz(value, done); // Use NULL as-is.
2132 STATIC_ASSERT(JNIHandles::weak_tag_mask == 1u);
2133 tbz(value, 0, not_weak); // Test for jweak tag.
2134 // Resolve jweak.
2135 ldr(value, Address(value, -JNIHandles::weak_tag_value));
2136 verify_oop(value);
2137 #if INCLUDE_ALL_GCS
2138 if (UseG1GC) {
2139 g1_write_barrier_pre(noreg, // store_addr
2140 noreg, // new_val
2141 value, // pre_val
2142 tmp1, // tmp1
2143 tmp2); // tmp2
2144 }
2145 #endif // INCLUDE_ALL_GCS
2146 b(done);
2147 bind(not_weak);
2148 // Resolve (untagged) jobject.
2149 ldr(value, Address(value));
2150 verify_oop(value);
2151 bind(done);
2152 }
2153
2154
2155 //////////////////////////////////////////////////////////////////////////////////
2156
2157 #if INCLUDE_ALL_GCS
2158
2159 // G1 pre-barrier.
2160 // Blows all volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64, Rtemp, LR).
2161 // If store_addr != noreg, then previous value is loaded from [store_addr];
2162 // in such case store_addr and new_val registers are preserved;
2163 // otherwise pre_val register is preserved.
2164 void MacroAssembler::g1_write_barrier_pre(Register store_addr,
2165 Register new_val,
2166 Register pre_val,
2167 Register tmp1,
2168 Register tmp2) {
2169 Label done;
2170 Label runtime;
2171
2172 if (store_addr != noreg) {
2173 assert_different_registers(store_addr, new_val, pre_val, tmp1, tmp2, noreg);
2174 } else {
2175 assert (new_val == noreg, "should be");
2176 assert_different_registers(pre_val, tmp1, tmp2, noreg);
2177 }
2178
2179 Address in_progress(Rthread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
2180 Address index(Rthread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()));
2181 Address buffer(Rthread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()));
2182
2183 // Is marking active?
2184 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "adjust this code");
2185 ldrb(tmp1, in_progress);
2186 cbz(tmp1, done);
2187
2188 // Do we need to load the previous value?
2189 if (store_addr != noreg) {
2190 load_heap_oop(pre_val, Address(store_addr, 0));
2191 }
2192
2193 // Is the previous value null?
2194 cbz(pre_val, done);
2195
2196 // Can we store original value in the thread's buffer?
2197 // Is index == 0?
2198 // (The index field is typed as size_t.)
2199
2200 ldr(tmp1, index); // tmp1 := *index_adr
2201 ldr(tmp2, buffer);
2202
2203 subs(tmp1, tmp1, wordSize); // tmp1 := tmp1 - wordSize
2204 b(runtime, lt); // If negative, goto runtime
2205
2206 str(tmp1, index); // *index_adr := tmp1
2207
2208 // Record the previous value
2209 str(pre_val, Address(tmp2, tmp1));
2210 b(done);
2211
2212 bind(runtime);
2213
2214 // save the live input values
2215 #ifdef AARCH64
2216 if (store_addr != noreg) {
2217 raw_push(store_addr, new_val);
2218 } else {
2219 raw_push(pre_val, ZR);
2220 }
2221 #else
2222 if (store_addr != noreg) {
2223 // avoid raw_push to support any ordering of store_addr and new_val
2224 push(RegisterSet(store_addr) | RegisterSet(new_val));
2225 } else {
2226 push(pre_val);
2227 }
2228 #endif // AARCH64
2229
2230 if (pre_val != R0) {
2231 mov(R0, pre_val);
2232 }
2233 mov(R1, Rthread);
2234
2235 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), R0, R1);
2236
2237 #ifdef AARCH64
2238 if (store_addr != noreg) {
2239 raw_pop(store_addr, new_val);
2240 } else {
2241 raw_pop(pre_val, ZR);
2242 }
2243 #else
2244 if (store_addr != noreg) {
2245 pop(RegisterSet(store_addr) | RegisterSet(new_val));
2246 } else {
2247 pop(pre_val);
2248 }
2249 #endif // AARCH64
2250
2251 bind(done);
2252 }
2253
2254 // G1 post-barrier.
2255 // Blows all volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64, Rtemp, LR).
2256 void MacroAssembler::g1_write_barrier_post(Register store_addr,
2257 Register new_val,
2258 Register tmp1,
2259 Register tmp2,
2260 Register tmp3) {
2261
2262 Address queue_index(Rthread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
2263 Address buffer(Rthread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
2264
2265 BarrierSet* bs = BarrierSet::barrier_set();
2266 CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs);
2267 CardTable* ct = ctbs->card_table();
2268 Label done;
2269 Label runtime;
2270
2271 // Does store cross heap regions?
2272
2273 eor(tmp1, store_addr, new_val);
2274 #ifdef AARCH64
2275 logical_shift_right(tmp1, tmp1, HeapRegion::LogOfHRGrainBytes);
2276 cbz(tmp1, done);
2277 #else
2278 movs(tmp1, AsmOperand(tmp1, lsr, HeapRegion::LogOfHRGrainBytes));
2279 b(done, eq);
2280 #endif
2281
2282 // crosses regions, storing NULL?
2283
2284 cbz(new_val, done);
2285
2286 // storing region crossing non-NULL, is card already dirty?
2287 const Register card_addr = tmp1;
2288 assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "adjust this code");
2289
2290 mov_address(tmp2, (address)ct->byte_map_base(), symbolic_Relocation::card_table_reference);
2291 add(card_addr, tmp2, AsmOperand(store_addr, lsr, CardTable::card_shift));
2292
2293 ldrb(tmp2, Address(card_addr));
2294 cmp(tmp2, (int)G1CardTable::g1_young_card_val());
2295 b(done, eq);
2296
2297 membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad), tmp2);
2298
2299 assert(CardTable::dirty_card_val() == 0, "adjust this code");
2300 ldrb(tmp2, Address(card_addr));
2301 cbz(tmp2, done);
2302
2303 // storing a region crossing, non-NULL oop, card is clean.
2304 // dirty card and log.
2305
2306 strb(zero_register(tmp2), Address(card_addr));
2307
2308 ldr(tmp2, queue_index);
2309 ldr(tmp3, buffer);
2310
2311 subs(tmp2, tmp2, wordSize);
2312 b(runtime, lt); // go to runtime if now negative
2313
2314 str(tmp2, queue_index);
2315
2316 str(card_addr, Address(tmp3, tmp2));
2317 b(done);
2318
2319 bind(runtime);
2320
2321 if (card_addr != R0) {
2322 mov(R0, card_addr);
2323 }
2324 mov(R1, Rthread);
2325 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), R0, R1);
2326
2327 bind(done);
2328 }
2329
2330 #endif // INCLUDE_ALL_GCS
2331
2332 //////////////////////////////////////////////////////////////////////////////////
2333
2334 #ifdef AARCH64
2335
2336 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed) {
2337 switch (size_in_bytes) {
2338 case 8: ldr(dst, src); break;
2339 case 4: is_signed ? ldr_s32(dst, src) : ldr_u32(dst, src); break;
2340 case 2: is_signed ? ldrsh(dst, src) : ldrh(dst, src); break;
2341 case 1: is_signed ? ldrsb(dst, src) : ldrb(dst, src); break;
2342 default: ShouldNotReachHere();
2343 }
2344 }
2345
2346 void MacroAssembler::store_sized_value(Register src, Address dst, size_t size_in_bytes) {
2347 switch (size_in_bytes) {
2348 case 8: str(src, dst); break;
2349 case 4: str_32(src, dst); break;
2350 case 2: strh(src, dst); break;
2351 case 1: strb(src, dst); break;
2352 default: ShouldNotReachHere();
2353 }
2856 assert(src_klass != dst_oop, "not enough registers");
2857 encode_klass_not_null(src_klass);
2858 str_w(src_klass, Address(dst_oop, oopDesc::klass_offset_in_bytes()));
2859 return;
2860 }
2861 #endif // AARCH64
2862 str(src_klass, Address(dst_oop, oopDesc::klass_offset_in_bytes()));
2863 }
2864
2865 #ifdef AARCH64
2866
2867 void MacroAssembler::store_klass_gap(Register dst) {
2868 if (UseCompressedClassPointers) {
2869 str_w(ZR, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
2870 }
2871 }
2872
2873 #endif // AARCH64
2874
2875
2876 void MacroAssembler::load_heap_oop(Register dst, Address src) {
2877 #ifdef AARCH64
2878 if (UseCompressedOops) {
2879 ldr_w(dst, src);
2880 decode_heap_oop(dst);
2881 return;
2882 }
2883 #endif // AARCH64
2884 ldr(dst, src);
2885 }
2886
2887 // Blows src and flags.
2888 void MacroAssembler::store_heap_oop(Register src, Address dst) {
2889 #ifdef AARCH64
2890 if (UseCompressedOops) {
2891 assert(!dst.uses(src), "not enough registers");
2892 encode_heap_oop(src);
2893 str_w(src, dst);
2894 return;
2895 }
2896 #endif // AARCH64
2897 str(src, dst);
2898 }
2899
2900 void MacroAssembler::store_heap_oop_null(Register src, Address dst) {
2901 #ifdef AARCH64
2902 if (UseCompressedOops) {
2903 str_w(src, dst);
2904 return;
2905 }
2906 #endif // AARCH64
2907 str(src, dst);
2908 }
2909
2910
2911 #ifdef AARCH64
2912
2913 // Algorithm must match oop.inline.hpp encode_heap_oop.
2914 void MacroAssembler::encode_heap_oop(Register dst, Register src) {
2915 // This code pattern is matched in NativeIntruction::skip_encode_heap_oop.
2916 // Update it at modifications.
2917 assert (UseCompressedOops, "must be compressed");
2918 assert (Universe::heap() != NULL, "java heap should be initialized");
2919 #ifdef ASSERT
2920 verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
2921 #endif
2922 verify_oop(src);
2923 if (Universe::narrow_oop_base() == NULL) {
2924 if (Universe::narrow_oop_shift() != 0) {
2925 assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
2926 _lsr(dst, src, Universe::narrow_oop_shift());
2927 } else if (dst != src) {
|
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/assembler.hpp"
27 #include "asm/assembler.inline.hpp"
28 #include "asm/macroAssembler.hpp"
29 #include "ci/ciEnv.hpp"
30 #include "code/nativeInst.hpp"
31 #include "compiler/disassembler.hpp"
32 #include "gc/shared/barrierSet.hpp"
33 #include "gc/shared/cardTable.hpp"
34 #include "gc/shared/barrierSetAssembler.hpp"
35 #include "gc/shared/cardTableBarrierSet.hpp"
36 #include "gc/shared/collectedHeap.inline.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "memory/resourceArea.hpp"
39 #include "oops/klass.inline.hpp"
40 #include "prims/methodHandles.hpp"
41 #include "runtime/biasedLocking.hpp"
42 #include "runtime/interfaceSupport.inline.hpp"
43 #include "runtime/objectMonitor.hpp"
44 #include "runtime/os.hpp"
45 #include "runtime/sharedRuntime.hpp"
46 #include "runtime/stubRoutines.hpp"
47 #include "utilities/macros.hpp"
48 #if INCLUDE_ALL_GCS
49 #include "gc/g1/g1BarrierSet.hpp"
50 #include "gc/g1/g1CardTable.hpp"
51 #include "gc/g1/g1ThreadLocalData.hpp"
52 #include "gc/g1/heapRegion.hpp"
53 #endif
54
2110 void MacroAssembler::biased_locking_exit(Register obj_reg, Register tmp_reg, Label& done) {
2111 assert(UseBiasedLocking, "why call this otherwise?");
2112
2113 // Check for biased locking unlock case, which is a no-op
2114 // Note: we do not have to check the thread ID for two reasons.
2115 // First, the interpreter checks for IllegalMonitorStateException at
2116 // a higher level. Second, if the bias was revoked while we held the
2117 // lock, the object could not be rebiased toward another thread, so
2118 // the bias bit would be clear.
2119 ldr(tmp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
2120
2121 andr(tmp_reg, tmp_reg, (uintx)markOopDesc::biased_lock_mask_in_place);
2122 cmp(tmp_reg, markOopDesc::biased_lock_pattern);
2123 b(done, eq);
2124 }
2125
2126
2127 void MacroAssembler::resolve_jobject(Register value,
2128 Register tmp1,
2129 Register tmp2) {
2130 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
2131
2132 assert_different_registers(value, tmp1, tmp2);
2133 Label done, not_weak;
2134 cbz(value, done); // Use NULL as-is.
2135 STATIC_ASSERT(JNIHandles::weak_tag_mask == 1u);
2136 tbz(value, 0, not_weak); // Test for jweak tag.
2137
2138 // Resolve jweak.
2139 bs->load_at(this, IN_ROOT | ON_PHANTOM_OOP_REF, T_OBJECT,
2140 value, Address(value, -JNIHandles::weak_tag_value), tmp1, tmp2, noreg);
2141 b(done);
2142 bind(not_weak);
2143 // Resolve (untagged) jobject.
2144 bs->load_at(this, IN_ROOT | IN_CONCURRENT_ROOT, T_OBJECT,
2145 value, Address(value, 0), tmp1, tmp2, noreg);
2146 verify_oop(value);
2147 bind(done);
2148 }
2149
2150
2151 //////////////////////////////////////////////////////////////////////////////////
2152
2153 #ifdef AARCH64
2154
2155 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed) {
2156 switch (size_in_bytes) {
2157 case 8: ldr(dst, src); break;
2158 case 4: is_signed ? ldr_s32(dst, src) : ldr_u32(dst, src); break;
2159 case 2: is_signed ? ldrsh(dst, src) : ldrh(dst, src); break;
2160 case 1: is_signed ? ldrsb(dst, src) : ldrb(dst, src); break;
2161 default: ShouldNotReachHere();
2162 }
2163 }
2164
2165 void MacroAssembler::store_sized_value(Register src, Address dst, size_t size_in_bytes) {
2166 switch (size_in_bytes) {
2167 case 8: str(src, dst); break;
2168 case 4: str_32(src, dst); break;
2169 case 2: strh(src, dst); break;
2170 case 1: strb(src, dst); break;
2171 default: ShouldNotReachHere();
2172 }
2675 assert(src_klass != dst_oop, "not enough registers");
2676 encode_klass_not_null(src_klass);
2677 str_w(src_klass, Address(dst_oop, oopDesc::klass_offset_in_bytes()));
2678 return;
2679 }
2680 #endif // AARCH64
2681 str(src_klass, Address(dst_oop, oopDesc::klass_offset_in_bytes()));
2682 }
2683
2684 #ifdef AARCH64
2685
2686 void MacroAssembler::store_klass_gap(Register dst) {
2687 if (UseCompressedClassPointers) {
2688 str_w(ZR, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
2689 }
2690 }
2691
2692 #endif // AARCH64
2693
2694
2695 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1, Register tmp2, Register tmp3, DecoratorSet decorators) {
2696 access_load_at(T_OBJECT, IN_HEAP | decorators, src, dst, tmp1, tmp2, tmp3);
2697 }
2698
2699 // Blows src and flags.
2700 void MacroAssembler::store_heap_oop(Address obj, Register new_val, Register tmp1, Register tmp2, Register tmp3, DecoratorSet decorators) {
2701 access_store_at(T_OBJECT, IN_HEAP | decorators, obj, new_val, tmp1, tmp2, tmp3, false);
2702 }
2703
2704 void MacroAssembler::store_heap_oop_null(Address obj, Register new_val, Register tmp1, Register tmp2, Register tmp3, DecoratorSet decorators) {
2705 access_store_at(T_OBJECT, IN_HEAP, obj, new_val, tmp1, tmp2, tmp3, true);
2706 }
2707
2708 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators,
2709 Address src, Register dst, Register tmp1, Register tmp2, Register tmp3) {
2710 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
2711 bool as_raw = (decorators & AS_RAW) != 0;
2712 if (as_raw) {
2713 bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, tmp2, tmp3);
2714 } else {
2715 bs->load_at(this, decorators, type, dst, src, tmp1, tmp2, tmp3);
2716 }
2717 }
2718
2719 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators,
2720 Address obj, Register new_val, Register tmp1, Register tmp2, Register tmp3, bool is_null) {
2721 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
2722 bool as_raw = (decorators & AS_RAW) != 0;
2723 if (as_raw) {
2724 bs->BarrierSetAssembler::store_at(this, decorators, type, obj, new_val, tmp1, tmp2, tmp3, is_null);
2725 } else {
2726 bs->store_at(this, decorators, type, obj, new_val, tmp1, tmp2, tmp3, is_null);
2727 }
2728 }
2729
2730
2731 #ifdef AARCH64
2732
2733 // Algorithm must match oop.inline.hpp encode_heap_oop.
2734 void MacroAssembler::encode_heap_oop(Register dst, Register src) {
2735 // This code pattern is matched in NativeIntruction::skip_encode_heap_oop.
2736 // Update it at modifications.
2737 assert (UseCompressedOops, "must be compressed");
2738 assert (Universe::heap() != NULL, "java heap should be initialized");
2739 #ifdef ASSERT
2740 verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
2741 #endif
2742 verify_oop(src);
2743 if (Universe::narrow_oop_base() == NULL) {
2744 if (Universe::narrow_oop_shift() != 0) {
2745 assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
2746 _lsr(dst, src, Universe::narrow_oop_shift());
2747 } else if (dst != src) {
|