1620
1621 // See if we can do this with two 12-bit offsets
1622 {
1623 unsigned long word_offset = byte_offset >> shift;
1624 unsigned long masked_offset = word_offset & 0xfff000;
1625 if (Address::offset_ok_for_immed(word_offset - masked_offset)
1626 && Assembler::operand_valid_for_add_sub_immediate(masked_offset << shift)) {
1627 add(Rd, base, masked_offset << shift);
1628 word_offset -= masked_offset;
1629 return Address(Rd, word_offset << shift);
1630 }
1631 }
1632
1633 // Do it the hard way
1634 mov(Rd, byte_offset);
1635 add(Rd, base, Rd);
1636 return Address(Rd);
1637 }
1638
1639 void MacroAssembler::atomic_incw(Register counter_addr, Register tmp, Register tmp2) {
1640 Label retry_load;
1641 prfm(Address(counter_addr), PSTL1STRM);
1642 bind(retry_load);
1643 // flush and load exclusive from the memory location
1644 ldxrw(tmp, counter_addr);
1645 addw(tmp, tmp, 1);
1646 // if we store+flush with no intervening write tmp wil be zero
1647 stxrw(tmp2, tmp, counter_addr);
1648 cbnzw(tmp2, retry_load);
1649 }
1650
1651
1652 int MacroAssembler::corrected_idivl(Register result, Register ra, Register rb,
1653 bool want_remainder, Register scratch)
1654 {
1655 // Full implementation of Java idiv and irem. The function
1656 // returns the (pc) offset of the div instruction - may be needed
1657 // for implicit exceptions.
1658 //
1659 // constraint : ra/rb =/= scratch
2155 load_exclusive(tmp, addr, size, acquire);
2156 if (size == xword)
2157 cmp(tmp, expected);
2158 else
2159 cmpw(tmp, expected);
2160 br(Assembler::NE, done);
2161 store_exclusive(tmp, new_val, addr, size, release);
2162 cbnzw(tmp, retry_load);
2163 bind(done);
2164 BLOCK_COMMENT("} cmpxchg");
2165 }
2166 }
2167
2168 static bool different(Register a, RegisterOrConstant b, Register c) {
2169 if (b.is_constant())
2170 return a != c;
2171 else
2172 return a != b.as_register() && a != c && b.as_register() != c;
2173 }
2174
2175 #define ATOMIC_OP(LDXR, OP, IOP, STXR) \
2176 void MacroAssembler::atomic_##OP(Register prev, RegisterOrConstant incr, Register addr) { \
2177 Register result = rscratch2; \
2178 if (prev->is_valid()) \
2179 result = different(prev, incr, addr) ? prev : rscratch2; \
2180 \
2181 Label retry_load; \
2182 prfm(Address(addr), PSTL1STRM); \
2183 bind(retry_load); \
2184 LDXR(result, addr); \
2185 OP(rscratch1, result, incr); \
2186 STXR(rscratch2, rscratch1, addr); \
2187 cbnzw(rscratch2, retry_load); \
2188 if (prev->is_valid() && prev != result) { \
2189 IOP(prev, rscratch1, incr); \
2190 } \
2191 }
2192
2193 ATOMIC_OP(ldxr, add, sub, stxr)
2194 ATOMIC_OP(ldxrw, addw, subw, stxrw)
2195
2196 #undef ATOMIC_OP
2197
2198 #define ATOMIC_XCHG(OP, LDXR, STXR) \
2199 void MacroAssembler::atomic_##OP(Register prev, Register newv, Register addr) { \
2200 Register result = rscratch2; \
2201 if (prev->is_valid()) \
2202 result = different(prev, newv, addr) ? prev : rscratch2; \
2203 \
2204 Label retry_load; \
2205 prfm(Address(addr), PSTL1STRM); \
2206 bind(retry_load); \
2207 LDXR(result, addr); \
2208 STXR(rscratch1, newv, addr); \
2209 cbnzw(rscratch1, retry_load); \
2210 if (prev->is_valid() && prev != result) \
2211 mov(prev, result); \
2212 }
2213
2214 ATOMIC_XCHG(xchg, ldxr, stxr)
2215 ATOMIC_XCHG(xchgw, ldxrw, stxrw)
2216
2217 #undef ATOMIC_XCHG
2218
2219 void MacroAssembler::incr_allocated_bytes(Register thread,
2220 Register var_size_in_bytes,
2221 int con_size_in_bytes,
2222 Register t1) {
2223 if (!thread->is_valid()) {
2224 thread = rthread;
2225 }
2226 assert(t1->is_valid(), "need temp reg");
2227
2228 ldr(t1, Address(thread, in_bytes(JavaThread::allocated_bytes_offset())));
2229 if (var_size_in_bytes->is_valid()) {
2230 add(t1, t1, var_size_in_bytes);
2231 } else {
2232 add(t1, t1, con_size_in_bytes);
2233 }
2234 str(t1, Address(thread, in_bytes(JavaThread::allocated_bytes_offset())));
2235 }
|
1620
1621 // See if we can do this with two 12-bit offsets
1622 {
1623 unsigned long word_offset = byte_offset >> shift;
1624 unsigned long masked_offset = word_offset & 0xfff000;
1625 if (Address::offset_ok_for_immed(word_offset - masked_offset)
1626 && Assembler::operand_valid_for_add_sub_immediate(masked_offset << shift)) {
1627 add(Rd, base, masked_offset << shift);
1628 word_offset -= masked_offset;
1629 return Address(Rd, word_offset << shift);
1630 }
1631 }
1632
1633 // Do it the hard way
1634 mov(Rd, byte_offset);
1635 add(Rd, base, Rd);
1636 return Address(Rd);
1637 }
1638
1639 void MacroAssembler::atomic_incw(Register counter_addr, Register tmp, Register tmp2) {
1640 if (UseLSE) {
1641 mov(tmp, 1);
1642 ldadd(Assembler::word, tmp, zr, counter_addr);
1643 return;
1644 }
1645 Label retry_load;
1646 prfm(Address(counter_addr), PSTL1STRM);
1647 bind(retry_load);
1648 // flush and load exclusive from the memory location
1649 ldxrw(tmp, counter_addr);
1650 addw(tmp, tmp, 1);
1651 // if we store+flush with no intervening write tmp wil be zero
1652 stxrw(tmp2, tmp, counter_addr);
1653 cbnzw(tmp2, retry_load);
1654 }
1655
1656
1657 int MacroAssembler::corrected_idivl(Register result, Register ra, Register rb,
1658 bool want_remainder, Register scratch)
1659 {
1660 // Full implementation of Java idiv and irem. The function
1661 // returns the (pc) offset of the div instruction - may be needed
1662 // for implicit exceptions.
1663 //
1664 // constraint : ra/rb =/= scratch
2160 load_exclusive(tmp, addr, size, acquire);
2161 if (size == xword)
2162 cmp(tmp, expected);
2163 else
2164 cmpw(tmp, expected);
2165 br(Assembler::NE, done);
2166 store_exclusive(tmp, new_val, addr, size, release);
2167 cbnzw(tmp, retry_load);
2168 bind(done);
2169 BLOCK_COMMENT("} cmpxchg");
2170 }
2171 }
2172
2173 static bool different(Register a, RegisterOrConstant b, Register c) {
2174 if (b.is_constant())
2175 return a != c;
2176 else
2177 return a != b.as_register() && a != c && b.as_register() != c;
2178 }
2179
2180 #define ATOMIC_OP(NAME, LDXR, OP, IOP, AOP, STXR, sz) \
2181 void MacroAssembler::atomic_##NAME(Register prev, RegisterOrConstant incr, Register addr) { \
2182 if (UseLSE) { \
2183 prev = prev->is_valid() ? prev : zr; \
2184 if (incr.is_register()) { \
2185 AOP(sz, incr.as_register(), prev, addr); \
2186 } else { \
2187 mov(rscratch2, incr.as_constant()); \
2188 AOP(sz, rscratch2, prev, addr); \
2189 } \
2190 return; \
2191 } \
2192 Register result = rscratch2; \
2193 if (prev->is_valid()) \
2194 result = different(prev, incr, addr) ? prev : rscratch2; \
2195 \
2196 Label retry_load; \
2197 prfm(Address(addr), PSTL1STRM); \
2198 bind(retry_load); \
2199 LDXR(result, addr); \
2200 OP(rscratch1, result, incr); \
2201 STXR(rscratch2, rscratch1, addr); \
2202 cbnzw(rscratch2, retry_load); \
2203 if (prev->is_valid() && prev != result) { \
2204 IOP(prev, rscratch1, incr); \
2205 } \
2206 }
2207
2208 ATOMIC_OP(add, ldxr, add, sub, ldadd, stxr, Assembler::xword)
2209 ATOMIC_OP(addw, ldxrw, addw, subw, ldadd, stxrw, Assembler::word)
2210 ATOMIC_OP(addal, ldaxr, add, sub, ldaddal, stlxr, Assembler::xword)
2211 ATOMIC_OP(addalw, ldaxrw, addw, subw, ldaddal, stlxrw, Assembler::word)
2212
2213 #undef ATOMIC_OP
2214
2215 #define ATOMIC_XCHG(OP, AOP, LDXR, STXR, sz) \
2216 void MacroAssembler::atomic_##OP(Register prev, Register newv, Register addr) { \
2217 if (UseLSE) { \
2218 prev = prev->is_valid() ? prev : zr; \
2219 AOP(sz, newv, prev, addr); \
2220 return; \
2221 } \
2222 Register result = rscratch2; \
2223 if (prev->is_valid()) \
2224 result = different(prev, newv, addr) ? prev : rscratch2; \
2225 \
2226 Label retry_load; \
2227 prfm(Address(addr), PSTL1STRM); \
2228 bind(retry_load); \
2229 LDXR(result, addr); \
2230 STXR(rscratch1, newv, addr); \
2231 cbnzw(rscratch1, retry_load); \
2232 if (prev->is_valid() && prev != result) \
2233 mov(prev, result); \
2234 }
2235
2236 ATOMIC_XCHG(xchg, swp, ldxr, stxr, Assembler::xword)
2237 ATOMIC_XCHG(xchgw, swp, ldxrw, stxrw, Assembler::word)
2238 ATOMIC_XCHG(xchgal, swpal, ldaxr, stlxr, Assembler::xword)
2239 ATOMIC_XCHG(xchgalw, swpal, ldaxrw, stlxrw, Assembler::word)
2240
2241 #undef ATOMIC_XCHG
2242
2243 void MacroAssembler::incr_allocated_bytes(Register thread,
2244 Register var_size_in_bytes,
2245 int con_size_in_bytes,
2246 Register t1) {
2247 if (!thread->is_valid()) {
2248 thread = rthread;
2249 }
2250 assert(t1->is_valid(), "need temp reg");
2251
2252 ldr(t1, Address(thread, in_bytes(JavaThread::allocated_bytes_offset())));
2253 if (var_size_in_bytes->is_valid()) {
2254 add(t1, t1, var_size_in_bytes);
2255 } else {
2256 add(t1, t1, con_size_in_bytes);
2257 }
2258 str(t1, Address(thread, in_bytes(JavaThread::allocated_bytes_offset())));
2259 }
|