147 } else {
148 return new AArch64AddressValue(address.getValueKind(), asAllocatable(address), Value.ILLEGAL, 0, 1, AddressingMode.BASE_REGISTER_ONLY);
149 }
150 }
151
152 @Override
153 public Variable emitLogicCompareAndSwap(LIRKind accessKind, Value address, Value expectedValue, Value newValue, Value trueValue, Value falseValue) {
154 Variable prevValue = newVariable(expectedValue.getValueKind());
155 Variable scratch = newVariable(LIRKind.value(AArch64Kind.DWORD));
156 append(new CompareAndSwapOp(prevValue, loadReg(expectedValue), loadReg(newValue), asAllocatable(address), scratch));
157 assert trueValue.getValueKind().equals(falseValue.getValueKind());
158 Variable result = newVariable(trueValue.getValueKind());
159 append(new CondMoveOp(result, ConditionFlag.EQ, asAllocatable(trueValue), asAllocatable(falseValue)));
160 return result;
161 }
162
163 @Override
164 public Variable emitValueCompareAndSwap(LIRKind accessKind, Value address, Value expectedValue, Value newValue) {
165 Variable result = newVariable(newValue.getValueKind());
166 Variable scratch = newVariable(LIRKind.value(AArch64Kind.WORD));
167 append(new CompareAndSwapOp(result, loadNonCompareConst(expectedValue), loadReg(newValue), asAllocatable(address), scratch));
168 return result;
169 }
170
171 @Override
172 public Value emitAtomicReadAndWrite(Value address, ValueKind<?> kind, Value newValue) {
173 Variable result = newVariable(kind);
174 Variable scratch = newVariable(kind);
175 append(new AtomicReadAndWriteOp((AArch64Kind) kind.getPlatformKind(), asAllocatable(result), asAllocatable(address), asAllocatable(newValue), asAllocatable(scratch)));
176 return result;
177 }
178
179 @Override
180 public Value emitAtomicReadAndAdd(Value address, ValueKind<?> kind, Value delta) {
181 Variable result = newVariable(kind);
182 if (AArch64LIRFlagsVersioned.useLSE(target().arch)) {
183 append(new AtomicReadAndAddLSEOp((AArch64Kind) kind.getPlatformKind(), asAllocatable(result), asAllocatable(address), asAllocatable(delta)));
184 } else {
185 append(new AtomicReadAndAddOp((AArch64Kind) kind.getPlatformKind(), asAllocatable(result), asAllocatable(address), delta));
186 }
187 return result;
221 ((AArch64ArithmeticLIRGenerator) getArithmetic()).emitBinary(LIRKind.combine(left, right), AArch64ArithmeticOp.ANDS, true, left, right);
222 append(new AArch64ControlFlow.BranchOp(ConditionFlag.EQ, trueDestination, falseDestination, trueSuccessorProbability));
223 }
224
225 /**
226 * Conditionally move trueValue into new variable if cond + unorderedIsTrue is true, else
227 * falseValue.
228 *
229 * @param left Arbitrary value. Has to have same type as right. Non null.
230 * @param right Arbitrary value. Has to have same type as left. Non null.
231 * @param cond condition that decides whether to move trueValue or falseValue into result. Non
232 * null.
233 * @param unorderedIsTrue defines whether floating-point comparisons consider unordered true or
234 * not. Ignored for integer comparisons.
235 * @param trueValue arbitrary value same type as falseValue. Non null.
236 * @param falseValue arbitrary value same type as trueValue. Non null.
237 * @return value containing trueValue if cond + unorderedIsTrue is true, else falseValue. Non
238 * null.
239 */
240 @Override
241 public Variable emitConditionalMove(PlatformKind cmpKind, Value left, Value right, Condition cond, boolean unorderedIsTrue, Value trueValue, Value falseValue) {
242 boolean mirrored = emitCompare(cmpKind, left, right, cond, unorderedIsTrue);
243 Condition finalCondition = mirrored ? cond.mirror() : cond;
244 boolean finalUnorderedIsTrue = mirrored ? !unorderedIsTrue : unorderedIsTrue;
245 ConditionFlag cmpCondition = toConditionFlag(((AArch64Kind) cmpKind).isInteger(), finalCondition, finalUnorderedIsTrue);
246 Variable result = newVariable(trueValue.getValueKind());
247
248 if (isIntConstant(trueValue, 1) && isIntConstant(falseValue, 0)) {
249 append(new CondSetOp(result, cmpCondition));
250 } else if (isIntConstant(trueValue, 0) && isIntConstant(falseValue, 1)) {
251 append(new CondSetOp(result, cmpCondition.negate()));
252 } else {
253 append(new CondMoveOp(result, cmpCondition, loadReg(trueValue), loadReg(falseValue)));
254 }
255 return result;
256 }
257
258 @Override
259 public void emitCompareBranch(PlatformKind cmpKind, Value left, Value right, Condition cond, boolean unorderedIsTrue, LabelRef trueDestination, LabelRef falseDestination,
260 double trueDestinationProbability) {
261 if (cond == Condition.EQ) {
262 // emit cbz instruction for IsNullNode.
263 assert !LIRValueUtil.isNullConstant(left) : "emitNullCheckBranch()'s null input should be in right.";
264 if (LIRValueUtil.isNullConstant(right)) {
265 append(new CompareBranchZeroOp(asAllocatable(left), trueDestination, falseDestination, trueDestinationProbability));
266 return;
267 }
268
269 // emit cbz instruction for IntegerEquals when any of the inputs is zero.
270 AArch64Kind kind = (AArch64Kind) cmpKind;
271 if (kind.isInteger()) {
272 if (isIntConstant(left, 0)) {
273 append(new CompareBranchZeroOp(asAllocatable(right), trueDestination, falseDestination, trueDestinationProbability));
274 return;
275 } else if (isIntConstant(right, 0)) {
276 append(new CompareBranchZeroOp(asAllocatable(left), trueDestination, falseDestination, trueDestinationProbability));
277 return;
278 }
279 }
280 }
281
282 boolean mirrored = emitCompare(cmpKind, left, right, cond, unorderedIsTrue);
283 Condition finalCondition = mirrored ? cond.mirror() : cond;
284 boolean finalUnorderedIsTrue = mirrored ? !unorderedIsTrue : unorderedIsTrue;
285 ConditionFlag cmpCondition = toConditionFlag(((AArch64Kind) cmpKind).isInteger(), finalCondition, finalUnorderedIsTrue);
286 append(new BranchOp(cmpCondition, trueDestination, falseDestination, trueDestinationProbability));
287 }
288
289 private static ConditionFlag toConditionFlag(boolean isInt, Condition cond, boolean unorderedIsTrue) {
290 return isInt ? toIntConditionFlag(cond) : toFloatConditionFlag(cond, unorderedIsTrue);
291 }
292
293 /**
294 * Takes a Condition and unorderedIsTrue flag and returns the correct Aarch64 specific
295 * ConditionFlag. Note: This is only correct if the emitCompare code for floats has correctly
296 * handled the case of 'EQ && unorderedIsTrue', respectively 'NE && !unorderedIsTrue'!
297 */
298 private static ConditionFlag toFloatConditionFlag(Condition cond, boolean unorderedIsTrue) {
299 switch (cond) {
300 case LT:
301 return unorderedIsTrue ? ConditionFlag.LT : ConditionFlag.LO;
302 case LE:
|
147 } else {
148 return new AArch64AddressValue(address.getValueKind(), asAllocatable(address), Value.ILLEGAL, 0, 1, AddressingMode.BASE_REGISTER_ONLY);
149 }
150 }
151
152 @Override
153 public Variable emitLogicCompareAndSwap(LIRKind accessKind, Value address, Value expectedValue, Value newValue, Value trueValue, Value falseValue) {
154 Variable prevValue = newVariable(expectedValue.getValueKind());
155 Variable scratch = newVariable(LIRKind.value(AArch64Kind.DWORD));
156 append(new CompareAndSwapOp(prevValue, loadReg(expectedValue), loadReg(newValue), asAllocatable(address), scratch));
157 assert trueValue.getValueKind().equals(falseValue.getValueKind());
158 Variable result = newVariable(trueValue.getValueKind());
159 append(new CondMoveOp(result, ConditionFlag.EQ, asAllocatable(trueValue), asAllocatable(falseValue)));
160 return result;
161 }
162
163 @Override
164 public Variable emitValueCompareAndSwap(LIRKind accessKind, Value address, Value expectedValue, Value newValue) {
165 Variable result = newVariable(newValue.getValueKind());
166 Variable scratch = newVariable(LIRKind.value(AArch64Kind.WORD));
167 append(new CompareAndSwapOp(result, loadReg(expectedValue), loadReg(newValue), asAllocatable(address), scratch));
168 return result;
169 }
170
171 @Override
172 public Value emitAtomicReadAndWrite(Value address, ValueKind<?> kind, Value newValue) {
173 Variable result = newVariable(kind);
174 Variable scratch = newVariable(kind);
175 append(new AtomicReadAndWriteOp((AArch64Kind) kind.getPlatformKind(), asAllocatable(result), asAllocatable(address), asAllocatable(newValue), asAllocatable(scratch)));
176 return result;
177 }
178
179 @Override
180 public Value emitAtomicReadAndAdd(Value address, ValueKind<?> kind, Value delta) {
181 Variable result = newVariable(kind);
182 if (AArch64LIRFlagsVersioned.useLSE(target().arch)) {
183 append(new AtomicReadAndAddLSEOp((AArch64Kind) kind.getPlatformKind(), asAllocatable(result), asAllocatable(address), asAllocatable(delta)));
184 } else {
185 append(new AtomicReadAndAddOp((AArch64Kind) kind.getPlatformKind(), asAllocatable(result), asAllocatable(address), delta));
186 }
187 return result;
221 ((AArch64ArithmeticLIRGenerator) getArithmetic()).emitBinary(LIRKind.combine(left, right), AArch64ArithmeticOp.ANDS, true, left, right);
222 append(new AArch64ControlFlow.BranchOp(ConditionFlag.EQ, trueDestination, falseDestination, trueSuccessorProbability));
223 }
224
225 /**
226 * Conditionally move trueValue into new variable if cond + unorderedIsTrue is true, else
227 * falseValue.
228 *
229 * @param left Arbitrary value. Has to have same type as right. Non null.
230 * @param right Arbitrary value. Has to have same type as left. Non null.
231 * @param cond condition that decides whether to move trueValue or falseValue into result. Non
232 * null.
233 * @param unorderedIsTrue defines whether floating-point comparisons consider unordered true or
234 * not. Ignored for integer comparisons.
235 * @param trueValue arbitrary value same type as falseValue. Non null.
236 * @param falseValue arbitrary value same type as trueValue. Non null.
237 * @return value containing trueValue if cond + unorderedIsTrue is true, else falseValue. Non
238 * null.
239 */
240 @Override
241 public Variable emitConditionalMove(PlatformKind cmpKind, Value left, final Value right, Condition cond, boolean unorderedIsTrue, Value trueValue, Value falseValue) {
242 AArch64ArithmeticLIRGenerator arithLir = ((AArch64ArithmeticLIRGenerator) arithmeticLIRGen);
243 Value actualRight = right;
244 if (isJavaConstant(actualRight) && arithLir.mustReplaceNullWithNullRegister((asJavaConstant(actualRight)))) {
245 actualRight = arithLir.getNullRegisterValue();
246 }
247 boolean mirrored = emitCompare(cmpKind, left, actualRight, cond, unorderedIsTrue);
248 Condition finalCondition = mirrored ? cond.mirror() : cond;
249 boolean finalUnorderedIsTrue = mirrored ? !unorderedIsTrue : unorderedIsTrue;
250 ConditionFlag cmpCondition = toConditionFlag(((AArch64Kind) cmpKind).isInteger(), finalCondition, finalUnorderedIsTrue);
251 Variable result = newVariable(trueValue.getValueKind());
252
253 if (isIntConstant(trueValue, 1) && isIntConstant(falseValue, 0)) {
254 append(new CondSetOp(result, cmpCondition));
255 } else if (isIntConstant(trueValue, 0) && isIntConstant(falseValue, 1)) {
256 append(new CondSetOp(result, cmpCondition.negate()));
257 } else {
258 append(new CondMoveOp(result, cmpCondition, loadReg(trueValue), loadReg(falseValue)));
259 }
260 return result;
261 }
262
263 @Override
264 public void emitCompareBranch(PlatformKind cmpKind, Value left, final Value right, Condition cond, boolean unorderedIsTrue, LabelRef trueDestination, LabelRef falseDestination,
265 double trueDestinationProbability) {
266 Value actualRight = right;
267 if (cond == Condition.EQ) {
268 // emit cbz instruction for IsNullNode.
269 assert !LIRValueUtil.isNullConstant(left) : "emitNullCheckBranch()'s null input should be in right.";
270 AArch64ArithmeticLIRGenerator arithLir = ((AArch64ArithmeticLIRGenerator) arithmeticLIRGen);
271 if (LIRValueUtil.isNullConstant(actualRight)) {
272 JavaConstant rightConstant = asJavaConstant(actualRight);
273 if (arithLir.mustReplaceNullWithNullRegister(rightConstant)) {
274 actualRight = arithLir.getNullRegisterValue();
275 } else {
276 append(new CompareBranchZeroOp(asAllocatable(left), trueDestination, falseDestination,
277 trueDestinationProbability));
278 return;
279 }
280 }
281
282 // emit cbz instruction for IntegerEquals when any of the inputs is zero.
283 AArch64Kind kind = (AArch64Kind) cmpKind;
284 if (kind.isInteger()) {
285 if (isIntConstant(left, 0)) {
286 append(new CompareBranchZeroOp(asAllocatable(actualRight), trueDestination, falseDestination, trueDestinationProbability));
287 return;
288 } else if (isIntConstant(actualRight, 0)) {
289 append(new CompareBranchZeroOp(asAllocatable(left), trueDestination, falseDestination, trueDestinationProbability));
290 return;
291 }
292 }
293 }
294
295 boolean mirrored = emitCompare(cmpKind, left, actualRight, cond, unorderedIsTrue);
296 Condition finalCondition = mirrored ? cond.mirror() : cond;
297 boolean finalUnorderedIsTrue = mirrored ? !unorderedIsTrue : unorderedIsTrue;
298 ConditionFlag cmpCondition = toConditionFlag(((AArch64Kind) cmpKind).isInteger(), finalCondition, finalUnorderedIsTrue);
299 append(new BranchOp(cmpCondition, trueDestination, falseDestination, trueDestinationProbability));
300 }
301
302 private static ConditionFlag toConditionFlag(boolean isInt, Condition cond, boolean unorderedIsTrue) {
303 return isInt ? toIntConditionFlag(cond) : toFloatConditionFlag(cond, unorderedIsTrue);
304 }
305
306 /**
307 * Takes a Condition and unorderedIsTrue flag and returns the correct Aarch64 specific
308 * ConditionFlag. Note: This is only correct if the emitCompare code for floats has correctly
309 * handled the case of 'EQ && unorderedIsTrue', respectively 'NE && !unorderedIsTrue'!
310 */
311 private static ConditionFlag toFloatConditionFlag(Condition cond, boolean unorderedIsTrue) {
312 switch (cond) {
313 case LT:
314 return unorderedIsTrue ? ConditionFlag.LT : ConditionFlag.LO;
315 case LE:
|