1 /*
2 * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2018, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 */
24
25
26 package org.graalvm.compiler.asm.aarch64;
27
28 import static jdk.vm.ci.aarch64.AArch64.CPU;
29 import static jdk.vm.ci.aarch64.AArch64.SIMD;
30 import static jdk.vm.ci.aarch64.AArch64.cpuRegisters;
31 import static jdk.vm.ci.aarch64.AArch64.r0;
32 import static jdk.vm.ci.aarch64.AArch64.sp;
33 import static jdk.vm.ci.aarch64.AArch64.zr;
34 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.ADD;
35 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.ADDS;
36 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.ADDV;
37 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.ADR;
38 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.ADRP;
39 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.AND;
40 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.ANDS;
41 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.ASRV;
42 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.BFM;
43 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.BIC;
44 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.BICS;
45 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.BLR;
46 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.BR;
47 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.BRK;
48 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.CAS;
49 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.CCMP;
50 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.CLREX;
51 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.CLS;
52 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.CLZ;
53 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.CNT;
54 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.CSEL;
55 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.CSINC;
56 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.CSNEG;
57 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.DMB;
58 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.EON;
59 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.EOR;
60 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.EXTR;
61 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.FABS;
62 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.FADD;
63 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.FCCMP;
64 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.FCMP;
65 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.FCMPZERO;
66 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.FCSEL;
67 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.FCVTDS;
68 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.FCVTSD;
69 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.FCVTZS;
70 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.FDIV;
71 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.FMADD;
72 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.FMOV;
73 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.FMSUB;
74 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.FMUL;
75 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.FNEG;
76 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.FRINTM;
77 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.FRINTN;
78 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.FRINTP;
79 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.FRINTZ;
80 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.FSQRT;
81 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.FSUB;
82 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.HINT;
83 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.HLT;
84 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.LDADD;
85 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.LDAR;
86 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.LDAXR;
87 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.LDP;
88 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.LDR;
89 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.LDRS;
90 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.LDXR;
91 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.LSLV;
92 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.LSRV;
93 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.MADD;
94 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.MOVK;
95 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.MOVN;
96 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.MOVZ;
97 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.MRS;
98 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.MSUB;
99 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.ORN;
100 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.ORR;
101 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.RBIT;
102 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.RET;
103 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.REVW;
104 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.REVX;
105 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.RORV;
106 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.SBFM;
107 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.SCVTF;
108 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.SDIV;
109 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.STLR;
110 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.STLXR;
111 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.STP;
112 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.STR;
113 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.STXR;
114 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.SUB;
115 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.SUBS;
116 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.SWP;
117 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.TBNZ;
118 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.TBZ;
119 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.UBFM;
120 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.UDIV;
121 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.Instruction.UMOV;
122 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.InstructionType.FP32;
123 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.InstructionType.FP64;
124 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.InstructionType.General32;
125 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.InstructionType.General64;
126 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.InstructionType.floatFromSize;
127 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.InstructionType.generalFromSize;
128 import static org.graalvm.compiler.asm.aarch64.AArch64Assembler.InstructionType.simdFromSize;
129
130 import java.util.Arrays;
131
132 import org.graalvm.compiler.asm.Assembler;
133 import org.graalvm.compiler.asm.aarch64.AArch64Address.AddressingMode;
134 import org.graalvm.compiler.core.common.NumUtil;
135 import org.graalvm.compiler.debug.GraalError;
136
137 import jdk.vm.ci.aarch64.AArch64;
138 import jdk.vm.ci.aarch64.AArch64.CPUFeature;
139 import jdk.vm.ci.aarch64.AArch64.Flag;
140 import jdk.vm.ci.code.Register;
141 import jdk.vm.ci.code.TargetDescription;
142
143 public abstract class AArch64Assembler extends Assembler {
144
145 public static class LogicalImmediateTable {
146
147 private static final Immediate[] IMMEDIATE_TABLE = buildImmediateTable();
148
149 private static final int ImmediateOffset = 10;
150 private static final int ImmediateRotateOffset = 16;
151 private static final int ImmediateSizeOffset = 22;
152
153 /**
154 * Specifies whether immediate can be represented in all cases (YES), as a 64bit instruction
155 * (SIXTY_FOUR_BIT_ONLY) or not at all (NO).
156 */
157 enum Representable {
158 YES,
159 SIXTY_FOUR_BIT_ONLY,
160 NO
161 }
162
163 /**
164 * Tests whether an immediate can be encoded for logical instructions.
165 *
166 * @param is64bit if true immediate is considered a 64-bit pattern. If false we may use a
167 * 64-bit instruction to load the 32-bit pattern into a register.
168 * @return enum specifying whether immediate can be used for 32- and 64-bit logical
169 * instructions ({@code #Representable.YES}), for 64-bit instructions only (
170 * {@link Representable#SIXTY_FOUR_BIT_ONLY}) or not at all (
171 * {@link Representable#NO}).
172 */
173 public static Representable isRepresentable(boolean is64bit, long immediate) {
174 int pos = getLogicalImmTablePos(is64bit, immediate);
175 if (pos < 0) {
176 // if 32bit instruction we can try again as 64bit immediate which may succeed.
177 // i.e. 0xffffffff fails as a 32bit immediate but works as 64bit one.
178 if (!is64bit) {
179 assert NumUtil.isUnsignedNbit(32, immediate);
180 pos = getLogicalImmTablePos(true, immediate);
181 return pos >= 0 ? Representable.SIXTY_FOUR_BIT_ONLY : Representable.NO;
182 }
183 return Representable.NO;
184 }
185 Immediate imm = IMMEDIATE_TABLE[pos];
186 return imm.only64bit() ? Representable.SIXTY_FOUR_BIT_ONLY : Representable.YES;
187 }
188
189 public static Representable isRepresentable(int immediate) {
190 return isRepresentable(false, immediate & 0xFFFF_FFFFL);
191 }
192
193 public static int getLogicalImmEncoding(boolean is64bit, long value) {
194 int pos = getLogicalImmTablePos(is64bit, value);
195 assert pos >= 0 : "Value cannot be represented as logical immediate: " + value + ", is64bit=" + is64bit;
196 Immediate imm = IMMEDIATE_TABLE[pos];
197 assert is64bit || !imm.only64bit() : "Immediate can only be represented for 64bit, but 32bit instruction specified";
198 return IMMEDIATE_TABLE[pos].encoding;
199 }
200
201 /**
202 * @param is64bit if true also allow 64-bit only encodings to be returned.
203 * @return If positive the return value is the position into the IMMEDIATE_TABLE for the
204 * given immediate, if negative the immediate cannot be encoded.
205 */
206 private static int getLogicalImmTablePos(boolean is64bit, long value) {
207 Immediate imm;
208 if (!is64bit) {
209 // 32bit instructions can only have 32bit immediates.
210 if (!NumUtil.isUnsignedNbit(32, value)) {
211 return -1;
212 }
213 // If we have a 32bit instruction (and therefore immediate) we have to duplicate it
214 // across 64bit to find it in the table.
215 imm = new Immediate(value << 32 | value);
216 } else {
217 imm = new Immediate(value);
218 }
219 int pos = Arrays.binarySearch(IMMEDIATE_TABLE, imm);
220 if (pos < 0) {
221 return -1;
222 }
223 if (!is64bit && IMMEDIATE_TABLE[pos].only64bit()) {
224 return -1;
225 }
226 return pos;
227 }
228
229 /**
230 * To quote 5.4.2: [..] an immediate is a 32 or 64 bit pattern viewed as a vector of
231 * identical elements of size e = 2, 4, 8, 16, 32 or (in the case of bimm64) 64 bits. Each
232 * element contains the same sub-pattern: a single run of 1 to e-1 non-zero bits, rotated by
233 * 0 to e-1 bits. It is encoded in the following: 10-16: rotation amount (6bit) starting
234 * from 1s in the LSB (i.e. 0111->1011->1101->1110) 16-22: This stores a combination of the
235 * number of set bits and the pattern size. The pattern size is encoded as follows (x is
236 * used to store the number of 1 bits - 1) e pattern 2 1111xx 4 1110xx 8 110xxx 16 10xxxx 32
237 * 0xxxxx 64 xxxxxx 22: if set we have an instruction with 64bit pattern?
238 */
239 private static final class Immediate implements Comparable<Immediate> {
240 public final long imm;
241 public final int encoding;
242
243 Immediate(long imm, boolean is64, int s, int r) {
244 this.imm = imm;
245 this.encoding = computeEncoding(is64, s, r);
246 }
247
248 // Used to be able to binary search for an immediate in the table.
249 Immediate(long imm) {
250 this(imm, false, 0, 0);
251 }
252
253 /**
254 * Returns true if this pattern is only representable as 64bit.
255 */
256 public boolean only64bit() {
257 return (encoding & (1 << ImmediateSizeOffset)) != 0;
258 }
259
260 private static int computeEncoding(boolean is64, int s, int r) {
261 int sf = is64 ? 1 : 0;
262 return sf << ImmediateSizeOffset | r << ImmediateRotateOffset | s << ImmediateOffset;
263 }
264
265 @Override
266 public int compareTo(Immediate o) {
267 return Long.compare(imm, o.imm);
268 }
269 }
270
271 private static Immediate[] buildImmediateTable() {
272 final int nrImmediates = 5334;
273 final Immediate[] table = new Immediate[nrImmediates];
274 int nrImms = 0;
275 for (int logE = 1; logE <= 6; logE++) {
276 int e = 1 << logE;
277 long mask = NumUtil.getNbitNumberLong(e);
278 for (int nrOnes = 1; nrOnes < e; nrOnes++) {
279 long val = (1L << nrOnes) - 1;
280 // r specifies how much we rotate the value
281 for (int r = 0; r < e; r++) {
282 long immediate = (val >>> r | val << (e - r)) & mask;
283 // Duplicate pattern to fill whole 64bit range.
284 switch (logE) {
285 case 1:
286 immediate |= immediate << 2;
287 immediate |= immediate << 4;
288 immediate |= immediate << 8;
289 immediate |= immediate << 16;
290 immediate |= immediate << 32;
291 break;
292 case 2:
293 immediate |= immediate << 4;
294 immediate |= immediate << 8;
295 immediate |= immediate << 16;
296 immediate |= immediate << 32;
297 break;
298 case 3:
299 immediate |= immediate << 8;
300 immediate |= immediate << 16;
301 immediate |= immediate << 32;
302 break;
303 case 4:
304 immediate |= immediate << 16;
305 immediate |= immediate << 32;
306 break;
307 case 5:
308 immediate |= immediate << 32;
309 break;
310 }
311 // 5 - logE can underflow to -1, but we shift this bogus result
312 // out of the masked area.
313 int sizeEncoding = (1 << (5 - logE)) - 1;
314 int s = ((sizeEncoding << (logE + 1)) & 0x3f) | (nrOnes - 1);
315 table[nrImms++] = new Immediate(immediate, /* is64bit */e == 64, s, r);
316 }
317 }
318 }
319 Arrays.sort(table);
320 assert nrImms == nrImmediates : nrImms + " instead of " + nrImmediates + " in table.";
321 assert checkDuplicates(table) : "Duplicate values in table.";
322 return table;
323 }
324
325 private static boolean checkDuplicates(Immediate[] table) {
326 for (int i = 0; i < table.length - 1; i++) {
327 if (table[i].imm >= table[i + 1].imm) {
328 return false;
329 }
330 }
331 return true;
332 }
333 }
334
335 private static final int RdOffset = 0;
336 private static final int Rs1Offset = 5;
337 private static final int Rs2Offset = 16;
338 private static final int Rs3Offset = 10;
339 private static final int RtOffset = 0;
340 private static final int RnOffset = 5;
341 private static final int Rt2Offset = 10;
342
343 /* Helper functions */
344 private static int rd(Register reg) {
345 return reg.encoding << RdOffset;
346 }
347
348 private static int rs1(Register reg) {
349 return reg.encoding << Rs1Offset;
350 }
351
352 private static int rs2(Register reg) {
353 return reg.encoding << Rs2Offset;
354 }
355
356 private static int rs3(Register reg) {
357 return reg.encoding << Rs3Offset;
358 }
359
360 private static int rt(Register reg) {
361 return reg.encoding << RtOffset;
362 }
363
364 private static int rt2(Register reg) {
365 return reg.encoding << Rt2Offset;
366 }
367
368 private static int rn(Register reg) {
369 return reg.encoding << RnOffset;
370 }
371
372 private static int maskField(int sizeInBits, int n) {
373 assert NumUtil.isSignedNbit(sizeInBits, n);
374 return n & NumUtil.getNbitNumberInt(sizeInBits);
375 }
376
377 /**
378 * Enumeration of all different lane types of SIMD register.
379 *
380 * Byte(B):8b/lane; HalfWord(H):16b/lane; Word(S):32b/lane; DoubleWord(D):64b/lane.
381 */
382 public enum SIMDElementSize {
383 Byte(0, 8),
384 HalfWord(1, 16),
385 Word(2, 32),
386 DoubleWord(3, 64);
387
388 public final int encoding;
389 public final int nbits;
390
391 SIMDElementSize(int encoding, int nbits) {
392 this.encoding = encoding;
393 this.nbits = nbits;
394 }
395 }
396
397 /**
398 * Enumeration of all different instruction kinds: General32/64 are the general instructions
399 * (integer, branch, etc.), for 32-, respectively 64-bit operands. FP32/64 is the encoding for
400 * the 32/64bit float operations. SIMDByte/HalfWord/Word/DoubleWord is the encoding for SIMD
401 * instructions
402 */
403 protected enum InstructionType {
404 General32(0b00 << 30, 32, true),
405 General64(0b10 << 30, 64, true),
406 FP32(0x00000000, 32, false),
407 FP64(0x00400000, 64, false),
408
409 SIMDByte(0x01, 8, false),
410 SIMDHalfWord(0x02, 16, false),
411 SIMDWord(0x04, 32, false),
412 SIMDDoubleWord(0x08, 64, false);
413
414 public final int encoding;
415 public final int width;
416 public final boolean isGeneral;
417
418 InstructionType(int encoding, int width, boolean isGeneral) {
419 this.encoding = encoding;
420 this.width = width;
421 this.isGeneral = isGeneral;
422 }
423
424 public static InstructionType generalFromSize(int size) {
425 assert size == 32 || size == 64;
426 return size == 32 ? General32 : General64;
427 }
428
429 public static InstructionType floatFromSize(int size) {
430 assert size == 32 || size == 64;
431 return size == 32 ? FP32 : FP64;
432 }
433
434 public static InstructionType simdFromSize(int size) {
435 switch (size) {
436 case 8:
437 return SIMDByte;
438 case 16:
439 return SIMDHalfWord;
440 case 32:
441 return SIMDWord;
442 case 64:
443 return SIMDDoubleWord;
444 default:
445 throw GraalError.shouldNotReachHere();
446 }
447 }
448 }
449
450 private static final int ImmediateOffset = 10;
451 private static final int ImmediateRotateOffset = 16;
452 private static final int ImmediateSizeOffset = 22;
453 private static final int ExtendTypeOffset = 13;
454
455 private static final int AddSubImmOp = 0x11000000;
456 private static final int AddSubShift12 = 0b01 << 22;
457 private static final int AddSubSetFlag = 0x20000000;
458
459 private static final int LogicalImmOp = 0x12000000;
460
461 private static final int MoveWideImmOp = 0x12800000;
462 private static final int MoveWideImmOffset = 5;
463 private static final int MoveWideShiftOffset = 21;
464
465 private static final int BitfieldImmOp = 0x13000000;
466
467 private static final int AddSubShiftedOp = 0x0B000000;
468 private static final int ShiftTypeOffset = 22;
469
470 private static final int AddSubExtendedOp = 0x0B200000;
471
472 private static final int MulOp = 0x1B000000;
473 private static final int DataProcessing1SourceOp = 0x5AC00000;
474 private static final int DataProcessing2SourceOp = 0x1AC00000;
475
476 private static final int Fp1SourceOp = 0x1E204000;
477 private static final int Fp2SourceOp = 0x1E200800;
478 private static final int Fp3SourceOp = 0x1F000000;
479
480 private static final int FpConvertOp = 0x1E200000;
481 private static final int FpImmOp = 0x1E201000;
482 private static final int FpImmOffset = 13;
483
484 private static final int FpCmpOp = 0x1E202000;
485 private static final int FpCmpeOp = 0x1E202010;
486
487 private static final int PcRelImmHiOffset = 5;
488 private static final int PcRelImmLoOffset = 29;
489
490 private static final int PcRelImmOp = 0x10000000;
491
492 private static final int UnconditionalBranchImmOp = 0x14000000;
493 private static final int UnconditionalBranchRegOp = 0xD6000000;
494 private static final int CompareBranchOp = 0x34000000;
495
496 private static final int ConditionalBranchImmOffset = 5;
497
498 private static final int ConditionalSelectOp = 0x1A800000;
499 private static final int ConditionalConditionOffset = 12;
500
501 private static final int LoadStoreScaledOp = 0b111_0_01_00 << 22;
502 private static final int LoadStoreUnscaledOp = 0b111_0_00_00 << 22;
503
504 private static final int LoadStoreRegisterOp = 0b111_0_00_00_1 << 21 | 0b10 << 10;
505
506 private static final int LoadLiteralOp = 0x18000000;
507
508 private static final int LoadStorePostIndexedOp = 0b111_0_00_00_0 << 21 | 0b01 << 10;
509 private static final int LoadStorePreIndexedOp = 0b111_0_00_00_0 << 21 | 0b11 << 10;
510
511 private static final int LoadStoreUnscaledImmOffset = 12;
512 private static final int LoadStoreScaledImmOffset = 10;
513 private static final int LoadStoreScaledRegOffset = 12;
514 private static final int LoadStoreIndexedImmOffset = 12;
515 private static final int LoadStoreTransferSizeOffset = 30;
516 private static final int LoadStoreFpFlagOffset = 26;
517 private static final int LoadLiteralImmeOffset = 5;
518
519 private static final int LoadStorePairOp = 0b101_0 << 26;
520 @SuppressWarnings("unused") private static final int LoadStorePairPostIndexOp = 0b101_0_001 << 23;
521 @SuppressWarnings("unused") private static final int LoadStorePairPreIndexOp = 0b101_0_011 << 23;
522 private static final int LoadStorePairImm7Offset = 15;
523
524 private static final int LogicalShiftOp = 0x0A000000;
525
526 private static final int ExceptionOp = 0xD4000000;
527 private static final int SystemImmediateOffset = 5;
528
529 @SuppressWarnings("unused") private static final int SimdImmediateOffset = 16;
530
531 private static final int BarrierOp = 0xD503301F;
532 private static final int BarrierKindOffset = 8;
533
534 private static final int CASAcquireOffset = 22;
535 private static final int CASReleaseOffset = 15;
536
537 private static final int LDADDAcquireOffset = 23;
538 private static final int LDADDReleaseOffset = 22;
539
540 private static final int SIMDImm5Offset = 16;
541 private static final int SIMDQBitOffset = 30;
542 private static final int SIMDSizeOffset = 22;
543
544 /**
545 * Encoding for all instructions.
546 */
547 public enum Instruction {
548 BCOND(0x54000000),
549 CBNZ(0x01000000),
550 CBZ(0x00000000),
551 TBZ(0x36000000),
552 TBNZ(0x37000000),
553
554 B(0x00000000),
555 BL(0x80000000),
556 BR(0x001F0000),
557 BLR(0x003F0000),
558 RET(0x005F0000),
559
560 LDR(0x00000000),
561 LDRS(0x00800000),
562 LDXR(0x081f7c00),
563 LDAR(0x8dffc00),
564 LDAXR(0x85ffc00),
565
566 STR(0x00000000),
567 STXR(0x08007c00),
568 STLR(0x089ffc00),
569 STLXR(0x0800fc00),
570
571 LDP(0b1 << 22),
572 STP(0b0 << 22),
573
574 CAS(0x08A07C00),
575 LDADD(0x38200000),
576 SWP(0x38208000),
577
578 ADR(0x00000000),
579 ADRP(0x80000000),
580
581 ADD(0x00000000),
582 ADDS(ADD.encoding | AddSubSetFlag),
583 SUB(0x40000000),
584 SUBS(SUB.encoding | AddSubSetFlag),
585
586 CCMP(0x7A400000),
587
588 NOT(0x00200000),
589 AND(0x00000000),
590 BIC(AND.encoding | NOT.encoding),
591 ORR(0x20000000),
592 ORN(ORR.encoding | NOT.encoding),
593 EOR(0x40000000),
594 EON(EOR.encoding | NOT.encoding),
595 ANDS(0x60000000),
596 BICS(ANDS.encoding | NOT.encoding),
597
598 ASRV(0x00002800),
599 RORV(0x00002C00),
600 LSRV(0x00002400),
601 LSLV(0x00002000),
602
603 CLS(0x00001400),
604 CLZ(0x00001000),
605 RBIT(0x00000000),
606 REVX(0x00000C00),
607 REVW(0x00000800),
608
609 MOVN(0x00000000),
610 MOVZ(0x40000000),
611 MOVK(0x60000000),
612
613 CSEL(0x00000000),
614 CSNEG(0x40000400),
615 CSINC(0x00000400),
616
617 BFM(0x20000000),
618 SBFM(0x00000000),
619 UBFM(0x40000000),
620 EXTR(0x13800000),
621
622 MADD(0x00000000),
623 MSUB(0x00008000),
624 SDIV(0x00000C00),
625 UDIV(0x00000800),
626
627 FMOV(0x00000000),
628 FMOVCPU2FPU(0x00070000),
629 FMOVFPU2CPU(0x00060000),
630
631 FCVTDS(0x00028000),
632 FCVTSD(0x00020000),
633
634 FCVTZS(0x00180000),
635 SCVTF(0x00020000),
636
637 FABS(0x00008000),
638 FSQRT(0x00018000),
639 FNEG(0x00010000),
640
641 FRINTM(0x00050000),
642 FRINTN(0x00040000),
643 FRINTP(0x00048000),
644 FRINTZ(0x00058000),
645
646 FADD(0x00002000),
647 FSUB(0x00003000),
648 FMUL(0x00000000),
649 FDIV(0x00001000),
650 FMAX(0x00004000),
651 FMIN(0x00005000),
652
653 FMADD(0x00000000),
654 FMSUB(0x00008000),
655
656 FCMP(0x00000000),
657 FCMPZERO(0x00000008),
658 FCCMP(0x1E200400),
659 FCSEL(0x1E200C00),
660
661 INS(0x4e081c00),
662 UMOV(0x0e003c00),
663
664 CNT(0xe205800),
665 USRA(0x6f001400),
666
667 HLT(0x00400000),
668 BRK(0x00200000),
669
670 CLREX(0xd5033f5f),
671 HINT(0xD503201F),
672 DMB(0x000000A0),
673
674 MRS(0xD5300000),
675 MSR(0xD5100000),
676
677 BLR_NATIVE(0xc0000000),
678
679 ADDV(0x0e31b800);
680
681 public final int encoding;
682
683 Instruction(int encoding) {
684 this.encoding = encoding;
685 }
686
687 }
688
689 public enum SystemRegister {
690 FPCR(0b11, 0b011, 0b0100, 0b0100, 0b000),
691 FPSR(0b11, 0b011, 0b0100, 0b0100, 0b001);
692
693 SystemRegister(int op0, int op1, int crn, int crm, int op2) {
694 this.op0 = op0;
695 this.op1 = op1;
696 this.crn = crn;
697 this.crm = crm;
698 this.op2 = op2;
699 }
700
701 public int encoding() {
702 return op0 << 19 | op1 << 16 | crn << 12 | crm << 8 | op2 << 5;
703 }
704
705 private final int op0;
706 private final int op1;
707 private final int crn;
708 private final int crm;
709 private final int op2;
710 }
711
712 public enum ShiftType {
713 LSL(0),
714 LSR(1),
715 ASR(2),
716 ROR(3);
717
718 public final int encoding;
719
720 ShiftType(int encoding) {
721 this.encoding = encoding;
722 }
723 }
724
725 public enum ExtendType {
726 UXTB(0),
727 UXTH(1),
728 UXTW(2),
729 UXTX(3),
730 SXTB(4),
731 SXTH(5),
732 SXTW(6),
733 SXTX(7);
734
735 public final int encoding;
736
737 ExtendType(int encoding) {
738 this.encoding = encoding;
739 }
740 }
741
742 /**
743 * Condition Flags for branches. See 4.3
744 */
745 public enum ConditionFlag {
746 // Integer | Floating-point meanings
747 /** Equal | Equal. */
748 EQ(0x0),
749
750 /** Not Equal | Not equal or unordered. */
751 NE(0x1),
752
753 /** Unsigned Higher or Same | Greater than, equal or unordered. */
754 HS(0x2),
755
756 /** Unsigned lower | less than. */
757 LO(0x3),
758
759 /** Minus (negative) | less than. */
760 MI(0x4),
761
762 /** Plus (positive or zero) | greater than, equal or unordered. */
763 PL(0x5),
764
765 /** Overflow set | unordered. */
766 VS(0x6),
767
768 /** Overflow clear | ordered. */
769 VC(0x7),
770
771 /** Unsigned higher | greater than or unordered. */
772 HI(0x8),
773
774 /** Unsigned lower or same | less than or equal. */
775 LS(0x9),
776
777 /** Signed greater than or equal | greater than or equal. */
778 GE(0xA),
779
780 /** Signed less than | less than or unordered. */
781 LT(0xB),
782
783 /** Signed greater than | greater than. */
784 GT(0xC),
785
786 /** Signed less than or equal | less than, equal or unordered. */
787 LE(0xD),
788
789 /** Always | always. */
790 AL(0xE),
791
792 /** Always | always (identical to AL, just to have valid 0b1111 encoding). */
793 NV(0xF);
794
795 public final int encoding;
796
797 ConditionFlag(int encoding) {
798 this.encoding = encoding;
799 }
800
801 /**
802 * @return ConditionFlag specified by decoding.
803 */
804 public static ConditionFlag fromEncoding(int encoding) {
805 return values()[encoding];
806 }
807
808 public ConditionFlag negate() {
809 switch (this) {
810 case EQ:
811 return NE;
812 case NE:
813 return EQ;
814 case HS:
815 return LO;
816 case LO:
817 return HS;
818 case MI:
819 return PL;
820 case PL:
821 return MI;
822 case VS:
823 return VC;
824 case VC:
825 return VS;
826 case HI:
827 return LS;
828 case LS:
829 return HI;
830 case GE:
831 return LT;
832 case LT:
833 return GE;
834 case GT:
835 return LE;
836 case LE:
837 return GT;
838 case AL:
839 case NV:
840 default:
841 throw GraalError.shouldNotReachHere();
842 }
843 }
844 }
845
846 public AArch64Assembler(TargetDescription target) {
847 super(target);
848 }
849
850 public boolean supports(CPUFeature feature) {
851 return ((AArch64) target.arch).getFeatures().contains(feature);
852 }
853
854 public boolean isFlagSet(Flag flag) {
855 return ((AArch64) target.arch).getFlags().contains(flag);
856 }
857
858 /* Conditional Branch (5.2.1) */
859
860 /**
861 * Branch conditionally.
862 *
863 * @param condition may not be null.
864 * @param imm21 Signed 21-bit offset, has to be word aligned.
865 */
866 protected void b(ConditionFlag condition, int imm21) {
867 b(condition, imm21, -1);
868 }
869
870 /**
871 * Branch conditionally. Inserts instruction into code buffer at pos.
872 *
873 * @param condition may not be null.
874 * @param imm21 Signed 21-bit offset, has to be word aligned.
875 * @param pos Position at which instruction is inserted into buffer. -1 means insert at end.
876 */
877 protected void b(ConditionFlag condition, int imm21, int pos) {
878 if (pos == -1) {
879 emitInt(Instruction.BCOND.encoding | getConditionalBranchImm(imm21) | condition.encoding);
880 } else {
881 emitInt(Instruction.BCOND.encoding | getConditionalBranchImm(imm21) | condition.encoding, pos);
882 }
883 }
884
885 /**
886 * Compare register and branch if non-zero.
887 *
888 * @param reg general purpose register. May not be null, zero-register or stackpointer.
889 * @param size Instruction size in bits. Should be either 32 or 64.
890 * @param imm21 Signed 21-bit offset, has to be word aligned.
891 */
892 protected void cbnz(int size, Register reg, int imm21) {
893 conditionalBranchInstruction(reg, imm21, generalFromSize(size), Instruction.CBNZ, -1);
894 }
895
896 /**
897 * Compare register and branch if non-zero.
898 *
899 * @param reg general purpose register. May not be null, zero-register or stackpointer.
900 * @param size Instruction size in bits. Should be either 32 or 64.
901 * @param imm21 Signed 21-bit offset, has to be word aligned.
902 * @param pos Position at which instruction is inserted into buffer. -1 means insert at end.
903 */
904 protected void cbnz(int size, Register reg, int imm21, int pos) {
905 conditionalBranchInstruction(reg, imm21, generalFromSize(size), Instruction.CBNZ, pos);
906 }
907
908 /**
909 * Compare and branch if zero.
910 *
911 * @param reg general purpose register. May not be null, zero-register or stackpointer.
912 * @param size Instruction size in bits. Should be either 32 or 64.
913 * @param imm21 Signed 21-bit offset, has to be word aligned.
914 */
915 protected void cbz(int size, Register reg, int imm21) {
916 conditionalBranchInstruction(reg, imm21, generalFromSize(size), Instruction.CBZ, -1);
917 }
918
919 /**
920 * Compare register and branch if zero.
921 *
922 * @param reg general purpose register. May not be null, zero-register or stackpointer.
923 * @param size Instruction size in bits. Should be either 32 or 64.
924 * @param imm21 Signed 21-bit offset, has to be word aligned.
925 * @param pos Position at which instruction is inserted into buffer. -1 means insert at end.
926 */
927 protected void cbz(int size, Register reg, int imm21, int pos) {
928 conditionalBranchInstruction(reg, imm21, generalFromSize(size), Instruction.CBZ, pos);
929 }
930
931 /**
932 * Test a single bit and branch if the bit is nonzero.
933 *
934 * @param reg general purpose register. May not be null, zero-register or stackpointer.
935 * @param uimm6 Unsigned 6-bit bit index.
936 * @param imm16 signed 16 bit offset
937 */
938 protected void tbnz(Register reg, int uimm6, int imm16) {
939 tbnz(reg, uimm6, imm16, -1);
940 }
941
942 /**
943 * Test a single bit and branch if the bit is zero.
944 *
945 * @param reg general purpose register. May not be null, zero-register or stackpointer.
946 * @param uimm6 Unsigned 6-bit bit index.
947 * @param imm16 signed 16 bit offset
948 */
949 protected void tbz(Register reg, int uimm6, int imm16) {
950 tbz(reg, uimm6, imm16, -1);
951 }
952
953 /**
954 * Test a single bit and branch if the bit is nonzero.
955 *
956 * @param reg general purpose register. May not be null, zero-register or stackpointer.
957 * @param uimm6 Unsigned 6-bit bit index.
958 * @param imm16 signed 16 bit offset
959 * @param pos Position at which instruction is inserted into buffer. -1 means insert at end.
960 */
961 protected void tbnz(Register reg, int uimm6, int imm16, int pos) {
962 assert reg.getRegisterCategory().equals(CPU);
963 assert NumUtil.isUnsignedNbit(6, uimm6);
964 assert NumUtil.isSignedNbit(16, imm16) : String.format("Offset value must fit in 16 bits signed: 0x%x", imm16);
965 assert (imm16 & 3) == 0 : String.format("Lower two bits must be zero: 0x%x", imm16 & 3);
966 // size bit is overloaded as top bit of uimm6 bit index
967 int size = (((uimm6 >> 5) & 1) == 0 ? 32 : 64);
968 // remaining 5 bits are encoded lower down
969 int uimm5 = uimm6 & 0x1F;
970 int imm14 = (imm16 & NumUtil.getNbitNumberInt(16)) >> 2;
971 InstructionType type = generalFromSize(size);
972 int encoding = type.encoding | TBNZ.encoding | (uimm5 << 19) | (imm14 << 5) | rd(reg);
973 if (pos == -1) {
974 emitInt(encoding);
975 } else {
976 emitInt(encoding, pos);
977 }
978 }
979
980 /**
981 * Test a single bit and branch if the bit is zero.
982 *
983 * @param reg general purpose register. May not be null, zero-register or stackpointer.
984 * @param uimm6 Unsigned 6-bit bit index.
985 * @param imm16 signed 16 bit offset
986 * @param pos Position at which instruction is inserted into buffer. -1 means insert at end.
987 */
988 protected void tbz(Register reg, int uimm6, int imm16, int pos) {
989 assert reg.getRegisterCategory().equals(CPU);
990 assert NumUtil.isUnsignedNbit(6, uimm6);
991 assert NumUtil.isSignedNbit(16, imm16) : String.format("Offset value must fit in 16 bits signed: 0x%x", imm16);
992 assert (imm16 & 3) == 0 : String.format("Lower two bits must be zero: 0x%x", imm16 & 3);
993 // size bit is overloaded as top bit of uimm6 bit index
994 int size = (((uimm6 >> 5) & 1) == 0 ? 32 : 64);
995 // remaining 5 bits are encoded lower down
996 int uimm5 = uimm6 & 0x1F;
997 int imm14 = (imm16 & NumUtil.getNbitNumberInt(16)) >> 2;
998 InstructionType type = generalFromSize(size);
999 int encoding = type.encoding | TBZ.encoding | (uimm5 << 19) | (imm14 << 5) | rd(reg);
1000 if (pos == -1) {
1001 emitInt(encoding);
1002 } else {
1003 emitInt(encoding, pos);
1004 }
1005 }
1006
1007 private void conditionalBranchInstruction(Register reg, int imm21, InstructionType type, Instruction instr, int pos) {
1008 assert reg.getRegisterCategory().equals(CPU);
1009 int instrEncoding = instr.encoding | CompareBranchOp;
1010 if (pos == -1) {
1011 emitInt(type.encoding | instrEncoding | getConditionalBranchImm(imm21) | rd(reg));
1012 } else {
1013 emitInt(type.encoding | instrEncoding | getConditionalBranchImm(imm21) | rd(reg), pos);
1014 }
1015 }
1016
1017 private static int getConditionalBranchImm(int imm21) {
1018 assert NumUtil.isSignedNbit(21, imm21) && (imm21 & 0x3) == 0 : "Immediate has to be 21bit signed number and word aligned";
1019 int imm = (imm21 & NumUtil.getNbitNumberInt(21)) >> 2;
1020 return imm << ConditionalBranchImmOffset;
1021 }
1022
1023 /* Unconditional Branch (immediate) (5.2.2) */
1024
1025 /**
1026 * @param imm28 Signed 28-bit offset, has to be word aligned.
1027 */
1028 protected void b(int imm28) {
1029 unconditionalBranchImmInstruction(imm28, Instruction.B, -1);
1030 }
1031
1032 /**
1033 *
1034 * @param imm28 Signed 28-bit offset, has to be word aligned.
1035 * @param pos Position where instruction is inserted into code buffer.
1036 */
1037 protected void b(int imm28, int pos) {
1038 unconditionalBranchImmInstruction(imm28, Instruction.B, pos);
1039 }
1040
1041 /**
1042 * Branch and link return address to register X30.
1043 *
1044 * @param imm28 Signed 28-bit offset, has to be word aligned.
1045 */
1046 public void bl(int imm28) {
1047 unconditionalBranchImmInstruction(imm28, Instruction.BL, -1);
1048 }
1049
1050 private void unconditionalBranchImmInstruction(int imm28, Instruction instr, int pos) {
1051 assert NumUtil.isSignedNbit(28, imm28) && (imm28 & 0x3) == 0 : "Immediate has to be 28bit signed number and word aligned";
1052 int imm = (imm28 & NumUtil.getNbitNumberInt(28)) >> 2;
1053 int instrEncoding = instr.encoding | UnconditionalBranchImmOp;
1054 if (pos == -1) {
1055 annotatePatchingImmediate(position(), instr, 26, 0, 2);
1056 emitInt(instrEncoding | imm);
1057 } else {
1058 annotatePatchingImmediate(pos, instr, 26, 0, 2);
1059 emitInt(instrEncoding | imm, pos);
1060 }
1061 }
1062
1063 /* Unconditional Branch (register) (5.2.3) */
1064
1065 /**
1066 * Branches to address in register and writes return address into register X30.
1067 *
1068 * @param reg general purpose register. May not be null, zero-register or stackpointer.
1069 */
1070 public void blr(Register reg) {
1071 unconditionalBranchRegInstruction(BLR, reg);
1072 }
1073
1074 /**
1075 * Branches to address in register.
1076 *
1077 * @param reg general purpose register. May not be null, zero-register or stackpointer.
1078 */
1079 protected void br(Register reg) {
1080 unconditionalBranchRegInstruction(BR, reg);
1081 }
1082
1083 /**
1084 * Return to address in register.
1085 *
1086 * @param reg general purpose register. May not be null, zero-register or stackpointer.
1087 */
1088 public void ret(Register reg) {
1089 unconditionalBranchRegInstruction(RET, reg);
1090 }
1091
1092 private void unconditionalBranchRegInstruction(Instruction instr, Register reg) {
1093 assert reg.getRegisterCategory().equals(CPU);
1094 assert !reg.equals(zr);
1095 assert !reg.equals(sp);
1096 emitInt(instr.encoding | UnconditionalBranchRegOp | rs1(reg));
1097
1098 }
1099
1100 /* Load-Store Single Register (5.3.1) */
1101
1102 /**
1103 * Loads a srcSize value from address into rt zero-extending it.
1104 *
1105 * @param srcSize size of memory read in bits. Must be 8, 16, 32 or 64.
1106 * @param rt general purpose register. May not be null or stackpointer.
1107 * @param address all addressing modes allowed. May not be null.
1108 */
1109 public void ldr(int srcSize, Register rt, AArch64Address address) {
1110 assert rt.getRegisterCategory().equals(CPU);
1111 assert srcSize == 8 || srcSize == 16 || srcSize == 32 || srcSize == 64;
1112 int transferSize = NumUtil.log2Ceil(srcSize / 8);
1113 loadStoreInstruction(LDR, rt, address, General32, transferSize);
1114 }
1115
1116 /**
1117 * Loads a srcSize value from address into rt sign-extending it.
1118 *
1119 * @param targetSize size of target register in bits. Must be 32 or 64.
1120 * @param srcSize size of memory read in bits. Must be 8, 16 or 32, but may not be equivalent to
1121 * targetSize.
1122 * @param rt general purpose register. May not be null or stackpointer.
1123 * @param address all addressing modes allowed. May not be null.
1124 */
1125 protected void ldrs(int targetSize, int srcSize, Register rt, AArch64Address address) {
1126 assert rt.getRegisterCategory().equals(CPU);
1127 assert (srcSize == 8 || srcSize == 16 || srcSize == 32) && srcSize != targetSize;
1128 int transferSize = NumUtil.log2Ceil(srcSize / 8);
1129 loadStoreInstruction(LDRS, rt, address, generalFromSize(targetSize), transferSize);
1130 }
1131
1132 public enum PrefetchMode {
1133 PLDL1KEEP(0b00000),
1134 PLDL1STRM(0b00001),
1135 PLDL2KEEP(0b00010),
1136 PLDL2STRM(0b00011),
1137 PLDL3KEEP(0b00100),
1138 PLDL3STRM(0b00101),
1139
1140 PLIL1KEEP(0b01000),
1141 PLIL1STRM(0b01001),
1142 PLIL2KEEP(0b01010),
1143 PLIL2STRM(0b01011),
1144 PLIL3KEEP(0b01100),
1145 PLIL3STRM(0b01101),
1146
1147 PSTL1KEEP(0b10000),
1148 PSTL1STRM(0b10001),
1149 PSTL2KEEP(0b10010),
1150 PSTL2STRM(0b10011),
1151 PSTL3KEEP(0b10100),
1152 PSTL3STRM(0b10101);
1153
1154 private final int encoding;
1155
1156 PrefetchMode(int encoding) {
1157 this.encoding = encoding;
1158 }
1159
1160 private static PrefetchMode[] modes = {
1161 PLDL1KEEP,
1162 PLDL1STRM,
1163 PLDL2KEEP,
1164 PLDL2STRM,
1165 PLDL3KEEP,
1166 PLDL3STRM,
1167
1168 null,
1169 null,
1170
1171 PLIL1KEEP,
1172 PLIL1STRM,
1173 PLIL2KEEP,
1174 PLIL2STRM,
1175 PLIL3KEEP,
1176 PLIL3STRM,
1177
1178 null,
1179 null,
1180
1181 PSTL1KEEP,
1182 PSTL1STRM,
1183 PSTL2KEEP,
1184 PSTL2STRM,
1185 PSTL3KEEP,
1186 PSTL3STRM
1187 };
1188
1189 public static PrefetchMode lookup(int enc) {
1190 assert enc >= 00 && enc < modes.length;
1191 return modes[enc];
1192 }
1193
1194 public Register toRegister() {
1195 return cpuRegisters.get(encoding);
1196 }
1197 }
1198
1199 /*
1200 * implements a prefetch at a 64-bit aligned address using a scaled 12 bit or unscaled 9 bit
1201 * displacement addressing mode
1202 *
1203 * @param rt general purpose register. May not be null, zr or stackpointer.
1204 *
1205 * @param address only displacement addressing modes allowed. May not be null.
1206 */
1207 public void prfm(AArch64Address address, PrefetchMode mode) {
1208 assert (address.getAddressingMode() == AddressingMode.IMMEDIATE_SCALED ||
1209 address.getAddressingMode() == AddressingMode.IMMEDIATE_UNSCALED ||
1210 address.getAddressingMode() == AddressingMode.REGISTER_OFFSET);
1211 assert mode != null;
1212 final int srcSize = 64;
1213 final int transferSize = NumUtil.log2Ceil(srcSize / 8);
1214 final Register rt = mode.toRegister();
1215 // this looks weird but that's because loadStoreInstruction is weird
1216 // instruction select fields are size [31:30], v [26] and opc [25:24]
1217 // prfm requires size == 0b11, v == 0b0 and opc == 0b11
1218 // passing LDRS ensures opc[1] == 0b1
1219 // (n.b. passing LDR/STR makes no difference to opc[1:0]!!)
1220 // passing General64 ensures opc[0] == 0b1 and v = 0b0
1221 // (n.b. passing General32 ensures opc[0] == 0b0 and v = 0b0)
1222 // srcSize 64 ensures size == 0b11
1223 loadStoreInstruction(LDRS, rt, address, General64, transferSize);
1224 }
1225
1226 /**
1227 * Stores register rt into memory pointed by address.
1228 *
1229 * @param destSize number of bits written to memory. Must be 8, 16, 32 or 64.
1230 * @param rt general purpose register. May not be null or stackpointer.
1231 * @param address all addressing modes allowed. May not be null.
1232 */
1233 public void str(int destSize, Register rt, AArch64Address address) {
1234 assert rt.getRegisterCategory().equals(CPU);
1235 assert destSize == 8 || destSize == 16 || destSize == 32 || destSize == 64;
1236 int transferSize = NumUtil.log2Ceil(destSize / 8);
1237 loadStoreInstruction(STR, rt, address, General64, transferSize);
1238 }
1239
1240 private void loadStoreInstruction(Instruction instr, Register reg, AArch64Address address, InstructionType type, int log2TransferSize) {
1241 assert log2TransferSize >= 0 && log2TransferSize < 4;
1242 int transferSizeEncoding = log2TransferSize << LoadStoreTransferSizeOffset;
1243 int is32Bit = type.width == 32 ? 1 << ImmediateSizeOffset : 0;
1244 int isFloat = !type.isGeneral ? 1 << LoadStoreFpFlagOffset : 0;
1245 int memop = instr.encoding | transferSizeEncoding | is32Bit | isFloat | rt(reg);
1246 switch (address.getAddressingMode()) {
1247 case IMMEDIATE_SCALED:
1248 annotatePatchingImmediate(position(), instr, 12, LoadStoreScaledImmOffset, log2TransferSize);
1249 emitInt(memop | LoadStoreScaledOp | address.getImmediate() << LoadStoreScaledImmOffset | rs1(address.getBase()));
1250 break;
1251 case IMMEDIATE_UNSCALED:
1252 annotatePatchingImmediate(position(), instr, 9, LoadStoreUnscaledImmOffset, 0);
1253 emitInt(memop | LoadStoreUnscaledOp | address.getImmediate() << LoadStoreUnscaledImmOffset | rs1(address.getBase()));
1254 break;
1255 case BASE_REGISTER_ONLY:
1256 emitInt(memop | LoadStoreScaledOp | rs1(address.getBase()));
1257 break;
1258 case EXTENDED_REGISTER_OFFSET:
1259 case REGISTER_OFFSET:
1260 ExtendType extendType = address.getAddressingMode() == AddressingMode.EXTENDED_REGISTER_OFFSET ? address.getExtendType() : ExtendType.UXTX;
1261 boolean shouldScale = address.isScaled() && log2TransferSize != 0;
1262 emitInt(memop | LoadStoreRegisterOp | rs2(address.getOffset()) | extendType.encoding << ExtendTypeOffset | (shouldScale ? 1 : 0) << LoadStoreScaledRegOffset | rs1(address.getBase()));
1263 break;
1264 case PC_LITERAL:
1265 assert log2TransferSize >= 2 : "PC literal loads only works for load/stores of 32-bit and larger";
1266 transferSizeEncoding = (log2TransferSize - 2) << LoadStoreTransferSizeOffset;
1267 annotatePatchingImmediate(position(), instr, 21, LoadLiteralImmeOffset, 2);
1268 emitInt(transferSizeEncoding | isFloat | LoadLiteralOp | rd(reg) | address.getImmediate() << LoadLiteralImmeOffset);
1269 break;
1270 case IMMEDIATE_POST_INDEXED:
1271 annotatePatchingImmediate(position(), instr, 9, LoadStoreIndexedImmOffset, 0);
1272 emitInt(memop | LoadStorePostIndexedOp | rs1(address.getBase()) | address.getImmediate() << LoadStoreIndexedImmOffset);
1273 break;
1274 case IMMEDIATE_PRE_INDEXED:
1275 annotatePatchingImmediate(position(), instr, 9, LoadStoreIndexedImmOffset, 0);
1276 emitInt(memop | LoadStorePreIndexedOp | rs1(address.getBase()) | address.getImmediate() << LoadStoreIndexedImmOffset);
1277 break;
1278 default:
1279 throw GraalError.shouldNotReachHere("Unhandled addressing mode: " + address.getAddressingMode());
1280 }
1281 }
1282
1283 /**
1284 * Load Pair of Registers calculates an address from a base register value and an immediate
1285 * offset, and stores two 32-bit words or two 64-bit doublewords to the calculated address, from
1286 * two registers.
1287 */
1288 public void ldp(int size, Register rt, Register rt2, AArch64Address address) {
1289 assert size == 32 || size == 64;
1290 loadStorePairInstruction(LDP, rt, rt2, address, generalFromSize(size));
1291 }
1292
1293 /**
1294 * Store Pair of Registers calculates an address from a base register value and an immediate
1295 * offset, and stores two 32-bit words or two 64-bit doublewords to the calculated address, from
1296 * two registers.
1297 */
1298 public void stp(int size, Register rt, Register rt2, AArch64Address address) {
1299 assert size == 32 || size == 64;
1300 loadStorePairInstruction(STP, rt, rt2, address, generalFromSize(size));
1301 }
1302
1303 private void loadStorePairInstruction(Instruction instr, Register rt, Register rt2, AArch64Address address, InstructionType type) {
1304 int scaledOffset = maskField(7, address.getImmediateRaw()); // LDP/STP use a 7-bit scaled
1305 // offset
1306 int memop = type.encoding | instr.encoding | scaledOffset << LoadStorePairImm7Offset | rt2(rt2) | rn(address.getBase()) | rt(rt);
1307 switch (address.getAddressingMode()) {
1308 case IMMEDIATE_SCALED:
1309 emitInt(memop | LoadStorePairOp | (0b010 << 23));
1310 break;
1311 case IMMEDIATE_POST_INDEXED:
1312 emitInt(memop | LoadStorePairOp | (0b001 << 23));
1313 break;
1314 case IMMEDIATE_PRE_INDEXED:
1315 emitInt(memop | LoadStorePairOp | (0b011 << 23));
1316 break;
1317 default:
1318 throw GraalError.shouldNotReachHere("Unhandled addressing mode: " + address.getAddressingMode());
1319 }
1320 }
1321
1322 /* Load-Store Exclusive (5.3.6) */
1323
1324 /**
1325 * Load address exclusive. Natural alignment of address is required.
1326 *
1327 * @param size size of memory read in bits. Must be 8, 16, 32 or 64.
1328 * @param rt general purpose register. May not be null or stackpointer.
1329 * @param rn general purpose register.
1330 */
1331 protected void ldxr(int size, Register rt, Register rn) {
1332 assert size == 8 || size == 16 || size == 32 || size == 64;
1333 int transferSize = NumUtil.log2Ceil(size / 8);
1334 exclusiveLoadInstruction(LDXR, rt, rn, transferSize);
1335 }
1336
1337 /**
1338 * Store address exclusive. Natural alignment of address is required. rs and rt may not point to
1339 * the same register.
1340 *
1341 * @param size size of bits written to memory. Must be 8, 16, 32 or 64.
1342 * @param rs general purpose register. Set to exclusive access status. 0 means success,
1343 * everything else failure. May not be null, or stackpointer.
1344 * @param rt general purpose register. May not be null or stackpointer.
1345 * @param rn general purpose register.
1346 */
1347 protected void stxr(int size, Register rs, Register rt, Register rn) {
1348 assert size == 8 || size == 16 || size == 32 || size == 64;
1349 int transferSize = NumUtil.log2Ceil(size / 8);
1350 exclusiveStoreInstruction(STXR, rs, rt, rn, transferSize);
1351 }
1352
1353 /* Load-Acquire/Store-Release (5.3.7) */
1354
1355 /* non exclusive access */
1356 /**
1357 * Load acquire. Natural alignment of address is required.
1358 *
1359 * @param size size of memory read in bits. Must be 8, 16, 32 or 64.
1360 * @param rt general purpose register. May not be null or stackpointer.
1361 * @param rn general purpose register.
1362 */
1363 protected void ldar(int size, Register rt, Register rn) {
1364 assert size == 8 || size == 16 || size == 32 || size == 64;
1365 int transferSize = NumUtil.log2Ceil(size / 8);
1366 exclusiveLoadInstruction(LDAR, rt, rn, transferSize);
1367 }
1368
1369 /**
1370 * Store-release. Natural alignment of address is required.
1371 *
1372 * @param size size of bits written to memory. Must be 8, 16, 32 or 64.
1373 * @param rt general purpose register. May not be null or stackpointer.
1374 * @param rn general purpose register.
1375 */
1376 protected void stlr(int size, Register rt, Register rn) {
1377 assert size == 8 || size == 16 || size == 32 || size == 64;
1378 int transferSize = NumUtil.log2Ceil(size / 8);
1379 // Hack: Passing the zero-register means it is ignored when building the encoding.
1380 exclusiveStoreInstruction(STLR, r0, rt, rn, transferSize);
1381 }
1382
1383 /* exclusive access */
1384 /**
1385 * Load acquire exclusive. Natural alignment of address is required.
1386 *
1387 * @param size size of memory read in bits. Must be 8, 16, 32 or 64.
1388 * @param rt general purpose register. May not be null or stackpointer.
1389 * @param rn general purpose register.
1390 */
1391 public void ldaxr(int size, Register rt, Register rn) {
1392 assert size == 8 || size == 16 || size == 32 || size == 64;
1393 int transferSize = NumUtil.log2Ceil(size / 8);
1394 exclusiveLoadInstruction(LDAXR, rt, rn, transferSize);
1395 }
1396
1397 /**
1398 * Store-release exclusive. Natural alignment of address is required. rs and rt may not point to
1399 * the same register.
1400 *
1401 * @param size size of bits written to memory. Must be 8, 16, 32 or 64.
1402 * @param rs general purpose register. Set to exclusive access status. 0 means success,
1403 * everything else failure. May not be null, or stackpointer.
1404 * @param rt general purpose register. May not be null or stackpointer.
1405 * @param rn general purpose register.
1406 */
1407 public void stlxr(int size, Register rs, Register rt, Register rn) {
1408 assert size == 8 || size == 16 || size == 32 || size == 64;
1409 int transferSize = NumUtil.log2Ceil(size / 8);
1410 exclusiveStoreInstruction(STLXR, rs, rt, rn, transferSize);
1411 }
1412
1413 private void exclusiveLoadInstruction(Instruction instr, Register reg, Register rn, int log2TransferSize) {
1414 assert log2TransferSize >= 0 && log2TransferSize < 4;
1415 assert reg.getRegisterCategory().equals(CPU);
1416 int transferSizeEncoding = log2TransferSize << LoadStoreTransferSizeOffset;
1417 emitInt(transferSizeEncoding | instr.encoding | 1 << ImmediateSizeOffset | rn(rn) | rt(reg));
1418 }
1419
1420 /**
1421 * Stores data from rt into address and sets rs to the returned exclusive access status.
1422 *
1423 * @param rs general purpose register into which the exclusive access status is written. May not
1424 * be null.
1425 * @param rt general purpose register containing data to be written to memory at address. May
1426 * not be null
1427 * @param rn general purpose register containing the address specifying where rt is written to.
1428 * @param log2TransferSize log2Ceil of memory transfer size.
1429 */
1430 private void exclusiveStoreInstruction(Instruction instr, Register rs, Register rt, Register rn, int log2TransferSize) {
1431 assert log2TransferSize >= 0 && log2TransferSize < 4;
1432 assert rt.getRegisterCategory().equals(CPU) && rs.getRegisterCategory().equals(CPU) && !rs.equals(rt);
1433 int transferSizeEncoding = log2TransferSize << LoadStoreTransferSizeOffset;
1434 emitInt(transferSizeEncoding | instr.encoding | rs2(rs) | rn(rn) | rt(rt));
1435 }
1436
1437 /**
1438 * Compare And Swap word or doubleword in memory. This reads a value from an address rn,
1439 * compares it against a given value rs, and, if equal, stores the value rt to memory. The value
1440 * read from address rn is stored in register rs.
1441 *
1442 * @param size size of bits read from memory. Must be 32 or 64.
1443 * @param rs general purpose register to be compared and loaded. May not be null.
1444 * @param rt general purpose register to be conditionally stored. May not be null.
1445 * @param rn general purpose register containing the address from which to read.
1446 * @param acquire boolean value signifying if the load should use acquire semantics.
1447 * @param release boolean value signifying if the store should use release semantics.
1448 */
1449 public void cas(int size, Register rs, Register rt, Register rn, boolean acquire, boolean release) {
1450 assert size == 32 || size == 64;
1451 int transferSize = NumUtil.log2Ceil(size / 8);
1452 compareAndSwapInstruction(CAS, rs, rt, rn, transferSize, acquire, release);
1453 }
1454
1455 private void compareAndSwapInstruction(Instruction instr, Register rs, Register rt, Register rn, int log2TransferSize, boolean acquire, boolean release) {
1456 assert log2TransferSize >= 0 && log2TransferSize < 4;
1457 assert rt.getRegisterCategory().equals(CPU) && rs.getRegisterCategory().equals(CPU) && !rs.equals(rt);
1458 int transferSizeEncoding = log2TransferSize << LoadStoreTransferSizeOffset;
1459 emitInt(transferSizeEncoding | instr.encoding | rs2(rs) | rn(rn) | rt(rt) | (acquire ? 1 : 0) << CASAcquireOffset | (release ? 1 : 0) << CASReleaseOffset);
1460 }
1461
1462 /**
1463 * Atomic add. This reads a value from an address rn, stores the value in rt, and adds the value
1464 * in rs to it, and stores the result back at address rn. The initial value read from memory is
1465 * stored in rt.
1466 *
1467 * @param size size of operand to read from memory. Must be 8, 16, 32, or 64.
1468 * @param rs general purpose register to be added to contents. May not be null.
1469 * @param rt general purpose register to be loaded. May not be null.
1470 * @param rn general purpose register or stack pointer holding an address from which to load.
1471 * @param acquire boolean value signifying if the load should use acquire semantics.
1472 * @param release boolean value signifying if the store should use release semantics.
1473 */
1474 public void ldadd(int size, Register rs, Register rt, Register rn, boolean acquire, boolean release) {
1475 assert size == 8 || size == 16 || size == 32 || size == 64;
1476 int transferSize = NumUtil.log2Ceil(size / 8);
1477 loadAndAddInstruction(LDADD, rs, rt, rn, transferSize, acquire, release);
1478 }
1479
1480 private void loadAndAddInstruction(Instruction instr, Register rs, Register rt, Register rn, int log2TransferSize, boolean acquire, boolean release) {
1481 assert log2TransferSize >= 0 && log2TransferSize < 4;
1482 assert rt.getRegisterCategory().equals(CPU) && rs.getRegisterCategory().equals(CPU) && !rs.equals(rt);
1483 int transferSizeEncoding = log2TransferSize << LoadStoreTransferSizeOffset;
1484 emitInt(transferSizeEncoding | instr.encoding | rs2(rs) | rn(rn) | rt(rt) | (acquire ? 1 : 0) << LDADDAcquireOffset | (release ? 1 : 0) << LDADDReleaseOffset);
1485 }
1486
1487 /**
1488 * Atomic swap. This reads a value from an address rn, stores the value in rt, and then stores
1489 * the value in rs back at address rn.
1490 *
1491 * @param size size of operand to read from memory. Must be 8, 16, 32, or 64.
1492 * @param rs general purpose register to be stored. May not be null.
1493 * @param rt general purpose register to be loaded. May not be null.
1494 * @param rn general purpose register or stack pointer holding an address from which to load.
1495 * @param acquire boolean value signifying if the load should use acquire semantics.
1496 * @param release boolean value signifying if the store should use release semantics.
1497 */
1498 public void swp(int size, Register rs, Register rt, Register rn, boolean acquire, boolean release) {
1499 assert size == 8 || size == 16 || size == 32 || size == 64;
1500 int transferSize = NumUtil.log2Ceil(size / 8);
1501 swapInstruction(SWP, rs, rt, rn, transferSize, acquire, release);
1502 }
1503
1504 private void swapInstruction(Instruction instr, Register rs, Register rt, Register rn, int log2TransferSize, boolean acquire, boolean release) {
1505 assert log2TransferSize >= 0 && log2TransferSize < 4;
1506 assert rt.getRegisterCategory().equals(CPU) && rs.getRegisterCategory().equals(CPU) && !rs.equals(rt);
1507 int transferSizeEncoding = log2TransferSize << LoadStoreTransferSizeOffset;
1508 emitInt(transferSizeEncoding | instr.encoding | rs2(rs) | rn(rn) | rt(rt) | (acquire ? 1 : 0) << LDADDAcquireOffset | (release ? 1 : 0) << LDADDReleaseOffset);
1509 }
1510
1511 /* PC-relative Address Calculation (5.4.4) */
1512
1513 /**
1514 * Address of page: sign extends 21-bit offset, shifts if left by 12 and adds it to the value of
1515 * the PC with its bottom 12-bits cleared, writing the result to dst. No offset is emitted; the
1516 * instruction will be patched later.
1517 *
1518 * @param dst general purpose register. May not be null, zero-register or stackpointer.
1519 */
1520 public void adrp(Register dst) {
1521 emitInt(ADRP.encoding | PcRelImmOp | rd(dst));
1522 }
1523
1524 /**
1525 * Adds a 21-bit signed offset to the program counter and writes the result to dst.
1526 *
1527 * @param dst general purpose register. May not be null, zero-register or stackpointer.
1528 * @param imm21 Signed 21-bit offset.
1529 */
1530 public void adr(Register dst, int imm21) {
1531 emitInt(ADR.encoding | PcRelImmOp | rd(dst) | getPcRelativeImmEncoding(imm21));
1532 }
1533
1534 /**
1535 * Adds a 21-bit signed offset to the program counter and writes the result to dst.
1536 *
1537 * @param dst general purpose register. May not be null, zero-register or stackpointer.
1538 * @param imm21 Signed 21-bit offset.
1539 * @param pos the position in the code that the instruction is emitted.
1540 */
1541 public void adr(Register dst, int imm21, int pos) {
1542 emitInt(ADR.encoding | PcRelImmOp | rd(dst) | getPcRelativeImmEncoding(imm21), pos);
1543 }
1544
1545 private static int getPcRelativeImmEncoding(int imm21) {
1546 assert NumUtil.isSignedNbit(21, imm21);
1547 int imm = imm21 & NumUtil.getNbitNumberInt(21);
1548 // higher 19 bit
1549 int immHi = (imm >> 2) << PcRelImmHiOffset;
1550 // lower 2 bit
1551 int immLo = (imm & 0x3) << PcRelImmLoOffset;
1552 return immHi | immLo;
1553 }
1554
1555 /* Arithmetic (Immediate) (5.4.1) */
1556
1557 /**
1558 * dst = src + aimm.
1559 *
1560 * @param size register size. Has to be 32 or 64.
1561 * @param dst general purpose register. May not be null or zero-register.
1562 * @param src general purpose register. May not be null or zero-register.
1563 * @param aimm arithmetic immediate. Either unsigned 12-bit value or unsigned 24-bit value with
1564 * the lower 12-bit cleared.
1565 */
1566 protected void add(int size, Register dst, Register src, int aimm) {
1567 assert !dst.equals(zr);
1568 assert !src.equals(zr);
1569 addSubImmInstruction(ADD, dst, src, aimm, generalFromSize(size));
1570 }
1571
1572 /**
1573 * dst = src + aimm and sets condition flags.
1574 *
1575 * @param size register size. Has to be 32 or 64.
1576 * @param dst general purpose register. May not be null or stackpointer.
1577 * @param src general purpose register. May not be null or zero-register.
1578 * @param aimm arithmetic immediate. Either unsigned 12-bit value or unsigned 24-bit value with
1579 * the lower 12-bit cleared.
1580 */
1581 protected void adds(int size, Register dst, Register src, int aimm) {
1582 assert !dst.equals(sp);
1583 assert !src.equals(zr);
1584 addSubImmInstruction(ADDS, dst, src, aimm, generalFromSize(size));
1585 }
1586
1587 /**
1588 * dst = src - aimm.
1589 *
1590 * @param size register size. Has to be 32 or 64.
1591 * @param dst general purpose register. May not be null or zero-register.
1592 * @param src general purpose register. May not be null or zero-register.
1593 * @param aimm arithmetic immediate. Either unsigned 12-bit value or unsigned 24-bit value with
1594 * the lower 12-bit cleared.
1595 */
1596 protected void sub(int size, Register dst, Register src, int aimm) {
1597 assert !dst.equals(zr);
1598 assert !src.equals(zr);
1599 addSubImmInstruction(SUB, dst, src, aimm, generalFromSize(size));
1600 }
1601
1602 /**
1603 * dst = src - aimm and sets condition flags.
1604 *
1605 * @param size register size. Has to be 32 or 64.
1606 * @param dst general purpose register. May not be null or stackpointer.
1607 * @param src general purpose register. May not be null or zero-register.
1608 * @param aimm arithmetic immediate. Either unsigned 12-bit value or unsigned 24-bit value with
1609 * the lower 12-bit cleared.
1610 */
1611 protected void subs(int size, Register dst, Register src, int aimm) {
1612 assert !dst.equals(sp);
1613 assert !src.equals(zr);
1614 addSubImmInstruction(SUBS, dst, src, aimm, generalFromSize(size));
1615 }
1616
1617 private void addSubImmInstruction(Instruction instr, Register dst, Register src, int aimm, InstructionType type) {
1618 emitInt(type.encoding | instr.encoding | AddSubImmOp | encodeAimm(aimm) | rd(dst) | rs1(src));
1619 }
1620
1621 public void ccmp(int size, Register x, Register y, int aimm, ConditionFlag condition) {
1622 emitInt(generalFromSize(size).encoding | CCMP.encoding | rs1(x) | rs2(y) | encodeAimm(aimm) | condition.encoding << ConditionalConditionOffset);
1623 }
1624
1625 /**
1626 * Encodes arithmetic immediate.
1627 *
1628 * @param imm Immediate has to be either an unsigned 12-bit value or an unsigned 24-bit value
1629 * with the lower 12 bits zero.
1630 * @return Representation of immediate for use with arithmetic instructions.
1631 */
1632 private static int encodeAimm(int imm) {
1633 assert isAimm(imm) : "Immediate has to be legal arithmetic immediate value " + imm;
1634 if (NumUtil.isUnsignedNbit(12, imm)) {
1635 return imm << ImmediateOffset;
1636 } else {
1637 // First 12-bit are zero, so shift immediate 12-bit and set flag to indicate
1638 // shifted immediate value.
1639 return (imm >>> 12 << ImmediateOffset) | AddSubShift12;
1640 }
1641 }
1642
1643 /**
1644 * Checks whether immediate can be encoded as an arithmetic immediate.
1645 *
1646 * @param imm Immediate has to be either an unsigned 12bit value or un unsigned 24bit value with
1647 * the lower 12 bits 0.
1648 * @return true if valid arithmetic immediate, false otherwise.
1649 */
1650 protected static boolean isAimm(int imm) {
1651 return NumUtil.isUnsignedNbit(12, imm) || NumUtil.isUnsignedNbit(12, imm >>> 12) && (imm & 0xfff) == 0;
1652 }
1653
1654 /* Logical (immediate) (5.4.2) */
1655
1656 /**
1657 * dst = src & bimm.
1658 *
1659 * @param size register size. Has to be 32 or 64.
1660 * @param dst general purpose register. May not be null or zero-register.
1661 * @param src general purpose register. May not be null or stack-pointer.
1662 * @param bimm logical immediate. See {@link LogicalImmediateTable} for exact definition.
1663 */
1664 public void and(int size, Register dst, Register src, long bimm) {
1665 assert !dst.equals(zr);
1666 assert !src.equals(sp);
1667 logicalImmInstruction(AND, dst, src, bimm, generalFromSize(size));
1668 }
1669
1670 /**
1671 * dst = src & bimm and sets condition flags.
1672 *
1673 * @param size register size. Has to be 32 or 64.
1674 * @param dst general purpose register. May not be null or stack-pointer.
1675 * @param src general purpose register. May not be null or stack-pointer.
1676 * @param bimm logical immediate. See {@link LogicalImmediateTable} for exact definition.
1677 */
1678 public void ands(int size, Register dst, Register src, long bimm) {
1679 assert !dst.equals(sp);
1680 assert !src.equals(sp);
1681 logicalImmInstruction(ANDS, dst, src, bimm, generalFromSize(size));
1682 }
1683
1684 /**
1685 * dst = src ^ bimm.
1686 *
1687 * @param size register size. Has to be 32 or 64.
1688 * @param dst general purpose register. May not be null or zero-register.
1689 * @param src general purpose register. May not be null or stack-pointer.
1690 * @param bimm logical immediate. See {@link LogicalImmediateTable} for exact definition.
1691 */
1692 public void eor(int size, Register dst, Register src, long bimm) {
1693 assert !dst.equals(zr);
1694 assert !src.equals(sp);
1695 logicalImmInstruction(EOR, dst, src, bimm, generalFromSize(size));
1696 }
1697
1698 /**
1699 * dst = src | bimm.
1700 *
1701 * @param size register size. Has to be 32 or 64.
1702 * @param dst general purpose register. May not be null or zero-register.
1703 * @param src general purpose register. May not be null or stack-pointer.
1704 * @param bimm logical immediate. See {@link LogicalImmediateTable} for exact definition.
1705 */
1706 protected void orr(int size, Register dst, Register src, long bimm) {
1707 assert !dst.equals(zr);
1708 assert !src.equals(sp);
1709 logicalImmInstruction(ORR, dst, src, bimm, generalFromSize(size));
1710 }
1711
1712 private void logicalImmInstruction(Instruction instr, Register dst, Register src, long bimm, InstructionType type) {
1713 // Mask higher bits off, since we always pass longs around even for the 32-bit instruction.
1714 long bimmValue;
1715 if (type == General32) {
1716 assert (bimm >> 32) == 0 || (bimm >> 32) == -1L : "Higher order bits for 32-bit instruction must either all be 0 or 1.";
1717 bimmValue = bimm & NumUtil.getNbitNumberLong(32);
1718 } else {
1719 bimmValue = bimm;
1720 }
1721 int immEncoding = LogicalImmediateTable.getLogicalImmEncoding(type == General64, bimmValue);
1722 emitInt(type.encoding | instr.encoding | LogicalImmOp | immEncoding | rd(dst) | rs1(src));
1723 }
1724
1725 /* Move (wide immediate) (5.4.3) */
1726
1727 /**
1728 * dst = uimm16 << shiftAmt.
1729 *
1730 * @param size register size. Has to be 32 or 64.
1731 * @param dst general purpose register. May not be null, stackpointer or zero-register.
1732 * @param uimm16 16-bit unsigned immediate
1733 * @param shiftAmt amount by which uimm16 is left shifted. Can be any multiple of 16 smaller
1734 * than size.
1735 */
1736 protected void movz(int size, Register dst, int uimm16, int shiftAmt) {
1737 moveWideImmInstruction(MOVZ, dst, uimm16, shiftAmt, generalFromSize(size));
1738 }
1739
1740 /**
1741 * dst = ~(uimm16 << shiftAmt).
1742 *
1743 * @param size register size. Has to be 32 or 64.
1744 * @param dst general purpose register. May not be null, stackpointer or zero-register.
1745 * @param uimm16 16-bit unsigned immediate
1746 * @param shiftAmt amount by which uimm16 is left shifted. Can be any multiple of 16 smaller
1747 * than size.
1748 */
1749 protected void movn(int size, Register dst, int uimm16, int shiftAmt) {
1750 moveWideImmInstruction(MOVN, dst, uimm16, shiftAmt, generalFromSize(size));
1751 }
1752
1753 /**
1754 * dst<pos+15:pos> = uimm16.
1755 *
1756 * @param size register size. Has to be 32 or 64.
1757 * @param dst general purpose register. May not be null, stackpointer or zero-register.
1758 * @param uimm16 16-bit unsigned immediate
1759 * @param pos position into which uimm16 is inserted. Can be any multiple of 16 smaller than
1760 * size.
1761 */
1762 protected void movk(int size, Register dst, int uimm16, int pos) {
1763 moveWideImmInstruction(MOVK, dst, uimm16, pos, generalFromSize(size));
1764 }
1765
1766 private void moveWideImmInstruction(Instruction instr, Register dst, int uimm16, int shiftAmt, InstructionType type) {
1767 assert dst.getRegisterCategory().equals(CPU);
1768 assert NumUtil.isUnsignedNbit(16, uimm16) : "Immediate has to be unsigned 16bit";
1769 assert shiftAmt == 0 || shiftAmt == 16 || (type == InstructionType.General64 && (shiftAmt == 32 || shiftAmt == 48)) : "Invalid shift amount: " + shiftAmt;
1770 int shiftValue = shiftAmt >> 4;
1771 emitInt(type.encoding | instr.encoding | MoveWideImmOp | rd(dst) | uimm16 << MoveWideImmOffset | shiftValue << MoveWideShiftOffset);
1772 }
1773
1774 /* Bitfield Operations (5.4.5) */
1775
1776 /**
1777 * Bitfield move.
1778 *
1779 * @param size register size. Has to be 32 or 64.
1780 * @param dst general purpose register. May not be null, stackpointer or zero-register.
1781 * @param src general purpose register. May not be null, stackpointer or zero-register.
1782 * @param r must be in the range 0 to size - 1
1783 * @param s must be in the range 0 to size - 1
1784 */
1785 public void bfm(int size, Register dst, Register src, int r, int s) {
1786 bitfieldInstruction(BFM, dst, src, r, s, generalFromSize(size));
1787 }
1788
1789 /**
1790 * Unsigned bitfield move.
1791 *
1792 * @param size register size. Has to be 32 or 64.
1793 * @param dst general purpose register. May not be null, stackpointer or zero-register.
1794 * @param src general purpose register. May not be null, stackpointer or zero-register.
1795 * @param r must be in the range 0 to size - 1
1796 * @param s must be in the range 0 to size - 1
1797 */
1798 public void ubfm(int size, Register dst, Register src, int r, int s) {
1799 bitfieldInstruction(UBFM, dst, src, r, s, generalFromSize(size));
1800 }
1801
1802 /**
1803 * Signed bitfield move.
1804 *
1805 * @param size register size. Has to be 32 or 64.
1806 * @param dst general purpose register. May not be null, stackpointer or zero-register.
1807 * @param src general purpose register. May not be null, stackpointer or zero-register.
1808 * @param r must be in the range 0 to size - 1
1809 * @param s must be in the range 0 to size - 1
1810 */
1811 protected void sbfm(int size, Register dst, Register src, int r, int s) {
1812 bitfieldInstruction(SBFM, dst, src, r, s, generalFromSize(size));
1813 }
1814
1815 private void bitfieldInstruction(Instruction instr, Register dst, Register src, int r, int s, InstructionType type) {
1816 assert !dst.equals(sp) && !dst.equals(zr);
1817 assert !src.equals(sp) && !src.equals(zr);
1818 assert s >= 0 && s < type.width && r >= 0 && r < type.width;
1819 int sf = type == General64 ? 1 << ImmediateSizeOffset : 0;
1820 emitInt(type.encoding | instr.encoding | BitfieldImmOp | sf | r << ImmediateRotateOffset | s << ImmediateOffset | rd(dst) | rs1(src));
1821 }
1822
1823 /* Extract (Immediate) (5.4.6) */
1824
1825 /**
1826 * Extract. dst = src1:src2<lsb+31:lsb>
1827 *
1828 * @param size register size. Has to be 32 or 64.
1829 * @param dst general purpose register. May not be null or stackpointer.
1830 * @param src1 general purpose register. May not be null or stackpointer.
1831 * @param src2 general purpose register. May not be null or stackpointer.
1832 * @param lsb must be in range 0 to size - 1.
1833 */
1834 protected void extr(int size, Register dst, Register src1, Register src2, int lsb) {
1835 assert !dst.equals(sp);
1836 assert !src1.equals(sp);
1837 assert !src2.equals(sp);
1838 InstructionType type = generalFromSize(size);
1839 assert lsb >= 0 && lsb < type.width;
1840 int sf = type == General64 ? 1 << ImmediateSizeOffset : 0;
1841 emitInt(type.encoding | EXTR.encoding | sf | lsb << ImmediateOffset | rd(dst) | rs1(src1) | rs2(src2));
1842 }
1843
1844 /* Arithmetic (shifted register) (5.5.1) */
1845
1846 /**
1847 * dst = src1 + shiftType(src2, imm).
1848 *
1849 * @param size register size. Has to be 32 or 64.
1850 * @param dst general purpose register. May not be null or stackpointer.
1851 * @param src1 general purpose register. May not be null or stackpointer.
1852 * @param src2 general purpose register. May not be null or stackpointer.
1853 * @param shiftType any type but ROR.
1854 * @param imm must be in range 0 to size - 1.
1855 */
1856 protected void add(int size, Register dst, Register src1, Register src2, ShiftType shiftType, int imm) {
1857 addSubShiftedInstruction(ADD, dst, src1, src2, shiftType, imm, generalFromSize(size));
1858 }
1859
1860 /**
1861 * dst = src1 + shiftType(src2, imm) and sets condition flags.
1862 *
1863 * @param size register size. Has to be 32 or 64.
1864 * @param dst general purpose register. May not be null or stackpointer.
1865 * @param src1 general purpose register. May not be null or stackpointer.
1866 * @param src2 general purpose register. May not be null or stackpointer.
1867 * @param shiftType any type but ROR.
1868 * @param imm must be in range 0 to size - 1.
1869 */
1870 public void adds(int size, Register dst, Register src1, Register src2, ShiftType shiftType, int imm) {
1871 addSubShiftedInstruction(ADDS, dst, src1, src2, shiftType, imm, generalFromSize(size));
1872 }
1873
1874 /**
1875 * dst = src1 - shiftType(src2, imm).
1876 *
1877 * @param size register size. Has to be 32 or 64.
1878 * @param dst general purpose register. May not be null or stackpointer.
1879 * @param src1 general purpose register. May not be null or stackpointer.
1880 * @param src2 general purpose register. May not be null or stackpointer.
1881 * @param shiftType any type but ROR.
1882 * @param imm must be in range 0 to size - 1.
1883 */
1884 protected void sub(int size, Register dst, Register src1, Register src2, ShiftType shiftType, int imm) {
1885 addSubShiftedInstruction(SUB, dst, src1, src2, shiftType, imm, generalFromSize(size));
1886 }
1887
1888 /**
1889 * dst = src1 - shiftType(src2, imm) and sets condition flags.
1890 *
1891 * @param size register size. Has to be 32 or 64.
1892 * @param dst general purpose register. May not be null or stackpointer.
1893 * @param src1 general purpose register. May not be null or stackpointer.
1894 * @param src2 general purpose register. May not be null or stackpointer.
1895 * @param shiftType any type but ROR.
1896 * @param imm must be in range 0 to size - 1.
1897 */
1898 public void subs(int size, Register dst, Register src1, Register src2, ShiftType shiftType, int imm) {
1899 addSubShiftedInstruction(SUBS, dst, src1, src2, shiftType, imm, generalFromSize(size));
1900 }
1901
1902 private void addSubShiftedInstruction(Instruction instr, Register dst, Register src1, Register src2, ShiftType shiftType, int imm, InstructionType type) {
1903 assert shiftType != ShiftType.ROR;
1904 assert imm >= 0 && imm < type.width;
1905 emitInt(type.encoding | instr.encoding | AddSubShiftedOp | imm << ImmediateOffset | shiftType.encoding << ShiftTypeOffset | rd(dst) | rs1(src1) | rs2(src2));
1906 }
1907
1908 /* Arithmetic (extended register) (5.5.2) */
1909 /**
1910 * dst = src1 + extendType(src2) << imm.
1911 *
1912 * @param size register size. Has to be 32 or 64.
1913 * @param dst general purpose register. May not be null or zero-register..
1914 * @param src1 general purpose register. May not be null or zero-register.
1915 * @param src2 general purpose register. May not be null or stackpointer.
1916 * @param extendType defines how src2 is extended to the same size as src1.
1917 * @param shiftAmt must be in range 0 to 4.
1918 */
1919 public void add(int size, Register dst, Register src1, Register src2, ExtendType extendType, int shiftAmt) {
1920 assert !dst.equals(zr);
1921 assert !src1.equals(zr);
1922 assert !src2.equals(sp);
1923 addSubExtendedInstruction(ADD, dst, src1, src2, extendType, shiftAmt, generalFromSize(size));
1924 }
1925
1926 /**
1927 * dst = src1 + extendType(src2) << imm and sets condition flags.
1928 *
1929 * @param size register size. Has to be 32 or 64.
1930 * @param dst general purpose register. May not be null or stackpointer..
1931 * @param src1 general purpose register. May not be null or zero-register.
1932 * @param src2 general purpose register. May not be null or stackpointer.
1933 * @param extendType defines how src2 is extended to the same size as src1.
1934 * @param shiftAmt must be in range 0 to 4.
1935 */
1936 protected void adds(int size, Register dst, Register src1, Register src2, ExtendType extendType, int shiftAmt) {
1937 assert !dst.equals(sp);
1938 assert !src1.equals(zr);
1939 assert !src2.equals(sp);
1940 addSubExtendedInstruction(ADDS, dst, src1, src2, extendType, shiftAmt, generalFromSize(size));
1941 }
1942
1943 /**
1944 * dst = src1 - extendType(src2) << imm.
1945 *
1946 * @param size register size. Has to be 32 or 64.
1947 * @param dst general purpose register. May not be null or zero-register..
1948 * @param src1 general purpose register. May not be null or zero-register.
1949 * @param src2 general purpose register. May not be null or stackpointer.
1950 * @param extendType defines how src2 is extended to the same size as src1.
1951 * @param shiftAmt must be in range 0 to 4.
1952 */
1953 protected void sub(int size, Register dst, Register src1, Register src2, ExtendType extendType, int shiftAmt) {
1954 assert !dst.equals(zr);
1955 assert !src1.equals(zr);
1956 assert !src2.equals(sp);
1957 addSubExtendedInstruction(SUB, dst, src1, src2, extendType, shiftAmt, generalFromSize(size));
1958 }
1959
1960 /**
1961 * dst = src1 - extendType(src2) << imm and sets flags.
1962 *
1963 * @param size register size. Has to be 32 or 64.
1964 * @param dst general purpose register. May not be null or stackpointer..
1965 * @param src1 general purpose register. May not be null or zero-register.
1966 * @param src2 general purpose register. May not be null or stackpointer.
1967 * @param extendType defines how src2 is extended to the same size as src1.
1968 * @param shiftAmt must be in range 0 to 4.
1969 */
1970 public void subs(int size, Register dst, Register src1, Register src2, ExtendType extendType, int shiftAmt) {
1971 assert !dst.equals(sp);
1972 assert !src1.equals(zr);
1973 assert !src2.equals(sp);
1974 addSubExtendedInstruction(SUBS, dst, src1, src2, extendType, shiftAmt, generalFromSize(size));
1975 }
1976
1977 private void addSubExtendedInstruction(Instruction instr, Register dst, Register src1, Register src2, ExtendType extendType, int shiftAmt, InstructionType type) {
1978 assert shiftAmt >= 0 && shiftAmt <= 4;
1979 emitInt(type.encoding | instr.encoding | AddSubExtendedOp | shiftAmt << ImmediateOffset | extendType.encoding << ExtendTypeOffset | rd(dst) | rs1(src1) | rs2(src2));
1980 }
1981
1982 /* Logical (shifted register) (5.5.3) */
1983 /**
1984 * dst = src1 & shiftType(src2, imm).
1985 *
1986 * @param size register size. Has to be 32 or 64.
1987 * @param dst general purpose register. May not be null or stackpointer.
1988 * @param src1 general purpose register. May not be null or stackpointer.
1989 * @param src2 general purpose register. May not be null or stackpointer.
1990 * @param shiftType all types allowed, may not be null.
1991 * @param shiftAmt must be in range 0 to size - 1.
1992 */
1993 protected void and(int size, Register dst, Register src1, Register src2, ShiftType shiftType, int shiftAmt) {
1994 logicalRegInstruction(AND, dst, src1, src2, shiftType, shiftAmt, generalFromSize(size));
1995 }
1996
1997 /**
1998 * dst = src1 & shiftType(src2, imm) and sets condition flags.
1999 *
2000 * @param size register size. Has to be 32 or 64.
2001 * @param dst general purpose register. May not be null or stackpointer.
2002 * @param src1 general purpose register. May not be null or stackpointer.
2003 * @param src2 general purpose register. May not be null or stackpointer.
2004 * @param shiftType all types allowed, may not be null.
2005 * @param shiftAmt must be in range 0 to size - 1.
2006 */
2007 protected void ands(int size, Register dst, Register src1, Register src2, ShiftType shiftType, int shiftAmt) {
2008 logicalRegInstruction(ANDS, dst, src1, src2, shiftType, shiftAmt, generalFromSize(size));
2009 }
2010
2011 /**
2012 * dst = src1 & ~(shiftType(src2, imm)).
2013 *
2014 * @param size register size. Has to be 32 or 64.
2015 * @param dst general purpose register. May not be null or stackpointer.
2016 * @param src1 general purpose register. May not be null or stackpointer.
2017 * @param src2 general purpose register. May not be null or stackpointer.
2018 * @param shiftType all types allowed, may not be null.
2019 * @param shiftAmt must be in range 0 to size - 1.
2020 */
2021 protected void bic(int size, Register dst, Register src1, Register src2, ShiftType shiftType, int shiftAmt) {
2022 logicalRegInstruction(BIC, dst, src1, src2, shiftType, shiftAmt, generalFromSize(size));
2023 }
2024
2025 /**
2026 * dst = src1 & ~(shiftType(src2, imm)) and sets condition flags.
2027 *
2028 * @param size register size. Has to be 32 or 64.
2029 * @param dst general purpose register. May not be null or stackpointer.
2030 * @param src1 general purpose register. May not be null or stackpointer.
2031 * @param src2 general purpose register. May not be null or stackpointer.
2032 * @param shiftType all types allowed, may not be null.
2033 * @param shiftAmt must be in range 0 to size - 1.
2034 */
2035 protected void bics(int size, Register dst, Register src1, Register src2, ShiftType shiftType, int shiftAmt) {
2036 logicalRegInstruction(BICS, dst, src1, src2, shiftType, shiftAmt, generalFromSize(size));
2037 }
2038
2039 /**
2040 * dst = src1 ^ ~(shiftType(src2, imm)).
2041 *
2042 * @param size register size. Has to be 32 or 64.
2043 * @param dst general purpose register. May not be null or stackpointer.
2044 * @param src1 general purpose register. May not be null or stackpointer.
2045 * @param src2 general purpose register. May not be null or stackpointer.
2046 * @param shiftType all types allowed, may not be null.
2047 * @param shiftAmt must be in range 0 to size - 1.
2048 */
2049 protected void eon(int size, Register dst, Register src1, Register src2, ShiftType shiftType, int shiftAmt) {
2050 logicalRegInstruction(EON, dst, src1, src2, shiftType, shiftAmt, generalFromSize(size));
2051 }
2052
2053 /**
2054 * dst = src1 ^ shiftType(src2, imm).
2055 *
2056 * @param size register size. Has to be 32 or 64.
2057 * @param dst general purpose register. May not be null or stackpointer.
2058 * @param src1 general purpose register. May not be null or stackpointer.
2059 * @param src2 general purpose register. May not be null or stackpointer.
2060 * @param shiftType all types allowed, may not be null.
2061 * @param shiftAmt must be in range 0 to size - 1.
2062 */
2063 protected void eor(int size, Register dst, Register src1, Register src2, ShiftType shiftType, int shiftAmt) {
2064 logicalRegInstruction(EOR, dst, src1, src2, shiftType, shiftAmt, generalFromSize(size));
2065 }
2066
2067 /**
2068 * dst = src1 | shiftType(src2, imm).
2069 *
2070 * @param size register size. Has to be 32 or 64.
2071 * @param dst general purpose register. May not be null or stackpointer.
2072 * @param src1 general purpose register. May not be null or stackpointer.
2073 * @param src2 general purpose register. May not be null or stackpointer.
2074 * @param shiftType all types allowed, may not be null.
2075 * @param shiftAmt must be in range 0 to size - 1.
2076 */
2077 protected void orr(int size, Register dst, Register src1, Register src2, ShiftType shiftType, int shiftAmt) {
2078 logicalRegInstruction(ORR, dst, src1, src2, shiftType, shiftAmt, generalFromSize(size));
2079 }
2080
2081 /**
2082 * dst = src1 | ~(shiftType(src2, imm)).
2083 *
2084 * @param size register size. Has to be 32 or 64.
2085 * @param dst general purpose register. May not be null or stackpointer.
2086 * @param src1 general purpose register. May not be null or stackpointer.
2087 * @param src2 general purpose register. May not be null or stackpointer.
2088 * @param shiftType all types allowed, may not be null.
2089 * @param shiftAmt must be in range 0 to size - 1.
2090 */
2091 protected void orn(int size, Register dst, Register src1, Register src2, ShiftType shiftType, int shiftAmt) {
2092 logicalRegInstruction(ORN, dst, src1, src2, shiftType, shiftAmt, generalFromSize(size));
2093 }
2094
2095 private void logicalRegInstruction(Instruction instr, Register dst, Register src1, Register src2, ShiftType shiftType, int shiftAmt, InstructionType type) {
2096 assert !dst.equals(sp);
2097 assert !src1.equals(sp);
2098 assert !src2.equals(sp);
2099 assert shiftAmt >= 0 && shiftAmt < type.width;
2100 emitInt(type.encoding | instr.encoding | LogicalShiftOp | shiftAmt << ImmediateOffset | shiftType.encoding << ShiftTypeOffset | rd(dst) | rs1(src1) | rs2(src2));
2101 }
2102
2103 /* Variable Shift (5.5.4) */
2104 /**
2105 * dst = src1 >> (src2 & log2(size)).
2106 *
2107 * @param size register size. Has to be 32 or 64.
2108 * @param dst general purpose register. May not be null or stackpointer.
2109 * @param src1 general purpose register. May not be null or stackpointer.
2110 * @param src2 general purpose register. May not be null or stackpointer.
2111 */
2112 protected void asr(int size, Register dst, Register src1, Register src2) {
2113 dataProcessing2SourceOp(ASRV, dst, src1, src2, generalFromSize(size));
2114 }
2115
2116 /**
2117 * dst = src1 << (src2 & log2(size)).
2118 *
2119 * @param size register size. Has to be 32 or 64.
2120 * @param dst general purpose register. May not be null or stackpointer.
2121 * @param src1 general purpose register. May not be null or stackpointer.
2122 * @param src2 general purpose register. May not be null or stackpointer.
2123 */
2124 protected void lsl(int size, Register dst, Register src1, Register src2) {
2125 dataProcessing2SourceOp(LSLV, dst, src1, src2, generalFromSize(size));
2126 }
2127
2128 /**
2129 * dst = src1 >>> (src2 & log2(size)).
2130 *
2131 * @param size register size. Has to be 32 or 64.
2132 * @param dst general purpose register. May not be null or stackpointer.
2133 * @param src1 general purpose register. May not be null or stackpointer.
2134 * @param src2 general purpose register. May not be null or stackpointer.
2135 */
2136 protected void lsr(int size, Register dst, Register src1, Register src2) {
2137 dataProcessing2SourceOp(LSRV, dst, src1, src2, generalFromSize(size));
2138 }
2139
2140 /**
2141 * dst = rotateRight(src1, (src2 & log2(size))).
2142 *
2143 * @param size register size. Has to be 32 or 64.
2144 * @param dst general purpose register. May not be null or stackpointer.
2145 * @param src1 general purpose register. May not be null or stackpointer.
2146 * @param src2 general purpose register. May not be null or stackpointer.
2147 */
2148 protected void ror(int size, Register dst, Register src1, Register src2) {
2149 dataProcessing2SourceOp(RORV, dst, src1, src2, generalFromSize(size));
2150 }
2151
2152 /* Bit Operations (5.5.5) */
2153
2154 /**
2155 * Counts leading sign bits. Sets Wd to the number of consecutive bits following the topmost bit
2156 * in dst, that are the same as the topmost bit. The count does not include the topmost bit
2157 * itself , so the result will be in the range 0 to size-1 inclusive.
2158 *
2159 * @param size register size. Has to be 32 or 64.
2160 * @param dst general purpose register. May not be null, zero-register or the stackpointer.
2161 * @param src source register. May not be null, zero-register or the stackpointer.
2162 */
2163 protected void cls(int size, Register dst, Register src) {
2164 dataProcessing1SourceOp(CLS, dst, src, generalFromSize(size));
2165 }
2166
2167 /**
2168 * Counts leading zeros.
2169 *
2170 * @param size register size. Has to be 32 or 64.
2171 * @param dst general purpose register. May not be null, zero-register or the stackpointer.
2172 * @param src source register. May not be null, zero-register or the stackpointer.
2173 */
2174 public void clz(int size, Register dst, Register src) {
2175 dataProcessing1SourceOp(CLZ, dst, src, generalFromSize(size));
2176 }
2177
2178 /**
2179 * Reverses bits.
2180 *
2181 * @param size register size. Has to be 32 or 64.
2182 * @param dst general purpose register. May not be null, zero-register or the stackpointer.
2183 * @param src source register. May not be null, zero-register or the stackpointer.
2184 */
2185 public void rbit(int size, Register dst, Register src) {
2186 dataProcessing1SourceOp(RBIT, dst, src, generalFromSize(size));
2187 }
2188
2189 /**
2190 * Reverses bytes.
2191 *
2192 * @param size register size. Has to be 32 or 64.
2193 * @param dst general purpose register. May not be null or the stackpointer.
2194 * @param src source register. May not be null or the stackpointer.
2195 */
2196 public void rev(int size, Register dst, Register src) {
2197 if (size == 64) {
2198 dataProcessing1SourceOp(REVX, dst, src, generalFromSize(size));
2199 } else {
2200 assert size == 32;
2201 dataProcessing1SourceOp(REVW, dst, src, generalFromSize(size));
2202 }
2203 }
2204
2205 /* Conditional Data Processing (5.5.6) */
2206
2207 /**
2208 * Conditional select. dst = src1 if condition else src2.
2209 *
2210 * @param size register size. Has to be 32 or 64.
2211 * @param dst general purpose register. May not be null or the stackpointer.
2212 * @param src1 general purpose register. May not be null or the stackpointer.
2213 * @param src2 general purpose register. May not be null or the stackpointer.
2214 * @param condition any condition flag. May not be null.
2215 */
2216 protected void csel(int size, Register dst, Register src1, Register src2, ConditionFlag condition) {
2217 conditionalSelectInstruction(CSEL, dst, src1, src2, condition, generalFromSize(size));
2218 }
2219
2220 /**
2221 * Conditional select negate. dst = src1 if condition else -src2.
2222 *
2223 * @param size register size. Has to be 32 or 64.
2224 * @param dst general purpose register. May not be null or the stackpointer.
2225 * @param src1 general purpose register. May not be null or the stackpointer.
2226 * @param src2 general purpose register. May not be null or the stackpointer.
2227 * @param condition any condition flag. May not be null.
2228 */
2229 protected void csneg(int size, Register dst, Register src1, Register src2, ConditionFlag condition) {
2230 conditionalSelectInstruction(CSNEG, dst, src1, src2, condition, generalFromSize(size));
2231 }
2232
2233 /**
2234 * Conditional increase. dst = src1 if condition else src2 + 1.
2235 *
2236 * @param size register size. Has to be 32 or 64.
2237 * @param dst general purpose register. May not be null or the stackpointer.
2238 * @param src1 general purpose register. May not be null or the stackpointer.
2239 * @param src2 general purpose register. May not be null or the stackpointer.
2240 * @param condition any condition flag. May not be null.
2241 */
2242 protected void csinc(int size, Register dst, Register src1, Register src2, ConditionFlag condition) {
2243 conditionalSelectInstruction(CSINC, dst, src1, src2, condition, generalFromSize(size));
2244 }
2245
2246 private void conditionalSelectInstruction(Instruction instr, Register dst, Register src1, Register src2, ConditionFlag condition, InstructionType type) {
2247 assert !dst.equals(sp);
2248 assert !src1.equals(sp);
2249 assert !src2.equals(sp);
2250 emitInt(type.encoding | instr.encoding | ConditionalSelectOp | rd(dst) | rs1(src1) | rs2(src2) | condition.encoding << ConditionalConditionOffset);
2251 }
2252
2253 /* Integer Multiply/Divide (5.6) */
2254
2255 /**
2256 * dst = src1 * src2 + src3.
2257 *
2258 * @param size register size. Has to be 32 or 64.
2259 * @param dst general purpose register. May not be null or the stackpointer.
2260 * @param src1 general purpose register. May not be null or the stackpointer.
2261 * @param src2 general purpose register. May not be null or the stackpointer.
2262 * @param src3 general purpose register. May not be null or the stackpointer.
2263 */
2264 protected void madd(int size, Register dst, Register src1, Register src2, Register src3) {
2265 mulInstruction(MADD, dst, src1, src2, src3, generalFromSize(size));
2266 }
2267
2268 /**
2269 * dst = src3 - src1 * src2.
2270 *
2271 * @param size register size. Has to be 32 or 64.
2272 * @param dst general purpose register. May not be null or the stackpointer.
2273 * @param src1 general purpose register. May not be null or the stackpointer.
2274 * @param src2 general purpose register. May not be null or the stackpointer.
2275 * @param src3 general purpose register. May not be null or the stackpointer.
2276 */
2277 protected void msub(int size, Register dst, Register src1, Register src2, Register src3) {
2278 mulInstruction(MSUB, dst, src1, src2, src3, generalFromSize(size));
2279 }
2280
2281 /**
2282 * Signed multiply high. dst = (src1 * src2)[127:64]
2283 *
2284 * @param dst general purpose register. May not be null or the stackpointer.
2285 * @param src1 general purpose register. May not be null or the stackpointer.
2286 * @param src2 general purpose register. May not be null or the stackpointer.
2287 */
2288 protected void smulh(Register dst, Register src1, Register src2) {
2289 assert !dst.equals(sp);
2290 assert !src1.equals(sp);
2291 assert !src2.equals(sp);
2292 emitInt(0b10011011010 << 21 | dst.encoding | rs1(src1) | rs2(src2) | 0b011111 << ImmediateOffset);
2293 }
2294
2295 /**
2296 * unsigned multiply high. dst = (src1 * src2)[127:64]
2297 *
2298 * @param dst general purpose register. May not be null or the stackpointer.
2299 * @param src1 general purpose register. May not be null or the stackpointer.
2300 * @param src2 general purpose register. May not be null or the stackpointer.
2301 */
2302 protected void umulh(Register dst, Register src1, Register src2) {
2303 assert !dst.equals(sp);
2304 assert !src1.equals(sp);
2305 assert !src2.equals(sp);
2306 emitInt(0b10011011110 << 21 | dst.encoding | rs1(src1) | rs2(src2) | 0b011111 << ImmediateOffset);
2307 }
2308
2309 /**
2310 * unsigned multiply add-long. xDst = xSrc3 + (wSrc1 * wSrc2)
2311 *
2312 * @param dst general purpose register. May not be null or the stackpointer.
2313 * @param src1 general purpose register. May not be null or the stackpointer.
2314 * @param src2 general purpose register. May not be null or the stackpointer.
2315 * @param src3 general purpose register. May not be null or the stackpointer.
2316 */
2317 protected void umaddl(Register dst, Register src1, Register src2, Register src3) {
2318 assert !dst.equals(sp);
2319 assert !src1.equals(sp);
2320 assert !src2.equals(sp);
2321 assert !src3.equals(sp);
2322 emitInt(0b10011011101 << 21 | dst.encoding | rs1(src1) | rs2(src2) | 0b011111 << ImmediateOffset);
2323 }
2324
2325 /**
2326 * signed multiply add-long. xDst = xSrc3 + (wSrc1 * wSrc2)
2327 *
2328 * @param dst general purpose register. May not be null or the stackpointer.
2329 * @param src1 general purpose register. May not be null or the stackpointer.
2330 * @param src2 general purpose register. May not be null or the stackpointer.
2331 * @param src3 general purpose register. May not be null or the stackpointer.
2332 */
2333 public void smaddl(Register dst, Register src1, Register src2, Register src3) {
2334 assert !dst.equals(sp);
2335 assert !src1.equals(sp);
2336 assert !src2.equals(sp);
2337 assert !src3.equals(sp);
2338 emitInt(0b10011011001 << 21 | dst.encoding | rs1(src1) | rs2(src2) | rs3(src3));
2339 }
2340
2341 private void mulInstruction(Instruction instr, Register dst, Register src1, Register src2, Register src3, InstructionType type) {
2342 assert !dst.equals(sp);
2343 assert !src1.equals(sp);
2344 assert !src2.equals(sp);
2345 assert !src3.equals(sp);
2346 emitInt(type.encoding | instr.encoding | MulOp | rd(dst) | rs1(src1) | rs2(src2) | rs3(src3));
2347 }
2348
2349 /**
2350 * Signed divide. dst = src1 / src2.
2351 *
2352 * @param size register size. Has to be 32 or 64.
2353 * @param dst general purpose register. May not be null or the stackpointer.
2354 * @param src1 general purpose register. May not be null or the stackpointer.
2355 * @param src2 general purpose register. May not be null or the stackpointer.
2356 */
2357 public void sdiv(int size, Register dst, Register src1, Register src2) {
2358 dataProcessing2SourceOp(SDIV, dst, src1, src2, generalFromSize(size));
2359 }
2360
2361 /**
2362 * Unsigned divide. dst = src1 / src2.
2363 *
2364 * @param size register size. Has to be 32 or 64.
2365 * @param dst general purpose register. May not be null or the stackpointer.
2366 * @param src1 general purpose register. May not be null or the stackpointer.
2367 * @param src2 general purpose register. May not be null or the stackpointer.
2368 */
2369 public void udiv(int size, Register dst, Register src1, Register src2) {
2370 dataProcessing2SourceOp(UDIV, dst, src1, src2, generalFromSize(size));
2371 }
2372
2373 private void dataProcessing1SourceOp(Instruction instr, Register dst, Register src, InstructionType type) {
2374 emitInt(type.encoding | instr.encoding | DataProcessing1SourceOp | rd(dst) | rs1(src));
2375 }
2376
2377 private void dataProcessing2SourceOp(Instruction instr, Register dst, Register src1, Register src2, InstructionType type) {
2378 assert !dst.equals(sp);
2379 assert !src1.equals(sp);
2380 assert !src2.equals(sp);
2381 emitInt(type.encoding | instr.encoding | DataProcessing2SourceOp | rd(dst) | rs1(src1) | rs2(src2));
2382 }
2383
2384 /* Floating point operations */
2385
2386 /* Load-Store Single FP register (5.7.1.1) */
2387 /**
2388 * Floating point load.
2389 *
2390 * @param size number of bits read from memory into rt. Must be 32 or 64.
2391 * @param rt floating point register. May not be null.
2392 * @param address all addressing modes allowed. May not be null.
2393 */
2394 public void fldr(int size, Register rt, AArch64Address address) {
2395 assert rt.getRegisterCategory().equals(SIMD);
2396 assert size == 32 || size == 64;
2397 int transferSize = NumUtil.log2Ceil(size / 8);
2398 loadStoreInstruction(LDR, rt, address, InstructionType.FP32, transferSize);
2399 }
2400
2401 /**
2402 * Floating point store.
2403 *
2404 * @param size number of bits read from memory into rt. Must be 32 or 64.
2405 * @param rt floating point register. May not be null.
2406 * @param address all addressing modes allowed. May not be null.
2407 */
2408 public void fstr(int size, Register rt, AArch64Address address) {
2409 assert rt.getRegisterCategory().equals(SIMD);
2410 assert size == 32 || size == 64;
2411 int transferSize = NumUtil.log2Ceil(size / 8);
2412 loadStoreInstruction(STR, rt, address, InstructionType.FP64, transferSize);
2413 }
2414
2415 /* Floating-point Move (register) (5.7.2) */
2416
2417 /**
2418 * Floating point move.
2419 *
2420 * @param size register size. Has to be 32 or 64.
2421 * @param dst floating point register. May not be null.
2422 * @param src floating point register. May not be null.
2423 */
2424 protected void fmov(int size, Register dst, Register src) {
2425 fpDataProcessing1Source(FMOV, dst, src, floatFromSize(size));
2426 }
2427
2428 /**
2429 * Move size bits from floating point register unchanged to general purpose register.
2430 *
2431 * @param size number of bits read from memory into rt. Must be 32 or 64.
2432 * @param dst general purpose register. May not be null, stack-pointer or zero-register
2433 * @param src floating point register. May not be null.
2434 */
2435 protected void fmovFpu2Cpu(int size, Register dst, Register src) {
2436 assert dst.getRegisterCategory().equals(CPU);
2437 assert src.getRegisterCategory().equals(SIMD);
2438 fmovCpuFpuInstruction(dst, src, size == 64, Instruction.FMOVFPU2CPU);
2439 }
2440
2441 /**
2442 * Move size bits from general purpose register unchanged to floating point register.
2443 *
2444 * @param size register size. Has to be 32 or 64.
2445 * @param dst floating point register. May not be null.
2446 * @param src general purpose register. May not be null or stack-pointer.
2447 */
2448 protected void fmovCpu2Fpu(int size, Register dst, Register src) {
2449 assert dst.getRegisterCategory().equals(SIMD);
2450 assert src.getRegisterCategory().equals(CPU);
2451 fmovCpuFpuInstruction(dst, src, size == 64, Instruction.FMOVCPU2FPU);
2452 }
2453
2454 private void fmovCpuFpuInstruction(Register dst, Register src, boolean is64bit, Instruction instr) {
2455 int sf = is64bit ? FP64.encoding | General64.encoding : FP32.encoding | General32.encoding;
2456 emitInt(sf | instr.encoding | FpConvertOp | rd(dst) | rs1(src));
2457 }
2458
2459 /* Floating-point Move (immediate) (5.7.3) */
2460
2461 /**
2462 * Move immediate into register.
2463 *
2464 * @param size register size. Has to be 32 or 64.
2465 * @param dst floating point register. May not be null.
2466 * @param imm immediate that is loaded into dst. If size is 32 only float immediates can be
2467 * loaded, i.e. (float) imm == imm must be true. In all cases
2468 * {@code isFloatImmediate}, respectively {@code #isDoubleImmediate} must be true
2469 * depending on size.
2470 */
2471 protected void fmov(int size, Register dst, double imm) {
2472 assert dst.getRegisterCategory().equals(SIMD);
2473 InstructionType type = floatFromSize(size);
2474 int immEncoding;
2475 if (type == FP64) {
2476 immEncoding = getDoubleImmediate(imm);
2477 } else {
2478 assert imm == (float) imm : "float mov must use an immediate that can be represented using a float.";
2479 immEncoding = getFloatImmediate((float) imm);
2480 }
2481 emitInt(type.encoding | FMOV.encoding | FpImmOp | immEncoding | rd(dst));
2482 }
2483
2484 private static int getDoubleImmediate(double imm) {
2485 assert isDoubleImmediate(imm);
2486 // bits: aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
2487 // 0000.0000.0000.0000.0000.0000.0000.0000
2488 long repr = Double.doubleToRawLongBits(imm);
2489 int a = (int) (repr >>> 63) << 7;
2490 int b = (int) ((repr >>> 61) & 0x1) << 6;
2491 int cToH = (int) (repr >>> 48) & 0x3f;
2492 return (a | b | cToH) << FpImmOffset;
2493 }
2494
2495 protected static boolean isDoubleImmediate(double imm) {
2496 // Valid values will have the form:
2497 // aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
2498 // 0000.0000.0000.0000.0000.0000.0000.0000
2499 long bits = Double.doubleToRawLongBits(imm);
2500 // lower 48 bits are cleared
2501 if ((bits & NumUtil.getNbitNumberLong(48)) != 0) {
2502 return false;
2503 }
2504 // bits[61..54] are all set or all cleared.
2505 long pattern = (bits >> 54) & NumUtil.getNbitNumberLong(7);
2506 if (pattern != 0 && pattern != NumUtil.getNbitNumberLong(7)) {
2507 return false;
2508 }
2509 // bits[62] and bits[61] are opposites.
2510 boolean result = ((bits ^ (bits << 1)) & (1L << 62)) != 0;
2511 return result;
2512 }
2513
2514 private static int getFloatImmediate(float imm) {
2515 assert isFloatImmediate(imm);
2516 // bits: aBbb.bbbc.defg.h000.0000.0000.0000.0000
2517 int repr = Float.floatToRawIntBits(imm);
2518 int a = (repr >>> 31) << 7;
2519 int b = ((repr >>> 29) & 0x1) << 6;
2520 int cToH = (repr >>> 19) & NumUtil.getNbitNumberInt(6);
2521 return (a | b | cToH) << FpImmOffset;
2522 }
2523
2524 protected static boolean isFloatImmediate(float imm) {
2525 // Valid values will have the form:
2526 // aBbb.bbbc.defg.h000.0000.0000.0000.0000
2527 int bits = Float.floatToRawIntBits(imm);
2528 // lower 20 bits are cleared.
2529 if ((bits & NumUtil.getNbitNumberInt(19)) != 0) {
2530 return false;
2531 }
2532 // bits[29..25] are all set or all cleared
2533 int pattern = (bits >> 25) & NumUtil.getNbitNumberInt(5);
2534 if (pattern != 0 && pattern != NumUtil.getNbitNumberInt(5)) {
2535 return false;
2536 }
2537 // bits[29] and bits[30] have to be opposite
2538 return ((bits ^ (bits << 1)) & (1 << 30)) != 0;
2539 }
2540
2541 /* Convert Floating-point Precision (5.7.4.1) */
2542 /* Converts float to double and vice-versa */
2543
2544 /**
2545 * Convert float to double and vice-versa.
2546 *
2547 * @param srcSize size of source register in bits.
2548 * @param dst floating point register. May not be null.
2549 * @param src floating point register. May not be null.
2550 */
2551 public void fcvt(int srcSize, Register dst, Register src) {
2552 if (srcSize == 32) {
2553 fpDataProcessing1Source(FCVTDS, dst, src, floatFromSize(srcSize));
2554 } else {
2555 fpDataProcessing1Source(FCVTSD, dst, src, floatFromSize(srcSize));
2556 }
2557 }
2558
2559 /* Convert to Integer (5.7.4.2) */
2560
2561 /**
2562 * Convert floating point to integer. Rounds towards zero.
2563 *
2564 * @param targetSize size of integer register. 32 or 64.
2565 * @param srcSize size of floating point register. 32 or 64.
2566 * @param dst general purpose register. May not be null, the zero-register or the stackpointer.
2567 * @param src floating point register. May not be null.
2568 */
2569 public void fcvtzs(int targetSize, int srcSize, Register dst, Register src) {
2570 assert !dst.equals(zr) && !dst.equals(sp);
2571 assert src.getRegisterCategory().equals(SIMD);
2572 fcvtCpuFpuInstruction(FCVTZS, dst, src, generalFromSize(targetSize), floatFromSize(srcSize));
2573 }
2574
2575 /* Convert from Integer (5.7.4.2) */
2576 /**
2577 * Converts integer to floating point. Uses rounding mode defined by FCPR.
2578 *
2579 * @param targetSize size of floating point register. 32 or 64.
2580 * @param srcSize size of integer register. 32 or 64.
2581 * @param dst floating point register. May not be null.
2582 * @param src general purpose register. May not be null or the stackpointer.
2583 */
2584 public void scvtf(int targetSize, int srcSize, Register dst, Register src) {
2585 assert dst.getRegisterCategory().equals(SIMD);
2586 assert !src.equals(sp);
2587 fcvtCpuFpuInstruction(SCVTF, dst, src, floatFromSize(targetSize), generalFromSize(srcSize));
2588 }
2589
2590 private void fcvtCpuFpuInstruction(Instruction instr, Register dst, Register src, InstructionType type1, InstructionType type2) {
2591 emitInt(type1.encoding | type2.encoding | instr.encoding | FpConvertOp | rd(dst) | rs1(src));
2592 }
2593
2594 /* Floating-point Round to Integral (5.7.5) */
2595
2596 /**
2597 * Rounds floating-point to integral. Rounds towards zero.
2598 *
2599 * @param size register size.
2600 * @param dst floating point register. May not be null.
2601 * @param src floating point register. May not be null.
2602 */
2603 protected void frintz(int size, Register dst, Register src) {
2604 fpDataProcessing1Source(FRINTZ, dst, src, floatFromSize(size));
2605 }
2606
2607 /**
2608 * Rounds floating-point to integral. Rounds towards nearest with ties to even.
2609 *
2610 * @param size register size.
2611 * @param dst floating point register. May not be null.
2612 * @param src floating point register. May not be null.
2613 */
2614 public void frintn(int size, Register dst, Register src) {
2615 fpDataProcessing1Source(FRINTN, dst, src, floatFromSize(size));
2616 }
2617
2618 /**
2619 * Rounds floating-point to integral. Rounds towards minus infinity.
2620 *
2621 * @param size register size.
2622 * @param dst floating point register. May not be null.
2623 * @param src floating point register. May not be null.
2624 */
2625 public void frintm(int size, Register dst, Register src) {
2626 fpDataProcessing1Source(FRINTM, dst, src, floatFromSize(size));
2627 }
2628
2629 /**
2630 * Rounds floating-point to integral. Rounds towards plus infinity.
2631 *
2632 * @param size register size.
2633 * @param dst floating point register. May not be null.
2634 * @param src floating point register. May not be null.
2635 */
2636 public void frintp(int size, Register dst, Register src) {
2637 fpDataProcessing1Source(FRINTP, dst, src, floatFromSize(size));
2638 }
2639
2640 /* Floating-point Arithmetic (1 source) (5.7.6) */
2641
2642 /**
2643 * dst = |src|.
2644 *
2645 * @param size register size.
2646 * @param dst floating point register. May not be null.
2647 * @param src floating point register. May not be null.
2648 */
2649 public void fabs(int size, Register dst, Register src) {
2650 fpDataProcessing1Source(FABS, dst, src, floatFromSize(size));
2651 }
2652
2653 /**
2654 * dst = -neg.
2655 *
2656 * @param size register size.
2657 * @param dst floating point register. May not be null.
2658 * @param src floating point register. May not be null.
2659 */
2660 public void fneg(int size, Register dst, Register src) {
2661 fpDataProcessing1Source(FNEG, dst, src, floatFromSize(size));
2662 }
2663
2664 /**
2665 * dst = Sqrt(src).
2666 *
2667 * @param size register size.
2668 * @param dst floating point register. May not be null.
2669 * @param src floating point register. May not be null.
2670 */
2671 public void fsqrt(int size, Register dst, Register src) {
2672 fpDataProcessing1Source(FSQRT, dst, src, floatFromSize(size));
2673 }
2674
2675 private void fpDataProcessing1Source(Instruction instr, Register dst, Register src, InstructionType type) {
2676 assert dst.getRegisterCategory().equals(SIMD);
2677 assert src.getRegisterCategory().equals(SIMD);
2678 emitInt(type.encoding | instr.encoding | Fp1SourceOp | rd(dst) | rs1(src));
2679 }
2680
2681 /* Floating-point Arithmetic (2 source) (5.7.7) */
2682
2683 /**
2684 * dst = src1 + src2.
2685 *
2686 * @param size register size.
2687 * @param dst floating point register. May not be null.
2688 * @param src1 floating point register. May not be null.
2689 * @param src2 floating point register. May not be null.
2690 */
2691 public void fadd(int size, Register dst, Register src1, Register src2) {
2692 fpDataProcessing2Source(FADD, dst, src1, src2, floatFromSize(size));
2693 }
2694
2695 /**
2696 * dst = src1 - src2.
2697 *
2698 * @param size register size.
2699 * @param dst floating point register. May not be null.
2700 * @param src1 floating point register. May not be null.
2701 * @param src2 floating point register. May not be null.
2702 */
2703 public void fsub(int size, Register dst, Register src1, Register src2) {
2704 fpDataProcessing2Source(FSUB, dst, src1, src2, floatFromSize(size));
2705 }
2706
2707 /**
2708 * dst = src1 * src2.
2709 *
2710 * @param size register size.
2711 * @param dst floating point register. May not be null.
2712 * @param src1 floating point register. May not be null.
2713 * @param src2 floating point register. May not be null.
2714 */
2715 public void fmul(int size, Register dst, Register src1, Register src2) {
2716 fpDataProcessing2Source(FMUL, dst, src1, src2, floatFromSize(size));
2717 }
2718
2719 /**
2720 * dst = src1 / src2.
2721 *
2722 * @param size register size.
2723 * @param dst floating point register. May not be null.
2724 * @param src1 floating point register. May not be null.
2725 * @param src2 floating point register. May not be null.
2726 */
2727 public void fdiv(int size, Register dst, Register src1, Register src2) {
2728 fpDataProcessing2Source(FDIV, dst, src1, src2, floatFromSize(size));
2729 }
2730
2731 private void fpDataProcessing2Source(Instruction instr, Register dst, Register src1, Register src2, InstructionType type) {
2732 assert dst.getRegisterCategory().equals(SIMD);
2733 assert src1.getRegisterCategory().equals(SIMD);
2734 assert src2.getRegisterCategory().equals(SIMD);
2735 emitInt(type.encoding | instr.encoding | Fp2SourceOp | rd(dst) | rs1(src1) | rs2(src2));
2736 }
2737
2738 /* Floating-point Multiply-Add (5.7.9) */
2739
2740 /**
2741 * dst = src1 * src2 + src3.
2742 *
2743 * @param size register size.
2744 * @param dst floating point register. May not be null.
2745 * @param src1 floating point register. May not be null.
2746 * @param src2 floating point register. May not be null.
2747 * @param src3 floating point register. May not be null.
2748 */
2749 protected void fmadd(int size, Register dst, Register src1, Register src2, Register src3) {
2750 fpDataProcessing3Source(FMADD, dst, src1, src2, src3, floatFromSize(size));
2751 }
2752
2753 /**
2754 * dst = src3 - src1 * src2.
2755 *
2756 * @param size register size.
2757 * @param dst floating point register. May not be null.
2758 * @param src1 floating point register. May not be null.
2759 * @param src2 floating point register. May not be null.
2760 * @param src3 floating point register. May not be null.
2761 */
2762 protected void fmsub(int size, Register dst, Register src1, Register src2, Register src3) {
2763 fpDataProcessing3Source(FMSUB, dst, src1, src2, src3, floatFromSize(size));
2764 }
2765
2766 private void fpDataProcessing3Source(Instruction instr, Register dst, Register src1, Register src2, Register src3, InstructionType type) {
2767 assert dst.getRegisterCategory().equals(SIMD);
2768 assert src1.getRegisterCategory().equals(SIMD);
2769 assert src2.getRegisterCategory().equals(SIMD);
2770 assert src3.getRegisterCategory().equals(SIMD);
2771 emitInt(type.encoding | instr.encoding | Fp3SourceOp | rd(dst) | rs1(src1) | rs2(src2) | rs3(src3));
2772 }
2773
2774 /* Floating-point Comparison (5.7.10) */
2775
2776 /**
2777 * Compares src1 to src2.
2778 *
2779 * @param size register size.
2780 * @param src1 floating point register. May not be null.
2781 * @param src2 floating point register. May not be null.
2782 */
2783 public void fcmp(int size, Register src1, Register src2) {
2784 assert src1.getRegisterCategory().equals(SIMD);
2785 assert src2.getRegisterCategory().equals(SIMD);
2786 InstructionType type = floatFromSize(size);
2787 emitInt(type.encoding | FCMP.encoding | FpCmpOp | rs1(src1) | rs2(src2));
2788 }
2789
2790 /**
2791 * Signalling compares src1 to src2.
2792 *
2793 * @param size register size.
2794 * @param src1 floating point register. May not be null.
2795 * @param src2 floating point register. May not be null.
2796 */
2797 public void fcmpe(int size, Register src1, Register src2) {
2798 assert src1.getRegisterCategory().equals(SIMD);
2799 assert src2.getRegisterCategory().equals(SIMD);
2800 InstructionType type = floatFromSize(size);
2801 emitInt(type.encoding | FCMP.encoding | FpCmpeOp | rs1(src1) | rs2(src2));
2802 }
2803
2804 /**
2805 * Conditional compare. NZCV = fcmp(src1, src2) if condition else uimm4.
2806 *
2807 * @param size register size.
2808 * @param src1 floating point register. May not be null.
2809 * @param src2 floating point register. May not be null.
2810 * @param uimm4 condition flags that are used if condition is false.
2811 * @param condition every condition allowed. May not be null.
2812 */
2813 public void fccmp(int size, Register src1, Register src2, int uimm4, ConditionFlag condition) {
2814 assert NumUtil.isUnsignedNbit(4, uimm4);
2815 assert src1.getRegisterCategory().equals(SIMD);
2816 assert src2.getRegisterCategory().equals(SIMD);
2817 InstructionType type = floatFromSize(size);
2818 emitInt(type.encoding | FCCMP.encoding | uimm4 | condition.encoding << ConditionalConditionOffset | rs1(src1) | rs2(src2));
2819 }
2820
2821 /**
2822 * Compare register to 0.0 .
2823 *
2824 * @param size register size.
2825 * @param src floating point register. May not be null.
2826 */
2827 public void fcmpZero(int size, Register src) {
2828 assert src.getRegisterCategory().equals(SIMD);
2829 InstructionType type = floatFromSize(size);
2830 emitInt(type.encoding | FCMPZERO.encoding | FpCmpOp | rs1(src));
2831 }
2832
2833 /**
2834 * Signalling compare register to 0.0 .
2835 *
2836 * @param size register size.
2837 * @param src floating point register. May not be null.
2838 */
2839 public void fcmpeZero(int size, Register src) {
2840 assert src.getRegisterCategory().equals(SIMD);
2841 InstructionType type = floatFromSize(size);
2842 emitInt(type.encoding | FCMPZERO.encoding | FpCmpeOp | rs1(src));
2843 }
2844
2845 /* Floating-point Conditional Select (5.7.11) */
2846
2847 /**
2848 * Conditional select. dst = src1 if condition else src2.
2849 *
2850 * @param size register size.
2851 * @param dst floating point register. May not be null.
2852 * @param src1 floating point register. May not be null.
2853 * @param src2 floating point register. May not be null.
2854 * @param condition every condition allowed. May not be null.
2855 */
2856 protected void fcsel(int size, Register dst, Register src1, Register src2, ConditionFlag condition) {
2857 assert dst.getRegisterCategory().equals(SIMD);
2858 assert src1.getRegisterCategory().equals(SIMD);
2859 assert src2.getRegisterCategory().equals(SIMD);
2860 InstructionType type = floatFromSize(size);
2861 emitInt(type.encoding | FCSEL.encoding | rd(dst) | rs1(src1) | rs2(src2) | condition.encoding << ConditionalConditionOffset);
2862 }
2863
2864 /* Debug exceptions (5.9.1.2) */
2865
2866 /**
2867 * Halting mode software breakpoint: Enters halting mode debug state if enabled, else treated as
2868 * UNALLOCATED instruction.
2869 *
2870 * @param uimm16 Arbitrary 16-bit unsigned payload.
2871 */
2872 protected void hlt(int uimm16) {
2873 exceptionInstruction(HLT, uimm16);
2874 }
2875
2876 /**
2877 * Monitor mode software breakpoint: exception routed to a debug monitor executing in a higher
2878 * exception level.
2879 *
2880 * @param uimm16 Arbitrary 16-bit unsigned payload.
2881 */
2882 protected void brk(int uimm16) {
2883 exceptionInstruction(BRK, uimm16);
2884 }
2885
2886 private void exceptionInstruction(Instruction instr, int uimm16) {
2887 assert NumUtil.isUnsignedNbit(16, uimm16);
2888 emitInt(instr.encoding | ExceptionOp | uimm16 << SystemImmediateOffset);
2889 }
2890
2891 /* Architectural hints (5.9.4) */
2892 public enum SystemHint {
2893 NOP(0x0),
2894 YIELD(0x1),
2895 WFE(0x2),
2896 WFI(0x3),
2897 SEV(0x4),
2898 SEVL(0x5),
2899 CSDB(0x14);
2900
2901 private final int encoding;
2902
2903 SystemHint(int encoding) {
2904 this.encoding = encoding;
2905 }
2906 }
2907
2908 /**
2909 * Architectural hints.
2910 *
2911 * @param hint Can be any of the defined hints. May not be null.
2912 */
2913 protected void hint(SystemHint hint) {
2914 emitInt(HINT.encoding | hint.encoding << SystemImmediateOffset);
2915 }
2916
2917 /**
2918 * Clear Exclusive: clears the local record of the executing processor that an address has had a
2919 * request for an exclusive access.
2920 */
2921 protected void clrex() {
2922 emitInt(CLREX.encoding);
2923 }
2924
2925 /**
2926 * Possible barrier definitions for Aarch64. LOAD_LOAD and LOAD_STORE map to the same underlying
2927 * barrier.
2928 *
2929 * We only need synchronization across the inner shareable domain (see B2-90 in the Reference
2930 * documentation).
2931 */
2932 public enum BarrierKind {
2933 LOAD_LOAD(0x9, "ISHLD"),
2934 LOAD_STORE(0x9, "ISHLD"),
2935 STORE_STORE(0xA, "ISHST"),
2936 ANY_ANY(0xB, "ISH");
2937
2938 public final int encoding;
2939 public final String optionName;
2940
2941 BarrierKind(int encoding, String optionName) {
2942 this.encoding = encoding;
2943 this.optionName = optionName;
2944 }
2945 }
2946
2947 /**
2948 * Data Memory Barrier.
2949 *
2950 * @param barrierKind barrier that is issued. May not be null.
2951 */
2952 public void dmb(BarrierKind barrierKind) {
2953 emitInt(DMB.encoding | BarrierOp | barrierKind.encoding << BarrierKindOffset);
2954 }
2955
2956 public void mrs(Register dst, SystemRegister systemRegister) {
2957 emitInt(MRS.encoding | systemRegister.encoding() | rt(dst));
2958 }
2959
2960 public void msr(SystemRegister systemRegister, Register src) {
2961 emitInt(MRS.encoding | systemRegister.encoding() | rt(src));
2962 }
2963
2964 public void annotatePatchingImmediate(int pos, Instruction instruction, int operandSizeBits, int offsetBits, int shift) {
2965 if (codePatchingAnnotationConsumer != null) {
2966 codePatchingAnnotationConsumer.accept(new SingleInstructionAnnotation(pos, instruction, operandSizeBits, offsetBits, shift));
2967 }
2968 }
2969
2970 void annotateImmediateMovSequence(int pos, int numInstrs) {
2971 if (codePatchingAnnotationConsumer != null) {
2972 codePatchingAnnotationConsumer.accept(new MovSequenceAnnotation(pos, numInstrs));
2973 }
2974 }
2975
2976 public static class SingleInstructionAnnotation extends CodeAnnotation {
2977
2978 /**
2979 * The size of the operand, in bytes.
2980 */
2981 public final int operandSizeBits;
2982 public final int offsetBits;
2983 public final Instruction instruction;
2984 public final int shift;
2985
2986 SingleInstructionAnnotation(int instructionPosition, Instruction instruction, int operandSizeBits, int offsetBits, int shift) {
2987 super(instructionPosition);
2988 this.operandSizeBits = operandSizeBits;
2989 this.offsetBits = offsetBits;
2990 this.shift = shift;
2991 this.instruction = instruction;
2992 }
2993 }
2994
2995 public static class MovSequenceAnnotation extends CodeAnnotation {
2996
2997 /**
2998 * The size of the operand, in bytes.
2999 */
3000 public final int numInstrs;
3001
3002 MovSequenceAnnotation(int instructionPosition, int numInstrs) {
3003 super(instructionPosition);
3004 this.numInstrs = numInstrs;
3005 }
3006 }
3007
3008 /**
3009 * dst[0...n] = countBitCountOfEachByte(src[0...n]), n = size/8.
3010 *
3011 * @param size register size. Has to be 64 or 128.
3012 * @param dst SIMD register. Should not be null.
3013 * @param src SIMD register. Should not be null.
3014 */
3015 public void cnt(int size, Register dst, Register src) {
3016 assert 64 == size || 128 == size : "Invalid size for cnt";
3017 emitInt((size >> 7) << SIMDQBitOffset | CNT.encoding | rd(dst) | rs1(src));
3018 }
3019
3020 /**
3021 * dst = src[0] + ....+ src[n].
3022 *
3023 * @param size register size. Has to be 64 or 128.
3024 * @param laneWidth the width that SIMD register is treated as different lanes with.
3025 * @param dst SIMD register. Should not be null.
3026 * @param src SIMD register. Should not be null.
3027 */
3028 public void addv(int size, SIMDElementSize laneWidth, Register dst, Register src) {
3029 assert 64 == size || 128 == size : "Invalid size for addv";
3030 assert SIMDElementSize.DoubleWord != laneWidth : "Invalid lane width for addv";
3031 assert 64 != size || SIMDElementSize.Word != laneWidth : "Invalid size and lane combination for addv";
3032 emitInt((size >> 7) << SIMDQBitOffset | laneWidth.encoding << SIMDSizeOffset | ADDV.encoding | rd(dst) | rs1(src));
3033 }
3034
3035 /**
3036 * dst = src[srcIdx].
3037 *
3038 * @param size register size. Can be 8, 16, 32 or 64.
3039 * @param dst general purpose register. Should not be null or zero-register.
3040 * @param srcIdx lane index of source register that dest data is from.
3041 * @param src SIMD register. Should not be null.
3042 */
3043 public void umov(int size, Register dst, int srcIdx, Register src) {
3044 assert (srcIdx + 1) * size <= 128 : "Invalid src vectRegister index";
3045 InstructionType simdDataType = simdFromSize(size);
3046 int imm5 = simdDataType.encoding | srcIdx << Integer.numberOfTrailingZeros(simdDataType.encoding) + 1;
3047 emitInt((size >> 6) << SIMDQBitOffset | imm5 << SIMDImm5Offset | UMOV.encoding | rd(dst) | rs1(src));
3048 }
3049 }