1 //
   2 // Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
   3 // Copyright (c) 2017, SAP SE. All rights reserved.
   4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5 //
   6 // This code is free software; you can redistribute it and/or modify it
   7 // under the terms of the GNU General Public License version 2 only, as
   8 // published by the Free Software Foundation.
   9 //
  10 // This code is distributed in the hope that it will be useful, but WITHOUT
  11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13 // version 2 for more details (a copy is included in the LICENSE file that
  14 // accompanied this code).
  15 //
  16 // You should have received a copy of the GNU General Public License version
  17 // 2 along with this work; if not, write to the Free Software Foundation,
  18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19 //
  20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21 // or visit www.oracle.com if you need additional information or have any
  22 // questions.
  23 //
  24 
  25 // z/Architecture Architecture Description File
  26 
  27 // Major contributions by AS, JL, LS.
  28 
  29 //
  30 // Following information is derived from private mail communication
  31 // (Oct. 2011).
  32 //
  33 // General branch target alignment considerations
  34 //
  35 // z/Architecture does not imply a general branch target alignment requirement.
  36 // There are side effects and side considerations, though, which may
  37 // provide some performance benefit. These are:
  38 //  - Align branch target on octoword (32-byte) boundary
  39 //    On more recent models (from z9 on), I-fetch is done on a Octoword
  40 //    (32 bytes at a time) basis. To avoid I-fetching unnecessary
  41 //    instructions, branch targets should be 32-byte aligend. If this
  42 //    exact alingment cannot be achieved, having the branch target in
  43 //    the first doubleword still provides some benefit.
  44 //  - Avoid branch targets at the end of cache lines (> 64 bytes distance).
  45 //    Sequential instruction prefetching after the branch target starts
  46 //    immediately after having fetched the octoword containing the
  47 //    branch target. When I-fetching crosses a cache line, there may be
  48 //    a small stall. The worst case: the branch target (at the end of
  49 //    a cache line) is a L1 I-cache miss and the next line as well.
  50 //    Then, the entire target line must be filled first (to contine at the
  51 //    branch target). Only then can the next sequential line be filled.
  52 //  - Avoid multiple poorly predicted branches in a row.
  53 //
  54 
  55 //----------REGISTER DEFINITION BLOCK------------------------------------------
  56 // This information is used by the matcher and the register allocator to
  57 // describe individual registers and classes of registers within the target
  58 // architecture.
  59 
  60 register %{
  61 
  62 //----------Architecture Description Register Definitions----------------------
  63 // General Registers
  64 // "reg_def" name (register save type, C convention save type,
  65 //                   ideal register type, encoding);
  66 //
  67 // Register Save Types:
  68 //
  69 //   NS  = No-Save:     The register allocator assumes that these registers
  70 //                      can be used without saving upon entry to the method, &
  71 //                      that they do not need to be saved at call sites.
  72 //
  73 //   SOC = Save-On-Call: The register allocator assumes that these registers
  74 //                      can be used without saving upon entry to the method,
  75 //                      but that they must be saved at call sites.
  76 //
  77 //   SOE = Save-On-Entry: The register allocator assumes that these registers
  78 //                      must be saved before using them upon entry to the
  79 //                      method, but they do not need to be saved at call sites.
  80 //
  81 //   AS  = Always-Save: The register allocator assumes that these registers
  82 //                      must be saved before using them upon entry to the
  83 //                      method, & that they must be saved at call sites.
  84 //
  85 // Ideal Register Type is used to determine how to save & restore a
  86 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  87 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
  88 //
  89 // The encoding number is the actual bit-pattern placed into the opcodes.
  90 
  91 // z/Architecture register definitions, based on the z/Architecture Principles
  92 // of Operation, 5th Edition, September 2005, and z/Linux Elf ABI Supplement,
  93 // 5th Edition, March 2001.
  94 //
  95 // For each 64-bit register we must define two registers: the register
  96 // itself, e.g. Z_R3, and a corresponding virtual other (32-bit-)'half',
  97 // e.g. Z_R3_H, which is needed by the allocator, but is not used
  98 // for stores, loads, etc.
  99 
 100   // Integer/Long Registers
 101   // ----------------------------
 102 
 103   // z/Architecture has 16 64-bit integer registers.
 104 
 105   // types: v = volatile, nv = non-volatile, s = system
 106   reg_def Z_R0   (SOC, SOC, Op_RegI,  0, Z_R0->as_VMReg());   // v   scratch1
 107   reg_def Z_R0_H (SOC, SOC, Op_RegI, 99, Z_R0->as_VMReg()->next());
 108   reg_def Z_R1   (SOC, SOC, Op_RegI,  1, Z_R1->as_VMReg());   // v   scratch2
 109   reg_def Z_R1_H (SOC, SOC, Op_RegI, 99, Z_R1->as_VMReg()->next());
 110   reg_def Z_R2   (SOC, SOC, Op_RegI,  2, Z_R2->as_VMReg());   // v   iarg1 & iret
 111   reg_def Z_R2_H (SOC, SOC, Op_RegI, 99, Z_R2->as_VMReg()->next());
 112   reg_def Z_R3   (SOC, SOC, Op_RegI,  3, Z_R3->as_VMReg());   // v   iarg2
 113   reg_def Z_R3_H (SOC, SOC, Op_RegI, 99, Z_R3->as_VMReg()->next());
 114   reg_def Z_R4   (SOC, SOC, Op_RegI,  4, Z_R4->as_VMReg());   // v   iarg3
 115   reg_def Z_R4_H (SOC, SOC, Op_RegI, 99, Z_R4->as_VMReg()->next());
 116   reg_def Z_R5   (SOC, SOC, Op_RegI,  5, Z_R5->as_VMReg());   // v   iarg4
 117   reg_def Z_R5_H (SOC, SOC, Op_RegI, 99, Z_R5->as_VMReg()->next());
 118   reg_def Z_R6   (SOC, SOE, Op_RegI,  6, Z_R6->as_VMReg());   // v   iarg5
 119   reg_def Z_R6_H (SOC, SOE, Op_RegI, 99, Z_R6->as_VMReg()->next());
 120   reg_def Z_R7   (SOC, SOE, Op_RegI,  7, Z_R7->as_VMReg());
 121   reg_def Z_R7_H (SOC, SOE, Op_RegI, 99, Z_R7->as_VMReg()->next());
 122   reg_def Z_R8   (SOC, SOE, Op_RegI,  8, Z_R8->as_VMReg());
 123   reg_def Z_R8_H (SOC, SOE, Op_RegI, 99, Z_R8->as_VMReg()->next());
 124   reg_def Z_R9   (SOC, SOE, Op_RegI,  9, Z_R9->as_VMReg());
 125   reg_def Z_R9_H (SOC, SOE, Op_RegI, 99, Z_R9->as_VMReg()->next());
 126   reg_def Z_R10  (SOC, SOE, Op_RegI, 10, Z_R10->as_VMReg());
 127   reg_def Z_R10_H(SOC, SOE, Op_RegI, 99, Z_R10->as_VMReg()->next());
 128   reg_def Z_R11  (SOC, SOE, Op_RegI, 11, Z_R11->as_VMReg());
 129   reg_def Z_R11_H(SOC, SOE, Op_RegI, 99, Z_R11->as_VMReg()->next());
 130   reg_def Z_R12  (SOC, SOE, Op_RegI, 12, Z_R12->as_VMReg());
 131   reg_def Z_R12_H(SOC, SOE, Op_RegI, 99, Z_R12->as_VMReg()->next());
 132   reg_def Z_R13  (SOC, SOE, Op_RegI, 13, Z_R13->as_VMReg());
 133   reg_def Z_R13_H(SOC, SOE, Op_RegI, 99, Z_R13->as_VMReg()->next());
 134   reg_def Z_R14  (NS,  NS,  Op_RegI, 14, Z_R14->as_VMReg());   // s  return_pc
 135   reg_def Z_R14_H(NS,  NS,  Op_RegI, 99, Z_R14->as_VMReg()->next());
 136   reg_def Z_R15  (NS,  NS,  Op_RegI, 15, Z_R15->as_VMReg());   // s  SP
 137   reg_def Z_R15_H(NS,  NS,  Op_RegI, 99, Z_R15->as_VMReg()->next());
 138 
 139   // Float/Double Registers
 140 
 141   // The rules of ADL require that double registers be defined in pairs.
 142   // Each pair must be two 32-bit values, but not necessarily a pair of
 143   // single float registers. In each pair, ADLC-assigned register numbers
 144   // must be adjacent, with the lower number even. Finally, when the
 145   // CPU stores such a register pair to memory, the word associated with
 146   // the lower ADLC-assigned number must be stored to the lower address.
 147 
 148   // z/Architecture has 16 64-bit floating-point registers. Each can store a single
 149   // or double precision floating-point value.
 150 
 151   // types: v = volatile, nv = non-volatile, s = system
 152   reg_def Z_F0   (SOC, SOC, Op_RegF,  0, Z_F0->as_VMReg());   // v   farg1 & fret
 153   reg_def Z_F0_H (SOC, SOC, Op_RegF, 99, Z_F0->as_VMReg()->next());
 154   reg_def Z_F1   (SOC, SOC, Op_RegF,  1, Z_F1->as_VMReg());
 155   reg_def Z_F1_H (SOC, SOC, Op_RegF, 99, Z_F1->as_VMReg()->next());
 156   reg_def Z_F2   (SOC, SOC, Op_RegF,  2, Z_F2->as_VMReg());   // v   farg2
 157   reg_def Z_F2_H (SOC, SOC, Op_RegF, 99, Z_F2->as_VMReg()->next());
 158   reg_def Z_F3   (SOC, SOC, Op_RegF,  3, Z_F3->as_VMReg());
 159   reg_def Z_F3_H (SOC, SOC, Op_RegF, 99, Z_F3->as_VMReg()->next());
 160   reg_def Z_F4   (SOC, SOC, Op_RegF,  4, Z_F4->as_VMReg());   // v   farg3
 161   reg_def Z_F4_H (SOC, SOC, Op_RegF, 99, Z_F4->as_VMReg()->next());
 162   reg_def Z_F5   (SOC, SOC, Op_RegF,  5, Z_F5->as_VMReg());
 163   reg_def Z_F5_H (SOC, SOC, Op_RegF, 99, Z_F5->as_VMReg()->next());
 164   reg_def Z_F6   (SOC, SOC, Op_RegF,  6, Z_F6->as_VMReg());
 165   reg_def Z_F6_H (SOC, SOC, Op_RegF, 99, Z_F6->as_VMReg()->next());
 166   reg_def Z_F7   (SOC, SOC, Op_RegF,  7, Z_F7->as_VMReg());
 167   reg_def Z_F7_H (SOC, SOC, Op_RegF, 99, Z_F7->as_VMReg()->next());
 168   reg_def Z_F8   (SOC, SOE, Op_RegF,  8, Z_F8->as_VMReg());
 169   reg_def Z_F8_H (SOC, SOE, Op_RegF, 99, Z_F8->as_VMReg()->next());
 170   reg_def Z_F9   (SOC, SOE, Op_RegF,  9, Z_F9->as_VMReg());
 171   reg_def Z_F9_H (SOC, SOE, Op_RegF, 99, Z_F9->as_VMReg()->next());
 172   reg_def Z_F10  (SOC, SOE, Op_RegF, 10, Z_F10->as_VMReg());
 173   reg_def Z_F10_H(SOC, SOE, Op_RegF, 99, Z_F10->as_VMReg()->next());
 174   reg_def Z_F11  (SOC, SOE, Op_RegF, 11, Z_F11->as_VMReg());
 175   reg_def Z_F11_H(SOC, SOE, Op_RegF, 99, Z_F11->as_VMReg()->next());
 176   reg_def Z_F12  (SOC, SOE, Op_RegF, 12, Z_F12->as_VMReg());
 177   reg_def Z_F12_H(SOC, SOE, Op_RegF, 99, Z_F12->as_VMReg()->next());
 178   reg_def Z_F13  (SOC, SOE, Op_RegF, 13, Z_F13->as_VMReg());
 179   reg_def Z_F13_H(SOC, SOE, Op_RegF, 99, Z_F13->as_VMReg()->next());
 180   reg_def Z_F14  (SOC, SOE, Op_RegF, 14, Z_F14->as_VMReg());
 181   reg_def Z_F14_H(SOC, SOE, Op_RegF, 99, Z_F14->as_VMReg()->next());
 182   reg_def Z_F15  (SOC, SOE, Op_RegF, 15, Z_F15->as_VMReg());
 183   reg_def Z_F15_H(SOC, SOE, Op_RegF, 99, Z_F15->as_VMReg()->next());
 184 
 185 
 186   // Special Registers
 187 
 188   // Condition Codes Flag Registers
 189 
 190   // z/Architecture has the PSW (program status word) that contains
 191   // (among other information) the condition code. We treat this
 192   // part of the PSW as a condition register CR. It consists of 4
 193   // bits. Floating point instructions influence the same condition register CR.
 194 
 195   reg_def Z_CR(SOC, SOC, Op_RegFlags, 0, Z_CR->as_VMReg());   // volatile
 196 
 197 
 198 // Specify priority of register selection within phases of register
 199 // allocation. Highest priority is first. A useful heuristic is to
 200 // give registers a low priority when they are required by machine
 201 // instructions, and choose no-save registers before save-on-call, and
 202 // save-on-call before save-on-entry. Registers which participate in
 203 // fix calling sequences should come last. Registers which are used
 204 // as pairs must fall on an even boundary.
 205 
 206 // It's worth about 1% on SPEC geomean to get this right.
 207 
 208 // Chunk0, chunk1, and chunk2 form the MachRegisterNumbers enumeration
 209 // in adGlobals_s390.hpp which defines the <register>_num values, e.g.
 210 // Z_R3_num. Therefore, Z_R3_num may not be (and in reality is not)
 211 // the same as Z_R3->encoding()! Furthermore, we cannot make any
 212 // assumptions on ordering, e.g. Z_R3_num may be less than Z_R2_num.
 213 // Additionally, the function
 214 //   static enum RC rc_class(OptoReg::Name reg)
 215 // maps a given <register>_num value to its chunk type (except for flags)
 216 // and its current implementation relies on chunk0 and chunk1 having a
 217 // size of 64 each.
 218 
 219 alloc_class chunk0(
 220   // chunk0 contains *all* 32 integer registers halves.
 221 
 222   // potential SOE regs
 223   Z_R13,Z_R13_H,
 224   Z_R12,Z_R12_H,
 225   Z_R11,Z_R11_H,
 226   Z_R10,Z_R10_H,
 227 
 228   Z_R9,Z_R9_H,
 229   Z_R8,Z_R8_H,
 230   Z_R7,Z_R7_H,
 231 
 232   Z_R1,Z_R1_H,
 233   Z_R0,Z_R0_H,
 234 
 235   // argument registers
 236   Z_R6,Z_R6_H,
 237   Z_R5,Z_R5_H,
 238   Z_R4,Z_R4_H,
 239   Z_R3,Z_R3_H,
 240   Z_R2,Z_R2_H,
 241 
 242   // special registers
 243   Z_R14,Z_R14_H,
 244   Z_R15,Z_R15_H
 245 );
 246 
 247 alloc_class chunk1(
 248   // Chunk1 contains *all* 64 floating-point registers halves.
 249 
 250   Z_F15,Z_F15_H,
 251   Z_F14,Z_F14_H,
 252   Z_F13,Z_F13_H,
 253   Z_F12,Z_F12_H,
 254   Z_F11,Z_F11_H,
 255   Z_F10,Z_F10_H,
 256   Z_F9,Z_F9_H,
 257   Z_F8,Z_F8_H,
 258   // scratch register
 259   Z_F7,Z_F7_H,
 260   Z_F5,Z_F5_H,
 261   Z_F3,Z_F3_H,
 262   Z_F1,Z_F1_H,
 263   // argument registers
 264   Z_F6,Z_F6_H,
 265   Z_F4,Z_F4_H,
 266   Z_F2,Z_F2_H,
 267   Z_F0,Z_F0_H
 268 );
 269 
 270 alloc_class chunk2(
 271   Z_CR
 272 );
 273 
 274 
 275 //-------Architecture Description Register Classes-----------------------
 276 
 277 // Several register classes are automatically defined based upon
 278 // information in this architecture description.
 279 
 280 // 1) reg_class inline_cache_reg           (as defined in frame section)
 281 // 2) reg_class compiler_method_oop_reg    (as defined in frame section)
 282 // 2) reg_class interpreter_method_oop_reg (as defined in frame section)
 283 // 3) reg_class stack_slots(/* one chunk of stack-based "registers" */)
 284 
 285 // Integer Register Classes
 286 reg_class z_int_reg(
 287 /*Z_R0*/              // R0
 288 /*Z_R1*/
 289   Z_R2,
 290   Z_R3,
 291   Z_R4,
 292   Z_R5,
 293   Z_R6,
 294   Z_R7,
 295 /*Z_R8,*/             // Z_thread
 296   Z_R9,
 297   Z_R10,
 298   Z_R11,
 299   Z_R12,
 300   Z_R13
 301 /*Z_R14*/             // return_pc
 302 /*Z_R15*/             // SP
 303 );
 304 
 305 reg_class z_no_odd_int_reg(
 306 /*Z_R0*/              // R0
 307 /*Z_R1*/
 308   Z_R2,
 309   Z_R3,
 310   Z_R4,
 311 /*Z_R5,*/             // odd part of fix register pair
 312   Z_R6,
 313   Z_R7,
 314 /*Z_R8,*/             // Z_thread
 315   Z_R9,
 316   Z_R10,
 317   Z_R11,
 318   Z_R12,
 319   Z_R13
 320 /*Z_R14*/             // return_pc
 321 /*Z_R15*/             // SP
 322 );
 323 
 324 reg_class z_no_arg_int_reg(
 325 /*Z_R0*/              // R0
 326 /*Z_R1*/              // scratch
 327 /*Z_R2*/
 328 /*Z_R3*/
 329 /*Z_R4*/
 330 /*Z_R5*/
 331 /*Z_R6*/
 332   Z_R7,
 333 /*Z_R8*/              // Z_thread
 334   Z_R9,
 335   Z_R10,
 336   Z_R11,
 337   Z_R12,
 338   Z_R13
 339 /*Z_R14*/             // return_pc
 340 /*Z_R15*/             // SP
 341 );
 342 
 343 reg_class z_rarg1_int_reg(Z_R2);
 344 reg_class z_rarg2_int_reg(Z_R3);
 345 reg_class z_rarg3_int_reg(Z_R4);
 346 reg_class z_rarg4_int_reg(Z_R5);
 347 reg_class z_rarg5_int_reg(Z_R6);
 348 
 349 // Pointer Register Classes
 350 
 351 // 64-bit build means 64-bit pointers means hi/lo pairs.
 352 
 353 reg_class z_rarg5_ptrN_reg(Z_R6);
 354 
 355 reg_class z_rarg1_ptr_reg(Z_R2_H,Z_R2);
 356 reg_class z_rarg2_ptr_reg(Z_R3_H,Z_R3);
 357 reg_class z_rarg3_ptr_reg(Z_R4_H,Z_R4);
 358 reg_class z_rarg4_ptr_reg(Z_R5_H,Z_R5);
 359 reg_class z_rarg5_ptr_reg(Z_R6_H,Z_R6);
 360 reg_class z_thread_ptr_reg(Z_R8_H,Z_R8);
 361 
 362 reg_class z_ptr_reg(
 363 /*Z_R0_H,Z_R0*/     // R0
 364 /*Z_R1_H,Z_R1*/
 365   Z_R2_H,Z_R2,
 366   Z_R3_H,Z_R3,
 367   Z_R4_H,Z_R4,
 368   Z_R5_H,Z_R5,
 369   Z_R6_H,Z_R6,
 370   Z_R7_H,Z_R7,
 371 /*Z_R8_H,Z_R8,*/    // Z_thread
 372   Z_R9_H,Z_R9,
 373   Z_R10_H,Z_R10,
 374   Z_R11_H,Z_R11,
 375   Z_R12_H,Z_R12,
 376   Z_R13_H,Z_R13
 377 /*Z_R14_H,Z_R14*/   // return_pc
 378 /*Z_R15_H,Z_R15*/   // SP
 379 );
 380 
 381 reg_class z_lock_ptr_reg(
 382 /*Z_R0_H,Z_R0*/     // R0
 383 /*Z_R1_H,Z_R1*/
 384   Z_R2_H,Z_R2,
 385   Z_R3_H,Z_R3,
 386   Z_R4_H,Z_R4,
 387 /*Z_R5_H,Z_R5,*/
 388 /*Z_R6_H,Z_R6,*/
 389   Z_R7_H,Z_R7,
 390 /*Z_R8_H,Z_R8,*/    // Z_thread
 391   Z_R9_H,Z_R9,
 392   Z_R10_H,Z_R10,
 393   Z_R11_H,Z_R11,
 394   Z_R12_H,Z_R12,
 395   Z_R13_H,Z_R13
 396 /*Z_R14_H,Z_R14*/   // return_pc
 397 /*Z_R15_H,Z_R15*/   // SP
 398 );
 399 
 400 reg_class z_no_arg_ptr_reg(
 401 /*Z_R0_H,Z_R0*/        // R0
 402 /*Z_R1_H,Z_R1*/        // scratch
 403 /*Z_R2_H,Z_R2*/
 404 /*Z_R3_H,Z_R3*/
 405 /*Z_R4_H,Z_R4*/
 406 /*Z_R5_H,Z_R5*/
 407 /*Z_R6_H,Z_R6*/
 408   Z_R7_H, Z_R7,
 409 /*Z_R8_H,Z_R8*/        // Z_thread
 410   Z_R9_H,Z_R9,
 411   Z_R10_H,Z_R10,
 412   Z_R11_H,Z_R11,
 413   Z_R12_H,Z_R12,
 414   Z_R13_H,Z_R13
 415 /*Z_R14_H,Z_R14*/      // return_pc
 416 /*Z_R15_H,Z_R15*/      // SP
 417 );
 418 
 419 // Special class for storeP instructions, which can store SP or RPC to
 420 // TLS. (Note: Do not generalize this to "any_reg". If you add
 421 // another register, such as FP, to this mask, the allocator may try
 422 // to put a temp in it.)
 423 // Register class for memory access base registers,
 424 // This class is a superset of z_ptr_reg including Z_thread.
 425 reg_class z_memory_ptr_reg(
 426 /*Z_R0_H,Z_R0*/     // R0
 427 /*Z_R1_H,Z_R1*/
 428   Z_R2_H,Z_R2,
 429   Z_R3_H,Z_R3,
 430   Z_R4_H,Z_R4,
 431   Z_R5_H,Z_R5,
 432   Z_R6_H,Z_R6,
 433   Z_R7_H,Z_R7,
 434   Z_R8_H,Z_R8,      // Z_thread
 435   Z_R9_H,Z_R9,
 436   Z_R10_H,Z_R10,
 437   Z_R11_H,Z_R11,
 438   Z_R12_H,Z_R12,
 439   Z_R13_H,Z_R13
 440 /*Z_R14_H,Z_R14*/   // return_pc
 441 /*Z_R15_H,Z_R15*/   // SP
 442 );
 443 
 444 // Other special pointer regs.
 445 reg_class z_r1_regP(Z_R1_H,Z_R1);
 446 reg_class z_r9_regP(Z_R9_H,Z_R9);
 447 
 448 
 449 // Long Register Classes
 450 
 451 reg_class z_rarg1_long_reg(Z_R2_H,Z_R2);
 452 reg_class z_rarg2_long_reg(Z_R3_H,Z_R3);
 453 reg_class z_rarg3_long_reg(Z_R4_H,Z_R4);
 454 reg_class z_rarg4_long_reg(Z_R5_H,Z_R5);
 455 reg_class z_rarg5_long_reg(Z_R6_H,Z_R6);
 456 
 457 // Longs in 1 register. Aligned adjacent hi/lo pairs.
 458 reg_class z_long_reg(
 459 /*Z_R0_H,Z_R0*/     // R0
 460 /*Z_R1_H,Z_R1*/
 461   Z_R2_H,Z_R2,
 462   Z_R3_H,Z_R3,
 463   Z_R4_H,Z_R4,
 464   Z_R5_H,Z_R5,
 465   Z_R6_H,Z_R6,
 466   Z_R7_H,Z_R7,
 467 /*Z_R8_H,Z_R8,*/    // Z_thread
 468   Z_R9_H,Z_R9,
 469   Z_R10_H,Z_R10,
 470   Z_R11_H,Z_R11,
 471   Z_R12_H,Z_R12,
 472   Z_R13_H,Z_R13
 473 /*Z_R14_H,Z_R14,*/  // return_pc
 474 /*Z_R15_H,Z_R15*/   // SP
 475 );
 476 
 477 // z_long_reg without even registers
 478 reg_class z_long_odd_reg(
 479 /*Z_R0_H,Z_R0*/     // R0
 480 /*Z_R1_H,Z_R1*/
 481   Z_R3_H,Z_R3,
 482   Z_R5_H,Z_R5,
 483   Z_R7_H,Z_R7,
 484   Z_R9_H,Z_R9,
 485   Z_R11_H,Z_R11,
 486   Z_R13_H,Z_R13
 487 /*Z_R14_H,Z_R14,*/  // return_pc
 488 /*Z_R15_H,Z_R15*/   // SP
 489 );
 490 
 491 // Special Class for Condition Code Flags Register
 492 
 493 reg_class z_condition_reg(
 494   Z_CR
 495 );
 496 
 497 // Scratch register for late profiling. Callee saved.
 498 reg_class z_rscratch2_bits64_reg(Z_R2_H, Z_R2);
 499 
 500 
 501 // Float Register Classes
 502 
 503 reg_class z_flt_reg(
 504   Z_F0,
 505 /*Z_F1,*/ // scratch
 506   Z_F2,
 507   Z_F3,
 508   Z_F4,
 509   Z_F5,
 510   Z_F6,
 511   Z_F7,
 512   Z_F8,
 513   Z_F9,
 514   Z_F10,
 515   Z_F11,
 516   Z_F12,
 517   Z_F13,
 518   Z_F14,
 519   Z_F15
 520 );
 521 reg_class z_rscratch1_flt_reg(Z_F1);
 522 
 523 // Double precision float registers have virtual `high halves' that
 524 // are needed by the allocator.
 525 reg_class z_dbl_reg(
 526   Z_F0,Z_F0_H,
 527 /*Z_F1,Z_F1_H,*/ // scratch
 528   Z_F2,Z_F2_H,
 529   Z_F3,Z_F3_H,
 530   Z_F4,Z_F4_H,
 531   Z_F5,Z_F5_H,
 532   Z_F6,Z_F6_H,
 533   Z_F7,Z_F7_H,
 534   Z_F8,Z_F8_H,
 535   Z_F9,Z_F9_H,
 536   Z_F10,Z_F10_H,
 537   Z_F11,Z_F11_H,
 538   Z_F12,Z_F12_H,
 539   Z_F13,Z_F13_H,
 540   Z_F14,Z_F14_H,
 541   Z_F15,Z_F15_H
 542 );
 543 reg_class z_rscratch1_dbl_reg(Z_F1,Z_F1_H);
 544 
 545 %}
 546 
 547 //----------DEFINITION BLOCK---------------------------------------------------
 548 // Define 'name --> value' mappings to inform the ADLC of an integer valued name.
 549 // Current support includes integer values in the range [0, 0x7FFFFFFF].
 550 // Format:
 551 //        int_def  <name>         (<int_value>, <expression>);
 552 // Generated Code in ad_<arch>.hpp
 553 //        #define  <name>   (<expression>)
 554 //        // value == <int_value>
 555 // Generated code in ad_<arch>.cpp adlc_verification()
 556 //        assert(<name> == <int_value>, "Expect (<expression>) to equal <int_value>");
 557 //
 558 definitions %{
 559   // The default cost (of an ALU instruction).
 560   int_def DEFAULT_COST      (   100,     100);
 561   int_def DEFAULT_COST_LOW  (    80,      80);
 562   int_def DEFAULT_COST_HIGH (   120,     120);
 563   int_def HUGE_COST         (1000000, 1000000);
 564 
 565   // Put an advantage on REG_MEM vs. MEM+REG_REG operations.
 566   int_def ALU_REG_COST      (   100, DEFAULT_COST);
 567   int_def ALU_MEMORY_COST   (   150,          150);
 568 
 569   // Memory refs are twice as expensive as run-of-the-mill.
 570   int_def MEMORY_REF_COST_HI (   220, 2 * DEFAULT_COST+20);
 571   int_def MEMORY_REF_COST    (   200, 2 * DEFAULT_COST);
 572   int_def MEMORY_REF_COST_LO (   180, 2 * DEFAULT_COST-20);
 573 
 574   // Branches are even more expensive.
 575   int_def BRANCH_COST       (   300, DEFAULT_COST * 3);
 576   int_def CALL_COST         (   300, DEFAULT_COST * 3);
 577 %}
 578 
 579 source %{
 580 
 581 #ifdef PRODUCT
 582 #define BLOCK_COMMENT(str)
 583 #define BIND(label)        __ bind(label)
 584 #else
 585 #define BLOCK_COMMENT(str) __ block_comment(str)
 586 #define BIND(label)        __ bind(label); BLOCK_COMMENT(#label ":")
 587 #endif
 588 
 589 #define __ _masm.
 590 
 591 #define Z_DISP_SIZE Immediate::is_uimm12((long)opnd_array(1)->disp(ra_,this,2)) ?  4 : 6
 592 #define Z_DISP3_SIZE 6
 593 
 594 // Tertiary op of a LoadP or StoreP encoding.
 595 #define REGP_OP true
 596 
 597 // Given a register encoding, produce an Integer Register object.
 598 static Register reg_to_register_object(int register_encoding);
 599 
 600 // ****************************************************************************
 601 
 602 // REQUIRED FUNCTIONALITY
 603 
 604 // !!!!! Special hack to get all type of calls to specify the byte offset
 605 //       from the start of the call to the point where the return address
 606 //       will point.
 607 
 608 int MachCallStaticJavaNode::ret_addr_offset() {
 609   if (_method) {
 610     return 8;
 611   } else {
 612     return MacroAssembler::call_far_patchable_ret_addr_offset();
 613   }
 614 }
 615 
 616 int MachCallDynamicJavaNode::ret_addr_offset() {
 617   // Consider size of receiver type profiling (C2 tiers).
 618   int profile_receiver_type_size = 0;
 619 
 620   int vtable_index = this->_vtable_index;
 621   if (vtable_index == -4) {
 622     return 14 + profile_receiver_type_size;
 623   } else {
 624     assert(!UseInlineCaches, "expect vtable calls only if not using ICs");
 625     return 36 + profile_receiver_type_size;
 626   }
 627 }
 628 
 629 int MachCallRuntimeNode::ret_addr_offset() {
 630   return 12 + MacroAssembler::call_far_patchable_ret_addr_offset();
 631 }
 632 
 633 // Compute padding required for nodes which need alignment
 634 //
 635 // The addresses of the call instructions needs to be 4-byte aligned to
 636 // ensure that they don't span a cache line so that they are atomically patchable.
 637 // The actual calls get emitted at different offsets within the node emitters.
 638 // ins_alignment needs to be set to 2 which means that up to 1 nop may get inserted.
 639 
 640 int CallStaticJavaDirect_dynTOCNode::compute_padding(int current_offset) const {
 641   return (0 - current_offset) & 2;
 642 }
 643 
 644 int CallDynamicJavaDirect_dynTOCNode::compute_padding(int current_offset) const {
 645   return (6 - current_offset) & 2;
 646 }
 647 
 648 int CallRuntimeDirectNode::compute_padding(int current_offset) const {
 649   return (12 - current_offset) & 2;
 650 }
 651 
 652 int CallLeafDirectNode::compute_padding(int current_offset) const {
 653   return (12 - current_offset) & 2;
 654 }
 655 
 656 int CallLeafNoFPDirectNode::compute_padding(int current_offset) const {
 657   return (12 - current_offset) & 2;
 658 }
 659 
 660 // Indicate if the safepoint node needs the polling page as an input.
 661 // Since z/Architecture does not have absolute addressing, it does.
 662 bool SafePointNode::needs_polling_address_input() {
 663   return true;
 664 }
 665 
 666 void emit_nop(CodeBuffer &cbuf) {
 667   MacroAssembler _masm(&cbuf);
 668   __ z_nop();
 669 }
 670 
 671 // Emit an interrupt that is caught by the debugger (for debugging compiler).
 672 void emit_break(CodeBuffer &cbuf) {
 673   MacroAssembler _masm(&cbuf);
 674   __ z_illtrap();
 675 }
 676 
 677 #if !defined(PRODUCT)
 678 void MachBreakpointNode::format(PhaseRegAlloc *, outputStream *os) const {
 679   os->print("TA");
 680 }
 681 #endif
 682 
 683 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 684   emit_break(cbuf);
 685 }
 686 
 687 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 688   return MachNode::size(ra_);
 689 }
 690 
 691 static inline void z_emit16(CodeBuffer &cbuf, long value) {
 692   // 32bit instructions may become sign extended.
 693   assert(value >= 0, "unintended sign extension (int->long)");
 694   assert(value < (1L << 16), "instruction too large");
 695   *((unsigned short*)(cbuf.insts_end())) = (unsigned short)value;
 696   cbuf.set_insts_end(cbuf.insts_end() + sizeof(unsigned short));
 697 }
 698 
 699 static inline void z_emit32(CodeBuffer &cbuf, long value) {
 700   // 32bit instructions may become sign extended.
 701   assert(value < (1L << 32), "instruction too large");
 702   *((unsigned int*)(cbuf.insts_end())) = (unsigned int)value;
 703   cbuf.set_insts_end(cbuf.insts_end() + sizeof(unsigned int));
 704 }
 705 
 706 static inline void z_emit48(CodeBuffer &cbuf, long value) {
 707   // 32bit instructions may become sign extended.
 708   assert(value >= 0, "unintended sign extension (int->long)");
 709   assert(value < (1L << 48), "instruction too large");
 710   value = value<<16;
 711   memcpy(cbuf.insts_end(), (unsigned char*)&value, 6);
 712   cbuf.set_insts_end(cbuf.insts_end() + 6);
 713 }
 714 
 715 static inline unsigned int z_emit_inst(CodeBuffer &cbuf, long value) {
 716   if (value < 0) {
 717     // There obviously has been an unintended sign extension (int->long). Revert it.
 718     value = (long)((unsigned long)((unsigned int)value));
 719   }
 720 
 721   if (value < (1L << 16)) { // 2-byte instruction
 722     z_emit16(cbuf, value);
 723     return 2;
 724   }
 725 
 726   if (value < (1L << 32)) { // 4-byte instruction, might be unaligned store
 727     z_emit32(cbuf, value);
 728     return 4;
 729   }
 730 
 731   // 6-byte instruction, probably unaligned store.
 732   z_emit48(cbuf, value);
 733   return 6;
 734 }
 735 
 736 // Check effective address (at runtime) for required alignment.
 737 static inline void z_assert_aligned(CodeBuffer &cbuf, int disp, Register index, Register base, int alignment) {
 738   MacroAssembler _masm(&cbuf);
 739 
 740   __ z_lay(Z_R0, disp, index, base);
 741   __ z_nill(Z_R0, alignment-1);
 742   __ z_brc(Assembler::bcondEqual, +3);
 743   __ z_illtrap();
 744 }
 745 
 746 int emit_call_reloc(MacroAssembler &_masm, intptr_t entry_point, relocInfo::relocType rtype,
 747                     PhaseRegAlloc* ra_, bool is_native_call = false) {
 748   __ set_inst_mark(); // Used in z_enc_java_static_call() and emit_java_to_interp().
 749   address old_mark = __ inst_mark();
 750   unsigned int start_off = __ offset();
 751 
 752   if (is_native_call) {
 753     ShouldNotReachHere();
 754   }
 755 
 756   if (rtype == relocInfo::runtime_call_w_cp_type) {
 757     assert((__ offset() & 2) == 0, "misaligned emit_call_reloc");
 758     address call_addr = __ call_c_opt((address)entry_point);
 759     if (call_addr == NULL) {
 760       Compile::current()->env()->record_out_of_memory_failure();
 761       return -1;
 762     }
 763   } else {
 764     assert(rtype == relocInfo::none || rtype == relocInfo::opt_virtual_call_type ||
 765            rtype == relocInfo::static_call_type, "unexpected rtype");
 766     __ relocate(rtype);
 767     // BRASL must be prepended with a nop to identify it in the instruction stream.
 768     __ z_nop();
 769     __ z_brasl(Z_R14, (address)entry_point);
 770   }
 771 
 772   unsigned int ret_off = __ offset();
 773 
 774   return (ret_off - start_off);
 775 }
 776 
 777 static int emit_call_reloc(MacroAssembler &_masm, intptr_t entry_point, RelocationHolder const& rspec) {
 778   __ set_inst_mark(); // Used in z_enc_java_static_call() and emit_java_to_interp().
 779   address old_mark = __ inst_mark();
 780   unsigned int start_off = __ offset();
 781 
 782   relocInfo::relocType rtype = rspec.type();
 783   assert(rtype == relocInfo::opt_virtual_call_type || rtype == relocInfo::static_call_type,
 784          "unexpected rtype");
 785 
 786   __ relocate(rspec);
 787   __ z_nop();
 788   __ z_brasl(Z_R14, (address)entry_point);
 789 
 790   unsigned int ret_off = __ offset();
 791 
 792   return (ret_off - start_off);
 793 }
 794 
 795 //=============================================================================
 796 
 797 const RegMask& MachConstantBaseNode::_out_RegMask = _Z_PTR_REG_mask;
 798 int Compile::ConstantTable::calculate_table_base_offset() const {
 799   return 0;  // absolute addressing, no offset
 800 }
 801 
 802 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 803 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 804   ShouldNotReachHere();
 805 }
 806 
 807 // Even with PC-relative TOC addressing, we still need this node.
 808 // Float loads/stores do not support PC-relative addresses.
 809 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 810   MacroAssembler _masm(&cbuf);
 811   Register Rtoc = as_Register(ra_->get_encode(this));
 812   __ load_toc(Rtoc);
 813 }
 814 
 815 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 816   // PCrelative TOC access.
 817   return 6;   // sizeof(LARL)
 818 }
 819 
 820 #if !defined(PRODUCT)
 821 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 822   Register r = as_Register(ra_->get_encode(this));
 823   st->print("LARL    %s,&constant_pool # MachConstantBaseNode", r->name());
 824 }
 825 #endif
 826 
 827 //=============================================================================
 828 
 829 #if !defined(PRODUCT)
 830 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
 831   Compile* C = ra_->C;
 832   st->print_cr("--- MachPrologNode ---");
 833   st->print("\t");
 834   for (int i = 0; i < OptoPrologueNops; i++) {
 835     st->print_cr("NOP"); st->print("\t");
 836   }
 837 
 838   if (VerifyThread) {
 839     st->print_cr("Verify_Thread");
 840     st->print("\t");
 841   }
 842 
 843   long framesize = C->frame_size_in_bytes();
 844   int bangsize   = C->bang_size_in_bytes();
 845 
 846   // Calls to C2R adapters often do not accept exceptional returns.
 847   // We require that their callers must bang for them. But be
 848   // careful, because some VM calls (such as call site linkage) can
 849   // use several kilobytes of stack. But the stack safety zone should
 850   // account for that. See bugs 4446381, 4468289, 4497237.
 851   if (C->need_stack_bang(bangsize) && UseStackBanging) {
 852     st->print_cr("# stack bang"); st->print("\t");
 853   }
 854   st->print_cr("push_frame %d", (int)-framesize);
 855   st->print("\t");
 856 }
 857 #endif
 858 
 859 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 860   Compile* C = ra_->C;
 861   MacroAssembler _masm(&cbuf);
 862 
 863   __ verify_thread();
 864 
 865   size_t framesize = C->frame_size_in_bytes();
 866   size_t bangsize  = C->bang_size_in_bytes();
 867 
 868   assert(framesize % wordSize == 0, "must preserve wordSize alignment");
 869 
 870   // Calls to C2R adapters often do not accept exceptional returns.
 871   // We require that their callers must bang for them. But be
 872   // careful, because some VM calls (such as call site linkage) can
 873   // use several kilobytes of stack. But the stack safety zone should
 874   // account for that. See bugs 4446381, 4468289, 4497237.
 875   if (C->need_stack_bang(bangsize) && UseStackBanging) {
 876     __ generate_stack_overflow_check(bangsize);
 877   }
 878 
 879   assert(Immediate::is_uimm32((long)framesize), "to do: choose suitable types!");
 880   __ save_return_pc();
 881 
 882   // The z/Architecture abi is already accounted for in `framesize' via the
 883   // 'out_preserve_stack_slots' declaration.
 884   __ push_frame((unsigned int)framesize/*includes JIT ABI*/);
 885 
 886   if (C->has_mach_constant_base_node()) {
 887     // NOTE: We set the table base offset here because users might be
 888     // emitted before MachConstantBaseNode.
 889     Compile::ConstantTable& constant_table = C->constant_table();
 890     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 891   }
 892 }
 893 
 894 uint MachPrologNode::size(PhaseRegAlloc *ra_) const {
 895   // Variable size. Determine dynamically.
 896   return MachNode::size(ra_);
 897 }
 898 
 899 int MachPrologNode::reloc() const {
 900   // Return number of relocatable values contained in this instruction.
 901   return 1; // One reloc entry for load_const(toc).
 902 }
 903 
 904 //=============================================================================
 905 
 906 #if !defined(PRODUCT)
 907 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *os) const {
 908   os->print_cr("epilog");
 909   os->print("\t");
 910   if (do_polling() && ra_->C->is_method_compilation()) {
 911     os->print_cr("load_from_polling_page Z_R1_scratch");
 912     os->print("\t");
 913   }
 914 }
 915 #endif
 916 
 917 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 918   MacroAssembler _masm(&cbuf);
 919   Compile* C = ra_->C;
 920   __ verify_thread();
 921 
 922   // If this does safepoint polling, then do it here.
 923   bool need_polling = do_polling() && C->is_method_compilation();
 924 
 925   // Pop frame, restore return_pc, and all stuff needed by interpreter.
 926   int frame_size_in_bytes = Assembler::align((C->frame_slots() << LogBytesPerInt), frame::alignment_in_bytes);
 927   __ pop_frame_restore_retPC(frame_size_in_bytes);
 928 
 929   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 930     __ reserved_stack_check(Z_R14);
 931   }
 932 
 933   // Touch the polling page.
 934   if (need_polling) {
 935     if (SafepointMechanism::uses_thread_local_poll()) {
 936       __ z_lg(Z_R1_scratch, Address(Z_thread, Thread::polling_page_offset()));
 937     } else {
 938       AddressLiteral pp(os::get_polling_page());
 939       __ load_const_optimized(Z_R1_scratch, pp);
 940     }
 941     // We need to mark the code position where the load from the safepoint
 942     // polling page was emitted as relocInfo::poll_return_type here.
 943     __ relocate(relocInfo::poll_return_type);
 944     __ load_from_polling_page(Z_R1_scratch);
 945   }
 946 }
 947 
 948 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 949   // Variable size. determine dynamically.
 950   return MachNode::size(ra_);
 951 }
 952 
 953 int MachEpilogNode::reloc() const {
 954   // Return number of relocatable values contained in this instruction.
 955   return 1; // One for load_from_polling_page.
 956 }
 957 
 958 const Pipeline * MachEpilogNode::pipeline() const {
 959   return MachNode::pipeline_class();
 960 }
 961 
 962 int MachEpilogNode::safepoint_offset() const {
 963   assert(do_polling(), "no return for this epilog node");
 964   return 0;
 965 }
 966 
 967 //=============================================================================
 968 
 969 // Figure out which register class each belongs in: rc_int, rc_float, rc_stack.
 970 enum RC { rc_bad, rc_int, rc_float, rc_stack };
 971 
 972 static enum RC rc_class(OptoReg::Name reg) {
 973   // Return the register class for the given register. The given register
 974   // reg is a <register>_num value, which is an index into the MachRegisterNumbers
 975   // enumeration in adGlobals_s390.hpp.
 976 
 977   if (reg == OptoReg::Bad) {
 978     return rc_bad;
 979   }
 980 
 981   // We have 32 integer register halves, starting at index 0.
 982   if (reg < 32) {
 983     return rc_int;
 984   }
 985 
 986   // We have 32 floating-point register halves, starting at index 32.
 987   if (reg < 32+32) {
 988     return rc_float;
 989   }
 990 
 991   // Between float regs & stack are the flags regs.
 992   assert(reg >= OptoReg::stack0(), "blow up if spilling flags");
 993   return rc_stack;
 994 }
 995 
 996 // Returns size as obtained from z_emit_instr.
 997 static unsigned int z_ld_st_helper(CodeBuffer *cbuf, const char *op_str, unsigned long opcode,
 998                                    int reg, int offset, bool do_print, outputStream *os) {
 999 
1000   if (cbuf) {
1001     if (opcode > (1L<<32)) {
1002       return z_emit_inst(*cbuf, opcode | Assembler::reg(Matcher::_regEncode[reg], 8, 48) |
1003                          Assembler::simm20(offset) | Assembler::reg(Z_R0, 12, 48) | Assembler::regz(Z_SP, 16, 48));
1004     } else {
1005       return z_emit_inst(*cbuf, opcode | Assembler::reg(Matcher::_regEncode[reg], 8, 32) |
1006                          Assembler::uimm12(offset, 20, 32) | Assembler::reg(Z_R0, 12, 32) | Assembler::regz(Z_SP, 16, 32));
1007     }
1008   }
1009 
1010 #if !defined(PRODUCT)
1011   if (do_print) {
1012     os->print("%s    %s,#%d[,SP]\t # MachCopy spill code",op_str, Matcher::regName[reg], offset);
1013   }
1014 #endif
1015   return (opcode > (1L << 32)) ? 6 : 4;
1016 }
1017 
1018 static unsigned int z_mvc_helper(CodeBuffer *cbuf, int len, int dst_off, int src_off, bool do_print, outputStream *os) {
1019   if (cbuf) {
1020     MacroAssembler _masm(cbuf);
1021     __ z_mvc(dst_off, len-1, Z_SP, src_off, Z_SP);
1022   }
1023 
1024 #if !defined(PRODUCT)
1025   else if (do_print) {
1026     os->print("MVC     %d(%d,SP),%d(SP)\t # MachCopy spill code",dst_off, len, src_off);
1027   }
1028 #endif
1029 
1030   return 6;
1031 }
1032 
1033 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *os) const {
1034   // Get registers to move.
1035   OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1036   OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1037   OptoReg::Name dst_hi = ra_->get_reg_second(this);
1038   OptoReg::Name dst_lo = ra_->get_reg_first(this);
1039 
1040   enum RC src_hi_rc = rc_class(src_hi);
1041   enum RC src_lo_rc = rc_class(src_lo);
1042   enum RC dst_hi_rc = rc_class(dst_hi);
1043   enum RC dst_lo_rc = rc_class(dst_lo);
1044 
1045   assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1046   bool is64 = (src_hi_rc != rc_bad);
1047   assert(!is64 ||
1048          ((src_lo&1) == 0 && src_lo+1 == src_hi && (dst_lo&1) == 0 && dst_lo+1 == dst_hi),
1049          "expected aligned-adjacent pairs");
1050 
1051   // Generate spill code!
1052 
1053   if (src_lo == dst_lo && src_hi == dst_hi) {
1054     return 0;            // Self copy, no move.
1055   }
1056 
1057   int  src_offset = ra_->reg2offset(src_lo);
1058   int  dst_offset = ra_->reg2offset(dst_lo);
1059   bool print = !do_size;
1060   bool src12 = Immediate::is_uimm12(src_offset);
1061   bool dst12 = Immediate::is_uimm12(dst_offset);
1062 
1063   const char   *mnemo = NULL;
1064   unsigned long opc = 0;
1065 
1066   // Memory->Memory Spill. Use Z_R0 to hold the value.
1067   if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1068 
1069     assert(!is64 || (src_hi_rc==rc_stack && dst_hi_rc==rc_stack),
1070            "expected same type of move for high parts");
1071 
1072     if (src12 && dst12) {
1073       return z_mvc_helper(cbuf, is64 ? 8 : 4, dst_offset, src_offset, print, os);
1074     }
1075 
1076     int r0 = Z_R0_num;
1077     if (is64) {
1078       return z_ld_st_helper(cbuf, "LG  ", LG_ZOPC, r0, src_offset, print, os) +
1079              z_ld_st_helper(cbuf, "STG ", STG_ZOPC, r0, dst_offset, print, os);
1080     }
1081 
1082     return z_ld_st_helper(cbuf, "LY   ", LY_ZOPC, r0, src_offset, print, os) +
1083            z_ld_st_helper(cbuf, "STY  ", STY_ZOPC, r0, dst_offset, print, os);
1084   }
1085 
1086   // Check for float->int copy. Requires a trip through memory.
1087   if (src_lo_rc == rc_float && dst_lo_rc == rc_int) {
1088     Unimplemented();  // Unsafe, do not remove!
1089   }
1090 
1091   // Check for integer reg-reg copy.
1092   if (src_lo_rc == rc_int && dst_lo_rc == rc_int) {
1093     if (cbuf) {
1094       MacroAssembler _masm(cbuf);
1095       Register Rsrc = as_Register(Matcher::_regEncode[src_lo]);
1096       Register Rdst = as_Register(Matcher::_regEncode[dst_lo]);
1097       __ z_lgr(Rdst, Rsrc);
1098       return 4;
1099     }
1100 #if !defined(PRODUCT)
1101     // else
1102     if (print) {
1103       os->print("LGR     %s,%s\t # MachCopy spill code", Matcher::regName[dst_lo], Matcher::regName[src_lo]);
1104     }
1105 #endif
1106     return 4;
1107   }
1108 
1109   // Check for integer store.
1110   if (src_lo_rc == rc_int && dst_lo_rc == rc_stack) {
1111     assert(!is64 || (src_hi_rc==rc_int && dst_hi_rc==rc_stack),
1112            "expected same type of move for high parts");
1113 
1114     if (is64) {
1115       return z_ld_st_helper(cbuf, "STG ", STG_ZOPC, src_lo, dst_offset, print, os);
1116     }
1117 
1118     // else
1119     mnemo = dst12 ? "ST  " : "STY ";
1120     opc = dst12 ? ST_ZOPC : STY_ZOPC;
1121 
1122     return z_ld_st_helper(cbuf, mnemo, opc, src_lo, dst_offset, print, os);
1123   }
1124 
1125   // Check for integer load
1126   // Always load cOops zero-extended. That doesn't hurt int loads.
1127   if (dst_lo_rc == rc_int && src_lo_rc == rc_stack) {
1128 
1129     assert(!is64 || (dst_hi_rc==rc_int && src_hi_rc==rc_stack),
1130            "expected same type of move for high parts");
1131 
1132     mnemo = is64 ? "LG  " : "LLGF";
1133     opc = is64 ? LG_ZOPC : LLGF_ZOPC;
1134 
1135     return z_ld_st_helper(cbuf, mnemo, opc, dst_lo, src_offset, print, os);
1136   }
1137 
1138   // Check for float reg-reg copy.
1139   if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1140     if (cbuf) {
1141       MacroAssembler _masm(cbuf);
1142       FloatRegister Rsrc = as_FloatRegister(Matcher::_regEncode[src_lo]);
1143       FloatRegister Rdst = as_FloatRegister(Matcher::_regEncode[dst_lo]);
1144       __ z_ldr(Rdst, Rsrc);
1145       return 2;
1146     }
1147 #if !defined(PRODUCT)
1148     // else
1149     if (print) {
1150       os->print("LDR      %s,%s\t # MachCopy spill code", Matcher::regName[dst_lo], Matcher::regName[src_lo]);
1151     }
1152 #endif
1153     return 2;
1154   }
1155 
1156   // Check for float store.
1157   if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1158     assert(!is64 || (src_hi_rc==rc_float && dst_hi_rc==rc_stack),
1159            "expected same type of move for high parts");
1160 
1161     if (is64) {
1162       mnemo = dst12 ? "STD  " : "STDY ";
1163       opc = dst12 ? STD_ZOPC : STDY_ZOPC;
1164       return z_ld_st_helper(cbuf, mnemo, opc, src_lo, dst_offset, print, os);
1165     }
1166     // else
1167 
1168     mnemo = dst12 ? "STE  " : "STEY ";
1169     opc = dst12 ? STE_ZOPC : STEY_ZOPC;
1170     return z_ld_st_helper(cbuf, mnemo, opc, src_lo, dst_offset, print, os);
1171   }
1172 
1173   // Check for float load.
1174   if (dst_lo_rc == rc_float && src_lo_rc == rc_stack) {
1175     assert(!is64 || (dst_hi_rc==rc_float && src_hi_rc==rc_stack),
1176            "expected same type of move for high parts");
1177 
1178     if (is64) {
1179       mnemo = src12 ? "LD   " : "LDY  ";
1180       opc = src12 ? LD_ZOPC : LDY_ZOPC;
1181       return z_ld_st_helper(cbuf, mnemo, opc, dst_lo, src_offset, print, os);
1182     }
1183     // else
1184 
1185     mnemo = src12 ? "LE   " : "LEY  ";
1186     opc = src12 ? LE_ZOPC : LEY_ZOPC;
1187     return z_ld_st_helper(cbuf, mnemo, opc, dst_lo, src_offset, print, os);
1188   }
1189 
1190   // --------------------------------------------------------------------
1191   // Check for hi bits still needing moving. Only happens for misaligned
1192   // arguments to native calls.
1193   if (src_hi == dst_hi) {
1194     return 0;               // Self copy, no move.
1195   }
1196 
1197   assert(is64 && dst_hi_rc != rc_bad, "src_hi & dst_hi cannot be Bad");
1198   Unimplemented();  // Unsafe, do not remove!
1199 
1200   return 0; // never reached, but make the compiler shut up!
1201 }
1202 
1203 #if !defined(PRODUCT)
1204 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *os) const {
1205   if (ra_ && ra_->node_regs_max_index() > 0) {
1206     implementation(NULL, ra_, false, os);
1207   } else {
1208     if (req() == 2 && in(1)) {
1209       os->print("N%d = N%d\n", _idx, in(1)->_idx);
1210     } else {
1211       const char *c = "(";
1212       os->print("N%d = ", _idx);
1213       for (uint i = 1; i < req(); ++i) {
1214         os->print("%sN%d", c, in(i)->_idx);
1215         c = ", ";
1216       }
1217       os->print(")");
1218     }
1219   }
1220 }
1221 #endif
1222 
1223 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1224   implementation(&cbuf, ra_, false, NULL);
1225 }
1226 
1227 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1228   return implementation(NULL, ra_, true, NULL);
1229 }
1230 
1231 //=============================================================================
1232 
1233 #if !defined(PRODUCT)
1234 void MachNopNode::format(PhaseRegAlloc *, outputStream *os) const {
1235   os->print("NOP     # pad for alignment (%d nops, %d bytes)", _count, _count*MacroAssembler::nop_size());
1236 }
1237 #endif
1238 
1239 void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc * ra_) const {
1240   MacroAssembler _masm(&cbuf);
1241 
1242   int rem_space = 0;
1243   if (!(ra_->C->in_scratch_emit_size())) {
1244     rem_space = cbuf.insts()->remaining();
1245     if (rem_space <= _count*2 + 8) {
1246       tty->print("NopNode: _count = %3.3d, remaining space before = %d", _count, rem_space);
1247     }
1248   }
1249 
1250   for (int i = 0; i < _count; i++) {
1251     __ z_nop();
1252   }
1253 
1254   if (!(ra_->C->in_scratch_emit_size())) {
1255     if (rem_space <= _count*2 + 8) {
1256       int rem_space2 = cbuf.insts()->remaining();
1257       tty->print_cr(", after = %d", rem_space2);
1258     }
1259   }
1260 }
1261 
1262 uint MachNopNode::size(PhaseRegAlloc *ra_) const {
1263    return 2 * _count;
1264 }
1265 
1266 #if !defined(PRODUCT)
1267 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *os) const {
1268   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1269   if (ra_ && ra_->node_regs_max_index() > 0) {
1270     int reg = ra_->get_reg_first(this);
1271     os->print("ADDHI  %s, SP, %d\t//box node", Matcher::regName[reg], offset);
1272   } else {
1273     os->print("ADDHI  N%d = SP + %d\t// box node", _idx, offset);
1274   }
1275 }
1276 #endif
1277 
1278 // Take care of the size function, if you make changes here!
1279 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1280   MacroAssembler _masm(&cbuf);
1281 
1282   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1283   int reg = ra_->get_encode(this);
1284   __ z_lay(as_Register(reg), offset, Z_SP);
1285 }
1286 
1287 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
1288   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_)
1289   return 6;
1290 }
1291 
1292  %} // end source section
1293 
1294 //----------SOURCE BLOCK-------------------------------------------------------
1295 // This is a block of C++ code which provides values, functions, and
1296 // definitions necessary in the rest of the architecture description
1297 
1298 source_hpp %{
1299 
1300 // Header information of the source block.
1301 // Method declarations/definitions which are used outside
1302 // the ad-scope can conveniently be defined here.
1303 //
1304 // To keep related declarations/definitions/uses close together,
1305 // we switch between source %{ }% and source_hpp %{ }% freely as needed.
1306 
1307 //--------------------------------------------------------------
1308 // Used for optimization in Compile::Shorten_branches
1309 //--------------------------------------------------------------
1310 
1311 class CallStubImpl {
1312  public:
1313 
1314   // call trampolines
1315   // Size of call trampoline stub. For add'l comments, see size_java_to_interp().
1316   static uint size_call_trampoline() {
1317     return 0; // no call trampolines on this platform
1318   }
1319 
1320   // call trampolines
1321   // Number of relocations needed by a call trampoline stub.
1322   static uint reloc_call_trampoline() {
1323     return 0; // No call trampolines on this platform.
1324   }
1325 };
1326 
1327 %} // end source_hpp section
1328 
1329 source %{
1330 
1331 #if !defined(PRODUCT)
1332 void MachUEPNode::format(PhaseRegAlloc *ra_, outputStream *os) const {
1333   os->print_cr("---- MachUEPNode ----");
1334   os->print_cr("\tTA");
1335   os->print_cr("\tload_const Z_R1, SharedRuntime::get_ic_miss_stub()");
1336   os->print_cr("\tBR(Z_R1)");
1337   os->print_cr("\tTA  # pad with illtraps");
1338   os->print_cr("\t...");
1339   os->print_cr("\tTA");
1340   os->print_cr("\tLTGR    Z_R2, Z_R2");
1341   os->print_cr("\tBRU     ic_miss");
1342 }
1343 #endif
1344 
1345 void MachUEPNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1346   MacroAssembler _masm(&cbuf);
1347   const int ic_miss_offset = 2;
1348 
1349   // Inline_cache contains a klass.
1350   Register ic_klass = as_Register(Matcher::inline_cache_reg_encode());
1351   // ARG1 is the receiver oop.
1352   Register R2_receiver = Z_ARG1;
1353   int      klass_offset = oopDesc::klass_offset_in_bytes();
1354   AddressLiteral icmiss(SharedRuntime::get_ic_miss_stub());
1355   Register R1_ic_miss_stub_addr = Z_R1_scratch;
1356 
1357   // Null check of receiver.
1358   // This is the null check of the receiver that actually should be
1359   // done in the caller. It's here because in case of implicit null
1360   // checks we get it for free.
1361   assert(!MacroAssembler::needs_explicit_null_check(oopDesc::klass_offset_in_bytes()),
1362          "second word in oop should not require explicit null check.");
1363   if (!ImplicitNullChecks) {
1364     Label valid;
1365     if (VM_Version::has_CompareBranch()) {
1366       __ z_cgij(R2_receiver, 0, Assembler::bcondNotEqual, valid);
1367     } else {
1368       __ z_ltgr(R2_receiver, R2_receiver);
1369       __ z_bre(valid);
1370     }
1371     // The ic_miss_stub will handle the null pointer exception.
1372     __ load_const_optimized(R1_ic_miss_stub_addr, icmiss);
1373     __ z_br(R1_ic_miss_stub_addr);
1374     __ bind(valid);
1375   }
1376 
1377   // Check whether this method is the proper implementation for the class of
1378   // the receiver (ic miss check).
1379   {
1380     Label valid;
1381     // Compare cached class against klass from receiver.
1382     // This also does an implicit null check!
1383     __ compare_klass_ptr(ic_klass, klass_offset, R2_receiver, false);
1384     __ z_bre(valid);
1385     // The inline cache points to the wrong method. Call the
1386     // ic_miss_stub to find the proper method.
1387     __ load_const_optimized(R1_ic_miss_stub_addr, icmiss);
1388     __ z_br(R1_ic_miss_stub_addr);
1389     __ bind(valid);
1390   }
1391 
1392 }
1393 
1394 uint MachUEPNode::size(PhaseRegAlloc *ra_) const {
1395   // Determine size dynamically.
1396   return MachNode::size(ra_);
1397 }
1398 
1399 //=============================================================================
1400 
1401 %} // interrupt source section
1402 
1403 source_hpp %{ // Header information of the source block.
1404 
1405 class HandlerImpl {
1406  public:
1407 
1408   static int emit_exception_handler(CodeBuffer &cbuf);
1409   static int emit_deopt_handler(CodeBuffer& cbuf);
1410 
1411   static uint size_exception_handler() {
1412     return NativeJump::max_instruction_size();
1413   }
1414 
1415   static uint size_deopt_handler() {
1416     return NativeCall::max_instruction_size();
1417   }
1418 };
1419 
1420 %} // end source_hpp section
1421 
1422 source %{
1423 
1424 // This exception handler code snippet is placed after the method's
1425 // code. It is the return point if an exception occurred. it jumps to
1426 // the exception blob.
1427 //
1428 // If the method gets deoptimized, the method and this code snippet
1429 // get patched.
1430 //
1431 // 1) Trampoline code gets patched into the end of this exception
1432 //   handler. the trampoline code jumps to the deoptimization blob.
1433 //
1434 // 2) The return address in the method's code will get patched such
1435 //   that it jumps to the trampoline.
1436 //
1437 // 3) The handler will get patched such that it does not jump to the
1438 //   exception blob, but to an entry in the deoptimization blob being
1439 //   aware of the exception.
1440 int HandlerImpl::emit_exception_handler(CodeBuffer &cbuf) {
1441   Register temp_reg = Z_R1;
1442   MacroAssembler _masm(&cbuf);
1443 
1444   address base = __ start_a_stub(size_exception_handler());
1445   if (base == NULL) {
1446     return 0;          // CodeBuffer::expand failed
1447   }
1448 
1449   int offset = __ offset();
1450   // Use unconditional pc-relative jump with 32-bit range here.
1451   __ load_const_optimized(temp_reg, (address)OptoRuntime::exception_blob()->content_begin());
1452   __ z_br(temp_reg);
1453 
1454   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
1455 
1456   __ end_a_stub();
1457 
1458   return offset;
1459 }
1460 
1461 // Emit deopt handler code.
1462 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf) {
1463   MacroAssembler _masm(&cbuf);
1464   address        base = __ start_a_stub(size_deopt_handler());
1465 
1466   if (base == NULL) {
1467     return 0;  // CodeBuffer::expand failed
1468   }
1469 
1470   int offset = __ offset();
1471 
1472   // Size_deopt_handler() must be exact on zarch, so for simplicity
1473   // we do not use load_const_opt here.
1474   __ load_const(Z_R1, SharedRuntime::deopt_blob()->unpack());
1475   __ call(Z_R1);
1476   assert(__ offset() - offset == (int) size_deopt_handler(), "must be fixed size");
1477 
1478   __ end_a_stub();
1479   return offset;
1480 }
1481 
1482 //=============================================================================
1483 
1484 
1485 // Given a register encoding, produce an Integer Register object.
1486 static Register reg_to_register_object(int register_encoding) {
1487   assert(Z_R12->encoding() == Z_R12_enc, "wrong coding");
1488   return as_Register(register_encoding);
1489 }
1490 
1491 const bool Matcher::match_rule_supported(int opcode) {
1492   if (!has_match_rule(opcode)) return false;
1493 
1494   switch (opcode) {
1495     case Op_CountLeadingZerosI:
1496     case Op_CountLeadingZerosL:
1497     case Op_CountTrailingZerosI:
1498     case Op_CountTrailingZerosL:
1499       // Implementation requires FLOGR instruction, which is available since z9.
1500       return true;
1501 
1502     case Op_ReverseBytesI:
1503     case Op_ReverseBytesL:
1504       return UseByteReverseInstruction;
1505 
1506     // PopCount supported by H/W from z/Architecture G5 (z196) on.
1507     case Op_PopCountI:
1508     case Op_PopCountL:
1509       return UsePopCountInstruction && VM_Version::has_PopCount();
1510 
1511     case Op_StrComp:
1512       return SpecialStringCompareTo;
1513     case Op_StrEquals:
1514       return SpecialStringEquals;
1515     case Op_StrIndexOf:
1516     case Op_StrIndexOfChar:
1517       return SpecialStringIndexOf;
1518 
1519     case Op_GetAndAddI:
1520     case Op_GetAndAddL:
1521       return true;
1522       // return VM_Version::has_AtomicMemWithImmALUOps();
1523     case Op_GetAndSetI:
1524     case Op_GetAndSetL:
1525     case Op_GetAndSetP:
1526     case Op_GetAndSetN:
1527       return true;  // General CAS implementation, always available.
1528 
1529     default:
1530       return true;  // Per default match rules are supported.
1531                     // BUT: make sure match rule is not disabled by a false predicate!
1532   }
1533 
1534   return true;  // Per default match rules are supported.
1535                 // BUT: make sure match rule is not disabled by a false predicate!
1536 }
1537 
1538 const bool Matcher::match_rule_supported_vector(int opcode, int vlen) {
1539   // TODO
1540   // Identify extra cases that we might want to provide match rules for
1541   // e.g. Op_ vector nodes and other intrinsics while guarding with vlen.
1542   bool ret_value = match_rule_supported(opcode);
1543   // Add rules here.
1544 
1545   return ret_value;  // Per default match rules are supported.
1546 }
1547 
1548 int Matcher::regnum_to_fpu_offset(int regnum) {
1549   ShouldNotReachHere();
1550   return regnum - 32; // The FP registers are in the second chunk.
1551 }
1552 
1553 const bool Matcher::has_predicated_vectors(void) {
1554   return false;
1555 }
1556 
1557 const int Matcher::float_pressure(int default_pressure_threshold) {
1558   return default_pressure_threshold;
1559 }
1560 
1561 const bool Matcher::convL2FSupported(void) {
1562   return true; // False means that conversion is done by runtime call.
1563 }
1564 
1565 //----------SUPERWORD HELPERS----------------------------------------
1566 
1567 // Vector width in bytes.
1568 const int Matcher::vector_width_in_bytes(BasicType bt) {
1569   assert(MaxVectorSize == 8, "");
1570   return 8;
1571 }
1572 
1573 // Vector ideal reg.
1574 const uint Matcher::vector_ideal_reg(int size) {
1575   assert(MaxVectorSize == 8 && size == 8, "");
1576   return Op_RegL;
1577 }
1578 
1579 // Limits on vector size (number of elements) loaded into vector.
1580 const int Matcher::max_vector_size(const BasicType bt) {
1581   assert(is_java_primitive(bt), "only primitive type vectors");
1582   return vector_width_in_bytes(bt)/type2aelembytes(bt);
1583 }
1584 
1585 const int Matcher::min_vector_size(const BasicType bt) {
1586   return max_vector_size(bt); // Same as max.
1587 }
1588 
1589 const uint Matcher::vector_shift_count_ideal_reg(int size) {
1590   fatal("vector shift is not supported");
1591   return Node::NotAMachineReg;
1592 }
1593 
1594 // z/Architecture does support misaligned store/load at minimal extra cost.
1595 const bool Matcher::misaligned_vectors_ok() {
1596   return true;
1597 }
1598 
1599 // Not yet ported to z/Architecture.
1600 const bool Matcher::pass_original_key_for_aes() {
1601   return false;
1602 }
1603 
1604 // RETURNS: whether this branch offset is short enough that a short
1605 // branch can be used.
1606 //
1607 // If the platform does not provide any short branch variants, then
1608 // this method should return `false' for offset 0.
1609 //
1610 // `Compile::Fill_buffer' will decide on basis of this information
1611 // whether to do the pass `Compile::Shorten_branches' at all.
1612 //
1613 // And `Compile::Shorten_branches' will decide on basis of this
1614 // information whether to replace particular branch sites by short
1615 // ones.
1616 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1617   // On zarch short branches use a 16 bit signed immediate that
1618   // is the pc-relative offset in halfword (= 2 bytes) units.
1619   return Assembler::is_within_range_of_RelAddr16((address)((long)offset), (address)0);
1620 }
1621 
1622 const bool Matcher::isSimpleConstant64(jlong value) {
1623   // Probably always true, even if a temp register is required.
1624   return true;
1625 }
1626 
1627 // Should correspond to setting above
1628 const bool Matcher::init_array_count_is_in_bytes = false;
1629 
1630 // Suppress CMOVL. Conditional move available on z/Architecture only from z196 onwards. Not exploited yet.
1631 const int Matcher::long_cmove_cost() { return ConditionalMoveLimit; }
1632 
1633 // Suppress CMOVF. Conditional move available on z/Architecture only from z196 onwards. Not exploited yet.
1634 const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
1635 
1636 // Does the CPU require postalloc expand (see block.cpp for description of postalloc expand)?
1637 const bool Matcher::require_postalloc_expand = false;
1638 
1639 // Do we need to mask the count passed to shift instructions or does
1640 // the cpu only look at the lower 5/6 bits anyway?
1641 // 32bit shifts mask in emitter, 64bit shifts need no mask.
1642 // Constant shift counts are handled in Ideal phase.
1643 const bool Matcher::need_masked_shift_count = false;
1644 
1645 // Set this as clone_shift_expressions.
1646 bool Matcher::narrow_oop_use_complex_address() {
1647   if (Universe::narrow_oop_base() == NULL && Universe::narrow_oop_shift() == 0) return true;
1648   return false;
1649 }
1650 
1651 bool Matcher::narrow_klass_use_complex_address() {
1652   NOT_LP64(ShouldNotCallThis());
1653   assert(UseCompressedClassPointers, "only for compressed klass code");
1654   // TODO HS25: z port if (MatchDecodeNodes) return true;
1655   return false;
1656 }
1657 
1658 bool Matcher::const_oop_prefer_decode() {
1659   // Prefer ConN+DecodeN over ConP in simple compressed oops mode.
1660   return Universe::narrow_oop_base() == NULL;
1661 }
1662 
1663 bool Matcher::const_klass_prefer_decode() {
1664   // Prefer ConNKlass+DecodeNKlass over ConP in simple compressed klass mode.
1665   return Universe::narrow_klass_base() == NULL;
1666 }
1667 
1668 // Is it better to copy float constants, or load them directly from memory?
1669 // Most RISCs will have to materialize an address into a
1670 // register first, so they would do better to copy the constant from stack.
1671 const bool Matcher::rematerialize_float_constants = false;
1672 
1673 // If CPU can load and store mis-aligned doubles directly then no fixup is
1674 // needed. Else we split the double into 2 integer pieces and move it
1675 // piece-by-piece. Only happens when passing doubles into C code as the
1676 // Java calling convention forces doubles to be aligned.
1677 const bool Matcher::misaligned_doubles_ok = true;
1678 
1679 // Advertise here if the CPU requires explicit rounding operations
1680 // to implement the UseStrictFP mode.
1681 const bool Matcher::strict_fp_requires_explicit_rounding = false;
1682 
1683 // Do floats take an entire double register or just half?
1684 //
1685 // A float in resides in a zarch double register. When storing it by
1686 // z_std, it cannot be restored in C-code by reloading it as a double
1687 // and casting it into a float afterwards.
1688 bool Matcher::float_in_double() { return false; }
1689 
1690 // Do ints take an entire long register or just half?
1691 // The relevant question is how the int is callee-saved:
1692 // the whole long is written but de-opt'ing will have to extract
1693 // the relevant 32 bits.
1694 const bool Matcher::int_in_long = true;
1695 
1696 // Constants for c2c and c calling conventions.
1697 
1698 const MachRegisterNumbers z_iarg_reg[5] = {
1699   Z_R2_num, Z_R3_num, Z_R4_num, Z_R5_num, Z_R6_num
1700 };
1701 
1702 const MachRegisterNumbers z_farg_reg[4] = {
1703   Z_F0_num, Z_F2_num, Z_F4_num, Z_F6_num
1704 };
1705 
1706 const int z_num_iarg_registers = sizeof(z_iarg_reg) / sizeof(z_iarg_reg[0]);
1707 
1708 const int z_num_farg_registers = sizeof(z_farg_reg) / sizeof(z_farg_reg[0]);
1709 
1710 // Return whether or not this register is ever used as an argument. This
1711 // function is used on startup to build the trampoline stubs in generateOptoStub.
1712 // Registers not mentioned will be killed by the VM call in the trampoline, and
1713 // arguments in those registers not be available to the callee.
1714 bool Matcher::can_be_java_arg(int reg) {
1715   // We return true for all registers contained in z_iarg_reg[] and
1716   // z_farg_reg[] and their virtual halves.
1717   // We must include the virtual halves in order to get STDs and LDs
1718   // instead of STWs and LWs in the trampoline stubs.
1719 
1720   if (reg == Z_R2_num || reg == Z_R2_H_num ||
1721       reg == Z_R3_num || reg == Z_R3_H_num ||
1722       reg == Z_R4_num || reg == Z_R4_H_num ||
1723       reg == Z_R5_num || reg == Z_R5_H_num ||
1724       reg == Z_R6_num || reg == Z_R6_H_num) {
1725     return true;
1726   }
1727 
1728   if (reg == Z_F0_num || reg == Z_F0_H_num ||
1729       reg == Z_F2_num || reg == Z_F2_H_num ||
1730       reg == Z_F4_num || reg == Z_F4_H_num ||
1731       reg == Z_F6_num || reg == Z_F6_H_num) {
1732     return true;
1733   }
1734 
1735   return false;
1736 }
1737 
1738 bool Matcher::is_spillable_arg(int reg) {
1739   return can_be_java_arg(reg);
1740 }
1741 
1742 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
1743   return false;
1744 }
1745 
1746 // Register for DIVI projection of divmodI
1747 RegMask Matcher::divI_proj_mask() {
1748   return _Z_RARG4_INT_REG_mask;
1749 }
1750 
1751 // Register for MODI projection of divmodI
1752 RegMask Matcher::modI_proj_mask() {
1753   return _Z_RARG3_INT_REG_mask;
1754 }
1755 
1756 // Register for DIVL projection of divmodL
1757 RegMask Matcher::divL_proj_mask() {
1758   return _Z_RARG4_LONG_REG_mask;
1759 }
1760 
1761 // Register for MODL projection of divmodL
1762 RegMask Matcher::modL_proj_mask() {
1763   return _Z_RARG3_LONG_REG_mask;
1764 }
1765 
1766 // Copied from sparc.
1767 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1768   return RegMask();
1769 }
1770 
1771 const bool Matcher::convi2l_type_required = true;
1772 
1773 // Should the Matcher clone shifts on addressing modes, expecting them
1774 // to be subsumed into complex addressing expressions or compute them
1775 // into registers?
1776 bool Matcher::clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
1777   return clone_base_plus_offset_address(m, mstack, address_visited);
1778 }
1779 
1780 void Compile::reshape_address(AddPNode* addp) {
1781 }
1782 
1783 %} // source
1784 
1785 //----------ENCODING BLOCK-----------------------------------------------------
1786 // This block specifies the encoding classes used by the compiler to output
1787 // byte streams. Encoding classes are parameterized macros used by
1788 // Machine Instruction Nodes in order to generate the bit encoding of the
1789 // instruction. Operands specify their base encoding interface with the
1790 // interface keyword. There are currently supported four interfaces,
1791 // REG_INTER, CONST_INTER, MEMORY_INTER, & COND_INTER. REG_INTER causes an
1792 // operand to generate a function which returns its register number when
1793 // queried. CONST_INTER causes an operand to generate a function which
1794 // returns the value of the constant when queried. MEMORY_INTER causes an
1795 // operand to generate four functions which return the Base Register, the
1796 // Index Register, the Scale Value, and the Offset Value of the operand when
1797 // queried. COND_INTER causes an operand to generate six functions which
1798 // return the encoding code (ie - encoding bits for the instruction)
1799 // associated with each basic boolean condition for a conditional instruction.
1800 //
1801 // Instructions specify two basic values for encoding. Again, a function
1802 // is available to check if the constant displacement is an oop. They use the
1803 // ins_encode keyword to specify their encoding classes (which must be
1804 // a sequence of enc_class names, and their parameters, specified in
1805 // the encoding block), and they use the
1806 // opcode keyword to specify, in order, their primary, secondary, and
1807 // tertiary opcode. Only the opcode sections which a particular instruction
1808 // needs for encoding need to be specified.
1809 encode %{
1810   enc_class enc_unimplemented %{
1811     MacroAssembler _masm(&cbuf);
1812     __ unimplemented("Unimplemented mach node encoding in AD file.", 13);
1813   %}
1814 
1815   enc_class enc_untested %{
1816 #ifdef ASSERT
1817     MacroAssembler _masm(&cbuf);
1818     __ untested("Untested mach node encoding in AD file.");
1819 #endif
1820   %}
1821 
1822   enc_class z_rrform(iRegI dst, iRegI src) %{
1823     assert((($primary >> 14) & 0x03) == 0, "Instruction format error");
1824     assert( ($primary >> 16)         == 0, "Instruction format error");
1825     z_emit16(cbuf, $primary |
1826              Assembler::reg($dst$$reg,8,16) |
1827              Assembler::reg($src$$reg,12,16));
1828   %}
1829 
1830   enc_class z_rreform(iRegI dst1, iRegI src2) %{
1831     assert((($primary >> 30) & 0x03) == 2, "Instruction format error");
1832     z_emit32(cbuf, $primary |
1833              Assembler::reg($dst1$$reg,24,32) |
1834              Assembler::reg($src2$$reg,28,32));
1835   %}
1836 
1837   enc_class z_rrfform(iRegI dst1, iRegI src2, iRegI src3) %{
1838     assert((($primary >> 30) & 0x03) == 2, "Instruction format error");
1839     z_emit32(cbuf, $primary |
1840              Assembler::reg($dst1$$reg,24,32) |
1841              Assembler::reg($src2$$reg,28,32) |
1842              Assembler::reg($src3$$reg,16,32));
1843   %}
1844 
1845   enc_class z_riform_signed(iRegI dst, immI16 src) %{
1846     assert((($primary>>30) & 0x03) == 2, "Instruction format error");
1847     z_emit32(cbuf, $primary |
1848              Assembler::reg($dst$$reg,8,32) |
1849              Assembler::simm16($src$$constant,16,32));
1850   %}
1851 
1852   enc_class z_riform_unsigned(iRegI dst, uimmI16 src) %{
1853     assert((($primary>>30) & 0x03) == 2, "Instruction format error");
1854     z_emit32(cbuf, $primary |
1855              Assembler::reg($dst$$reg,8,32) |
1856              Assembler::uimm16($src$$constant,16,32));
1857   %}
1858 
1859   enc_class z_rieform_d(iRegI dst1, iRegI src3, immI src2) %{
1860     assert((($primary>>46) & 0x03) == 3, "Instruction format error");
1861     z_emit48(cbuf, $primary |
1862              Assembler::reg($dst1$$reg,8,48) |
1863              Assembler::reg($src3$$reg,12,48) |
1864              Assembler::simm16($src2$$constant,16,48));
1865   %}
1866 
1867   enc_class z_rilform_signed(iRegI dst, immL32 src) %{
1868     assert((($primary>>46) & 0x03) == 3, "Instruction format error");
1869     z_emit48(cbuf, $primary |
1870              Assembler::reg($dst$$reg,8,48) |
1871              Assembler::simm32($src$$constant,16,48));
1872   %}
1873 
1874   enc_class z_rilform_unsigned(iRegI dst, uimmL32 src) %{
1875     assert((($primary>>46) & 0x03) == 3, "Instruction format error");
1876     z_emit48(cbuf, $primary |
1877              Assembler::reg($dst$$reg,8,48) |
1878              Assembler::uimm32($src$$constant,16,48));
1879   %}
1880 
1881   enc_class z_rsyform_const(iRegI dst, iRegI src1, immI src2) %{
1882     z_emit48(cbuf, $primary |
1883              Assembler::reg($dst$$reg,8,48) |
1884              Assembler::reg($src1$$reg,12,48) |
1885              Assembler::simm20($src2$$constant));
1886   %}
1887 
1888   enc_class z_rsyform_reg_reg(iRegI dst, iRegI src, iRegI shft) %{
1889     z_emit48(cbuf, $primary |
1890              Assembler::reg($dst$$reg,8,48) |
1891              Assembler::reg($src$$reg,12,48) |
1892              Assembler::reg($shft$$reg,16,48) |
1893              Assembler::simm20(0));
1894   %}
1895 
1896   enc_class z_rxform_imm_reg_reg(iRegL dst, immL con, iRegL src1, iRegL src2) %{
1897     assert((($primary>>30) & 0x03) == 1, "Instruction format error");
1898     z_emit32(cbuf, $primary |
1899              Assembler::reg($dst$$reg,8,32) |
1900              Assembler::reg($src1$$reg,12,32) |
1901              Assembler::reg($src2$$reg,16,32) |
1902              Assembler::uimm12($con$$constant,20,32));
1903   %}
1904 
1905   enc_class z_rxform_imm_reg(iRegL dst, immL con, iRegL src) %{
1906     assert((($primary>>30) & 0x03) == 1, "Instruction format error");
1907     z_emit32(cbuf, $primary |
1908              Assembler::reg($dst$$reg,8,32) |
1909              Assembler::reg($src$$reg,16,32) |
1910              Assembler::uimm12($con$$constant,20,32));
1911   %}
1912 
1913   enc_class z_rxyform_imm_reg_reg(iRegL dst, immL con, iRegL src1, iRegL src2) %{
1914     z_emit48(cbuf, $primary |
1915              Assembler::reg($dst$$reg,8,48) |
1916              Assembler::reg($src1$$reg,12,48) |
1917              Assembler::reg($src2$$reg,16,48) |
1918              Assembler::simm20($con$$constant));
1919   %}
1920 
1921   enc_class z_rxyform_imm_reg(iRegL dst, immL con, iRegL src) %{
1922     z_emit48(cbuf, $primary |
1923              Assembler::reg($dst$$reg,8,48) |
1924              Assembler::reg($src$$reg,16,48) |
1925              Assembler::simm20($con$$constant));
1926   %}
1927 
1928   // Direct memory arithmetic.
1929   enc_class z_siyform(memoryRSY mem, immI8 src) %{
1930     int      disp = $mem$$disp;
1931     Register base = reg_to_register_object($mem$$base);
1932     int      con  = $src$$constant;
1933 
1934     assert(VM_Version::has_MemWithImmALUOps(), "unsupported CPU");
1935     z_emit_inst(cbuf, $primary |
1936                 Assembler::regz(base,16,48) |
1937                 Assembler::simm20(disp) |
1938                 Assembler::simm8(con,8,48));
1939   %}
1940 
1941   enc_class z_silform(memoryRS mem, immI16 src) %{
1942     z_emit_inst(cbuf, $primary |
1943                 Assembler::regz(reg_to_register_object($mem$$base),16,48) |
1944                 Assembler::uimm12($mem$$disp,20,48) |
1945                 Assembler::simm16($src$$constant,32,48));
1946   %}
1947 
1948   // Encoder for FP ALU reg/mem instructions (support only short displacements).
1949   enc_class z_form_rt_memFP(RegF dst, memoryRX mem) %{
1950     Register Ridx = $mem$$index$$Register;
1951     if (Ridx == noreg) { Ridx = Z_R0; } // Index is 0.
1952     if ($primary > (1L << 32)) {
1953       z_emit_inst(cbuf, $primary |
1954                   Assembler::reg($dst$$reg, 8, 48) |
1955                   Assembler::uimm12($mem$$disp, 20, 48) |
1956                   Assembler::reg(Ridx, 12, 48) |
1957                   Assembler::regz(reg_to_register_object($mem$$base), 16, 48));
1958     } else {
1959       z_emit_inst(cbuf, $primary |
1960                   Assembler::reg($dst$$reg, 8, 32) |
1961                   Assembler::uimm12($mem$$disp, 20, 32) |
1962                   Assembler::reg(Ridx, 12, 32) |
1963                   Assembler::regz(reg_to_register_object($mem$$base), 16, 32));
1964     }
1965   %}
1966 
1967   enc_class z_form_rt_mem(iRegI dst, memory mem) %{
1968     Register Ridx = $mem$$index$$Register;
1969     if (Ridx == noreg) { Ridx = Z_R0; } // Index is 0.
1970     if ($primary > (1L<<32)) {
1971       z_emit_inst(cbuf, $primary |
1972                   Assembler::reg($dst$$reg, 8, 48) |
1973                   Assembler::simm20($mem$$disp) |
1974                   Assembler::reg(Ridx, 12, 48) |
1975                   Assembler::regz(reg_to_register_object($mem$$base), 16, 48));
1976     } else {
1977       z_emit_inst(cbuf, $primary |
1978                   Assembler::reg($dst$$reg, 8, 32) |
1979                   Assembler::uimm12($mem$$disp, 20, 32) |
1980                   Assembler::reg(Ridx, 12, 32) |
1981                   Assembler::regz(reg_to_register_object($mem$$base), 16, 32));
1982     }
1983   %}
1984 
1985   enc_class z_form_rt_mem_opt(iRegI dst, memory mem) %{
1986     int isize = $secondary > 1L << 32 ? 48 : 32;
1987     Register Ridx = $mem$$index$$Register;
1988     if (Ridx == noreg) { Ridx = Z_R0; } // Index is 0.
1989 
1990     if (Displacement::is_shortDisp((long)$mem$$disp)) {
1991       z_emit_inst(cbuf, $secondary |
1992                   Assembler::reg($dst$$reg, 8, isize) |
1993                   Assembler::uimm12($mem$$disp, 20, isize) |
1994                   Assembler::reg(Ridx, 12, isize) |
1995                   Assembler::regz(reg_to_register_object($mem$$base), 16, isize));
1996     } else if (Displacement::is_validDisp((long)$mem$$disp)) {
1997       z_emit_inst(cbuf, $primary |
1998                   Assembler::reg($dst$$reg, 8, 48) |
1999                   Assembler::simm20($mem$$disp) |
2000                   Assembler::reg(Ridx, 12, 48) |
2001                   Assembler::regz(reg_to_register_object($mem$$base), 16, 48));
2002     } else {
2003         MacroAssembler _masm(&cbuf);
2004         __ load_const_optimized(Z_R1_scratch, $mem$$disp);
2005         if (Ridx != Z_R0) { __ z_agr(Z_R1_scratch, Ridx); }
2006         z_emit_inst(cbuf, $secondary |
2007                     Assembler::reg($dst$$reg, 8, isize) |
2008                     Assembler::uimm12(0, 20, isize) |
2009                     Assembler::reg(Z_R1_scratch, 12, isize) |
2010                     Assembler::regz(reg_to_register_object($mem$$base), 16, isize));
2011     }
2012   %}
2013 
2014   enc_class z_enc_brul(Label lbl) %{
2015     MacroAssembler _masm(&cbuf);
2016     Label* p = $lbl$$label;
2017 
2018     // 'p' is `NULL' when this encoding class is used only to
2019     // determine the size of the encoded instruction.
2020     // Use a bound dummy label in that case.
2021     Label d;
2022     __ bind(d);
2023     Label& l = (NULL == p) ? d : *(p);
2024     __ z_brul(l);
2025   %}
2026 
2027   enc_class z_enc_bru(Label lbl) %{
2028     MacroAssembler _masm(&cbuf);
2029     Label* p = $lbl$$label;
2030 
2031     // 'p' is `NULL' when this encoding class is used only to
2032     // determine the size of the encoded instruction.
2033     // Use a bound dummy label in that case.
2034     Label d;
2035     __ bind(d);
2036     Label& l = (NULL == p) ? d : *(p);
2037     __ z_bru(l);
2038   %}
2039 
2040   enc_class z_enc_branch_con_far(cmpOp cmp, Label lbl) %{
2041     MacroAssembler _masm(&cbuf);
2042     Label* p = $lbl$$label;
2043 
2044     // 'p' is `NULL' when this encoding class is used only to
2045     // determine the size of the encoded instruction.
2046     // Use a bound dummy label in that case.
2047     Label d;
2048     __ bind(d);
2049     Label& l = (NULL == p) ? d : *(p);
2050     __ z_brcl((Assembler::branch_condition)$cmp$$cmpcode, l);
2051   %}
2052 
2053   enc_class z_enc_branch_con_short(cmpOp cmp, Label lbl) %{
2054     MacroAssembler _masm(&cbuf);
2055     Label* p = $lbl$$label;
2056 
2057     // 'p' is `NULL' when this encoding class is used only to
2058     // determine the size of the encoded instruction.
2059     // Use a bound dummy label in that case.
2060     Label d;
2061     __ bind(d);
2062     Label& l = (NULL == p) ? d : *(p);
2063     __ z_brc((Assembler::branch_condition)$cmp$$cmpcode, l);
2064   %}
2065 
2066   enc_class z_enc_cmpb_regreg(iRegI src1, iRegI src2, Label lbl, cmpOpT cmp) %{
2067     MacroAssembler _masm(&cbuf);
2068     Label* p = $lbl$$label;
2069 
2070     // 'p' is `NULL' when this encoding class is used only to
2071     // determine the size of the encoded instruction.
2072     // Use a bound dummy label in that case.
2073     Label d;
2074     __ bind(d);
2075     Label& l = (NULL == p) ? d : *(p);
2076     Assembler::branch_condition cc = (Assembler::branch_condition)$cmp$$cmpcode;
2077     unsigned long instr = $primary;
2078     if (instr == CRJ_ZOPC) {
2079       __ z_crj($src1$$Register, $src2$$Register, cc, l);
2080     } else if (instr == CLRJ_ZOPC) {
2081       __ z_clrj($src1$$Register, $src2$$Register, cc, l);
2082     } else if (instr == CGRJ_ZOPC) {
2083       __ z_cgrj($src1$$Register, $src2$$Register, cc, l);
2084     } else {
2085       guarantee(instr == CLGRJ_ZOPC, "opcode not implemented");
2086       __ z_clgrj($src1$$Register, $src2$$Register, cc, l);
2087     }
2088   %}
2089 
2090   enc_class z_enc_cmpb_regregFar(iRegI src1, iRegI src2, Label lbl, cmpOpT cmp) %{
2091     MacroAssembler _masm(&cbuf);
2092     Label* p = $lbl$$label;
2093 
2094     // 'p' is `NULL' when this encoding class is used only to
2095     // determine the size of the encoded instruction.
2096     // Use a bound dummy label in that case.
2097     Label d;
2098     __ bind(d);
2099     Label& l = (NULL == p) ? d : *(p);
2100 
2101     unsigned long instr = $primary;
2102     if (instr == CR_ZOPC) {
2103       __ z_cr($src1$$Register, $src2$$Register);
2104     } else if (instr == CLR_ZOPC) {
2105       __ z_clr($src1$$Register, $src2$$Register);
2106     } else if (instr == CGR_ZOPC) {
2107       __ z_cgr($src1$$Register, $src2$$Register);
2108     } else {
2109       guarantee(instr == CLGR_ZOPC, "opcode not implemented");
2110       __ z_clgr($src1$$Register, $src2$$Register);
2111     }
2112 
2113     __ z_brcl((Assembler::branch_condition)$cmp$$cmpcode, l);
2114   %}
2115 
2116   enc_class z_enc_cmpb_regimm(iRegI src1, immI8 src2, Label lbl, cmpOpT cmp) %{
2117     MacroAssembler _masm(&cbuf);
2118     Label* p = $lbl$$label;
2119 
2120     // 'p' is `NULL' when this encoding class is used only to
2121     // determine the size of the encoded instruction.
2122     // Use a bound dummy label in that case.
2123     Label d;
2124     __ bind(d);
2125     Label& l = (NULL == p) ? d : *(p);
2126 
2127     Assembler::branch_condition cc = (Assembler::branch_condition)$cmp$$cmpcode;
2128     unsigned long instr = $primary;
2129     if (instr == CIJ_ZOPC) {
2130       __ z_cij($src1$$Register, $src2$$constant, cc, l);
2131     } else if (instr == CLIJ_ZOPC) {
2132       __ z_clij($src1$$Register, $src2$$constant, cc, l);
2133     } else if (instr == CGIJ_ZOPC) {
2134       __ z_cgij($src1$$Register, $src2$$constant, cc, l);
2135     } else {
2136       guarantee(instr == CLGIJ_ZOPC, "opcode not implemented");
2137       __ z_clgij($src1$$Register, $src2$$constant, cc, l);
2138     }
2139   %}
2140 
2141   enc_class z_enc_cmpb_regimmFar(iRegI src1, immI8 src2, Label lbl, cmpOpT cmp) %{
2142     MacroAssembler _masm(&cbuf);
2143     Label* p = $lbl$$label;
2144 
2145     // 'p' is `NULL' when this encoding class is used only to
2146     // determine the size of the encoded instruction.
2147     // Use a bound dummy label in that case.
2148     Label d;
2149     __ bind(d);
2150     Label& l = (NULL == p) ? d : *(p);
2151 
2152     unsigned long instr = $primary;
2153     if (instr == CHI_ZOPC) {
2154       __ z_chi($src1$$Register, $src2$$constant);
2155     } else if (instr == CLFI_ZOPC) {
2156       __ z_clfi($src1$$Register, $src2$$constant);
2157     } else if (instr == CGHI_ZOPC) {
2158       __ z_cghi($src1$$Register, $src2$$constant);
2159     } else {
2160       guarantee(instr == CLGFI_ZOPC, "opcode not implemented");
2161       __ z_clgfi($src1$$Register, $src2$$constant);
2162     }
2163 
2164     __ z_brcl((Assembler::branch_condition)$cmp$$cmpcode, l);
2165   %}
2166 
2167   // Call from Java to runtime.
2168   enc_class z_enc_java_to_runtime_call(method meth) %{
2169     MacroAssembler _masm(&cbuf);
2170 
2171     // Save return pc before call to the place where we need it, since
2172     // callee doesn't.
2173     unsigned int start_off = __ offset();
2174     // Compute size of "larl + stg + call_c_opt".
2175     const int size_of_code = 6 + 6 + MacroAssembler::call_far_patchable_size();
2176     __ get_PC(Z_R14, size_of_code);
2177     __ save_return_pc();
2178     assert(__ offset() - start_off == 12, "bad prelude len: %d", __ offset() - start_off);
2179 
2180     assert((__ offset() & 2) == 0, "misaligned z_enc_java_to_runtime_call");
2181     address call_addr = __ call_c_opt((address)$meth$$method);
2182     if (call_addr == NULL) {
2183       Compile::current()->env()->record_out_of_memory_failure();
2184       return;
2185     }
2186 
2187 #ifdef ASSERT
2188     // Plausibility check for size_of_code assumptions.
2189     unsigned int actual_ret_off = __ offset();
2190     assert(start_off + size_of_code == actual_ret_off, "wrong return_pc");
2191 #endif
2192   %}
2193 
2194   enc_class z_enc_java_static_call(method meth) %{
2195     // Call to fixup routine. Fixup routine uses ScopeDesc info to determine
2196     // whom we intended to call.
2197     MacroAssembler _masm(&cbuf);
2198     int ret_offset = 0;
2199 
2200     if (!_method) {
2201       ret_offset = emit_call_reloc(_masm, $meth$$method,
2202                                    relocInfo::runtime_call_w_cp_type, ra_);
2203     } else {
2204       int method_index = resolved_method_index(cbuf);
2205       if (_optimized_virtual) {
2206         ret_offset = emit_call_reloc(_masm, $meth$$method,
2207                                      opt_virtual_call_Relocation::spec(method_index));
2208       } else {
2209         ret_offset = emit_call_reloc(_masm, $meth$$method,
2210                                      static_call_Relocation::spec(method_index));
2211       }
2212     }
2213     assert(__ inst_mark() != NULL, "emit_call_reloc must set_inst_mark()");
2214 
2215     if (_method) { // Emit stub for static call.
2216       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
2217       if (stub == NULL) {
2218         ciEnv::current()->record_failure("CodeCache is full");
2219         return;
2220       }
2221     }
2222   %}
2223 
2224   // Java dynamic call
2225   enc_class z_enc_java_dynamic_call(method meth) %{
2226     MacroAssembler _masm(&cbuf);
2227     unsigned int start_off = __ offset();
2228 
2229     int vtable_index = this->_vtable_index;
2230     if (vtable_index == -4) {
2231       Register ic_reg = reg_to_register_object(Matcher::inline_cache_reg_encode());
2232       address virtual_call_oop_addr = NULL;
2233 
2234       AddressLiteral empty_ic((address) Universe::non_oop_word());
2235       virtual_call_oop_addr = __ pc();
2236       bool success = __ load_const_from_toc(ic_reg, empty_ic);
2237       if (!success) {
2238         Compile::current()->env()->record_out_of_memory_failure();
2239         return;
2240       }
2241 
2242       // Call to fixup routine. Fixup routine uses ScopeDesc info
2243       // to determine who we intended to call.
2244       int method_index = resolved_method_index(cbuf);
2245       __ relocate(virtual_call_Relocation::spec(virtual_call_oop_addr, method_index));
2246       unsigned int ret_off = __ offset();
2247       assert(__ offset() - start_off == 6, "bad prelude len: %d", __ offset() - start_off);
2248       ret_off += emit_call_reloc(_masm, $meth$$method, relocInfo::none, ra_);
2249       assert(_method, "lazy_constant may be wrong when _method==null");
2250     } else {
2251       assert(!UseInlineCaches, "expect vtable calls only if not using ICs");
2252       // Go through the vtable. Get receiver klass. Receiver already
2253       // checked for non-null. If we'll go thru a C2I adapter, the
2254       // interpreter expects method in Z_method.
2255       // Use Z_method to temporarily hold the klass oop.
2256       // Z_R1_scratch is destroyed.
2257       __ load_klass(Z_method, Z_R2);
2258 
2259       int entry_offset = in_bytes(Klass::vtable_start_offset()) + vtable_index * vtableEntry::size_in_bytes();
2260       int v_off        = entry_offset + vtableEntry::method_offset_in_bytes();
2261 
2262       if (Displacement::is_validDisp(v_off) ) {
2263         // Can use load instruction with large offset.
2264         __ z_lg(Z_method, Address(Z_method /*class oop*/, v_off /*method offset*/));
2265       } else {
2266         // Worse case, must load offset into register.
2267         __ load_const(Z_R1_scratch, v_off);
2268         __ z_lg(Z_method, Address(Z_method /*class oop*/, Z_R1_scratch /*method offset*/));
2269       }
2270       // NOTE: for vtable dispatches, the vtable entry will never be
2271       // null. However it may very well end up in handle_wrong_method
2272       // if the method is abstract for the particular class.
2273       __ z_lg(Z_R1_scratch, Address(Z_method, Method::from_compiled_offset()));
2274       // Call target. Either compiled code or C2I adapter.
2275       __ z_basr(Z_R14, Z_R1_scratch);
2276       unsigned int ret_off = __ offset();
2277     }
2278   %}
2279 
2280   enc_class z_enc_cmov_reg(cmpOp cmp, iRegI dst, iRegI src) %{
2281     MacroAssembler _masm(&cbuf);
2282     Register Rdst = reg_to_register_object($dst$$reg);
2283     Register Rsrc = reg_to_register_object($src$$reg);
2284 
2285     // Don't emit code if operands are identical (same register).
2286     if (Rsrc != Rdst) {
2287       Assembler::branch_condition cc = (Assembler::branch_condition)$cmp$$cmpcode;
2288 
2289       if (VM_Version::has_LoadStoreConditional()) {
2290         __ z_locgr(Rdst, Rsrc, cc);
2291       } else {
2292         // Branch if not (cmp cr).
2293         Label done;
2294         __ z_brc(Assembler::inverse_condition(cc), done);
2295         __ z_lgr(Rdst, Rsrc); // Used for int and long+ptr.
2296         __ bind(done);
2297       }
2298     }
2299   %}
2300 
2301   enc_class z_enc_cmov_imm(cmpOp cmp, iRegI dst, immI16 src) %{
2302     MacroAssembler _masm(&cbuf);
2303     Register Rdst = reg_to_register_object($dst$$reg);
2304     int      Csrc = $src$$constant;
2305     Assembler::branch_condition cc = (Assembler::branch_condition)$cmp$$cmpcode;
2306     Label done;
2307     // Branch if not (cmp cr).
2308     __ z_brc(Assembler::inverse_condition(cc), done);
2309     if (Csrc == 0) {
2310       // Don't set CC.
2311       __ clear_reg(Rdst, true, false);  // Use for int, long & ptr.
2312     } else {
2313       __ z_lghi(Rdst, Csrc); // Use for int, long & ptr.
2314     }
2315     __ bind(done);
2316   %}
2317 
2318   enc_class z_enc_cctobool(iRegI res) %{
2319     MacroAssembler _masm(&cbuf);
2320     Register Rres = reg_to_register_object($res$$reg);
2321 
2322     if (VM_Version::has_LoadStoreConditional()) {
2323       __ load_const_optimized(Z_R0_scratch, 0L); // false (failed)
2324       __ load_const_optimized(Rres, 1L);         // true  (succeed)
2325       __ z_locgr(Rres, Z_R0_scratch, Assembler::bcondNotEqual);
2326     } else {
2327       Label done;
2328       __ load_const_optimized(Rres, 0L); // false (failed)
2329       __ z_brne(done);                   // Assume true to be the common case.
2330       __ load_const_optimized(Rres, 1L); // true  (succeed)
2331       __ bind(done);
2332     }
2333   %}
2334 
2335   enc_class z_enc_casI(iRegI compare_value, iRegI exchange_value, iRegP addr_ptr) %{
2336     MacroAssembler _masm(&cbuf);
2337     Register Rcomp = reg_to_register_object($compare_value$$reg);
2338     Register Rnew  = reg_to_register_object($exchange_value$$reg);
2339     Register Raddr = reg_to_register_object($addr_ptr$$reg);
2340 
2341     __ z_cs(Rcomp, Rnew, 0, Raddr);
2342   %}
2343 
2344   enc_class z_enc_casL(iRegL compare_value, iRegL exchange_value, iRegP addr_ptr) %{
2345     MacroAssembler _masm(&cbuf);
2346     Register Rcomp = reg_to_register_object($compare_value$$reg);
2347     Register Rnew  = reg_to_register_object($exchange_value$$reg);
2348     Register Raddr = reg_to_register_object($addr_ptr$$reg);
2349 
2350     __ z_csg(Rcomp, Rnew, 0, Raddr);
2351   %}
2352 
2353   enc_class z_enc_SwapI(memoryRSY mem, iRegI dst, iRegI tmp) %{
2354     MacroAssembler _masm(&cbuf);
2355     Register Rdst = reg_to_register_object($dst$$reg);
2356     Register Rtmp = reg_to_register_object($tmp$$reg);
2357     guarantee(Rdst != Rtmp, "Fix match rule to use TEMP_DEF");
2358     Label    retry;
2359 
2360     // Iterate until swap succeeds.
2361     __ z_llgf(Rtmp, $mem$$Address);  // current contents
2362     __ bind(retry);
2363       // Calculate incremented value.
2364       __ z_csy(Rtmp, Rdst, $mem$$Address); // Try to store new value.
2365       __ z_brne(retry);                    // Yikes, concurrent update, need to retry.
2366     __ z_lgr(Rdst, Rtmp);                  // Exchanged value from memory is return value.
2367   %}
2368 
2369   enc_class z_enc_SwapL(memoryRSY mem, iRegL dst, iRegL tmp) %{
2370     MacroAssembler _masm(&cbuf);
2371     Register Rdst = reg_to_register_object($dst$$reg);
2372     Register Rtmp = reg_to_register_object($tmp$$reg);
2373     guarantee(Rdst != Rtmp, "Fix match rule to use TEMP_DEF");
2374     Label    retry;
2375 
2376     // Iterate until swap succeeds.
2377     __ z_lg(Rtmp, $mem$$Address);  // current contents
2378     __ bind(retry);
2379       // Calculate incremented value.
2380       __ z_csg(Rtmp, Rdst, $mem$$Address); // Try to store new value.
2381       __ z_brne(retry);                    // Yikes, concurrent update, need to retry.
2382     __ z_lgr(Rdst, Rtmp);                  // Exchanged value from memory is return value.
2383   %}
2384 
2385 %} // encode
2386 
2387 source %{
2388 
2389   // Check whether outs are all Stores. If so, we can omit clearing the upper
2390   // 32 bits after encoding.
2391   static bool all_outs_are_Stores(const Node *n) {
2392     for (DUIterator_Fast imax, k = n->fast_outs(imax); k < imax; k++) {
2393       Node *out = n->fast_out(k);
2394       if (!out->is_Mach() || out->as_Mach()->ideal_Opcode() != Op_StoreN) {
2395         // Most other outs are SpillCopy, but there are various other.
2396         // jvm98 has arond 9% Encodes where we return false.
2397         return false;
2398       }
2399     }
2400     return true;
2401   }
2402 
2403 %} // source
2404 
2405 
2406 //----------FRAME--------------------------------------------------------------
2407 // Definition of frame structure and management information.
2408 
2409 frame %{
2410   // What direction does stack grow in (assumed to be same for native & Java).
2411   stack_direction(TOWARDS_LOW);
2412 
2413   // These two registers define part of the calling convention between
2414   // compiled code and the interpreter.
2415 
2416   // Inline Cache Register
2417   inline_cache_reg(Z_R9); // Z_inline_cache
2418 
2419   // Argument pointer for I2C adapters
2420   //
2421   // Tos is loaded in run_compiled_code to Z_ARG5=Z_R6.
2422   // interpreter_arg_ptr_reg(Z_R6);
2423 
2424   // Temporary in compiled entry-points
2425   // compiler_method_oop_reg(Z_R1);//Z_R1_scratch
2426 
2427   // Method Oop Register when calling interpreter
2428   interpreter_method_oop_reg(Z_R9);//Z_method
2429 
2430   // Optional: name the operand used by cisc-spilling to access
2431   // [stack_pointer + offset].
2432   cisc_spilling_operand_name(indOffset12);
2433 
2434   // Number of stack slots consumed by a Monitor enter.
2435   sync_stack_slots(frame::jit_monitor_size_in_4_byte_units);
2436 
2437   // Compiled code's Frame Pointer
2438   //
2439   // z/Architecture stack pointer
2440   frame_pointer(Z_R15); // Z_SP
2441 
2442   // Interpreter stores its frame pointer in a register which is
2443   // stored to the stack by I2CAdaptors. I2CAdaptors convert from
2444   // interpreted java to compiled java.
2445   //
2446   // Z_state holds pointer to caller's cInterpreter.
2447   interpreter_frame_pointer(Z_R7); // Z_state
2448 
2449   // Use alignment_in_bytes instead of log_2_of_alignment_in_bits.
2450   stack_alignment(frame::alignment_in_bytes);
2451 
2452   in_preserve_stack_slots(frame::jit_in_preserve_size_in_4_byte_units);
2453 
2454   // A `slot' is assumed 4 bytes here!
2455   // out_preserve_stack_slots(frame::jit_out_preserve_size_in_4_byte_units);
2456 
2457   // Number of outgoing stack slots killed above the
2458   // out_preserve_stack_slots for calls to C. Supports the var-args
2459   // backing area for register parms.
2460   varargs_C_out_slots_killed(((frame::z_abi_160_size - frame::z_jit_out_preserve_size) / VMRegImpl::stack_slot_size));
2461 
2462   // The after-PROLOG location of the return address. Location of
2463   // return address specifies a type (REG or STACK) and a number
2464   // representing the register number (i.e. - use a register name) or
2465   // stack slot.
2466   return_addr(REG Z_R14);
2467 
2468   // This is the body of the function
2469   //
2470   // void Matcher::calling_convention(OptoRegPair* sig /* array of ideal regs */,
2471   //                                  uint length      /* length of array */,
2472   //                                  bool is_outgoing)
2473   //
2474   // The `sig' array is to be updated. Sig[j] represents the location
2475   // of the j-th argument, either a register or a stack slot.
2476 
2477   // Body of function which returns an integer array locating
2478   // arguments either in registers or in stack slots. Passed an array
2479   // of ideal registers called "sig" and a "length" count. Stack-slot
2480   // offsets are based on outgoing arguments, i.e. a CALLER setting up
2481   // arguments for a CALLEE. Incoming stack arguments are
2482   // automatically biased by the preserve_stack_slots field above.
2483   calling_convention %{
2484     // No difference between ingoing/outgoing just pass false.
2485     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
2486   %}
2487 
2488   // Body of function which returns an integer array locating
2489   // arguments either in registers or in stack slots. Passed an array
2490   // of ideal registers called "sig" and a "length" count. Stack-slot
2491   // offsets are based on outgoing arguments, i.e. a CALLER setting up
2492   // arguments for a CALLEE. Incoming stack arguments are
2493   // automatically biased by the preserve_stack_slots field above.
2494   c_calling_convention %{
2495     // This is obviously always outgoing.
2496     // C argument must be in register AND stack slot.
2497     (void) SharedRuntime::c_calling_convention(sig_bt, regs, /*regs2=*/NULL, length);
2498   %}
2499 
2500   // Location of native (C/C++) and interpreter return values. This
2501   // is specified to be the same as Java. In the 32-bit VM, long
2502   // values are actually returned from native calls in O0:O1 and
2503   // returned to the interpreter in I0:I1. The copying to and from
2504   // the register pairs is done by the appropriate call and epilog
2505   // opcodes. This simplifies the register allocator.
2506   //
2507   // Use register pair for c return value.
2508   c_return_value %{
2509     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL, "only return normal values");
2510     static int typeToRegLo[Op_RegL+1] = { 0, 0, Z_R2_num, Z_R2_num, Z_R2_num, Z_F0_num, Z_F0_num, Z_R2_num };
2511     static int typeToRegHi[Op_RegL+1] = { 0, 0, OptoReg::Bad, OptoReg::Bad, Z_R2_H_num, OptoReg::Bad, Z_F0_H_num, Z_R2_H_num };
2512     return OptoRegPair(typeToRegHi[ideal_reg], typeToRegLo[ideal_reg]);
2513   %}
2514 
2515   // Use register pair for return value.
2516   // Location of compiled Java return values. Same as C
2517   return_value %{
2518     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL, "only return normal values");
2519     static int typeToRegLo[Op_RegL+1] = { 0, 0, Z_R2_num, Z_R2_num, Z_R2_num, Z_F0_num, Z_F0_num, Z_R2_num };
2520     static int typeToRegHi[Op_RegL+1] = { 0, 0, OptoReg::Bad, OptoReg::Bad, Z_R2_H_num, OptoReg::Bad, Z_F0_H_num, Z_R2_H_num };
2521     return OptoRegPair(typeToRegHi[ideal_reg], typeToRegLo[ideal_reg]);
2522   %}
2523 %}
2524 
2525 
2526 //----------ATTRIBUTES---------------------------------------------------------
2527 
2528 //----------Operand Attributes-------------------------------------------------
2529 op_attrib op_cost(1);          // Required cost attribute
2530 
2531 //----------Instruction Attributes---------------------------------------------
2532 
2533 // Cost attribute. required.
2534 ins_attrib ins_cost(DEFAULT_COST);
2535 
2536 // Is this instruction a non-matching short branch variant of some
2537 // long branch? Not required.
2538 ins_attrib ins_short_branch(0);
2539 
2540 // Indicates this is a trap based check node and final control-flow fixup
2541 // must generate a proper fall through.
2542 ins_attrib ins_is_TrapBasedCheckNode(true);
2543 
2544 // Attribute of instruction to tell how many constants the instruction will generate.
2545 // (optional attribute). Default: 0.
2546 ins_attrib ins_num_consts(0);
2547 
2548 // Required alignment attribute (must be a power of 2)
2549 // specifies the alignment that some part of the instruction (not
2550 // necessarily the start) requires. If > 1, a compute_padding()
2551 // function must be provided for the instruction.
2552 //
2553 // WARNING: Don't use size(FIXED_SIZE) or size(VARIABLE_SIZE) in
2554 // instructions which depend on the proper alignment, because the
2555 // desired alignment isn't guaranteed for the call to "emit()" during
2556 // the size computation.
2557 ins_attrib ins_alignment(1);
2558 
2559 // Enforce/prohibit rematerializations.
2560 // - If an instruction is attributed with 'ins_cannot_rematerialize(true)'
2561 //   then rematerialization of that instruction is prohibited and the
2562 //   instruction's value will be spilled if necessary.
2563 // - If an instruction is attributed with 'ins_should_rematerialize(true)'
2564 //   then rematerialization is enforced and the instruction's value will
2565 //   never get spilled. a copy of the instruction will be inserted if
2566 //   necessary.
2567 //   Note: this may result in rematerializations in front of every use.
2568 // (optional attribute)
2569 ins_attrib ins_cannot_rematerialize(false);
2570 ins_attrib ins_should_rematerialize(false);
2571 
2572 //----------OPERANDS-----------------------------------------------------------
2573 // Operand definitions must precede instruction definitions for correct
2574 // parsing in the ADLC because operands constitute user defined types
2575 // which are used in instruction definitions.
2576 
2577 //----------Simple Operands----------------------------------------------------
2578 // Immediate Operands
2579 // Please note:
2580 // Formats are generated automatically for constants and base registers.
2581 
2582 //----------------------------------------------
2583 // SIGNED (shorter than INT) immediate operands
2584 //----------------------------------------------
2585 
2586 // Byte Immediate: constant 'int -1'
2587 operand immB_minus1() %{
2588   //         sign-ext constant      zero-ext constant
2589   predicate((n->get_int() == -1) || ((n->get_int()&0x000000ff) == 0x000000ff));
2590   match(ConI);
2591   op_cost(1);
2592   format %{ %}
2593   interface(CONST_INTER);
2594 %}
2595 
2596 // Byte Immediate: constant, but not 'int 0' nor 'int -1'.
2597 operand immB_n0m1() %{
2598   //                             sign-ext constant     zero-ext constant
2599   predicate(n->get_int() != 0 && n->get_int() != -1 && (n->get_int()&0x000000ff) != 0x000000ff);
2600   match(ConI);
2601   op_cost(1);
2602   format %{ %}
2603   interface(CONST_INTER);
2604 %}
2605 
2606 // Short Immediate: constant 'int -1'
2607 operand immS_minus1() %{
2608   //         sign-ext constant      zero-ext constant
2609   predicate((n->get_int() == -1) || ((n->get_int()&0x0000ffff) == 0x0000ffff));
2610   match(ConI);
2611   op_cost(1);
2612   format %{ %}
2613   interface(CONST_INTER);
2614 %}
2615 
2616 // Short Immediate: constant, but not 'int 0' nor 'int -1'.
2617 operand immS_n0m1() %{
2618   //                             sign-ext constant     zero-ext constant
2619   predicate(n->get_int() != 0 && n->get_int() != -1 && (n->get_int()&0x0000ffff) != 0x0000ffff);
2620   match(ConI);
2621   op_cost(1);
2622   format %{ %}
2623   interface(CONST_INTER);
2624 %}
2625 
2626 //-----------------------------------------
2627 //  SIGNED INT immediate operands
2628 //-----------------------------------------
2629 
2630 // Integer Immediate: 32-bit
2631 operand immI() %{
2632   match(ConI);
2633   op_cost(1);
2634   format %{ %}
2635   interface(CONST_INTER);
2636 %}
2637 
2638 // Int Immediate: 20-bit
2639 operand immI20() %{
2640   predicate(Immediate::is_simm20(n->get_int()));
2641   match(ConI);
2642   op_cost(1);
2643   format %{ %}
2644   interface(CONST_INTER);
2645 %}
2646 
2647 // Integer Immediate: 16-bit
2648 operand immI16() %{
2649   predicate(Immediate::is_simm16(n->get_int()));
2650   match(ConI);
2651   op_cost(1);
2652   format %{ %}
2653   interface(CONST_INTER);
2654 %}
2655 
2656 // Integer Immediate: 8-bit
2657 operand immI8() %{
2658   predicate(Immediate::is_simm8(n->get_int()));
2659   match(ConI);
2660   op_cost(1);
2661   format %{ %}
2662   interface(CONST_INTER);
2663 %}
2664 
2665 // Integer Immediate: constant 'int 0'
2666 operand immI_0() %{
2667   predicate(n->get_int() == 0);
2668   match(ConI);
2669   op_cost(1);
2670   format %{ %}
2671   interface(CONST_INTER);
2672 %}
2673 
2674 // Integer Immediate: constant 'int -1'
2675 operand immI_minus1() %{
2676   predicate(n->get_int() == -1);
2677   match(ConI);
2678   op_cost(1);
2679   format %{ %}
2680   interface(CONST_INTER);
2681 %}
2682 
2683 // Integer Immediate: constant, but not 'int 0' nor 'int -1'.
2684 operand immI_n0m1() %{
2685   predicate(n->get_int() != 0 && n->get_int() != -1);
2686   match(ConI);
2687   op_cost(1);
2688   format %{ %}
2689   interface(CONST_INTER);
2690 %}
2691 
2692 //-------------------------------------------
2693 // UNSIGNED INT immediate operands
2694 //-------------------------------------------
2695 
2696 // Unsigned Integer Immediate: 32-bit
2697 operand uimmI() %{
2698   match(ConI);
2699   op_cost(1);
2700   format %{ %}
2701   interface(CONST_INTER);
2702 %}
2703 
2704 // Unsigned Integer Immediate: 16-bit
2705 operand uimmI16() %{
2706   predicate(Immediate::is_uimm16(n->get_int()));
2707   match(ConI);
2708   op_cost(1);
2709   format %{ %}
2710   interface(CONST_INTER);
2711 %}
2712 
2713 // Unsigned Integer Immediate: 12-bit
2714 operand uimmI12() %{
2715   predicate(Immediate::is_uimm12(n->get_int()));
2716   match(ConI);
2717   op_cost(1);
2718   format %{ %}
2719   interface(CONST_INTER);
2720 %}
2721 
2722 // Unsigned Integer Immediate: 12-bit
2723 operand uimmI8() %{
2724   predicate(Immediate::is_uimm8(n->get_int()));
2725   match(ConI);
2726   op_cost(1);
2727   format %{ %}
2728   interface(CONST_INTER);
2729 %}
2730 
2731 // Integer Immediate: 6-bit
2732 operand uimmI6() %{
2733   predicate(Immediate::is_uimm(n->get_int(), 6));
2734   match(ConI);
2735   op_cost(1);
2736   format %{ %}
2737   interface(CONST_INTER);
2738 %}
2739 
2740 // Integer Immediate: 5-bit
2741 operand uimmI5() %{
2742   predicate(Immediate::is_uimm(n->get_int(), 5));
2743   match(ConI);
2744   op_cost(1);
2745   format %{ %}
2746   interface(CONST_INTER);
2747 %}
2748 
2749 // Length for SS instructions, given in DWs,
2750 //   possible range [1..512], i.e. [8..4096] Bytes
2751 //   used     range [1..256], i.e. [8..2048] Bytes
2752 //   operand type int
2753 // Unsigned Integer Immediate: 9-bit
2754 operand SSlenDW() %{
2755   predicate(Immediate::is_uimm8(n->get_long()-1));
2756   match(ConL);
2757   op_cost(1);
2758   format %{ %}
2759   interface(CONST_INTER);
2760 %}
2761 
2762 //------------------------------------------
2763 // (UN)SIGNED INT specific values
2764 //------------------------------------------
2765 
2766 // Integer Immediate: the value 1
2767 operand immI_1() %{
2768   predicate(n->get_int() == 1);
2769   match(ConI);
2770   op_cost(1);
2771   format %{ %}
2772   interface(CONST_INTER);
2773 %}
2774 
2775 // Integer Immediate: the value 16.
2776 operand immI_16() %{
2777   predicate(n->get_int() == 16);
2778   match(ConI);
2779   op_cost(1);
2780   format %{ %}
2781   interface(CONST_INTER);
2782 %}
2783 
2784 // Integer Immediate: the value 24.
2785 operand immI_24() %{
2786   predicate(n->get_int() == 24);
2787   match(ConI);
2788   op_cost(1);
2789   format %{ %}
2790   interface(CONST_INTER);
2791 %}
2792 
2793 // Integer Immediate: the value 255
2794 operand immI_255() %{
2795   predicate(n->get_int() == 255);
2796   match(ConI);
2797   op_cost(1);
2798   format %{ %}
2799   interface(CONST_INTER);
2800 %}
2801 
2802 // Integer Immediate: the values 32-63
2803 operand immI_32_63() %{
2804   predicate(n->get_int() >= 32 && n->get_int() <= 63);
2805   match(ConI);
2806   op_cost(1);
2807   format %{ %}
2808   interface(CONST_INTER);
2809 %}
2810 
2811 // Unsigned Integer Immediate: LL-part, extended by 1s.
2812 operand uimmI_LL1() %{
2813   predicate((n->get_int() & 0xFFFF0000) == 0xFFFF0000);
2814   match(ConI);
2815   op_cost(1);
2816   format %{ %}
2817   interface(CONST_INTER);
2818 %}
2819 
2820 // Unsigned Integer Immediate: LH-part, extended by 1s.
2821 operand uimmI_LH1() %{
2822   predicate((n->get_int() & 0xFFFF) == 0xFFFF);
2823   match(ConI);
2824   op_cost(1);
2825   format %{ %}
2826   interface(CONST_INTER);
2827 %}
2828 
2829 //------------------------------------------
2830 // SIGNED LONG immediate operands
2831 //------------------------------------------
2832 
2833 operand immL() %{
2834   match(ConL);
2835   op_cost(1);
2836   format %{ %}
2837   interface(CONST_INTER);
2838 %}
2839 
2840 // Long Immediate: 32-bit
2841 operand immL32() %{
2842   predicate(Immediate::is_simm32(n->get_long()));
2843   match(ConL);
2844   op_cost(1);
2845   format %{ %}
2846   interface(CONST_INTER);
2847 %}
2848 
2849 // Long Immediate: 20-bit
2850 operand immL20() %{
2851   predicate(Immediate::is_simm20(n->get_long()));
2852   match(ConL);
2853   op_cost(1);
2854   format %{ %}
2855   interface(CONST_INTER);
2856 %}
2857 
2858 // Long Immediate: 16-bit
2859 operand immL16() %{
2860   predicate(Immediate::is_simm16(n->get_long()));
2861   match(ConL);
2862   op_cost(1);
2863   format %{ %}
2864   interface(CONST_INTER);
2865 %}
2866 
2867 // Long Immediate: 8-bit
2868 operand immL8() %{
2869   predicate(Immediate::is_simm8(n->get_long()));
2870   match(ConL);
2871   op_cost(1);
2872   format %{ %}
2873   interface(CONST_INTER);
2874 %}
2875 
2876 //--------------------------------------------
2877 // UNSIGNED LONG immediate operands
2878 //--------------------------------------------
2879 
2880 operand uimmL32() %{
2881   predicate(Immediate::is_uimm32(n->get_long()));
2882   match(ConL);
2883   op_cost(1);
2884   format %{ %}
2885   interface(CONST_INTER);
2886 %}
2887 
2888 // Unsigned Long Immediate: 16-bit
2889 operand uimmL16() %{
2890   predicate(Immediate::is_uimm16(n->get_long()));
2891   match(ConL);
2892   op_cost(1);
2893   format %{ %}
2894   interface(CONST_INTER);
2895 %}
2896 
2897 // Unsigned Long Immediate: 12-bit
2898 operand uimmL12() %{
2899   predicate(Immediate::is_uimm12(n->get_long()));
2900   match(ConL);
2901   op_cost(1);
2902   format %{ %}
2903   interface(CONST_INTER);
2904 %}
2905 
2906 // Unsigned Long Immediate: 8-bit
2907 operand uimmL8() %{
2908   predicate(Immediate::is_uimm8(n->get_long()));
2909   match(ConL);
2910   op_cost(1);
2911   format %{ %}
2912   interface(CONST_INTER);
2913 %}
2914 
2915 //-------------------------------------------
2916 // (UN)SIGNED LONG specific values
2917 //-------------------------------------------
2918 
2919 // Long Immediate: the value FF
2920 operand immL_FF() %{
2921   predicate(n->get_long() == 0xFFL);
2922   match(ConL);
2923   op_cost(1);
2924   format %{ %}
2925   interface(CONST_INTER);
2926 %}
2927 
2928 // Long Immediate: the value FFFF
2929 operand immL_FFFF() %{
2930   predicate(n->get_long() == 0xFFFFL);
2931   match(ConL);
2932   op_cost(1);
2933   format %{ %}
2934   interface(CONST_INTER);
2935 %}
2936 
2937 // Long Immediate: the value FFFFFFFF
2938 operand immL_FFFFFFFF() %{
2939   predicate(n->get_long() == 0xFFFFFFFFL);
2940   match(ConL);
2941   op_cost(1);
2942   format %{ %}
2943   interface(CONST_INTER);
2944 %}
2945 
2946 operand immL_0() %{
2947   predicate(n->get_long() == 0L);
2948   match(ConL);
2949   op_cost(1);
2950   format %{ %}
2951   interface(CONST_INTER);
2952 %}
2953 
2954 // Unsigned Long Immediate: LL-part, extended by 1s.
2955 operand uimmL_LL1() %{
2956   predicate((n->get_long() & 0xFFFFFFFFFFFF0000L) == 0xFFFFFFFFFFFF0000L);
2957   match(ConL);
2958   op_cost(1);
2959   format %{ %}
2960   interface(CONST_INTER);
2961 %}
2962 
2963 // Unsigned Long Immediate: LH-part, extended by 1s.
2964 operand uimmL_LH1() %{
2965   predicate((n->get_long() & 0xFFFFFFFF0000FFFFL) == 0xFFFFFFFF0000FFFFL);
2966   match(ConL);
2967   op_cost(1);
2968   format %{ %}
2969   interface(CONST_INTER);
2970 %}
2971 
2972 // Unsigned Long Immediate: HL-part, extended by 1s.
2973 operand uimmL_HL1() %{
2974   predicate((n->get_long() & 0xFFFF0000FFFFFFFFL) == 0xFFFF0000FFFFFFFFL);
2975   match(ConL);
2976   op_cost(1);
2977   format %{ %}
2978   interface(CONST_INTER);
2979 %}
2980 
2981 // Unsigned Long Immediate: HH-part, extended by 1s.
2982 operand uimmL_HH1() %{
2983   predicate((n->get_long() & 0xFFFFFFFFFFFFL) == 0xFFFFFFFFFFFFL);
2984   match(ConL);
2985   op_cost(1);
2986   format %{ %}
2987   interface(CONST_INTER);
2988 %}
2989 
2990 // Long Immediate: low 32-bit mask
2991 operand immL_32bits() %{
2992   predicate(n->get_long() == 0xFFFFFFFFL);
2993   match(ConL);
2994   op_cost(1);
2995   format %{ %}
2996   interface(CONST_INTER);
2997 %}
2998 
2999 //--------------------------------------
3000 //  POINTER immediate operands
3001 //--------------------------------------
3002 
3003 // Pointer Immediate: 64-bit
3004 operand immP() %{
3005   match(ConP);
3006   op_cost(1);
3007   format %{ %}
3008   interface(CONST_INTER);
3009 %}
3010 
3011 // Pointer Immediate: 32-bit
3012 operand immP32() %{
3013   predicate(Immediate::is_uimm32(n->get_ptr()));
3014   match(ConP);
3015   op_cost(1);
3016   format %{ %}
3017   interface(CONST_INTER);
3018 %}
3019 
3020 // Pointer Immediate: 16-bit
3021 operand immP16() %{
3022   predicate(Immediate::is_uimm16(n->get_ptr()));
3023   match(ConP);
3024   op_cost(1);
3025   format %{ %}
3026   interface(CONST_INTER);
3027 %}
3028 
3029 // Pointer Immediate: 8-bit
3030 operand immP8() %{
3031   predicate(Immediate::is_uimm8(n->get_ptr()));
3032   match(ConP);
3033   op_cost(1);
3034   format %{ %}
3035   interface(CONST_INTER);
3036 %}
3037 
3038 //-----------------------------------
3039 // POINTER specific values
3040 //-----------------------------------
3041 
3042 // Pointer Immediate: NULL
3043 operand immP0() %{
3044   predicate(n->get_ptr() == 0);
3045   match(ConP);
3046   op_cost(1);
3047   format %{ %}
3048   interface(CONST_INTER);
3049 %}
3050 
3051 //---------------------------------------------
3052 // NARROW POINTER immediate operands
3053 //---------------------------------------------
3054 
3055 // Narrow Pointer Immediate
3056 operand immN() %{
3057   match(ConN);
3058   op_cost(1);
3059   format %{ %}
3060   interface(CONST_INTER);
3061 %}
3062 
3063 operand immNKlass() %{
3064   match(ConNKlass);
3065   op_cost(1);
3066   format %{ %}
3067   interface(CONST_INTER);
3068 %}
3069 
3070 // Narrow Pointer Immediate
3071 operand immN8() %{
3072   predicate(Immediate::is_uimm8(n->get_narrowcon()));
3073   match(ConN);
3074   op_cost(1);
3075   format %{ %}
3076   interface(CONST_INTER);
3077 %}
3078 
3079 // Narrow NULL Pointer Immediate
3080 operand immN0() %{
3081   predicate(n->get_narrowcon() == 0);
3082   match(ConN);
3083   op_cost(1);
3084   format %{ %}
3085   interface(CONST_INTER);
3086 %}
3087 
3088 // FLOAT and DOUBLE immediate operands
3089 
3090 // Double Immediate
3091 operand immD() %{
3092   match(ConD);
3093   op_cost(1);
3094   format %{ %}
3095   interface(CONST_INTER);
3096 %}
3097 
3098 // Double Immediate: +-0
3099 operand immDpm0() %{
3100   predicate(n->getd() == 0);
3101   match(ConD);
3102   op_cost(1);
3103   format %{ %}
3104   interface(CONST_INTER);
3105 %}
3106 
3107 // Double Immediate: +0
3108 operand immDp0() %{
3109   predicate(jlong_cast(n->getd()) == 0);
3110   match(ConD);
3111   op_cost(1);
3112   format %{ %}
3113   interface(CONST_INTER);
3114 %}
3115 
3116 // Float Immediate
3117 operand immF() %{
3118   match(ConF);
3119   op_cost(1);
3120   format %{ %}
3121   interface(CONST_INTER);
3122 %}
3123 
3124 // Float Immediate: +-0
3125 operand immFpm0() %{
3126   predicate(n->getf() == 0);
3127   match(ConF);
3128   op_cost(1);
3129   format %{ %}
3130   interface(CONST_INTER);
3131 %}
3132 
3133 // Float Immediate: +0
3134 operand immFp0() %{
3135   predicate(jint_cast(n->getf()) == 0);
3136   match(ConF);
3137   op_cost(1);
3138   format %{ %}
3139   interface(CONST_INTER);
3140 %}
3141 
3142 // End of Immediate Operands
3143 
3144 // Integer Register Operands
3145 // Integer Register
3146 operand iRegI() %{
3147   constraint(ALLOC_IN_RC(z_int_reg));
3148   match(RegI);
3149   match(noArg_iRegI);
3150   match(rarg1RegI);
3151   match(rarg2RegI);
3152   match(rarg3RegI);
3153   match(rarg4RegI);
3154   match(rarg5RegI);
3155   match(noOdd_iRegI);
3156   match(revenRegI);
3157   match(roddRegI);
3158   format %{ %}
3159   interface(REG_INTER);
3160 %}
3161 
3162 operand noArg_iRegI() %{
3163   constraint(ALLOC_IN_RC(z_no_arg_int_reg));
3164   match(RegI);
3165   format %{ %}
3166   interface(REG_INTER);
3167 %}
3168 
3169 // revenRegI and roddRegI constitute and even-odd-pair.
3170 operand revenRegI() %{
3171   constraint(ALLOC_IN_RC(z_rarg3_int_reg));
3172   match(iRegI);
3173   format %{ %}
3174   interface(REG_INTER);
3175 %}
3176 
3177 // revenRegI and roddRegI constitute and even-odd-pair.
3178 operand roddRegI() %{
3179   constraint(ALLOC_IN_RC(z_rarg4_int_reg));
3180   match(iRegI);
3181   format %{ %}
3182   interface(REG_INTER);
3183 %}
3184 
3185 operand rarg1RegI() %{
3186   constraint(ALLOC_IN_RC(z_rarg1_int_reg));
3187   match(iRegI);
3188   format %{ %}
3189   interface(REG_INTER);
3190 %}
3191 
3192 operand rarg2RegI() %{
3193   constraint(ALLOC_IN_RC(z_rarg2_int_reg));
3194   match(iRegI);
3195   format %{ %}
3196   interface(REG_INTER);
3197 %}
3198 
3199 operand rarg3RegI() %{
3200   constraint(ALLOC_IN_RC(z_rarg3_int_reg));
3201   match(iRegI);
3202   format %{ %}
3203   interface(REG_INTER);
3204 %}
3205 
3206 operand rarg4RegI() %{
3207   constraint(ALLOC_IN_RC(z_rarg4_int_reg));
3208   match(iRegI);
3209   format %{ %}
3210   interface(REG_INTER);
3211 %}
3212 
3213 operand rarg5RegI() %{
3214   constraint(ALLOC_IN_RC(z_rarg5_int_reg));
3215   match(iRegI);
3216   format %{ %}
3217   interface(REG_INTER);
3218 %}
3219 
3220 operand noOdd_iRegI() %{
3221   constraint(ALLOC_IN_RC(z_no_odd_int_reg));
3222   match(RegI);
3223   match(revenRegI);
3224   format %{ %}
3225   interface(REG_INTER);
3226 %}
3227 
3228 // Pointer Register
3229 operand iRegP() %{
3230   constraint(ALLOC_IN_RC(z_ptr_reg));
3231   match(RegP);
3232   match(noArg_iRegP);
3233   match(rarg1RegP);
3234   match(rarg2RegP);
3235   match(rarg3RegP);
3236   match(rarg4RegP);
3237   match(rarg5RegP);
3238   match(revenRegP);
3239   match(roddRegP);
3240   format %{ %}
3241   interface(REG_INTER);
3242 %}
3243 
3244 // thread operand
3245 operand threadRegP() %{
3246   constraint(ALLOC_IN_RC(z_thread_ptr_reg));
3247   match(RegP);
3248   format %{ "Z_THREAD" %}
3249   interface(REG_INTER);
3250 %}
3251 
3252 operand noArg_iRegP() %{
3253   constraint(ALLOC_IN_RC(z_no_arg_ptr_reg));
3254   match(iRegP);
3255   format %{ %}
3256   interface(REG_INTER);
3257 %}
3258 
3259 operand rarg1RegP() %{
3260   constraint(ALLOC_IN_RC(z_rarg1_ptr_reg));
3261   match(iRegP);
3262   format %{ %}
3263   interface(REG_INTER);
3264 %}
3265 
3266 operand rarg2RegP() %{
3267   constraint(ALLOC_IN_RC(z_rarg2_ptr_reg));
3268   match(iRegP);
3269   format %{ %}
3270   interface(REG_INTER);
3271 %}
3272 
3273 operand rarg3RegP() %{
3274   constraint(ALLOC_IN_RC(z_rarg3_ptr_reg));
3275   match(iRegP);
3276   format %{ %}
3277   interface(REG_INTER);
3278 %}
3279 
3280 operand rarg4RegP() %{
3281   constraint(ALLOC_IN_RC(z_rarg4_ptr_reg));
3282   match(iRegP);
3283   format %{ %}
3284   interface(REG_INTER);
3285 %}
3286 
3287 operand rarg5RegP() %{
3288   constraint(ALLOC_IN_RC(z_rarg5_ptr_reg));
3289   match(iRegP);
3290   format %{ %}
3291   interface(REG_INTER);
3292 %}
3293 
3294 operand memoryRegP() %{
3295   constraint(ALLOC_IN_RC(z_memory_ptr_reg));
3296   match(RegP);
3297   match(iRegP);
3298   match(threadRegP);
3299   format %{ %}
3300   interface(REG_INTER);
3301 %}
3302 
3303 // revenRegP and roddRegP constitute and even-odd-pair.
3304 operand revenRegP() %{
3305   constraint(ALLOC_IN_RC(z_rarg3_ptr_reg));
3306   match(iRegP);
3307   format %{ %}
3308   interface(REG_INTER);
3309 %}
3310 
3311 // revenRegP and roddRegP constitute and even-odd-pair.
3312 operand roddRegP() %{
3313   constraint(ALLOC_IN_RC(z_rarg4_ptr_reg));
3314   match(iRegP);
3315   format %{ %}
3316   interface(REG_INTER);
3317 %}
3318 
3319 operand lock_ptr_RegP() %{
3320   constraint(ALLOC_IN_RC(z_lock_ptr_reg));
3321   match(RegP);
3322   format %{ %}
3323   interface(REG_INTER);
3324 %}
3325 
3326 operand rscratch2RegP() %{
3327   constraint(ALLOC_IN_RC(z_rscratch2_bits64_reg));
3328   match(RegP);
3329   format %{ %}
3330   interface(REG_INTER);
3331 %}
3332 
3333 operand iRegN() %{
3334   constraint(ALLOC_IN_RC(z_int_reg));
3335   match(RegN);
3336   match(noArg_iRegN);
3337   match(rarg1RegN);
3338   match(rarg2RegN);
3339   match(rarg3RegN);
3340   match(rarg4RegN);
3341   match(rarg5RegN);
3342   format %{ %}
3343   interface(REG_INTER);
3344 %}
3345 
3346 operand noArg_iRegN() %{
3347   constraint(ALLOC_IN_RC(z_no_arg_int_reg));
3348   match(iRegN);
3349   format %{ %}
3350   interface(REG_INTER);
3351 %}
3352 
3353 operand rarg1RegN() %{
3354   constraint(ALLOC_IN_RC(z_rarg1_int_reg));
3355   match(iRegN);
3356   format %{ %}
3357   interface(REG_INTER);
3358 %}
3359 
3360 operand rarg2RegN() %{
3361   constraint(ALLOC_IN_RC(z_rarg2_int_reg));
3362   match(iRegN);
3363   format %{ %}
3364   interface(REG_INTER);
3365 %}
3366 
3367 operand rarg3RegN() %{
3368   constraint(ALLOC_IN_RC(z_rarg3_int_reg));
3369   match(iRegN);
3370   format %{ %}
3371   interface(REG_INTER);
3372 %}
3373 
3374 operand rarg4RegN() %{
3375   constraint(ALLOC_IN_RC(z_rarg4_int_reg));
3376   match(iRegN);
3377   format %{ %}
3378   interface(REG_INTER);
3379 %}
3380 
3381 operand rarg5RegN() %{
3382   constraint(ALLOC_IN_RC(z_rarg5_ptrN_reg));
3383   match(iRegN);
3384   format %{ %}
3385   interface(REG_INTER);
3386 %}
3387 
3388 // Long Register
3389 operand iRegL() %{
3390   constraint(ALLOC_IN_RC(z_long_reg));
3391   match(RegL);
3392   match(revenRegL);
3393   match(roddRegL);
3394   match(allRoddRegL);
3395   match(rarg1RegL);
3396   match(rarg5RegL);
3397   format %{ %}
3398   interface(REG_INTER);
3399 %}
3400 
3401 // revenRegL and roddRegL constitute and even-odd-pair.
3402 operand revenRegL() %{
3403   constraint(ALLOC_IN_RC(z_rarg3_long_reg));
3404   match(iRegL);
3405   format %{ %}
3406   interface(REG_INTER);
3407 %}
3408 
3409 // revenRegL and roddRegL constitute and even-odd-pair.
3410 operand roddRegL() %{
3411   constraint(ALLOC_IN_RC(z_rarg4_long_reg));
3412   match(iRegL);
3413   format %{ %}
3414   interface(REG_INTER);
3415 %}
3416 
3417 // available odd registers for iRegL
3418 operand allRoddRegL() %{
3419   constraint(ALLOC_IN_RC(z_long_odd_reg));
3420   match(iRegL);
3421   format %{ %}
3422   interface(REG_INTER);
3423 %}
3424 
3425 operand rarg1RegL() %{
3426   constraint(ALLOC_IN_RC(z_rarg1_long_reg));
3427   match(iRegL);
3428   format %{ %}
3429   interface(REG_INTER);
3430 %}
3431 
3432 operand rarg5RegL() %{
3433   constraint(ALLOC_IN_RC(z_rarg5_long_reg));
3434   match(iRegL);
3435   format %{ %}
3436   interface(REG_INTER);
3437 %}
3438 
3439 // Condition Code Flag Registers
3440 operand flagsReg() %{
3441   constraint(ALLOC_IN_RC(z_condition_reg));
3442   match(RegFlags);
3443   format %{ "CR" %}
3444   interface(REG_INTER);
3445 %}
3446 
3447 // Condition Code Flag Registers for rules with result tuples
3448 operand TD_flagsReg() %{
3449   constraint(ALLOC_IN_RC(z_condition_reg));
3450   match(RegFlags);
3451   format %{ "CR" %}
3452   interface(REG_TUPLE_DEST_INTER);
3453 %}
3454 
3455 operand regD() %{
3456   constraint(ALLOC_IN_RC(z_dbl_reg));
3457   match(RegD);
3458   format %{ %}
3459   interface(REG_INTER);
3460 %}
3461 
3462 operand rscratchRegD() %{
3463   constraint(ALLOC_IN_RC(z_rscratch1_dbl_reg));
3464   match(RegD);
3465   format %{ %}
3466   interface(REG_INTER);
3467 %}
3468 
3469 operand regF() %{
3470   constraint(ALLOC_IN_RC(z_flt_reg));
3471   match(RegF);
3472   format %{ %}
3473   interface(REG_INTER);
3474 %}
3475 
3476 operand rscratchRegF() %{
3477   constraint(ALLOC_IN_RC(z_rscratch1_flt_reg));
3478   match(RegF);
3479   format %{ %}
3480   interface(REG_INTER);
3481 %}
3482 
3483 // Special Registers
3484 
3485 // Method Register
3486 operand inline_cache_regP(iRegP reg) %{
3487   constraint(ALLOC_IN_RC(z_r9_regP)); // inline_cache_reg
3488   match(reg);
3489   format %{ %}
3490   interface(REG_INTER);
3491 %}
3492 
3493 operand compiler_method_oop_regP(iRegP reg) %{
3494   constraint(ALLOC_IN_RC(z_r1_RegP)); // compiler_method_oop_reg
3495   match(reg);
3496   format %{ %}
3497   interface(REG_INTER);
3498 %}
3499 
3500 operand interpreter_method_oop_regP(iRegP reg) %{
3501   constraint(ALLOC_IN_RC(z_r9_regP)); // interpreter_method_oop_reg
3502   match(reg);
3503   format %{ %}
3504   interface(REG_INTER);
3505 %}
3506 
3507 // Operands to remove register moves in unscaled mode.
3508 // Match read/write registers with an EncodeP node if neither shift nor add are required.
3509 operand iRegP2N(iRegP reg) %{
3510   predicate(Universe::narrow_oop_shift() == 0 && _leaf->as_EncodeP()->in(0) == NULL);
3511   constraint(ALLOC_IN_RC(z_memory_ptr_reg));
3512   match(EncodeP reg);
3513   format %{ "$reg" %}
3514   interface(REG_INTER)
3515 %}
3516 
3517 operand iRegN2P(iRegN reg) %{
3518   predicate(Universe::narrow_oop_base() == NULL && Universe::narrow_oop_shift() == 0 &&
3519             _leaf->as_DecodeN()->in(0) == NULL);
3520   constraint(ALLOC_IN_RC(z_memory_ptr_reg));
3521   match(DecodeN reg);
3522   format %{ "$reg" %}
3523   interface(REG_INTER)
3524 %}
3525 
3526 
3527 //----------Complex Operands---------------------------------------------------
3528 
3529 // Indirect Memory Reference
3530 operand indirect(memoryRegP base) %{
3531   constraint(ALLOC_IN_RC(z_memory_ptr_reg));
3532   match(base);
3533   op_cost(1);
3534   format %{ "#0[,$base]" %}
3535   interface(MEMORY_INTER) %{
3536     base($base);
3537     index(0xffffFFFF); // noreg
3538     scale(0x0);
3539     disp(0x0);
3540   %}
3541 %}
3542 
3543 // Indirect with Offset (long)
3544 operand indOffset20(memoryRegP base, immL20 offset) %{
3545   constraint(ALLOC_IN_RC(z_memory_ptr_reg));
3546   match(AddP base offset);
3547   op_cost(1);
3548   format %{ "$offset[,$base]" %}
3549   interface(MEMORY_INTER) %{
3550     base($base);
3551     index(0xffffFFFF); // noreg
3552     scale(0x0);
3553     disp($offset);
3554   %}
3555 %}
3556 
3557 operand indOffset20Narrow(iRegN base, immL20 offset) %{
3558   predicate(Matcher::narrow_oop_use_complex_address());
3559   constraint(ALLOC_IN_RC(z_memory_ptr_reg));
3560   match(AddP (DecodeN base) offset);
3561   op_cost(1);
3562   format %{ "$offset[,$base]" %}
3563   interface(MEMORY_INTER) %{
3564     base($base);
3565     index(0xffffFFFF); // noreg
3566     scale(0x0);
3567     disp($offset);
3568   %}
3569 %}
3570 
3571 // Indirect with Offset (short)
3572 operand indOffset12(memoryRegP base, uimmL12 offset) %{
3573   constraint(ALLOC_IN_RC(z_memory_ptr_reg));
3574   match(AddP base offset);
3575   op_cost(1);
3576   format %{ "$offset[[,$base]]" %}
3577   interface(MEMORY_INTER) %{
3578     base($base);
3579     index(0xffffFFFF); // noreg
3580     scale(0x0);
3581     disp($offset);
3582   %}
3583 %}
3584 
3585 operand indOffset12Narrow(iRegN base, uimmL12 offset) %{
3586   predicate(Matcher::narrow_oop_use_complex_address());
3587   constraint(ALLOC_IN_RC(z_memory_ptr_reg));
3588   match(AddP (DecodeN base) offset);
3589   op_cost(1);
3590   format %{ "$offset[[,$base]]" %}
3591   interface(MEMORY_INTER) %{
3592     base($base);
3593     index(0xffffFFFF); // noreg
3594     scale(0x0);
3595     disp($offset);
3596   %}
3597 %}
3598 
3599 // Indirect with Register Index
3600 operand indIndex(memoryRegP base, iRegL index) %{
3601   constraint(ALLOC_IN_RC(z_memory_ptr_reg));
3602   match(AddP base index);
3603   op_cost(1);
3604   format %{ "#0[($index,$base)]" %}
3605   interface(MEMORY_INTER) %{
3606     base($base);
3607     index($index);
3608     scale(0x0);
3609     disp(0x0);
3610   %}
3611 %}
3612 
3613 // Indirect with Offset (long) and index
3614 operand indOffset20index(memoryRegP base, immL20 offset, iRegL index) %{
3615   constraint(ALLOC_IN_RC(z_memory_ptr_reg));
3616   match(AddP (AddP base index) offset);
3617   op_cost(1);
3618   format %{ "$offset[($index,$base)]" %}
3619   interface(MEMORY_INTER) %{
3620     base($base);
3621     index($index);
3622     scale(0x0);
3623     disp($offset);
3624   %}
3625 %}
3626 
3627 operand indOffset20indexNarrow(iRegN base, immL20 offset, iRegL index) %{
3628   predicate(Matcher::narrow_oop_use_complex_address());
3629   constraint(ALLOC_IN_RC(z_memory_ptr_reg));
3630   match(AddP (AddP (DecodeN base) index) offset);
3631   op_cost(1);
3632   format %{ "$offset[($index,$base)]" %}
3633   interface(MEMORY_INTER) %{
3634     base($base);
3635     index($index);
3636     scale(0x0);
3637     disp($offset);
3638   %}
3639 %}
3640 
3641 // Indirect with Offset (short) and index
3642 operand indOffset12index(memoryRegP base, uimmL12 offset, iRegL index) %{
3643   constraint(ALLOC_IN_RC(z_memory_ptr_reg));
3644   match(AddP (AddP base index) offset);
3645   op_cost(1);
3646   format %{ "$offset[[($index,$base)]]" %}
3647   interface(MEMORY_INTER) %{
3648     base($base);
3649     index($index);
3650     scale(0x0);
3651     disp($offset);
3652   %}
3653 %}
3654 
3655 operand indOffset12indexNarrow(iRegN base, uimmL12 offset, iRegL index) %{
3656   predicate(Matcher::narrow_oop_use_complex_address());
3657   constraint(ALLOC_IN_RC(z_memory_ptr_reg));
3658   match(AddP (AddP (DecodeN base) index) offset);
3659   op_cost(1);
3660   format %{ "$offset[[($index,$base)]]" %}
3661   interface(MEMORY_INTER) %{
3662     base($base);
3663     index($index);
3664     scale(0x0);
3665     disp($offset);
3666   %}
3667 %}
3668 
3669 //----------Special Memory Operands--------------------------------------------
3670 
3671 // Stack Slot Operand
3672 // This operand is used for loading and storing temporary values on
3673 // the stack where a match requires a value to flow through memory.
3674 operand stackSlotI(sRegI reg) %{
3675   constraint(ALLOC_IN_RC(stack_slots));
3676   op_cost(1);
3677   format %{ "[$reg(stackSlotI)]" %}
3678   interface(MEMORY_INTER) %{
3679     base(0xf);   // Z_SP
3680     index(0xffffFFFF); // noreg
3681     scale(0x0);
3682     disp($reg);  // stack offset
3683   %}
3684 %}
3685 
3686 operand stackSlotP(sRegP reg) %{
3687   constraint(ALLOC_IN_RC(stack_slots));
3688   op_cost(1);
3689   format %{ "[$reg(stackSlotP)]" %}
3690   interface(MEMORY_INTER) %{
3691     base(0xf);   // Z_SP
3692     index(0xffffFFFF); // noreg
3693     scale(0x0);
3694     disp($reg);  // Stack Offset
3695   %}
3696 %}
3697 
3698 operand stackSlotF(sRegF reg) %{
3699   constraint(ALLOC_IN_RC(stack_slots));
3700   op_cost(1);
3701   format %{ "[$reg(stackSlotF)]" %}
3702   interface(MEMORY_INTER) %{
3703     base(0xf);   // Z_SP
3704     index(0xffffFFFF); // noreg
3705     scale(0x0);
3706     disp($reg);  // Stack Offset
3707   %}
3708 %}
3709 
3710 operand stackSlotD(sRegD reg) %{
3711   constraint(ALLOC_IN_RC(stack_slots));
3712   op_cost(1);
3713   //match(RegD);
3714   format %{ "[$reg(stackSlotD)]" %}
3715   interface(MEMORY_INTER) %{
3716     base(0xf);   // Z_SP
3717     index(0xffffFFFF); // noreg
3718     scale(0x0);
3719     disp($reg);  // Stack Offset
3720   %}
3721 %}
3722 
3723 operand stackSlotL(sRegL reg) %{
3724   constraint(ALLOC_IN_RC(stack_slots));
3725   op_cost(1);  //match(RegL);
3726   format %{ "[$reg(stackSlotL)]" %}
3727   interface(MEMORY_INTER) %{
3728     base(0xf);   // Z_SP
3729     index(0xffffFFFF); // noreg
3730     scale(0x0);
3731     disp($reg);  // Stack Offset
3732   %}
3733 %}
3734 
3735 // Operands for expressing Control Flow
3736 // NOTE: Label is a predefined operand which should not be redefined in
3737 // the AD file. It is generically handled within the ADLC.
3738 
3739 //----------Conditional Branch Operands----------------------------------------
3740 // Comparison Op  - This is the operation of the comparison, and is limited to
3741 //                  the following set of codes:
3742 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
3743 //
3744 // Other attributes of the comparison, such as unsignedness, are specified
3745 // by the comparison instruction that sets a condition code flags register.
3746 // That result is represented by a flags operand whose subtype is appropriate
3747 // to the unsignedness (etc.) of the comparison.
3748 //
3749 // Later, the instruction which matches both the Comparison Op (a Bool) and
3750 // the flags (produced by the Cmp) specifies the coding of the comparison op
3751 // by matching a specific subtype of Bool operand below.
3752 
3753 // INT cmpOps for CompareAndBranch and CompareAndTrap instructions should not
3754 // have mask bit #3 set.
3755 operand cmpOpT() %{
3756   match(Bool);
3757   format %{ "" %}
3758   interface(COND_INTER) %{
3759     equal(0x8);         // Assembler::bcondEqual
3760     not_equal(0x6);     // Assembler::bcondNotEqual
3761     less(0x4);          // Assembler::bcondLow
3762     greater_equal(0xa); // Assembler::bcondNotLow
3763     less_equal(0xc);    // Assembler::bcondNotHigh
3764     greater(0x2);       // Assembler::bcondHigh
3765     overflow(0x1);      // Assembler::bcondOverflow
3766     no_overflow(0xe);   // Assembler::bcondNotOverflow
3767   %}
3768 %}
3769 
3770 // When used for floating point comparisons: unordered is treated as less.
3771 operand cmpOpF() %{
3772   match(Bool);
3773   format %{ "" %}
3774   interface(COND_INTER) %{
3775     equal(0x8);
3776     not_equal(0x7);     // Includes 'unordered'.
3777     less(0x5);          // Includes 'unordered'.
3778     greater_equal(0xa);
3779     less_equal(0xd);    // Includes 'unordered'.
3780     greater(0x2);
3781     overflow(0x0);      // Not meaningful on z/Architecture.
3782     no_overflow(0x0);   // leave unchanged (zero) therefore
3783   %}
3784 %}
3785 
3786 // "Regular" cmpOp for int comparisons, includes bit #3 (overflow).
3787 operand cmpOp() %{
3788   match(Bool);
3789   format %{ "" %}
3790   interface(COND_INTER) %{
3791     equal(0x8);
3792     not_equal(0x7);     // Includes 'unordered'.
3793     less(0x5);          // Includes 'unordered'.
3794     greater_equal(0xa);
3795     less_equal(0xd);    // Includes 'unordered'.
3796     greater(0x2);
3797     overflow(0x1);      // Assembler::bcondOverflow
3798     no_overflow(0xe);   // Assembler::bcondNotOverflow
3799   %}
3800 %}
3801 
3802 //----------OPERAND CLASSES----------------------------------------------------
3803 // Operand Classes are groups of operands that are used to simplify
3804 // instruction definitions by not requiring the AD writer to specify
3805 // seperate instructions for every form of operand when the
3806 // instruction accepts multiple operand types with the same basic
3807 // encoding and format.  The classic case of this is memory operands.
3808 // Indirect is not included since its use is limited to Compare & Swap
3809 
3810 // Most general memory operand, allows base, index, and long displacement.
3811 opclass memory(indirect, indIndex, indOffset20, indOffset20Narrow, indOffset20index, indOffset20indexNarrow);
3812 opclass memoryRXY(indirect, indIndex, indOffset20, indOffset20Narrow, indOffset20index, indOffset20indexNarrow);
3813 
3814 // General memory operand, allows base, index, and short displacement.
3815 opclass memoryRX(indirect, indIndex, indOffset12, indOffset12Narrow, indOffset12index, indOffset12indexNarrow);
3816 
3817 // Memory operand, allows only base and long displacement.
3818 opclass memoryRSY(indirect, indOffset20, indOffset20Narrow);
3819 
3820 // Memory operand, allows only base and short displacement.
3821 opclass memoryRS(indirect, indOffset12, indOffset12Narrow);
3822 
3823 // Operand classes to match encode and decode.
3824 opclass iRegN_P2N(iRegN);
3825 opclass iRegP_N2P(iRegP);
3826 
3827 
3828 //----------PIPELINE-----------------------------------------------------------
3829 pipeline %{
3830 
3831 //----------ATTRIBUTES---------------------------------------------------------
3832 attributes %{
3833   // z/Architecture instructions are of length 2, 4, or 6 bytes.
3834   variable_size_instructions;
3835   instruction_unit_size = 2;
3836 
3837   // Meaningless on z/Architecture.
3838   max_instructions_per_bundle = 1;
3839 
3840   // The z/Architecture processor fetches 64 bytes...
3841   instruction_fetch_unit_size = 64;
3842 
3843   // ...in one line.
3844   instruction_fetch_units = 1
3845 %}
3846 
3847 //----------RESOURCES----------------------------------------------------------
3848 // Resources are the functional units available to the machine.
3849 resources(
3850    Z_BR,     // branch unit
3851    Z_CR,     // condition unit
3852    Z_FX1,    // integer arithmetic unit 1
3853    Z_FX2,    // integer arithmetic unit 2
3854    Z_LDST1,  // load/store unit 1
3855    Z_LDST2,  // load/store unit 2
3856    Z_FP1,    // float arithmetic unit 1
3857    Z_FP2,    // float arithmetic unit 2
3858    Z_LDST = Z_LDST1 | Z_LDST2,
3859    Z_FX   = Z_FX1 | Z_FX2,
3860    Z_FP   = Z_FP1 | Z_FP2
3861   );
3862 
3863 //----------PIPELINE DESCRIPTION-----------------------------------------------
3864 // Pipeline Description specifies the stages in the machine's pipeline.
3865 pipe_desc(
3866    // TODO: adapt
3867    Z_IF,  // instruction fetch
3868    Z_IC,
3869    Z_D0,  // decode
3870    Z_D1,  // decode
3871    Z_D2,  // decode
3872    Z_D3,  // decode
3873    Z_Xfer1,
3874    Z_GD,  // group definition
3875    Z_MP,  // map
3876    Z_ISS, // issue
3877    Z_RF,  // resource fetch
3878    Z_EX1, // execute (all units)
3879    Z_EX2, // execute (FP, LDST)
3880    Z_EX3, // execute (FP, LDST)
3881    Z_EX4, // execute (FP)
3882    Z_EX5, // execute (FP)
3883    Z_EX6, // execute (FP)
3884    Z_WB,  // write back
3885    Z_Xfer2,
3886    Z_CP
3887   );
3888 
3889 //----------PIPELINE CLASSES---------------------------------------------------
3890 // Pipeline Classes describe the stages in which input and output are
3891 // referenced by the hardware pipeline.
3892 
3893 // Providing the `ins_pipe' declarations in the instruction
3894 // specifications seems to be of little use. So we use
3895 // `pipe_class_dummy' for all our instructions at present.
3896 pipe_class pipe_class_dummy() %{
3897   single_instruction;
3898   fixed_latency(4);
3899 %}
3900 
3901 // SIGTRAP based implicit range checks in compiled code.
3902 // Currently, no pipe classes are used on z/Architecture.
3903 pipe_class pipe_class_trap() %{
3904   single_instruction;
3905 %}
3906 
3907 pipe_class pipe_class_fx_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
3908   single_instruction;
3909   dst  : Z_EX1(write);
3910   src1 : Z_RF(read);
3911   src2 : Z_RF(read);
3912   Z_FX : Z_RF;
3913 %}
3914 
3915 pipe_class pipe_class_ldst(iRegP dst, memory mem) %{
3916   single_instruction;
3917   mem : Z_RF(read);
3918   dst : Z_WB(write);
3919   Z_LDST : Z_RF;
3920 %}
3921 
3922 define %{
3923   MachNop = pipe_class_dummy;
3924 %}
3925 
3926 %}
3927 
3928 //----------INSTRUCTIONS-------------------------------------------------------
3929 
3930 //---------- Chain stack slots between similar types --------
3931 
3932 // Load integer from stack slot.
3933 instruct stkI_to_regI(iRegI dst, stackSlotI src) %{
3934   match(Set dst src);
3935   ins_cost(MEMORY_REF_COST);
3936   // TODO: s390 port size(FIXED_SIZE);
3937   format %{ "L       $dst,$src\t # stk reload int" %}
3938   opcode(L_ZOPC);
3939   ins_encode(z_form_rt_mem(dst, src));
3940   ins_pipe(pipe_class_dummy);
3941 %}
3942 
3943 // Store integer to stack slot.
3944 instruct regI_to_stkI(stackSlotI dst, iRegI src) %{
3945   match(Set dst src);
3946   ins_cost(MEMORY_REF_COST);
3947   // TODO: s390 port size(FIXED_SIZE);
3948   format %{ "ST      $src,$dst\t # stk spill int" %}
3949   opcode(ST_ZOPC);
3950   ins_encode(z_form_rt_mem(src, dst)); // rs=rt
3951   ins_pipe(pipe_class_dummy);
3952 %}
3953 
3954 // Load long from stack slot.
3955 instruct stkL_to_regL(iRegL dst, stackSlotL src) %{
3956   match(Set dst src);
3957   ins_cost(MEMORY_REF_COST);
3958   // TODO: s390 port size(FIXED_SIZE);
3959   format %{ "LG      $dst,$src\t # stk reload long" %}
3960   opcode(LG_ZOPC);
3961   ins_encode(z_form_rt_mem(dst, src));
3962   ins_pipe(pipe_class_dummy);
3963 %}
3964 
3965 // Store long to stack slot.
3966 instruct regL_to_stkL(stackSlotL dst, iRegL src) %{
3967   match(Set dst src);
3968   ins_cost(MEMORY_REF_COST);
3969   size(6);
3970   format %{ "STG     $src,$dst\t # stk spill long" %}
3971   opcode(STG_ZOPC);
3972   ins_encode(z_form_rt_mem(src, dst)); // rs=rt
3973   ins_pipe(pipe_class_dummy);
3974 %}
3975 
3976 // Load pointer from stack slot, 64-bit encoding.
3977 instruct stkP_to_regP(iRegP dst, stackSlotP src) %{
3978   match(Set dst src);
3979   ins_cost(MEMORY_REF_COST);
3980   // TODO: s390 port size(FIXED_SIZE);
3981   format %{ "LG      $dst,$src\t # stk reload ptr" %}
3982   opcode(LG_ZOPC);
3983   ins_encode(z_form_rt_mem(dst, src));
3984   ins_pipe(pipe_class_dummy);
3985 %}
3986 
3987 // Store pointer to stack slot.
3988 instruct regP_to_stkP(stackSlotP dst, iRegP src) %{
3989   match(Set dst src);
3990   ins_cost(MEMORY_REF_COST);
3991   // TODO: s390 port size(FIXED_SIZE);
3992   format %{ "STG     $src,$dst\t # stk spill ptr" %}
3993   opcode(STG_ZOPC);
3994   ins_encode(z_form_rt_mem(src, dst)); // rs=rt
3995   ins_pipe(pipe_class_dummy);
3996 %}
3997 
3998 //  Float types
3999 
4000 // Load float value from stack slot.
4001 instruct stkF_to_regF(regF dst, stackSlotF src) %{
4002   match(Set dst src);
4003   ins_cost(MEMORY_REF_COST);
4004   size(4);
4005   format %{ "LE(Y)   $dst,$src\t # stk reload float" %}
4006   opcode(LE_ZOPC);
4007   ins_encode(z_form_rt_mem(dst, src));
4008   ins_pipe(pipe_class_dummy);
4009 %}
4010 
4011 // Store float value to stack slot.
4012 instruct regF_to_stkF(stackSlotF dst, regF src) %{
4013   match(Set dst src);
4014   ins_cost(MEMORY_REF_COST);
4015   size(4);
4016   format %{ "STE(Y)  $src,$dst\t # stk spill float" %}
4017   opcode(STE_ZOPC);
4018   ins_encode(z_form_rt_mem(src, dst));
4019   ins_pipe(pipe_class_dummy);
4020 %}
4021 
4022 // Load double value from stack slot.
4023 instruct stkD_to_regD(regD dst, stackSlotD src) %{
4024   match(Set dst src);
4025   ins_cost(MEMORY_REF_COST);
4026   // TODO: s390 port size(FIXED_SIZE);
4027   format %{ "LD(Y)   $dst,$src\t # stk reload double" %}
4028   opcode(LD_ZOPC);
4029   ins_encode(z_form_rt_mem(dst, src));
4030   ins_pipe(pipe_class_dummy);
4031 %}
4032 
4033 // Store double value to stack slot.
4034 instruct regD_to_stkD(stackSlotD dst, regD src) %{
4035   match(Set dst src);
4036   ins_cost(MEMORY_REF_COST);
4037   size(4);
4038   format %{ "STD(Y)  $src,$dst\t # stk spill double" %}
4039   opcode(STD_ZOPC);
4040   ins_encode(z_form_rt_mem(src, dst));
4041   ins_pipe(pipe_class_dummy);
4042 %}
4043 
4044 //----------Load/Store/Move Instructions---------------------------------------
4045 
4046 //----------Load Instructions--------------------------------------------------
4047 
4048 //------------------
4049 //  MEMORY
4050 //------------------
4051 
4052 //  BYTE
4053 // Load Byte (8bit signed)
4054 instruct loadB(iRegI dst, memory mem) %{
4055   match(Set dst (LoadB mem));
4056   ins_cost(MEMORY_REF_COST);
4057   size(Z_DISP3_SIZE);
4058   format %{ "LB      $dst, $mem\t # sign-extend byte to int" %}
4059   opcode(LB_ZOPC, LB_ZOPC);
4060   ins_encode(z_form_rt_mem_opt(dst, mem));
4061   ins_pipe(pipe_class_dummy);
4062 %}
4063 
4064 // Load Byte (8bit signed)
4065 instruct loadB2L(iRegL dst, memory mem) %{
4066   match(Set dst (ConvI2L (LoadB mem)));
4067   ins_cost(MEMORY_REF_COST);
4068   size(Z_DISP3_SIZE);
4069   format %{ "LGB     $dst, $mem\t # sign-extend byte to long" %}
4070   opcode(LGB_ZOPC, LGB_ZOPC);
4071   ins_encode(z_form_rt_mem_opt(dst, mem));
4072   ins_pipe(pipe_class_dummy);
4073 %}
4074 
4075 // Load Unsigned Byte (8bit UNsigned) into an int reg.
4076 instruct loadUB(iRegI dst, memory mem) %{
4077   match(Set dst (LoadUB mem));
4078   ins_cost(MEMORY_REF_COST);
4079   size(Z_DISP3_SIZE);
4080   format %{ "LLGC    $dst,$mem\t # zero-extend byte to int" %}
4081   opcode(LLGC_ZOPC, LLGC_ZOPC);
4082   ins_encode(z_form_rt_mem_opt(dst, mem));
4083   ins_pipe(pipe_class_dummy);
4084 %}
4085 
4086 // Load Unsigned Byte (8bit UNsigned) into a Long Register.
4087 instruct loadUB2L(iRegL dst, memory mem) %{
4088   match(Set dst (ConvI2L (LoadUB mem)));
4089   ins_cost(MEMORY_REF_COST);
4090   size(Z_DISP3_SIZE);
4091   format %{ "LLGC    $dst,$mem\t # zero-extend byte to long" %}
4092   opcode(LLGC_ZOPC, LLGC_ZOPC);
4093   ins_encode(z_form_rt_mem_opt(dst, mem));
4094   ins_pipe(pipe_class_dummy);
4095 %}
4096 
4097 // CHAR/SHORT
4098 
4099 // Load Short (16bit signed)
4100 instruct loadS(iRegI dst, memory mem) %{
4101   match(Set dst (LoadS mem));
4102   ins_cost(MEMORY_REF_COST);
4103   size(Z_DISP_SIZE);
4104   format %{ "LH(Y)   $dst,$mem\t # sign-extend short to int" %}
4105   opcode(LHY_ZOPC, LH_ZOPC);
4106   ins_encode(z_form_rt_mem_opt(dst, mem));
4107   ins_pipe(pipe_class_dummy);
4108 %}
4109 
4110 // Load Short (16bit signed)
4111 instruct loadS2L(iRegL dst, memory mem) %{
4112   match(Set dst (ConvI2L (LoadS mem)));
4113   ins_cost(MEMORY_REF_COST);
4114   size(Z_DISP3_SIZE);
4115   format %{ "LGH     $dst,$mem\t # sign-extend short to long" %}
4116   opcode(LGH_ZOPC, LGH_ZOPC);
4117   ins_encode(z_form_rt_mem_opt(dst, mem));
4118   ins_pipe(pipe_class_dummy);
4119 %}
4120 
4121 // Load Char (16bit Unsigned)
4122 instruct loadUS(iRegI dst, memory mem) %{
4123   match(Set dst (LoadUS mem));
4124   ins_cost(MEMORY_REF_COST);
4125   size(Z_DISP3_SIZE);
4126   format %{ "LLGH    $dst,$mem\t # zero-extend short to int" %}
4127   opcode(LLGH_ZOPC, LLGH_ZOPC);
4128   ins_encode(z_form_rt_mem_opt(dst, mem));
4129   ins_pipe(pipe_class_dummy);
4130 %}
4131 
4132 // Load Unsigned Short/Char (16bit UNsigned) into a Long Register.
4133 instruct loadUS2L(iRegL dst, memory mem) %{
4134   match(Set dst (ConvI2L (LoadUS mem)));
4135   ins_cost(MEMORY_REF_COST);
4136   size(Z_DISP3_SIZE);
4137   format %{ "LLGH    $dst,$mem\t # zero-extend short to long" %}
4138   opcode(LLGH_ZOPC, LLGH_ZOPC);
4139   ins_encode(z_form_rt_mem_opt(dst, mem));
4140   ins_pipe(pipe_class_dummy);
4141 %}
4142 
4143 // INT
4144 
4145 // Load Integer
4146 instruct loadI(iRegI dst, memory mem) %{
4147   match(Set dst (LoadI mem));
4148   ins_cost(MEMORY_REF_COST);
4149   size(Z_DISP_SIZE);
4150   format %{ "L(Y)    $dst,$mem\t #" %}
4151   opcode(LY_ZOPC, L_ZOPC);
4152   ins_encode(z_form_rt_mem_opt(dst, mem));
4153   ins_pipe(pipe_class_dummy);
4154 %}
4155 
4156 // Load and convert to long.
4157 instruct loadI2L(iRegL dst, memory mem) %{
4158   match(Set dst (ConvI2L (LoadI mem)));
4159   ins_cost(MEMORY_REF_COST);
4160   size(Z_DISP3_SIZE);
4161   format %{ "LGF     $dst,$mem\t #" %}
4162   opcode(LGF_ZOPC, LGF_ZOPC);
4163   ins_encode(z_form_rt_mem_opt(dst, mem));
4164   ins_pipe(pipe_class_dummy);
4165 %}
4166 
4167 // Load Unsigned Integer into a Long Register
4168 instruct loadUI2L(iRegL dst, memory mem, immL_FFFFFFFF mask) %{
4169   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
4170   ins_cost(MEMORY_REF_COST);
4171   size(Z_DISP3_SIZE);
4172   format %{ "LLGF    $dst,$mem\t # zero-extend int to long" %}
4173   opcode(LLGF_ZOPC, LLGF_ZOPC);
4174   ins_encode(z_form_rt_mem_opt(dst, mem));
4175   ins_pipe(pipe_class_dummy);
4176 %}
4177 
4178 // range = array length (=jint)
4179 // Load Range
4180 instruct loadRange(iRegI dst, memory mem) %{
4181   match(Set dst (LoadRange mem));
4182   ins_cost(MEMORY_REF_COST);
4183   size(Z_DISP_SIZE);
4184   format %{ "L(Y)    $dst,$mem\t # range" %}
4185   opcode(LY_ZOPC, L_ZOPC);
4186   ins_encode(z_form_rt_mem_opt(dst, mem));
4187   ins_pipe(pipe_class_dummy);
4188 %}
4189 
4190 // LONG
4191 
4192 // Load Long - aligned
4193 instruct loadL(iRegL dst, memory mem) %{
4194   match(Set dst (LoadL mem));
4195   ins_cost(MEMORY_REF_COST);
4196   size(Z_DISP3_SIZE);
4197   format %{ "LG      $dst,$mem\t # long" %}
4198   opcode(LG_ZOPC, LG_ZOPC);
4199   ins_encode(z_form_rt_mem_opt(dst, mem));
4200   ins_pipe(pipe_class_dummy);
4201 %}
4202 
4203 // Load Long - UNaligned
4204 instruct loadL_unaligned(iRegL dst, memory mem) %{
4205   match(Set dst (LoadL_unaligned mem));
4206   ins_cost(MEMORY_REF_COST);
4207   size(Z_DISP3_SIZE);
4208   format %{ "LG      $dst,$mem\t # unaligned long" %}
4209   opcode(LG_ZOPC, LG_ZOPC);
4210   ins_encode(z_form_rt_mem_opt(dst, mem));
4211   ins_pipe(pipe_class_dummy);
4212 %}
4213 
4214 
4215 // PTR
4216 
4217 // Load Pointer
4218 instruct loadP(iRegP dst, memory mem) %{
4219   match(Set dst (LoadP mem));
4220   ins_cost(MEMORY_REF_COST);
4221   size(Z_DISP3_SIZE);
4222   format %{ "LG      $dst,$mem\t # ptr" %}
4223   opcode(LG_ZOPC, LG_ZOPC);
4224   ins_encode(z_form_rt_mem_opt(dst, mem));
4225   ins_pipe(pipe_class_dummy);
4226 %}
4227 
4228 // LoadP + CastP2L
4229 instruct castP2X_loadP(iRegL dst, memory mem) %{
4230   match(Set dst (CastP2X (LoadP mem)));
4231   ins_cost(MEMORY_REF_COST);
4232   size(Z_DISP3_SIZE);
4233   format %{ "LG      $dst,$mem\t # ptr + p2x" %}
4234   opcode(LG_ZOPC, LG_ZOPC);
4235   ins_encode(z_form_rt_mem_opt(dst, mem));
4236   ins_pipe(pipe_class_dummy);
4237 %}
4238 
4239 // Load Klass Pointer
4240 instruct loadKlass(iRegP dst, memory mem) %{
4241   match(Set dst (LoadKlass mem));
4242   ins_cost(MEMORY_REF_COST);
4243   size(Z_DISP3_SIZE);
4244   format %{ "LG      $dst,$mem\t # klass ptr" %}
4245   opcode(LG_ZOPC, LG_ZOPC);
4246   ins_encode(z_form_rt_mem_opt(dst, mem));
4247   ins_pipe(pipe_class_dummy);
4248 %}
4249 
4250 instruct loadTOC(iRegL dst) %{
4251   effect(DEF dst);
4252   ins_cost(DEFAULT_COST);
4253   // TODO: s390 port size(FIXED_SIZE);
4254   // TODO: check why this attribute causes many unnecessary rematerializations.
4255   //
4256   // The graphs I saw just had high register pressure. Further the
4257   // register TOC is loaded to is overwritten by the constant short
4258   // after. Here something as round robin register allocation might
4259   // help. But rematerializing seems not to hurt, jack even seems to
4260   // improve slightly.
4261   //
4262   // Without this flag we get spill-split recycle sanity check
4263   // failures in
4264   // spec.benchmarks._228_jack.NfaState::GenerateCode. This happens in
4265   // a block with three loadConP_dynTOC nodes and a tlsLoadP. The
4266   // tlsLoadP has a huge amount of outs and forces the TOC down to the
4267   // stack. Later tlsLoadP is rematerialized, leaving the register
4268   // allocator with TOC on the stack and a badly placed reload.
4269   ins_should_rematerialize(true);
4270   format %{ "LARL    $dst, &constant_pool\t; load dynTOC" %}
4271   ins_encode %{ __ load_toc($dst$$Register); %}
4272   ins_pipe(pipe_class_dummy);
4273 %}
4274 
4275 // FLOAT
4276 
4277 // Load Float
4278 instruct loadF(regF dst, memory mem) %{
4279   match(Set dst (LoadF mem));
4280   ins_cost(MEMORY_REF_COST);
4281   size(Z_DISP_SIZE);
4282   format %{ "LE(Y)    $dst,$mem" %}
4283   opcode(LEY_ZOPC, LE_ZOPC);
4284   ins_encode(z_form_rt_mem_opt(dst, mem));
4285   ins_pipe(pipe_class_dummy);
4286 %}
4287 
4288 // DOUBLE
4289 
4290 // Load Double
4291 instruct loadD(regD dst, memory mem) %{
4292   match(Set dst (LoadD mem));
4293   ins_cost(MEMORY_REF_COST);
4294   size(Z_DISP_SIZE);
4295   format %{ "LD(Y)    $dst,$mem" %}
4296   opcode(LDY_ZOPC, LD_ZOPC);
4297   ins_encode(z_form_rt_mem_opt(dst, mem));
4298   ins_pipe(pipe_class_dummy);
4299 %}
4300 
4301 // Load Double - UNaligned
4302 instruct loadD_unaligned(regD dst, memory mem) %{
4303   match(Set dst (LoadD_unaligned mem));
4304   ins_cost(MEMORY_REF_COST);
4305   size(Z_DISP_SIZE);
4306   format %{ "LD(Y)    $dst,$mem" %}
4307   opcode(LDY_ZOPC, LD_ZOPC);
4308   ins_encode(z_form_rt_mem_opt(dst, mem));
4309   ins_pipe(pipe_class_dummy);
4310 %}
4311 
4312 
4313 //----------------------
4314 //  IMMEDIATES
4315 //----------------------
4316 
4317 instruct loadConI(iRegI dst, immI src) %{
4318   match(Set dst src);
4319   ins_cost(DEFAULT_COST);
4320   size(6);
4321   format %{ "LGFI     $dst,$src\t # (int)" %}
4322   ins_encode %{ __ z_lgfi($dst$$Register, $src$$constant); %}  // Sign-extend to 64 bit, it's at no cost.
4323   ins_pipe(pipe_class_dummy);
4324 %}
4325 
4326 instruct loadConI16(iRegI dst, immI16 src) %{
4327   match(Set dst src);
4328   ins_cost(DEFAULT_COST_LOW);
4329   size(4);
4330   format %{ "LGHI     $dst,$src\t # (int)" %}
4331   ins_encode %{ __ z_lghi($dst$$Register, $src$$constant); %}  // Sign-extend to 64 bit, it's at no cost.
4332   ins_pipe(pipe_class_dummy);
4333 %}
4334 
4335 instruct loadConI_0(iRegI dst, immI_0 src, flagsReg cr) %{
4336   match(Set dst src);
4337   effect(KILL cr);
4338   ins_cost(DEFAULT_COST_LOW);
4339   size(4);
4340   format %{ "loadConI $dst,$src\t # (int) XGR because ZERO is loaded" %}
4341   opcode(XGR_ZOPC);
4342   ins_encode(z_rreform(dst, dst));
4343   ins_pipe(pipe_class_dummy);
4344 %}
4345 
4346 instruct loadConUI16(iRegI dst, uimmI16 src) %{
4347   match(Set dst src);
4348   // TODO: s390 port size(FIXED_SIZE);
4349   format %{ "LLILL    $dst,$src" %}
4350   opcode(LLILL_ZOPC);
4351   ins_encode(z_riform_unsigned(dst, src) );
4352   ins_pipe(pipe_class_dummy);
4353 %}
4354 
4355 // Load long constant from TOC with pcrelative address.
4356 instruct loadConL_pcrelTOC(iRegL dst, immL src) %{
4357   match(Set dst src);
4358   ins_cost(MEMORY_REF_COST_LO);
4359   size(6);
4360   format %{ "LGRL    $dst,[pcrelTOC]\t # load long $src from table" %}
4361   ins_encode %{
4362     address long_address = __ long_constant($src$$constant);
4363     if (long_address == NULL) {
4364       Compile::current()->env()->record_out_of_memory_failure();
4365       return;
4366     }
4367     __ load_long_pcrelative($dst$$Register, long_address);
4368   %}
4369   ins_pipe(pipe_class_dummy);
4370 %}
4371 
4372 instruct loadConL32(iRegL dst, immL32 src) %{
4373   match(Set dst src);
4374   ins_cost(DEFAULT_COST);
4375   size(6);
4376   format %{ "LGFI     $dst,$src\t # (long)" %}
4377   ins_encode %{ __ z_lgfi($dst$$Register, $src$$constant); %}  // Sign-extend to 64 bit, it's at no cost.
4378   ins_pipe(pipe_class_dummy);
4379 %}
4380 
4381 instruct loadConL16(iRegL dst, immL16 src) %{
4382   match(Set dst src);
4383   ins_cost(DEFAULT_COST_LOW);
4384   size(4);
4385   format %{ "LGHI     $dst,$src\t # (long)" %}
4386   ins_encode %{ __ z_lghi($dst$$Register, $src$$constant); %}  // Sign-extend to 64 bit, it's at no cost.
4387   ins_pipe(pipe_class_dummy);
4388 %}
4389 
4390 instruct loadConL_0(iRegL dst, immL_0 src, flagsReg cr) %{
4391   match(Set dst src);
4392   effect(KILL cr);
4393   ins_cost(DEFAULT_COST_LOW);
4394   format %{ "LoadConL    $dst,$src\t # (long) XGR because ZERO is loaded" %}
4395   opcode(XGR_ZOPC);
4396   ins_encode(z_rreform(dst, dst));
4397   ins_pipe(pipe_class_dummy);
4398 %}
4399 
4400 // Load ptr constant from TOC with pc relative address.
4401 // Special handling for oop constants required.
4402 instruct loadConP_pcrelTOC(iRegP dst, immP src) %{
4403   match(Set dst src);
4404   ins_cost(MEMORY_REF_COST_LO);
4405   size(6);
4406   format %{ "LGRL    $dst,[pcrelTOC]\t # load ptr $src from table" %}
4407   ins_encode %{
4408     relocInfo::relocType constant_reloc = $src->constant_reloc();
4409     if (constant_reloc == relocInfo::oop_type) {
4410       AddressLiteral a = __ allocate_oop_address((jobject)$src$$constant);
4411       bool success = __ load_oop_from_toc($dst$$Register, a);
4412       if (!success) {
4413         Compile::current()->env()->record_out_of_memory_failure();
4414         return;
4415       }
4416     } else if (constant_reloc == relocInfo::metadata_type) {
4417       AddressLiteral a = __ constant_metadata_address((Metadata *)$src$$constant);
4418       address const_toc_addr = __ address_constant((address)a.value(), RelocationHolder::none);
4419       if (const_toc_addr == NULL) {
4420         Compile::current()->env()->record_out_of_memory_failure();
4421         return;
4422       }
4423       __ load_long_pcrelative($dst$$Register, const_toc_addr);
4424     } else {          // Non-oop pointers, e.g. card mark base, heap top.
4425       address long_address = __ long_constant((jlong)$src$$constant);
4426       if (long_address == NULL) {
4427         Compile::current()->env()->record_out_of_memory_failure();
4428         return;
4429       }
4430       __ load_long_pcrelative($dst$$Register, long_address);
4431     }
4432   %}
4433   ins_pipe(pipe_class_dummy);
4434 %}
4435 
4436 // We don't use immP16 to avoid problems with oops.
4437 instruct loadConP0(iRegP dst, immP0 src, flagsReg cr) %{
4438   match(Set dst src);
4439   effect(KILL cr);
4440   size(4);
4441   format %{ "XGR     $dst,$dst\t # NULL ptr" %}
4442   opcode(XGR_ZOPC);
4443   ins_encode(z_rreform(dst, dst));
4444   ins_pipe(pipe_class_dummy);
4445 %}
4446 
4447 //----------Load Float Constant Instructions-------------------------------------------------
4448 
4449 // We may not specify this instruction via an `expand' rule. If we do,
4450 // code selection will forget that this instruction needs a floating
4451 // point constant inserted into the code buffer. So `Shorten_branches'
4452 // will fail.
4453 instruct loadConF_dynTOC(regF dst, immF src, flagsReg cr) %{
4454   match(Set dst src);
4455   effect(KILL cr);
4456   ins_cost(MEMORY_REF_COST);
4457   size(6);
4458   // If this instruction rematerializes, it prolongs the live range
4459   // of the toc node, causing illegal graphs.
4460   ins_cannot_rematerialize(true);
4461   format %{ "LE(Y)    $dst,$constantoffset[,$constanttablebase]\t # load FLOAT $src from table" %}
4462   ins_encode %{
4463     __ load_float_largeoffset($dst$$FloatRegister, $constantoffset($src), $constanttablebase, Z_R1_scratch);
4464   %}
4465   ins_pipe(pipe_class_dummy);
4466 %}
4467 
4468 // E may not specify this instruction via an `expand' rule. If we do,
4469 // code selection will forget that this instruction needs a floating
4470 // point constant inserted into the code buffer. So `Shorten_branches'
4471 // will fail.
4472 instruct loadConD_dynTOC(regD dst, immD src, flagsReg cr) %{
4473   match(Set dst src);
4474   effect(KILL cr);
4475   ins_cost(MEMORY_REF_COST);
4476   size(6);
4477   // If this instruction rematerializes, it prolongs the live range
4478   // of the toc node, causing illegal graphs.
4479   ins_cannot_rematerialize(true);
4480   format %{ "LD(Y)    $dst,$constantoffset[,$constanttablebase]\t # load DOUBLE $src from table" %}
4481   ins_encode %{
4482     __ load_double_largeoffset($dst$$FloatRegister, $constantoffset($src), $constanttablebase, Z_R1_scratch);
4483   %}
4484   ins_pipe(pipe_class_dummy);
4485 %}
4486 
4487 // Special case: Load Const 0.0F
4488 
4489 // There's a special instr to clear a FP register.
4490 instruct loadConF0(regF dst, immFp0 src) %{
4491   match(Set dst src);
4492   ins_cost(DEFAULT_COST_LOW);
4493   size(4);
4494   format %{ "LZER     $dst,$src\t # clear to zero" %}
4495   opcode(LZER_ZOPC);
4496   ins_encode(z_rreform(dst, Z_F0));
4497   ins_pipe(pipe_class_dummy);
4498 %}
4499 
4500 // There's a special instr to clear a FP register.
4501 instruct loadConD0(regD dst, immDp0 src) %{
4502   match(Set dst src);
4503   ins_cost(DEFAULT_COST_LOW);
4504   size(4);
4505   format %{ "LZDR     $dst,$src\t # clear to zero" %}
4506   opcode(LZDR_ZOPC);
4507   ins_encode(z_rreform(dst, Z_F0));
4508   ins_pipe(pipe_class_dummy);
4509 %}
4510 
4511 
4512 //----------Store Instructions-------------------------------------------------
4513 
4514 // BYTE
4515 
4516 // Store Byte
4517 instruct storeB(memory mem, iRegI src) %{
4518   match(Set mem (StoreB mem src));
4519   ins_cost(MEMORY_REF_COST);
4520   size(Z_DISP_SIZE);
4521   format %{ "STC(Y)  $src,$mem\t # byte" %}
4522   opcode(STCY_ZOPC, STC_ZOPC);
4523   ins_encode(z_form_rt_mem_opt(src, mem));
4524   ins_pipe(pipe_class_dummy);
4525 %}
4526 
4527 instruct storeCM(memory mem, immI_0 src) %{
4528   match(Set mem (StoreCM mem src));
4529   ins_cost(MEMORY_REF_COST);
4530   // TODO: s390 port size(VARIABLE_SIZE);
4531   format %{ "STC(Y)  $src,$mem\t # CMS card-mark byte (must be 0!)" %}
4532   ins_encode %{
4533     guarantee($mem$$index$$Register != Z_R0, "content will not be used.");
4534     if ($mem$$index$$Register != noreg) {
4535       // Can't use clear_mem --> load const zero and store character.
4536       __ load_const_optimized(Z_R0_scratch, (long)0);
4537       if (Immediate::is_uimm12($mem$$disp)) {
4538         __ z_stc(Z_R0_scratch, $mem$$Address);
4539       } else {
4540         __ z_stcy(Z_R0_scratch, $mem$$Address);
4541       }
4542     } else {
4543       __ clear_mem(Address($mem$$Address), 1);
4544     }
4545   %}
4546   ins_pipe(pipe_class_dummy);
4547 %}
4548 
4549 // CHAR/SHORT
4550 
4551 // Store Char/Short
4552 instruct storeC(memory mem, iRegI src) %{
4553   match(Set mem (StoreC mem src));
4554   ins_cost(MEMORY_REF_COST);
4555   size(Z_DISP_SIZE);
4556   format %{ "STH(Y)  $src,$mem\t # short" %}
4557   opcode(STHY_ZOPC, STH_ZOPC);
4558   ins_encode(z_form_rt_mem_opt(src, mem));
4559   ins_pipe(pipe_class_dummy);
4560 %}
4561 
4562 // INT
4563 
4564 // Store Integer
4565 instruct storeI(memory mem, iRegI src) %{
4566   match(Set mem (StoreI mem src));
4567   ins_cost(MEMORY_REF_COST);
4568   size(Z_DISP_SIZE);
4569   format %{ "ST(Y)   $src,$mem\t # int" %}
4570   opcode(STY_ZOPC, ST_ZOPC);
4571   ins_encode(z_form_rt_mem_opt(src, mem));
4572   ins_pipe(pipe_class_dummy);
4573 %}
4574 
4575 // LONG
4576 
4577 // Store Long
4578 instruct storeL(memory mem, iRegL src) %{
4579   match(Set mem (StoreL mem src));
4580   ins_cost(MEMORY_REF_COST);
4581   size(Z_DISP3_SIZE);
4582   format %{ "STG     $src,$mem\t # long" %}
4583   opcode(STG_ZOPC, STG_ZOPC);
4584   ins_encode(z_form_rt_mem_opt(src, mem));
4585   ins_pipe(pipe_class_dummy);
4586 %}
4587 
4588 // PTR
4589 
4590 // Store Pointer
4591 instruct storeP(memory dst, memoryRegP src) %{
4592   match(Set dst (StoreP dst src));
4593   ins_cost(MEMORY_REF_COST);
4594   size(Z_DISP3_SIZE);
4595   format %{ "STG     $src,$dst\t # ptr" %}
4596   opcode(STG_ZOPC, STG_ZOPC);
4597   ins_encode(z_form_rt_mem_opt(src, dst));
4598   ins_pipe(pipe_class_dummy);
4599 %}
4600 
4601 // FLOAT
4602 
4603 // Store Float
4604 instruct storeF(memory mem, regF src) %{
4605   match(Set mem (StoreF mem src));
4606   ins_cost(MEMORY_REF_COST);
4607   size(Z_DISP_SIZE);
4608   format %{ "STE(Y)   $src,$mem\t # float" %}
4609   opcode(STEY_ZOPC, STE_ZOPC);
4610   ins_encode(z_form_rt_mem_opt(src, mem));
4611   ins_pipe(pipe_class_dummy);
4612 %}
4613 
4614 // DOUBLE
4615 
4616 // Store Double
4617 instruct storeD(memory mem, regD src) %{
4618   match(Set mem (StoreD mem src));
4619   ins_cost(MEMORY_REF_COST);
4620   size(Z_DISP_SIZE);
4621   format %{ "STD(Y)   $src,$mem\t # double" %}
4622   opcode(STDY_ZOPC, STD_ZOPC);
4623   ins_encode(z_form_rt_mem_opt(src, mem));
4624   ins_pipe(pipe_class_dummy);
4625 %}
4626 
4627 // Prefetch instructions. Must be safe to execute with invalid address (cannot fault).
4628 
4629 // Should support match rule for PrefetchAllocation.
4630 // Still needed after 8068977 for PrefetchAllocate.
4631 instruct prefetchAlloc(memory mem) %{
4632   match(PrefetchAllocation mem);
4633   predicate(VM_Version::has_Prefetch());
4634   ins_cost(DEFAULT_COST);
4635   format %{ "PREFETCH 2, $mem\t # Prefetch allocation, z10 only" %}
4636   ins_encode %{ __ z_pfd(0x02, $mem$$Address); %}
4637   ins_pipe(pipe_class_dummy);
4638 %}
4639 
4640 //----------Memory init instructions------------------------------------------
4641 
4642 // Move Immediate to 1-byte memory.
4643 instruct memInitB(memoryRSY mem, immI8 src) %{
4644   match(Set mem (StoreB mem src));
4645   ins_cost(MEMORY_REF_COST);
4646   // TODO: s390 port size(VARIABLE_SIZE);
4647   format %{ "MVI     $mem,$src\t # direct mem init 1" %}
4648   ins_encode %{
4649     if (Immediate::is_uimm12((long)$mem$$disp)) {
4650       __ z_mvi($mem$$Address, $src$$constant);
4651     } else {
4652       __ z_mviy($mem$$Address, $src$$constant);
4653     }
4654   %}
4655   ins_pipe(pipe_class_dummy);
4656 %}
4657 
4658 // Move Immediate to 2-byte memory.
4659 instruct memInitC(memoryRS mem, immI16 src) %{
4660   match(Set mem (StoreC mem src));
4661   ins_cost(MEMORY_REF_COST);
4662   size(6);
4663   format %{ "MVHHI   $mem,$src\t # direct mem init 2" %}
4664   opcode(MVHHI_ZOPC);
4665   ins_encode(z_silform(mem, src));
4666   ins_pipe(pipe_class_dummy);
4667 %}
4668 
4669 // Move Immediate to 4-byte memory.
4670 instruct memInitI(memoryRS mem, immI16 src) %{
4671   match(Set mem (StoreI mem src));
4672   ins_cost(MEMORY_REF_COST);
4673   size(6);
4674   format %{ "MVHI    $mem,$src\t # direct mem init 4" %}
4675   opcode(MVHI_ZOPC);
4676   ins_encode(z_silform(mem, src));
4677   ins_pipe(pipe_class_dummy);
4678 %}
4679 
4680 
4681 // Move Immediate to 8-byte memory.
4682 instruct memInitL(memoryRS mem, immL16 src) %{
4683   match(Set mem (StoreL mem src));
4684   ins_cost(MEMORY_REF_COST);
4685   size(6);
4686   format %{ "MVGHI   $mem,$src\t # direct mem init 8" %}
4687   opcode(MVGHI_ZOPC);
4688   ins_encode(z_silform(mem, src));
4689   ins_pipe(pipe_class_dummy);
4690 %}
4691 
4692 // Move Immediate to 8-byte memory.
4693 instruct memInitP(memoryRS mem, immP16 src) %{
4694   match(Set mem (StoreP mem src));
4695   ins_cost(MEMORY_REF_COST);
4696   size(6);
4697   format %{ "MVGHI   $mem,$src\t # direct mem init 8" %}
4698   opcode(MVGHI_ZOPC);
4699   ins_encode(z_silform(mem, src));
4700   ins_pipe(pipe_class_dummy);
4701 %}
4702 
4703 
4704 //----------Instructions for compressed pointers (cOop and NKlass)-------------
4705 
4706 // See cOop encoding classes for elaborate comment.
4707 
4708 // Moved here because it is needed in expand rules for encode.
4709 // Long negation.
4710 instruct negL_reg_reg(iRegL dst, immL_0 zero, iRegL src, flagsReg cr) %{
4711   match(Set dst (SubL zero src));
4712   effect(KILL cr);
4713   size(4);
4714   format %{ "NEG     $dst, $src\t # long" %}
4715   ins_encode %{ __ z_lcgr($dst$$Register, $src$$Register); %}
4716   ins_pipe(pipe_class_dummy);
4717 %}
4718 
4719 // Load Compressed Pointer
4720 
4721 // Load narrow oop
4722 instruct loadN(iRegN dst, memory mem) %{
4723   match(Set dst (LoadN mem));
4724   ins_cost(MEMORY_REF_COST);
4725   size(Z_DISP3_SIZE);
4726   format %{ "LoadN  $dst,$mem\t# (cOop)" %}
4727   opcode(LLGF_ZOPC, LLGF_ZOPC);
4728   ins_encode(z_form_rt_mem_opt(dst, mem));
4729   ins_pipe(pipe_class_dummy);
4730 %}
4731 
4732 // Load narrow Klass Pointer
4733 instruct loadNKlass(iRegN dst, memory mem) %{
4734   match(Set dst (LoadNKlass mem));
4735   ins_cost(MEMORY_REF_COST);
4736   size(Z_DISP3_SIZE);
4737   format %{ "LoadNKlass $dst,$mem\t# (klass cOop)" %}
4738   opcode(LLGF_ZOPC, LLGF_ZOPC);
4739   ins_encode(z_form_rt_mem_opt(dst, mem));
4740   ins_pipe(pipe_class_dummy);
4741 %}
4742 
4743 // Load constant Compressed Pointer
4744 
4745 instruct loadConN(iRegN dst, immN src) %{
4746   match(Set dst src);
4747   ins_cost(DEFAULT_COST);
4748   size(6);
4749   format %{ "loadConN    $dst,$src\t # (cOop)" %}
4750   ins_encode %{
4751     AddressLiteral cOop = __ constant_oop_address((jobject)$src$$constant);
4752     __ relocate(cOop.rspec(), 1);
4753     __ load_narrow_oop($dst$$Register, (narrowOop)cOop.value());
4754   %}
4755   ins_pipe(pipe_class_dummy);
4756 %}
4757 
4758 instruct loadConN0(iRegN dst, immN0 src, flagsReg cr) %{
4759   match(Set dst src);
4760   effect(KILL cr);
4761   ins_cost(DEFAULT_COST_LOW);
4762   size(4);
4763   format %{ "loadConN    $dst,$src\t # (cOop) XGR because ZERO is loaded" %}
4764   opcode(XGR_ZOPC);
4765   ins_encode(z_rreform(dst, dst));
4766   ins_pipe(pipe_class_dummy);
4767 %}
4768 
4769 instruct loadConNKlass(iRegN dst, immNKlass src) %{
4770   match(Set dst src);
4771   ins_cost(DEFAULT_COST);
4772   size(6);
4773   format %{ "loadConNKlass $dst,$src\t # (cKlass)" %}
4774   ins_encode %{
4775     AddressLiteral NKlass = __ constant_metadata_address((Metadata*)$src$$constant);
4776     __ relocate(NKlass.rspec(), 1);
4777     __ load_narrow_klass($dst$$Register, (Klass*)NKlass.value());
4778   %}
4779   ins_pipe(pipe_class_dummy);
4780 %}
4781 
4782 // Load and Decode Compressed Pointer
4783 // optimized variants for Unscaled cOops
4784 
4785 instruct decodeLoadN(iRegP dst, memory mem) %{
4786   match(Set dst (DecodeN (LoadN mem)));
4787   predicate(false && (Universe::narrow_oop_base()==NULL)&&(Universe::narrow_oop_shift()==0));
4788   ins_cost(MEMORY_REF_COST);
4789   size(Z_DISP3_SIZE);
4790   format %{ "DecodeLoadN  $dst,$mem\t# (cOop Load+Decode)" %}
4791   opcode(LLGF_ZOPC, LLGF_ZOPC);
4792   ins_encode(z_form_rt_mem_opt(dst, mem));
4793   ins_pipe(pipe_class_dummy);
4794 %}
4795 
4796 instruct decodeLoadNKlass(iRegP dst, memory mem) %{
4797   match(Set dst (DecodeNKlass (LoadNKlass mem)));
4798   predicate(false && (Universe::narrow_klass_base()==NULL)&&(Universe::narrow_klass_shift()==0));
4799   ins_cost(MEMORY_REF_COST);
4800   size(Z_DISP3_SIZE);
4801   format %{ "DecodeLoadNKlass  $dst,$mem\t# (load/decode NKlass)" %}
4802   opcode(LLGF_ZOPC, LLGF_ZOPC);
4803   ins_encode(z_form_rt_mem_opt(dst, mem));
4804   ins_pipe(pipe_class_dummy);
4805 %}
4806 
4807 instruct decodeLoadConNKlass(iRegP dst, immNKlass src) %{
4808   match(Set dst (DecodeNKlass src));
4809   ins_cost(3 * DEFAULT_COST);
4810   size(12);
4811   format %{ "DecodeLoadConNKlass  $dst,$src\t # decode(cKlass)" %}
4812   ins_encode %{
4813     AddressLiteral NKlass = __ constant_metadata_address((Metadata*)$src$$constant);
4814     __ relocate(NKlass.rspec(), 1);
4815     __ load_const($dst$$Register, (Klass*)NKlass.value());
4816   %}
4817   ins_pipe(pipe_class_dummy);
4818 %}
4819 
4820 // Decode Compressed Pointer
4821 
4822 // General decoder
4823 instruct decodeN(iRegP dst, iRegN src, flagsReg cr) %{
4824   match(Set dst (DecodeN src));
4825   effect(KILL cr);
4826   predicate(Universe::narrow_oop_base() == NULL || !ExpandLoadingBaseDecode);
4827   ins_cost(MEMORY_REF_COST+3 * DEFAULT_COST + BRANCH_COST);
4828   // TODO: s390 port size(VARIABLE_SIZE);
4829   format %{ "decodeN  $dst,$src\t# (decode cOop)" %}
4830   ins_encode %{  __ oop_decoder($dst$$Register, $src$$Register, true); %}
4831   ins_pipe(pipe_class_dummy);
4832 %}
4833 
4834 // General Klass decoder
4835 instruct decodeKlass(iRegP dst, iRegN src, flagsReg cr) %{
4836   match(Set dst (DecodeNKlass src));
4837   effect(KILL cr);
4838   ins_cost(3 * DEFAULT_COST);
4839   format %{ "decode_klass $dst,$src" %}
4840   ins_encode %{ __ decode_klass_not_null($dst$$Register, $src$$Register); %}
4841   ins_pipe(pipe_class_dummy);
4842 %}
4843 
4844 // General decoder
4845 instruct decodeN_NN(iRegP dst, iRegN src, flagsReg cr) %{
4846   match(Set dst (DecodeN src));
4847   effect(KILL cr);
4848   predicate((n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull ||
4849              n->bottom_type()->is_oopptr()->ptr() == TypePtr::Constant) &&
4850             (Universe::narrow_oop_base()== NULL || !ExpandLoadingBaseDecode_NN));
4851   ins_cost(MEMORY_REF_COST+2 * DEFAULT_COST);
4852   // TODO: s390 port size(VARIABLE_SIZE);
4853   format %{ "decodeN  $dst,$src\t# (decode cOop NN)" %}
4854   ins_encode %{ __ oop_decoder($dst$$Register, $src$$Register, false); %}
4855   ins_pipe(pipe_class_dummy);
4856 %}
4857 
4858   instruct loadBase(iRegL dst, immL baseImm) %{
4859     effect(DEF dst, USE baseImm);
4860     predicate(false);
4861     format %{ "llihl    $dst=$baseImm \t// load heap base" %}
4862     ins_encode %{ __ get_oop_base($dst$$Register, $baseImm$$constant); %}
4863     ins_pipe(pipe_class_dummy);
4864   %}
4865 
4866   // Decoder for heapbased mode peeling off loading the base.
4867   instruct decodeN_base(iRegP dst, iRegN src, iRegL base, flagsReg cr) %{
4868     match(Set dst (DecodeN src base));
4869     // Note: Effect TEMP dst was used with the intention to get
4870     // different regs for dst and base, but this has caused ADLC to
4871     // generate wrong code. Oop_decoder generates additional lgr when
4872     // dst==base.
4873     effect(KILL cr);
4874     predicate(false);
4875     // TODO: s390 port size(VARIABLE_SIZE);
4876     format %{ "decodeN  $dst = ($src == 0) ? NULL : ($src << 3) + $base + pow2_offset\t# (decode cOop)" %}
4877     ins_encode %{
4878       __ oop_decoder($dst$$Register, $src$$Register, true, $base$$Register,
4879                      (jlong)MacroAssembler::get_oop_base_pow2_offset((uint64_t)(intptr_t)Universe::narrow_oop_base()));
4880     %}
4881     ins_pipe(pipe_class_dummy);
4882   %}
4883 
4884   // Decoder for heapbased mode peeling off loading the base.
4885   instruct decodeN_NN_base(iRegP dst, iRegN src, iRegL base, flagsReg cr) %{
4886     match(Set dst (DecodeN src base));
4887     effect(KILL cr);
4888     predicate(false);
4889     // TODO: s390 port size(VARIABLE_SIZE);
4890     format %{ "decodeN  $dst = ($src << 3) + $base + pow2_offset\t# (decode cOop)" %}
4891     ins_encode %{
4892       __ oop_decoder($dst$$Register, $src$$Register, false, $base$$Register,
4893                      (jlong)MacroAssembler::get_oop_base_pow2_offset((uint64_t)(intptr_t)Universe::narrow_oop_base()));
4894     %}
4895     ins_pipe(pipe_class_dummy);
4896   %}
4897 
4898 // Decoder for heapbased mode peeling off loading the base.
4899 instruct decodeN_Ex(iRegP dst, iRegN src, flagsReg cr) %{
4900   match(Set dst (DecodeN src));
4901   predicate(Universe::narrow_oop_base() != NULL && ExpandLoadingBaseDecode);
4902   ins_cost(MEMORY_REF_COST+3 * DEFAULT_COST + BRANCH_COST);
4903   // TODO: s390 port size(VARIABLE_SIZE);
4904   expand %{
4905     immL baseImm %{ (jlong)(intptr_t)Universe::narrow_oop_base() %}
4906     iRegL base;
4907     loadBase(base, baseImm);
4908     decodeN_base(dst, src, base, cr);
4909   %}
4910 %}
4911 
4912 // Decoder for heapbased mode peeling off loading the base.
4913 instruct decodeN_NN_Ex(iRegP dst, iRegN src, flagsReg cr) %{
4914   match(Set dst (DecodeN src));
4915   predicate((n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull ||
4916              n->bottom_type()->is_oopptr()->ptr() == TypePtr::Constant) &&
4917             Universe::narrow_oop_base() != NULL && ExpandLoadingBaseDecode_NN);
4918   ins_cost(MEMORY_REF_COST+2 * DEFAULT_COST);
4919   // TODO: s390 port size(VARIABLE_SIZE);
4920   expand %{
4921     immL baseImm %{ (jlong)(intptr_t)Universe::narrow_oop_base() %}
4922     iRegL base;
4923     loadBase(base, baseImm);
4924     decodeN_NN_base(dst, src, base, cr);
4925   %}
4926 %}
4927 
4928 //  Encode Compressed Pointer
4929 
4930 // General encoder
4931 instruct encodeP(iRegN dst, iRegP src, flagsReg cr) %{
4932   match(Set dst (EncodeP src));
4933   effect(KILL cr);
4934   predicate((n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull) &&
4935             (Universe::narrow_oop_base() == 0 ||
4936              Universe::narrow_oop_base_disjoint() ||
4937              !ExpandLoadingBaseEncode));
4938   ins_cost(MEMORY_REF_COST+3 * DEFAULT_COST);
4939   // TODO: s390 port size(VARIABLE_SIZE);
4940   format %{ "encodeP  $dst,$src\t# (encode cOop)" %}
4941   ins_encode %{ __ oop_encoder($dst$$Register, $src$$Register, true, Z_R1_scratch, -1, all_outs_are_Stores(this)); %}
4942   ins_pipe(pipe_class_dummy);
4943 %}
4944 
4945 // General class encoder
4946 instruct encodeKlass(iRegN dst, iRegP src, flagsReg cr) %{
4947   match(Set dst (EncodePKlass src));
4948   effect(KILL cr);
4949   format %{ "encode_klass $dst,$src" %}
4950   ins_encode %{ __ encode_klass_not_null($dst$$Register, $src$$Register); %}
4951   ins_pipe(pipe_class_dummy);
4952 %}
4953 
4954 instruct encodeP_NN(iRegN dst, iRegP src, flagsReg cr) %{
4955   match(Set dst (EncodeP src));
4956   effect(KILL cr);
4957   predicate((n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull) &&
4958             (Universe::narrow_oop_base() == 0 ||
4959              Universe::narrow_oop_base_disjoint() ||
4960              !ExpandLoadingBaseEncode_NN));
4961   ins_cost(MEMORY_REF_COST+3 * DEFAULT_COST);
4962   // TODO: s390 port size(VARIABLE_SIZE);
4963   format %{ "encodeP  $dst,$src\t# (encode cOop)" %}
4964   ins_encode %{ __ oop_encoder($dst$$Register, $src$$Register, false, Z_R1_scratch, -1, all_outs_are_Stores(this)); %}
4965   ins_pipe(pipe_class_dummy);
4966 %}
4967 
4968   // Encoder for heapbased mode peeling off loading the base.
4969   instruct encodeP_base(iRegN dst, iRegP src, iRegL base) %{
4970     match(Set dst (EncodeP src (Binary base dst)));
4971     effect(TEMP_DEF dst);
4972     predicate(false);
4973     ins_cost(MEMORY_REF_COST+2 * DEFAULT_COST);
4974     // TODO: s390 port size(VARIABLE_SIZE);
4975     format %{ "encodeP  $dst = ($src>>3) +$base + pow2_offset\t# (encode cOop)" %}
4976     ins_encode %{
4977       jlong offset = -(jlong)MacroAssembler::get_oop_base_pow2_offset
4978         (((uint64_t)(intptr_t)Universe::narrow_oop_base()) >> Universe::narrow_oop_shift());
4979       __ oop_encoder($dst$$Register, $src$$Register, true, $base$$Register, offset);
4980     %}
4981     ins_pipe(pipe_class_dummy);
4982   %}
4983 
4984   // Encoder for heapbased mode peeling off loading the base.
4985   instruct encodeP_NN_base(iRegN dst, iRegP src, iRegL base, immL pow2_offset) %{
4986     match(Set dst (EncodeP src base));
4987     effect(USE pow2_offset);
4988     predicate(false);
4989     ins_cost(MEMORY_REF_COST+2 * DEFAULT_COST);
4990     // TODO: s390 port size(VARIABLE_SIZE);
4991     format %{ "encodeP  $dst = ($src>>3) +$base + $pow2_offset\t# (encode cOop)" %}
4992     ins_encode %{ __ oop_encoder($dst$$Register, $src$$Register, false, $base$$Register, $pow2_offset$$constant); %}
4993     ins_pipe(pipe_class_dummy);
4994   %}
4995 
4996 // Encoder for heapbased mode peeling off loading the base.
4997 instruct encodeP_Ex(iRegN dst, iRegP src, flagsReg cr) %{
4998   match(Set dst (EncodeP src));
4999   effect(KILL cr);
5000   predicate((n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull) &&
5001             (Universe::narrow_oop_base_overlaps() && ExpandLoadingBaseEncode));
5002   ins_cost(MEMORY_REF_COST+3 * DEFAULT_COST);
5003   // TODO: s390 port size(VARIABLE_SIZE);
5004   expand %{
5005     immL baseImm %{ ((jlong)(intptr_t)Universe::narrow_oop_base()) >> Universe::narrow_oop_shift() %}
5006     immL_0 zero %{ (0) %}
5007     flagsReg ccr;
5008     iRegL base;
5009     iRegL negBase;
5010     loadBase(base, baseImm);
5011     negL_reg_reg(negBase, zero, base, ccr);
5012     encodeP_base(dst, src, negBase);
5013   %}
5014 %}
5015 
5016 // Encoder for heapbased mode peeling off loading the base.
5017 instruct encodeP_NN_Ex(iRegN dst, iRegP src, flagsReg cr) %{
5018   match(Set dst (EncodeP src));
5019   effect(KILL cr);
5020   predicate((n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull) &&
5021             (Universe::narrow_oop_base_overlaps() && ExpandLoadingBaseEncode_NN));
5022   ins_cost(MEMORY_REF_COST+3 * DEFAULT_COST);
5023   // TODO: s390 port size(VARIABLE_SIZE);
5024   expand %{
5025     immL baseImm %{ (jlong)(intptr_t)Universe::narrow_oop_base() %}
5026     immL pow2_offset %{ -(jlong)MacroAssembler::get_oop_base_pow2_offset(((uint64_t)(intptr_t)Universe::narrow_oop_base())) %}
5027     immL_0 zero %{ 0 %}
5028     flagsReg ccr;
5029     iRegL base;
5030     iRegL negBase;
5031     loadBase(base, baseImm);
5032     negL_reg_reg(negBase, zero, base, ccr);
5033     encodeP_NN_base(dst, src, negBase, pow2_offset);
5034   %}
5035 %}
5036 
5037 //  Store Compressed Pointer
5038 
5039 // Store Compressed Pointer
5040 instruct storeN(memory mem, iRegN_P2N src) %{
5041   match(Set mem (StoreN mem src));
5042   ins_cost(MEMORY_REF_COST);
5043   size(Z_DISP_SIZE);
5044   format %{ "ST      $src,$mem\t# (cOop)" %}
5045   opcode(STY_ZOPC, ST_ZOPC);
5046   ins_encode(z_form_rt_mem_opt(src, mem));
5047   ins_pipe(pipe_class_dummy);
5048 %}
5049 
5050 // Store Compressed Klass pointer
5051 instruct storeNKlass(memory mem, iRegN src) %{
5052   match(Set mem (StoreNKlass mem src));
5053   ins_cost(MEMORY_REF_COST);
5054   size(Z_DISP_SIZE);
5055   format %{ "ST      $src,$mem\t# (cKlass)" %}
5056   opcode(STY_ZOPC, ST_ZOPC);
5057   ins_encode(z_form_rt_mem_opt(src, mem));
5058   ins_pipe(pipe_class_dummy);
5059 %}
5060 
5061 // Compare Compressed Pointers
5062 
5063 instruct compN_iRegN(iRegN_P2N src1, iRegN_P2N src2, flagsReg cr) %{
5064   match(Set cr (CmpN src1 src2));
5065   ins_cost(DEFAULT_COST);
5066   size(2);
5067   format %{ "CLR     $src1,$src2\t# (cOop)" %}
5068   opcode(CLR_ZOPC);
5069   ins_encode(z_rrform(src1, src2));
5070   ins_pipe(pipe_class_dummy);
5071 %}
5072 
5073 instruct compN_iRegN_immN(iRegN_P2N src1, immN src2, flagsReg cr) %{
5074   match(Set cr (CmpN src1 src2));
5075   ins_cost(DEFAULT_COST);
5076   size(6);
5077   format %{ "CLFI    $src1,$src2\t# (cOop) compare immediate narrow" %}
5078   ins_encode %{
5079     AddressLiteral cOop = __ constant_oop_address((jobject)$src2$$constant);
5080     __ relocate(cOop.rspec(), 1);
5081     __ compare_immediate_narrow_oop($src1$$Register, (narrowOop)cOop.value());
5082   %}
5083   ins_pipe(pipe_class_dummy);
5084 %}
5085 
5086 instruct compNKlass_iRegN_immN(iRegN src1, immNKlass src2, flagsReg cr) %{
5087   match(Set cr (CmpN src1 src2));
5088   ins_cost(DEFAULT_COST);
5089   size(6);
5090   format %{ "CLFI    $src1,$src2\t# (NKlass) compare immediate narrow" %}
5091   ins_encode %{
5092     AddressLiteral NKlass = __ constant_metadata_address((Metadata*)$src2$$constant);
5093     __ relocate(NKlass.rspec(), 1);
5094     __ compare_immediate_narrow_klass($src1$$Register, (Klass*)NKlass.value());
5095   %}
5096   ins_pipe(pipe_class_dummy);
5097 %}
5098 
5099 instruct compN_iRegN_immN0(iRegN_P2N src1, immN0 src2, flagsReg cr) %{
5100   match(Set cr (CmpN src1 src2));
5101   ins_cost(DEFAULT_COST);
5102   size(2);
5103   format %{ "LTR     $src1,$src2\t# (cOop) LTR because comparing against zero" %}
5104   opcode(LTR_ZOPC);
5105   ins_encode(z_rrform(src1, src1));
5106   ins_pipe(pipe_class_dummy);
5107 %}
5108 
5109 
5110 //----------MemBar Instructions-----------------------------------------------
5111 
5112 // Memory barrier flavors
5113 
5114 instruct membar_acquire() %{
5115   match(MemBarAcquire);
5116   match(LoadFence);
5117   ins_cost(4*MEMORY_REF_COST);
5118   size(0);
5119   format %{ "MEMBAR-acquire" %}
5120   ins_encode %{ __ z_acquire(); %}
5121   ins_pipe(pipe_class_dummy);
5122 %}
5123 
5124 instruct membar_acquire_lock() %{
5125   match(MemBarAcquireLock);
5126   ins_cost(0);
5127   size(0);
5128   format %{ "MEMBAR-acquire (CAS in prior FastLock so empty encoding)" %}
5129   ins_encode(/*empty*/);
5130   ins_pipe(pipe_class_dummy);
5131 %}
5132 
5133 instruct membar_release() %{
5134   match(MemBarRelease);
5135   match(StoreFence);
5136   ins_cost(4 * MEMORY_REF_COST);
5137   size(0);
5138   format %{ "MEMBAR-release" %}
5139   ins_encode %{ __ z_release(); %}
5140   ins_pipe(pipe_class_dummy);
5141 %}
5142 
5143 instruct membar_release_lock() %{
5144   match(MemBarReleaseLock);
5145   ins_cost(0);
5146   size(0);
5147   format %{ "MEMBAR-release (CAS in succeeding FastUnlock so empty encoding)" %}
5148   ins_encode(/*empty*/);
5149   ins_pipe(pipe_class_dummy);
5150 %}
5151 
5152 instruct membar_volatile() %{
5153   match(MemBarVolatile);
5154   ins_cost(4 * MEMORY_REF_COST);
5155   size(2);
5156   format %{ "MEMBAR-volatile" %}
5157   ins_encode %{ __ z_fence(); %}
5158   ins_pipe(pipe_class_dummy);
5159 %}
5160 
5161 instruct unnecessary_membar_volatile() %{
5162   match(MemBarVolatile);
5163   predicate(Matcher::post_store_load_barrier(n));
5164   ins_cost(0);
5165   size(0);
5166   format %{ "# MEMBAR-volatile (empty)" %}
5167   ins_encode(/*empty*/);
5168   ins_pipe(pipe_class_dummy);
5169 %}
5170 
5171 instruct membar_CPUOrder() %{
5172   match(MemBarCPUOrder);
5173   ins_cost(0);
5174   // TODO: s390 port size(FIXED_SIZE);
5175   format %{ "MEMBAR-CPUOrder (empty)" %}
5176   ins_encode(/*empty*/);
5177   ins_pipe(pipe_class_dummy);
5178 %}
5179 
5180 instruct membar_storestore() %{
5181   match(MemBarStoreStore);
5182   ins_cost(0);
5183   size(0);
5184   format %{ "MEMBAR-storestore (empty)" %}
5185   ins_encode();
5186   ins_pipe(pipe_class_dummy);
5187 %}
5188 
5189 
5190 //----------Register Move Instructions-----------------------------------------
5191 instruct roundDouble_nop(regD dst) %{
5192   match(Set dst (RoundDouble dst));
5193   ins_cost(0);
5194   // TODO: s390 port size(FIXED_SIZE);
5195   // z/Architecture results are already "rounded" (i.e., normal-format IEEE).
5196   ins_encode();
5197   ins_pipe(pipe_class_dummy);
5198 %}
5199 
5200 instruct roundFloat_nop(regF dst) %{
5201   match(Set dst (RoundFloat dst));
5202   ins_cost(0);
5203   // TODO: s390 port size(FIXED_SIZE);
5204   // z/Architecture results are already "rounded" (i.e., normal-format IEEE).
5205   ins_encode();
5206   ins_pipe(pipe_class_dummy);
5207 %}
5208 
5209 // Cast Long to Pointer for unsafe natives.
5210 instruct castX2P(iRegP dst, iRegL src) %{
5211   match(Set dst (CastX2P src));
5212   // TODO: s390 port size(VARIABLE_SIZE);
5213   format %{ "LGR     $dst,$src\t # CastX2P" %}
5214   ins_encode %{ __ lgr_if_needed($dst$$Register, $src$$Register); %}
5215   ins_pipe(pipe_class_dummy);
5216 %}
5217 
5218 // Cast Pointer to Long for unsafe natives.
5219 instruct castP2X(iRegL dst, iRegP_N2P src) %{
5220   match(Set dst (CastP2X src));
5221   // TODO: s390 port size(VARIABLE_SIZE);
5222   format %{ "LGR     $dst,$src\t # CastP2X" %}
5223   ins_encode %{ __ lgr_if_needed($dst$$Register, $src$$Register); %}
5224   ins_pipe(pipe_class_dummy);
5225 %}
5226 
5227 instruct stfSSD(stackSlotD stkSlot, regD src) %{
5228   // %%%% TODO: Tell the coalescer that this kind of node is a copy!
5229   match(Set stkSlot src);   // chain rule
5230   ins_cost(MEMORY_REF_COST);
5231   // TODO: s390 port size(FIXED_SIZE);
5232   format %{ " STD   $src,$stkSlot\t # stk" %}
5233   opcode(STD_ZOPC);
5234   ins_encode(z_form_rt_mem(src, stkSlot));
5235   ins_pipe(pipe_class_dummy);
5236 %}
5237 
5238 instruct stfSSF(stackSlotF stkSlot, regF src) %{
5239   // %%%% TODO: Tell the coalescer that this kind of node is a copy!
5240   match(Set stkSlot src);   // chain rule
5241   ins_cost(MEMORY_REF_COST);
5242   // TODO: s390 port size(FIXED_SIZE);
5243   format %{ "STE   $src,$stkSlot\t # stk" %}
5244   opcode(STE_ZOPC);
5245   ins_encode(z_form_rt_mem(src, stkSlot));
5246   ins_pipe(pipe_class_dummy);
5247 %}
5248 
5249 //----------Conditional Move---------------------------------------------------
5250 
5251 instruct cmovN_reg(cmpOp cmp, flagsReg cr, iRegN dst, iRegN_P2N src) %{
5252   match(Set dst (CMoveN (Binary cmp cr) (Binary dst src)));
5253   ins_cost(DEFAULT_COST + BRANCH_COST);
5254   // TODO: s390 port size(VARIABLE_SIZE);
5255   format %{ "CMoveN,$cmp   $dst,$src" %}
5256   ins_encode(z_enc_cmov_reg(cmp,dst,src));
5257   ins_pipe(pipe_class_dummy);
5258 %}
5259 
5260 instruct cmovN_imm(cmpOp cmp, flagsReg cr, iRegN dst, immN0 src) %{
5261   match(Set dst (CMoveN (Binary cmp cr) (Binary dst src)));
5262   ins_cost(DEFAULT_COST + BRANCH_COST);
5263   // TODO: s390 port size(VARIABLE_SIZE);
5264   format %{ "CMoveN,$cmp   $dst,$src" %}
5265   ins_encode(z_enc_cmov_imm(cmp,dst,src));
5266   ins_pipe(pipe_class_dummy);
5267 %}
5268 
5269 instruct cmovI_reg(cmpOp cmp, flagsReg cr, iRegI dst, iRegI src) %{
5270   match(Set dst (CMoveI (Binary cmp cr) (Binary dst src)));
5271   ins_cost(DEFAULT_COST + BRANCH_COST);
5272   // TODO: s390 port size(VARIABLE_SIZE);
5273   format %{ "CMoveI,$cmp   $dst,$src" %}
5274   ins_encode(z_enc_cmov_reg(cmp,dst,src));
5275   ins_pipe(pipe_class_dummy);
5276 %}
5277 
5278 instruct cmovI_imm(cmpOp cmp, flagsReg cr, iRegI dst, immI16 src) %{
5279   match(Set dst (CMoveI (Binary cmp cr) (Binary dst src)));
5280   ins_cost(DEFAULT_COST + BRANCH_COST);
5281   // TODO: s390 port size(VARIABLE_SIZE);
5282   format %{ "CMoveI,$cmp   $dst,$src" %}
5283   ins_encode(z_enc_cmov_imm(cmp,dst,src));
5284   ins_pipe(pipe_class_dummy);
5285 %}
5286 
5287 instruct cmovP_reg(cmpOp cmp, flagsReg cr, iRegP dst, iRegP_N2P src) %{
5288   match(Set dst (CMoveP (Binary cmp cr) (Binary dst src)));
5289   ins_cost(DEFAULT_COST + BRANCH_COST);
5290   // TODO: s390 port size(VARIABLE_SIZE);
5291   format %{ "CMoveP,$cmp    $dst,$src" %}
5292   ins_encode(z_enc_cmov_reg(cmp,dst,src));
5293   ins_pipe(pipe_class_dummy);
5294 %}
5295 
5296 instruct cmovP_imm(cmpOp cmp, flagsReg cr, iRegP dst, immP0 src) %{
5297   match(Set dst (CMoveP (Binary cmp cr) (Binary dst src)));
5298   ins_cost(DEFAULT_COST + BRANCH_COST);
5299   // TODO: s390 port size(VARIABLE_SIZE);
5300   format %{ "CMoveP,$cmp  $dst,$src" %}
5301   ins_encode(z_enc_cmov_imm(cmp,dst,src));
5302   ins_pipe(pipe_class_dummy);
5303 %}
5304 
5305 instruct cmovF_reg(cmpOpF cmp, flagsReg cr, regF dst, regF src) %{
5306   match(Set dst (CMoveF (Binary cmp cr) (Binary dst src)));
5307   ins_cost(DEFAULT_COST + BRANCH_COST);
5308   // TODO: s390 port size(VARIABLE_SIZE);
5309   format %{ "CMoveF,$cmp   $dst,$src" %}
5310   ins_encode %{
5311     // Don't emit code if operands are identical (same register).
5312     if ($dst$$FloatRegister != $src$$FloatRegister) {
5313       Label done;
5314       __ z_brc(Assembler::inverse_float_condition((Assembler::branch_condition)$cmp$$cmpcode), done);
5315       __ z_ler($dst$$FloatRegister, $src$$FloatRegister);
5316       __ bind(done);
5317     }
5318   %}
5319   ins_pipe(pipe_class_dummy);
5320 %}
5321 
5322 instruct cmovD_reg(cmpOpF cmp, flagsReg cr, regD dst, regD src) %{
5323   match(Set dst (CMoveD (Binary cmp cr) (Binary dst src)));
5324   ins_cost(DEFAULT_COST + BRANCH_COST);
5325   // TODO: s390 port size(VARIABLE_SIZE);
5326   format %{ "CMoveD,$cmp   $dst,$src" %}
5327   ins_encode %{
5328     // Don't emit code if operands are identical (same register).
5329     if ($dst$$FloatRegister != $src$$FloatRegister) {
5330       Label done;
5331       __ z_brc(Assembler::inverse_float_condition((Assembler::branch_condition)$cmp$$cmpcode), done);
5332       __ z_ldr($dst$$FloatRegister, $src$$FloatRegister);
5333       __ bind(done);
5334     }
5335   %}
5336   ins_pipe(pipe_class_dummy);
5337 %}
5338 
5339 instruct cmovL_reg(cmpOp cmp, flagsReg cr, iRegL dst, iRegL src) %{
5340   match(Set dst (CMoveL (Binary cmp cr) (Binary dst src)));
5341   ins_cost(DEFAULT_COST + BRANCH_COST);
5342   // TODO: s390 port size(VARIABLE_SIZE);
5343   format %{ "CMoveL,$cmp  $dst,$src" %}
5344   ins_encode(z_enc_cmov_reg(cmp,dst,src));
5345   ins_pipe(pipe_class_dummy);
5346 %}
5347 
5348 instruct cmovL_imm(cmpOp cmp, flagsReg cr, iRegL dst, immL16 src) %{
5349   match(Set dst (CMoveL (Binary cmp cr) (Binary dst src)));
5350   ins_cost(DEFAULT_COST + BRANCH_COST);
5351   // TODO: s390 port size(VARIABLE_SIZE);
5352   format %{ "CMoveL,$cmp  $dst,$src" %}
5353   ins_encode(z_enc_cmov_imm(cmp,dst,src));
5354   ins_pipe(pipe_class_dummy);
5355 %}
5356 
5357 //----------OS and Locking Instructions----------------------------------------
5358 
5359 // This name is KNOWN by the ADLC and cannot be changed.
5360 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
5361 // for this guy.
5362 instruct tlsLoadP(threadRegP dst) %{
5363   match(Set dst (ThreadLocal));
5364   ins_cost(0);
5365   size(0);
5366   ins_should_rematerialize(true);
5367   format %{ "# $dst=ThreadLocal" %}
5368   ins_encode(/* empty */);
5369   ins_pipe(pipe_class_dummy);
5370 %}
5371 
5372 instruct checkCastPP(iRegP dst) %{
5373   match(Set dst (CheckCastPP dst));
5374   size(0);
5375   format %{ "# checkcastPP of $dst" %}
5376   ins_encode(/*empty*/);
5377   ins_pipe(pipe_class_dummy);
5378 %}
5379 
5380 instruct castPP(iRegP dst) %{
5381   match(Set dst (CastPP dst));
5382   size(0);
5383   format %{ "# castPP of $dst" %}
5384   ins_encode(/*empty*/);
5385   ins_pipe(pipe_class_dummy);
5386 %}
5387 
5388 instruct castII(iRegI dst) %{
5389   match(Set dst (CastII dst));
5390   size(0);
5391   format %{ "# castII of $dst" %}
5392   ins_encode(/*empty*/);
5393   ins_pipe(pipe_class_dummy);
5394 %}
5395 
5396 
5397 //----------Conditional_store--------------------------------------------------
5398 // Conditional-store of the updated heap-top.
5399 // Used during allocation of the shared heap.
5400 // Sets flags (EQ) on success.
5401 
5402 // Implement LoadPLocked. Must be ordered against changes of the memory location
5403 // by storePConditional.
5404 // Don't know whether this is ever used.
5405 instruct loadPLocked(iRegP dst, memory mem) %{
5406   match(Set dst (LoadPLocked mem));
5407   ins_cost(MEMORY_REF_COST);
5408   size(Z_DISP3_SIZE);
5409   format %{ "LG      $dst,$mem\t # LoadPLocked" %}
5410   opcode(LG_ZOPC, LG_ZOPC);
5411   ins_encode(z_form_rt_mem_opt(dst, mem));
5412   ins_pipe(pipe_class_dummy);
5413 %}
5414 
5415 // As compareAndSwapP, but return flag register instead of boolean value in
5416 // int register.
5417 // This instruction is matched if UseTLAB is off. Needed to pass
5418 // option tests.  Mem_ptr must be a memory operand, else this node
5419 // does not get Flag_needs_anti_dependence_check set by adlc. If this
5420 // is not set this node can be rematerialized which leads to errors.
5421 instruct storePConditional(indirect mem_ptr, rarg5RegP oldval, iRegP_N2P newval, flagsReg cr) %{
5422   match(Set cr (StorePConditional mem_ptr (Binary oldval newval)));
5423   effect(KILL oldval);
5424   // TODO: s390 port size(FIXED_SIZE);
5425   format %{ "storePConditional $oldval,$newval,$mem_ptr" %}
5426   ins_encode(z_enc_casL(oldval, newval, mem_ptr));
5427   ins_pipe(pipe_class_dummy);
5428 %}
5429 
5430 // As compareAndSwapL, but return flag register instead of boolean value in
5431 // int register.
5432 // Used by sun/misc/AtomicLongCSImpl.java. Mem_ptr must be a memory
5433 // operand, else this node does not get
5434 // Flag_needs_anti_dependence_check set by adlc. If this is not set
5435 // this node can be rematerialized which leads to errors.
5436 instruct storeLConditional(indirect mem_ptr, rarg5RegL oldval, iRegL newval, flagsReg cr) %{
5437   match(Set cr (StoreLConditional mem_ptr (Binary oldval newval)));
5438   effect(KILL oldval);
5439   // TODO: s390 port size(FIXED_SIZE);
5440   format %{ "storePConditional $oldval,$newval,$mem_ptr" %}
5441   ins_encode(z_enc_casL(oldval, newval, mem_ptr));
5442   ins_pipe(pipe_class_dummy);
5443 %}
5444 
5445 // No flag versions for CompareAndSwap{P,I,L,N} because matcher can't match them.
5446 
5447 instruct compareAndSwapI_bool(iRegP mem_ptr, rarg5RegI oldval, iRegI newval, iRegI res, flagsReg cr) %{
5448   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
5449   effect(USE mem_ptr, USE_KILL oldval, KILL cr);
5450   size(16);
5451   format %{ "$res = CompareAndSwapI $oldval,$newval,$mem_ptr" %}
5452   ins_encode(z_enc_casI(oldval, newval, mem_ptr),
5453              z_enc_cctobool(res));
5454   ins_pipe(pipe_class_dummy);
5455 %}
5456 
5457 instruct compareAndSwapL_bool(iRegP mem_ptr, rarg5RegL oldval, iRegL newval, iRegI res, flagsReg cr) %{
5458   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
5459   effect(USE mem_ptr, USE_KILL oldval, KILL cr);
5460   size(18);
5461   format %{ "$res = CompareAndSwapL $oldval,$newval,$mem_ptr" %}
5462   ins_encode(z_enc_casL(oldval, newval, mem_ptr),
5463              z_enc_cctobool(res));
5464   ins_pipe(pipe_class_dummy);
5465 %}
5466 
5467 instruct compareAndSwapP_bool(iRegP mem_ptr, rarg5RegP oldval, iRegP_N2P newval, iRegI res, flagsReg cr) %{
5468   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
5469   effect(USE mem_ptr, USE_KILL oldval, KILL cr);
5470   size(18);
5471   format %{ "$res = CompareAndSwapP $oldval,$newval,$mem_ptr" %}
5472   ins_encode(z_enc_casL(oldval, newval, mem_ptr),
5473              z_enc_cctobool(res));
5474   ins_pipe(pipe_class_dummy);
5475 %}
5476 
5477 instruct compareAndSwapN_bool(iRegP mem_ptr, rarg5RegN oldval, iRegN_P2N newval, iRegI res, flagsReg cr) %{
5478   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
5479   effect(USE mem_ptr, USE_KILL oldval, KILL cr);
5480   size(16);
5481   format %{ "$res = CompareAndSwapN $oldval,$newval,$mem_ptr" %}
5482   ins_encode(z_enc_casI(oldval, newval, mem_ptr),
5483              z_enc_cctobool(res));
5484   ins_pipe(pipe_class_dummy);
5485 %}
5486 
5487 //----------Atomic operations on memory (GetAndSet*, GetAndAdd*)---------------
5488 
5489 // Exploit: direct memory arithmetic
5490 // Prereqs: - instructions available
5491 //          - instructions guarantee atomicity
5492 //          - immediate operand to be added
5493 //          - immediate operand is small enough (8-bit signed).
5494 //          - result of instruction is not used
5495 instruct addI_mem_imm8_atomic_no_res(memoryRSY mem, Universe dummy, immI8 src, flagsReg cr) %{
5496   match(Set dummy (GetAndAddI mem src));
5497   effect(KILL cr);
5498   predicate(VM_Version::has_AtomicMemWithImmALUOps() && n->as_LoadStore()->result_not_used());
5499   ins_cost(MEMORY_REF_COST);
5500   size(6);
5501   format %{ "ASI     [$mem],$src\t # GetAndAddI (atomic)" %}
5502   opcode(ASI_ZOPC);
5503   ins_encode(z_siyform(mem, src));
5504   ins_pipe(pipe_class_dummy);
5505 %}
5506 
5507 // Fallback: direct memory arithmetic not available
5508 // Disadvantages: - CS-Loop required, very expensive.
5509 //                - more code generated (26 to xx bytes vs. 6 bytes)
5510 instruct addI_mem_imm16_atomic(memoryRSY mem, iRegI dst, immI16 src, iRegI tmp, flagsReg cr) %{
5511   match(Set dst (GetAndAddI mem src));
5512   effect(KILL cr, TEMP_DEF dst, TEMP tmp);
5513   ins_cost(MEMORY_REF_COST+100*DEFAULT_COST);
5514   format %{ "BEGIN ATOMIC {\n\t"
5515             "  LGF     $dst,[$mem]\n\t"
5516             "  AHIK    $tmp,$dst,$src\n\t"
5517             "  CSY     $dst,$tmp,$mem\n\t"
5518             "  retry if failed\n\t"
5519             "} END ATOMIC"
5520          %}
5521   ins_encode %{
5522     Register Rdst = $dst$$Register;
5523     Register Rtmp = $tmp$$Register;
5524     int      Isrc = $src$$constant;
5525     Label    retry;
5526 
5527     // Iterate until update with incremented value succeeds.
5528     __ z_lgf(Rdst, $mem$$Address);    // current contents
5529     __ bind(retry);
5530       // Calculate incremented value.
5531       if (VM_Version::has_DistinctOpnds()) {
5532         __ z_ahik(Rtmp, Rdst, Isrc);
5533       } else {
5534         __ z_lr(Rtmp, Rdst);
5535         __ z_ahi(Rtmp, Isrc);
5536       }
5537       // Swap into memory location.
5538       __ z_csy(Rdst, Rtmp, $mem$$Address); // Try to store new value.
5539     __ z_brne(retry);                      // Yikes, concurrent update, need to retry.
5540   %}
5541   ins_pipe(pipe_class_dummy);
5542 %}
5543 
5544 instruct addI_mem_imm32_atomic(memoryRSY mem, iRegI dst, immI src, iRegI tmp, flagsReg cr) %{
5545   match(Set dst (GetAndAddI mem src));
5546   effect(KILL cr, TEMP_DEF dst, TEMP tmp);
5547   ins_cost(MEMORY_REF_COST+200*DEFAULT_COST);
5548   format %{ "BEGIN ATOMIC {\n\t"
5549             "  LGF     $dst,[$mem]\n\t"
5550             "  LGR     $tmp,$dst\n\t"
5551             "  AFI     $tmp,$src\n\t"
5552             "  CSY     $dst,$tmp,$mem\n\t"
5553             "  retry if failed\n\t"
5554             "} END ATOMIC"
5555          %}
5556   ins_encode %{
5557     Register Rdst = $dst$$Register;
5558     Register Rtmp = $tmp$$Register;
5559     int      Isrc = $src$$constant;
5560     Label    retry;
5561 
5562     // Iterate until update with incremented value succeeds.
5563     __ z_lgf(Rdst, $mem$$Address);    // current contents
5564     __ bind(retry);
5565       // Calculate incremented value.
5566       __ z_lr(Rtmp, Rdst);
5567       __ z_afi(Rtmp, Isrc);
5568       // Swap into memory location.
5569       __ z_csy(Rdst, Rtmp, $mem$$Address); // Try to store new value.
5570     __ z_brne(retry);                      // Yikes, concurrent update, need to retry.
5571   %}
5572   ins_pipe(pipe_class_dummy);
5573 %}
5574 
5575 instruct addI_mem_reg_atomic(memoryRSY mem, iRegI dst, iRegI src, iRegI tmp, flagsReg cr) %{
5576   match(Set dst (GetAndAddI mem src));
5577   effect(KILL cr, TEMP_DEF dst, TEMP tmp);
5578   ins_cost(MEMORY_REF_COST+100*DEFAULT_COST);
5579   format %{ "BEGIN ATOMIC {\n\t"
5580             "  LGF     $dst,[$mem]\n\t"
5581             "  ARK     $tmp,$dst,$src\n\t"
5582             "  CSY     $dst,$tmp,$mem\n\t"
5583             "  retry if failed\n\t"
5584             "} END ATOMIC"
5585          %}
5586   ins_encode %{
5587     Register Rsrc = $src$$Register;
5588     Register Rdst = $dst$$Register;
5589     Register Rtmp = $tmp$$Register;
5590     Label    retry;
5591 
5592     // Iterate until update with incremented value succeeds.
5593     __ z_lgf(Rdst, $mem$$Address);  // current contents
5594     __ bind(retry);
5595       // Calculate incremented value.
5596       if (VM_Version::has_DistinctOpnds()) {
5597         __ z_ark(Rtmp, Rdst, Rsrc);
5598       } else {
5599         __ z_lr(Rtmp, Rdst);
5600         __ z_ar(Rtmp, Rsrc);
5601       }
5602       __ z_csy(Rdst, Rtmp, $mem$$Address); // Try to store new value.
5603     __ z_brne(retry);                      // Yikes, concurrent update, need to retry.
5604   %}
5605   ins_pipe(pipe_class_dummy);
5606 %}
5607 
5608 
5609 // Exploit: direct memory arithmetic
5610 // Prereqs: - instructions available
5611 //          - instructions guarantee atomicity
5612 //          - immediate operand to be added
5613 //          - immediate operand is small enough (8-bit signed).
5614 //          - result of instruction is not used
5615 instruct addL_mem_imm8_atomic_no_res(memoryRSY mem, Universe dummy, immL8 src, flagsReg cr) %{
5616   match(Set dummy (GetAndAddL mem src));
5617   effect(KILL cr);
5618   predicate(VM_Version::has_AtomicMemWithImmALUOps() && n->as_LoadStore()->result_not_used());
5619   ins_cost(MEMORY_REF_COST);
5620   size(6);
5621   format %{ "AGSI    [$mem],$src\t # GetAndAddL (atomic)" %}
5622   opcode(AGSI_ZOPC);
5623   ins_encode(z_siyform(mem, src));
5624   ins_pipe(pipe_class_dummy);
5625 %}
5626 
5627 // Fallback: direct memory arithmetic not available
5628 // Disadvantages: - CS-Loop required, very expensive.
5629 //                - more code generated (26 to xx bytes vs. 6 bytes)
5630 instruct addL_mem_imm16_atomic(memoryRSY mem, iRegL dst, immL16 src, iRegL tmp, flagsReg cr) %{
5631   match(Set dst (GetAndAddL mem src));
5632   effect(KILL cr, TEMP_DEF dst, TEMP tmp);
5633   ins_cost(MEMORY_REF_COST+100*DEFAULT_COST);
5634   format %{ "BEGIN ATOMIC {\n\t"
5635             "  LG      $dst,[$mem]\n\t"
5636             "  AGHIK   $tmp,$dst,$src\n\t"
5637             "  CSG     $dst,$tmp,$mem\n\t"
5638             "  retry if failed\n\t"
5639             "} END ATOMIC"
5640          %}
5641   ins_encode %{
5642     Register Rdst = $dst$$Register;
5643     Register Rtmp = $tmp$$Register;
5644     int      Isrc = $src$$constant;
5645     Label    retry;
5646 
5647     // Iterate until update with incremented value succeeds.
5648     __ z_lg(Rdst, $mem$$Address);  // current contents
5649     __ bind(retry);
5650       // Calculate incremented value.
5651       if (VM_Version::has_DistinctOpnds()) {
5652         __ z_aghik(Rtmp, Rdst, Isrc);
5653       } else {
5654         __ z_lgr(Rtmp, Rdst);
5655         __ z_aghi(Rtmp, Isrc);
5656       }
5657       __ z_csg(Rdst, Rtmp, $mem$$Address); // Try to store new value.
5658     __ z_brne(retry);                      // Yikes, concurrent update, need to retry.
5659   %}
5660   ins_pipe(pipe_class_dummy);
5661 %}
5662 
5663 instruct addL_mem_imm32_atomic(memoryRSY mem, iRegL dst, immL32 src, iRegL tmp, flagsReg cr) %{
5664   match(Set dst (GetAndAddL mem src));
5665   effect(KILL cr, TEMP_DEF dst, TEMP tmp);
5666   ins_cost(MEMORY_REF_COST+100*DEFAULT_COST);
5667   format %{ "BEGIN ATOMIC {\n\t"
5668             "  LG      $dst,[$mem]\n\t"
5669             "  LGR     $tmp,$dst\n\t"
5670             "  AGFI    $tmp,$src\n\t"
5671             "  CSG     $dst,$tmp,$mem\n\t"
5672             "  retry if failed\n\t"
5673             "} END ATOMIC"
5674          %}
5675   ins_encode %{
5676     Register Rdst = $dst$$Register;
5677     Register Rtmp = $tmp$$Register;
5678     int      Isrc = $src$$constant;
5679     Label    retry;
5680 
5681     // Iterate until update with incremented value succeeds.
5682     __ z_lg(Rdst, $mem$$Address);  // current contents
5683     __ bind(retry);
5684       // Calculate incremented value.
5685       __ z_lgr(Rtmp, Rdst);
5686       __ z_agfi(Rtmp, Isrc);
5687       __ z_csg(Rdst, Rtmp, $mem$$Address); // Try to store new value.
5688     __ z_brne(retry);                      // Yikes, concurrent update, need to retry.
5689   %}
5690   ins_pipe(pipe_class_dummy);
5691 %}
5692 
5693 instruct addL_mem_reg_atomic(memoryRSY mem, iRegL dst, iRegL src, iRegL tmp, flagsReg cr) %{
5694   match(Set dst (GetAndAddL mem src));
5695   effect(KILL cr, TEMP_DEF dst, TEMP tmp);
5696   ins_cost(MEMORY_REF_COST+100*DEFAULT_COST);
5697   format %{ "BEGIN ATOMIC {\n\t"
5698             "  LG      $dst,[$mem]\n\t"
5699             "  AGRK    $tmp,$dst,$src\n\t"
5700             "  CSG     $dst,$tmp,$mem\n\t"
5701             "  retry if failed\n\t"
5702             "} END ATOMIC"
5703          %}
5704   ins_encode %{
5705     Register Rsrc = $src$$Register;
5706     Register Rdst = $dst$$Register;
5707     Register Rtmp = $tmp$$Register;
5708     Label    retry;
5709 
5710     // Iterate until update with incremented value succeeds.
5711     __ z_lg(Rdst, $mem$$Address);  // current contents
5712     __ bind(retry);
5713       // Calculate incremented value.
5714       if (VM_Version::has_DistinctOpnds()) {
5715         __ z_agrk(Rtmp, Rdst, Rsrc);
5716       } else {
5717         __ z_lgr(Rtmp, Rdst);
5718         __ z_agr(Rtmp, Rsrc);
5719       }
5720       __ z_csg(Rdst, Rtmp, $mem$$Address); // Try to store new value.
5721     __ z_brne(retry);                      // Yikes, concurrent update, need to retry.
5722   %}
5723   ins_pipe(pipe_class_dummy);
5724 %}
5725 
5726 // Increment value in memory, save old value in dst.
5727 instruct addI_mem_reg_atomic_z196(memoryRSY mem, iRegI dst, iRegI src) %{
5728   match(Set dst (GetAndAddI mem src));
5729   predicate(VM_Version::has_LoadAndALUAtomicV1());
5730   ins_cost(MEMORY_REF_COST + DEFAULT_COST);
5731   size(6);
5732   format %{ "LAA     $dst,$src,[$mem]" %}
5733   ins_encode %{ __ z_laa($dst$$Register, $src$$Register, $mem$$Address); %}
5734   ins_pipe(pipe_class_dummy);
5735 %}
5736 
5737 // Increment value in memory, save old value in dst.
5738 instruct addL_mem_reg_atomic_z196(memoryRSY mem, iRegL dst, iRegL src) %{
5739   match(Set dst (GetAndAddL mem src));
5740   predicate(VM_Version::has_LoadAndALUAtomicV1());
5741   ins_cost(MEMORY_REF_COST + DEFAULT_COST);
5742   size(6);
5743   format %{ "LAAG    $dst,$src,[$mem]" %}
5744   ins_encode %{ __ z_laag($dst$$Register, $src$$Register, $mem$$Address); %}
5745   ins_pipe(pipe_class_dummy);
5746 %}
5747 
5748 
5749 instruct xchgI_reg_mem(memoryRSY mem, iRegI dst, iRegI tmp, flagsReg cr) %{
5750   match(Set dst (GetAndSetI mem dst));
5751   effect(KILL cr, TEMP tmp); // USE_DEF dst by match rule.
5752   format %{ "XCHGI   $dst,[$mem]\t # EXCHANGE (int, atomic), temp $tmp" %}
5753   ins_encode(z_enc_SwapI(mem, dst, tmp));
5754   ins_pipe(pipe_class_dummy);
5755 %}
5756 
5757 instruct xchgL_reg_mem(memoryRSY mem, iRegL dst, iRegL tmp, flagsReg cr) %{
5758   match(Set dst (GetAndSetL mem dst));
5759   effect(KILL cr, TEMP tmp); // USE_DEF dst by match rule.
5760   format %{ "XCHGL   $dst,[$mem]\t # EXCHANGE (long, atomic), temp $tmp" %}
5761   ins_encode(z_enc_SwapL(mem, dst, tmp));
5762   ins_pipe(pipe_class_dummy);
5763 %}
5764 
5765 instruct xchgN_reg_mem(memoryRSY mem, iRegN dst, iRegI tmp, flagsReg cr) %{
5766   match(Set dst (GetAndSetN mem dst));
5767   effect(KILL cr, TEMP tmp); // USE_DEF dst by match rule.
5768   format %{ "XCHGN   $dst,[$mem]\t # EXCHANGE (coop, atomic), temp $tmp" %}
5769   ins_encode(z_enc_SwapI(mem, dst, tmp));
5770   ins_pipe(pipe_class_dummy);
5771 %}
5772 
5773 instruct xchgP_reg_mem(memoryRSY mem, iRegP dst, iRegL tmp, flagsReg cr) %{
5774   match(Set dst (GetAndSetP mem dst));
5775   effect(KILL cr, TEMP tmp); // USE_DEF dst by match rule.
5776   format %{ "XCHGP   $dst,[$mem]\t # EXCHANGE (oop, atomic), temp $tmp" %}
5777   ins_encode(z_enc_SwapL(mem, dst, tmp));
5778   ins_pipe(pipe_class_dummy);
5779 %}
5780 
5781 
5782 //----------Arithmetic Instructions--------------------------------------------
5783 
5784 // The rules are sorted by right operand type and operand length. Please keep
5785 // it that way.
5786 // Left operand type is always reg. Left operand len is I, L, P
5787 // Right operand type is reg, imm, mem. Right operand len is S, I, L, P
5788 // Special instruction formats, e.g. multi-operand, are inserted at the end.
5789 
5790 // ADD
5791 
5792 // REG = REG + REG
5793 
5794 // Register Addition
5795 instruct addI_reg_reg_CISC(iRegI dst, iRegI src, flagsReg cr) %{
5796   match(Set dst (AddI dst src));
5797   effect(KILL cr);
5798   // TODO: s390 port size(FIXED_SIZE);
5799   format %{ "AR      $dst,$src\t # int  CISC ALU" %}
5800   opcode(AR_ZOPC);
5801   ins_encode(z_rrform(dst, src));
5802   ins_pipe(pipe_class_dummy);
5803 %}
5804 
5805 // Avoid use of LA(Y) for general ALU operation.
5806 instruct addI_reg_reg_RISC(iRegI dst, iRegI src1, iRegI src2, flagsReg cr) %{
5807   match(Set dst (AddI src1 src2));
5808   effect(KILL cr);
5809   predicate(VM_Version::has_DistinctOpnds());
5810   ins_cost(DEFAULT_COST);
5811   size(4);
5812   format %{ "ARK     $dst,$src1,$src2\t # int  RISC ALU" %}
5813   opcode(ARK_ZOPC);
5814   ins_encode(z_rrfform(dst, src1, src2));
5815   ins_pipe(pipe_class_dummy);
5816 %}
5817 
5818 // REG = REG + IMM
5819 
5820 // Avoid use of LA(Y) for general ALU operation.
5821 // Immediate Addition
5822 instruct addI_reg_imm16_CISC(iRegI dst, immI16 con, flagsReg cr) %{
5823   match(Set dst (AddI dst con));
5824   effect(KILL cr);
5825   ins_cost(DEFAULT_COST);
5826   // TODO: s390 port size(FIXED_SIZE);
5827   format %{ "AHI     $dst,$con\t # int  CISC ALU" %}
5828   opcode(AHI_ZOPC);
5829   ins_encode(z_riform_signed(dst, con));
5830   ins_pipe(pipe_class_dummy);
5831 %}
5832 
5833 // Avoid use of LA(Y) for general ALU operation.
5834 // Immediate Addition
5835 instruct addI_reg_imm16_RISC(iRegI dst, iRegI src, immI16 con, flagsReg cr) %{
5836   match(Set dst (AddI src con));
5837   effect(KILL cr);
5838   predicate( VM_Version::has_DistinctOpnds());
5839   ins_cost(DEFAULT_COST);
5840   // TODO: s390 port size(FIXED_SIZE);
5841   format %{ "AHIK    $dst,$src,$con\t # int  RISC ALU" %}
5842   opcode(AHIK_ZOPC);
5843   ins_encode(z_rieform_d(dst, src, con));
5844   ins_pipe(pipe_class_dummy);
5845 %}
5846 
5847 // Immediate Addition
5848 instruct addI_reg_imm32(iRegI dst, immI src, flagsReg cr) %{
5849   match(Set dst (AddI dst src));
5850   effect(KILL cr);
5851   ins_cost(DEFAULT_COST_HIGH);
5852   size(6);
5853   format %{ "AFI     $dst,$src" %}
5854   opcode(AFI_ZOPC);
5855   ins_encode(z_rilform_signed(dst, src));
5856   ins_pipe(pipe_class_dummy);
5857 %}
5858 
5859 // Immediate Addition
5860 instruct addI_reg_imm12(iRegI dst, iRegI src, uimmI12 con) %{
5861   match(Set dst (AddI src con));
5862   predicate(PreferLAoverADD);
5863   ins_cost(DEFAULT_COST_LOW);
5864   size(4);
5865   format %{ "LA      $dst,$con(,$src)\t # int d12(,b)" %}
5866   opcode(LA_ZOPC);
5867   ins_encode(z_rxform_imm_reg(dst, con, src));
5868   ins_pipe(pipe_class_dummy);
5869 %}
5870 
5871 // Immediate Addition
5872 instruct addI_reg_imm20(iRegI dst, iRegI src, immI20 con) %{
5873   match(Set dst (AddI src con));
5874   predicate(PreferLAoverADD);
5875   ins_cost(DEFAULT_COST);
5876   size(6);
5877   format %{ "LAY     $dst,$con(,$src)\t # int d20(,b)" %}
5878   opcode(LAY_ZOPC);
5879   ins_encode(z_rxyform_imm_reg(dst, con, src));
5880   ins_pipe(pipe_class_dummy);
5881 %}
5882 
5883 instruct addI_reg_reg_imm12(iRegI dst, iRegI src1, iRegI src2, uimmI12 con) %{
5884   match(Set dst (AddI (AddI src1 src2) con));
5885   predicate( PreferLAoverADD);
5886   ins_cost(DEFAULT_COST_LOW);
5887   size(4);
5888   format %{ "LA      $dst,$con($src1,$src2)\t # int d12(x,b)" %}
5889   opcode(LA_ZOPC);
5890   ins_encode(z_rxform_imm_reg_reg(dst, con, src1, src2));
5891   ins_pipe(pipe_class_dummy);
5892 %}
5893 
5894 instruct addI_reg_reg_imm20(iRegI dst, iRegI src1, iRegI src2, immI20 con) %{
5895   match(Set dst (AddI (AddI src1 src2) con));
5896   predicate(PreferLAoverADD);
5897   ins_cost(DEFAULT_COST);
5898   size(6);
5899   format %{ "LAY     $dst,$con($src1,$src2)\t # int d20(x,b)" %}
5900   opcode(LAY_ZOPC);
5901   ins_encode(z_rxyform_imm_reg_reg(dst, con, src1, src2));
5902   ins_pipe(pipe_class_dummy);
5903 %}
5904 
5905 // REG = REG + MEM
5906 
5907 instruct addI_Reg_mem(iRegI dst, memory src, flagsReg cr)%{
5908   match(Set dst (AddI dst (LoadI src)));
5909   effect(KILL cr);
5910   ins_cost(MEMORY_REF_COST);
5911   // TODO: s390 port size(VARIABLE_SIZE);
5912   format %{ "A(Y)    $dst, $src\t # int" %}
5913   opcode(AY_ZOPC, A_ZOPC);
5914   ins_encode(z_form_rt_mem_opt(dst, src));
5915   ins_pipe(pipe_class_dummy);
5916 %}
5917 
5918 // MEM = MEM + IMM
5919 
5920 // Add Immediate to 4-byte memory operand and result
5921 instruct addI_mem_imm(memoryRSY mem, immI8 src, flagsReg cr) %{
5922   match(Set mem (StoreI mem (AddI (LoadI mem) src)));
5923   effect(KILL cr);
5924   predicate(VM_Version::has_MemWithImmALUOps());
5925   ins_cost(MEMORY_REF_COST);
5926   size(6);
5927   format %{ "ASI     $mem,$src\t # direct mem add 4" %}
5928   opcode(ASI_ZOPC);
5929   ins_encode(z_siyform(mem, src));
5930   ins_pipe(pipe_class_dummy);
5931 %}
5932 
5933 
5934 //
5935 
5936 // REG = REG + REG
5937 
5938 instruct addL_reg_regI(iRegL dst, iRegI src, flagsReg cr) %{
5939   match(Set dst (AddL dst (ConvI2L src)));
5940   effect(KILL cr);
5941   size(4);
5942   format %{ "AGFR    $dst,$src\t # long<-int CISC ALU" %}
5943   opcode(AGFR_ZOPC);
5944   ins_encode(z_rreform(dst, src));
5945   ins_pipe(pipe_class_dummy);
5946 %}
5947 
5948 instruct addL_reg_reg_CISC(iRegL dst, iRegL src, flagsReg cr) %{
5949   match(Set dst (AddL dst src));
5950   effect(KILL cr);
5951   // TODO: s390 port size(FIXED_SIZE);
5952   format %{ "AGR     $dst, $src\t # long CISC ALU" %}
5953   opcode(AGR_ZOPC);
5954   ins_encode(z_rreform(dst, src));
5955   ins_pipe(pipe_class_dummy);
5956 %}
5957 
5958 // Avoid use of LA(Y) for general ALU operation.
5959 instruct addL_reg_reg_RISC(iRegL dst, iRegL src1, iRegL src2, flagsReg cr) %{
5960   match(Set dst (AddL src1 src2));
5961   effect(KILL cr);
5962   predicate(VM_Version::has_DistinctOpnds());
5963   ins_cost(DEFAULT_COST);
5964   size(4);
5965   format %{ "AGRK    $dst,$src1,$src2\t # long RISC ALU" %}
5966   opcode(AGRK_ZOPC);
5967   ins_encode(z_rrfform(dst, src1, src2));
5968   ins_pipe(pipe_class_dummy);
5969 %}
5970 
5971 // REG = REG + IMM
5972 
5973 instruct addL_reg_imm12(iRegL dst, iRegL src, uimmL12 con) %{
5974   match(Set dst (AddL src con));
5975   predicate( PreferLAoverADD);
5976   ins_cost(DEFAULT_COST_LOW);
5977   size(4);
5978   format %{ "LA      $dst,$con(,$src)\t # long d12(,b)" %}
5979   opcode(LA_ZOPC);
5980   ins_encode(z_rxform_imm_reg(dst, con, src));
5981   ins_pipe(pipe_class_dummy);
5982 %}
5983 
5984 instruct addL_reg_imm20(iRegL dst, iRegL src, immL20 con) %{
5985   match(Set dst (AddL src con));
5986   predicate(PreferLAoverADD);
5987   ins_cost(DEFAULT_COST);
5988   size(6);
5989   format %{ "LAY     $dst,$con(,$src)\t # long d20(,b)" %}
5990   opcode(LAY_ZOPC);
5991   ins_encode(z_rxyform_imm_reg(dst, con, src));
5992   ins_pipe(pipe_class_dummy);
5993 %}
5994 
5995 instruct addL_reg_imm32(iRegL dst, immL32 con, flagsReg cr) %{
5996   match(Set dst (AddL dst con));
5997   effect(KILL cr);
5998   ins_cost(DEFAULT_COST_HIGH);
5999   size(6);
6000   format %{ "AGFI    $dst,$con\t # long CISC ALU" %}
6001   opcode(AGFI_ZOPC);
6002   ins_encode(z_rilform_signed(dst, con));
6003   ins_pipe(pipe_class_dummy);
6004 %}
6005 
6006 // Avoid use of LA(Y) for general ALU operation.
6007 instruct addL_reg_imm16_CISC(iRegL dst, immL16 con, flagsReg cr) %{
6008   match(Set dst (AddL dst con));
6009   effect(KILL cr);
6010   ins_cost(DEFAULT_COST);
6011   // TODO: s390 port size(FIXED_SIZE);
6012   format %{ "AGHI    $dst,$con\t # long CISC ALU" %}
6013   opcode(AGHI_ZOPC);
6014   ins_encode(z_riform_signed(dst, con));
6015   ins_pipe(pipe_class_dummy);
6016 %}
6017 
6018 // Avoid use of LA(Y) for general ALU operation.
6019 instruct addL_reg_imm16_RISC(iRegL dst, iRegL src, immL16 con, flagsReg cr) %{
6020   match(Set dst (AddL src con));
6021   effect(KILL cr);
6022   predicate( VM_Version::has_DistinctOpnds());
6023   ins_cost(DEFAULT_COST);
6024   size(6);
6025   format %{ "AGHIK   $dst,$src,$con\t # long RISC ALU" %}
6026   opcode(AGHIK_ZOPC);
6027   ins_encode(z_rieform_d(dst, src, con));
6028   ins_pipe(pipe_class_dummy);
6029 %}
6030 
6031 // REG = REG + MEM
6032 
6033 instruct addL_Reg_memI(iRegL dst, memory src, flagsReg cr)%{
6034   match(Set dst (AddL dst (ConvI2L (LoadI src))));
6035   effect(KILL cr);
6036   ins_cost(MEMORY_REF_COST);
6037   size(Z_DISP3_SIZE);
6038   format %{ "AGF     $dst, $src\t # long/int" %}
6039   opcode(AGF_ZOPC, AGF_ZOPC);
6040   ins_encode(z_form_rt_mem_opt(dst, src));
6041   ins_pipe(pipe_class_dummy);
6042 %}
6043 
6044 instruct addL_Reg_mem(iRegL dst, memory src, flagsReg cr)%{
6045   match(Set dst (AddL dst (LoadL src)));
6046   effect(KILL cr);
6047   ins_cost(MEMORY_REF_COST);
6048   size(Z_DISP3_SIZE);
6049   format %{ "AG      $dst, $src\t # long" %}
6050   opcode(AG_ZOPC, AG_ZOPC);
6051   ins_encode(z_form_rt_mem_opt(dst, src));
6052   ins_pipe(pipe_class_dummy);
6053 %}
6054 
6055 instruct addL_reg_reg_imm12(iRegL dst, iRegL src1, iRegL src2, uimmL12 con) %{
6056   match(Set dst (AddL (AddL src1 src2) con));
6057   predicate( PreferLAoverADD);
6058   ins_cost(DEFAULT_COST_LOW);
6059   size(4);
6060   format %{ "LA     $dst,$con($src1,$src2)\t # long d12(x,b)" %}
6061   opcode(LA_ZOPC);
6062   ins_encode(z_rxform_imm_reg_reg(dst, con, src1, src2));
6063   ins_pipe(pipe_class_dummy);
6064 %}
6065 
6066 instruct addL_reg_reg_imm20(iRegL dst, iRegL src1, iRegL src2, immL20 con) %{
6067   match(Set dst (AddL (AddL src1 src2) con));
6068   predicate(PreferLAoverADD);
6069   ins_cost(DEFAULT_COST);
6070   size(6);
6071   format %{ "LAY    $dst,$con($src1,$src2)\t # long d20(x,b)" %}
6072   opcode(LAY_ZOPC);
6073   ins_encode(z_rxyform_imm_reg_reg(dst, con, src1, src2));
6074   ins_pipe(pipe_class_dummy);
6075 %}
6076 
6077 // MEM = MEM + IMM
6078 
6079 // Add Immediate to 8-byte memory operand and result.
6080 instruct addL_mem_imm(memoryRSY mem, immL8 src, flagsReg cr) %{
6081   match(Set mem (StoreL mem (AddL (LoadL mem) src)));
6082   effect(KILL cr);
6083   predicate(VM_Version::has_MemWithImmALUOps());
6084   ins_cost(MEMORY_REF_COST);
6085   size(6);
6086   format %{ "AGSI    $mem,$src\t # direct mem add 8" %}
6087   opcode(AGSI_ZOPC);
6088   ins_encode(z_siyform(mem, src));
6089   ins_pipe(pipe_class_dummy);
6090 %}
6091 
6092 
6093 // REG = REG + REG
6094 
6095 // Ptr Addition
6096 instruct addP_reg_reg_LA(iRegP dst, iRegP_N2P src1, iRegL src2) %{
6097   match(Set dst (AddP src1 src2));
6098   predicate( PreferLAoverADD);
6099   ins_cost(DEFAULT_COST);
6100   size(4);
6101   format %{ "LA      $dst,#0($src1,$src2)\t # ptr 0(x,b)" %}
6102   opcode(LA_ZOPC);
6103   ins_encode(z_rxform_imm_reg_reg(dst, 0x0, src1, src2));
6104   ins_pipe(pipe_class_dummy);
6105 %}
6106 
6107 // Ptr Addition
6108 // Avoid use of LA(Y) for general ALU operation.
6109 instruct addP_reg_reg_CISC(iRegP dst, iRegL src, flagsReg cr) %{
6110   match(Set dst (AddP dst src));
6111   effect(KILL cr);
6112   predicate(!PreferLAoverADD && !VM_Version::has_DistinctOpnds());
6113   ins_cost(DEFAULT_COST);
6114   // TODO: s390 port size(FIXED_SIZE);
6115   format %{ "ALGR    $dst,$src\t # ptr CICS ALU" %}
6116   opcode(ALGR_ZOPC);
6117   ins_encode(z_rreform(dst, src));
6118   ins_pipe(pipe_class_dummy);
6119 %}
6120 
6121 // Ptr Addition
6122 // Avoid use of LA(Y) for general ALU operation.
6123 instruct addP_reg_reg_RISC(iRegP dst, iRegP_N2P src1, iRegL src2, flagsReg cr) %{
6124   match(Set dst (AddP src1 src2));
6125   effect(KILL cr);
6126   predicate(!PreferLAoverADD && VM_Version::has_DistinctOpnds());
6127   ins_cost(DEFAULT_COST);
6128   // TODO: s390 port size(FIXED_SIZE);
6129   format %{ "ALGRK   $dst,$src1,$src2\t # ptr RISC ALU" %}
6130   opcode(ALGRK_ZOPC);
6131   ins_encode(z_rrfform(dst, src1, src2));
6132   ins_pipe(pipe_class_dummy);
6133 %}
6134 
6135 // REG = REG + IMM
6136 
6137 instruct addP_reg_imm12(iRegP dst, iRegP_N2P src, uimmL12 con) %{
6138   match(Set dst (AddP src con));
6139   predicate( PreferLAoverADD);
6140   ins_cost(DEFAULT_COST_LOW);
6141   size(4);
6142   format %{ "LA      $dst,$con(,$src)\t # ptr d12(,b)" %}
6143   opcode(LA_ZOPC);
6144   ins_encode(z_rxform_imm_reg(dst, con, src));
6145   ins_pipe(pipe_class_dummy);
6146 %}
6147 
6148 // Avoid use of LA(Y) for general ALU operation.
6149 instruct addP_reg_imm16_CISC(iRegP dst, immL16 src, flagsReg cr) %{
6150   match(Set dst (AddP dst src));
6151   effect(KILL cr);
6152   predicate(!PreferLAoverADD && !VM_Version::has_DistinctOpnds());
6153   ins_cost(DEFAULT_COST);
6154   // TODO: s390 port size(FIXED_SIZE);
6155   format %{ "AGHI    $dst,$src\t # ptr CISC ALU" %}
6156   opcode(AGHI_ZOPC);
6157   ins_encode(z_riform_signed(dst, src));
6158   ins_pipe(pipe_class_dummy);
6159 %}
6160 
6161 // Avoid use of LA(Y) for general ALU operation.
6162 instruct addP_reg_imm16_RISC(iRegP dst, iRegP_N2P src, immL16 con, flagsReg cr) %{
6163   match(Set dst (AddP src con));
6164   effect(KILL cr);
6165   predicate(!PreferLAoverADD && VM_Version::has_DistinctOpnds());
6166   ins_cost(DEFAULT_COST);
6167   // TODO: s390 port size(FIXED_SIZE);
6168   format %{ "ALGHSIK $dst,$src,$con\t # ptr RISC ALU" %}
6169   opcode(ALGHSIK_ZOPC);
6170   ins_encode(z_rieform_d(dst, src, con));
6171   ins_pipe(pipe_class_dummy);
6172 %}
6173 
6174 instruct addP_reg_imm20(iRegP dst, memoryRegP src, immL20 con) %{
6175   match(Set dst (AddP src con));
6176   predicate(PreferLAoverADD);
6177   ins_cost(DEFAULT_COST);
6178   size(6);
6179   format %{ "LAY     $dst,$con(,$src)\t # ptr d20(,b)" %}
6180   opcode(LAY_ZOPC);
6181   ins_encode(z_rxyform_imm_reg(dst, con, src));
6182   ins_pipe(pipe_class_dummy);
6183 %}
6184 
6185 // Pointer Immediate Addition
6186 instruct addP_reg_imm32(iRegP dst, immL32 src, flagsReg cr) %{
6187   match(Set dst (AddP dst src));
6188   effect(KILL cr);
6189   ins_cost(DEFAULT_COST_HIGH);
6190   // TODO: s390 port size(FIXED_SIZE);
6191   format %{ "AGFI    $dst,$src\t # ptr" %}
6192   opcode(AGFI_ZOPC);
6193   ins_encode(z_rilform_signed(dst, src));
6194   ins_pipe(pipe_class_dummy);
6195 %}
6196 
6197 // REG = REG1 + REG2 + IMM
6198 
6199 instruct addP_reg_reg_imm12(iRegP dst, memoryRegP src1, iRegL src2, uimmL12 con) %{
6200   match(Set dst (AddP (AddP src1 src2) con));
6201   predicate( PreferLAoverADD);
6202   ins_cost(DEFAULT_COST_LOW);
6203   size(4);
6204   format %{ "LA      $dst,$con($src1,$src2)\t # ptr d12(x,b)" %}
6205   opcode(LA_ZOPC);
6206   ins_encode(z_rxform_imm_reg_reg(dst, con, src1, src2));
6207   ins_pipe(pipe_class_dummy);
6208 %}
6209 
6210 instruct addP_regN_reg_imm12(iRegP dst, iRegP_N2P src1, iRegL src2, uimmL12 con) %{
6211   match(Set dst (AddP (AddP src1 src2) con));
6212   predicate( PreferLAoverADD && Universe::narrow_oop_base() == NULL && Universe::narrow_oop_shift() == 0);
6213   ins_cost(DEFAULT_COST_LOW);
6214   size(4);
6215   format %{ "LA      $dst,$con($src1,$src2)\t # ptr d12(x,b)" %}
6216   opcode(LA_ZOPC);
6217   ins_encode(z_rxform_imm_reg_reg(dst, con, src1, src2));
6218   ins_pipe(pipe_class_dummy);
6219 %}
6220 
6221 instruct addP_reg_reg_imm20(iRegP dst, memoryRegP src1, iRegL src2, immL20 con) %{
6222   match(Set dst (AddP (AddP src1 src2) con));
6223   predicate(PreferLAoverADD);
6224   ins_cost(DEFAULT_COST);
6225   // TODO: s390 port size(FIXED_SIZE);
6226   format %{ "LAY     $dst,$con($src1,$src2)\t # ptr d20(x,b)" %}
6227   opcode(LAY_ZOPC);
6228   ins_encode(z_rxyform_imm_reg_reg(dst, con, src1, src2));
6229   ins_pipe(pipe_class_dummy);
6230 %}
6231 
6232 instruct addP_regN_reg_imm20(iRegP dst, iRegP_N2P src1, iRegL src2, immL20 con) %{
6233   match(Set dst (AddP (AddP src1 src2) con));
6234   predicate( PreferLAoverADD && Universe::narrow_oop_base() == NULL && Universe::narrow_oop_shift() == 0);
6235   ins_cost(DEFAULT_COST);
6236   // TODO: s390 port size(FIXED_SIZE);
6237   format %{ "LAY     $dst,$con($src1,$src2)\t # ptr d20(x,b)" %}
6238   opcode(LAY_ZOPC);
6239   ins_encode(z_rxyform_imm_reg_reg(dst, con, src1, src2));
6240   ins_pipe(pipe_class_dummy);
6241 %}
6242 
6243 // MEM = MEM + IMM
6244 
6245 // Add Immediate to 8-byte memory operand and result
6246 instruct addP_mem_imm(memoryRSY mem, immL8 src, flagsReg cr) %{
6247   match(Set mem (StoreP mem (AddP (LoadP mem) src)));
6248   effect(KILL cr);
6249   predicate(VM_Version::has_MemWithImmALUOps());
6250   ins_cost(MEMORY_REF_COST);
6251   size(6);
6252   format %{ "AGSI    $mem,$src\t # direct mem add 8 (ptr)" %}
6253   opcode(AGSI_ZOPC);
6254   ins_encode(z_siyform(mem, src));
6255   ins_pipe(pipe_class_dummy);
6256 %}
6257 
6258 // SUB
6259 
6260 // Register Subtraction
6261 instruct subI_reg_reg_CISC(iRegI dst, iRegI src, flagsReg cr) %{
6262   match(Set dst (SubI dst src));
6263   effect(KILL cr);
6264   // TODO: s390 port size(FIXED_SIZE);
6265   format %{ "SR      $dst,$src\t # int  CISC ALU" %}
6266   opcode(SR_ZOPC);
6267   ins_encode(z_rrform(dst, src));
6268   ins_pipe(pipe_class_dummy);
6269 %}
6270 
6271 instruct subI_reg_reg_RISC(iRegI dst, iRegI src1, iRegI src2, flagsReg cr) %{
6272   match(Set dst (SubI src1 src2));
6273   effect(KILL cr);
6274   predicate(VM_Version::has_DistinctOpnds());
6275   ins_cost(DEFAULT_COST);
6276   size(4);
6277   format %{ "SRK     $dst,$src1,$src2\t # int  RISC ALU" %}
6278   opcode(SRK_ZOPC);
6279   ins_encode(z_rrfform(dst, src1, src2));
6280   ins_pipe(pipe_class_dummy);
6281 %}
6282 
6283 instruct subI_Reg_mem(iRegI dst, memory src, flagsReg cr)%{
6284   match(Set dst (SubI dst (LoadI src)));
6285   effect(KILL cr);
6286   ins_cost(MEMORY_REF_COST);
6287   // TODO: s390 port size(VARIABLE_SIZE);
6288   format %{ "S(Y)    $dst, $src\t # int" %}
6289   opcode(SY_ZOPC, S_ZOPC);
6290   ins_encode(z_form_rt_mem_opt(dst, src));
6291   ins_pipe(pipe_class_dummy);
6292 %}
6293 
6294 instruct subI_zero_reg(iRegI dst, immI_0 zero, iRegI src, flagsReg cr) %{
6295   match(Set dst (SubI zero src));
6296   effect(KILL cr);
6297   size(2);
6298   format %{ "NEG     $dst, $src" %}
6299   ins_encode %{ __ z_lcr($dst$$Register, $src$$Register); %}
6300   ins_pipe(pipe_class_dummy);
6301 %}
6302 
6303 //
6304 
6305 // Long subtraction
6306 instruct subL_reg_reg_CISC(iRegL dst, iRegL src, flagsReg cr) %{
6307   match(Set dst (SubL dst src));
6308   effect(KILL cr);
6309   // TODO: s390 port size(FIXED_SIZE);
6310   format %{ "SGR     $dst,$src\t # int  CISC ALU" %}
6311   opcode(SGR_ZOPC);
6312   ins_encode(z_rreform(dst, src));
6313   ins_pipe(pipe_class_dummy);
6314 %}
6315 
6316 // Avoid use of LA(Y) for general ALU operation.
6317 instruct subL_reg_reg_RISC(iRegL dst, iRegL src1, iRegL src2, flagsReg cr) %{
6318   match(Set dst (SubL src1 src2));
6319   effect(KILL cr);
6320   predicate(VM_Version::has_DistinctOpnds());
6321   ins_cost(DEFAULT_COST);
6322   size(4);
6323   format %{ "SGRK    $dst,$src1,$src2\t # int  RISC ALU" %}
6324   opcode(SGRK_ZOPC);
6325   ins_encode(z_rrfform(dst, src1, src2));
6326   ins_pipe(pipe_class_dummy);
6327 %}
6328 
6329 instruct subL_reg_regI_CISC(iRegL dst, iRegI src, flagsReg cr) %{
6330   match(Set dst (SubL dst (ConvI2L src)));
6331   effect(KILL cr);
6332   size(4);
6333   format %{ "SGFR    $dst, $src\t # int  CISC ALU" %}
6334   opcode(SGFR_ZOPC);
6335   ins_encode(z_rreform(dst, src));
6336   ins_pipe(pipe_class_dummy);
6337 %}
6338 
6339 instruct subL_Reg_memI(iRegL dst, memory src, flagsReg cr)%{
6340   match(Set dst (SubL dst (ConvI2L (LoadI src))));
6341   effect(KILL cr);
6342   ins_cost(MEMORY_REF_COST);
6343   size(Z_DISP3_SIZE);
6344   format %{ "SGF     $dst, $src\t # long/int" %}
6345   opcode(SGF_ZOPC, SGF_ZOPC);
6346   ins_encode(z_form_rt_mem_opt(dst, src));
6347   ins_pipe(pipe_class_dummy);
6348 %}
6349 
6350 instruct subL_Reg_mem(iRegL dst, memory src, flagsReg cr)%{
6351   match(Set dst (SubL dst (LoadL src)));
6352   effect(KILL cr);
6353   ins_cost(MEMORY_REF_COST);
6354   size(Z_DISP3_SIZE);
6355   format %{ "SG      $dst, $src\t # long" %}
6356   opcode(SG_ZOPC, SG_ZOPC);
6357   ins_encode(z_form_rt_mem_opt(dst, src));
6358   ins_pipe(pipe_class_dummy);
6359 %}
6360 
6361 // Moved declaration of negL_reg_reg before encode nodes, where it is used.
6362 
6363 //  MUL
6364 
6365 // Register Multiplication
6366 instruct mulI_reg_reg(iRegI dst, iRegI src) %{
6367   match(Set dst (MulI dst src));
6368   ins_cost(DEFAULT_COST);
6369   size(4);
6370   format %{ "MSR     $dst, $src" %}
6371   opcode(MSR_ZOPC);
6372   ins_encode(z_rreform(dst, src));
6373   ins_pipe(pipe_class_dummy);
6374 %}
6375 
6376 // Immediate Multiplication
6377 instruct mulI_reg_imm16(iRegI dst, immI16 con) %{
6378   match(Set dst (MulI dst con));
6379   ins_cost(DEFAULT_COST);
6380   // TODO: s390 port size(FIXED_SIZE);
6381   format %{ "MHI     $dst,$con" %}
6382   opcode(MHI_ZOPC);
6383   ins_encode(z_riform_signed(dst,con));
6384   ins_pipe(pipe_class_dummy);
6385 %}
6386 
6387 // Immediate (32bit) Multiplication
6388 instruct mulI_reg_imm32(iRegI dst, immI con) %{
6389   match(Set dst (MulI dst con));
6390   ins_cost(DEFAULT_COST);
6391   size(6);
6392   format %{ "MSFI    $dst,$con" %}
6393   opcode(MSFI_ZOPC);
6394   ins_encode(z_rilform_signed(dst,con));
6395   ins_pipe(pipe_class_dummy);
6396 %}
6397 
6398 instruct mulI_Reg_mem(iRegI dst, memory src)%{
6399   match(Set dst (MulI dst (LoadI src)));
6400   ins_cost(MEMORY_REF_COST);
6401   // TODO: s390 port size(VARIABLE_SIZE);
6402   format %{ "MS(Y)   $dst, $src\t # int" %}
6403   opcode(MSY_ZOPC, MS_ZOPC);
6404   ins_encode(z_form_rt_mem_opt(dst, src));
6405   ins_pipe(pipe_class_dummy);
6406 %}
6407 
6408 //
6409 
6410 instruct mulL_reg_regI(iRegL dst, iRegI src) %{
6411   match(Set dst (MulL dst (ConvI2L src)));
6412   ins_cost(DEFAULT_COST);
6413   // TODO: s390 port size(FIXED_SIZE);
6414   format %{ "MSGFR   $dst $src\t # long/int" %}
6415   opcode(MSGFR_ZOPC);
6416   ins_encode(z_rreform(dst, src));
6417   ins_pipe(pipe_class_dummy);
6418 %}
6419 
6420 instruct mulL_reg_reg(iRegL dst, iRegL src) %{
6421   match(Set dst (MulL dst src));
6422   ins_cost(DEFAULT_COST);
6423   size(4);
6424   format %{ "MSGR    $dst $src\t # long" %}
6425   opcode(MSGR_ZOPC);
6426   ins_encode(z_rreform(dst, src));
6427   ins_pipe(pipe_class_dummy);
6428 %}
6429 
6430 // Immediate Multiplication
6431 instruct mulL_reg_imm16(iRegL dst, immL16 src) %{
6432   match(Set dst (MulL dst src));
6433   ins_cost(DEFAULT_COST);
6434   // TODO: s390 port size(FIXED_SIZE);
6435   format %{ "MGHI    $dst,$src\t # long" %}
6436   opcode(MGHI_ZOPC);
6437   ins_encode(z_riform_signed(dst, src));
6438   ins_pipe(pipe_class_dummy);
6439 %}
6440 
6441 // Immediate (32bit) Multiplication
6442 instruct mulL_reg_imm32(iRegL dst, immL32 con) %{
6443   match(Set dst (MulL dst con));
6444   ins_cost(DEFAULT_COST);
6445   size(6);
6446   format %{ "MSGFI   $dst,$con" %}
6447   opcode(MSGFI_ZOPC);
6448   ins_encode(z_rilform_signed(dst,con));
6449   ins_pipe(pipe_class_dummy);
6450 %}
6451 
6452 instruct mulL_Reg_memI(iRegL dst, memory src)%{
6453   match(Set dst (MulL dst (ConvI2L (LoadI src))));
6454   ins_cost(MEMORY_REF_COST);
6455   size(Z_DISP3_SIZE);
6456   format %{ "MSGF    $dst, $src\t # long" %}
6457   opcode(MSGF_ZOPC, MSGF_ZOPC);
6458   ins_encode(z_form_rt_mem_opt(dst, src));
6459   ins_pipe(pipe_class_dummy);
6460 %}
6461 
6462 instruct mulL_Reg_mem(iRegL dst, memory src)%{
6463   match(Set dst (MulL dst (LoadL src)));
6464   ins_cost(MEMORY_REF_COST);
6465   size(Z_DISP3_SIZE);
6466   format %{ "MSG     $dst, $src\t # long" %}
6467   opcode(MSG_ZOPC, MSG_ZOPC);
6468   ins_encode(z_form_rt_mem_opt(dst, src));
6469   ins_pipe(pipe_class_dummy);
6470 %}
6471 
6472 instruct mulHiL_reg_reg(revenRegL Rdst, roddRegL Rsrc1, iRegL Rsrc2, iRegL Rtmp1, flagsReg cr)%{
6473   match(Set Rdst (MulHiL Rsrc1 Rsrc2));
6474   effect(TEMP_DEF Rdst, USE_KILL Rsrc1, TEMP Rtmp1, KILL cr);
6475   ins_cost(7*DEFAULT_COST);
6476   // TODO: s390 port size(VARIABLE_SIZE);
6477   format %{ "MulHiL  $Rdst, $Rsrc1, $Rsrc2\t # Multiply High Long" %}
6478   ins_encode%{
6479     Register dst  = $Rdst$$Register;
6480     Register src1 = $Rsrc1$$Register;
6481     Register src2 = $Rsrc2$$Register;
6482     Register tmp1 = $Rtmp1$$Register;
6483     Register tmp2 = $Rdst$$Register;
6484     // z/Architecture has only unsigned multiply (64 * 64 -> 128).
6485     // implementing mulhs(a,b) = mulhu(a,b) – (a & (b>>63)) – (b & (a>>63))
6486     __ z_srag(tmp2, src1, 63);  // a>>63
6487     __ z_srag(tmp1, src2, 63);  // b>>63
6488     __ z_ngr(tmp2, src2);       // b & (a>>63)
6489     __ z_ngr(tmp1, src1);       // a & (b>>63)
6490     __ z_agr(tmp1, tmp2);       // ((a & (b>>63)) + (b & (a>>63)))
6491     __ z_mlgr(dst, src2);       // tricky: 128-bit product is written to even/odd pair (dst,src1),
6492                                 //         multiplicand is taken from oddReg (src1), multiplier in src2.
6493     __ z_sgr(dst, tmp1);
6494   %}
6495   ins_pipe(pipe_class_dummy);
6496 %}
6497 
6498 //  DIV
6499 
6500 // Integer DIVMOD with Register, both quotient and mod results
6501 instruct divModI_reg_divmod(roddRegI dst1src1, revenRegI dst2, noOdd_iRegI src2, flagsReg cr) %{
6502   match(DivModI dst1src1 src2);
6503   effect(KILL cr);
6504   ins_cost(2 * DEFAULT_COST + BRANCH_COST);
6505   size((VM_Version::has_CompareBranch() ? 24 : 26));
6506   format %{ "DIVMODI ($dst1src1, $dst2) $src2" %}
6507   ins_encode %{
6508     Register d1s1 = $dst1src1$$Register;
6509     Register d2   = $dst2$$Register;
6510     Register s2   = $src2$$Register;
6511 
6512     assert_different_registers(d1s1, s2);
6513 
6514     Label do_div, done_div;
6515     if (VM_Version::has_CompareBranch()) {
6516       __ z_cij(s2, -1, Assembler::bcondNotEqual, do_div);
6517     } else {
6518       __ z_chi(s2, -1);
6519       __ z_brne(do_div);
6520     }
6521     __ z_lcr(d1s1, d1s1);
6522     __ clear_reg(d2, false, false);
6523     __ z_bru(done_div);
6524     __ bind(do_div);
6525     __ z_lgfr(d1s1, d1s1);
6526     __ z_dsgfr(d2, s2);
6527     __ bind(done_div);
6528   %}
6529   ins_pipe(pipe_class_dummy);
6530 %}
6531 
6532 
6533 // Register Division
6534 instruct divI_reg_reg(roddRegI dst, iRegI src1, noOdd_iRegI src2, revenRegI tmp, flagsReg cr) %{
6535   match(Set dst (DivI src1 src2));
6536   effect(KILL tmp, KILL cr);
6537   ins_cost(2 * DEFAULT_COST + BRANCH_COST);
6538   size((VM_Version::has_CompareBranch() ? 20 : 22));
6539   format %{ "DIV_checked $dst, $src1,$src2\t # treats special case 0x80../-1" %}
6540   ins_encode %{
6541     Register a = $src1$$Register;
6542     Register b = $src2$$Register;
6543     Register t = $dst$$Register;
6544 
6545     assert_different_registers(t, b);
6546 
6547     Label do_div, done_div;
6548     if (VM_Version::has_CompareBranch()) {
6549       __ z_cij(b, -1, Assembler::bcondNotEqual, do_div);
6550     } else {
6551       __ z_chi(b, -1);
6552       __ z_brne(do_div);
6553     }
6554     __ z_lcr(t, a);
6555     __ z_bru(done_div);
6556     __ bind(do_div);
6557     __ z_lgfr(t, a);
6558     __ z_dsgfr(t->predecessor()/* t is odd part of a register pair. */, b);
6559     __ bind(done_div);
6560   %}
6561   ins_pipe(pipe_class_dummy);
6562 %}
6563 
6564 // Immediate Division
6565 instruct divI_reg_imm16(roddRegI dst, iRegI src1, immI16 src2, revenRegI tmp, flagsReg cr) %{
6566   match(Set dst (DivI src1 src2));
6567   effect(KILL tmp, KILL cr);  // R0 is killed, too.
6568   ins_cost(2 * DEFAULT_COST);
6569   // TODO: s390 port size(VARIABLE_SIZE);
6570   format %{ "DIV_const  $dst,$src1,$src2" %}
6571   ins_encode %{
6572     // No sign extension of Rdividend needed here.
6573     if ($src2$$constant != -1) {
6574       __ z_lghi(Z_R0_scratch, $src2$$constant);
6575       __ z_lgfr($dst$$Register, $src1$$Register);
6576       __ z_dsgfr($dst$$Register->predecessor()/* Dst is odd part of a register pair. */, Z_R0_scratch);
6577     } else {
6578       __ z_lcr($dst$$Register, $src1$$Register);
6579     }
6580   %}
6581   ins_pipe(pipe_class_dummy);
6582 %}
6583 
6584 // Long DIVMOD with Register, both quotient and mod results
6585 instruct divModL_reg_divmod(roddRegL dst1src1, revenRegL dst2, iRegL src2, flagsReg cr) %{
6586   match(DivModL dst1src1 src2);
6587   effect(KILL cr);
6588   ins_cost(2 * DEFAULT_COST + BRANCH_COST);
6589   size((VM_Version::has_CompareBranch() ? 22 : 24));
6590   format %{ "DIVMODL ($dst1src1, $dst2) $src2" %}
6591   ins_encode %{
6592     Register d1s1 = $dst1src1$$Register;
6593     Register d2   = $dst2$$Register;
6594     Register s2   = $src2$$Register;
6595 
6596     Label do_div, done_div;
6597     if (VM_Version::has_CompareBranch()) {
6598       __ z_cgij(s2, -1, Assembler::bcondNotEqual, do_div);
6599     } else {
6600       __ z_cghi(s2, -1);
6601       __ z_brne(do_div);
6602     }
6603     __ z_lcgr(d1s1, d1s1);
6604     // indicate unused result
6605     (void) __ clear_reg(d2, true, false);
6606     __ z_bru(done_div);
6607     __ bind(do_div);
6608     __ z_dsgr(d2, s2);
6609     __ bind(done_div);
6610   %}
6611   ins_pipe(pipe_class_dummy);
6612 %}
6613 
6614 // Register Long Division
6615 instruct divL_reg_reg(roddRegL dst, iRegL src, revenRegL tmp, flagsReg cr) %{
6616   match(Set dst (DivL dst src));
6617   effect(KILL tmp, KILL cr);
6618   ins_cost(2 * DEFAULT_COST + BRANCH_COST);
6619   size((VM_Version::has_CompareBranch() ? 18 : 20));
6620   format %{ "DIVG_checked  $dst, $src\t # long, treats special case 0x80../-1" %}
6621   ins_encode %{
6622     Register b = $src$$Register;
6623     Register t = $dst$$Register;
6624 
6625     Label done_div;
6626     __ z_lcgr(t, t);    // Does no harm. divisor is in other register.
6627     if (VM_Version::has_CompareBranch()) {
6628       __ z_cgij(b, -1, Assembler::bcondEqual, done_div);
6629     } else {
6630       __ z_cghi(b, -1);
6631       __ z_bre(done_div);
6632     }
6633     __ z_lcgr(t, t);    // Restore sign.
6634     __ z_dsgr(t->predecessor()/* t is odd part of a register pair. */, b);
6635     __ bind(done_div);
6636   %}
6637   ins_pipe(pipe_class_dummy);
6638 %}
6639 
6640 // Immediate Long Division
6641 instruct divL_reg_imm16(roddRegL dst, iRegL src1, immL16 src2, revenRegL tmp, flagsReg cr) %{
6642   match(Set dst (DivL src1 src2));
6643   effect(KILL tmp, KILL cr);  // R0 is killed, too.
6644   ins_cost(2 * DEFAULT_COST);
6645   // TODO: s390 port size(VARIABLE_SIZE);
6646   format %{ "DIVG_const  $dst,$src1,$src2\t # long" %}
6647   ins_encode %{
6648     if ($src2$$constant != -1) {
6649       __ z_lghi(Z_R0_scratch, $src2$$constant);
6650       __ lgr_if_needed($dst$$Register, $src1$$Register);
6651       __ z_dsgr($dst$$Register->predecessor()/* Dst is odd part of a register pair. */, Z_R0_scratch);
6652     } else {
6653       __ z_lcgr($dst$$Register, $src1$$Register);
6654     }
6655   %}
6656   ins_pipe(pipe_class_dummy);
6657 %}
6658 
6659 // REM
6660 
6661 // Integer Remainder
6662 // Register Remainder
6663 instruct modI_reg_reg(revenRegI dst, iRegI src1, noOdd_iRegI src2, roddRegI tmp, flagsReg cr) %{
6664   match(Set dst (ModI src1 src2));
6665   effect(KILL tmp, KILL cr);
6666   ins_cost(2 * DEFAULT_COST + BRANCH_COST);
6667   // TODO: s390 port size(VARIABLE_SIZE);
6668   format %{ "MOD_checked   $dst,$src1,$src2" %}
6669   ins_encode %{
6670     Register a = $src1$$Register;
6671     Register b = $src2$$Register;
6672     Register t = $dst$$Register;
6673     assert_different_registers(t->successor(), b);
6674 
6675     Label do_div, done_div;
6676 
6677     if ((t->encoding() != b->encoding()) && (t->encoding() != a->encoding())) {
6678       (void) __ clear_reg(t, true, false);  // Does no harm. Operands are in other regs.
6679       if (VM_Version::has_CompareBranch()) {
6680         __ z_cij(b, -1, Assembler::bcondEqual, done_div);
6681       } else {
6682         __ z_chi(b, -1);
6683         __ z_bre(done_div);
6684       }
6685       __ z_lgfr(t->successor(), a);
6686       __ z_dsgfr(t/* t is even part of a register pair. */, b);
6687     } else {
6688       if (VM_Version::has_CompareBranch()) {
6689         __ z_cij(b, -1, Assembler::bcondNotEqual, do_div);
6690       } else {
6691         __ z_chi(b, -1);
6692         __ z_brne(do_div);
6693       }
6694       __ clear_reg(t, true, false);
6695       __ z_bru(done_div);
6696       __ bind(do_div);
6697       __ z_lgfr(t->successor(), a);
6698       __ z_dsgfr(t/* t is even part of a register pair. */, b);
6699     }
6700     __ bind(done_div);
6701   %}
6702   ins_pipe(pipe_class_dummy);
6703 %}
6704 
6705 // Immediate Remainder
6706 instruct modI_reg_imm16(revenRegI dst, iRegI src1, immI16 src2, roddRegI tmp, flagsReg cr) %{
6707   match(Set dst (ModI src1 src2));
6708   effect(KILL tmp, KILL cr); // R0 is killed, too.
6709   ins_cost(3 * DEFAULT_COST);
6710   // TODO: s390 port size(VARIABLE_SIZE);
6711   format %{ "MOD_const  $dst,src1,$src2" %}
6712   ins_encode %{
6713     assert_different_registers($dst$$Register, $src1$$Register);
6714     assert_different_registers($dst$$Register->successor(), $src1$$Register);
6715     int divisor = $src2$$constant;
6716 
6717     if (divisor != -1) {
6718       __ z_lghi(Z_R0_scratch, divisor);
6719       __ z_lgfr($dst$$Register->successor(), $src1$$Register);
6720       __ z_dsgfr($dst$$Register/* Dst is even part of a register pair. */, Z_R0_scratch); // Instruction kills tmp.
6721     } else {
6722       __ clear_reg($dst$$Register, true, false);
6723     }
6724   %}
6725   ins_pipe(pipe_class_dummy);
6726 %}
6727 
6728 // Register Long Remainder
6729 instruct modL_reg_reg(revenRegL dst, roddRegL src1, iRegL src2, flagsReg cr) %{
6730   match(Set dst (ModL src1 src2));
6731   effect(KILL src1, KILL cr); // R0 is killed, too.
6732   ins_cost(2 * DEFAULT_COST + BRANCH_COST);
6733   // TODO: s390 port size(VARIABLE_SIZE);
6734   format %{ "MODG_checked   $dst,$src1,$src2" %}
6735   ins_encode %{
6736     Register a = $src1$$Register;
6737     Register b = $src2$$Register;
6738     Register t = $dst$$Register;
6739     assert(t->successor() == a, "(t,a) is an even-odd pair" );
6740 
6741     Label do_div, done_div;
6742     if (t->encoding() != b->encoding()) {
6743       (void) __ clear_reg(t, true, false); // Does no harm. Dividend is in successor.
6744       if (VM_Version::has_CompareBranch()) {
6745         __ z_cgij(b, -1, Assembler::bcondEqual, done_div);
6746       } else {
6747         __ z_cghi(b, -1);
6748         __ z_bre(done_div);
6749       }
6750       __ z_dsgr(t, b);
6751     } else {
6752       if (VM_Version::has_CompareBranch()) {
6753         __ z_cgij(b, -1, Assembler::bcondNotEqual, do_div);
6754       } else {
6755         __ z_cghi(b, -1);
6756         __ z_brne(do_div);
6757       }
6758       __ clear_reg(t, true, false);
6759       __ z_bru(done_div);
6760       __ bind(do_div);
6761       __ z_dsgr(t, b);
6762     }
6763     __ bind(done_div);
6764   %}
6765   ins_pipe(pipe_class_dummy);
6766 %}
6767 
6768 // Register Long Remainder
6769 instruct modL_reg_imm16(revenRegL dst, iRegL src1, immL16 src2, roddRegL tmp, flagsReg cr) %{
6770   match(Set dst (ModL src1 src2));
6771   effect(KILL tmp, KILL cr); // R0 is killed, too.
6772   ins_cost(3 * DEFAULT_COST);
6773   // TODO: s390 port size(VARIABLE_SIZE);
6774   format %{ "MODG_const  $dst,src1,$src2\t # long" %}
6775   ins_encode %{
6776     int divisor = $src2$$constant;
6777     if (divisor != -1) {
6778       __ z_lghi(Z_R0_scratch, divisor);
6779       __ z_lgr($dst$$Register->successor(), $src1$$Register);
6780       __ z_dsgr($dst$$Register /* Dst is even part of a register pair. */, Z_R0_scratch);  // Instruction kills tmp.
6781     } else {
6782       __ clear_reg($dst$$Register, true, false);
6783     }
6784   %}
6785   ins_pipe(pipe_class_dummy);
6786 %}
6787 
6788 // SHIFT
6789 
6790 // Shift left logical
6791 
6792 // Register Shift Left variable
6793 instruct sllI_reg_reg(iRegI dst, iRegI src, iRegI nbits, flagsReg cr) %{
6794   match(Set dst (LShiftI src nbits));
6795   effect(KILL cr); // R1 is killed, too.
6796   ins_cost(3 * DEFAULT_COST);
6797   size(14);
6798   format %{ "SLL     $dst,$src,[$nbits] & 31\t# use RISC-like SLLG also for int" %}
6799   ins_encode %{
6800     __ z_lgr(Z_R1_scratch, $nbits$$Register);
6801     __ z_nill(Z_R1_scratch, BitsPerJavaInteger-1);
6802     __ z_sllg($dst$$Register, $src$$Register, 0, Z_R1_scratch);
6803   %}
6804   ins_pipe(pipe_class_dummy);
6805 %}
6806 
6807 // Register Shift Left Immediate
6808 // Constant shift count is masked in ideal graph already.
6809 instruct sllI_reg_imm(iRegI dst, iRegI src, immI nbits) %{
6810   match(Set dst (LShiftI src nbits));
6811   size(6);
6812   format %{ "SLL     $dst,$src,$nbits\t# use RISC-like SLLG also for int" %}
6813   ins_encode %{
6814     int Nbit = $nbits$$constant;
6815     assert((Nbit & (BitsPerJavaInteger - 1)) == Nbit, "Check shift mask in ideal graph");
6816     __ z_sllg($dst$$Register, $src$$Register, Nbit & (BitsPerJavaInteger - 1), Z_R0);
6817   %}
6818   ins_pipe(pipe_class_dummy);
6819 %}
6820 
6821 // Register Shift Left Immediate by 1bit
6822 instruct sllI_reg_imm_1(iRegI dst, iRegI src, immI_1 nbits) %{
6823   match(Set dst (LShiftI src nbits));
6824   predicate(PreferLAoverADD);
6825   ins_cost(DEFAULT_COST_LOW);
6826   size(4);
6827   format %{ "LA      $dst,#0($src,$src)\t # SLL by 1 (int)" %}
6828   ins_encode %{ __ z_la($dst$$Register, 0, $src$$Register, $src$$Register); %}
6829   ins_pipe(pipe_class_dummy);
6830 %}
6831 
6832 // Register Shift Left Long
6833 instruct sllL_reg_reg(iRegL dst, iRegL src1, iRegI nbits) %{
6834   match(Set dst (LShiftL src1 nbits));
6835   size(6);
6836   format %{ "SLLG    $dst,$src1,[$nbits]" %}
6837   opcode(SLLG_ZOPC);
6838   ins_encode(z_rsyform_reg_reg(dst, src1, nbits));
6839   ins_pipe(pipe_class_dummy);
6840 %}
6841 
6842 // Register Shift Left Long Immediate
6843 instruct sllL_reg_imm(iRegL dst, iRegL src1, immI nbits) %{
6844   match(Set dst (LShiftL src1 nbits));
6845   size(6);
6846   format %{ "SLLG    $dst,$src1,$nbits" %}
6847   opcode(SLLG_ZOPC);
6848   ins_encode(z_rsyform_const(dst, src1, nbits));
6849   ins_pipe(pipe_class_dummy);
6850 %}
6851 
6852 // Register Shift Left Long Immediate by 1bit
6853 instruct sllL_reg_imm_1(iRegL dst, iRegL src1, immI_1 nbits) %{
6854   match(Set dst (LShiftL src1 nbits));
6855   predicate(PreferLAoverADD);
6856   ins_cost(DEFAULT_COST_LOW);
6857   size(4);
6858   format %{ "LA      $dst,#0($src1,$src1)\t # SLLG by 1 (long)" %}
6859   ins_encode %{ __ z_la($dst$$Register, 0, $src1$$Register, $src1$$Register); %}
6860   ins_pipe(pipe_class_dummy);
6861 %}
6862 
6863 // Shift right arithmetic
6864 
6865 // Register Arithmetic Shift Right
6866 instruct sraI_reg_reg(iRegI dst, iRegI src, flagsReg cr) %{
6867   match(Set dst (RShiftI dst src));
6868   effect(KILL cr); // R1 is killed, too.
6869   ins_cost(3 * DEFAULT_COST);
6870   size(12);
6871   format %{ "SRA     $dst,[$src] & 31" %}
6872   ins_encode %{
6873     __ z_lgr(Z_R1_scratch, $src$$Register);
6874     __ z_nill(Z_R1_scratch, BitsPerJavaInteger-1);
6875     __ z_sra($dst$$Register, 0, Z_R1_scratch);
6876   %}
6877   ins_pipe(pipe_class_dummy);
6878 %}
6879 
6880 // Register Arithmetic Shift Right Immediate
6881 // Constant shift count is masked in ideal graph already.
6882 instruct sraI_reg_imm(iRegI dst, immI src, flagsReg cr) %{
6883   match(Set dst (RShiftI dst src));
6884   effect(KILL cr);
6885   size(4);
6886   format %{ "SRA     $dst,$src" %}
6887   ins_encode %{
6888     int Nbit = $src$$constant;
6889     assert((Nbit & (BitsPerJavaInteger - 1)) == Nbit, "Check shift mask in ideal graph");
6890     __ z_sra($dst$$Register, Nbit & (BitsPerJavaInteger - 1), Z_R0);
6891   %}
6892   ins_pipe(pipe_class_dummy);
6893 %}
6894 
6895 // Register Arithmetic Shift Right Long
6896 instruct sraL_reg_reg(iRegL dst, iRegL src1, iRegI src2, flagsReg cr) %{
6897   match(Set dst (RShiftL src1 src2));
6898   effect(KILL cr);
6899   size(6);
6900   format %{ "SRAG    $dst,$src1,[$src2]" %}
6901   opcode(SRAG_ZOPC);
6902   ins_encode(z_rsyform_reg_reg(dst, src1, src2));
6903   ins_pipe(pipe_class_dummy);
6904 %}
6905 
6906 // Register Arithmetic Shift Right Long Immediate
6907 instruct sraL_reg_imm(iRegL dst, iRegL src1, immI src2, flagsReg cr) %{
6908   match(Set dst (RShiftL src1 src2));
6909   effect(KILL cr);
6910   size(6);
6911   format %{ "SRAG    $dst,$src1,$src2" %}
6912   opcode(SRAG_ZOPC);
6913   ins_encode(z_rsyform_const(dst, src1, src2));
6914   ins_pipe(pipe_class_dummy);
6915 %}
6916 
6917 //  Shift right logical
6918 
6919 // Register Shift Right
6920 instruct srlI_reg_reg(iRegI dst, iRegI src, flagsReg cr) %{
6921   match(Set dst (URShiftI dst src));
6922   effect(KILL cr); // R1 is killed, too.
6923   ins_cost(3 * DEFAULT_COST);
6924   size(12);
6925   format %{ "SRL     $dst,[$src] & 31" %}
6926   ins_encode %{
6927     __ z_lgr(Z_R1_scratch, $src$$Register);
6928     __ z_nill(Z_R1_scratch, BitsPerJavaInteger-1);
6929     __ z_srl($dst$$Register, 0, Z_R1_scratch);
6930   %}
6931   ins_pipe(pipe_class_dummy);
6932 %}
6933 
6934 // Register Shift Right Immediate
6935 // Constant shift count is masked in ideal graph already.
6936 instruct srlI_reg_imm(iRegI dst, immI src) %{
6937   match(Set dst (URShiftI dst src));
6938   size(4);
6939   format %{ "SRL     $dst,$src" %}
6940   ins_encode %{
6941     int Nbit = $src$$constant;
6942     assert((Nbit & (BitsPerJavaInteger - 1)) == Nbit, "Check shift mask in ideal graph");
6943     __ z_srl($dst$$Register, Nbit & (BitsPerJavaInteger - 1), Z_R0);
6944   %}
6945   ins_pipe(pipe_class_dummy);
6946 %}
6947 
6948 // Register Shift Right Long
6949 instruct srlL_reg_reg(iRegL dst, iRegL src1, iRegI src2) %{
6950   match(Set dst (URShiftL src1 src2));
6951   size(6);
6952   format %{ "SRLG    $dst,$src1,[$src2]" %}
6953   opcode(SRLG_ZOPC);
6954   ins_encode(z_rsyform_reg_reg(dst, src1, src2));
6955   ins_pipe(pipe_class_dummy);
6956 %}
6957 
6958 // Register Shift Right Long Immediate
6959 instruct srlL_reg_imm(iRegL dst, iRegL src1, immI src2) %{
6960   match(Set dst (URShiftL src1 src2));
6961   size(6);
6962   format %{ "SRLG    $dst,$src1,$src2" %}
6963   opcode(SRLG_ZOPC);
6964   ins_encode(z_rsyform_const(dst, src1, src2));
6965   ins_pipe(pipe_class_dummy);
6966 %}
6967 
6968 // Register Shift Right Immediate with a CastP2X
6969 instruct srlP_reg_imm(iRegL dst, iRegP_N2P src1, immI src2) %{
6970   match(Set dst (URShiftL (CastP2X src1) src2));
6971   size(6);
6972   format %{ "SRLG    $dst,$src1,$src2\t # Cast ptr $src1 to long and shift" %}
6973   opcode(SRLG_ZOPC);
6974   ins_encode(z_rsyform_const(dst, src1, src2));
6975   ins_pipe(pipe_class_dummy);
6976 %}
6977 
6978 //----------Rotate Instructions------------------------------------------------
6979 
6980 // Rotate left 32bit.
6981 instruct rotlI_reg_immI8(iRegI dst, iRegI src, immI8 lshift, immI8 rshift) %{
6982   match(Set dst (OrI (LShiftI src lshift) (URShiftI src rshift)));
6983   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
6984   size(6);
6985   format %{ "RLL     $dst,$src,$lshift\t # ROTL32" %}
6986   opcode(RLL_ZOPC);
6987   ins_encode(z_rsyform_const(dst, src, lshift));
6988   ins_pipe(pipe_class_dummy);
6989 %}
6990 
6991 // Rotate left 64bit.
6992 instruct rotlL_reg_immI8(iRegL dst, iRegL src, immI8 lshift, immI8 rshift) %{
6993   match(Set dst (OrL (LShiftL src lshift) (URShiftL src rshift)));
6994   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
6995   size(6);
6996   format %{ "RLLG    $dst,$src,$lshift\t # ROTL64" %}
6997   opcode(RLLG_ZOPC);
6998   ins_encode(z_rsyform_const(dst, src, lshift));
6999   ins_pipe(pipe_class_dummy);
7000 %}
7001 
7002 // Rotate right 32bit.
7003 instruct rotrI_reg_immI8(iRegI dst, iRegI src, immI8 rshift, immI8 lshift) %{
7004   match(Set dst (OrI (URShiftI src rshift) (LShiftI src lshift)));
7005   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
7006   // TODO: s390 port size(FIXED_SIZE);
7007   format %{ "RLL     $dst,$src,$rshift\t # ROTR32" %}
7008   opcode(RLL_ZOPC);
7009   ins_encode(z_rsyform_const(dst, src, rshift));
7010   ins_pipe(pipe_class_dummy);
7011 %}
7012 
7013 // Rotate right 64bit.
7014 instruct rotrL_reg_immI8(iRegL dst, iRegL src, immI8 rshift, immI8 lshift) %{
7015   match(Set dst (OrL (URShiftL src rshift) (LShiftL src lshift)));
7016   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
7017   // TODO: s390 port size(FIXED_SIZE);
7018   format %{ "RLLG    $dst,$src,$rshift\t # ROTR64" %}
7019   opcode(RLLG_ZOPC);
7020   ins_encode(z_rsyform_const(dst, src, rshift));
7021   ins_pipe(pipe_class_dummy);
7022 %}
7023 
7024 
7025 //----------Overflow Math Instructions-----------------------------------------
7026 
7027 instruct overflowAddI_reg_reg(flagsReg cr, iRegI op1, iRegI op2) %{
7028   match(Set cr (OverflowAddI op1 op2));
7029   effect(DEF cr, USE op1, USE op2);
7030   // TODO: s390 port size(FIXED_SIZE);
7031   format %{ "AR      $op1,$op2\t # overflow check int" %}
7032   ins_encode %{
7033     __ z_lr(Z_R0_scratch, $op1$$Register);
7034     __ z_ar(Z_R0_scratch, $op2$$Register);
7035   %}
7036   ins_pipe(pipe_class_dummy);
7037 %}
7038 
7039 instruct overflowAddI_reg_imm(flagsReg cr, iRegI op1, immI op2) %{
7040   match(Set cr (OverflowAddI op1 op2));
7041   effect(DEF cr, USE op1, USE op2);
7042   // TODO: s390 port size(VARIABLE_SIZE);
7043   format %{ "AR      $op1,$op2\t # overflow check int" %}
7044   ins_encode %{
7045     __ load_const_optimized(Z_R0_scratch, $op2$$constant);
7046     __ z_ar(Z_R0_scratch, $op1$$Register);
7047   %}
7048   ins_pipe(pipe_class_dummy);
7049 %}
7050 
7051 instruct overflowAddL_reg_reg(flagsReg cr, iRegL op1, iRegL op2) %{
7052   match(Set cr (OverflowAddL op1 op2));
7053   effect(DEF cr, USE op1, USE op2);
7054   // TODO: s390 port size(FIXED_SIZE);
7055   format %{ "AGR     $op1,$op2\t # overflow check long" %}
7056   ins_encode %{
7057     __ z_lgr(Z_R0_scratch, $op1$$Register);
7058     __ z_agr(Z_R0_scratch, $op2$$Register);
7059   %}
7060   ins_pipe(pipe_class_dummy);
7061 %}
7062 
7063 instruct overflowAddL_reg_imm(flagsReg cr, iRegL op1, immL op2) %{
7064   match(Set cr (OverflowAddL op1 op2));
7065   effect(DEF cr, USE op1, USE op2);
7066   // TODO: s390 port size(VARIABLE_SIZE);
7067   format %{ "AGR     $op1,$op2\t # overflow check long" %}
7068   ins_encode %{
7069     __ load_const_optimized(Z_R0_scratch, $op2$$constant);
7070     __ z_agr(Z_R0_scratch, $op1$$Register);
7071   %}
7072   ins_pipe(pipe_class_dummy);
7073 %}
7074 
7075 instruct overflowSubI_reg_reg(flagsReg cr, iRegI op1, iRegI op2) %{
7076   match(Set cr (OverflowSubI op1 op2));
7077   effect(DEF cr, USE op1, USE op2);
7078   // TODO: s390 port size(FIXED_SIZE);
7079   format %{ "SR      $op1,$op2\t # overflow check int" %}
7080   ins_encode %{
7081     __ z_lr(Z_R0_scratch, $op1$$Register);
7082     __ z_sr(Z_R0_scratch, $op2$$Register);
7083   %}
7084   ins_pipe(pipe_class_dummy);
7085 %}
7086 
7087 instruct overflowSubI_reg_imm(flagsReg cr, iRegI op1, immI op2) %{
7088   match(Set cr (OverflowSubI op1 op2));
7089   effect(DEF cr, USE op1, USE op2);
7090   // TODO: s390 port size(VARIABLE_SIZE);
7091   format %{ "SR      $op1,$op2\t # overflow check int" %}
7092   ins_encode %{
7093     __ load_const_optimized(Z_R1_scratch, $op2$$constant);
7094     __ z_lr(Z_R0_scratch, $op1$$Register);
7095     __ z_sr(Z_R0_scratch, Z_R1_scratch);
7096   %}
7097   ins_pipe(pipe_class_dummy);
7098 %}
7099 
7100 instruct overflowSubL_reg_reg(flagsReg cr, iRegL op1, iRegL op2) %{
7101   match(Set cr (OverflowSubL op1 op2));
7102   effect(DEF cr, USE op1, USE op2);
7103   // TODO: s390 port size(FIXED_SIZE);
7104   format %{ "SGR     $op1,$op2\t # overflow check long" %}
7105   ins_encode %{
7106     __ z_lgr(Z_R0_scratch, $op1$$Register);
7107     __ z_sgr(Z_R0_scratch, $op2$$Register);
7108   %}
7109   ins_pipe(pipe_class_dummy);
7110 %}
7111 
7112 instruct overflowSubL_reg_imm(flagsReg cr, iRegL op1, immL op2) %{
7113   match(Set cr (OverflowSubL op1 op2));
7114   effect(DEF cr, USE op1, USE op2);
7115   // TODO: s390 port size(VARIABLE_SIZE);
7116   format %{ "SGR     $op1,$op2\t # overflow check long" %}
7117   ins_encode %{
7118     __ load_const_optimized(Z_R1_scratch, $op2$$constant);
7119     __ z_lgr(Z_R0_scratch, $op1$$Register);
7120     __ z_sgr(Z_R0_scratch, Z_R1_scratch);
7121   %}
7122   ins_pipe(pipe_class_dummy);
7123 %}
7124 
7125 instruct overflowNegI_rReg(flagsReg cr, immI_0 zero, iRegI op2) %{
7126   match(Set cr (OverflowSubI zero op2));
7127   effect(DEF cr, USE op2);
7128   format %{ "NEG    $op2\t# overflow check int" %}
7129   ins_encode %{
7130     __ clear_reg(Z_R0_scratch, false, false);
7131     __ z_sr(Z_R0_scratch, $op2$$Register);
7132   %}
7133   ins_pipe(pipe_class_dummy);
7134 %}
7135 
7136 instruct overflowNegL_rReg(flagsReg cr, immL_0 zero, iRegL op2) %{
7137   match(Set cr (OverflowSubL zero op2));
7138   effect(DEF cr, USE op2);
7139   format %{ "NEGG    $op2\t# overflow check long" %}
7140   ins_encode %{
7141     __ clear_reg(Z_R0_scratch, true, false);
7142     __ z_sgr(Z_R0_scratch, $op2$$Register);
7143   %}
7144   ins_pipe(pipe_class_dummy);
7145 %}
7146 
7147 // No intrinsics for multiplication, since there is no easy way
7148 // to check for overflow.
7149 
7150 
7151 //----------Floating Point Arithmetic Instructions-----------------------------
7152 
7153 //  ADD
7154 
7155 //  Add float single precision
7156 instruct addF_reg_reg(regF dst, regF src, flagsReg cr) %{
7157   match(Set dst (AddF dst src));
7158   effect(KILL cr);
7159   ins_cost(ALU_REG_COST);
7160   size(4);
7161   format %{ "AEBR     $dst,$src" %}
7162   opcode(AEBR_ZOPC);
7163   ins_encode(z_rreform(dst, src));
7164   ins_pipe(pipe_class_dummy);
7165 %}
7166 
7167 instruct addF_reg_mem(regF dst, memoryRX src, flagsReg cr)%{
7168   match(Set dst (AddF dst (LoadF src)));
7169   effect(KILL cr);
7170   ins_cost(ALU_MEMORY_COST);
7171   size(6);
7172   format %{ "AEB      $dst,$src\t # floatMemory" %}
7173   opcode(AEB_ZOPC);
7174   ins_encode(z_form_rt_memFP(dst, src));
7175   ins_pipe(pipe_class_dummy);
7176 %}
7177 
7178 // Add float double precision
7179 instruct addD_reg_reg(regD dst, regD src, flagsReg cr) %{
7180   match(Set dst (AddD dst src));
7181   effect(KILL cr);
7182   ins_cost(ALU_REG_COST);
7183   size(4);
7184   format %{ "ADBR     $dst,$src" %}
7185   opcode(ADBR_ZOPC);
7186   ins_encode(z_rreform(dst, src));
7187   ins_pipe(pipe_class_dummy);
7188 %}
7189 
7190 instruct addD_reg_mem(regD dst, memoryRX src, flagsReg cr)%{
7191   match(Set dst (AddD dst (LoadD src)));
7192   effect(KILL cr);
7193   ins_cost(ALU_MEMORY_COST);
7194   size(6);
7195   format %{ "ADB      $dst,$src\t # doubleMemory" %}
7196   opcode(ADB_ZOPC);
7197   ins_encode(z_form_rt_memFP(dst, src));
7198   ins_pipe(pipe_class_dummy);
7199 %}
7200 
7201 // SUB
7202 
7203 // Sub float single precision
7204 instruct subF_reg_reg(regF dst, regF src, flagsReg cr) %{
7205   match(Set dst (SubF dst src));
7206   effect(KILL cr);
7207   ins_cost(ALU_REG_COST);
7208   size(4);
7209   format %{ "SEBR     $dst,$src" %}
7210   opcode(SEBR_ZOPC);
7211   ins_encode(z_rreform(dst, src));
7212   ins_pipe(pipe_class_dummy);
7213 %}
7214 
7215 instruct subF_reg_mem(regF dst, memoryRX src, flagsReg cr)%{
7216   match(Set dst (SubF dst (LoadF src)));
7217   effect(KILL cr);
7218   ins_cost(ALU_MEMORY_COST);
7219   size(6);
7220   format %{ "SEB      $dst,$src\t # floatMemory" %}
7221   opcode(SEB_ZOPC);
7222   ins_encode(z_form_rt_memFP(dst, src));
7223   ins_pipe(pipe_class_dummy);
7224 %}
7225 
7226 //  Sub float double precision
7227 instruct subD_reg_reg(regD dst, regD src, flagsReg cr) %{
7228   match(Set dst (SubD dst src));
7229   effect(KILL cr);
7230   ins_cost(ALU_REG_COST);
7231   size(4);
7232   format %{ "SDBR     $dst,$src" %}
7233   opcode(SDBR_ZOPC);
7234   ins_encode(z_rreform(dst, src));
7235   ins_pipe(pipe_class_dummy);
7236 %}
7237 
7238 instruct subD_reg_mem(regD dst, memoryRX src, flagsReg cr)%{
7239   match(Set dst (SubD dst (LoadD src)));
7240   effect(KILL cr);
7241   ins_cost(ALU_MEMORY_COST);
7242   size(6);
7243   format %{ "SDB      $dst,$src\t # doubleMemory" %}
7244   opcode(SDB_ZOPC);
7245   ins_encode(z_form_rt_memFP(dst, src));
7246   ins_pipe(pipe_class_dummy);
7247 %}
7248 
7249 // MUL
7250 
7251 // Mul float single precision
7252 instruct mulF_reg_reg(regF dst, regF src) %{
7253   match(Set dst (MulF dst src));
7254   // CC unchanged by MUL.
7255   ins_cost(ALU_REG_COST);
7256   size(4);
7257   format %{ "MEEBR    $dst,$src" %}
7258   opcode(MEEBR_ZOPC);
7259   ins_encode(z_rreform(dst, src));
7260   ins_pipe(pipe_class_dummy);
7261 %}
7262 
7263 instruct mulF_reg_mem(regF dst, memoryRX src)%{
7264   match(Set dst (MulF dst (LoadF src)));
7265   // CC unchanged by MUL.
7266   ins_cost(ALU_MEMORY_COST);
7267   size(6);
7268   format %{ "MEEB     $dst,$src\t # floatMemory" %}
7269   opcode(MEEB_ZOPC);
7270   ins_encode(z_form_rt_memFP(dst, src));
7271   ins_pipe(pipe_class_dummy);
7272 %}
7273 
7274 //  Mul float double precision
7275 instruct mulD_reg_reg(regD dst, regD src) %{
7276   match(Set dst (MulD dst src));
7277   // CC unchanged by MUL.
7278   ins_cost(ALU_REG_COST);
7279   size(4);
7280   format %{ "MDBR     $dst,$src" %}
7281   opcode(MDBR_ZOPC);
7282   ins_encode(z_rreform(dst, src));
7283   ins_pipe(pipe_class_dummy);
7284 %}
7285 
7286 instruct mulD_reg_mem(regD dst, memoryRX src)%{
7287   match(Set dst (MulD dst (LoadD src)));
7288   // CC unchanged by MUL.
7289   ins_cost(ALU_MEMORY_COST);
7290   size(6);
7291   format %{ "MDB      $dst,$src\t # doubleMemory" %}
7292   opcode(MDB_ZOPC);
7293   ins_encode(z_form_rt_memFP(dst, src));
7294   ins_pipe(pipe_class_dummy);
7295 %}
7296 
7297 // Multiply-Accumulate
7298 // src1 * src2 + dst
7299 instruct maddF_reg_reg(regF dst, regF src1, regF src2) %{
7300   match(Set dst (FmaF dst (Binary src1 src2)));
7301   // CC unchanged by MUL-ADD.
7302   ins_cost(ALU_REG_COST);
7303   size(4);
7304   format %{ "MAEBR    $dst, $src1, $src2" %}
7305   ins_encode %{
7306     __ z_maebr($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
7307   %}
7308   ins_pipe(pipe_class_dummy);
7309 %}
7310 
7311 // src1 * src2 + dst
7312 instruct maddD_reg_reg(regD dst, regD src1, regD src2) %{
7313   match(Set dst (FmaD dst (Binary src1 src2)));
7314   // CC unchanged by MUL-ADD.
7315   ins_cost(ALU_REG_COST);
7316   size(4);
7317   format %{ "MADBR    $dst, $src1, $src2" %}
7318   ins_encode %{
7319     __ z_madbr($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
7320   %}
7321   ins_pipe(pipe_class_dummy);
7322 %}
7323 
7324 // src1 * src2 - dst
7325 instruct msubF_reg_reg(regF dst, regF src1, regF src2) %{
7326   match(Set dst (FmaF (NegF dst) (Binary src1 src2)));
7327   // CC unchanged by MUL-SUB.
7328   ins_cost(ALU_REG_COST);
7329   size(4);
7330   format %{ "MSEBR    $dst, $src1, $src2" %}
7331   ins_encode %{
7332     __ z_msebr($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
7333   %}
7334   ins_pipe(pipe_class_dummy);
7335 %}
7336 
7337 // src1 * src2 - dst
7338 instruct msubD_reg_reg(regD dst, regD src1, regD src2) %{
7339   match(Set dst (FmaD (NegD dst) (Binary src1 src2)));
7340   // CC unchanged by MUL-SUB.
7341   ins_cost(ALU_REG_COST);
7342   size(4);
7343   format %{ "MSDBR    $dst, $src1, $src2" %}
7344   ins_encode %{
7345     __ z_msdbr($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
7346   %}
7347   ins_pipe(pipe_class_dummy);
7348 %}
7349 
7350 // src1 * src2 + dst
7351 instruct maddF_reg_mem(regF dst, regF src1, memoryRX src2) %{
7352   match(Set dst (FmaF dst (Binary src1 (LoadF src2))));
7353   // CC unchanged by MUL-ADD.
7354   ins_cost(ALU_MEMORY_COST);
7355   size(6);
7356   format %{ "MAEB     $dst, $src1, $src2" %}
7357   ins_encode %{
7358     __ z_maeb($dst$$FloatRegister, $src1$$FloatRegister,
7359               Address(reg_to_register_object($src2$$base), $src2$$index$$Register, $src2$$disp));
7360   %}
7361   ins_pipe(pipe_class_dummy);
7362 %}
7363 
7364 // src1 * src2 + dst
7365 instruct maddD_reg_mem(regD dst, regD src1, memoryRX src2) %{
7366   match(Set dst (FmaD dst (Binary src1 (LoadD src2))));
7367   // CC unchanged by MUL-ADD.
7368   ins_cost(ALU_MEMORY_COST);
7369   size(6);
7370   format %{ "MADB     $dst, $src1, $src2" %}
7371   ins_encode %{
7372     __ z_madb($dst$$FloatRegister, $src1$$FloatRegister,
7373               Address(reg_to_register_object($src2$$base), $src2$$index$$Register, $src2$$disp));
7374   %}
7375   ins_pipe(pipe_class_dummy);
7376 %}
7377 
7378 // src1 * src2 - dst
7379 instruct msubF_reg_mem(regF dst, regF src1, memoryRX src2) %{
7380   match(Set dst (FmaF (NegF dst) (Binary src1 (LoadF src2))));
7381   // CC unchanged by MUL-SUB.
7382   ins_cost(ALU_MEMORY_COST);
7383   size(6);
7384   format %{ "MSEB     $dst, $src1, $src2" %}
7385   ins_encode %{
7386     __ z_mseb($dst$$FloatRegister, $src1$$FloatRegister,
7387               Address(reg_to_register_object($src2$$base), $src2$$index$$Register, $src2$$disp));
7388   %}
7389   ins_pipe(pipe_class_dummy);
7390 %}
7391 
7392 // src1 * src2 - dst
7393 instruct msubD_reg_mem(regD dst, regD src1, memoryRX src2) %{
7394   match(Set dst (FmaD (NegD dst) (Binary src1 (LoadD src2))));
7395   // CC unchanged by MUL-SUB.
7396   ins_cost(ALU_MEMORY_COST);
7397   size(6);
7398   format %{ "MSDB    $dst, $src1, $src2" %}
7399   ins_encode %{
7400     __ z_msdb($dst$$FloatRegister, $src1$$FloatRegister,
7401               Address(reg_to_register_object($src2$$base), $src2$$index$$Register, $src2$$disp));
7402   %}
7403   ins_pipe(pipe_class_dummy);
7404 %}
7405 
7406 // src1 * src2 + dst
7407 instruct maddF_mem_reg(regF dst, memoryRX src1, regF src2) %{
7408   match(Set dst (FmaF dst (Binary (LoadF src1) src2)));
7409   // CC unchanged by MUL-ADD.
7410   ins_cost(ALU_MEMORY_COST);
7411   size(6);
7412   format %{ "MAEB     $dst, $src1, $src2" %}
7413   ins_encode %{
7414     __ z_maeb($dst$$FloatRegister, $src2$$FloatRegister,
7415               Address(reg_to_register_object($src1$$base), $src1$$index$$Register, $src1$$disp));
7416   %}
7417   ins_pipe(pipe_class_dummy);
7418 %}
7419 
7420 // src1 * src2 + dst
7421 instruct maddD_mem_reg(regD dst, memoryRX src1, regD src2) %{
7422   match(Set dst (FmaD dst (Binary (LoadD src1) src2)));
7423   // CC unchanged by MUL-ADD.
7424   ins_cost(ALU_MEMORY_COST);
7425   size(6);
7426   format %{ "MADB     $dst, $src1, $src2" %}
7427   ins_encode %{
7428     __ z_madb($dst$$FloatRegister, $src2$$FloatRegister,
7429               Address(reg_to_register_object($src1$$base), $src1$$index$$Register, $src1$$disp));
7430   %}
7431   ins_pipe(pipe_class_dummy);
7432 %}
7433 
7434 // src1 * src2 - dst
7435 instruct msubF_mem_reg(regF dst, memoryRX src1, regF src2) %{
7436   match(Set dst (FmaF (NegF dst) (Binary (LoadF src1) src2)));
7437   // CC unchanged by MUL-SUB.
7438   ins_cost(ALU_MEMORY_COST);
7439   size(6);
7440   format %{ "MSEB     $dst, $src1, $src2" %}
7441   ins_encode %{
7442     __ z_mseb($dst$$FloatRegister, $src2$$FloatRegister,
7443               Address(reg_to_register_object($src1$$base), $src1$$index$$Register, $src1$$disp));
7444   %}
7445   ins_pipe(pipe_class_dummy);
7446 %}
7447 
7448 // src1 * src2 - dst
7449 instruct msubD_mem_reg(regD dst, memoryRX src1, regD src2) %{
7450   match(Set dst (FmaD (NegD dst) (Binary (LoadD src1) src2)));
7451   // CC unchanged by MUL-SUB.
7452   ins_cost(ALU_MEMORY_COST);
7453   size(6);
7454   format %{ "MSDB    $dst, $src1, $src2" %}
7455   ins_encode %{
7456     __ z_msdb($dst$$FloatRegister, $src2$$FloatRegister,
7457               Address(reg_to_register_object($src1$$base), $src1$$index$$Register, $src1$$disp));
7458   %}
7459   ins_pipe(pipe_class_dummy);
7460 %}
7461 
7462 //  DIV
7463 
7464 //  Div float single precision
7465 instruct divF_reg_reg(regF dst, regF src) %{
7466   match(Set dst (DivF dst src));
7467   // CC unchanged by DIV.
7468   ins_cost(ALU_REG_COST);
7469   size(4);
7470   format %{ "DEBR     $dst,$src" %}
7471   opcode(DEBR_ZOPC);
7472   ins_encode(z_rreform(dst, src));
7473   ins_pipe(pipe_class_dummy);
7474 %}
7475 
7476 instruct divF_reg_mem(regF dst, memoryRX src)%{
7477   match(Set dst (DivF dst (LoadF src)));
7478   // CC unchanged by DIV.
7479   ins_cost(ALU_MEMORY_COST);
7480   size(6);
7481   format %{ "DEB      $dst,$src\t # floatMemory" %}
7482   opcode(DEB_ZOPC);
7483   ins_encode(z_form_rt_memFP(dst, src));
7484   ins_pipe(pipe_class_dummy);
7485 %}
7486 
7487 //  Div float double precision
7488 instruct divD_reg_reg(regD dst, regD src) %{
7489   match(Set dst (DivD dst src));
7490   // CC unchanged by DIV.
7491   ins_cost(ALU_REG_COST);
7492   size(4);
7493   format %{ "DDBR     $dst,$src" %}
7494   opcode(DDBR_ZOPC);
7495   ins_encode(z_rreform(dst, src));
7496   ins_pipe(pipe_class_dummy);
7497 %}
7498 
7499 instruct divD_reg_mem(regD dst, memoryRX src)%{
7500   match(Set dst (DivD dst (LoadD src)));
7501   // CC unchanged by DIV.
7502   ins_cost(ALU_MEMORY_COST);
7503   size(6);
7504   format %{ "DDB      $dst,$src\t # doubleMemory" %}
7505   opcode(DDB_ZOPC);
7506   ins_encode(z_form_rt_memFP(dst, src));
7507   ins_pipe(pipe_class_dummy);
7508 %}
7509 
7510 // ABS
7511 
7512 // Absolute float single precision
7513 instruct absF_reg(regF dst, regF src, flagsReg cr) %{
7514   match(Set dst (AbsF src));
7515   effect(KILL cr);
7516   size(4);
7517   format %{ "LPEBR    $dst,$src\t float" %}
7518   opcode(LPEBR_ZOPC);
7519   ins_encode(z_rreform(dst, src));
7520   ins_pipe(pipe_class_dummy);
7521 %}
7522 
7523 // Absolute float double precision
7524 instruct absD_reg(regD dst, regD src, flagsReg cr) %{
7525   match(Set dst (AbsD src));
7526   effect(KILL cr);
7527   size(4);
7528   format %{ "LPDBR    $dst,$src\t double" %}
7529   opcode(LPDBR_ZOPC);
7530   ins_encode(z_rreform(dst, src));
7531   ins_pipe(pipe_class_dummy);
7532 %}
7533 
7534 //  NEG(ABS)
7535 
7536 // Negative absolute float single precision
7537 instruct nabsF_reg(regF dst, regF src, flagsReg cr) %{
7538   match(Set dst (NegF (AbsF src)));
7539   effect(KILL cr);
7540   size(4);
7541   format %{ "LNEBR    $dst,$src\t float" %}
7542   opcode(LNEBR_ZOPC);
7543   ins_encode(z_rreform(dst, src));
7544   ins_pipe(pipe_class_dummy);
7545 %}
7546 
7547 // Negative absolute float double precision
7548 instruct nabsD_reg(regD dst, regD src, flagsReg cr) %{
7549   match(Set dst (NegD (AbsD src)));
7550   effect(KILL cr);
7551   size(4);
7552   format %{ "LNDBR    $dst,$src\t double" %}
7553   opcode(LNDBR_ZOPC);
7554   ins_encode(z_rreform(dst, src));
7555   ins_pipe(pipe_class_dummy);
7556 %}
7557 
7558 // NEG
7559 
7560 instruct negF_reg(regF dst, regF src, flagsReg cr) %{
7561   match(Set dst (NegF src));
7562   effect(KILL cr);
7563   size(4);
7564   format %{ "NegF     $dst,$src\t float" %}
7565   ins_encode %{ __ z_lcebr($dst$$FloatRegister, $src$$FloatRegister); %}
7566   ins_pipe(pipe_class_dummy);
7567 %}
7568 
7569 instruct negD_reg(regD dst, regD src, flagsReg cr) %{
7570   match(Set dst (NegD src));
7571   effect(KILL cr);
7572   size(4);
7573   format %{ "NegD     $dst,$src\t double" %}
7574   ins_encode %{ __ z_lcdbr($dst$$FloatRegister, $src$$FloatRegister); %}
7575   ins_pipe(pipe_class_dummy);
7576 %}
7577 
7578 // SQRT
7579 
7580 // Sqrt float precision
7581 instruct sqrtF_reg(regF dst, regF src) %{
7582   match(Set dst (ConvD2F (SqrtD (ConvF2D src))));
7583   // CC remains unchanged.
7584   ins_cost(ALU_REG_COST);
7585   size(4);
7586   format %{ "SQEBR    $dst,$src" %}
7587   opcode(SQEBR_ZOPC);
7588   ins_encode(z_rreform(dst, src));
7589   ins_pipe(pipe_class_dummy);
7590 %}
7591 
7592 // Sqrt double precision
7593 instruct sqrtD_reg(regD dst, regD src) %{
7594   match(Set dst (SqrtD src));
7595   // CC remains unchanged.
7596   ins_cost(ALU_REG_COST);
7597   size(4);
7598   format %{ "SQDBR    $dst,$src" %}
7599   opcode(SQDBR_ZOPC);
7600   ins_encode(z_rreform(dst, src));
7601   ins_pipe(pipe_class_dummy);
7602 %}
7603 
7604 instruct sqrtF_mem(regF dst, memoryRX src) %{
7605   match(Set dst (ConvD2F (SqrtD (ConvF2D src))));
7606   // CC remains unchanged.
7607   ins_cost(ALU_MEMORY_COST);
7608   size(6);
7609   format %{ "SQEB     $dst,$src\t # floatMemory" %}
7610   opcode(SQEB_ZOPC);
7611   ins_encode(z_form_rt_memFP(dst, src));
7612   ins_pipe(pipe_class_dummy);
7613 %}
7614 
7615 instruct sqrtD_mem(regD dst, memoryRX src) %{
7616   match(Set dst (SqrtD src));
7617   // CC remains unchanged.
7618   ins_cost(ALU_MEMORY_COST);
7619   // TODO: s390 port size(FIXED_SIZE);
7620   format %{ "SQDB     $dst,$src\t # doubleMemory" %}
7621   opcode(SQDB_ZOPC);
7622   ins_encode(z_form_rt_memFP(dst, src));
7623   ins_pipe(pipe_class_dummy);
7624 %}
7625 
7626 //----------Logical Instructions-----------------------------------------------
7627 
7628 // Register And
7629 instruct andI_reg_reg(iRegI dst, iRegI src, flagsReg cr) %{
7630   match(Set dst (AndI dst src));
7631   effect(KILL cr);
7632   ins_cost(DEFAULT_COST_LOW);
7633   size(2);
7634   format %{ "NR      $dst,$src\t # int" %}
7635   opcode(NR_ZOPC);
7636   ins_encode(z_rrform(dst, src));
7637   ins_pipe(pipe_class_dummy);
7638 %}
7639 
7640 instruct andI_Reg_mem(iRegI dst, memory src, flagsReg cr)%{
7641   match(Set dst (AndI dst (LoadI src)));
7642   effect(KILL cr);
7643   ins_cost(MEMORY_REF_COST);
7644   // TODO: s390 port size(VARIABLE_SIZE);
7645   format %{ "N(Y)    $dst, $src\t # int" %}
7646   opcode(NY_ZOPC, N_ZOPC);
7647   ins_encode(z_form_rt_mem_opt(dst, src));
7648   ins_pipe(pipe_class_dummy);
7649 %}
7650 
7651 // Immediate And
7652 instruct andI_reg_uimm32(iRegI dst, uimmI src, flagsReg cr) %{
7653   match(Set dst (AndI dst src));
7654   effect(KILL cr);
7655   ins_cost(DEFAULT_COST_HIGH);
7656   size(6);
7657   format %{ "NILF    $dst,$src" %}
7658   opcode(NILF_ZOPC);
7659   ins_encode(z_rilform_unsigned(dst, src));
7660   ins_pipe(pipe_class_dummy);
7661 %}
7662 
7663 instruct andI_reg_uimmI_LH1(iRegI dst, uimmI_LH1 src, flagsReg cr) %{
7664   match(Set dst (AndI dst src));
7665   effect(KILL cr);
7666   ins_cost(DEFAULT_COST);
7667   size(4);
7668   format %{ "NILH    $dst,$src" %}
7669   ins_encode %{ __ z_nilh($dst$$Register, ($src$$constant >> 16) & 0xFFFF); %}
7670   ins_pipe(pipe_class_dummy);
7671 %}
7672 
7673 instruct andI_reg_uimmI_LL1(iRegI dst, uimmI_LL1 src, flagsReg cr) %{
7674   match(Set dst (AndI dst src));
7675   effect(KILL cr);
7676   ins_cost(DEFAULT_COST);
7677   size(4);
7678   format %{ "NILL    $dst,$src" %}
7679   ins_encode %{ __ z_nill($dst$$Register, $src$$constant & 0xFFFF); %}
7680   ins_pipe(pipe_class_dummy);
7681 %}
7682 
7683 // Register And Long
7684 instruct andL_reg_reg(iRegL dst, iRegL src, flagsReg cr) %{
7685   match(Set dst (AndL dst src));
7686   effect(KILL cr);
7687   ins_cost(DEFAULT_COST);
7688   size(4);
7689   format %{ "NGR     $dst,$src\t # long" %}
7690   opcode(NGR_ZOPC);
7691   ins_encode(z_rreform(dst, src));
7692   ins_pipe(pipe_class_dummy);
7693 %}
7694 
7695 instruct andL_Reg_mem(iRegL dst, memory src, flagsReg cr)%{
7696   match(Set dst (AndL dst (LoadL src)));
7697   effect(KILL cr);
7698   ins_cost(MEMORY_REF_COST);
7699   size(Z_DISP3_SIZE);
7700   format %{ "NG      $dst, $src\t # long" %}
7701   opcode(NG_ZOPC, NG_ZOPC);
7702   ins_encode(z_form_rt_mem_opt(dst, src));
7703   ins_pipe(pipe_class_dummy);
7704 %}
7705 
7706 instruct andL_reg_uimmL_LL1(iRegL dst, uimmL_LL1 src, flagsReg cr) %{
7707   match(Set dst (AndL dst src));
7708   effect(KILL cr);
7709   ins_cost(DEFAULT_COST);
7710   size(4);
7711   format %{ "NILL    $dst,$src\t # long" %}
7712   ins_encode %{ __ z_nill($dst$$Register, $src$$constant & 0xFFFF); %}
7713   ins_pipe(pipe_class_dummy);
7714 %}
7715 
7716 instruct andL_reg_uimmL_LH1(iRegL dst, uimmL_LH1 src, flagsReg cr) %{
7717   match(Set dst (AndL dst src));
7718   effect(KILL cr);
7719   ins_cost(DEFAULT_COST);
7720   size(4);
7721   format %{ "NILH    $dst,$src\t # long" %}
7722   ins_encode %{ __ z_nilh($dst$$Register, ($src$$constant >> 16) & 0xFFFF); %}
7723   ins_pipe(pipe_class_dummy);
7724 %}
7725 
7726 instruct andL_reg_uimmL_HL1(iRegL dst, uimmL_HL1 src, flagsReg cr) %{
7727   match(Set dst (AndL dst src));
7728   effect(KILL cr);
7729   ins_cost(DEFAULT_COST);
7730   size(4);
7731   format %{ "NIHL    $dst,$src\t # long" %}
7732   ins_encode %{ __ z_nihl($dst$$Register, ($src$$constant >> 32) & 0xFFFF); %}
7733   ins_pipe(pipe_class_dummy);
7734 %}
7735 
7736 instruct andL_reg_uimmL_HH1(iRegL dst, uimmL_HH1 src, flagsReg cr) %{
7737   match(Set dst (AndL dst src));
7738   effect(KILL cr);
7739   ins_cost(DEFAULT_COST);
7740   size(4);
7741   format %{ "NIHH    $dst,$src\t # long" %}
7742   ins_encode %{ __ z_nihh($dst$$Register, ($src$$constant >> 48) & 0xFFFF); %}
7743   ins_pipe(pipe_class_dummy);
7744 %}
7745 
7746 //  OR
7747 
7748 // Or Instructions
7749 // Register Or
7750 instruct orI_reg_reg(iRegI dst, iRegI src, flagsReg cr) %{
7751   match(Set dst (OrI dst src));
7752   effect(KILL cr);
7753   size(2);
7754   format %{ "OR      $dst,$src" %}
7755   opcode(OR_ZOPC);
7756   ins_encode(z_rrform(dst, src));
7757   ins_pipe(pipe_class_dummy);
7758 %}
7759 
7760 instruct orI_Reg_mem(iRegI dst, memory src, flagsReg cr)%{
7761   match(Set dst (OrI dst (LoadI src)));
7762   effect(KILL cr);
7763   ins_cost(MEMORY_REF_COST);
7764   // TODO: s390 port size(VARIABLE_SIZE);
7765   format %{ "O(Y)    $dst, $src\t # int" %}
7766   opcode(OY_ZOPC, O_ZOPC);
7767   ins_encode(z_form_rt_mem_opt(dst, src));
7768   ins_pipe(pipe_class_dummy);
7769 %}
7770 
7771 // Immediate Or
7772 instruct orI_reg_uimm16(iRegI dst, uimmI16 con, flagsReg cr) %{
7773   match(Set dst (OrI dst con));
7774   effect(KILL cr);
7775   size(4);
7776   format %{ "OILL    $dst,$con" %}
7777   opcode(OILL_ZOPC);
7778   ins_encode(z_riform_unsigned(dst,con));
7779   ins_pipe(pipe_class_dummy);
7780 %}
7781 
7782 instruct orI_reg_uimm32(iRegI dst, uimmI con, flagsReg cr) %{
7783   match(Set dst (OrI dst con));
7784   effect(KILL cr);
7785   ins_cost(DEFAULT_COST_HIGH);
7786   size(6);
7787   format %{ "OILF    $dst,$con" %}
7788   opcode(OILF_ZOPC);
7789   ins_encode(z_rilform_unsigned(dst,con));
7790   ins_pipe(pipe_class_dummy);
7791 %}
7792 
7793 // Register Or Long
7794 instruct orL_reg_reg(iRegL dst, iRegL src, flagsReg cr) %{
7795   match(Set dst (OrL dst src));
7796   effect(KILL cr);
7797   ins_cost(DEFAULT_COST);
7798   size(4);
7799   format %{ "OGR      $dst,$src\t # long" %}
7800   opcode(OGR_ZOPC);
7801   ins_encode(z_rreform(dst, src));
7802   ins_pipe(pipe_class_dummy);
7803 %}
7804 
7805 instruct orL_Reg_mem(iRegL dst, memory src, flagsReg cr)%{
7806   match(Set dst (OrL dst (LoadL src)));
7807   effect(KILL cr);
7808   ins_cost(MEMORY_REF_COST);
7809   size(Z_DISP3_SIZE);
7810   format %{ "OG      $dst, $src\t # long" %}
7811   opcode(OG_ZOPC, OG_ZOPC);
7812   ins_encode(z_form_rt_mem_opt(dst, src));
7813   ins_pipe(pipe_class_dummy);
7814 %}
7815 
7816 // Immediate Or long
7817 instruct orL_reg_uimm16(iRegL dst, uimmL16 con, flagsReg cr) %{
7818   match(Set dst (OrL dst con));
7819   effect(KILL cr);
7820   ins_cost(DEFAULT_COST);
7821   size(4);
7822   format %{ "OILL    $dst,$con\t # long" %}
7823   opcode(OILL_ZOPC);
7824   ins_encode(z_riform_unsigned(dst,con));
7825   ins_pipe(pipe_class_dummy);
7826 %}
7827 
7828 instruct orL_reg_uimm32(iRegI dst, uimmL32 con, flagsReg cr) %{
7829   match(Set dst (OrI dst con));
7830   effect(KILL cr);
7831   ins_cost(DEFAULT_COST_HIGH);
7832   // TODO: s390 port size(FIXED_SIZE);
7833   format %{ "OILF    $dst,$con\t # long" %}
7834   opcode(OILF_ZOPC);
7835   ins_encode(z_rilform_unsigned(dst,con));
7836   ins_pipe(pipe_class_dummy);
7837 %}
7838 
7839 // XOR
7840 
7841 // Register Xor
7842 instruct xorI_reg_reg(iRegI dst, iRegI src, flagsReg cr) %{
7843   match(Set dst (XorI dst src));
7844   effect(KILL cr);
7845   size(2);
7846   format %{ "XR      $dst,$src" %}
7847   opcode(XR_ZOPC);
7848   ins_encode(z_rrform(dst, src));
7849   ins_pipe(pipe_class_dummy);
7850 %}
7851 
7852 instruct xorI_Reg_mem(iRegI dst, memory src, flagsReg cr)%{
7853   match(Set dst (XorI dst (LoadI src)));
7854   effect(KILL cr);
7855   ins_cost(MEMORY_REF_COST);
7856   // TODO: s390 port size(VARIABLE_SIZE);
7857   format %{ "X(Y)    $dst, $src\t # int" %}
7858   opcode(XY_ZOPC, X_ZOPC);
7859   ins_encode(z_form_rt_mem_opt(dst, src));
7860   ins_pipe(pipe_class_dummy);
7861 %}
7862 
7863 // Immediate Xor
7864 instruct xorI_reg_uimm32(iRegI dst, uimmI src, flagsReg cr) %{
7865   match(Set dst (XorI dst src));
7866   effect(KILL cr);
7867   ins_cost(DEFAULT_COST_HIGH);
7868   size(6);
7869   format %{ "XILF    $dst,$src" %}
7870   opcode(XILF_ZOPC);
7871   ins_encode(z_rilform_unsigned(dst, src));
7872   ins_pipe(pipe_class_dummy);
7873 %}
7874 
7875 // Register Xor Long
7876 instruct xorL_reg_reg(iRegL dst, iRegL src, flagsReg cr) %{
7877   match(Set dst (XorL dst src));
7878   effect(KILL cr);
7879   ins_cost(DEFAULT_COST);
7880   size(4);
7881   format %{ "XGR     $dst,$src\t # long" %}
7882   opcode(XGR_ZOPC);
7883   ins_encode(z_rreform(dst, src));
7884   ins_pipe(pipe_class_dummy);
7885 %}
7886 
7887 instruct xorL_Reg_mem(iRegL dst, memory src, flagsReg cr)%{
7888   match(Set dst (XorL dst (LoadL src)));
7889   effect(KILL cr);
7890   ins_cost(MEMORY_REF_COST);
7891   size(Z_DISP3_SIZE);
7892   format %{ "XG      $dst, $src\t # long" %}
7893   opcode(XG_ZOPC, XG_ZOPC);
7894   ins_encode(z_form_rt_mem_opt(dst, src));
7895   ins_pipe(pipe_class_dummy);
7896 %}
7897 
7898 // Immediate Xor Long
7899 instruct xorL_reg_uimm32(iRegL dst, uimmL32 con, flagsReg cr) %{
7900   match(Set dst (XorL dst con));
7901   effect(KILL cr);
7902   ins_cost(DEFAULT_COST_HIGH);
7903   size(6);
7904   format %{ "XILF    $dst,$con\t # long" %}
7905   opcode(XILF_ZOPC);
7906   ins_encode(z_rilform_unsigned(dst,con));
7907   ins_pipe(pipe_class_dummy);
7908 %}
7909 
7910 //----------Convert to Boolean-------------------------------------------------
7911 
7912 // Convert integer to boolean.
7913 instruct convI2B(iRegI dst, iRegI src, flagsReg cr) %{
7914   match(Set dst (Conv2B src));
7915   effect(KILL cr);
7916   ins_cost(3 * DEFAULT_COST);
7917   size(6);
7918   format %{ "convI2B $dst,$src" %}
7919   ins_encode %{
7920     __ z_lnr($dst$$Register, $src$$Register);  // Rdst := -|Rsrc|, i.e. Rdst == 0 <=> Rsrc == 0
7921     __ z_srl($dst$$Register, 31);              // Rdst := sign(Rdest)
7922   %}
7923   ins_pipe(pipe_class_dummy);
7924 %}
7925 
7926 instruct convP2B(iRegI dst, iRegP_N2P src, flagsReg cr) %{
7927   match(Set dst (Conv2B src));
7928   effect(KILL cr);
7929   ins_cost(3 * DEFAULT_COST);
7930   size(10);
7931   format %{ "convP2B $dst,$src" %}
7932   ins_encode %{
7933     __ z_lngr($dst$$Register, $src$$Register);     // Rdst := -|Rsrc| i.e. Rdst == 0 <=> Rsrc == 0
7934     __ z_srlg($dst$$Register, $dst$$Register, 63); // Rdst := sign(Rdest)
7935   %}
7936   ins_pipe(pipe_class_dummy);
7937 %}
7938 
7939 instruct cmpLTMask_reg_reg(iRegI dst, iRegI src, flagsReg cr) %{
7940   match(Set dst (CmpLTMask dst src));
7941   effect(KILL cr);
7942   ins_cost(2 * DEFAULT_COST);
7943   size(18);
7944   format %{ "Set $dst CmpLTMask $dst,$src" %}
7945   ins_encode %{
7946     // Avoid signed 32 bit overflow: Do sign extend and sub 64 bit.
7947     __ z_lgfr(Z_R0_scratch, $src$$Register);
7948     __ z_lgfr($dst$$Register, $dst$$Register);
7949     __ z_sgr($dst$$Register, Z_R0_scratch);
7950     __ z_srag($dst$$Register, $dst$$Register, 63);
7951   %}
7952   ins_pipe(pipe_class_dummy);
7953 %}
7954 
7955 instruct cmpLTMask_reg_zero(iRegI dst, immI_0 zero, flagsReg cr) %{
7956   match(Set dst (CmpLTMask dst zero));
7957   effect(KILL cr);
7958   ins_cost(DEFAULT_COST);
7959   size(4);
7960   format %{ "Set $dst CmpLTMask $dst,$zero" %}
7961   ins_encode %{ __ z_sra($dst$$Register, 31); %}
7962   ins_pipe(pipe_class_dummy);
7963 %}
7964 
7965 
7966 //----------Arithmetic Conversion Instructions---------------------------------
7967 // The conversions operations are all Alpha sorted. Please keep it that way!
7968 
7969 instruct convD2F_reg(regF dst, regD src) %{
7970   match(Set dst (ConvD2F src));
7971   // CC remains unchanged.
7972   size(4);
7973   format %{ "LEDBR   $dst,$src" %}
7974   opcode(LEDBR_ZOPC);
7975   ins_encode(z_rreform(dst, src));
7976   ins_pipe(pipe_class_dummy);
7977 %}
7978 
7979 instruct convF2I_reg(iRegI dst, regF src, flagsReg cr) %{
7980   match(Set dst (ConvF2I src));
7981   effect(KILL cr);
7982   ins_cost(2 * DEFAULT_COST + BRANCH_COST);
7983   size(16);
7984   format %{ "convF2I  $dst,$src" %}
7985   ins_encode %{
7986     Label done;
7987     __ clear_reg($dst$$Register, false, false);  // Initialize with result for unordered: 0.
7988     __ z_cebr($src$$FloatRegister, $src$$FloatRegister);   // Round.
7989     __ z_brno(done);                             // Result is zero if unordered argument.
7990     __ z_cfebr($dst$$Register, $src$$FloatRegister, Assembler::to_zero);
7991     __ bind(done);
7992   %}
7993   ins_pipe(pipe_class_dummy);
7994 %}
7995 
7996 instruct convD2I_reg(iRegI dst, regD src, flagsReg cr) %{
7997   match(Set dst (ConvD2I src));
7998   effect(KILL cr);
7999   ins_cost(2 * DEFAULT_COST + BRANCH_COST);
8000   size(16);
8001   format %{ "convD2I  $dst,$src" %}
8002   ins_encode %{
8003     Label done;
8004     __ clear_reg($dst$$Register, false, false);  // Initialize with result for unordered: 0.
8005     __ z_cdbr($src$$FloatRegister, $src$$FloatRegister);   // Round.
8006     __ z_brno(done);                             // Result is zero if unordered argument.
8007     __ z_cfdbr($dst$$Register, $src$$FloatRegister, Assembler::to_zero);
8008     __ bind(done);
8009   %}
8010   ins_pipe(pipe_class_dummy);
8011 %}
8012 
8013 instruct convF2L_reg(iRegL dst, regF src, flagsReg cr) %{
8014   match(Set dst (ConvF2L src));
8015   effect(KILL cr);
8016   ins_cost(2 * DEFAULT_COST + BRANCH_COST);
8017   size(16);
8018   format %{ "convF2L  $dst,$src" %}
8019   ins_encode %{
8020     Label done;
8021     __ clear_reg($dst$$Register, true, false);  // Initialize with result for unordered: 0.
8022     __ z_cebr($src$$FloatRegister, $src$$FloatRegister);   // Round.
8023     __ z_brno(done);                             // Result is zero if unordered argument.
8024     __ z_cgebr($dst$$Register, $src$$FloatRegister, Assembler::to_zero);
8025     __ bind(done);
8026   %}
8027   ins_pipe(pipe_class_dummy);
8028 %}
8029 
8030 instruct convD2L_reg(iRegL dst, regD src, flagsReg cr) %{
8031   match(Set dst (ConvD2L src));
8032   effect(KILL cr);
8033   ins_cost(2 * DEFAULT_COST + BRANCH_COST);
8034   size(16);
8035   format %{ "convD2L  $dst,$src" %}
8036   ins_encode %{
8037     Label done;
8038     __ clear_reg($dst$$Register, true, false);  // Initialize with result for unordered: 0.
8039     __ z_cdbr($src$$FloatRegister, $src$$FloatRegister);   // Round.
8040     __ z_brno(done);                             // Result is zero if unordered argument.
8041     __ z_cgdbr($dst$$Register, $src$$FloatRegister, Assembler::to_zero);
8042     __ bind(done);
8043   %}
8044   ins_pipe(pipe_class_dummy);
8045 %}
8046 
8047 instruct convF2D_reg(regD dst, regF src) %{
8048   match(Set dst (ConvF2D src));
8049   // CC remains unchanged.
8050   size(4);
8051   format %{ "LDEBR   $dst,$src" %}
8052   opcode(LDEBR_ZOPC);
8053   ins_encode(z_rreform(dst, src));
8054   ins_pipe(pipe_class_dummy);
8055 %}
8056 
8057 instruct convF2D_mem(regD dst, memoryRX src) %{
8058   match(Set dst (ConvF2D src));
8059   // CC remains unchanged.
8060   size(6);
8061   format %{ "LDEB    $dst,$src" %}
8062   opcode(LDEB_ZOPC);
8063   ins_encode(z_form_rt_memFP(dst, src));
8064   ins_pipe(pipe_class_dummy);
8065 %}
8066 
8067 instruct convI2D_reg(regD dst, iRegI src) %{
8068   match(Set dst (ConvI2D src));
8069   // CC remains unchanged.
8070   ins_cost(DEFAULT_COST);
8071   size(4);
8072   format %{ "CDFBR   $dst,$src" %}
8073   opcode(CDFBR_ZOPC);
8074   ins_encode(z_rreform(dst, src));
8075   ins_pipe(pipe_class_dummy);
8076 %}
8077 
8078 // Optimization that saves up to two memory operations for each conversion.
8079 instruct convI2F_ireg(regF dst, iRegI src) %{
8080   match(Set dst (ConvI2F src));
8081   // CC remains unchanged.
8082   ins_cost(DEFAULT_COST);
8083   size(4);
8084   format %{ "CEFBR   $dst,$src\t # convert int to float" %}
8085   opcode(CEFBR_ZOPC);
8086   ins_encode(z_rreform(dst, src));
8087   ins_pipe(pipe_class_dummy);
8088 %}
8089 
8090 instruct convI2L_reg(iRegL dst, iRegI src) %{
8091   match(Set dst (ConvI2L src));
8092   size(4);
8093   format %{ "LGFR    $dst,$src\t # int->long" %}
8094   opcode(LGFR_ZOPC);
8095   ins_encode(z_rreform(dst, src));
8096   ins_pipe(pipe_class_dummy);
8097 %}
8098 
8099 // Zero-extend convert int to long.
8100 instruct convI2L_reg_zex(iRegL dst, iRegI src, immL_32bits mask) %{
8101   match(Set dst (AndL (ConvI2L src) mask));
8102   size(4);
8103   format %{ "LLGFR   $dst, $src \t # zero-extend int to long" %}
8104   ins_encode %{ __ z_llgfr($dst$$Register, $src$$Register); %}
8105   ins_pipe(pipe_class_dummy);
8106 %}
8107 
8108 // Zero-extend convert int to long.
8109 instruct convI2L_mem_zex(iRegL dst, memory src, immL_32bits mask) %{
8110   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
8111   // Uses load_const_optmized, so size can vary.
8112   // TODO: s390 port size(VARIABLE_SIZE);
8113   format %{ "LLGF    $dst, $src \t # zero-extend int to long" %}
8114   opcode(LLGF_ZOPC, LLGF_ZOPC);
8115   ins_encode(z_form_rt_mem_opt(dst, src));
8116   ins_pipe(pipe_class_dummy);
8117 %}
8118 
8119 // Zero-extend long
8120 instruct zeroExtend_long(iRegL dst, iRegL src, immL_32bits mask) %{
8121   match(Set dst (AndL src mask));
8122   size(4);
8123   format %{ "LLGFR   $dst, $src \t # zero-extend long to long" %}
8124   ins_encode %{ __ z_llgfr($dst$$Register, $src$$Register); %}
8125   ins_pipe(pipe_class_dummy);
8126 %}
8127 
8128 instruct rShiftI16_lShiftI16_reg(iRegI dst, iRegI src, immI_16 amount) %{
8129   match(Set dst (RShiftI (LShiftI src amount) amount));
8130   size(4);
8131   format %{ "LHR     $dst,$src\t short->int" %}
8132   opcode(LHR_ZOPC);
8133   ins_encode(z_rreform(dst, src));
8134   ins_pipe(pipe_class_dummy);
8135 %}
8136 
8137 instruct rShiftI24_lShiftI24_reg(iRegI dst, iRegI src, immI_24 amount) %{
8138   match(Set dst (RShiftI (LShiftI src amount) amount));
8139   size(4);
8140   format %{ "LBR     $dst,$src\t byte->int" %}
8141   opcode(LBR_ZOPC);
8142   ins_encode(z_rreform(dst, src));
8143   ins_pipe(pipe_class_dummy);
8144 %}
8145 
8146 instruct MoveF2I_stack_reg(iRegI dst, stackSlotF src) %{
8147   match(Set dst (MoveF2I src));
8148   ins_cost(MEMORY_REF_COST);
8149   size(4);
8150   format %{ "L       $dst,$src\t # MoveF2I" %}
8151   opcode(L_ZOPC);
8152   ins_encode(z_form_rt_mem(dst, src));
8153   ins_pipe(pipe_class_dummy);
8154 %}
8155 
8156 // javax.imageio.stream.ImageInputStreamImpl.toFloats([B[FII)
8157 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
8158   match(Set dst (MoveI2F src));
8159   ins_cost(MEMORY_REF_COST);
8160   // TODO: s390 port size(FIXED_SIZE);
8161   format %{ "LE      $dst,$src\t # MoveI2F" %}
8162   opcode(LE_ZOPC);
8163   ins_encode(z_form_rt_mem(dst, src));
8164   ins_pipe(pipe_class_dummy);
8165 %}
8166 
8167 instruct MoveD2L_stack_reg(iRegL dst, stackSlotD src) %{
8168   match(Set dst (MoveD2L src));
8169   ins_cost(MEMORY_REF_COST);
8170   size(6);
8171   format %{ "LG      $src,$dst\t # MoveD2L" %}
8172   opcode(LG_ZOPC);
8173   ins_encode(z_form_rt_mem(dst, src));
8174   ins_pipe(pipe_class_dummy);
8175 %}
8176 
8177 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
8178   match(Set dst (MoveL2D src));
8179   ins_cost(MEMORY_REF_COST);
8180   size(4);
8181   format %{ "LD      $dst,$src\t # MoveL2D" %}
8182   opcode(LD_ZOPC);
8183   ins_encode(z_form_rt_mem(dst, src));
8184   ins_pipe(pipe_class_dummy);
8185 %}
8186 
8187 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
8188   match(Set dst (MoveI2F src));
8189   ins_cost(MEMORY_REF_COST);
8190   size(4);
8191   format %{ "ST      $src,$dst\t # MoveI2F" %}
8192   opcode(ST_ZOPC);
8193   ins_encode(z_form_rt_mem(src, dst));
8194   ins_pipe(pipe_class_dummy);
8195 %}
8196 
8197 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
8198   match(Set dst (MoveD2L src));
8199   effect(DEF dst, USE src);
8200   ins_cost(MEMORY_REF_COST);
8201   size(4);
8202   format %{ "STD     $src,$dst\t # MoveD2L" %}
8203   opcode(STD_ZOPC);
8204   ins_encode(z_form_rt_mem(src,dst));
8205   ins_pipe(pipe_class_dummy);
8206 %}
8207 
8208 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
8209   match(Set dst (MoveL2D src));
8210   ins_cost(MEMORY_REF_COST);
8211   size(6);
8212   format %{ "STG     $src,$dst\t # MoveL2D" %}
8213   opcode(STG_ZOPC);
8214   ins_encode(z_form_rt_mem(src,dst));
8215   ins_pipe(pipe_class_dummy);
8216 %}
8217 
8218 instruct convL2F_reg(regF dst, iRegL src) %{
8219   match(Set dst (ConvL2F src));
8220   // CC remains unchanged.
8221   ins_cost(DEFAULT_COST);
8222   size(4);
8223   format %{ "CEGBR   $dst,$src" %}
8224   opcode(CEGBR_ZOPC);
8225   ins_encode(z_rreform(dst, src));
8226   ins_pipe(pipe_class_dummy);
8227 %}
8228 
8229 instruct convL2D_reg(regD dst, iRegL src) %{
8230   match(Set dst (ConvL2D src));
8231   // CC remains unchanged.
8232   ins_cost(DEFAULT_COST);
8233   size(4);
8234   format %{ "CDGBR   $dst,$src" %}
8235   opcode(CDGBR_ZOPC);
8236   ins_encode(z_rreform(dst, src));
8237   ins_pipe(pipe_class_dummy);
8238 %}
8239 
8240 instruct convL2I_reg(iRegI dst, iRegL src) %{
8241   match(Set dst (ConvL2I src));
8242   // TODO: s390 port size(VARIABLE_SIZE);
8243   format %{ "LR      $dst,$src\t # long->int (if needed)" %}
8244   ins_encode %{ __ lr_if_needed($dst$$Register, $src$$Register); %}
8245   ins_pipe(pipe_class_dummy);
8246 %}
8247 
8248 // Register Shift Right Immediate
8249 instruct shrL_reg_imm6_L2I(iRegI dst, iRegL src, immI_32_63 cnt, flagsReg cr) %{
8250   match(Set dst (ConvL2I (RShiftL src cnt)));
8251   effect(KILL cr);
8252   size(6);
8253   format %{ "SRAG    $dst,$src,$cnt" %}
8254   opcode(SRAG_ZOPC);
8255   ins_encode(z_rsyform_const(dst, src, cnt));
8256   ins_pipe(pipe_class_dummy);
8257 %}
8258 
8259 //----------TRAP based zero checks and range checks----------------------------
8260 
8261 // SIGTRAP based implicit range checks in compiled code.
8262 // A range check in the ideal world has one of the following shapes:
8263 //   - (If le (CmpU length index)), (IfTrue  throw exception)
8264 //   - (If lt (CmpU index length)), (IfFalse throw exception)
8265 //
8266 // Match range check 'If le (CmpU length index)'
8267 instruct rangeCheck_iReg_uimmI16(cmpOpT cmp, iRegI length, uimmI16 index, label labl) %{
8268   match(If cmp (CmpU length index));
8269   effect(USE labl);
8270   predicate(TrapBasedRangeChecks &&
8271             _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le &&
8272             PROB_UNLIKELY(_leaf->as_If ()->_prob) >= PROB_ALWAYS &&
8273             Matcher::branches_to_uncommon_trap(_leaf));
8274   ins_cost(1);
8275   // TODO: s390 port size(FIXED_SIZE);
8276 
8277   ins_is_TrapBasedCheckNode(true);
8278 
8279   format %{ "RangeCheck len=$length cmp=$cmp idx=$index => trap $labl" %}
8280   ins_encode %{ __ z_clfit($length$$Register, $index$$constant, $cmp$$cmpcode); %}
8281   ins_pipe(pipe_class_trap);
8282 %}
8283 
8284 // Match range check 'If lt (CmpU index length)'
8285 instruct rangeCheck_iReg_iReg(cmpOpT cmp, iRegI index, iRegI length, label labl, flagsReg cr) %{
8286   match(If cmp (CmpU index length));
8287   effect(USE labl, KILL cr);
8288   predicate(TrapBasedRangeChecks &&
8289             _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt &&
8290             _leaf->as_If ()->_prob >= PROB_ALWAYS &&
8291             Matcher::branches_to_uncommon_trap(_leaf));
8292   ins_cost(1);
8293   // TODO: s390 port size(FIXED_SIZE);
8294 
8295   ins_is_TrapBasedCheckNode(true);
8296 
8297   format %{ "RangeCheck idx=$index cmp=$cmp len=$length => trap $labl" %}
8298   ins_encode %{ __ z_clrt($index$$Register, $length$$Register, $cmp$$cmpcode); %}
8299   ins_pipe(pipe_class_trap);
8300 %}
8301 
8302 // Match range check 'If lt (CmpU index length)'
8303 instruct rangeCheck_uimmI16_iReg(cmpOpT cmp, iRegI index, uimmI16 length, label labl) %{
8304   match(If cmp (CmpU index length));
8305   effect(USE labl);
8306   predicate(TrapBasedRangeChecks &&
8307             _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt &&
8308             _leaf->as_If ()->_prob >= PROB_ALWAYS &&
8309             Matcher::branches_to_uncommon_trap(_leaf));
8310   ins_cost(1);
8311   // TODO: s390 port size(FIXED_SIZE);
8312 
8313   ins_is_TrapBasedCheckNode(true);
8314 
8315   format %{ "RangeCheck idx=$index cmp=$cmp len= $length => trap $labl" %}
8316   ins_encode %{ __ z_clfit($index$$Register, $length$$constant, $cmp$$cmpcode); %}
8317   ins_pipe(pipe_class_trap);
8318 %}
8319 
8320 // Implicit zero checks (more implicit null checks).
8321 instruct zeroCheckP_iReg_imm0(cmpOpT cmp, iRegP_N2P value, immP0 zero, label labl) %{
8322   match(If cmp (CmpP value zero));
8323   effect(USE labl);
8324   predicate(TrapBasedNullChecks &&
8325             _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne &&
8326             _leaf->as_If ()->_prob >= PROB_LIKELY_MAG(4) &&
8327             Matcher::branches_to_uncommon_trap(_leaf));
8328   size(6);
8329 
8330   ins_is_TrapBasedCheckNode(true);
8331 
8332   format %{ "ZeroCheckP value=$value cmp=$cmp zero=$zero => trap $labl" %}
8333   ins_encode %{ __ z_cgit($value$$Register, 0, $cmp$$cmpcode); %}
8334   ins_pipe(pipe_class_trap);
8335 %}
8336 
8337 // Implicit zero checks (more implicit null checks).
8338 instruct zeroCheckN_iReg_imm0(cmpOpT cmp, iRegN_P2N value, immN0 zero, label labl) %{
8339   match(If cmp (CmpN value zero));
8340   effect(USE labl);
8341   predicate(TrapBasedNullChecks &&
8342             _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne &&
8343             _leaf->as_If ()->_prob >= PROB_LIKELY_MAG(4) &&
8344             Matcher::branches_to_uncommon_trap(_leaf));
8345   size(6);
8346 
8347   ins_is_TrapBasedCheckNode(true);
8348 
8349   format %{ "ZeroCheckN value=$value cmp=$cmp zero=$zero => trap $labl" %}
8350   ins_encode %{ __ z_cit($value$$Register, 0, $cmp$$cmpcode); %}
8351   ins_pipe(pipe_class_trap);
8352 %}
8353 
8354 //----------Compare instructions-----------------------------------------------
8355 
8356 // INT signed
8357 
8358 // Compare Integers
8359 instruct compI_reg_reg(flagsReg cr, iRegI op1, iRegI op2) %{
8360   match(Set cr (CmpI op1 op2));
8361   size(2);
8362   format %{ "CR      $op1,$op2" %}
8363   opcode(CR_ZOPC);
8364   ins_encode(z_rrform(op1, op2));
8365   ins_pipe(pipe_class_dummy);
8366 %}
8367 
8368 instruct compI_reg_imm(flagsReg cr, iRegI op1, immI op2) %{
8369   match(Set cr (CmpI op1 op2));
8370   size(6);
8371   format %{ "CFI     $op1,$op2" %}
8372   opcode(CFI_ZOPC);
8373   ins_encode(z_rilform_signed(op1, op2));
8374   ins_pipe(pipe_class_dummy);
8375 %}
8376 
8377 instruct compI_reg_imm16(flagsReg cr, iRegI op1, immI16 op2) %{
8378   match(Set cr (CmpI op1 op2));
8379   size(4);
8380   format %{ "CHI     $op1,$op2" %}
8381   opcode(CHI_ZOPC);
8382   ins_encode(z_riform_signed(op1, op2));
8383   ins_pipe(pipe_class_dummy);
8384 %}
8385 
8386 instruct compI_reg_imm0(flagsReg cr, iRegI op1, immI_0 zero) %{
8387   match(Set cr (CmpI op1 zero));
8388   ins_cost(DEFAULT_COST_LOW);
8389   size(2);
8390   format %{ "LTR     $op1,$op1" %}
8391   opcode(LTR_ZOPC);
8392   ins_encode(z_rrform(op1, op1));
8393   ins_pipe(pipe_class_dummy);
8394 %}
8395 
8396 instruct compI_reg_mem(flagsReg cr, iRegI op1, memory op2)%{
8397   match(Set cr (CmpI op1 (LoadI op2)));
8398   ins_cost(MEMORY_REF_COST);
8399   // TODO: s390 port size(VARIABLE_SIZE);
8400   format %{ "C(Y)    $op1, $op2\t # int" %}
8401   opcode(CY_ZOPC, C_ZOPC);
8402   ins_encode(z_form_rt_mem_opt(op1, op2));
8403   ins_pipe(pipe_class_dummy);
8404 %}
8405 
8406 // INT unsigned
8407 
8408 instruct compU_reg_reg(flagsReg cr, iRegI op1, iRegI op2) %{
8409   match(Set cr (CmpU op1 op2));
8410   size(2);
8411   format %{ "CLR     $op1,$op2\t # unsigned" %}
8412   opcode(CLR_ZOPC);
8413   ins_encode(z_rrform(op1, op2));
8414   ins_pipe(pipe_class_dummy);
8415 %}
8416 
8417 instruct compU_reg_uimm(flagsReg cr, iRegI op1, uimmI op2) %{
8418   match(Set cr (CmpU op1 op2));
8419   size(6);
8420   format %{ "CLFI    $op1,$op2\t # unsigned" %}
8421   opcode(CLFI_ZOPC);
8422   ins_encode(z_rilform_unsigned(op1, op2));
8423   ins_pipe(pipe_class_dummy);
8424 %}
8425 
8426 instruct compU_reg_mem(flagsReg cr, iRegI op1, memory op2)%{
8427   match(Set cr (CmpU op1 (LoadI op2)));
8428   ins_cost(MEMORY_REF_COST);
8429   // TODO: s390 port size(VARIABLE_SIZE);
8430   format %{ "CL(Y)   $op1, $op2\t # unsigned" %}
8431   opcode(CLY_ZOPC, CL_ZOPC);
8432   ins_encode(z_form_rt_mem_opt(op1, op2));
8433   ins_pipe(pipe_class_dummy);
8434 %}
8435 
8436 // LONG signed
8437 
8438 instruct compL_reg_reg(flagsReg cr, iRegL op1, iRegL op2) %{
8439   match(Set cr (CmpL op1 op2));
8440   size(4);
8441   format %{ "CGR     $op1,$op2\t # long" %}
8442   opcode(CGR_ZOPC);
8443   ins_encode(z_rreform(op1, op2));
8444   ins_pipe(pipe_class_dummy);
8445 %}
8446 
8447 instruct compL_reg_regI(flagsReg cr, iRegL op1, iRegI op2) %{
8448   match(Set cr (CmpL op1 (ConvI2L op2)));
8449   size(4);
8450   format %{ "CGFR    $op1,$op2\t # long/int" %}
8451   opcode(CGFR_ZOPC);
8452   ins_encode(z_rreform(op1, op2));
8453   ins_pipe(pipe_class_dummy);
8454 %}
8455 
8456 instruct compL_reg_imm32(flagsReg cr, iRegL op1, immL32 con) %{
8457   match(Set cr (CmpL op1 con));
8458   size(6);
8459   format %{ "CGFI    $op1,$con" %}
8460   opcode(CGFI_ZOPC);
8461   ins_encode(z_rilform_signed(op1, con));
8462   ins_pipe(pipe_class_dummy);
8463 %}
8464 
8465 instruct compL_reg_imm16(flagsReg cr, iRegL op1, immL16 con) %{
8466   match(Set cr (CmpL op1 con));
8467   size(4);
8468   format %{ "CGHI    $op1,$con" %}
8469   opcode(CGHI_ZOPC);
8470   ins_encode(z_riform_signed(op1, con));
8471   ins_pipe(pipe_class_dummy);
8472 %}
8473 
8474 instruct compL_reg_imm0(flagsReg cr, iRegL op1, immL_0 con) %{
8475   match(Set cr (CmpL op1 con));
8476   ins_cost(DEFAULT_COST_LOW);
8477   size(4);
8478   format %{ "LTGR    $op1,$op1" %}
8479   opcode(LTGR_ZOPC);
8480   ins_encode(z_rreform(op1, op1));
8481   ins_pipe(pipe_class_dummy);
8482 %}
8483 
8484 instruct compL_conv_reg_imm0(flagsReg cr, iRegI op1, immL_0 con) %{
8485   match(Set cr (CmpL (ConvI2L op1) con));
8486   ins_cost(DEFAULT_COST_LOW);
8487   size(4);
8488   format %{ "LTGFR    $op1,$op1" %}
8489   opcode(LTGFR_ZOPC);
8490   ins_encode(z_rreform(op1, op1));
8491   ins_pipe(pipe_class_dummy);
8492 %}
8493 
8494 instruct compL_reg_mem(iRegL dst, memory src, flagsReg cr)%{
8495   match(Set cr (CmpL dst (LoadL src)));
8496   ins_cost(MEMORY_REF_COST);
8497   size(Z_DISP3_SIZE);
8498   format %{ "CG      $dst, $src\t # long" %}
8499   opcode(CG_ZOPC, CG_ZOPC);
8500   ins_encode(z_form_rt_mem_opt(dst, src));
8501   ins_pipe(pipe_class_dummy);
8502 %}
8503 
8504 instruct compL_reg_memI(iRegL dst, memory src, flagsReg cr)%{
8505   match(Set cr (CmpL dst (ConvI2L (LoadI src))));
8506   ins_cost(MEMORY_REF_COST);
8507   size(Z_DISP3_SIZE);
8508   format %{ "CGF     $dst, $src\t # long/int" %}
8509   opcode(CGF_ZOPC, CGF_ZOPC);
8510   ins_encode(z_form_rt_mem_opt(dst, src));
8511   ins_pipe(pipe_class_dummy);
8512 %}
8513 
8514 //  LONG unsigned
8515 // Added CmpUL for LoopPredicate.
8516 instruct compUL_reg_reg(flagsReg cr, iRegL op1, iRegL op2) %{
8517   match(Set cr (CmpUL op1 op2));
8518   size(4);
8519   format %{ "CLGR    $op1,$op2\t # long" %}
8520   opcode(CLGR_ZOPC);
8521   ins_encode(z_rreform(op1, op2));
8522   ins_pipe(pipe_class_dummy);
8523 %}
8524 
8525 instruct compUL_reg_imm32(flagsReg cr, iRegL op1, uimmL32 con) %{
8526   match(Set cr (CmpUL op1 con));
8527   size(6);
8528   format %{ "CLGFI   $op1,$con" %}
8529   opcode(CLGFI_ZOPC);
8530   ins_encode(z_rilform_unsigned(op1, con));
8531   ins_pipe(pipe_class_dummy);
8532 %}
8533 
8534 //  PTR unsigned
8535 
8536 instruct compP_reg_reg(flagsReg cr, iRegP_N2P op1, iRegP_N2P op2) %{
8537   match(Set cr (CmpP op1 op2));
8538   size(4);
8539   format %{ "CLGR    $op1,$op2\t # ptr" %}
8540   opcode(CLGR_ZOPC);
8541   ins_encode(z_rreform(op1, op2));
8542   ins_pipe(pipe_class_dummy);
8543 %}
8544 
8545 instruct compP_reg_imm0(flagsReg cr, iRegP_N2P op1, immP0 op2) %{
8546   match(Set cr (CmpP op1 op2));
8547   ins_cost(DEFAULT_COST_LOW);
8548   size(4);
8549   format %{ "LTGR    $op1, $op1\t # ptr" %}
8550   opcode(LTGR_ZOPC);
8551   ins_encode(z_rreform(op1, op1));
8552   ins_pipe(pipe_class_dummy);
8553 %}
8554 
8555 // Don't use LTGFR which performs sign extend.
8556 instruct compP_decode_reg_imm0(flagsReg cr, iRegN op1, immP0 op2) %{
8557   match(Set cr (CmpP (DecodeN op1) op2));
8558   predicate(Universe::narrow_oop_base() == NULL && Universe::narrow_oop_shift() == 0);
8559   ins_cost(DEFAULT_COST_LOW);
8560   size(2);
8561   format %{ "LTR    $op1, $op1\t # ptr" %}
8562   opcode(LTR_ZOPC);
8563   ins_encode(z_rrform(op1, op1));
8564   ins_pipe(pipe_class_dummy);
8565 %}
8566 
8567 instruct compP_reg_mem(iRegP dst, memory src, flagsReg cr)%{
8568   match(Set cr (CmpP dst (LoadP src)));
8569   ins_cost(MEMORY_REF_COST);
8570   size(Z_DISP3_SIZE);
8571   format %{ "CLG     $dst, $src\t # ptr" %}
8572   opcode(CLG_ZOPC, CLG_ZOPC);
8573   ins_encode(z_form_rt_mem_opt(dst, src));
8574   ins_pipe(pipe_class_dummy);
8575 %}
8576 
8577 //----------Max and Min--------------------------------------------------------
8578 
8579 // Max Register with Register
8580 instruct z196_minI_reg_reg(iRegI dst, iRegI src1, iRegI src2, flagsReg cr) %{
8581   match(Set dst (MinI src1 src2));
8582   effect(KILL cr);
8583   predicate(VM_Version::has_LoadStoreConditional());
8584   ins_cost(3 * DEFAULT_COST);
8585   // TODO: s390 port size(VARIABLE_SIZE);
8586   format %{ "MinI $dst $src1,$src2\t MinI (z196 only)" %}
8587   ins_encode %{
8588     Register Rdst = $dst$$Register;
8589     Register Rsrc1 = $src1$$Register;
8590     Register Rsrc2 = $src2$$Register;
8591 
8592     if (Rsrc1 == Rsrc2) {
8593       if (Rdst != Rsrc1) {
8594         __ z_lgfr(Rdst, Rsrc1);
8595       }
8596     } else if (Rdst == Rsrc1) {   // Rdst preset with src1.
8597       __ z_cr(Rsrc1, Rsrc2);      // Move src2 only if src1 is NotLow.
8598       __ z_locr(Rdst, Rsrc2, Assembler::bcondNotLow);
8599     } else if (Rdst == Rsrc2) {   // Rdst preset with src2.
8600       __ z_cr(Rsrc2, Rsrc1);      // Move src1 only if src2 is NotLow.
8601       __ z_locr(Rdst, Rsrc1, Assembler::bcondNotLow);
8602     } else {
8603       // Rdst is disjoint from operands, move in either case.
8604       __ z_cr(Rsrc1, Rsrc2);
8605       __ z_locr(Rdst, Rsrc2, Assembler::bcondNotLow);
8606       __ z_locr(Rdst, Rsrc1, Assembler::bcondLow);
8607     }
8608   %}
8609   ins_pipe(pipe_class_dummy);
8610 %}
8611 
8612 // Min Register with Register.
8613 instruct z10_minI_reg_reg(iRegI dst, iRegI src1, iRegI src2, flagsReg cr) %{
8614   match(Set dst (MinI src1 src2));
8615   effect(KILL cr);
8616   predicate(VM_Version::has_CompareBranch());
8617   ins_cost(2 * DEFAULT_COST + BRANCH_COST);
8618   // TODO: s390 port size(VARIABLE_SIZE);
8619   format %{ "MinI $dst $src1,$src2\t MinI (z10 only)" %}
8620   ins_encode %{
8621     Register Rdst = $dst$$Register;
8622     Register Rsrc1 = $src1$$Register;
8623     Register Rsrc2 = $src2$$Register;
8624     Label done;
8625 
8626     if (Rsrc1 == Rsrc2) {
8627       if (Rdst != Rsrc1) {
8628         __ z_lgfr(Rdst, Rsrc1);
8629       }
8630     } else if (Rdst == Rsrc1) {
8631       __ z_crj(Rsrc1, Rsrc2, Assembler::bcondLow, done);
8632       __ z_lgfr(Rdst, Rsrc2);
8633     } else if (Rdst == Rsrc2) {
8634       __ z_crj(Rsrc2, Rsrc1, Assembler::bcondLow, done);
8635       __ z_lgfr(Rdst, Rsrc1);
8636     } else {
8637       __ z_lgfr(Rdst, Rsrc1);
8638       __ z_crj(Rsrc1, Rsrc2, Assembler::bcondLow, done);
8639       __ z_lgfr(Rdst, Rsrc2);
8640     }
8641     __ bind(done);
8642   %}
8643   ins_pipe(pipe_class_dummy);
8644 %}
8645 
8646 instruct minI_reg_reg(iRegI dst, iRegI src1, iRegI src2, flagsReg cr) %{
8647   match(Set dst (MinI src1 src2));
8648   effect(KILL cr);
8649   predicate(!VM_Version::has_CompareBranch());
8650   ins_cost(3 * DEFAULT_COST + BRANCH_COST);
8651   // TODO: s390 port size(VARIABLE_SIZE);
8652   format %{ "MinI $dst $src1,$src2\t MinI" %}
8653   ins_encode %{
8654     Register Rdst = $dst$$Register;
8655     Register Rsrc1 = $src1$$Register;
8656     Register Rsrc2 = $src2$$Register;
8657     Label done;
8658 
8659     if (Rsrc1 == Rsrc2) {
8660       if (Rdst != Rsrc1) {
8661         __ z_lgfr(Rdst, Rsrc1);
8662       }
8663     } else if (Rdst == Rsrc1) {
8664       __ z_cr(Rsrc1, Rsrc2);
8665       __ z_brl(done);
8666       __ z_lgfr(Rdst, Rsrc2);
8667     } else if (Rdst == Rsrc2) {
8668       __ z_cr(Rsrc2, Rsrc1);
8669       __ z_brl(done);
8670       __ z_lgfr(Rdst, Rsrc1);
8671     } else {
8672       __ z_lgfr(Rdst, Rsrc1);
8673       __ z_cr(Rsrc1, Rsrc2);
8674       __ z_brl(done);
8675       __ z_lgfr(Rdst, Rsrc2);
8676     }
8677     __ bind(done);
8678   %}
8679   ins_pipe(pipe_class_dummy);
8680 %}
8681 
8682 instruct z196_minI_reg_imm32(iRegI dst, iRegI src1, immI src2, flagsReg cr) %{
8683   match(Set dst (MinI src1 src2));
8684   effect(KILL cr);
8685   predicate(VM_Version::has_LoadStoreConditional());
8686   ins_cost(3 * DEFAULT_COST);
8687   // TODO: s390 port size(VARIABLE_SIZE);
8688   format %{ "MinI $dst $src1,$src2\t MinI const32 (z196 only)" %}
8689   ins_encode %{
8690     Register Rdst = $dst$$Register;
8691     Register Rsrc1 = $src1$$Register;
8692     int      Isrc2 = $src2$$constant;
8693 
8694     if (Rdst == Rsrc1) {
8695       __ load_const_optimized(Z_R0_scratch, Isrc2);
8696       __ z_cfi(Rsrc1, Isrc2);
8697       __ z_locr(Rdst, Z_R0_scratch, Assembler::bcondNotLow);
8698     } else {
8699       __ load_const_optimized(Rdst, Isrc2);
8700       __ z_cfi(Rsrc1, Isrc2);
8701       __ z_locr(Rdst, Rsrc1, Assembler::bcondLow);
8702     }
8703   %}
8704   ins_pipe(pipe_class_dummy);
8705 %}
8706 
8707 instruct minI_reg_imm32(iRegI dst, iRegI src1, immI src2, flagsReg cr) %{
8708   match(Set dst (MinI src1 src2));
8709   effect(KILL cr);
8710   ins_cost(2 * DEFAULT_COST + BRANCH_COST);
8711   // TODO: s390 port size(VARIABLE_SIZE);
8712   format %{ "MinI $dst $src1,$src2\t MinI const32" %}
8713   ins_encode %{
8714     Label done;
8715     if ($dst$$Register != $src1$$Register) {
8716       __ z_lgfr($dst$$Register, $src1$$Register);
8717     }
8718     __ z_cfi($src1$$Register, $src2$$constant);
8719     __ z_brl(done);
8720     __ z_lgfi($dst$$Register, $src2$$constant);
8721     __ bind(done);
8722   %}
8723   ins_pipe(pipe_class_dummy);
8724 %}
8725 
8726 instruct z196_minI_reg_imm16(iRegI dst, iRegI src1, immI16 src2, flagsReg cr) %{
8727   match(Set dst (MinI src1 src2));
8728   effect(KILL cr);
8729   predicate(VM_Version::has_LoadStoreConditional());
8730   ins_cost(3 * DEFAULT_COST);
8731   // TODO: s390 port size(VARIABLE_SIZE);
8732   format %{ "MinI $dst $src1,$src2\t MinI const16 (z196 only)" %}
8733   ins_encode %{
8734     Register Rdst = $dst$$Register;
8735     Register Rsrc1 = $src1$$Register;
8736     int      Isrc2 = $src2$$constant;
8737 
8738     if (Rdst == Rsrc1) {
8739       __ load_const_optimized(Z_R0_scratch, Isrc2);
8740       __ z_chi(Rsrc1, Isrc2);
8741       __ z_locr(Rdst, Z_R0_scratch, Assembler::bcondNotLow);
8742     } else {
8743       __ load_const_optimized(Rdst, Isrc2);
8744       __ z_chi(Rsrc1, Isrc2);
8745       __ z_locr(Rdst, Rsrc1, Assembler::bcondLow);
8746     }
8747   %}
8748   ins_pipe(pipe_class_dummy);
8749 %}
8750 
8751 instruct minI_reg_imm16(iRegI dst, iRegI src1, immI16 src2, flagsReg cr) %{
8752   match(Set dst (MinI src1 src2));
8753   effect(KILL cr);
8754   ins_cost(2 * DEFAULT_COST + BRANCH_COST);
8755   // TODO: s390 port size(VARIABLE_SIZE);
8756   format %{ "MinI $dst $src1,$src2\t MinI const16" %}
8757   ins_encode %{
8758     Label done;
8759     if ($dst$$Register != $src1$$Register) {
8760       __ z_lgfr($dst$$Register, $src1$$Register);
8761     }
8762     __ z_chi($src1$$Register, $src2$$constant);
8763     __ z_brl(done);
8764     __ z_lghi($dst$$Register, $src2$$constant);
8765     __ bind(done);
8766   %}
8767   ins_pipe(pipe_class_dummy);
8768 %}
8769 
8770 instruct z10_minI_reg_imm8(iRegI dst, iRegI src1, immI8 src2, flagsReg cr) %{
8771   match(Set dst (MinI src1 src2));
8772   effect(KILL cr);
8773   predicate(VM_Version::has_CompareBranch());
8774   ins_cost(DEFAULT_COST + BRANCH_COST);
8775   // TODO: s390 port size(VARIABLE_SIZE);
8776   format %{ "MinI $dst $src1,$src2\t MinI const8 (z10 only)" %}
8777   ins_encode %{
8778     Label done;
8779     if ($dst$$Register != $src1$$Register) {
8780       __ z_lgfr($dst$$Register, $src1$$Register);
8781     }
8782     __ z_cij($src1$$Register, $src2$$constant, Assembler::bcondLow, done);
8783     __ z_lghi($dst$$Register, $src2$$constant);
8784     __ bind(done);
8785   %}
8786   ins_pipe(pipe_class_dummy);
8787 %}
8788 
8789 // Max Register with Register
8790 instruct z196_maxI_reg_reg(iRegI dst, iRegI src1, iRegI src2, flagsReg cr) %{
8791   match(Set dst (MaxI src1 src2));
8792   effect(KILL cr);
8793   predicate(VM_Version::has_LoadStoreConditional());
8794   ins_cost(3 * DEFAULT_COST);
8795   // TODO: s390 port size(VARIABLE_SIZE);
8796   format %{ "MaxI $dst $src1,$src2\t MaxI (z196 only)" %}
8797   ins_encode %{
8798     Register Rdst = $dst$$Register;
8799     Register Rsrc1 = $src1$$Register;
8800     Register Rsrc2 = $src2$$Register;
8801 
8802     if (Rsrc1 == Rsrc2) {
8803       if (Rdst != Rsrc1) {
8804         __ z_lgfr(Rdst, Rsrc1);
8805       }
8806     } else if (Rdst == Rsrc1) { // Rdst preset with src1.
8807       __ z_cr(Rsrc1, Rsrc2);    // Move src2 only if src1 is NotHigh.
8808       __ z_locr(Rdst, Rsrc2, Assembler::bcondNotHigh);
8809     } else if (Rdst == Rsrc2) { // Rdst preset with src2.
8810       __ z_cr(Rsrc2, Rsrc1);    // Move src1 only if src2 is NotHigh.
8811       __ z_locr(Rdst, Rsrc1, Assembler::bcondNotHigh);
8812     } else {                    // Rdst is disjoint from operands, move in either case.
8813       __ z_cr(Rsrc1, Rsrc2);
8814       __ z_locr(Rdst, Rsrc2, Assembler::bcondNotHigh);
8815       __ z_locr(Rdst, Rsrc1, Assembler::bcondHigh);
8816     }
8817   %}
8818   ins_pipe(pipe_class_dummy);
8819 %}
8820 
8821 // Max Register with Register
8822 instruct z10_maxI_reg_reg(iRegI dst, iRegI src1, iRegI src2, flagsReg cr) %{
8823   match(Set dst (MaxI src1 src2));
8824   effect(KILL cr);
8825   predicate(VM_Version::has_CompareBranch());
8826   ins_cost(2 * DEFAULT_COST + BRANCH_COST);
8827   // TODO: s390 port size(VARIABLE_SIZE);
8828   format %{ "MaxI $dst $src1,$src2\t MaxI (z10 only)" %}
8829   ins_encode %{
8830     Register Rdst = $dst$$Register;
8831     Register Rsrc1 = $src1$$Register;
8832     Register Rsrc2 = $src2$$Register;
8833     Label done;
8834 
8835     if (Rsrc1 == Rsrc2) {
8836       if (Rdst != Rsrc1) {
8837         __ z_lgfr(Rdst, Rsrc1);
8838       }
8839     } else if (Rdst == Rsrc1) {
8840       __ z_crj(Rsrc1, Rsrc2, Assembler::bcondHigh, done);
8841       __ z_lgfr(Rdst, Rsrc2);
8842     } else if (Rdst == Rsrc2) {
8843       __ z_crj(Rsrc2, Rsrc1, Assembler::bcondHigh, done);
8844       __ z_lgfr(Rdst, Rsrc1);
8845     } else {
8846       __ z_lgfr(Rdst, Rsrc1);
8847       __ z_crj(Rsrc1, Rsrc2, Assembler::bcondHigh, done);
8848       __ z_lgfr(Rdst, Rsrc2);
8849     }
8850     __ bind(done);
8851   %}
8852   ins_pipe(pipe_class_dummy);
8853 %}
8854 
8855 instruct maxI_reg_reg(iRegI dst, iRegI src1, iRegI src2, flagsReg cr) %{
8856   match(Set dst (MaxI src1 src2));
8857   effect(KILL cr);
8858   predicate(!VM_Version::has_CompareBranch());
8859   ins_cost(3 * DEFAULT_COST + BRANCH_COST);
8860   // TODO: s390 port size(VARIABLE_SIZE);
8861   format %{ "MaxI $dst $src1,$src2\t MaxI" %}
8862   ins_encode %{
8863     Register Rdst = $dst$$Register;
8864     Register Rsrc1 = $src1$$Register;
8865     Register Rsrc2 = $src2$$Register;
8866     Label done;
8867 
8868     if (Rsrc1 == Rsrc2) {
8869       if (Rdst != Rsrc1) {
8870         __ z_lgfr(Rdst, Rsrc1);
8871       }
8872     } else if (Rdst == Rsrc1) {
8873       __ z_cr(Rsrc1, Rsrc2);
8874       __ z_brh(done);
8875       __ z_lgfr(Rdst, Rsrc2);
8876     } else if (Rdst == Rsrc2) {
8877       __ z_cr(Rsrc2, Rsrc1);
8878       __ z_brh(done);
8879       __ z_lgfr(Rdst, Rsrc1);
8880     } else {
8881       __ z_lgfr(Rdst, Rsrc1);
8882       __ z_cr(Rsrc1, Rsrc2);
8883       __ z_brh(done);
8884       __ z_lgfr(Rdst, Rsrc2);
8885     }
8886 
8887     __ bind(done);
8888   %}
8889 
8890   ins_pipe(pipe_class_dummy);
8891 %}
8892 
8893 instruct z196_maxI_reg_imm32(iRegI dst, iRegI src1, immI src2, flagsReg cr) %{
8894   match(Set dst (MaxI src1 src2));
8895   effect(KILL cr);
8896   predicate(VM_Version::has_LoadStoreConditional());
8897   ins_cost(3 * DEFAULT_COST);
8898   // TODO: s390 port size(VARIABLE_SIZE);
8899   format %{ "MaxI $dst $src1,$src2\t MaxI const32 (z196 only)" %}
8900   ins_encode %{
8901     Register Rdst = $dst$$Register;
8902     Register Rsrc1 = $src1$$Register;
8903     int      Isrc2 = $src2$$constant;
8904 
8905     if (Rdst == Rsrc1) {
8906       __ load_const_optimized(Z_R0_scratch, Isrc2);
8907       __ z_cfi(Rsrc1, Isrc2);
8908       __ z_locr(Rdst, Z_R0_scratch, Assembler::bcondNotHigh);
8909     } else {
8910       __ load_const_optimized(Rdst, Isrc2);
8911       __ z_cfi(Rsrc1, Isrc2);
8912       __ z_locr(Rdst, Rsrc1, Assembler::bcondHigh);
8913     }
8914   %}
8915   ins_pipe(pipe_class_dummy);
8916 %}
8917 
8918 instruct maxI_reg_imm32(iRegI dst, iRegI src1, immI src2, flagsReg cr) %{
8919   match(Set dst (MaxI src1 src2));
8920   effect(KILL cr);
8921   ins_cost(2 * DEFAULT_COST + BRANCH_COST);
8922   // TODO: s390 port size(VARIABLE_SIZE);
8923   format %{ "MaxI $dst $src1,$src2\t MaxI const32" %}
8924   ins_encode %{
8925     Label done;
8926     if ($dst$$Register != $src1$$Register) {
8927       __ z_lgfr($dst$$Register, $src1$$Register);
8928     }
8929     __ z_cfi($src1$$Register, $src2$$constant);
8930     __ z_brh(done);
8931     __ z_lgfi($dst$$Register, $src2$$constant);
8932     __ bind(done);
8933   %}
8934   ins_pipe(pipe_class_dummy);
8935 %}
8936 
8937 instruct z196_maxI_reg_imm16(iRegI dst, iRegI src1, immI16 src2, flagsReg cr) %{
8938   match(Set dst (MaxI src1 src2));
8939   effect(KILL cr);
8940   predicate(VM_Version::has_LoadStoreConditional());
8941   ins_cost(3 * DEFAULT_COST);
8942   // TODO: s390 port size(VARIABLE_SIZE);
8943   format %{ "MaxI $dst $src1,$src2\t MaxI const16 (z196 only)" %}
8944   ins_encode %{
8945     Register Rdst = $dst$$Register;
8946     Register Rsrc1 = $src1$$Register;
8947     int      Isrc2 = $src2$$constant;
8948     if (Rdst == Rsrc1) {
8949       __ load_const_optimized(Z_R0_scratch, Isrc2);
8950       __ z_chi(Rsrc1, Isrc2);
8951       __ z_locr(Rdst, Z_R0_scratch, Assembler::bcondNotHigh);
8952     } else {
8953       __ load_const_optimized(Rdst, Isrc2);
8954       __ z_chi(Rsrc1, Isrc2);
8955       __ z_locr(Rdst, Rsrc1, Assembler::bcondHigh);
8956     }
8957   %}
8958   ins_pipe(pipe_class_dummy);
8959 %}
8960 
8961 instruct maxI_reg_imm16(iRegI dst, iRegI src1, immI16 src2, flagsReg cr) %{
8962   match(Set dst (MaxI src1 src2));
8963   effect(KILL cr);
8964   ins_cost(2 * DEFAULT_COST + BRANCH_COST);
8965   // TODO: s390 port size(VARIABLE_SIZE);
8966   format %{ "MaxI $dst $src1,$src2\t MaxI const16" %}
8967   ins_encode %{
8968     Label done;
8969     if ($dst$$Register != $src1$$Register) {
8970       __ z_lgfr($dst$$Register, $src1$$Register);
8971     }
8972     __ z_chi($src1$$Register, $src2$$constant);
8973     __ z_brh(done);
8974     __ z_lghi($dst$$Register, $src2$$constant);
8975     __ bind(done);
8976   %}
8977   ins_pipe(pipe_class_dummy);
8978 %}
8979 
8980 instruct z10_maxI_reg_imm8(iRegI dst, iRegI src1, immI8 src2, flagsReg cr) %{
8981   match(Set dst (MaxI src1 src2));
8982   effect(KILL cr);
8983   predicate(VM_Version::has_CompareBranch());
8984   ins_cost(DEFAULT_COST + BRANCH_COST);
8985   // TODO: s390 port size(VARIABLE_SIZE);
8986   format %{ "MaxI $dst $src1,$src2\t MaxI const8" %}
8987   ins_encode %{
8988     Label done;
8989     if ($dst$$Register != $src1$$Register) {
8990       __ z_lgfr($dst$$Register, $src1$$Register);
8991     }
8992     __ z_cij($src1$$Register, $src2$$constant, Assembler::bcondHigh, done);
8993     __ z_lghi($dst$$Register, $src2$$constant);
8994     __ bind(done);
8995   %}
8996   ins_pipe(pipe_class_dummy);
8997 %}
8998 
8999 //----------Abs---------------------------------------------------------------
9000 
9001 instruct absI_reg(iRegI dst, iRegI src, flagsReg cr) %{
9002   match(Set dst (AbsI src));
9003   effect(KILL cr);
9004   ins_cost(DEFAULT_COST_LOW);
9005   // TODO: s390 port size(FIXED_SIZE);
9006   format %{ "LPR     $dst, $src" %}
9007   opcode(LPR_ZOPC);
9008   ins_encode(z_rrform(dst, src));
9009   ins_pipe(pipe_class_dummy);
9010 %}
9011 
9012 instruct negabsI_reg(iRegI dst, iRegI src, immI_0 zero, flagsReg cr) %{
9013   match(Set dst (SubI zero (AbsI src)));
9014   effect(KILL cr);
9015   ins_cost(DEFAULT_COST_LOW);
9016   // TODO: s390 port size(FIXED_SIZE);
9017   format %{ "LNR     $dst, $src" %}
9018   opcode(LNR_ZOPC);
9019   ins_encode(z_rrform(dst, src));
9020   ins_pipe(pipe_class_dummy);
9021 %}
9022 
9023 //----------Float Compares----------------------------------------------------
9024 
9025 // Compare floating, generate condition code.
9026 instruct cmpF_cc(flagsReg cr, regF src1, regF src2) %{
9027   match(Set cr (CmpF src1 src2));
9028   ins_cost(ALU_REG_COST);
9029   size(4);
9030   format %{ "FCMPcc   $src1,$src2\t # float" %}
9031   ins_encode %{ __ z_cebr($src1$$FloatRegister, $src2$$FloatRegister); %}
9032   ins_pipe(pipe_class_dummy);
9033 %}
9034 
9035 instruct cmpD_cc(flagsReg cr, regD src1, regD src2) %{
9036   match(Set cr (CmpD src1 src2));
9037   ins_cost(ALU_REG_COST);
9038   size(4);
9039   format %{ "FCMPcc   $src1,$src2 \t # double" %}
9040   ins_encode %{ __ z_cdbr($src1$$FloatRegister, $src2$$FloatRegister); %}
9041   ins_pipe(pipe_class_dummy);
9042 %}
9043 
9044 instruct cmpF_cc_mem(flagsReg cr, regF src1, memoryRX src2) %{
9045   match(Set cr (CmpF src1 (LoadF src2)));
9046   ins_cost(ALU_MEMORY_COST);
9047   size(6);
9048   format %{ "FCMPcc_mem $src1,$src2\t # floatMemory" %}
9049   opcode(CEB_ZOPC);
9050   ins_encode(z_form_rt_memFP(src1, src2));
9051   ins_pipe(pipe_class_dummy);
9052 %}
9053 
9054 instruct cmpD_cc_mem(flagsReg cr, regD src1, memoryRX src2) %{
9055   match(Set cr (CmpD src1 (LoadD src2)));
9056   ins_cost(ALU_MEMORY_COST);
9057   size(6);
9058   format %{ "DCMPcc_mem $src1,$src2\t # doubleMemory" %}
9059   opcode(CDB_ZOPC);
9060   ins_encode(z_form_rt_memFP(src1, src2));
9061   ins_pipe(pipe_class_dummy);
9062 %}
9063 
9064 // Compare floating, generate condition code
9065 instruct cmpF0_cc(flagsReg cr, regF src1, immFpm0 src2) %{
9066   match(Set cr (CmpF src1 src2));
9067   ins_cost(DEFAULT_COST);
9068   size(4);
9069   format %{ "LTEBR    $src1,$src1\t # float" %}
9070   opcode(LTEBR_ZOPC);
9071   ins_encode(z_rreform(src1, src1));
9072   ins_pipe(pipe_class_dummy);
9073 %}
9074 
9075 instruct cmpD0_cc(flagsReg cr, regD src1, immDpm0 src2) %{
9076   match(Set cr (CmpD src1 src2));
9077   ins_cost(DEFAULT_COST);
9078   size(4);
9079   format %{ "LTDBR    $src1,$src1 \t # double" %}
9080   opcode(LTDBR_ZOPC);
9081   ins_encode(z_rreform(src1, src1));
9082   ins_pipe(pipe_class_dummy);
9083 %}
9084 
9085 // Compare floating, generate -1,0,1
9086 instruct cmpF_reg(iRegI dst, regF src1, regF src2, flagsReg cr) %{
9087   match(Set dst (CmpF3 src1 src2));
9088   effect(KILL cr);
9089   ins_cost(DEFAULT_COST * 5 + BRANCH_COST);
9090   size(24);
9091   format %{ "CmpF3    $dst,$src1,$src2" %}
9092   ins_encode %{
9093     // compare registers
9094     __ z_cebr($src1$$FloatRegister, $src2$$FloatRegister);
9095     // Convert condition code into -1,0,1, where
9096     // -1 means unordered or less
9097     //  0 means equal
9098     //  1 means greater.
9099     if (VM_Version::has_LoadStoreConditional()) {
9100       Register one       = Z_R0_scratch;
9101       Register minus_one = Z_R1_scratch;
9102       __ z_lghi(minus_one, -1);
9103       __ z_lghi(one, 1);
9104       __ z_lghi( $dst$$Register, 0);
9105       __ z_locgr($dst$$Register, one,       Assembler::bcondHigh);
9106       __ z_locgr($dst$$Register, minus_one, Assembler::bcondLowOrNotOrdered);
9107     } else {
9108       Label done;
9109       __ clear_reg($dst$$Register, true, false);
9110       __ z_bre(done);
9111       __ z_lhi($dst$$Register, 1);
9112       __ z_brh(done);
9113       __ z_lhi($dst$$Register, -1);
9114       __ bind(done);
9115     }
9116   %}
9117   ins_pipe(pipe_class_dummy);
9118 %}
9119 
9120 instruct cmpD_reg(iRegI dst, regD src1, regD src2, flagsReg cr) %{
9121   match(Set dst (CmpD3 src1 src2));
9122   effect(KILL cr);
9123   ins_cost(DEFAULT_COST * 5 + BRANCH_COST);
9124   size(24);
9125   format %{ "CmpD3    $dst,$src1,$src2" %}
9126   ins_encode %{
9127     // compare registers
9128     __ z_cdbr($src1$$FloatRegister, $src2$$FloatRegister);
9129     // Convert condition code into -1,0,1, where
9130     // -1 means unordered or less
9131     //  0 means equal
9132     //  1 means greater.
9133     if (VM_Version::has_LoadStoreConditional()) {
9134       Register one       = Z_R0_scratch;
9135       Register minus_one = Z_R1_scratch;
9136       __ z_lghi(minus_one, -1);
9137       __ z_lghi(one, 1);
9138       __ z_lghi( $dst$$Register, 0);
9139       __ z_locgr($dst$$Register, one,       Assembler::bcondHigh);
9140       __ z_locgr($dst$$Register, minus_one, Assembler::bcondLowOrNotOrdered);
9141     } else {
9142       Label done;
9143       // indicate unused result
9144       (void) __ clear_reg($dst$$Register, true, false);
9145       __ z_bre(done);
9146       __ z_lhi($dst$$Register, 1);
9147       __ z_brh(done);
9148       __ z_lhi($dst$$Register, -1);
9149       __ bind(done);
9150     }
9151   %}
9152   ins_pipe(pipe_class_dummy);
9153 %}
9154 
9155 //----------Branches---------------------------------------------------------
9156 // Jump
9157 
9158 // Direct Branch.
9159 instruct branch(label labl) %{
9160   match(Goto);
9161   effect(USE labl);
9162   ins_cost(BRANCH_COST);
9163   size(4);
9164   format %{ "BRU     $labl" %}
9165   ins_encode(z_enc_bru(labl));
9166   ins_pipe(pipe_class_dummy);
9167   // If set to 1 this indicates that the current instruction is a
9168   // short variant of a long branch. This avoids using this
9169   // instruction in first-pass matching. It will then only be used in
9170   // the `Shorten_branches' pass.
9171   ins_short_branch(1);
9172 %}
9173 
9174 // Direct Branch.
9175 instruct branchFar(label labl) %{
9176   match(Goto);
9177   effect(USE labl);
9178   ins_cost(BRANCH_COST);
9179   size(6);
9180   format %{ "BRUL   $labl" %}
9181   ins_encode(z_enc_brul(labl));
9182   ins_pipe(pipe_class_dummy);
9183   // This is not a short variant of a branch, but the long variant.
9184   ins_short_branch(0);
9185 %}
9186 
9187 // Conditional Near Branch
9188 instruct branchCon(cmpOp cmp, flagsReg cr, label lbl) %{
9189   // Same match rule as `branchConFar'.
9190   match(If cmp cr);
9191   effect(USE lbl);
9192   ins_cost(BRANCH_COST);
9193   size(4);
9194   format %{ "branch_con_short,$cmp   $cr, $lbl" %}
9195   ins_encode(z_enc_branch_con_short(cmp, lbl));
9196   ins_pipe(pipe_class_dummy);
9197   // If set to 1 this indicates that the current instruction is a
9198   // short variant of a long branch. This avoids using this
9199   // instruction in first-pass matching. It will then only be used in
9200   // the `Shorten_branches' pass.
9201   ins_short_branch(1);
9202 %}
9203 
9204 // This is for cases when the z/Architecture conditional branch instruction
9205 // does not reach far enough. So we emit a far branch here, which is
9206 // more expensive.
9207 //
9208 // Conditional Far Branch
9209 instruct branchConFar(cmpOp cmp, flagsReg cr, label lbl) %{
9210   // Same match rule as `branchCon'.
9211   match(If cmp cr);
9212   effect(USE cr, USE lbl);
9213   // Make more expensive to prefer compare_and_branch over separate instructions.
9214   ins_cost(2 * BRANCH_COST);
9215   size(6);
9216   format %{ "branch_con_far,$cmp   $cr, $lbl" %}
9217   ins_encode(z_enc_branch_con_far(cmp, lbl));
9218   ins_pipe(pipe_class_dummy);
9219   // This is not a short variant of a branch, but the long variant..
9220   ins_short_branch(0);
9221 %}
9222 
9223 instruct branchLoopEnd(cmpOp cmp, flagsReg cr, label labl) %{
9224   match(CountedLoopEnd cmp cr);
9225   effect(USE labl);
9226   ins_cost(BRANCH_COST);
9227   size(4);
9228   format %{ "branch_con_short,$cmp   $labl\t # counted loop end" %}
9229   ins_encode(z_enc_branch_con_short(cmp, labl));
9230   ins_pipe(pipe_class_dummy);
9231   // If set to 1 this indicates that the current instruction is a
9232   // short variant of a long branch. This avoids using this
9233   // instruction in first-pass matching. It will then only be used in
9234   // the `Shorten_branches' pass.
9235   ins_short_branch(1);
9236 %}
9237 
9238 instruct branchLoopEndFar(cmpOp cmp, flagsReg cr, label labl) %{
9239   match(CountedLoopEnd cmp cr);
9240   effect(USE labl);
9241   ins_cost(BRANCH_COST);
9242   size(6);
9243   format %{ "branch_con_far,$cmp   $labl\t # counted loop end" %}
9244   ins_encode(z_enc_branch_con_far(cmp, labl));
9245   ins_pipe(pipe_class_dummy);
9246   // This is not a short variant of a branch, but the long variant.
9247   ins_short_branch(0);
9248 %}
9249 
9250 //----------Compare and Branch (short distance)------------------------------
9251 
9252 // INT REG operands for loop counter processing.
9253 instruct testAndBranchLoopEnd_Reg(cmpOpT boolnode, iRegI src1, iRegI src2, label labl, flagsReg cr) %{
9254   match(CountedLoopEnd boolnode (CmpI src1 src2));
9255   effect(USE labl, KILL cr);
9256   predicate(VM_Version::has_CompareBranch());
9257   ins_cost(BRANCH_COST);
9258   // TODO: s390 port size(FIXED_SIZE);
9259   format %{ "test_and_branch_loop_end,$boolnode  $src1,$src2,$labl\t # counted loop end SHORT" %}
9260   opcode(CRJ_ZOPC);
9261   ins_encode(z_enc_cmpb_regreg(src1, src2, labl, boolnode));
9262   ins_pipe(pipe_class_dummy);
9263   ins_short_branch(1);
9264 %}
9265 
9266 // INT REG operands.
9267 instruct cmpb_RegI(cmpOpT boolnode, iRegI src1, iRegI src2, label labl, flagsReg cr) %{
9268   match(If boolnode (CmpI src1 src2));
9269   effect(USE labl, KILL cr);
9270   predicate(VM_Version::has_CompareBranch());
9271   ins_cost(BRANCH_COST);
9272   // TODO: s390 port size(FIXED_SIZE);
9273   format %{ "CRJ,$boolnode  $src1,$src2,$labl\t # SHORT" %}
9274   opcode(CRJ_ZOPC);
9275   ins_encode(z_enc_cmpb_regreg(src1, src2, labl, boolnode));
9276   ins_pipe(pipe_class_dummy);
9277   ins_short_branch(1);
9278 %}
9279 
9280 // Unsigned INT REG operands
9281 instruct cmpbU_RegI(cmpOpT boolnode, iRegI src1, iRegI src2, label labl, flagsReg cr) %{
9282   match(If boolnode (CmpU src1 src2));
9283   effect(USE labl, KILL cr);
9284   predicate(VM_Version::has_CompareBranch());
9285   ins_cost(BRANCH_COST);
9286   // TODO: s390 port size(FIXED_SIZE);
9287   format %{ "CLRJ,$boolnode  $src1,$src2,$labl\t # SHORT" %}
9288   opcode(CLRJ_ZOPC);
9289   ins_encode(z_enc_cmpb_regreg(src1, src2, labl, boolnode));
9290   ins_pipe(pipe_class_dummy);
9291   ins_short_branch(1);
9292 %}
9293 
9294 // LONG REG operands
9295 instruct cmpb_RegL(cmpOpT boolnode, iRegL src1, iRegL src2, label labl, flagsReg cr) %{
9296   match(If boolnode (CmpL src1 src2));
9297   effect(USE labl, KILL cr);
9298   predicate(VM_Version::has_CompareBranch());
9299   ins_cost(BRANCH_COST);
9300   // TODO: s390 port size(FIXED_SIZE);
9301   format %{ "CGRJ,$boolnode $src1,$src2,$labl\t # SHORT" %}
9302   opcode(CGRJ_ZOPC);
9303   ins_encode(z_enc_cmpb_regreg(src1, src2, labl, boolnode));
9304   ins_pipe(pipe_class_dummy);
9305   ins_short_branch(1);
9306 %}
9307 
9308 //  PTR REG operands
9309 
9310 // Separate rules for regular and narrow oops.  ADLC can't recognize
9311 // rules with polymorphic operands to be sisters -> shorten_branches
9312 // will not shorten.
9313 
9314 instruct cmpb_RegPP(cmpOpT boolnode, iRegP src1, iRegP src2, label labl, flagsReg cr) %{
9315   match(If boolnode (CmpP src1 src2));
9316   effect(USE labl, KILL cr);
9317   predicate(VM_Version::has_CompareBranch());
9318   ins_cost(BRANCH_COST);
9319   // TODO: s390 port size(FIXED_SIZE);
9320   format %{ "CLGRJ,$boolnode $src1,$src2,$labl\t # SHORT" %}
9321   opcode(CLGRJ_ZOPC);
9322   ins_encode(z_enc_cmpb_regreg(src1, src2, labl, boolnode));
9323   ins_pipe(pipe_class_dummy);
9324   ins_short_branch(1);
9325 %}
9326 
9327 instruct cmpb_RegNN(cmpOpT boolnode, iRegN src1, iRegN src2, label labl, flagsReg cr) %{
9328   match(If boolnode (CmpP (DecodeN src1) (DecodeN src2)));
9329   effect(USE labl, KILL cr);
9330   predicate(VM_Version::has_CompareBranch());
9331   ins_cost(BRANCH_COST);
9332   // TODO: s390 port size(FIXED_SIZE);
9333   format %{ "CLGRJ,$boolnode $src1,$src2,$labl\t # SHORT" %}
9334   opcode(CLGRJ_ZOPC);
9335   ins_encode(z_enc_cmpb_regreg(src1, src2, labl, boolnode));
9336   ins_pipe(pipe_class_dummy);
9337   ins_short_branch(1);
9338 %}
9339 
9340 // INT REG/IMM operands for loop counter processing
9341 instruct testAndBranchLoopEnd_Imm(cmpOpT boolnode, iRegI src1, immI8 src2, label labl, flagsReg cr) %{
9342   match(CountedLoopEnd boolnode (CmpI src1 src2));
9343   effect(USE labl, KILL cr);
9344   predicate(VM_Version::has_CompareBranch());
9345   ins_cost(BRANCH_COST);
9346   // TODO: s390 port size(FIXED_SIZE);
9347   format %{ "test_and_branch_loop_end,$boolnode  $src1,$src2,$labl\t # counted loop end SHORT" %}
9348   opcode(CIJ_ZOPC);
9349   ins_encode(z_enc_cmpb_regimm(src1, src2, labl, boolnode));
9350   ins_pipe(pipe_class_dummy);
9351   ins_short_branch(1);
9352 %}
9353 
9354 // INT REG/IMM operands
9355 instruct cmpb_RegI_imm(cmpOpT boolnode, iRegI src1, immI8 src2, label labl, flagsReg cr) %{
9356   match(If boolnode (CmpI src1 src2));
9357   effect(USE labl, KILL cr);
9358   predicate(VM_Version::has_CompareBranch());
9359   ins_cost(BRANCH_COST);
9360   // TODO: s390 port size(FIXED_SIZE);
9361   format %{ "CIJ,$boolnode  $src1,$src2,$labl\t # SHORT" %}
9362   opcode(CIJ_ZOPC);
9363   ins_encode(z_enc_cmpb_regimm(src1, src2, labl, boolnode));
9364   ins_pipe(pipe_class_dummy);
9365   ins_short_branch(1);
9366 %}
9367 
9368 // INT REG/IMM operands
9369 instruct cmpbU_RegI_imm(cmpOpT boolnode, iRegI src1, uimmI8 src2, label labl, flagsReg cr) %{
9370   match(If boolnode (CmpU src1 src2));
9371   effect(USE labl, KILL cr);
9372   predicate(VM_Version::has_CompareBranch());
9373   ins_cost(BRANCH_COST);
9374   // TODO: s390 port size(FIXED_SIZE);
9375   format %{ "CLIJ,$boolnode $src1,$src2,$labl\t # SHORT" %}
9376   opcode(CLIJ_ZOPC);
9377   ins_encode(z_enc_cmpb_regimm(src1, src2, labl, boolnode));
9378   ins_pipe(pipe_class_dummy);
9379   ins_short_branch(1);
9380 %}
9381 
9382 // LONG REG/IMM operands
9383 instruct cmpb_RegL_imm(cmpOpT boolnode, iRegL src1, immL8 src2, label labl, flagsReg cr) %{
9384   match(If boolnode (CmpL src1 src2));
9385   effect(USE labl, KILL cr);
9386   predicate(VM_Version::has_CompareBranch());
9387   ins_cost(BRANCH_COST);
9388   // TODO: s390 port size(FIXED_SIZE);
9389   format %{ "CGIJ,$boolnode $src1,$src2,$labl\t # SHORT" %}
9390   opcode(CGIJ_ZOPC);
9391   ins_encode(z_enc_cmpb_regimm(src1, src2, labl, boolnode));
9392   ins_pipe(pipe_class_dummy);
9393   ins_short_branch(1);
9394 %}
9395 
9396 // PTR REG-imm operands
9397 
9398 // Separate rules for regular and narrow oops. ADLC can't recognize
9399 // rules with polymorphic operands to be sisters -> shorten_branches
9400 // will not shorten.
9401 
9402 instruct cmpb_RegP_immP(cmpOpT boolnode, iRegP src1, immP8 src2, label labl, flagsReg cr) %{
9403   match(If boolnode (CmpP src1 src2));
9404   effect(USE labl, KILL cr);
9405   predicate(VM_Version::has_CompareBranch());
9406   ins_cost(BRANCH_COST);
9407   // TODO: s390 port size(FIXED_SIZE);
9408   format %{ "CLGIJ,$boolnode $src1,$src2,$labl\t # SHORT" %}
9409   opcode(CLGIJ_ZOPC);
9410   ins_encode(z_enc_cmpb_regimm(src1, src2, labl, boolnode));
9411   ins_pipe(pipe_class_dummy);
9412   ins_short_branch(1);
9413 %}
9414 
9415 // Compare against zero only, do not mix N and P oops (encode/decode required).
9416 instruct cmpb_RegN_immP0(cmpOpT boolnode, iRegN src1, immP0 src2, label labl, flagsReg cr) %{
9417   match(If boolnode (CmpP (DecodeN src1) src2));
9418   effect(USE labl, KILL cr);
9419   predicate(VM_Version::has_CompareBranch());
9420   ins_cost(BRANCH_COST);
9421   // TODO: s390 port size(FIXED_SIZE);
9422   format %{ "CLGIJ,$boolnode $src1,$src2,$labl\t # SHORT" %}
9423   opcode(CLGIJ_ZOPC);
9424   ins_encode(z_enc_cmpb_regimm(src1, src2, labl, boolnode));
9425   ins_pipe(pipe_class_dummy);
9426   ins_short_branch(1);
9427 %}
9428 
9429 instruct cmpb_RegN_imm(cmpOpT boolnode, iRegN src1, immN8 src2, label labl, flagsReg cr) %{
9430   match(If boolnode (CmpP (DecodeN src1) (DecodeN src2)));
9431   effect(USE labl, KILL cr);
9432   predicate(VM_Version::has_CompareBranch());
9433   ins_cost(BRANCH_COST);
9434   // TODO: s390 port size(FIXED_SIZE);
9435   format %{ "CLGIJ,$boolnode $src1,$src2,$labl\t # SHORT" %}
9436   opcode(CLGIJ_ZOPC);
9437   ins_encode(z_enc_cmpb_regimm(src1, src2, labl, boolnode));
9438   ins_pipe(pipe_class_dummy);
9439   ins_short_branch(1);
9440 %}
9441 
9442 
9443 //----------Compare and Branch (far distance)------------------------------
9444 
9445 // INT REG operands for loop counter processing
9446 instruct testAndBranchLoopEnd_RegFar(cmpOpT boolnode, iRegI src1, iRegI src2, label labl, flagsReg cr) %{
9447   match(CountedLoopEnd boolnode (CmpI src1 src2));
9448   effect(USE labl, KILL cr);
9449   predicate(VM_Version::has_CompareBranch());
9450   ins_cost(BRANCH_COST+DEFAULT_COST);
9451   // TODO: s390 port size(FIXED_SIZE);
9452   format %{ "test_and_branch_loop_end,$boolnode  $src1,$src2,$labl\t # counted loop end FAR" %}
9453   opcode(CR_ZOPC, BRCL_ZOPC);
9454   ins_encode(z_enc_cmpb_regregFar(src1, src2, labl, boolnode));
9455   ins_pipe(pipe_class_dummy);
9456   ins_short_branch(0);
9457 %}
9458 
9459 // INT REG operands
9460 instruct cmpb_RegI_Far(cmpOpT boolnode, iRegI src1, iRegI src2, label labl, flagsReg cr) %{
9461   match(If boolnode (CmpI src1 src2));
9462   effect(USE labl, KILL cr);
9463   predicate(VM_Version::has_CompareBranch());
9464   ins_cost(BRANCH_COST+DEFAULT_COST);
9465   // TODO: s390 port size(FIXED_SIZE);
9466   format %{ "CRJ,$boolnode   $src1,$src2,$labl\t # FAR(substituted)" %}
9467   opcode(CR_ZOPC, BRCL_ZOPC);
9468   ins_encode(z_enc_cmpb_regregFar(src1, src2, labl, boolnode));
9469   ins_pipe(pipe_class_dummy);
9470   ins_short_branch(0);
9471 %}
9472 
9473 // INT REG operands
9474 instruct cmpbU_RegI_Far(cmpOpT boolnode, iRegI src1, iRegI src2, label labl, flagsReg cr) %{
9475   match(If boolnode (CmpU src1 src2));
9476   effect(USE labl, KILL cr);
9477   predicate(VM_Version::has_CompareBranch());
9478   ins_cost(BRANCH_COST+DEFAULT_COST);
9479   // TODO: s390 port size(FIXED_SIZE);
9480   format %{ "CLRJ,$boolnode   $src1,$src2,$labl\t # FAR(substituted)" %}
9481   opcode(CLR_ZOPC, BRCL_ZOPC);
9482   ins_encode(z_enc_cmpb_regregFar(src1, src2, labl, boolnode));
9483   ins_pipe(pipe_class_dummy);
9484   ins_short_branch(0);
9485 %}
9486 
9487 // LONG REG operands
9488 instruct cmpb_RegL_Far(cmpOpT boolnode, iRegL src1, iRegL src2, label labl, flagsReg cr) %{
9489   match(If boolnode (CmpL src1 src2));
9490   effect(USE labl, KILL cr);
9491   predicate(VM_Version::has_CompareBranch());
9492   ins_cost(BRANCH_COST+DEFAULT_COST);
9493   // TODO: s390 port size(FIXED_SIZE);
9494   format %{ "CGRJ,$boolnode   $src1,$src2,$labl\t # FAR(substituted)" %}
9495   opcode(CGR_ZOPC, BRCL_ZOPC);
9496   ins_encode(z_enc_cmpb_regregFar(src1, src2, labl, boolnode));
9497   ins_pipe(pipe_class_dummy);
9498   ins_short_branch(0);
9499 %}
9500 
9501 // PTR REG operands
9502 
9503 // Separate rules for regular and narrow oops. ADLC can't recognize
9504 // rules with polymorphic operands to be sisters -> shorten_branches
9505 // will not shorten.
9506 
9507 instruct cmpb_RegPP_Far(cmpOpT boolnode, iRegP src1, iRegP src2, label labl, flagsReg cr) %{
9508   match(If boolnode (CmpP src1 src2));
9509   effect(USE labl, KILL cr);
9510   predicate(VM_Version::has_CompareBranch());
9511   ins_cost(BRANCH_COST+DEFAULT_COST);
9512   // TODO: s390 port size(FIXED_SIZE);
9513   format %{ "CLGRJ,$boolnode   $src1,$src2,$labl\t # FAR(substituted)" %}
9514   opcode(CLGR_ZOPC, BRCL_ZOPC);
9515   ins_encode(z_enc_cmpb_regregFar(src1, src2, labl, boolnode));
9516   ins_pipe(pipe_class_dummy);
9517   ins_short_branch(0);
9518 %}
9519 
9520 instruct cmpb_RegNN_Far(cmpOpT boolnode, iRegN src1, iRegN src2, label labl, flagsReg cr) %{
9521   match(If boolnode (CmpP (DecodeN src1) (DecodeN src2)));
9522   effect(USE labl, KILL cr);
9523   predicate(VM_Version::has_CompareBranch());
9524   ins_cost(BRANCH_COST+DEFAULT_COST);
9525   // TODO: s390 port size(FIXED_SIZE);
9526   format %{ "CLGRJ,$boolnode   $src1,$src2,$labl\t # FAR(substituted)" %}
9527   opcode(CLGR_ZOPC, BRCL_ZOPC);
9528   ins_encode(z_enc_cmpb_regregFar(src1, src2, labl, boolnode));
9529   ins_pipe(pipe_class_dummy);
9530   ins_short_branch(0);
9531 %}
9532 
9533 // INT REG/IMM operands for loop counter processing
9534 instruct testAndBranchLoopEnd_ImmFar(cmpOpT boolnode, iRegI src1, immI8 src2, label labl, flagsReg cr) %{
9535   match(CountedLoopEnd boolnode (CmpI src1 src2));
9536   effect(USE labl, KILL cr);
9537   predicate(VM_Version::has_CompareBranch());
9538   ins_cost(BRANCH_COST+DEFAULT_COST);
9539   // TODO: s390 port size(FIXED_SIZE);
9540   format %{ "test_and_branch_loop_end,$boolnode  $src1,$src2,$labl\t # counted loop end FAR" %}
9541   opcode(CHI_ZOPC, BRCL_ZOPC);
9542   ins_encode(z_enc_cmpb_regimmFar(src1, src2, labl, boolnode));
9543   ins_pipe(pipe_class_dummy);
9544   ins_short_branch(0);
9545 %}
9546 
9547 // INT REG/IMM operands
9548 instruct cmpb_RegI_imm_Far(cmpOpT boolnode, iRegI src1, immI8 src2, label labl, flagsReg cr) %{
9549   match(If boolnode (CmpI src1 src2));
9550   effect(USE labl, KILL cr);
9551   predicate(VM_Version::has_CompareBranch());
9552   ins_cost(BRANCH_COST+DEFAULT_COST);
9553   // TODO: s390 port size(FIXED_SIZE);
9554   format %{ "CIJ,$boolnode   $src1,$src2,$labl\t # FAR(substituted)" %}
9555   opcode(CHI_ZOPC, BRCL_ZOPC);
9556   ins_encode(z_enc_cmpb_regimmFar(src1, src2, labl, boolnode));
9557   ins_pipe(pipe_class_dummy);
9558   ins_short_branch(0);
9559 %}
9560 
9561 // INT REG/IMM operands
9562 instruct cmpbU_RegI_imm_Far(cmpOpT boolnode, iRegI src1, uimmI8 src2, label labl, flagsReg cr) %{
9563   match(If boolnode (CmpU src1 src2));
9564   effect(USE labl, KILL cr);
9565   predicate(VM_Version::has_CompareBranch());
9566   ins_cost(BRANCH_COST+DEFAULT_COST);
9567   // TODO: s390 port size(FIXED_SIZE);
9568   format %{ "CLIJ,$boolnode   $src1,$src2,$labl\t # FAR(substituted)" %}
9569   opcode(CLFI_ZOPC, BRCL_ZOPC);
9570   ins_encode(z_enc_cmpb_regimmFar(src1, src2, labl, boolnode));
9571   ins_pipe(pipe_class_dummy);
9572   ins_short_branch(0);
9573 %}
9574 
9575 // LONG REG/IMM operands
9576 instruct cmpb_RegL_imm_Far(cmpOpT boolnode, iRegL src1, immL8 src2, label labl, flagsReg cr) %{
9577   match(If boolnode (CmpL src1 src2));
9578   effect(USE labl, KILL cr);
9579   predicate(VM_Version::has_CompareBranch());
9580   ins_cost(BRANCH_COST+DEFAULT_COST);
9581   // TODO: s390 port size(FIXED_SIZE);
9582   format %{ "CGIJ,$boolnode   $src1,$src2,$labl\t # FAR(substituted)" %}
9583   opcode(CGHI_ZOPC, BRCL_ZOPC);
9584   ins_encode(z_enc_cmpb_regimmFar(src1, src2, labl, boolnode));
9585   ins_pipe(pipe_class_dummy);
9586   ins_short_branch(0);
9587 %}
9588 
9589 // PTR REG-imm operands
9590 
9591 // Separate rules for regular and narrow oops. ADLC can't recognize
9592 // rules with polymorphic operands to be sisters -> shorten_branches
9593 // will not shorten.
9594 
9595 instruct cmpb_RegP_immP_Far(cmpOpT boolnode, iRegP src1, immP8 src2, label labl, flagsReg cr) %{
9596   match(If boolnode (CmpP src1 src2));
9597   effect(USE labl, KILL cr);
9598   predicate(VM_Version::has_CompareBranch());
9599   ins_cost(BRANCH_COST+DEFAULT_COST);
9600   // TODO: s390 port size(FIXED_SIZE);
9601   format %{ "CLGIJ,$boolnode   $src1,$src2,$labl\t # FAR(substituted)" %}
9602   opcode(CLGFI_ZOPC, BRCL_ZOPC);
9603   ins_encode(z_enc_cmpb_regimmFar(src1, src2, labl, boolnode));
9604   ins_pipe(pipe_class_dummy);
9605   ins_short_branch(0);
9606 %}
9607 
9608 // Compare against zero only, do not mix N and P oops (encode/decode required).
9609 instruct cmpb_RegN_immP0_Far(cmpOpT boolnode, iRegN src1, immP0 src2, label labl, flagsReg cr) %{
9610   match(If boolnode (CmpP (DecodeN src1) src2));
9611   effect(USE labl, KILL cr);
9612   predicate(VM_Version::has_CompareBranch());
9613   ins_cost(BRANCH_COST+DEFAULT_COST);
9614   // TODO: s390 port size(FIXED_SIZE);
9615   format %{ "CLGIJ,$boolnode   $src1,$src2,$labl\t # FAR(substituted)" %}
9616   opcode(CLGFI_ZOPC, BRCL_ZOPC);
9617   ins_encode(z_enc_cmpb_regimmFar(src1, src2, labl, boolnode));
9618   ins_pipe(pipe_class_dummy);
9619   ins_short_branch(0);
9620 %}
9621 
9622 instruct cmpb_RegN_immN_Far(cmpOpT boolnode, iRegN src1, immN8 src2, label labl, flagsReg cr) %{
9623   match(If boolnode (CmpP (DecodeN src1) (DecodeN src2)));
9624   effect(USE labl, KILL cr);
9625   predicate(VM_Version::has_CompareBranch());
9626   ins_cost(BRANCH_COST+DEFAULT_COST);
9627   // TODO: s390 port size(FIXED_SIZE);
9628   format %{ "CLGIJ,$boolnode   $src1,$src2,$labl\t # FAR(substituted)" %}
9629   opcode(CLGFI_ZOPC, BRCL_ZOPC);
9630   ins_encode(z_enc_cmpb_regimmFar(src1, src2, labl, boolnode));
9631   ins_pipe(pipe_class_dummy);
9632   ins_short_branch(0);
9633 %}
9634 
9635 // ============================================================================
9636 // Long Compare
9637 
9638 // Due to a shortcoming in the ADLC, it mixes up expressions like:
9639 // (foo (CmpI (CmpL X Y) 0)) and (bar (CmpI (CmpL X 0L) 0)). Note the
9640 // difference between 'Y' and '0L'. The tree-matches for the CmpI sections
9641 // are collapsed internally in the ADLC's dfa-gen code. The match for
9642 // (CmpI (CmpL X Y) 0) is silently replaced with (CmpI (CmpL X 0L) 0) and the
9643 // foo match ends up with the wrong leaf. One fix is to not match both
9644 // reg-reg and reg-zero forms of long-compare. This is unfortunate because
9645 // both forms beat the trinary form of long-compare and both are very useful
9646 // on platforms which have few registers.
9647 
9648 // Manifest a CmpL3 result in an integer register. Very painful.
9649 // This is the test to avoid.
9650 instruct cmpL3_reg_reg(iRegI dst, iRegL src1, iRegL src2, flagsReg cr) %{
9651   match(Set dst (CmpL3 src1 src2));
9652   effect(KILL cr);
9653   ins_cost(DEFAULT_COST * 5 + BRANCH_COST);
9654   size(24);
9655   format %{ "CmpL3 $dst,$src1,$src2" %}
9656   ins_encode %{
9657     Label done;
9658     // compare registers
9659     __ z_cgr($src1$$Register, $src2$$Register);
9660     // Convert condition code into -1,0,1, where
9661     // -1 means less
9662     //  0 means equal
9663     //  1 means greater.
9664     if (VM_Version::has_LoadStoreConditional()) {
9665       Register one       = Z_R0_scratch;
9666       Register minus_one = Z_R1_scratch;
9667       __ z_lghi(minus_one, -1);
9668       __ z_lghi(one, 1);
9669       __ z_lghi( $dst$$Register, 0);
9670       __ z_locgr($dst$$Register, one,       Assembler::bcondHigh);
9671       __ z_locgr($dst$$Register, minus_one, Assembler::bcondLow);
9672     } else {
9673       __ clear_reg($dst$$Register, true, false);
9674       __ z_bre(done);
9675       __ z_lhi($dst$$Register, 1);
9676       __ z_brh(done);
9677       __ z_lhi($dst$$Register, -1);
9678     }
9679     __ bind(done);
9680   %}
9681   ins_pipe(pipe_class_dummy);
9682 %}
9683 
9684 // ============================================================================
9685 // Safepoint Instruction
9686 
9687 instruct safePoint() %{
9688   match(SafePoint);
9689   predicate(false);
9690   // TODO: s390 port size(FIXED_SIZE);
9691   format %{ "UNIMPLEMENTED Safepoint_ " %}
9692   ins_encode(enc_unimplemented());
9693   ins_pipe(pipe_class_dummy);
9694 %}
9695 
9696 instruct safePoint_poll(iRegP poll, flagsReg cr) %{
9697   match(SafePoint poll);
9698   effect(USE poll, KILL cr); // R0 is killed, too.
9699   // TODO: s390 port size(FIXED_SIZE);
9700   format %{ "TM      #0[,$poll],#111\t # Safepoint: poll for GC" %}
9701   ins_encode %{
9702     // Mark the code position where the load from the safepoint
9703     // polling page was emitted as relocInfo::poll_type.
9704     __ relocate(relocInfo::poll_type);
9705     __ load_from_polling_page($poll$$Register);
9706   %}
9707   ins_pipe(pipe_class_dummy);
9708 %}
9709 
9710 // ============================================================================
9711 
9712 // Call Instructions
9713 
9714 // Call Java Static Instruction
9715 instruct CallStaticJavaDirect_dynTOC(method meth) %{
9716   match(CallStaticJava);
9717   effect(USE meth);
9718   ins_cost(CALL_COST);
9719   // TODO: s390 port size(VARIABLE_SIZE);
9720   format %{ "CALL,static dynTOC $meth; ==> " %}
9721   ins_encode( z_enc_java_static_call(meth) );
9722   ins_pipe(pipe_class_dummy);
9723   ins_alignment(2);
9724 %}
9725 
9726 // Call Java Dynamic Instruction
9727 instruct CallDynamicJavaDirect_dynTOC(method meth) %{
9728   match(CallDynamicJava);
9729   effect(USE meth);
9730   ins_cost(CALL_COST);
9731   // TODO: s390 port size(VARIABLE_SIZE);
9732   format %{ "CALL,dynamic dynTOC $meth; ==> " %}
9733   ins_encode(z_enc_java_dynamic_call(meth));
9734   ins_pipe(pipe_class_dummy);
9735   ins_alignment(2);
9736 %}
9737 
9738 // Call Runtime Instruction
9739 instruct CallRuntimeDirect(method meth) %{
9740   match(CallRuntime);
9741   effect(USE meth);
9742   ins_cost(CALL_COST);
9743   // TODO: s390 port size(VARIABLE_SIZE);
9744   ins_num_consts(1);
9745   ins_alignment(2);
9746   format %{ "CALL,runtime" %}
9747   ins_encode( z_enc_java_to_runtime_call(meth) );
9748   ins_pipe(pipe_class_dummy);
9749 %}
9750 
9751 // Call runtime without safepoint - same as CallRuntime
9752 instruct CallLeafDirect(method meth) %{
9753   match(CallLeaf);
9754   effect(USE meth);
9755   ins_cost(CALL_COST);
9756   // TODO: s390 port size(VARIABLE_SIZE);
9757   ins_num_consts(1);
9758   ins_alignment(2);
9759   format %{ "CALL,runtime leaf $meth" %}
9760   ins_encode( z_enc_java_to_runtime_call(meth) );
9761   ins_pipe(pipe_class_dummy);
9762 %}
9763 
9764 // Call runtime without safepoint - same as CallLeaf
9765 instruct CallLeafNoFPDirect(method meth) %{
9766   match(CallLeafNoFP);
9767   effect(USE meth);
9768   ins_cost(CALL_COST);
9769   // TODO: s390 port size(VARIABLE_SIZE);
9770   ins_num_consts(1);
9771   format %{ "CALL,runtime leaf nofp $meth" %}
9772   ins_encode( z_enc_java_to_runtime_call(meth) );
9773   ins_pipe(pipe_class_dummy);
9774   ins_alignment(2);
9775 %}
9776 
9777 // Tail Call; Jump from runtime stub to Java code.
9778 // Also known as an 'interprocedural jump'.
9779 // Target of jump will eventually return to caller.
9780 // TailJump below removes the return address.
9781 instruct TailCalljmpInd(iRegP jump_target, inline_cache_regP method_oop) %{
9782   match(TailCall jump_target method_oop);
9783   ins_cost(CALL_COST);
9784   size(2);
9785   format %{ "Jmp     $jump_target\t# $method_oop holds method oop" %}
9786   ins_encode %{ __ z_br($jump_target$$Register); %}
9787   ins_pipe(pipe_class_dummy);
9788 %}
9789 
9790 // Return Instruction
9791 instruct Ret() %{
9792   match(Return);
9793   size(2);
9794   format %{ "BR(Z_R14) // branch to link register" %}
9795   ins_encode %{ __ z_br(Z_R14); %}
9796   ins_pipe(pipe_class_dummy);
9797 %}
9798 
9799 // Tail Jump; remove the return address; jump to target.
9800 // TailCall above leaves the return address around.
9801 // TailJump is used in only one place, the rethrow_Java stub (fancy_jump=2).
9802 // ex_oop (Exception Oop) is needed in %o0 at the jump. As there would be a
9803 // "restore" before this instruction (in Epilogue), we need to materialize it
9804 // in %i0.
9805 instruct tailjmpInd(iRegP jump_target, rarg1RegP ex_oop) %{
9806   match(TailJump jump_target ex_oop);
9807   ins_cost(CALL_COST);
9808   size(8);
9809   format %{ "TailJump $jump_target" %}
9810   ins_encode %{
9811     __ z_lg(Z_ARG2/* issuing pc */, _z_abi(return_pc), Z_SP);
9812     __ z_br($jump_target$$Register);
9813   %}
9814   ins_pipe(pipe_class_dummy);
9815 %}
9816 
9817 // Create exception oop: created by stack-crawling runtime code.
9818 // Created exception is now available to this handler, and is setup
9819 // just prior to jumping to this handler. No code emitted.
9820 instruct CreateException(rarg1RegP ex_oop) %{
9821   match(Set ex_oop (CreateEx));
9822   ins_cost(0);
9823   size(0);
9824   format %{ "# exception oop; no code emitted" %}
9825   ins_encode(/*empty*/);
9826   ins_pipe(pipe_class_dummy);
9827 %}
9828 
9829 // Rethrow exception: The exception oop will come in the first
9830 // argument position. Then JUMP (not call) to the rethrow stub code.
9831 instruct RethrowException() %{
9832   match(Rethrow);
9833   ins_cost(CALL_COST);
9834   // TODO: s390 port size(VARIABLE_SIZE);
9835   format %{ "Jmp    rethrow_stub" %}
9836   ins_encode %{
9837     cbuf.set_insts_mark();
9838     __ load_const_optimized(Z_R1_scratch, (address)OptoRuntime::rethrow_stub());
9839     __ z_br(Z_R1_scratch);
9840   %}
9841   ins_pipe(pipe_class_dummy);
9842 %}
9843 
9844 // Die now.
9845 instruct ShouldNotReachHere() %{
9846   match(Halt);
9847   ins_cost(CALL_COST);
9848   size(2);
9849   format %{ "ILLTRAP; ShouldNotReachHere" %}
9850   ins_encode %{ __ z_illtrap(); %}
9851   ins_pipe(pipe_class_dummy);
9852 %}
9853 
9854 // ============================================================================
9855 // The 2nd slow-half of a subtype check. Scan the subklass's 2ndary superklass
9856 // array for an instance of the superklass. Set a hidden internal cache on a
9857 // hit (cache is checked with exposed code in gen_subtype_check()). Return
9858 // not zero for a miss or zero for a hit. The encoding ALSO sets flags.
9859 instruct partialSubtypeCheck(rarg1RegP index, rarg2RegP sub, rarg3RegP super, flagsReg pcc,
9860                              rarg4RegP scratch1, rarg5RegP scratch2) %{
9861   match(Set index (PartialSubtypeCheck sub super));
9862   effect(KILL pcc, KILL scratch1, KILL scratch2);
9863   ins_cost(10 * DEFAULT_COST);
9864   // TODO: s390 port size(FIXED_SIZE);
9865   format %{ "  CALL   PartialSubtypeCheck\n" %}
9866   ins_encode %{
9867     AddressLiteral stub_address(StubRoutines::zarch::partial_subtype_check());
9868     __ load_const_optimized(Z_ARG4, stub_address);
9869     __ z_basr(Z_R14, Z_ARG4);
9870   %}
9871   ins_pipe(pipe_class_dummy);
9872 %}
9873 
9874 instruct partialSubtypeCheck_vs_zero(flagsReg pcc, rarg2RegP sub, rarg3RegP super, immP0 zero,
9875                                      rarg1RegP index, rarg4RegP scratch1, rarg5RegP scratch2) %{
9876   match(Set pcc (CmpI (PartialSubtypeCheck sub super) zero));
9877   effect(KILL scratch1, KILL scratch2, KILL index);
9878   ins_cost(10 * DEFAULT_COST);
9879   // TODO: s390 port size(FIXED_SIZE);
9880   format %{ "CALL   PartialSubtypeCheck_vs_zero\n" %}
9881   ins_encode %{
9882     AddressLiteral stub_address(StubRoutines::zarch::partial_subtype_check());
9883     __ load_const_optimized(Z_ARG4, stub_address);
9884     __ z_basr(Z_R14, Z_ARG4);
9885   %}
9886   ins_pipe(pipe_class_dummy);
9887 %}
9888 
9889 // ============================================================================
9890 // inlined locking and unlocking
9891 
9892 instruct cmpFastLock(flagsReg pcc, iRegP_N2P oop, iRegP_N2P box, iRegP tmp1, iRegP tmp2) %{
9893   match(Set pcc (FastLock oop box));
9894   effect(TEMP tmp1, TEMP tmp2);
9895   ins_cost(100);
9896   // TODO: s390 port size(VARIABLE_SIZE); // Uses load_const_optimized.
9897   format %{ "FASTLOCK  $oop, $box; KILL Z_ARG4, Z_ARG5" %}
9898   ins_encode %{ __ compiler_fast_lock_object($oop$$Register, $box$$Register, $tmp1$$Register, $tmp2$$Register,
9899                                              UseBiasedLocking && !UseOptoBiasInlining); %}
9900   ins_pipe(pipe_class_dummy);
9901 %}
9902 
9903 instruct cmpFastUnlock(flagsReg pcc, iRegP_N2P oop, iRegP_N2P box, iRegP tmp1, iRegP tmp2) %{
9904   match(Set pcc (FastUnlock oop box));
9905   effect(TEMP tmp1, TEMP tmp2);
9906   ins_cost(100);
9907   // TODO: s390 port size(FIXED_SIZE);  // emitted code depends on UseBiasedLocking being on/off.
9908   format %{ "FASTUNLOCK  $oop, $box; KILL Z_ARG4, Z_ARG5" %}
9909   ins_encode %{ __ compiler_fast_unlock_object($oop$$Register, $box$$Register, $tmp1$$Register, $tmp2$$Register,
9910                                                UseBiasedLocking && !UseOptoBiasInlining); %}
9911   ins_pipe(pipe_class_dummy);
9912 %}
9913 
9914 instruct inlineCallClearArrayConst(SSlenDW cnt, iRegP_N2P base, Universe dummy, flagsReg cr) %{
9915   match(Set dummy (ClearArray cnt base));
9916   effect(KILL cr);
9917   ins_cost(100);
9918   // TODO: s390 port size(VARIABLE_SIZE);       // Variable in size due to varying #instructions.
9919   format %{ "ClearArrayConst $cnt,$base" %}
9920   ins_encode %{ __ Clear_Array_Const($cnt$$constant, $base$$Register); %}
9921   ins_pipe(pipe_class_dummy);
9922 %}
9923 
9924 instruct inlineCallClearArrayConstBig(immL cnt, iRegP_N2P base, Universe dummy, allRoddRegL tmpL, flagsReg cr) %{
9925   match(Set dummy (ClearArray cnt base));
9926   effect(TEMP tmpL, KILL cr); // R0, R1 are killed, too.
9927   ins_cost(200);
9928   // TODO: s390 port size(VARIABLE_SIZE);       // Variable in size due to optimized constant loader.
9929   format %{ "ClearArrayConstBig $cnt,$base" %}
9930   ins_encode %{ __ Clear_Array_Const_Big($cnt$$constant, $base$$Register, $tmpL$$Register); %}
9931   ins_pipe(pipe_class_dummy);
9932 %}
9933 
9934 instruct inlineCallClearArray(iRegL cnt, iRegP_N2P base, Universe dummy, allRoddRegL tmpL, flagsReg cr) %{
9935   match(Set dummy (ClearArray cnt base));
9936   effect(TEMP tmpL, KILL cr); // R0, R1 are killed, too.
9937   ins_cost(300);
9938   // TODO: s390 port size(FIXED_SIZE);  // z/Architecture: emitted code depends on PreferLAoverADD being on/off.
9939   format %{ "ClearArrayVar $cnt,$base" %}
9940   ins_encode %{ __ Clear_Array($cnt$$Register, $base$$Register, $tmpL$$Register); %}
9941   ins_pipe(pipe_class_dummy);
9942 %}
9943 
9944 // ============================================================================
9945 // CompactStrings
9946 
9947 // String equals
9948 instruct string_equalsL(iRegP str1, iRegP str2, iRegI cnt, iRegI result, roddRegL oddReg, revenRegL evenReg, flagsReg cr) %{
9949   match(Set result (StrEquals (Binary str1 str2) cnt));
9950   effect(TEMP oddReg, TEMP evenReg, KILL cr); // R0, R1 are killed, too.
9951   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
9952   ins_cost(300);
9953   format %{ "String Equals byte[] $str1,$str2,$cnt -> $result" %}
9954   ins_encode %{
9955     __ array_equals(false, $str1$$Register, $str2$$Register,
9956                     $cnt$$Register, $oddReg$$Register, $evenReg$$Register,
9957                     $result$$Register, true /* byte */);
9958   %}
9959   ins_pipe(pipe_class_dummy);
9960 %}
9961 
9962 instruct string_equalsU(iRegP str1, iRegP str2, iRegI cnt, iRegI result, roddRegL oddReg, revenRegL evenReg, flagsReg cr) %{
9963   match(Set result (StrEquals (Binary str1 str2) cnt));
9964   effect(TEMP oddReg, TEMP evenReg, KILL cr); // R0, R1 are killed, too.
9965   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU || ((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::none);
9966   ins_cost(300);
9967   format %{ "String Equals char[] $str1,$str2,$cnt -> $result" %}
9968   ins_encode %{
9969     __ array_equals(false, $str1$$Register, $str2$$Register,
9970                     $cnt$$Register, $oddReg$$Register, $evenReg$$Register,
9971                     $result$$Register, false /* byte */);
9972   %}
9973   ins_pipe(pipe_class_dummy);
9974 %}
9975 
9976 instruct string_equals_imm(iRegP str1, iRegP str2, uimmI8 cnt, iRegI result, flagsReg cr) %{
9977   match(Set result (StrEquals (Binary str1 str2) cnt));
9978   effect(KILL cr); // R0 is killed, too.
9979   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL || ((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
9980   ins_cost(100);
9981   format %{ "String Equals byte[] $str1,$str2,$cnt -> $result" %}
9982   ins_encode %{
9983     const int cnt_imm = $cnt$$constant;
9984     if (cnt_imm) { __ z_clc(0, cnt_imm - 1, $str1$$Register, 0, $str2$$Register); }
9985     __ z_lhi($result$$Register, 1);
9986     if (cnt_imm) {
9987       if (VM_Version::has_LoadStoreConditional()) {
9988         __ z_lhi(Z_R0_scratch, 0);
9989         __ z_locr($result$$Register, Z_R0_scratch, Assembler::bcondNotEqual);
9990       } else {
9991         Label Lskip;
9992         __ z_bre(Lskip);
9993         __ clear_reg($result$$Register);
9994         __ bind(Lskip);
9995       }
9996     }
9997   %}
9998   ins_pipe(pipe_class_dummy);
9999 %}
10000 
10001 instruct string_equalsC_imm(iRegP str1, iRegP str2, immI8 cnt, iRegI result, flagsReg cr) %{
10002   match(Set result (StrEquals (Binary str1 str2) cnt));
10003   effect(KILL cr); // R0 is killed, too.
10004   predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::none);
10005   ins_cost(100);
10006   format %{ "String Equals $str1,$str2,$cnt -> $result" %}
10007   ins_encode %{
10008     const int cnt_imm = $cnt$$constant; // positive immI8 (7 bits used)
10009     if (cnt_imm) { __ z_clc(0, (cnt_imm << 1) - 1, $str1$$Register, 0, $str2$$Register); }
10010     __ z_lhi($result$$Register, 1);
10011     if (cnt_imm) {
10012       if (VM_Version::has_LoadStoreConditional()) {
10013         __ z_lhi(Z_R0_scratch, 0);
10014         __ z_locr($result$$Register, Z_R0_scratch, Assembler::bcondNotEqual);
10015       } else {
10016         Label Lskip;
10017         __ z_bre(Lskip);
10018         __ clear_reg($result$$Register);
10019         __ bind(Lskip);
10020       }
10021     }
10022   %}
10023   ins_pipe(pipe_class_dummy);
10024 %}
10025 
10026 // Array equals
10027 instruct array_equalsB(iRegP ary1, iRegP ary2, iRegI result, roddRegL oddReg, revenRegL evenReg, flagsReg cr) %{
10028   match(Set result (AryEq ary1 ary2));
10029   effect(TEMP oddReg, TEMP evenReg, KILL cr); // R0, R1 are killed, too.
10030   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
10031   ins_cost(300);
10032   format %{ "Array Equals $ary1,$ary2 -> $result" %}
10033   ins_encode %{
10034     __ array_equals(true, $ary1$$Register, $ary2$$Register,
10035                     noreg, $oddReg$$Register, $evenReg$$Register,
10036                     $result$$Register, true /* byte */);
10037   %}
10038   ins_pipe(pipe_class_dummy);
10039 %}
10040 
10041 instruct array_equalsC(iRegP ary1, iRegP ary2, iRegI result, roddRegL oddReg, revenRegL evenReg, flagsReg cr) %{
10042   match(Set result (AryEq ary1 ary2));
10043   effect(TEMP oddReg, TEMP evenReg, KILL cr); // R0, R1 are killed, too.
10044   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
10045   ins_cost(300);
10046   format %{ "Array Equals $ary1,$ary2 -> $result" %}
10047   ins_encode %{
10048     __ array_equals(true, $ary1$$Register, $ary2$$Register,
10049                     noreg, $oddReg$$Register, $evenReg$$Register,
10050                     $result$$Register, false /* byte */);
10051   %}
10052   ins_pipe(pipe_class_dummy);
10053 %}
10054 
10055 // String CompareTo
10056 instruct string_compareL(iRegP str1, iRegP str2, rarg2RegI cnt1, rarg5RegI cnt2, iRegI result, roddRegL oddReg, revenRegL evenReg, flagsReg cr) %{
10057   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10058   effect(TEMP_DEF result, USE_KILL cnt1, USE_KILL cnt2, TEMP oddReg, TEMP evenReg, KILL cr); // R0, R1 are killed, too.
10059   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
10060   ins_cost(300);
10061   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result" %}
10062   ins_encode %{
10063     __ string_compare($str1$$Register, $str2$$Register,
10064                       $cnt1$$Register, $cnt2$$Register,
10065                       $oddReg$$Register, $evenReg$$Register,
10066                       $result$$Register, StrIntrinsicNode::LL);
10067   %}
10068   ins_pipe(pipe_class_dummy);
10069 %}
10070 
10071 instruct string_compareU(iRegP str1, iRegP str2, rarg2RegI cnt1, rarg5RegI cnt2, iRegI result, roddRegL oddReg, revenRegL evenReg, flagsReg cr) %{
10072   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10073   effect(TEMP_DEF result, USE_KILL cnt1, USE_KILL cnt2, TEMP oddReg, TEMP evenReg, KILL cr); // R0, R1 are killed, too.
10074   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU || ((StrCompNode*)n)->encoding() == StrIntrinsicNode::none);
10075   ins_cost(300);
10076   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result" %}
10077   ins_encode %{
10078     __ string_compare($str1$$Register, $str2$$Register,
10079                       $cnt1$$Register, $cnt2$$Register,
10080                       $oddReg$$Register, $evenReg$$Register,
10081                       $result$$Register, StrIntrinsicNode::UU);
10082   %}
10083   ins_pipe(pipe_class_dummy);
10084 %}
10085 
10086 instruct string_compareLU(iRegP str1, iRegP str2, rarg2RegI cnt1, rarg5RegI cnt2, iRegI result, roddRegL oddReg, revenRegL evenReg, flagsReg cr) %{
10087   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10088   effect(TEMP_DEF result, USE_KILL cnt1, USE_KILL cnt2, TEMP oddReg, TEMP evenReg, KILL cr); // R0, R1 are killed, too.
10089   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
10090   ins_cost(300);
10091   format %{ "String Compare byte[],char[] $str1,$cnt1,$str2,$cnt2 -> $result" %}
10092   ins_encode %{
10093     __ string_compare($str1$$Register, $str2$$Register,
10094                       $cnt1$$Register, $cnt2$$Register,
10095                       $oddReg$$Register, $evenReg$$Register,
10096                       $result$$Register, StrIntrinsicNode::LU);
10097   %}
10098   ins_pipe(pipe_class_dummy);
10099 %}
10100 
10101 instruct string_compareUL(iRegP str1, iRegP str2, rarg2RegI cnt1, rarg5RegI cnt2, iRegI result, roddRegL oddReg, revenRegL evenReg, flagsReg cr) %{
10102   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10103   effect(TEMP_DEF result, USE_KILL cnt1, USE_KILL cnt2, TEMP oddReg, TEMP evenReg, KILL cr); // R0, R1 are killed, too.
10104   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
10105   ins_cost(300);
10106   format %{ "String Compare char[],byte[] $str1,$cnt1,$str2,$cnt2 -> $result" %}
10107   ins_encode %{
10108     __ string_compare($str2$$Register, $str1$$Register,
10109                       $cnt2$$Register, $cnt1$$Register,
10110                       $oddReg$$Register, $evenReg$$Register,
10111                       $result$$Register, StrIntrinsicNode::UL);
10112   %}
10113   ins_pipe(pipe_class_dummy);
10114 %}
10115 
10116 // String IndexOfChar
10117 instruct indexOfChar_U(iRegP haystack, iRegI haycnt, iRegI ch, iRegI result, roddRegL oddReg, revenRegL evenReg, flagsReg cr) %{
10118   match(Set result (StrIndexOfChar (Binary haystack haycnt) ch));
10119   effect(TEMP_DEF result, TEMP evenReg, TEMP oddReg, KILL cr); // R0, R1 are killed, too.
10120   ins_cost(200);
10121   format %{ "String IndexOfChar [0..$haycnt]($haystack), $ch -> $result" %}
10122   ins_encode %{
10123     __ string_indexof_char($result$$Register,
10124                            $haystack$$Register, $haycnt$$Register,
10125                            $ch$$Register, 0 /* unused, ch is in register */,
10126                            $oddReg$$Register, $evenReg$$Register, false /*is_byte*/);
10127   %}
10128   ins_pipe(pipe_class_dummy);
10129 %}
10130 
10131 instruct indexOf_imm1_U(iRegP haystack, iRegI haycnt, immP needle, immI_1 needlecnt, iRegI result, roddRegL oddReg, revenRegL evenReg, flagsReg cr) %{
10132   match(Set result (StrIndexOf (Binary haystack haycnt) (Binary needle needlecnt)));
10133   effect(TEMP_DEF result, TEMP evenReg, TEMP oddReg, KILL cr); // R0, R1 are killed, too.
10134   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU || ((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::none);
10135   ins_cost(200);
10136   format %{ "String IndexOf UL [0..$haycnt]($haystack), [0]($needle) -> $result" %}
10137   ins_encode %{
10138     immPOper *needleOper = (immPOper *)$needle;
10139     const TypeOopPtr *t = needleOper->type()->isa_oopptr();
10140     ciTypeArray* needle_values = t->const_oop()->as_type_array();  // Pointer to live char *
10141     jchar chr;
10142 #ifdef VM_LITTLE_ENDIAN
10143     Unimplemented();
10144 #else
10145     chr = (((jchar)(unsigned char)needle_values->element_value(0).as_byte()) << 8) |
10146            ((jchar)(unsigned char)needle_values->element_value(1).as_byte());
10147 #endif
10148     __ string_indexof_char($result$$Register,
10149                            $haystack$$Register, $haycnt$$Register,
10150                            noreg, chr,
10151                            $oddReg$$Register, $evenReg$$Register, false /*is_byte*/);
10152   %}
10153   ins_pipe(pipe_class_dummy);
10154 %}
10155 
10156 instruct indexOf_imm1_L(iRegP haystack, iRegI haycnt, immP needle, immI_1 needlecnt, iRegI result, roddRegL oddReg, revenRegL evenReg, flagsReg cr) %{
10157   match(Set result (StrIndexOf (Binary haystack haycnt) (Binary needle needlecnt)));
10158   effect(TEMP_DEF result, TEMP evenReg, TEMP oddReg, KILL cr); // R0, R1 are killed, too.
10159   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
10160   ins_cost(200);
10161   format %{ "String IndexOf L [0..$haycnt]($haystack), [0]($needle) -> $result" %}
10162   ins_encode %{
10163     immPOper *needleOper = (immPOper *)$needle;
10164     const TypeOopPtr *t = needleOper->type()->isa_oopptr();
10165     ciTypeArray* needle_values = t->const_oop()->as_type_array();  // Pointer to live char *
10166     jchar chr = (jchar)needle_values->element_value(0).as_byte();
10167     __ string_indexof_char($result$$Register,
10168                            $haystack$$Register, $haycnt$$Register,
10169                            noreg, chr,
10170                            $oddReg$$Register, $evenReg$$Register, true /*is_byte*/);
10171   %}
10172   ins_pipe(pipe_class_dummy);
10173 %}
10174 
10175 instruct indexOf_imm1_UL(iRegP haystack, iRegI haycnt, immP needle, immI_1 needlecnt, iRegI result, roddRegL oddReg, revenRegL evenReg, flagsReg cr) %{
10176   match(Set result (StrIndexOf (Binary haystack haycnt) (Binary needle needlecnt)));
10177   effect(TEMP_DEF result, TEMP evenReg, TEMP oddReg, KILL cr); // R0, R1 are killed, too.
10178   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
10179   ins_cost(200);
10180   format %{ "String IndexOf UL [0..$haycnt]($haystack), [0]($needle) -> $result" %}
10181   ins_encode %{
10182     immPOper *needleOper = (immPOper *)$needle;
10183     const TypeOopPtr *t = needleOper->type()->isa_oopptr();
10184     ciTypeArray* needle_values = t->const_oop()->as_type_array();  // Pointer to live char *
10185     jchar chr = (jchar)needle_values->element_value(0).as_byte();
10186     __ string_indexof_char($result$$Register,
10187                            $haystack$$Register, $haycnt$$Register,
10188                            noreg, chr,
10189                            $oddReg$$Register, $evenReg$$Register, false /*is_byte*/);
10190   %}
10191   ins_pipe(pipe_class_dummy);
10192 %}
10193 
10194 // String IndexOf
10195 instruct indexOf_imm_U(iRegP haystack, rarg2RegI haycnt, iRegP needle, immI16 needlecntImm, iRegI result, roddRegL oddReg, revenRegL evenReg, flagsReg cr) %{
10196   match(Set result (StrIndexOf (Binary haystack haycnt) (Binary needle needlecntImm)));
10197   effect(TEMP_DEF result, USE_KILL haycnt, TEMP oddReg, TEMP evenReg, KILL cr); // R0, R1 are killed, too.
10198   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU || ((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::none);
10199   ins_cost(250);
10200   format %{ "String IndexOf U [0..$needlecntImm]($needle) .in. [0..$haycnt]($haystack) -> $result" %}
10201   ins_encode %{
10202     __ string_indexof($result$$Register,
10203                       $haystack$$Register, $haycnt$$Register,
10204                       $needle$$Register, noreg, $needlecntImm$$constant,
10205                       $oddReg$$Register, $evenReg$$Register, StrIntrinsicNode::UU);
10206   %}
10207   ins_pipe(pipe_class_dummy);
10208 %}
10209 
10210 instruct indexOf_imm_L(iRegP haystack, rarg2RegI haycnt, iRegP needle, immI16 needlecntImm, iRegI result, roddRegL oddReg, revenRegL evenReg, flagsReg cr) %{
10211   match(Set result (StrIndexOf (Binary haystack haycnt) (Binary needle needlecntImm)));
10212   effect(TEMP_DEF result, USE_KILL haycnt, TEMP oddReg, TEMP evenReg, KILL cr); // R0, R1 are killed, too.
10213   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
10214   ins_cost(250);
10215   format %{ "String IndexOf L [0..$needlecntImm]($needle) .in. [0..$haycnt]($haystack) -> $result" %}
10216   ins_encode %{
10217     __ string_indexof($result$$Register,
10218                       $haystack$$Register, $haycnt$$Register,
10219                       $needle$$Register, noreg, $needlecntImm$$constant,
10220                       $oddReg$$Register, $evenReg$$Register, StrIntrinsicNode::LL);
10221   %}
10222   ins_pipe(pipe_class_dummy);
10223 %}
10224 
10225 instruct indexOf_imm_UL(iRegP haystack, rarg2RegI haycnt, iRegP needle, immI16 needlecntImm, iRegI result, roddRegL oddReg, revenRegL evenReg, flagsReg cr) %{
10226   match(Set result (StrIndexOf (Binary haystack haycnt) (Binary needle needlecntImm)));
10227   effect(TEMP_DEF result, USE_KILL haycnt, TEMP oddReg, TEMP evenReg, KILL cr); // R0, R1 are killed, too.
10228   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
10229   ins_cost(250);
10230   format %{ "String IndexOf UL [0..$needlecntImm]($needle) .in. [0..$haycnt]($haystack) -> $result" %}
10231   ins_encode %{
10232     __ string_indexof($result$$Register,
10233                       $haystack$$Register, $haycnt$$Register,
10234                       $needle$$Register, noreg, $needlecntImm$$constant,
10235                       $oddReg$$Register, $evenReg$$Register, StrIntrinsicNode::UL);
10236   %}
10237   ins_pipe(pipe_class_dummy);
10238 %}
10239 
10240 instruct indexOf_U(iRegP haystack, rarg2RegI haycnt, iRegP needle, rarg5RegI needlecnt, iRegI result, roddRegL oddReg, revenRegL evenReg, flagsReg cr) %{
10241   match(Set result (StrIndexOf (Binary haystack haycnt) (Binary needle needlecnt)));
10242   effect(TEMP_DEF result, USE_KILL haycnt, USE_KILL needlecnt, TEMP oddReg, TEMP evenReg, KILL cr); // R0, R1 are killed, too.
10243   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU || ((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::none);
10244   ins_cost(300);
10245   format %{ "String IndexOf U [0..$needlecnt]($needle) .in. [0..$haycnt]($haystack) -> $result" %}
10246   ins_encode %{
10247     __ string_indexof($result$$Register,
10248                       $haystack$$Register, $haycnt$$Register,
10249                       $needle$$Register, $needlecnt$$Register, 0,
10250                       $oddReg$$Register, $evenReg$$Register, StrIntrinsicNode::UU);
10251   %}
10252   ins_pipe(pipe_class_dummy);
10253 %}
10254 
10255 instruct indexOf_L(iRegP haystack, rarg2RegI haycnt, iRegP needle, rarg5RegI needlecnt, iRegI result, roddRegL oddReg, revenRegL evenReg, flagsReg cr) %{
10256   match(Set result (StrIndexOf (Binary haystack haycnt) (Binary needle needlecnt)));
10257   effect(TEMP_DEF result, USE_KILL haycnt, USE_KILL needlecnt, TEMP oddReg, TEMP evenReg, KILL cr); // R0, R1 are killed, too.
10258   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
10259   ins_cost(300);
10260   format %{ "String IndexOf L [0..$needlecnt]($needle) .in. [0..$haycnt]($haystack) -> $result" %}
10261   ins_encode %{
10262     __ string_indexof($result$$Register,
10263                       $haystack$$Register, $haycnt$$Register,
10264                       $needle$$Register, $needlecnt$$Register, 0,
10265                       $oddReg$$Register, $evenReg$$Register, StrIntrinsicNode::LL);
10266   %}
10267   ins_pipe(pipe_class_dummy);
10268 %}
10269 
10270 instruct indexOf_UL(iRegP haystack, rarg2RegI haycnt, iRegP needle, rarg5RegI needlecnt, iRegI result, roddRegL oddReg, revenRegL evenReg, flagsReg cr) %{
10271   match(Set result (StrIndexOf (Binary haystack haycnt) (Binary needle needlecnt)));
10272   effect(TEMP_DEF result, USE_KILL haycnt, USE_KILL needlecnt, TEMP oddReg, TEMP evenReg, KILL cr); // R0, R1 are killed, too.
10273   predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
10274   ins_cost(300);
10275   format %{ "String IndexOf UL [0..$needlecnt]($needle) .in. [0..$haycnt]($haystack) -> $result" %}
10276   ins_encode %{
10277     __ string_indexof($result$$Register,
10278                       $haystack$$Register, $haycnt$$Register,
10279                       $needle$$Register, $needlecnt$$Register, 0,
10280                       $oddReg$$Register, $evenReg$$Register, StrIntrinsicNode::UL);
10281   %}
10282   ins_pipe(pipe_class_dummy);
10283 %}
10284 
10285 // char[] to byte[] compression
10286 instruct string_compress(iRegP src, iRegP dst, iRegI result, iRegI len, iRegI tmp, flagsReg cr) %{
10287   match(Set result (StrCompressedCopy src (Binary dst len)));
10288   effect(TEMP_DEF result, TEMP tmp, KILL cr); // R0, R1 are killed, too.
10289   ins_cost(300);
10290   format %{ "String Compress $src->$dst($len) -> $result" %}
10291   ins_encode %{
10292     __ string_compress($result$$Register, $src$$Register, $dst$$Register, $len$$Register,
10293                        $tmp$$Register, false);
10294   %}
10295   ins_pipe(pipe_class_dummy);
10296 %}
10297 
10298 // byte[] to char[] inflation. trot implementation is shorter, but slower than the unrolled icm(h) loop.
10299 //instruct string_inflate_trot(Universe dummy, iRegP src, revenRegP dst, roddRegI len, iRegI tmp, flagsReg cr) %{
10300 //  match(Set dummy (StrInflatedCopy src (Binary dst len)));
10301 //  effect(USE_KILL dst, USE_KILL len, TEMP tmp, KILL cr); // R0, R1 are killed, too.
10302 //  predicate(VM_Version::has_ETF2Enhancements());
10303 //  ins_cost(300);
10304 //  format %{ "String Inflate (trot) $dst,$src($len)" %}
10305 //  ins_encode %{
10306 //    __ string_inflate_trot($src$$Register, $dst$$Register, $len$$Register, $tmp$$Register);
10307 //  %}
10308 //  ins_pipe(pipe_class_dummy);
10309 //%}
10310 
10311 // byte[] to char[] inflation
10312 instruct string_inflate(Universe dummy, iRegP src, iRegP dst, iRegI len, iRegI tmp, flagsReg cr) %{
10313   match(Set dummy (StrInflatedCopy src (Binary dst len)));
10314   effect(TEMP tmp, KILL cr); // R0, R1 are killed, too.
10315   ins_cost(300);
10316   format %{ "String Inflate $src->$dst($len)" %}
10317   ins_encode %{
10318     __ string_inflate($src$$Register, $dst$$Register, $len$$Register, $tmp$$Register);
10319   %}
10320   ins_pipe(pipe_class_dummy);
10321 %}
10322 
10323 // byte[] to char[] inflation
10324 instruct string_inflate_const(Universe dummy, iRegP src, iRegP dst, iRegI tmp, immI len, flagsReg cr) %{
10325   match(Set dummy (StrInflatedCopy src (Binary dst len)));
10326   effect(TEMP tmp, KILL cr); // R0, R1 are killed, too.
10327   ins_cost(300);
10328   format %{ "String Inflate (constLen) $src->$dst($len)" %}
10329   ins_encode %{
10330     __ string_inflate_const($src$$Register, $dst$$Register, $tmp$$Register, $len$$constant);
10331   %}
10332   ins_pipe(pipe_class_dummy);
10333 %}
10334 
10335 // StringCoding.java intrinsics
10336 instruct has_negatives(rarg5RegP ary1, iRegI len, iRegI result, roddRegI oddReg, revenRegI evenReg, iRegI tmp, flagsReg cr) %{
10337   match(Set result (HasNegatives ary1 len));
10338   effect(TEMP_DEF result, USE_KILL ary1, TEMP oddReg, TEMP evenReg, TEMP tmp, KILL cr); // R0, R1 are killed, too.
10339   ins_cost(300);
10340   format %{ "has negatives byte[] $ary1($len) -> $result" %}
10341   ins_encode %{
10342     __ has_negatives($result$$Register, $ary1$$Register, $len$$Register,
10343                      $oddReg$$Register, $evenReg$$Register, $tmp$$Register);
10344   %}
10345   ins_pipe(pipe_class_dummy);
10346 %}
10347 
10348 // encode char[] to byte[] in ISO_8859_1
10349 instruct encode_iso_array(iRegP src, iRegP dst, iRegI result, iRegI len, iRegI tmp, flagsReg cr) %{
10350   match(Set result (EncodeISOArray src (Binary dst len)));
10351   effect(TEMP_DEF result, TEMP tmp, KILL cr); // R0, R1 are killed, too.
10352   ins_cost(300);
10353   format %{ "Encode array $src->$dst($len) -> $result" %}
10354   ins_encode %{
10355     __ string_compress($result$$Register, $src$$Register, $dst$$Register, $len$$Register,
10356                        $tmp$$Register, true);
10357   %}
10358   ins_pipe(pipe_class_dummy);
10359 %}
10360 
10361 
10362 //----------PEEPHOLE RULES-----------------------------------------------------
10363 // These must follow all instruction definitions as they use the names
10364 // defined in the instructions definitions.
10365 //
10366 // peepmatch (root_instr_name [preceeding_instruction]*);
10367 //
10368 // peepconstraint %{
10369 // (instruction_number.operand_name relational_op instruction_number.operand_name
10370 //  [, ...]);
10371 // // instruction numbers are zero-based using left to right order in peepmatch
10372 //
10373 // peepreplace (instr_name([instruction_number.operand_name]*));
10374 // // provide an instruction_number.operand_name for each operand that appears
10375 // // in the replacement instruction's match rule
10376 //
10377 // ---------VM FLAGS---------------------------------------------------------
10378 //
10379 // All peephole optimizations can be turned off using -XX:-OptoPeephole
10380 //
10381 // Each peephole rule is given an identifying number starting with zero and
10382 // increasing by one in the order seen by the parser. An individual peephole
10383 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
10384 // on the command-line.
10385 //
10386 // ---------CURRENT LIMITATIONS----------------------------------------------
10387 //
10388 // Only match adjacent instructions in same basic block
10389 // Only equality constraints
10390 // Only constraints between operands, not (0.dest_reg == EAX_enc)
10391 // Only one replacement instruction
10392 //
10393 // ---------EXAMPLE----------------------------------------------------------
10394 //
10395 // // pertinent parts of existing instructions in architecture description
10396 // instruct movI(eRegI dst, eRegI src) %{
10397 //   match(Set dst (CopyI src));
10398 // %}
10399 //
10400 // instruct incI_eReg(eRegI dst, immI1 src, eFlagsReg cr) %{
10401 //   match(Set dst (AddI dst src));
10402 //   effect(KILL cr);
10403 // %}
10404 //
10405 // // Change (inc mov) to lea
10406 // peephole %{
10407 //   // increment preceeded by register-register move
10408 //   peepmatch (incI_eReg movI);
10409 //   // require that the destination register of the increment
10410 //   // match the destination register of the move
10411 //   peepconstraint (0.dst == 1.dst);
10412 //   // construct a replacement instruction that sets
10413 //   // the destination to (move's source register + one)
10414 //   peepreplace (leaI_eReg_immI(0.dst 1.src 0.src));
10415 // %}
10416 //
10417 // Implementation no longer uses movX instructions since
10418 // machine-independent system no longer uses CopyX nodes.
10419 //
10420 // peephole %{
10421 //   peepmatch (incI_eReg movI);
10422 //   peepconstraint (0.dst == 1.dst);
10423 //   peepreplace (leaI_eReg_immI(0.dst 1.src 0.src));
10424 // %}
10425 //
10426 // peephole %{
10427 //   peepmatch (decI_eReg movI);
10428 //   peepconstraint (0.dst == 1.dst);
10429 //   peepreplace (leaI_eReg_immI(0.dst 1.src 0.src));
10430 // %}
10431 //
10432 // peephole %{
10433 //   peepmatch (addI_eReg_imm movI);
10434 //   peepconstraint (0.dst == 1.dst);
10435 //   peepreplace (leaI_eReg_immI(0.dst 1.src 0.src));
10436 // %}
10437 //
10438 // peephole %{
10439 //   peepmatch (addP_eReg_imm movP);
10440 //   peepconstraint (0.dst == 1.dst);
10441 //   peepreplace (leaP_eReg_immI(0.dst 1.src 0.src));
10442 // %}
10443 
10444 
10445 //  This peephole rule does not work, probably because ADLC can't handle two effects:
10446 //  Effect 1 is defining 0.op1 and effect 2 is setting CC
10447 // condense a load from memory and subsequent test for zero
10448 // into a single, more efficient ICM instruction.
10449 // peephole %{
10450 //   peepmatch (compI_iReg_imm0 loadI);
10451 //   peepconstraint (1.dst == 0.op1);
10452 //   peepreplace (loadtest15_iReg_mem(0.op1 0.op1 1.mem));
10453 // %}
10454 
10455 // // Change load of spilled value to only a spill
10456 // instruct storeI(memory mem, eRegI src) %{
10457 //   match(Set mem (StoreI mem src));
10458 // %}
10459 //
10460 // instruct loadI(eRegI dst, memory mem) %{
10461 //   match(Set dst (LoadI mem));
10462 // %}
10463 //
10464 peephole %{
10465   peepmatch (loadI storeI);
10466   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
10467   peepreplace (storeI(1.mem 1.mem 1.src));
10468 %}
10469 
10470 peephole %{
10471   peepmatch (loadL storeL);
10472   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
10473   peepreplace (storeL(1.mem 1.mem 1.src));
10474 %}
10475 
10476 peephole %{
10477   peepmatch (loadP storeP);
10478   peepconstraint (1.src == 0.dst, 1.dst == 0.mem);
10479   peepreplace (storeP(1.dst 1.dst 1.src));
10480 %}
10481 
10482 //----------SUPERWORD RULES---------------------------------------------------
10483 
10484 //  Expand rules for special cases
10485 
10486 instruct expand_storeF(stackSlotF mem, regF src) %{
10487   // No match rule, false predicate, for expand only.
10488   effect(DEF mem, USE src);
10489   predicate(false);
10490   ins_cost(MEMORY_REF_COST);
10491   // TODO: s390 port size(FIXED_SIZE);
10492   format %{ "STE      $src,$mem\t # replicate(float2stack)" %}
10493   opcode(STE_ZOPC, STE_ZOPC);
10494   ins_encode(z_form_rt_mem(src, mem));
10495   ins_pipe(pipe_class_dummy);
10496 %}
10497 
10498 instruct expand_LoadLogical_I2L(iRegL dst, stackSlotF mem) %{
10499   // No match rule, false predicate, for expand only.
10500   effect(DEF dst, USE mem);
10501   predicate(false);
10502   ins_cost(MEMORY_REF_COST);
10503   // TODO: s390 port size(FIXED_SIZE);
10504   format %{ "LLGF     $dst,$mem\t # replicate(stack2reg(unsigned))" %}
10505   opcode(LLGF_ZOPC, LLGF_ZOPC);
10506   ins_encode(z_form_rt_mem(dst, mem));
10507   ins_pipe(pipe_class_dummy);
10508 %}
10509 
10510 // Replicate scalar int to packed int values (8 Bytes)
10511 instruct expand_Repl2I_reg(iRegL dst, iRegL src) %{
10512   // Dummy match rule, false predicate, for expand only.
10513   match(Set dst (ConvI2L src));
10514   predicate(false);
10515   ins_cost(DEFAULT_COST);
10516   // TODO: s390 port size(FIXED_SIZE);
10517   format %{ "REPLIC2F $dst,$src\t # replicate(pack2F)" %}
10518   ins_encode %{
10519     if ($dst$$Register == $src$$Register) {
10520       __ z_sllg(Z_R0_scratch, $src$$Register, 64-32);
10521       __ z_ogr($dst$$Register, Z_R0_scratch);
10522     }  else {
10523       __ z_sllg($dst$$Register, $src$$Register, 64-32);
10524       __ z_ogr( $dst$$Register, $src$$Register);
10525     }
10526   %}
10527   ins_pipe(pipe_class_dummy);
10528 %}
10529 
10530 // Replication
10531 
10532 // Exploit rotate_then_insert, if available
10533 // Replicate scalar byte to packed byte values (8 Bytes).
10534 instruct Repl8B_reg_risbg(iRegL dst, iRegI src, flagsReg cr) %{
10535   match(Set dst (ReplicateB src));
10536   effect(KILL cr);
10537   predicate((n->as_Vector()->length() == 8));
10538   format %{ "REPLIC8B $dst,$src\t # pack8B" %}
10539   ins_encode %{
10540     if ($dst$$Register != $src$$Register) {
10541       __ z_lgr($dst$$Register, $src$$Register);
10542     }
10543     __ rotate_then_insert($dst$$Register, $dst$$Register, 48, 55,  8, false);
10544     __ rotate_then_insert($dst$$Register, $dst$$Register, 32, 47, 16, false);
10545     __ rotate_then_insert($dst$$Register, $dst$$Register,  0, 31, 32, false);
10546   %}
10547   ins_pipe(pipe_class_dummy);
10548 %}
10549 
10550 // Replicate scalar byte to packed byte values (8 Bytes).
10551 instruct Repl8B_imm(iRegL dst, immB_n0m1 src) %{
10552   match(Set dst (ReplicateB src));
10553   predicate(n->as_Vector()->length() == 8);
10554   ins_should_rematerialize(true);
10555   format %{ "REPLIC8B $dst,$src\t # pack8B imm" %}
10556   ins_encode %{
10557     int64_t  Isrc8 = $src$$constant & 0x000000ff;
10558     int64_t Isrc16 =  Isrc8 <<  8 |  Isrc8;
10559     int64_t Isrc32 = Isrc16 << 16 | Isrc16;
10560     assert(Isrc8 != 0x000000ff && Isrc8 != 0, "should be handled by other match rules.");
10561 
10562     __ z_llilf($dst$$Register, Isrc32);
10563     __ z_iihf($dst$$Register, Isrc32);
10564   %}
10565   ins_pipe(pipe_class_dummy);
10566 %}
10567 
10568 // Replicate scalar byte to packed byte values (8 Bytes).
10569 instruct Repl8B_imm0(iRegL dst, immI_0 src) %{
10570   match(Set dst (ReplicateB src));
10571   predicate(n->as_Vector()->length() == 8);
10572   ins_should_rematerialize(true);
10573   format %{ "REPLIC8B $dst,$src\t # pack8B imm0" %}
10574   ins_encode %{ __ z_laz($dst$$Register, 0, Z_R0); %}
10575   ins_pipe(pipe_class_dummy);
10576 %}
10577 
10578 // Replicate scalar byte to packed byte values (8 Bytes).
10579 instruct Repl8B_immm1(iRegL dst, immB_minus1 src) %{
10580   match(Set dst (ReplicateB src));
10581   predicate(n->as_Vector()->length() == 8);
10582   ins_should_rematerialize(true);
10583   format %{ "REPLIC8B $dst,$src\t # pack8B immm1" %}
10584   ins_encode %{ __ z_lghi($dst$$Register, -1); %}
10585   ins_pipe(pipe_class_dummy);
10586 %}
10587 
10588 // Exploit rotate_then_insert, if available
10589 // Replicate scalar short to packed short values (8 Bytes).
10590 instruct Repl4S_reg_risbg(iRegL dst, iRegI src, flagsReg cr) %{
10591   match(Set dst (ReplicateS src));
10592   effect(KILL cr);
10593   predicate((n->as_Vector()->length() == 4));
10594   format %{ "REPLIC4S $dst,$src\t # pack4S" %}
10595   ins_encode %{
10596     if ($dst$$Register != $src$$Register) {
10597       __ z_lgr($dst$$Register, $src$$Register);
10598     }
10599     __ rotate_then_insert($dst$$Register, $dst$$Register, 32, 47, 16, false);
10600     __ rotate_then_insert($dst$$Register, $dst$$Register,  0, 31, 32, false);
10601   %}
10602   ins_pipe(pipe_class_dummy);
10603 %}
10604 
10605 // Replicate scalar short to packed short values (8 Bytes).
10606 instruct Repl4S_imm(iRegL dst, immS_n0m1 src) %{
10607   match(Set dst (ReplicateS src));
10608   predicate(n->as_Vector()->length() == 4);
10609   ins_should_rematerialize(true);
10610   format %{ "REPLIC4S $dst,$src\t # pack4S imm" %}
10611   ins_encode %{
10612     int64_t Isrc16 = $src$$constant & 0x0000ffff;
10613     int64_t Isrc32 = Isrc16 << 16 | Isrc16;
10614     assert(Isrc16 != 0x0000ffff && Isrc16 != 0, "Repl4S_imm: (src == " INT64_FORMAT
10615            ") should be handled by other match rules.", $src$$constant);
10616 
10617     __ z_llilf($dst$$Register, Isrc32);
10618     __ z_iihf($dst$$Register, Isrc32);
10619   %}
10620   ins_pipe(pipe_class_dummy);
10621 %}
10622 
10623 // Replicate scalar short to packed short values (8 Bytes).
10624 instruct Repl4S_imm0(iRegL dst, immI_0 src) %{
10625   match(Set dst (ReplicateS src));
10626   predicate(n->as_Vector()->length() == 4);
10627   ins_should_rematerialize(true);
10628   format %{ "REPLIC4S $dst,$src\t # pack4S imm0" %}
10629   ins_encode %{ __ z_laz($dst$$Register, 0, Z_R0); %}
10630   ins_pipe(pipe_class_dummy);
10631 %}
10632 
10633 // Replicate scalar short to packed short values (8 Bytes).
10634 instruct Repl4S_immm1(iRegL dst, immS_minus1 src) %{
10635   match(Set dst (ReplicateS src));
10636   predicate(n->as_Vector()->length() == 4);
10637   ins_should_rematerialize(true);
10638   format %{ "REPLIC4S $dst,$src\t # pack4S immm1" %}
10639   ins_encode %{ __ z_lghi($dst$$Register, -1); %}
10640   ins_pipe(pipe_class_dummy);
10641 %}
10642 
10643 // Exploit rotate_then_insert, if available.
10644 // Replicate scalar int to packed int values (8 Bytes).
10645 instruct Repl2I_reg_risbg(iRegL dst, iRegI src, flagsReg cr) %{
10646   match(Set dst (ReplicateI src));
10647   effect(KILL cr);
10648   predicate((n->as_Vector()->length() == 2));
10649   format %{ "REPLIC2I $dst,$src\t # pack2I" %}
10650   ins_encode %{
10651     if ($dst$$Register != $src$$Register) {
10652       __ z_lgr($dst$$Register, $src$$Register);
10653     }
10654     __ rotate_then_insert($dst$$Register, $dst$$Register, 0, 31, 32, false);
10655   %}
10656   ins_pipe(pipe_class_dummy);
10657 %}
10658 
10659 // Replicate scalar int to packed int values (8 Bytes).
10660 instruct Repl2I_imm(iRegL dst, immI_n0m1 src) %{
10661   match(Set dst (ReplicateI src));
10662   predicate(n->as_Vector()->length() == 2);
10663   ins_should_rematerialize(true);
10664   format %{ "REPLIC2I $dst,$src\t # pack2I imm" %}
10665   ins_encode %{
10666     int64_t Isrc32 = $src$$constant;
10667     assert(Isrc32 != -1 && Isrc32 != 0, "should be handled by other match rules.");
10668 
10669     __ z_llilf($dst$$Register, Isrc32);
10670     __ z_iihf($dst$$Register, Isrc32);
10671   %}
10672   ins_pipe(pipe_class_dummy);
10673 %}
10674 
10675 // Replicate scalar int to packed int values (8 Bytes).
10676 instruct Repl2I_imm0(iRegL dst, immI_0 src) %{
10677   match(Set dst (ReplicateI src));
10678   predicate(n->as_Vector()->length() == 2);
10679   ins_should_rematerialize(true);
10680   format %{ "REPLIC2I $dst,$src\t # pack2I imm0" %}
10681   ins_encode %{ __ z_laz($dst$$Register, 0, Z_R0); %}
10682   ins_pipe(pipe_class_dummy);
10683 %}
10684 
10685 // Replicate scalar int to packed int values (8 Bytes).
10686 instruct Repl2I_immm1(iRegL dst, immI_minus1 src) %{
10687   match(Set dst (ReplicateI src));
10688   predicate(n->as_Vector()->length() == 2);
10689   ins_should_rematerialize(true);
10690   format %{ "REPLIC2I $dst,$src\t # pack2I immm1" %}
10691   ins_encode %{ __ z_lghi($dst$$Register, -1); %}
10692   ins_pipe(pipe_class_dummy);
10693 %}
10694 
10695 //
10696 
10697 instruct Repl2F_reg_indirect(iRegL dst, regF src, flagsReg cr) %{
10698   match(Set dst (ReplicateF src));
10699   effect(KILL cr);
10700   predicate(!VM_Version::has_FPSupportEnhancements() && n->as_Vector()->length() == 2);
10701   format %{ "REPLIC2F $dst,$src\t # pack2F indirect" %}
10702   expand %{
10703     stackSlotF tmp;
10704     iRegL      tmp2;
10705     expand_storeF(tmp, src);
10706     expand_LoadLogical_I2L(tmp2, tmp);
10707     expand_Repl2I_reg(dst, tmp2);
10708   %}
10709 %}
10710 
10711 // Replicate scalar float to packed float values in GREG (8 Bytes).
10712 instruct Repl2F_reg_direct(iRegL dst, regF src, flagsReg cr) %{
10713   match(Set dst (ReplicateF src));
10714   effect(KILL cr);
10715   predicate(VM_Version::has_FPSupportEnhancements() && n->as_Vector()->length() == 2);
10716   format %{ "REPLIC2F $dst,$src\t # pack2F direct" %}
10717   ins_encode %{
10718     assert(VM_Version::has_FPSupportEnhancements(), "encoder should never be called on old H/W");
10719     __ z_lgdr($dst$$Register, $src$$FloatRegister);
10720 
10721     __ z_srlg(Z_R0_scratch, $dst$$Register, 32);  // Floats are left-justified in 64bit reg.
10722     __ z_iilf($dst$$Register, 0);                 // Save a "result not ready" stall.
10723     __ z_ogr($dst$$Register, Z_R0_scratch);
10724   %}
10725   ins_pipe(pipe_class_dummy);
10726 %}
10727 
10728 // Replicate scalar float immediate to packed float values in GREG (8 Bytes).
10729 instruct Repl2F_imm(iRegL dst, immF src) %{
10730   match(Set dst (ReplicateF src));
10731   predicate(n->as_Vector()->length() == 2);
10732   ins_should_rematerialize(true);
10733   format %{ "REPLIC2F $dst,$src\t # pack2F imm" %}
10734   ins_encode %{
10735     union {
10736       int   Isrc32;
10737       float Fsrc32;
10738     };
10739     Fsrc32 = $src$$constant;
10740     __ z_llilf($dst$$Register, Isrc32);
10741     __ z_iihf($dst$$Register, Isrc32);
10742   %}
10743   ins_pipe(pipe_class_dummy);
10744 %}
10745 
10746 // Replicate scalar float immediate zeroes to packed float values in GREG (8 Bytes).
10747 // Do this only for 'real' zeroes, especially don't loose sign of negative zeroes.
10748 instruct Repl2F_imm0(iRegL dst, immFp0 src) %{
10749   match(Set dst (ReplicateF src));
10750   predicate(n->as_Vector()->length() == 2);
10751   ins_should_rematerialize(true);
10752   format %{ "REPLIC2F $dst,$src\t # pack2F imm0" %}
10753   ins_encode %{ __ z_laz($dst$$Register, 0, Z_R0); %}
10754   ins_pipe(pipe_class_dummy);
10755 %}
10756 
10757 // Store
10758 
10759 // Store Aligned Packed Byte register to memory (8 Bytes).
10760 instruct storeA8B(memory mem, iRegL src) %{
10761   match(Set mem (StoreVector mem src));
10762   predicate(n->as_StoreVector()->memory_size() == 8);
10763   ins_cost(MEMORY_REF_COST);
10764   // TODO: s390 port size(VARIABLE_SIZE);
10765   format %{ "STG     $src,$mem\t # ST(packed8B)" %}
10766   opcode(STG_ZOPC, STG_ZOPC);
10767   ins_encode(z_form_rt_mem_opt(src, mem));
10768   ins_pipe(pipe_class_dummy);
10769 %}
10770 
10771 // Load
10772 
10773 instruct loadV8(iRegL dst, memory mem) %{
10774   match(Set dst (LoadVector mem));
10775   predicate(n->as_LoadVector()->memory_size() == 8);
10776   ins_cost(MEMORY_REF_COST);
10777   // TODO: s390 port size(VARIABLE_SIZE);
10778   format %{ "LG      $dst,$mem\t # L(packed8B)" %}
10779   opcode(LG_ZOPC, LG_ZOPC);
10780   ins_encode(z_form_rt_mem_opt(dst, mem));
10781   ins_pipe(pipe_class_dummy);
10782 %}
10783 
10784 //----------POPULATION COUNT RULES--------------------------------------------
10785 
10786 // Byte reverse
10787 
10788 instruct bytes_reverse_int(iRegI dst, iRegI src) %{
10789   match(Set dst (ReverseBytesI src));
10790   predicate(UseByteReverseInstruction);  // See Matcher::match_rule_supported
10791   ins_cost(DEFAULT_COST);
10792   size(4);
10793   format %{ "LRVR    $dst,$src\t# byte reverse int" %}
10794   opcode(LRVR_ZOPC);
10795   ins_encode(z_rreform(dst, src));
10796   ins_pipe(pipe_class_dummy);
10797 %}
10798 
10799 instruct bytes_reverse_long(iRegL dst, iRegL src) %{
10800   match(Set dst (ReverseBytesL src));
10801   predicate(UseByteReverseInstruction);  // See Matcher::match_rule_supported
10802   ins_cost(DEFAULT_COST);
10803   // TODO: s390 port size(FIXED_SIZE);
10804   format %{ "LRVGR   $dst,$src\t# byte reverse long" %}
10805   opcode(LRVGR_ZOPC);
10806   ins_encode(z_rreform(dst, src));
10807   ins_pipe(pipe_class_dummy);
10808 %}
10809 
10810 // Leading zeroes
10811 
10812 // The instruction FLOGR (Find Leftmost One in Grande (64bit) Register)
10813 // returns the bit position of the leftmost 1 in the 64bit source register.
10814 // As the bits are numbered from left to right (0..63), the returned
10815 // position index is equivalent to the number of leading zeroes.
10816 // If no 1-bit is found (i.e. the regsiter contains zero), the instruction
10817 // returns position 64. That's exactly what we need.
10818 
10819 instruct countLeadingZerosI(revenRegI dst, iRegI src, roddRegI tmp, flagsReg cr) %{
10820   match(Set dst (CountLeadingZerosI src));
10821   effect(KILL tmp, KILL cr);
10822   ins_cost(3 * DEFAULT_COST);
10823   size(14);
10824   format %{ "SLLG    $dst,$src,32\t# no need to always count 32 zeroes first\n\t"
10825             "IILH    $dst,0x8000 \t# insert \"stop bit\" to force result 32 for zero src.\n\t"
10826             "FLOGR   $dst,$dst"
10827          %}
10828   ins_encode %{
10829     // Performance experiments indicate that "FLOGR" is using some kind of
10830     // iteration to find the leftmost "1" bit.
10831     //
10832     // The prior implementation zero-extended the 32-bit argument to 64 bit,
10833     // thus forcing "FLOGR" to count 32 bits of which we know they are zero.
10834     // We could gain measurable speedup in micro benchmark:
10835     //
10836     //               leading   trailing
10837     //   z10:   int     2.04       1.68
10838     //         long     1.00       1.02
10839     //   z196:  int     0.99       1.23
10840     //         long     1.00       1.11
10841     //
10842     // By shifting the argument into the high-word instead of zero-extending it.
10843     // The add'l branch on condition (taken for a zero argument, very infrequent,
10844     // good prediction) is well compensated for by the savings.
10845     //
10846     // We leave the previous implementation in for some time in the future when
10847     // the "FLOGR" instruction may become less iterative.
10848 
10849     // Version 2: shows 62%(z9), 204%(z10), -1%(z196) improvement over original
10850     __ z_sllg($dst$$Register, $src$$Register, 32); // No need to always count 32 zeroes first.
10851     __ z_iilh($dst$$Register, 0x8000);   // Insert "stop bit" to force result 32 for zero src.
10852     __ z_flogr($dst$$Register, $dst$$Register);
10853   %}
10854   ins_pipe(pipe_class_dummy);
10855 %}
10856 
10857 instruct countLeadingZerosL(revenRegI dst, iRegL src, roddRegI tmp, flagsReg cr) %{
10858   match(Set dst (CountLeadingZerosL src));
10859   effect(KILL tmp, KILL cr);
10860   ins_cost(DEFAULT_COST);
10861   size(4);
10862   format %{ "FLOGR   $dst,$src \t# count leading zeros (long)\n\t" %}
10863   ins_encode %{ __ z_flogr($dst$$Register, $src$$Register); %}
10864   ins_pipe(pipe_class_dummy);
10865 %}
10866 
10867 // trailing zeroes
10868 
10869 // We transform the trailing zeroes problem to a leading zeroes problem
10870 // such that can use the FLOGR instruction to our advantage.
10871 
10872 // With
10873 //   tmp1 = src - 1
10874 // we flip all trailing zeroes to ones and the rightmost one to zero.
10875 // All other bits remain unchanged.
10876 // With the complement
10877 //   tmp2 = ~src
10878 // we get all ones in the trailing zeroes positions. Thus,
10879 //   tmp3 = tmp1 & tmp2
10880 // yields ones in the trailing zeroes positions and zeroes elsewhere.
10881 // Now we can apply FLOGR and get 64-(trailing zeroes).
10882 instruct countTrailingZerosI(revenRegI dst, iRegI src, roddRegI tmp, flagsReg cr) %{
10883   match(Set dst (CountTrailingZerosI src));
10884   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
10885   ins_cost(8 * DEFAULT_COST);
10886   // TODO: s390 port size(FIXED_SIZE);  // Emitted code depends on PreferLAoverADD being on/off.
10887   format %{ "LLGFR   $dst,$src  \t# clear upper 32 bits (we are dealing with int)\n\t"
10888             "LCGFR   $tmp,$src  \t# load 2's complement (32->64 bit)\n\t"
10889             "AGHI    $dst,-1    \t# tmp1 = src-1\n\t"
10890             "AGHI    $tmp,-1    \t# tmp2 = -src-1 = ~src\n\t"
10891             "NGR     $dst,$tmp  \t# tmp3 = tmp1&tmp2\n\t"
10892             "FLOGR   $dst,$dst  \t# count trailing zeros (int)\n\t"
10893             "AHI     $dst,-64   \t# tmp4 = 64-(trailing zeroes)-64\n\t"
10894             "LCR     $dst,$dst  \t# res = -tmp4"
10895          %}
10896   ins_encode %{
10897     Register Rdst = $dst$$Register;
10898     Register Rsrc = $src$$Register;
10899     // Rtmp only needed for for zero-argument shortcut. With kill effect in
10900     // match rule Rsrc = roddReg would be possible, saving one register.
10901     Register Rtmp = $tmp$$Register;
10902 
10903     assert_different_registers(Rdst, Rsrc, Rtmp);
10904 
10905     // Algorithm:
10906     // - Isolate the least significant (rightmost) set bit using (src & (-src)).
10907     //   All other bits in the result are zero.
10908     // - Find the "leftmost one" bit position in the single-bit result from previous step.
10909     // - 63-("leftmost one" bit position) gives the # of trailing zeros.
10910 
10911     // Version 2: shows 79%(z9), 68%(z10), 23%(z196) improvement over original.
10912     Label done;
10913     __ load_const_optimized(Rdst, 32); // Prepare for shortcut (zero argument), result will be 32.
10914     __ z_lcgfr(Rtmp, Rsrc);
10915     __ z_bre(done);                    // Taken very infrequently, good prediction, no BHT entry.
10916 
10917     __ z_nr(Rtmp, Rsrc);               // (src) & (-src) leaves nothing but least significant bit.
10918     __ z_ahi(Rtmp,  -1);               // Subtract one to fill all trailing zero positions with ones.
10919                                        // Use 32bit op to prevent borrow propagation (case Rdst = 0x80000000)
10920                                        // into upper half of reg. Not relevant with sllg below.
10921     __ z_sllg(Rdst, Rtmp, 32);         // Shift interesting contents to upper half of register.
10922     __ z_bre(done);                    // Shortcut for argument = 1, result will be 0.
10923                                        // Depends on CC set by ahi above.
10924                                        // Taken very infrequently, good prediction, no BHT entry.
10925                                        // Branch delayed to have Rdst set correctly (Rtmp == 0(32bit)
10926                                        // after SLLG Rdst == 0(64bit)).
10927     __ z_flogr(Rdst, Rdst);            // Kills tmp which is the oddReg for dst.
10928     __ add2reg(Rdst,  -32);            // 32-pos(leftmost1) is #trailing zeros
10929     __ z_lcgfr(Rdst, Rdst);            // Provide 64bit result at no cost.
10930     __ bind(done);
10931   %}
10932   ins_pipe(pipe_class_dummy);
10933 %}
10934 
10935 instruct countTrailingZerosL(revenRegI dst, iRegL src, roddRegL tmp, flagsReg cr) %{
10936   match(Set dst (CountTrailingZerosL src));
10937   effect(TEMP_DEF dst, KILL tmp, KILL cr);
10938   ins_cost(8 * DEFAULT_COST);
10939   // TODO: s390 port size(FIXED_SIZE);  // Emitted code depends on PreferLAoverADD being on/off.
10940   format %{ "LCGR    $dst,$src  \t# preserve src\n\t"
10941             "NGR     $dst,$src  \t#"
10942             "AGHI    $dst,-1    \t# tmp1 = src-1\n\t"
10943             "FLOGR   $dst,$dst  \t# count trailing zeros (long), kill $tmp\n\t"
10944             "AHI     $dst,-64   \t# tmp4 = 64-(trailing zeroes)-64\n\t"
10945             "LCR     $dst,$dst  \t#"
10946          %}
10947   ins_encode %{
10948     Register Rdst = $dst$$Register;
10949     Register Rsrc = $src$$Register;
10950     assert_different_registers(Rdst, Rsrc); // Rtmp == Rsrc allowed.
10951 
10952     // New version: shows 5%(z9), 2%(z10), 11%(z196) improvement over original.
10953     __ z_lcgr(Rdst, Rsrc);
10954     __ z_ngr(Rdst, Rsrc);
10955     __ add2reg(Rdst,   -1);
10956     __ z_flogr(Rdst, Rdst); // Kills tmp which is the oddReg for dst.
10957     __ add2reg(Rdst,  -64);
10958     __ z_lcgfr(Rdst, Rdst); // Provide 64bit result at no cost.
10959   %}
10960   ins_pipe(pipe_class_dummy);
10961 %}
10962 
10963 
10964 // bit count
10965 
10966 instruct popCountI(iRegI dst, iRegI src, iRegI tmp, flagsReg cr) %{
10967   match(Set dst (PopCountI src));
10968   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
10969   predicate(UsePopCountInstruction && VM_Version::has_PopCount());
10970   ins_cost(DEFAULT_COST);
10971   size(24);
10972   format %{ "POPCNT  $dst,$src\t# pop count int" %}
10973   ins_encode %{
10974     Register Rdst = $dst$$Register;
10975     Register Rsrc = $src$$Register;
10976     Register Rtmp = $tmp$$Register;
10977 
10978     // Prefer compile-time assertion over run-time SIGILL.
10979     assert(VM_Version::has_PopCount(), "bad predicate for countLeadingZerosI");
10980     assert_different_registers(Rdst, Rtmp);
10981 
10982     // Version 2: shows 10%(z196) improvement over original.
10983     __ z_popcnt(Rdst, Rsrc);
10984     __ z_srlg(Rtmp, Rdst, 16); // calc  byte4+byte6 and byte5+byte7
10985     __ z_alr(Rdst, Rtmp);      //   into byte6 and byte7
10986     __ z_srlg(Rtmp, Rdst,  8); // calc (byte4+byte6) + (byte5+byte7)
10987     __ z_alr(Rdst, Rtmp);      //   into byte7
10988     __ z_llgcr(Rdst, Rdst);    // zero-extend sum
10989   %}
10990   ins_pipe(pipe_class_dummy);
10991 %}
10992 
10993 instruct popCountL(iRegI dst, iRegL src, iRegL tmp, flagsReg cr) %{
10994   match(Set dst (PopCountL src));
10995   effect(TEMP_DEF dst, TEMP tmp, KILL cr);
10996   predicate(UsePopCountInstruction && VM_Version::has_PopCount());
10997   ins_cost(DEFAULT_COST);
10998   // TODO: s390 port size(FIXED_SIZE);
10999   format %{ "POPCNT  $dst,$src\t# pop count long" %}
11000   ins_encode %{
11001     Register Rdst = $dst$$Register;
11002     Register Rsrc = $src$$Register;
11003     Register Rtmp = $tmp$$Register;
11004 
11005     // Prefer compile-time assertion over run-time SIGILL.
11006     assert(VM_Version::has_PopCount(), "bad predicate for countLeadingZerosI");
11007     assert_different_registers(Rdst, Rtmp);
11008 
11009     // Original version. Using LA instead of algr seems to be a really bad idea (-35%).
11010     __ z_popcnt(Rdst, Rsrc);
11011     __ z_ahhlr(Rdst, Rdst, Rdst);
11012     __ z_sllg(Rtmp, Rdst, 16);
11013     __ z_algr(Rdst, Rtmp);
11014     __ z_sllg(Rtmp, Rdst,  8);
11015     __ z_algr(Rdst, Rtmp);
11016     __ z_srlg(Rdst, Rdst, 56);
11017   %}
11018   ins_pipe(pipe_class_dummy);
11019 %}
11020 
11021 //----------SMARTSPILL RULES---------------------------------------------------
11022 // These must follow all instruction definitions as they use the names
11023 // defined in the instructions definitions.
11024 
11025 // ============================================================================
11026 // TYPE PROFILING RULES
11027