1 //
   2 // Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 //
  24 
  25 // X86 Common Architecture Description File
  26 
  27 //----------REGISTER DEFINITION BLOCK------------------------------------------
  28 // This information is used by the matcher and the register allocator to
  29 // describe individual registers and classes of registers within the target
  30 // archtecture.
  31 
  32 register %{
  33 //----------Architecture Description Register Definitions----------------------
  34 // General Registers
  35 // "reg_def"  name ( register save type, C convention save type,
  36 //                   ideal register type, encoding );
  37 // Register Save Types:
  38 //
  39 // NS  = No-Save:       The register allocator assumes that these registers
  40 //                      can be used without saving upon entry to the method, &
  41 //                      that they do not need to be saved at call sites.
  42 //
  43 // SOC = Save-On-Call:  The register allocator assumes that these registers
  44 //                      can be used without saving upon entry to the method,
  45 //                      but that they must be saved at call sites.
  46 //
  47 // SOE = Save-On-Entry: The register allocator assumes that these registers
  48 //                      must be saved before using them upon entry to the
  49 //                      method, but they do not need to be saved at call
  50 //                      sites.
  51 //
  52 // AS  = Always-Save:   The register allocator assumes that these registers
  53 //                      must be saved before using them upon entry to the
  54 //                      method, & that they must be saved at call sites.
  55 //
  56 // Ideal Register Type is used to determine how to save & restore a
  57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  59 //
  60 // The encoding number is the actual bit-pattern placed into the opcodes.
  61 
  62 // XMM registers.  512-bit registers or 8 words each, labeled (a)-p.
  63 // Word a in each register holds a Float, words ab hold a Double.
  64 // The whole registers are used in SSE4.2 version intrinsics,
  65 // array copy stubs and superword operations (see UseSSE42Intrinsics,
  66 // UseXMMForArrayCopy and UseSuperword flags).
  67 // For pre EVEX enabled architectures:
  68 //      XMM8-XMM15 must be encoded with REX (VEX for UseAVX)
  69 // For EVEX enabled architectures:
  70 //      XMM8-XMM31 must be encoded with REX (EVEX for UseAVX).
  71 //
  72 // Linux ABI:   No register preserved across function calls
  73 //              XMM0-XMM7 might hold parameters
  74 // Windows ABI: XMM6-XMM31 preserved across function calls
  75 //              XMM0-XMM3 might hold parameters
  76 
  77 reg_def XMM0 ( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg());
  78 reg_def XMM0b( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(1));
  79 reg_def XMM0c( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(2));
  80 reg_def XMM0d( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(3));
  81 reg_def XMM0e( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(4));
  82 reg_def XMM0f( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(5));
  83 reg_def XMM0g( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(6));
  84 reg_def XMM0h( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(7));
  85 reg_def XMM0i( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(8));
  86 reg_def XMM0j( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(9));
  87 reg_def XMM0k( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(10));
  88 reg_def XMM0l( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(11));
  89 reg_def XMM0m( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(12));
  90 reg_def XMM0n( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(13));
  91 reg_def XMM0o( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(14));
  92 reg_def XMM0p( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(15));
  93 
  94 reg_def XMM1 ( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg());
  95 reg_def XMM1b( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(1));
  96 reg_def XMM1c( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(2));
  97 reg_def XMM1d( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(3));
  98 reg_def XMM1e( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(4));
  99 reg_def XMM1f( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(5));
 100 reg_def XMM1g( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(6));
 101 reg_def XMM1h( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(7));
 102 reg_def XMM1i( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(8));
 103 reg_def XMM1j( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(9));
 104 reg_def XMM1k( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(10));
 105 reg_def XMM1l( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(11));
 106 reg_def XMM1m( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(12));
 107 reg_def XMM1n( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(13));
 108 reg_def XMM1o( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(14));
 109 reg_def XMM1p( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(15));
 110 
 111 reg_def XMM2 ( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg());
 112 reg_def XMM2b( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(1));
 113 reg_def XMM2c( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(2));
 114 reg_def XMM2d( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(3));
 115 reg_def XMM2e( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(4));
 116 reg_def XMM2f( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(5));
 117 reg_def XMM2g( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(6));
 118 reg_def XMM2h( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(7));
 119 reg_def XMM2i( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(8));
 120 reg_def XMM2j( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(9));
 121 reg_def XMM2k( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(10));
 122 reg_def XMM2l( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(11));
 123 reg_def XMM2m( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(12));
 124 reg_def XMM2n( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(13));
 125 reg_def XMM2o( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(14));
 126 reg_def XMM2p( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(15));
 127 
 128 reg_def XMM3 ( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg());
 129 reg_def XMM3b( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(1));
 130 reg_def XMM3c( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(2));
 131 reg_def XMM3d( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(3));
 132 reg_def XMM3e( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(4));
 133 reg_def XMM3f( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(5));
 134 reg_def XMM3g( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(6));
 135 reg_def XMM3h( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(7));
 136 reg_def XMM3i( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(8));
 137 reg_def XMM3j( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(9));
 138 reg_def XMM3k( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(10));
 139 reg_def XMM3l( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(11));
 140 reg_def XMM3m( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(12));
 141 reg_def XMM3n( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(13));
 142 reg_def XMM3o( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(14));
 143 reg_def XMM3p( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(15));
 144 
 145 reg_def XMM4 ( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg());
 146 reg_def XMM4b( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(1));
 147 reg_def XMM4c( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(2));
 148 reg_def XMM4d( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(3));
 149 reg_def XMM4e( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(4));
 150 reg_def XMM4f( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(5));
 151 reg_def XMM4g( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(6));
 152 reg_def XMM4h( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(7));
 153 reg_def XMM4i( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(8));
 154 reg_def XMM4j( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(9));
 155 reg_def XMM4k( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(10));
 156 reg_def XMM4l( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(11));
 157 reg_def XMM4m( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(12));
 158 reg_def XMM4n( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(13));
 159 reg_def XMM4o( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(14));
 160 reg_def XMM4p( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(15));
 161 
 162 reg_def XMM5 ( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg());
 163 reg_def XMM5b( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(1));
 164 reg_def XMM5c( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(2));
 165 reg_def XMM5d( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(3));
 166 reg_def XMM5e( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(4));
 167 reg_def XMM5f( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(5));
 168 reg_def XMM5g( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(6));
 169 reg_def XMM5h( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(7));
 170 reg_def XMM5i( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(8));
 171 reg_def XMM5j( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(9));
 172 reg_def XMM5k( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(10));
 173 reg_def XMM5l( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(11));
 174 reg_def XMM5m( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(12));
 175 reg_def XMM5n( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(13));
 176 reg_def XMM5o( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(14));
 177 reg_def XMM5p( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(15));
 178 
 179 reg_def XMM6 ( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg());
 180 reg_def XMM6b( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(1));
 181 reg_def XMM6c( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(2));
 182 reg_def XMM6d( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(3));
 183 reg_def XMM6e( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(4));
 184 reg_def XMM6f( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(5));
 185 reg_def XMM6g( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(6));
 186 reg_def XMM6h( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(7));
 187 reg_def XMM6i( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(8));
 188 reg_def XMM6j( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(9));
 189 reg_def XMM6k( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(10));
 190 reg_def XMM6l( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(11));
 191 reg_def XMM6m( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(12));
 192 reg_def XMM6n( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(13));
 193 reg_def XMM6o( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(14));
 194 reg_def XMM6p( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(15));
 195 
 196 reg_def XMM7 ( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg());
 197 reg_def XMM7b( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(1));
 198 reg_def XMM7c( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(2));
 199 reg_def XMM7d( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(3));
 200 reg_def XMM7e( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(4));
 201 reg_def XMM7f( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(5));
 202 reg_def XMM7g( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(6));
 203 reg_def XMM7h( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(7));
 204 reg_def XMM7i( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(8));
 205 reg_def XMM7j( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(9));
 206 reg_def XMM7k( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(10));
 207 reg_def XMM7l( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(11));
 208 reg_def XMM7m( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(12));
 209 reg_def XMM7n( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(13));
 210 reg_def XMM7o( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(14));
 211 reg_def XMM7p( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(15));
 212 
 213 #ifdef _LP64
 214 
 215 reg_def XMM8 ( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg());
 216 reg_def XMM8b( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(1));
 217 reg_def XMM8c( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(2));
 218 reg_def XMM8d( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(3));
 219 reg_def XMM8e( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(4));
 220 reg_def XMM8f( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(5));
 221 reg_def XMM8g( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(6));
 222 reg_def XMM8h( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(7));
 223 reg_def XMM8i( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(8));
 224 reg_def XMM8j( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(9));
 225 reg_def XMM8k( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(10));
 226 reg_def XMM8l( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(11));
 227 reg_def XMM8m( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(12));
 228 reg_def XMM8n( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(13));
 229 reg_def XMM8o( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(14));
 230 reg_def XMM8p( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(15));
 231 
 232 reg_def XMM9 ( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg());
 233 reg_def XMM9b( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(1));
 234 reg_def XMM9c( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(2));
 235 reg_def XMM9d( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(3));
 236 reg_def XMM9e( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(4));
 237 reg_def XMM9f( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(5));
 238 reg_def XMM9g( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(6));
 239 reg_def XMM9h( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(7));
 240 reg_def XMM9i( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(8));
 241 reg_def XMM9j( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(9));
 242 reg_def XMM9k( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(10));
 243 reg_def XMM9l( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(11));
 244 reg_def XMM9m( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(12));
 245 reg_def XMM9n( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(13));
 246 reg_def XMM9o( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(14));
 247 reg_def XMM9p( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(15));
 248 
 249 reg_def XMM10 ( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg());
 250 reg_def XMM10b( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(1));
 251 reg_def XMM10c( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(2));
 252 reg_def XMM10d( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(3));
 253 reg_def XMM10e( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(4));
 254 reg_def XMM10f( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(5));
 255 reg_def XMM10g( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(6));
 256 reg_def XMM10h( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(7));
 257 reg_def XMM10i( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(8));
 258 reg_def XMM10j( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(9));
 259 reg_def XMM10k( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(10));
 260 reg_def XMM10l( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(11));
 261 reg_def XMM10m( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(12));
 262 reg_def XMM10n( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(13));
 263 reg_def XMM10o( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(14));
 264 reg_def XMM10p( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(15));
 265 
 266 reg_def XMM11 ( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg());
 267 reg_def XMM11b( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(1));
 268 reg_def XMM11c( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(2));
 269 reg_def XMM11d( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(3));
 270 reg_def XMM11e( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(4));
 271 reg_def XMM11f( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(5));
 272 reg_def XMM11g( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(6));
 273 reg_def XMM11h( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(7));
 274 reg_def XMM11i( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(8));
 275 reg_def XMM11j( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(9));
 276 reg_def XMM11k( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(10));
 277 reg_def XMM11l( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(11));
 278 reg_def XMM11m( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(12));
 279 reg_def XMM11n( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(13));
 280 reg_def XMM11o( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(14));
 281 reg_def XMM11p( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(15));
 282 
 283 reg_def XMM12 ( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg());
 284 reg_def XMM12b( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(1));
 285 reg_def XMM12c( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(2));
 286 reg_def XMM12d( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(3));
 287 reg_def XMM12e( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(4));
 288 reg_def XMM12f( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(5));
 289 reg_def XMM12g( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(6));
 290 reg_def XMM12h( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(7));
 291 reg_def XMM12i( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(8));
 292 reg_def XMM12j( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(9));
 293 reg_def XMM12k( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(10));
 294 reg_def XMM12l( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(11));
 295 reg_def XMM12m( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(12));
 296 reg_def XMM12n( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(13));
 297 reg_def XMM12o( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(14));
 298 reg_def XMM12p( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(15));
 299 
 300 reg_def XMM13 ( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg());
 301 reg_def XMM13b( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(1));
 302 reg_def XMM13c( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(2));
 303 reg_def XMM13d( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(3));
 304 reg_def XMM13e( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(4));
 305 reg_def XMM13f( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(5));
 306 reg_def XMM13g( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(6));
 307 reg_def XMM13h( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(7));
 308 reg_def XMM13i( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(8));
 309 reg_def XMM13j( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(9));
 310 reg_def XMM13k( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(10));
 311 reg_def XMM13l( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(11));
 312 reg_def XMM13m( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(12));
 313 reg_def XMM13n( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(13));
 314 reg_def XMM13o( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(14));
 315 reg_def XMM13p( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(15));
 316 
 317 reg_def XMM14 ( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg());
 318 reg_def XMM14b( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(1));
 319 reg_def XMM14c( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(2));
 320 reg_def XMM14d( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(3));
 321 reg_def XMM14e( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(4));
 322 reg_def XMM14f( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(5));
 323 reg_def XMM14g( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(6));
 324 reg_def XMM14h( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(7));
 325 reg_def XMM14i( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(8));
 326 reg_def XMM14j( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(9));
 327 reg_def XMM14k( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(10));
 328 reg_def XMM14l( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(11));
 329 reg_def XMM14m( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(12));
 330 reg_def XMM14n( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(13));
 331 reg_def XMM14o( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(14));
 332 reg_def XMM14p( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(15));
 333 
 334 reg_def XMM15 ( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg());
 335 reg_def XMM15b( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(1));
 336 reg_def XMM15c( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(2));
 337 reg_def XMM15d( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(3));
 338 reg_def XMM15e( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(4));
 339 reg_def XMM15f( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(5));
 340 reg_def XMM15g( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(6));
 341 reg_def XMM15h( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(7));
 342 reg_def XMM15i( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(8));
 343 reg_def XMM15j( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(9));
 344 reg_def XMM15k( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(10));
 345 reg_def XMM15l( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(11));
 346 reg_def XMM15m( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(12));
 347 reg_def XMM15n( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(13));
 348 reg_def XMM15o( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(14));
 349 reg_def XMM15p( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(15));
 350 
 351 reg_def XMM16 ( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg());
 352 reg_def XMM16b( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(1));
 353 reg_def XMM16c( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(2));
 354 reg_def XMM16d( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(3));
 355 reg_def XMM16e( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(4));
 356 reg_def XMM16f( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(5));
 357 reg_def XMM16g( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(6));
 358 reg_def XMM16h( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(7));
 359 reg_def XMM16i( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(8));
 360 reg_def XMM16j( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(9));
 361 reg_def XMM16k( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(10));
 362 reg_def XMM16l( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(11));
 363 reg_def XMM16m( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(12));
 364 reg_def XMM16n( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(13));
 365 reg_def XMM16o( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(14));
 366 reg_def XMM16p( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(15));
 367 
 368 reg_def XMM17 ( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg());
 369 reg_def XMM17b( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(1));
 370 reg_def XMM17c( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(2));
 371 reg_def XMM17d( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(3));
 372 reg_def XMM17e( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(4));
 373 reg_def XMM17f( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(5));
 374 reg_def XMM17g( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(6));
 375 reg_def XMM17h( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(7));
 376 reg_def XMM17i( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(8));
 377 reg_def XMM17j( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(9));
 378 reg_def XMM17k( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(10));
 379 reg_def XMM17l( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(11));
 380 reg_def XMM17m( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(12));
 381 reg_def XMM17n( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(13));
 382 reg_def XMM17o( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(14));
 383 reg_def XMM17p( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(15));
 384 
 385 reg_def XMM18 ( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg());
 386 reg_def XMM18b( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(1));
 387 reg_def XMM18c( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(2));
 388 reg_def XMM18d( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(3));
 389 reg_def XMM18e( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(4));
 390 reg_def XMM18f( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(5));
 391 reg_def XMM18g( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(6));
 392 reg_def XMM18h( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(7));
 393 reg_def XMM18i( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(8));
 394 reg_def XMM18j( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(9));
 395 reg_def XMM18k( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(10));
 396 reg_def XMM18l( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(11));
 397 reg_def XMM18m( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(12));
 398 reg_def XMM18n( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(13));
 399 reg_def XMM18o( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(14));
 400 reg_def XMM18p( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(15));
 401 
 402 reg_def XMM19 ( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg());
 403 reg_def XMM19b( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(1));
 404 reg_def XMM19c( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(2));
 405 reg_def XMM19d( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(3));
 406 reg_def XMM19e( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(4));
 407 reg_def XMM19f( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(5));
 408 reg_def XMM19g( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(6));
 409 reg_def XMM19h( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(7));
 410 reg_def XMM19i( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(8));
 411 reg_def XMM19j( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(9));
 412 reg_def XMM19k( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(10));
 413 reg_def XMM19l( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(11));
 414 reg_def XMM19m( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(12));
 415 reg_def XMM19n( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(13));
 416 reg_def XMM19o( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(14));
 417 reg_def XMM19p( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(15));
 418 
 419 reg_def XMM20 ( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg());
 420 reg_def XMM20b( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(1));
 421 reg_def XMM20c( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(2));
 422 reg_def XMM20d( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(3));
 423 reg_def XMM20e( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(4));
 424 reg_def XMM20f( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(5));
 425 reg_def XMM20g( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(6));
 426 reg_def XMM20h( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(7));
 427 reg_def XMM20i( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(8));
 428 reg_def XMM20j( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(9));
 429 reg_def XMM20k( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(10));
 430 reg_def XMM20l( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(11));
 431 reg_def XMM20m( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(12));
 432 reg_def XMM20n( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(13));
 433 reg_def XMM20o( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(14));
 434 reg_def XMM20p( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(15));
 435 
 436 reg_def XMM21 ( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg());
 437 reg_def XMM21b( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(1));
 438 reg_def XMM21c( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(2));
 439 reg_def XMM21d( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(3));
 440 reg_def XMM21e( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(4));
 441 reg_def XMM21f( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(5));
 442 reg_def XMM21g( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(6));
 443 reg_def XMM21h( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(7));
 444 reg_def XMM21i( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(8));
 445 reg_def XMM21j( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(9));
 446 reg_def XMM21k( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(10));
 447 reg_def XMM21l( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(11));
 448 reg_def XMM21m( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(12));
 449 reg_def XMM21n( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(13));
 450 reg_def XMM21o( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(14));
 451 reg_def XMM21p( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(15));
 452 
 453 reg_def XMM22 ( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg());
 454 reg_def XMM22b( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(1));
 455 reg_def XMM22c( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(2));
 456 reg_def XMM22d( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(3));
 457 reg_def XMM22e( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(4));
 458 reg_def XMM22f( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(5));
 459 reg_def XMM22g( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(6));
 460 reg_def XMM22h( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(7));
 461 reg_def XMM22i( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(8));
 462 reg_def XMM22j( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(9));
 463 reg_def XMM22k( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(10));
 464 reg_def XMM22l( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(11));
 465 reg_def XMM22m( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(12));
 466 reg_def XMM22n( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(13));
 467 reg_def XMM22o( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(14));
 468 reg_def XMM22p( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(15));
 469 
 470 reg_def XMM23 ( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg());
 471 reg_def XMM23b( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(1));
 472 reg_def XMM23c( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(2));
 473 reg_def XMM23d( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(3));
 474 reg_def XMM23e( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(4));
 475 reg_def XMM23f( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(5));
 476 reg_def XMM23g( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(6));
 477 reg_def XMM23h( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(7));
 478 reg_def XMM23i( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(8));
 479 reg_def XMM23j( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(9));
 480 reg_def XMM23k( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(10));
 481 reg_def XMM23l( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(11));
 482 reg_def XMM23m( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(12));
 483 reg_def XMM23n( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(13));
 484 reg_def XMM23o( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(14));
 485 reg_def XMM23p( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(15));
 486 
 487 reg_def XMM24 ( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg());
 488 reg_def XMM24b( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(1));
 489 reg_def XMM24c( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(2));
 490 reg_def XMM24d( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(3));
 491 reg_def XMM24e( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(4));
 492 reg_def XMM24f( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(5));
 493 reg_def XMM24g( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(6));
 494 reg_def XMM24h( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(7));
 495 reg_def XMM24i( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(8));
 496 reg_def XMM24j( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(9));
 497 reg_def XMM24k( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(10));
 498 reg_def XMM24l( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(11));
 499 reg_def XMM24m( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(12));
 500 reg_def XMM24n( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(13));
 501 reg_def XMM24o( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(14));
 502 reg_def XMM24p( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(15));
 503 
 504 reg_def XMM25 ( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg());
 505 reg_def XMM25b( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(1));
 506 reg_def XMM25c( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(2));
 507 reg_def XMM25d( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(3));
 508 reg_def XMM25e( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(4));
 509 reg_def XMM25f( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(5));
 510 reg_def XMM25g( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(6));
 511 reg_def XMM25h( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(7));
 512 reg_def XMM25i( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(8));
 513 reg_def XMM25j( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(9));
 514 reg_def XMM25k( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(10));
 515 reg_def XMM25l( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(11));
 516 reg_def XMM25m( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(12));
 517 reg_def XMM25n( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(13));
 518 reg_def XMM25o( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(14));
 519 reg_def XMM25p( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(15));
 520 
 521 reg_def XMM26 ( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg());
 522 reg_def XMM26b( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(1));
 523 reg_def XMM26c( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(2));
 524 reg_def XMM26d( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(3));
 525 reg_def XMM26e( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(4));
 526 reg_def XMM26f( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(5));
 527 reg_def XMM26g( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(6));
 528 reg_def XMM26h( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(7));
 529 reg_def XMM26i( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(8));
 530 reg_def XMM26j( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(9));
 531 reg_def XMM26k( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(10));
 532 reg_def XMM26l( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(11));
 533 reg_def XMM26m( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(12));
 534 reg_def XMM26n( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(13));
 535 reg_def XMM26o( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(14));
 536 reg_def XMM26p( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(15));
 537 
 538 reg_def XMM27 ( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg());
 539 reg_def XMM27b( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(1));
 540 reg_def XMM27c( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(2));
 541 reg_def XMM27d( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(3));
 542 reg_def XMM27e( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(4));
 543 reg_def XMM27f( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(5));
 544 reg_def XMM27g( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(6));
 545 reg_def XMM27h( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(7));
 546 reg_def XMM27i( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(8));
 547 reg_def XMM27j( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(9));
 548 reg_def XMM27k( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(10));
 549 reg_def XMM27l( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(11));
 550 reg_def XMM27m( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(12));
 551 reg_def XMM27n( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(13));
 552 reg_def XMM27o( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(14));
 553 reg_def XMM27p( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(15));
 554 
 555 reg_def XMM28 ( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg());
 556 reg_def XMM28b( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(1));
 557 reg_def XMM28c( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(2));
 558 reg_def XMM28d( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(3));
 559 reg_def XMM28e( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(4));
 560 reg_def XMM28f( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(5));
 561 reg_def XMM28g( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(6));
 562 reg_def XMM28h( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(7));
 563 reg_def XMM28i( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(8));
 564 reg_def XMM28j( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(9));
 565 reg_def XMM28k( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(10));
 566 reg_def XMM28l( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(11));
 567 reg_def XMM28m( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(12));
 568 reg_def XMM28n( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(13));
 569 reg_def XMM28o( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(14));
 570 reg_def XMM28p( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(15));
 571 
 572 reg_def XMM29 ( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg());
 573 reg_def XMM29b( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(1));
 574 reg_def XMM29c( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(2));
 575 reg_def XMM29d( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(3));
 576 reg_def XMM29e( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(4));
 577 reg_def XMM29f( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(5));
 578 reg_def XMM29g( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(6));
 579 reg_def XMM29h( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(7));
 580 reg_def XMM29i( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(8));
 581 reg_def XMM29j( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(9));
 582 reg_def XMM29k( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(10));
 583 reg_def XMM29l( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(11));
 584 reg_def XMM29m( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(12));
 585 reg_def XMM29n( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(13));
 586 reg_def XMM29o( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(14));
 587 reg_def XMM29p( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(15));
 588 
 589 reg_def XMM30 ( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg());
 590 reg_def XMM30b( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(1));
 591 reg_def XMM30c( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(2));
 592 reg_def XMM30d( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(3));
 593 reg_def XMM30e( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(4));
 594 reg_def XMM30f( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(5));
 595 reg_def XMM30g( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(6));
 596 reg_def XMM30h( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(7));
 597 reg_def XMM30i( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(8));
 598 reg_def XMM30j( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(9));
 599 reg_def XMM30k( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(10));
 600 reg_def XMM30l( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(11));
 601 reg_def XMM30m( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(12));
 602 reg_def XMM30n( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(13));
 603 reg_def XMM30o( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(14));
 604 reg_def XMM30p( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(15));
 605 
 606 reg_def XMM31 ( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg());
 607 reg_def XMM31b( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(1));
 608 reg_def XMM31c( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(2));
 609 reg_def XMM31d( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(3));
 610 reg_def XMM31e( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(4));
 611 reg_def XMM31f( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(5));
 612 reg_def XMM31g( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(6));
 613 reg_def XMM31h( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(7));
 614 reg_def XMM31i( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(8));
 615 reg_def XMM31j( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(9));
 616 reg_def XMM31k( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(10));
 617 reg_def XMM31l( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(11));
 618 reg_def XMM31m( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(12));
 619 reg_def XMM31n( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(13));
 620 reg_def XMM31o( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(14));
 621 reg_def XMM31p( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(15));
 622 
 623 #endif // _LP64
 624 
 625 #ifdef _LP64
 626 reg_def RFLAGS(SOC, SOC, 0, 16, VMRegImpl::Bad());
 627 #else
 628 reg_def RFLAGS(SOC, SOC, 0, 8, VMRegImpl::Bad());
 629 #endif // _LP64
 630 
 631 alloc_class chunk1(XMM0,  XMM0b,  XMM0c,  XMM0d,  XMM0e,  XMM0f,  XMM0g,  XMM0h,  XMM0i,  XMM0j,  XMM0k,  XMM0l,  XMM0m,  XMM0n,  XMM0o,  XMM0p,
 632                    XMM1,  XMM1b,  XMM1c,  XMM1d,  XMM1e,  XMM1f,  XMM1g,  XMM1h,  XMM1i,  XMM1j,  XMM1k,  XMM1l,  XMM1m,  XMM1n,  XMM1o,  XMM1p,
 633                    XMM2,  XMM2b,  XMM2c,  XMM2d,  XMM2e,  XMM2f,  XMM2g,  XMM2h,  XMM2i,  XMM2j,  XMM2k,  XMM2l,  XMM2m,  XMM2n,  XMM2o,  XMM2p,
 634                    XMM3,  XMM3b,  XMM3c,  XMM3d,  XMM3e,  XMM3f,  XMM3g,  XMM3h,  XMM3i,  XMM3j,  XMM3k,  XMM3l,  XMM3m,  XMM3n,  XMM3o,  XMM3p,
 635                    XMM4,  XMM4b,  XMM4c,  XMM4d,  XMM4e,  XMM4f,  XMM4g,  XMM4h,  XMM4i,  XMM4j,  XMM4k,  XMM4l,  XMM4m,  XMM4n,  XMM4o,  XMM4p,
 636                    XMM5,  XMM5b,  XMM5c,  XMM5d,  XMM5e,  XMM5f,  XMM5g,  XMM5h,  XMM5i,  XMM5j,  XMM5k,  XMM5l,  XMM5m,  XMM5n,  XMM5o,  XMM5p,
 637                    XMM6,  XMM6b,  XMM6c,  XMM6d,  XMM6e,  XMM6f,  XMM6g,  XMM6h,  XMM6i,  XMM6j,  XMM6k,  XMM6l,  XMM6m,  XMM6n,  XMM6o,  XMM6p,
 638                    XMM7,  XMM7b,  XMM7c,  XMM7d,  XMM7e,  XMM7f,  XMM7g,  XMM7h,  XMM7i,  XMM7j,  XMM7k,  XMM7l,  XMM7m,  XMM7n,  XMM7o,  XMM7p
 639 #ifdef _LP64
 640                   ,XMM8,  XMM8b,  XMM8c,  XMM8d,  XMM8e,  XMM8f,  XMM8g,  XMM8h,  XMM8i,  XMM8j,  XMM8k,  XMM8l,  XMM8m,  XMM8n,  XMM8o,  XMM8p,
 641                    XMM9,  XMM9b,  XMM9c,  XMM9d,  XMM9e,  XMM9f,  XMM9g,  XMM9h,  XMM9i,  XMM9j,  XMM9k,  XMM9l,  XMM9m,  XMM9n,  XMM9o,  XMM9p,
 642                    XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h, XMM10i, XMM10j, XMM10k, XMM10l, XMM10m, XMM10n, XMM10o, XMM10p,
 643                    XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h, XMM11i, XMM11j, XMM11k, XMM11l, XMM11m, XMM11n, XMM11o, XMM11p,
 644                    XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h, XMM12i, XMM12j, XMM12k, XMM12l, XMM12m, XMM12n, XMM12o, XMM12p,
 645                    XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h, XMM13i, XMM13j, XMM13k, XMM13l, XMM13m, XMM13n, XMM13o, XMM13p,
 646                    XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h, XMM14i, XMM14j, XMM14k, XMM14l, XMM14m, XMM14n, XMM14o, XMM14p,
 647                    XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h, XMM15i, XMM15j, XMM15k, XMM15l, XMM15m, XMM15n, XMM15o, XMM15p
 648                   ,XMM16, XMM16b, XMM16c, XMM16d, XMM16e, XMM16f, XMM16g, XMM16h, XMM16i, XMM16j, XMM16k, XMM16l, XMM16m, XMM16n, XMM16o, XMM16p,
 649                    XMM17, XMM17b, XMM17c, XMM17d, XMM17e, XMM17f, XMM17g, XMM17h, XMM17i, XMM17j, XMM17k, XMM17l, XMM17m, XMM17n, XMM17o, XMM17p,
 650                    XMM18, XMM18b, XMM18c, XMM18d, XMM18e, XMM18f, XMM18g, XMM18h, XMM18i, XMM18j, XMM18k, XMM18l, XMM18m, XMM18n, XMM18o, XMM18p,
 651                    XMM19, XMM19b, XMM19c, XMM19d, XMM19e, XMM19f, XMM19g, XMM19h, XMM19i, XMM19j, XMM19k, XMM19l, XMM19m, XMM19n, XMM19o, XMM19p,
 652                    XMM20, XMM20b, XMM20c, XMM20d, XMM20e, XMM20f, XMM20g, XMM20h, XMM20i, XMM20j, XMM20k, XMM20l, XMM20m, XMM20n, XMM20o, XMM20p,
 653                    XMM21, XMM21b, XMM21c, XMM21d, XMM21e, XMM21f, XMM21g, XMM21h, XMM21i, XMM21j, XMM21k, XMM21l, XMM21m, XMM21n, XMM21o, XMM21p,
 654                    XMM22, XMM22b, XMM22c, XMM22d, XMM22e, XMM22f, XMM22g, XMM22h, XMM22i, XMM22j, XMM22k, XMM22l, XMM22m, XMM22n, XMM22o, XMM22p,
 655                    XMM23, XMM23b, XMM23c, XMM23d, XMM23e, XMM23f, XMM23g, XMM23h, XMM23i, XMM23j, XMM23k, XMM23l, XMM23m, XMM23n, XMM23o, XMM23p,
 656                    XMM24, XMM24b, XMM24c, XMM24d, XMM24e, XMM24f, XMM24g, XMM24h, XMM24i, XMM24j, XMM24k, XMM24l, XMM24m, XMM24n, XMM24o, XMM24p,
 657                    XMM25, XMM25b, XMM25c, XMM25d, XMM25e, XMM25f, XMM25g, XMM25h, XMM25i, XMM25j, XMM25k, XMM25l, XMM25m, XMM25n, XMM25o, XMM25p,
 658                    XMM26, XMM26b, XMM26c, XMM26d, XMM26e, XMM26f, XMM26g, XMM26h, XMM26i, XMM26j, XMM26k, XMM26l, XMM26m, XMM26n, XMM26o, XMM26p,
 659                    XMM27, XMM27b, XMM27c, XMM27d, XMM27e, XMM27f, XMM27g, XMM27h, XMM27i, XMM27j, XMM27k, XMM27l, XMM27m, XMM27n, XMM27o, XMM27p,
 660                    XMM28, XMM28b, XMM28c, XMM28d, XMM28e, XMM28f, XMM28g, XMM28h, XMM28i, XMM28j, XMM28k, XMM28l, XMM28m, XMM28n, XMM28o, XMM28p,
 661                    XMM29, XMM29b, XMM29c, XMM29d, XMM29e, XMM29f, XMM29g, XMM29h, XMM29i, XMM29j, XMM29k, XMM29l, XMM29m, XMM29n, XMM29o, XMM29p,
 662                    XMM30, XMM30b, XMM30c, XMM30d, XMM30e, XMM30f, XMM30g, XMM30h, XMM30i, XMM30j, XMM30k, XMM30l, XMM30m, XMM30n, XMM30o, XMM30p,
 663                    XMM31, XMM31b, XMM31c, XMM31d, XMM31e, XMM31f, XMM31g, XMM31h, XMM31i, XMM31j, XMM31k, XMM31l, XMM31m, XMM31n, XMM31o, XMM31p
 664 #endif
 665                       );
 666 
 667 // flags allocation class should be last.
 668 alloc_class chunk2(RFLAGS);
 669 
 670 // Singleton class for condition codes
 671 reg_class int_flags(RFLAGS);
 672 
 673 // Class for pre evex float registers
 674 reg_class float_reg_legacy(XMM0,
 675                     XMM1,
 676                     XMM2,
 677                     XMM3,
 678                     XMM4,
 679                     XMM5,
 680                     XMM6,
 681                     XMM7
 682 #ifdef _LP64
 683                    ,XMM8,
 684                     XMM9,
 685                     XMM10,
 686                     XMM11,
 687                     XMM12,
 688                     XMM13,
 689                     XMM14,
 690                     XMM15
 691 #endif
 692                     );
 693 
 694 // Class for evex float registers
 695 reg_class float_reg_evex(XMM0,
 696                     XMM1,
 697                     XMM2,
 698                     XMM3,
 699                     XMM4,
 700                     XMM5,
 701                     XMM6,
 702                     XMM7
 703 #ifdef _LP64
 704                    ,XMM8,
 705                     XMM9,
 706                     XMM10,
 707                     XMM11,
 708                     XMM12,
 709                     XMM13,
 710                     XMM14,
 711                     XMM15,
 712                     XMM16,
 713                     XMM17,
 714                     XMM18,
 715                     XMM19,
 716                     XMM20,
 717                     XMM21,
 718                     XMM22,
 719                     XMM23,
 720                     XMM24,
 721                     XMM25,
 722                     XMM26,
 723                     XMM27,
 724                     XMM28,
 725                     XMM29,
 726                     XMM30,
 727                     XMM31
 728 #endif
 729                     );
 730 
 731 reg_class_dynamic float_reg(float_reg_evex, float_reg_legacy, %{ VM_Version::supports_evex() %} );
 732 
 733 // Class for pre evex double registers
 734 reg_class double_reg_legacy(XMM0,  XMM0b,
 735                      XMM1,  XMM1b,
 736                      XMM2,  XMM2b,
 737                      XMM3,  XMM3b,
 738                      XMM4,  XMM4b,
 739                      XMM5,  XMM5b,
 740                      XMM6,  XMM6b,
 741                      XMM7,  XMM7b
 742 #ifdef _LP64
 743                     ,XMM8,  XMM8b,
 744                      XMM9,  XMM9b,
 745                      XMM10, XMM10b,
 746                      XMM11, XMM11b,
 747                      XMM12, XMM12b,
 748                      XMM13, XMM13b,
 749                      XMM14, XMM14b,
 750                      XMM15, XMM15b
 751 #endif
 752                      );
 753 
 754 // Class for evex double registers
 755 reg_class double_reg_evex(XMM0,  XMM0b,
 756                      XMM1,  XMM1b,
 757                      XMM2,  XMM2b,
 758                      XMM3,  XMM3b,
 759                      XMM4,  XMM4b,
 760                      XMM5,  XMM5b,
 761                      XMM6,  XMM6b,
 762                      XMM7,  XMM7b
 763 #ifdef _LP64
 764                     ,XMM8,  XMM8b,
 765                      XMM9,  XMM9b,
 766                      XMM10, XMM10b,
 767                      XMM11, XMM11b,
 768                      XMM12, XMM12b,
 769                      XMM13, XMM13b,
 770                      XMM14, XMM14b,
 771                      XMM15, XMM15b,
 772                      XMM16, XMM16b,
 773                      XMM17, XMM17b,
 774                      XMM18, XMM18b,
 775                      XMM19, XMM19b,
 776                      XMM20, XMM20b,
 777                      XMM21, XMM21b,
 778                      XMM22, XMM22b,
 779                      XMM23, XMM23b,
 780                      XMM24, XMM24b,
 781                      XMM25, XMM25b,
 782                      XMM26, XMM26b,
 783                      XMM27, XMM27b,
 784                      XMM28, XMM28b,
 785                      XMM29, XMM29b,
 786                      XMM30, XMM30b,
 787                      XMM31, XMM31b
 788 #endif
 789                      );
 790 
 791 reg_class_dynamic double_reg(double_reg_evex, double_reg_legacy, %{ VM_Version::supports_evex() %} );
 792 
 793 // Class for pre evex 32bit vector registers
 794 reg_class vectors_reg_legacy(XMM0,
 795                       XMM1,
 796                       XMM2,
 797                       XMM3,
 798                       XMM4,
 799                       XMM5,
 800                       XMM6,
 801                       XMM7
 802 #ifdef _LP64
 803                      ,XMM8,
 804                       XMM9,
 805                       XMM10,
 806                       XMM11,
 807                       XMM12,
 808                       XMM13,
 809                       XMM14,
 810                       XMM15
 811 #endif
 812                       );
 813 
 814 // Class for evex 32bit vector registers
 815 reg_class vectors_reg_evex(XMM0,
 816                       XMM1,
 817                       XMM2,
 818                       XMM3,
 819                       XMM4,
 820                       XMM5,
 821                       XMM6,
 822                       XMM7
 823 #ifdef _LP64
 824                      ,XMM8,
 825                       XMM9,
 826                       XMM10,
 827                       XMM11,
 828                       XMM12,
 829                       XMM13,
 830                       XMM14,
 831                       XMM15,
 832                       XMM16,
 833                       XMM17,
 834                       XMM18,
 835                       XMM19,
 836                       XMM20,
 837                       XMM21,
 838                       XMM22,
 839                       XMM23,
 840                       XMM24,
 841                       XMM25,
 842                       XMM26,
 843                       XMM27,
 844                       XMM28,
 845                       XMM29,
 846                       XMM30,
 847                       XMM31
 848 #endif
 849                       );
 850 
 851 reg_class_dynamic vectors_reg(vectors_reg_evex, vectors_reg_legacy, %{ VM_Version::supports_evex() %} );
 852 
 853 // Class for all 64bit vector registers
 854 reg_class vectord_reg_legacy(XMM0,  XMM0b,
 855                       XMM1,  XMM1b,
 856                       XMM2,  XMM2b,
 857                       XMM3,  XMM3b,
 858                       XMM4,  XMM4b,
 859                       XMM5,  XMM5b,
 860                       XMM6,  XMM6b,
 861                       XMM7,  XMM7b
 862 #ifdef _LP64
 863                      ,XMM8,  XMM8b,
 864                       XMM9,  XMM9b,
 865                       XMM10, XMM10b,
 866                       XMM11, XMM11b,
 867                       XMM12, XMM12b,
 868                       XMM13, XMM13b,
 869                       XMM14, XMM14b,
 870                       XMM15, XMM15b
 871 #endif
 872                       );
 873 
 874 // Class for all 64bit vector registers
 875 reg_class vectord_reg_evex(XMM0,  XMM0b,
 876                       XMM1,  XMM1b,
 877                       XMM2,  XMM2b,
 878                       XMM3,  XMM3b,
 879                       XMM4,  XMM4b,
 880                       XMM5,  XMM5b,
 881                       XMM6,  XMM6b,
 882                       XMM7,  XMM7b
 883 #ifdef _LP64
 884                      ,XMM8,  XMM8b,
 885                       XMM9,  XMM9b,
 886                       XMM10, XMM10b,
 887                       XMM11, XMM11b,
 888                       XMM12, XMM12b,
 889                       XMM13, XMM13b,
 890                       XMM14, XMM14b,
 891                       XMM15, XMM15b,
 892                       XMM16, XMM16b,
 893                       XMM17, XMM17b,
 894                       XMM18, XMM18b,
 895                       XMM19, XMM19b,
 896                       XMM20, XMM20b,
 897                       XMM21, XMM21b,
 898                       XMM22, XMM22b,
 899                       XMM23, XMM23b,
 900                       XMM24, XMM24b,
 901                       XMM25, XMM25b,
 902                       XMM26, XMM26b,
 903                       XMM27, XMM27b,
 904                       XMM28, XMM28b,
 905                       XMM29, XMM29b,
 906                       XMM30, XMM30b,
 907                       XMM31, XMM31b
 908 #endif
 909                       );
 910 
 911 reg_class_dynamic vectord_reg(vectord_reg_evex, vectord_reg_legacy, %{ VM_Version::supports_evex() %} );
 912 
 913 // Class for all 128bit vector registers
 914 reg_class vectorx_reg_legacy(XMM0,  XMM0b,  XMM0c,  XMM0d,
 915                       XMM1,  XMM1b,  XMM1c,  XMM1d,
 916                       XMM2,  XMM2b,  XMM2c,  XMM2d,
 917                       XMM3,  XMM3b,  XMM3c,  XMM3d,
 918                       XMM4,  XMM4b,  XMM4c,  XMM4d,
 919                       XMM5,  XMM5b,  XMM5c,  XMM5d,
 920                       XMM6,  XMM6b,  XMM6c,  XMM6d,
 921                       XMM7,  XMM7b,  XMM7c,  XMM7d
 922 #ifdef _LP64
 923                      ,XMM8,  XMM8b,  XMM8c,  XMM8d,
 924                       XMM9,  XMM9b,  XMM9c,  XMM9d,
 925                       XMM10, XMM10b, XMM10c, XMM10d,
 926                       XMM11, XMM11b, XMM11c, XMM11d,
 927                       XMM12, XMM12b, XMM12c, XMM12d,
 928                       XMM13, XMM13b, XMM13c, XMM13d,
 929                       XMM14, XMM14b, XMM14c, XMM14d,
 930                       XMM15, XMM15b, XMM15c, XMM15d
 931 #endif
 932                       );
 933 
 934 // Class for all 128bit vector registers
 935 reg_class vectorx_reg_evex(XMM0,  XMM0b,  XMM0c,  XMM0d,
 936                       XMM1,  XMM1b,  XMM1c,  XMM1d,
 937                       XMM2,  XMM2b,  XMM2c,  XMM2d,
 938                       XMM3,  XMM3b,  XMM3c,  XMM3d,
 939                       XMM4,  XMM4b,  XMM4c,  XMM4d,
 940                       XMM5,  XMM5b,  XMM5c,  XMM5d,
 941                       XMM6,  XMM6b,  XMM6c,  XMM6d,
 942                       XMM7,  XMM7b,  XMM7c,  XMM7d
 943 #ifdef _LP64
 944                      ,XMM8,  XMM8b,  XMM8c,  XMM8d,
 945                       XMM9,  XMM9b,  XMM9c,  XMM9d,
 946                       XMM10, XMM10b, XMM10c, XMM10d,
 947                       XMM11, XMM11b, XMM11c, XMM11d,
 948                       XMM12, XMM12b, XMM12c, XMM12d,
 949                       XMM13, XMM13b, XMM13c, XMM13d,
 950                       XMM14, XMM14b, XMM14c, XMM14d,
 951                       XMM15, XMM15b, XMM15c, XMM15d,
 952                       XMM16, XMM16b, XMM16c, XMM16d,
 953                       XMM17, XMM17b, XMM17c, XMM17d,
 954                       XMM18, XMM18b, XMM18c, XMM18d,
 955                       XMM19, XMM19b, XMM19c, XMM19d,
 956                       XMM20, XMM20b, XMM20c, XMM20d,
 957                       XMM21, XMM21b, XMM21c, XMM21d,
 958                       XMM22, XMM22b, XMM22c, XMM22d,
 959                       XMM23, XMM23b, XMM23c, XMM23d,
 960                       XMM24, XMM24b, XMM24c, XMM24d,
 961                       XMM25, XMM25b, XMM25c, XMM25d,
 962                       XMM26, XMM26b, XMM26c, XMM26d,
 963                       XMM27, XMM27b, XMM27c, XMM27d,
 964                       XMM28, XMM28b, XMM28c, XMM28d,
 965                       XMM29, XMM29b, XMM29c, XMM29d,
 966                       XMM30, XMM30b, XMM30c, XMM30d,
 967                       XMM31, XMM31b, XMM31c, XMM31d
 968 #endif
 969                       );
 970 
 971 reg_class_dynamic vectorx_reg(vectorx_reg_evex, vectorx_reg_legacy, %{ VM_Version::supports_evex() %} );
 972 
 973 // Class for all 256bit vector registers
 974 reg_class vectory_reg_legacy(XMM0,  XMM0b,  XMM0c,  XMM0d,  XMM0e,  XMM0f,  XMM0g,  XMM0h,
 975                       XMM1,  XMM1b,  XMM1c,  XMM1d,  XMM1e,  XMM1f,  XMM1g,  XMM1h,
 976                       XMM2,  XMM2b,  XMM2c,  XMM2d,  XMM2e,  XMM2f,  XMM2g,  XMM2h,
 977                       XMM3,  XMM3b,  XMM3c,  XMM3d,  XMM3e,  XMM3f,  XMM3g,  XMM3h,
 978                       XMM4,  XMM4b,  XMM4c,  XMM4d,  XMM4e,  XMM4f,  XMM4g,  XMM4h,
 979                       XMM5,  XMM5b,  XMM5c,  XMM5d,  XMM5e,  XMM5f,  XMM5g,  XMM5h,
 980                       XMM6,  XMM6b,  XMM6c,  XMM6d,  XMM6e,  XMM6f,  XMM6g,  XMM6h,
 981                       XMM7,  XMM7b,  XMM7c,  XMM7d,  XMM7e,  XMM7f,  XMM7g,  XMM7h
 982 #ifdef _LP64
 983                      ,XMM8,  XMM8b,  XMM8c,  XMM8d,  XMM8e,  XMM8f,  XMM8g,  XMM8h,
 984                       XMM9,  XMM9b,  XMM9c,  XMM9d,  XMM9e,  XMM9f,  XMM9g,  XMM9h,
 985                       XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h,
 986                       XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h,
 987                       XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h,
 988                       XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h,
 989                       XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h,
 990                       XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h
 991 #endif
 992                       );
 993 
 994 // Class for all 256bit vector registers
 995 reg_class vectory_reg_evex(XMM0,  XMM0b,  XMM0c,  XMM0d,  XMM0e,  XMM0f,  XMM0g,  XMM0h,
 996                       XMM1,  XMM1b,  XMM1c,  XMM1d,  XMM1e,  XMM1f,  XMM1g,  XMM1h,
 997                       XMM2,  XMM2b,  XMM2c,  XMM2d,  XMM2e,  XMM2f,  XMM2g,  XMM2h,
 998                       XMM3,  XMM3b,  XMM3c,  XMM3d,  XMM3e,  XMM3f,  XMM3g,  XMM3h,
 999                       XMM4,  XMM4b,  XMM4c,  XMM4d,  XMM4e,  XMM4f,  XMM4g,  XMM4h,
1000                       XMM5,  XMM5b,  XMM5c,  XMM5d,  XMM5e,  XMM5f,  XMM5g,  XMM5h,
1001                       XMM6,  XMM6b,  XMM6c,  XMM6d,  XMM6e,  XMM6f,  XMM6g,  XMM6h,
1002                       XMM7,  XMM7b,  XMM7c,  XMM7d,  XMM7e,  XMM7f,  XMM7g,  XMM7h
1003 #ifdef _LP64
1004                      ,XMM8,  XMM8b,  XMM8c,  XMM8d,  XMM8e,  XMM8f,  XMM8g,  XMM8h,
1005                       XMM9,  XMM9b,  XMM9c,  XMM9d,  XMM9e,  XMM9f,  XMM9g,  XMM9h,
1006                       XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h,
1007                       XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h,
1008                       XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h,
1009                       XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h,
1010                       XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h,
1011                       XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h,
1012                       XMM16, XMM16b, XMM16c, XMM16d, XMM16e, XMM16f, XMM16g, XMM16h,
1013                       XMM17, XMM17b, XMM17c, XMM17d, XMM17e, XMM17f, XMM17g, XMM17h,
1014                       XMM18, XMM18b, XMM18c, XMM18d, XMM18e, XMM18f, XMM18g, XMM18h,
1015                       XMM19, XMM19b, XMM19c, XMM19d, XMM19e, XMM19f, XMM19g, XMM19h,
1016                       XMM20, XMM20b, XMM20c, XMM20d, XMM20e, XMM20f, XMM20g, XMM20h,
1017                       XMM21, XMM21b, XMM21c, XMM21d, XMM21e, XMM21f, XMM21g, XMM21h,
1018                       XMM22, XMM22b, XMM22c, XMM22d, XMM22e, XMM22f, XMM22g, XMM22h,
1019                       XMM23, XMM23b, XMM23c, XMM23d, XMM23e, XMM23f, XMM23g, XMM23h,
1020                       XMM24, XMM24b, XMM24c, XMM24d, XMM24e, XMM24f, XMM24g, XMM24h,
1021                       XMM25, XMM25b, XMM25c, XMM25d, XMM25e, XMM25f, XMM25g, XMM25h,
1022                       XMM26, XMM26b, XMM26c, XMM26d, XMM26e, XMM26f, XMM26g, XMM26h,
1023                       XMM27, XMM27b, XMM27c, XMM27d, XMM27e, XMM27f, XMM27g, XMM27h,
1024                       XMM28, XMM28b, XMM28c, XMM28d, XMM28e, XMM28f, XMM28g, XMM28h,
1025                       XMM29, XMM29b, XMM29c, XMM29d, XMM29e, XMM29f, XMM29g, XMM29h,
1026                       XMM30, XMM30b, XMM30c, XMM30d, XMM30e, XMM30f, XMM30g, XMM30h,
1027                       XMM31, XMM31b, XMM31c, XMM31d, XMM31e, XMM31f, XMM31g, XMM31h
1028 #endif
1029                       );
1030 
1031 reg_class_dynamic vectory_reg(vectory_reg_evex, vectory_reg_legacy, %{ VM_Version::supports_evex() %} );
1032 
1033 // Class for all 512bit vector registers
1034 reg_class vectorz_reg(XMM0,  XMM0b,  XMM0c,  XMM0d,  XMM0e,  XMM0f,  XMM0g,  XMM0h,  XMM0i,  XMM0j,  XMM0k,  XMM0l,  XMM0m,  XMM0n,  XMM0o,  XMM0p,
1035                       XMM1,  XMM1b,  XMM1c,  XMM1d,  XMM1e,  XMM1f,  XMM1g,  XMM1h,  XMM1i,  XMM1j,  XMM1k,  XMM1l,  XMM1m,  XMM1n,  XMM1o,  XMM1p,
1036                       XMM2,  XMM2b,  XMM2c,  XMM2d,  XMM2e,  XMM2f,  XMM2g,  XMM2h,  XMM2i,  XMM2j,  XMM2k,  XMM2l,  XMM2m,  XMM2n,  XMM2o,  XMM2p,
1037                       XMM3,  XMM3b,  XMM3c,  XMM3d,  XMM3e,  XMM3f,  XMM3g,  XMM3h,  XMM3i,  XMM3j,  XMM3k,  XMM3l,  XMM3m,  XMM3n,  XMM3o,  XMM3p,
1038                       XMM4,  XMM4b,  XMM4c,  XMM4d,  XMM4e,  XMM4f,  XMM4g,  XMM4h,  XMM4i,  XMM4j,  XMM4k,  XMM4l,  XMM4m,  XMM4n,  XMM4o,  XMM4p,
1039                       XMM5,  XMM5b,  XMM5c,  XMM5d,  XMM5e,  XMM5f,  XMM5g,  XMM5h,  XMM5i,  XMM5j,  XMM5k,  XMM5l,  XMM5m,  XMM5n,  XMM5o,  XMM5p,
1040                       XMM6,  XMM6b,  XMM6c,  XMM6d,  XMM6e,  XMM6f,  XMM6g,  XMM6h,  XMM6i,  XMM6j,  XMM6k,  XMM6l,  XMM6m,  XMM6n,  XMM6o,  XMM6p,
1041                       XMM7,  XMM7b,  XMM7c,  XMM7d,  XMM7e,  XMM7f,  XMM7g,  XMM7h,  XMM7i,  XMM7j,  XMM7k,  XMM7l,  XMM7m,  XMM7n,  XMM7o,  XMM7p
1042 #ifdef _LP64
1043                      ,XMM8,  XMM8b,  XMM8c,  XMM8d,  XMM8e,  XMM8f,  XMM8g,  XMM8h,  XMM8i,  XMM8j,  XMM8k,  XMM8l,  XMM8m,  XMM8n,  XMM8o,  XMM8p,
1044                       XMM9,  XMM9b,  XMM9c,  XMM9d,  XMM9e,  XMM9f,  XMM9g,  XMM9h,  XMM9i,  XMM9j,  XMM9k,  XMM9l,  XMM9m,  XMM9n,  XMM9o,  XMM9p,
1045                       XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h, XMM10i, XMM10j, XMM10k, XMM10l, XMM10m, XMM10n, XMM10o, XMM10p,
1046                       XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h, XMM11i, XMM11j, XMM11k, XMM11l, XMM11m, XMM11n, XMM11o, XMM11p,
1047                       XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h, XMM12i, XMM12j, XMM12k, XMM12l, XMM12m, XMM12n, XMM12o, XMM12p,
1048                       XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h, XMM13i, XMM13j, XMM13k, XMM13l, XMM13m, XMM13n, XMM13o, XMM13p,
1049                       XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h, XMM14i, XMM14j, XMM14k, XMM14l, XMM14m, XMM14n, XMM14o, XMM14p,
1050                       XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h, XMM15i, XMM15j, XMM15k, XMM15l, XMM15m, XMM15n, XMM15o, XMM15p
1051                      ,XMM16, XMM16b, XMM16c, XMM16d, XMM16e, XMM16f, XMM16g, XMM16h, XMM16i, XMM16j, XMM16k, XMM16l, XMM16m, XMM16n, XMM16o, XMM16p,
1052                       XMM17, XMM17b, XMM17c, XMM17d, XMM17e, XMM17f, XMM17g, XMM17h, XMM17i, XMM17j, XMM17k, XMM17l, XMM17m, XMM17n, XMM17o, XMM17p,
1053                       XMM18, XMM18b, XMM18c, XMM18d, XMM18e, XMM18f, XMM18g, XMM18h, XMM18i, XMM18j, XMM18k, XMM18l, XMM18m, XMM18n, XMM18o, XMM18p,
1054                       XMM19, XMM19b, XMM19c, XMM19d, XMM19e, XMM19f, XMM19g, XMM19h, XMM19i, XMM19j, XMM19k, XMM19l, XMM19m, XMM19n, XMM19o, XMM19p,
1055                       XMM20, XMM20b, XMM20c, XMM20d, XMM20e, XMM20f, XMM20g, XMM20h, XMM20i, XMM20j, XMM20k, XMM20l, XMM20m, XMM20n, XMM20o, XMM20p,
1056                       XMM21, XMM21b, XMM21c, XMM21d, XMM21e, XMM21f, XMM21g, XMM21h, XMM21i, XMM21j, XMM21k, XMM21l, XMM21m, XMM21n, XMM21o, XMM21p,
1057                       XMM22, XMM22b, XMM22c, XMM22d, XMM22e, XMM22f, XMM22g, XMM22h, XMM22i, XMM22j, XMM22k, XMM22l, XMM22m, XMM22n, XMM22o, XMM22p,
1058                       XMM23, XMM23b, XMM23c, XMM23d, XMM23e, XMM23f, XMM23g, XMM23h, XMM23i, XMM23j, XMM23k, XMM23l, XMM23m, XMM23n, XMM23o, XMM23p,
1059                       XMM24, XMM24b, XMM24c, XMM24d, XMM24e, XMM24f, XMM24g, XMM24h, XMM24i, XMM24j, XMM24k, XMM24l, XMM24m, XMM24n, XMM24o, XMM24p,
1060                       XMM25, XMM25b, XMM25c, XMM25d, XMM25e, XMM25f, XMM25g, XMM25h, XMM25i, XMM25j, XMM25k, XMM25l, XMM25m, XMM25n, XMM25o, XMM25p,
1061                       XMM26, XMM26b, XMM26c, XMM26d, XMM26e, XMM26f, XMM26g, XMM26h, XMM26i, XMM26j, XMM26k, XMM26l, XMM26m, XMM26n, XMM26o, XMM26p,
1062                       XMM27, XMM27b, XMM27c, XMM27d, XMM27e, XMM27f, XMM27g, XMM27h, XMM27i, XMM27j, XMM27k, XMM27l, XMM27m, XMM27n, XMM27o, XMM27p,
1063                       XMM28, XMM28b, XMM28c, XMM28d, XMM28e, XMM28f, XMM28g, XMM28h, XMM28i, XMM28j, XMM28k, XMM28l, XMM28m, XMM28n, XMM28o, XMM28p,
1064                       XMM29, XMM29b, XMM29c, XMM29d, XMM29e, XMM29f, XMM29g, XMM29h, XMM29i, XMM29j, XMM29k, XMM29l, XMM29m, XMM29n, XMM29o, XMM29p,
1065                       XMM30, XMM30b, XMM30c, XMM30d, XMM30e, XMM30f, XMM30g, XMM30h, XMM30i, XMM30j, XMM30k, XMM30l, XMM30m, XMM30n, XMM30o, XMM30p,
1066                       XMM31, XMM31b, XMM31c, XMM31d, XMM31e, XMM31f, XMM31g, XMM31h, XMM31i, XMM31j, XMM31k, XMM31l, XMM31m, XMM31n, XMM31o, XMM31p
1067 #endif
1068                       );
1069 
1070 reg_class xmm0_reg(XMM0, XMM0b, XMM0c, XMM0d);
1071 reg_class ymm0_reg(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h);
1072 reg_class zmm0_reg(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h, XMM0i, XMM0j, XMM0k, XMM0l, XMM0m, XMM0n, XMM0o, XMM0p);
1073 
1074 reg_class xmm1_reg(XMM1, XMM1b, XMM1c, XMM1d);
1075 reg_class ymm1_reg(XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h);
1076 reg_class zmm1_reg(XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h, XMM1i, XMM1j, XMM1k, XMM1l, XMM1m, XMM1n, XMM1o, XMM1p);
1077 
1078 reg_class xmm2_reg(XMM2, XMM2b, XMM2c, XMM2d);
1079 reg_class ymm2_reg(XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h);
1080 reg_class zmm2_reg(XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h, XMM2i, XMM2j, XMM2k, XMM2l, XMM2m, XMM2n, XMM2o, XMM2p);
1081 
1082 reg_class xmm3_reg(XMM3, XMM3b, XMM3c, XMM3d);
1083 reg_class ymm3_reg(XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h);
1084 reg_class zmm3_reg(XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h, XMM3i, XMM3j, XMM3k, XMM3l, XMM3m, XMM3n, XMM3o, XMM3p);
1085 
1086 reg_class xmm4_reg(XMM4, XMM4b, XMM4c, XMM4d);
1087 reg_class ymm4_reg(XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h);
1088 reg_class zmm4_reg(XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h, XMM4i, XMM4j, XMM4k, XMM4l, XMM4m, XMM4n, XMM4o, XMM4p);
1089 
1090 reg_class xmm5_reg(XMM5, XMM5b, XMM5c, XMM5d);
1091 reg_class ymm5_reg(XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h);
1092 reg_class zmm5_reg(XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h, XMM5i, XMM5j, XMM5k, XMM5l, XMM5m, XMM5n, XMM5o, XMM5p);
1093 
1094 reg_class xmm6_reg(XMM6, XMM6b, XMM6c, XMM6d);
1095 reg_class ymm6_reg(XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h);
1096 reg_class zmm6_reg(XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h, XMM6i, XMM6j, XMM6k, XMM6l, XMM6m, XMM6n, XMM6o, XMM6p);
1097 
1098 reg_class xmm7_reg(XMM7, XMM7b, XMM7c, XMM7d);
1099 reg_class ymm7_reg(XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h);
1100 reg_class zmm7_reg(XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h, XMM7i, XMM7j, XMM7k, XMM7l, XMM7m, XMM7n, XMM7o, XMM7p);
1101 
1102 #ifdef _LP64
1103 
1104 reg_class xmm8_reg(XMM8, XMM8b, XMM8c, XMM8d);
1105 reg_class ymm8_reg(XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h);
1106 reg_class zmm8_reg(XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h, XMM8i, XMM8j, XMM8k, XMM8l, XMM8m, XMM8n, XMM8o, XMM8p);
1107 
1108 reg_class xmm9_reg(XMM9, XMM9b, XMM9c, XMM9d);
1109 reg_class ymm9_reg(XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h);
1110 reg_class zmm9_reg(XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h, XMM9i, XMM9j, XMM9k, XMM9l, XMM9m, XMM9n, XMM9o, XMM9p);
1111 
1112 reg_class xmm10_reg(XMM10, XMM10b, XMM10c, XMM10d);
1113 reg_class ymm10_reg(XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h);
1114 reg_class zmm10_reg(XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h, XMM10i, XMM10j, XMM10k, XMM10l, XMM10m, XMM10n, XMM10o, XMM10p);
1115 
1116 reg_class xmm11_reg(XMM11, XMM11b, XMM11c, XMM11d);
1117 reg_class ymm11_reg(XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h);
1118 reg_class zmm11_reg(XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h, XMM11i, XMM11j, XMM11k, XMM11l, XMM11m, XMM11n, XMM11o, XMM11p);
1119 
1120 reg_class xmm12_reg(XMM12, XMM12b, XMM12c, XMM12d);
1121 reg_class ymm12_reg(XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h);
1122 reg_class zmm12_reg(XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h, XMM12i, XMM12j, XMM12k, XMM12l, XMM12m, XMM12n, XMM12o, XMM12p);
1123 
1124 reg_class xmm13_reg(XMM13, XMM13b, XMM13c, XMM13d);
1125 reg_class ymm13_reg(XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h);
1126 reg_class zmm13_reg(XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h, XMM13i, XMM13j, XMM13k, XMM13l, XMM13m, XMM13n, XMM13o, XMM13p);
1127 
1128 reg_class xmm14_reg(XMM14, XMM14b, XMM14c, XMM14d);
1129 reg_class ymm14_reg(XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h);
1130 reg_class zmm14_reg(XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h, XMM14i, XMM14j, XMM14k, XMM14l, XMM14m, XMM14n, XMM14o, XMM14p);
1131 
1132 reg_class xmm15_reg(XMM15, XMM15b, XMM15c, XMM15d);
1133 reg_class ymm15_reg(XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h);
1134 reg_class zmm15_reg(XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h, XMM15i, XMM15j, XMM15k, XMM15l, XMM15m, XMM15n, XMM15o, XMM15p);
1135 
1136 reg_class xmm16_reg(XMM16, XMM16b, XMM16c, XMM16d);
1137 reg_class ymm16_reg(XMM16, XMM16b, XMM16c, XMM16d, XMM16e, XMM16f, XMM16g, XMM16h);
1138 reg_class zmm16_reg(XMM16, XMM16b, XMM16c, XMM16d, XMM16e, XMM16f, XMM16g, XMM16h, XMM16i, XMM16j, XMM16k, XMM16l, XMM16m, XMM16n, XMM16o, XMM16p);
1139 
1140 reg_class xmm17_reg(XMM17, XMM17b, XMM17c, XMM17d);
1141 reg_class ymm17_reg(XMM17, XMM17b, XMM17c, XMM17d, XMM17e, XMM17f, XMM17g, XMM17h);
1142 reg_class zmm17_reg(XMM17, XMM17b, XMM17c, XMM17d, XMM17e, XMM17f, XMM17g, XMM17h, XMM17i, XMM17j, XMM17k, XMM17l, XMM17m, XMM17n, XMM17o, XMM17p);
1143 
1144 reg_class xmm18_reg(XMM18, XMM18b, XMM18c, XMM18d);
1145 reg_class ymm18_reg(XMM18, XMM18b, XMM18c, XMM18d, XMM18e, XMM18f, XMM18g, XMM18h);
1146 reg_class zmm18_reg(XMM18, XMM18b, XMM18c, XMM18d, XMM18e, XMM18f, XMM18g, XMM18h, XMM18i, XMM18j, XMM18k, XMM18l, XMM18m, XMM18n, XMM18o, XMM18p);
1147 
1148 reg_class xmm19_reg(XMM19, XMM19b, XMM19c, XMM19d);
1149 reg_class ymm19_reg(XMM19, XMM19b, XMM19c, XMM19d, XMM19e, XMM19f, XMM19g, XMM19h);
1150 reg_class zmm19_reg(XMM19, XMM19b, XMM19c, XMM19d, XMM19e, XMM19f, XMM19g, XMM19h, XMM19i, XMM19j, XMM19k, XMM19l, XMM19m, XMM19n, XMM19o, XMM19p);
1151 
1152 reg_class xmm20_reg(XMM20, XMM20b, XMM20c, XMM20d);
1153 reg_class ymm20_reg(XMM20, XMM20b, XMM20c, XMM20d, XMM20e, XMM20f, XMM20g, XMM20h);
1154 reg_class zmm20_reg(XMM20, XMM20b, XMM20c, XMM20d, XMM20e, XMM20f, XMM20g, XMM20h, XMM20i, XMM20j, XMM20k, XMM20l, XMM20m, XMM20n, XMM20o, XMM20p);
1155 
1156 reg_class xmm21_reg(XMM21, XMM21b, XMM21c, XMM21d);
1157 reg_class ymm21_reg(XMM21, XMM21b, XMM21c, XMM21d, XMM21e, XMM21f, XMM21g, XMM21h);
1158 reg_class zmm21_reg(XMM21, XMM21b, XMM21c, XMM21d, XMM21e, XMM21f, XMM21g, XMM21h, XMM21i, XMM21j, XMM21k, XMM21l, XMM21m, XMM21n, XMM21o, XMM21p);
1159 
1160 reg_class xmm22_reg(XMM22, XMM22b, XMM22c, XMM22d);
1161 reg_class ymm22_reg(XMM22, XMM22b, XMM22c, XMM22d, XMM22e, XMM22f, XMM22g, XMM22h);
1162 reg_class zmm22_reg(XMM22, XMM22b, XMM22c, XMM22d, XMM22e, XMM22f, XMM22g, XMM22h, XMM22i, XMM22j, XMM22k, XMM22l, XMM22m, XMM22n, XMM22o, XMM22p);
1163 
1164 reg_class xmm23_reg(XMM23, XMM23b, XMM23c, XMM23d);
1165 reg_class ymm23_reg(XMM23, XMM23b, XMM23c, XMM23d, XMM23e, XMM23f, XMM23g, XMM23h);
1166 reg_class zmm23_reg(XMM23, XMM23b, XMM23c, XMM23d, XMM23e, XMM23f, XMM23g, XMM23h, XMM23i, XMM23j, XMM23k, XMM23l, XMM23m, XMM23n, XMM23o, XMM23p);
1167 
1168 reg_class xmm24_reg(XMM24, XMM24b, XMM24c, XMM24d);
1169 reg_class ymm24_reg(XMM24, XMM24b, XMM24c, XMM24d, XMM24e, XMM24f, XMM24g, XMM24h);
1170 reg_class zmm24_reg(XMM24, XMM24b, XMM24c, XMM24d, XMM24e, XMM24f, XMM24g, XMM24h, XMM24i, XMM24j, XMM24k, XMM24l, XMM24m, XMM24n, XMM24o, XMM24p);
1171 
1172 reg_class xmm25_reg(XMM25, XMM25b, XMM25c, XMM25d);
1173 reg_class ymm25_reg(XMM25, XMM25b, XMM25c, XMM25d, XMM25e, XMM25f, XMM25g, XMM25h);
1174 reg_class zmm25_reg(XMM25, XMM25b, XMM25c, XMM25d, XMM25e, XMM25f, XMM25g, XMM25h, XMM25i, XMM25j, XMM25k, XMM25l, XMM25m, XMM25n, XMM25o, XMM25p);
1175 
1176 reg_class xmm26_reg(XMM26, XMM26b, XMM26c, XMM26d);
1177 reg_class ymm26_reg(XMM26, XMM26b, XMM26c, XMM26d, XMM26e, XMM26f, XMM26g, XMM26h);
1178 reg_class zmm26_reg(XMM26, XMM26b, XMM26c, XMM26d, XMM26e, XMM26f, XMM26g, XMM26h, XMM26i, XMM26j, XMM26k, XMM26l, XMM26m, XMM26n, XMM26o, XMM26p);
1179 
1180 reg_class xmm27_reg(XMM27, XMM27b, XMM27c, XMM27d);
1181 reg_class ymm27_reg(XMM27, XMM27b, XMM27c, XMM27d, XMM27e, XMM27f, XMM27g, XMM27h);
1182 reg_class zmm27_reg(XMM27, XMM27b, XMM27c, XMM27d, XMM27e, XMM27f, XMM27g, XMM27h, XMM27i, XMM27j, XMM27k, XMM27l, XMM27m, XMM27n, XMM27o, XMM27p);
1183 
1184 reg_class xmm28_reg(XMM28, XMM28b, XMM28c, XMM28d);
1185 reg_class ymm28_reg(XMM28, XMM28b, XMM28c, XMM28d, XMM28e, XMM28f, XMM28g, XMM28h);
1186 reg_class zmm28_reg(XMM28, XMM28b, XMM28c, XMM28d, XMM28e, XMM28f, XMM28g, XMM28h, XMM28i, XMM28j, XMM28k, XMM28l, XMM28m, XMM28n, XMM28o, XMM28p);
1187 
1188 reg_class xmm29_reg(XMM29, XMM29b, XMM29c, XMM29d);
1189 reg_class ymm29_reg(XMM29, XMM29b, XMM29c, XMM29d, XMM29e, XMM29f, XMM29g, XMM29h);
1190 reg_class zmm29_reg(XMM29, XMM29b, XMM29c, XMM29d, XMM29e, XMM29f, XMM29g, XMM29h, XMM29i, XMM29j, XMM29k, XMM29l, XMM29m, XMM29n, XMM29o, XMM29p);
1191 
1192 reg_class xmm30_reg(XMM30, XMM30b, XMM30c, XMM30d);
1193 reg_class ymm30_reg(XMM30, XMM30b, XMM30c, XMM30d, XMM30e, XMM30f, XMM30g, XMM30h);
1194 reg_class zmm30_reg(XMM30, XMM30b, XMM30c, XMM30d, XMM30e, XMM30f, XMM30g, XMM30h, XMM30i, XMM30j, XMM30k, XMM30l, XMM30m, XMM30n, XMM30o, XMM30p);
1195 
1196 reg_class xmm31_reg(XMM31, XMM31b, XMM31c, XMM31d);
1197 reg_class ymm31_reg(XMM31, XMM31b, XMM31c, XMM31d, XMM31e, XMM31f, XMM31g, XMM31h);
1198 reg_class zmm31_reg(XMM31, XMM31b, XMM31c, XMM31d, XMM31e, XMM31f, XMM31g, XMM31h, XMM31i, XMM31j, XMM31k, XMM31l, XMM31m, XMM31n, XMM31o, XMM31p);
1199 
1200 #endif
1201 
1202 %}
1203 
1204 
1205 //----------SOURCE BLOCK-------------------------------------------------------
1206 // This is a block of C++ code which provides values, functions, and
1207 // definitions necessary in the rest of the architecture description
1208 
1209 source_hpp %{
1210 // Header information of the source block.
1211 // Method declarations/definitions which are used outside
1212 // the ad-scope can conveniently be defined here.
1213 //
1214 // To keep related declarations/definitions/uses close together,
1215 // we switch between source %{ }% and source_hpp %{ }% freely as needed.
1216 
1217 class NativeJump;
1218 
1219 class CallStubImpl {
1220 
1221   //--------------------------------------------------------------
1222   //---<  Used for optimization in Compile::shorten_branches  >---
1223   //--------------------------------------------------------------
1224 
1225  public:
1226   // Size of call trampoline stub.
1227   static uint size_call_trampoline() {
1228     return 0; // no call trampolines on this platform
1229   }
1230 
1231   // number of relocations needed by a call trampoline stub
1232   static uint reloc_call_trampoline() {
1233     return 0; // no call trampolines on this platform
1234   }
1235 };
1236 
1237 class HandlerImpl {
1238 
1239  public:
1240 
1241   static int emit_exception_handler(CodeBuffer &cbuf);
1242   static int emit_deopt_handler(CodeBuffer& cbuf);
1243 
1244   static uint size_exception_handler() {
1245     // NativeCall instruction size is the same as NativeJump.
1246     // exception handler starts out as jump and can be patched to
1247     // a call be deoptimization.  (4932387)
1248     // Note that this value is also credited (in output.cpp) to
1249     // the size of the code section.
1250     return NativeJump::instruction_size;
1251   }
1252 
1253 #ifdef _LP64
1254   static uint size_deopt_handler() {
1255     // three 5 byte instructions plus one move for unreachable address.
1256     return 15+3;
1257   }
1258 #else
1259   static uint size_deopt_handler() {
1260     // NativeCall instruction size is the same as NativeJump.
1261     // exception handler starts out as jump and can be patched to
1262     // a call be deoptimization.  (4932387)
1263     // Note that this value is also credited (in output.cpp) to
1264     // the size of the code section.
1265     return 5 + NativeJump::instruction_size; // pushl(); jmp;
1266   }
1267 #endif
1268 };
1269 
1270 %} // end source_hpp
1271 
1272 source %{
1273 
1274 #include "opto/addnode.hpp"
1275 
1276 // Emit exception handler code.
1277 // Stuff framesize into a register and call a VM stub routine.
1278 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf) {
1279 
1280   // Note that the code buffer's insts_mark is always relative to insts.
1281   // That's why we must use the macroassembler to generate a handler.
1282   MacroAssembler _masm(&cbuf);
1283   address base = __ start_a_stub(size_exception_handler());
1284   if (base == NULL) {
1285     ciEnv::current()->record_failure("CodeCache is full");
1286     return 0;  // CodeBuffer::expand failed
1287   }
1288   int offset = __ offset();
1289   __ jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
1290   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
1291   __ end_a_stub();
1292   return offset;
1293 }
1294 
1295 // Emit deopt handler code.
1296 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf) {
1297 
1298   // Note that the code buffer's insts_mark is always relative to insts.
1299   // That's why we must use the macroassembler to generate a handler.
1300   MacroAssembler _masm(&cbuf);
1301   address base = __ start_a_stub(size_deopt_handler());
1302   if (base == NULL) {
1303     ciEnv::current()->record_failure("CodeCache is full");
1304     return 0;  // CodeBuffer::expand failed
1305   }
1306   int offset = __ offset();
1307 
1308 #ifdef _LP64
1309   address the_pc = (address) __ pc();
1310   Label next;
1311   // push a "the_pc" on the stack without destroying any registers
1312   // as they all may be live.
1313 
1314   // push address of "next"
1315   __ call(next, relocInfo::none); // reloc none is fine since it is a disp32
1316   __ bind(next);
1317   // adjust it so it matches "the_pc"
1318   __ subptr(Address(rsp, 0), __ offset() - offset);
1319 #else
1320   InternalAddress here(__ pc());
1321   __ pushptr(here.addr());
1322 #endif
1323 
1324   __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
1325   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow %d", (__ offset() - offset));
1326   __ end_a_stub();
1327   return offset;
1328 }
1329 
1330 
1331 //=============================================================================
1332 
1333   // Float masks come from different places depending on platform.
1334 #ifdef _LP64
1335   static address float_signmask()  { return StubRoutines::x86::float_sign_mask(); }
1336   static address float_signflip()  { return StubRoutines::x86::float_sign_flip(); }
1337   static address double_signmask() { return StubRoutines::x86::double_sign_mask(); }
1338   static address double_signflip() { return StubRoutines::x86::double_sign_flip(); }
1339   static address vector_float_signmask() { return StubRoutines::x86::vector_float_sign_mask(); }
1340   static address vector_float_signflip() { return StubRoutines::x86::vector_float_sign_flip(); }
1341   static address vector_double_signmask() { return StubRoutines::x86::vector_double_sign_mask(); }
1342   static address vector_double_signflip() { return StubRoutines::x86::vector_double_sign_flip(); }
1343   static address vector_all_bits_set() { return StubRoutines::x86::vector_all_bits_set(); }
1344   static address vector_byte_bitset() { return StubRoutines::x86::vector_byte_bitset(); }
1345   static address vector_long_perm_mask() { return StubRoutines::x86::vector_long_perm_mask(); }
1346   static address vector_short_to_byte_mask() { return StubRoutines::x86::vector_short_to_byte_mask(); }
1347   static address vector_int_to_byte_mask() { return StubRoutines::x86::vector_int_to_byte_mask(); }
1348   static address vector_int_to_short_mask() { return StubRoutines::x86::vector_int_to_short_mask(); }
1349   static address vector_32_bit_mask() { return StubRoutines::x86::vector_32_bit_mask(); }
1350   static address vector_64_bit_mask() { return StubRoutines::x86::vector_64_bit_mask(); }
1351   static address vector_all_ones_mask() { return StubRoutines::x86::vector_all_ones_mask(); }
1352   static address vector_int_shufflemask() { return StubRoutines::x86::vector_int_shuffle_mask(); }
1353   static address vector_int_sizemask() { return StubRoutines::x86::vector_int_size_mask(); }
1354   static address vector_short_shufflemask() { return StubRoutines::x86::vector_short_shuffle_mask(); }
1355   static address vector_short_sizemask() { return StubRoutines::x86::vector_short_size_mask(); }
1356   static address vector_long_shufflemask() { return StubRoutines::x86::vector_long_shuffle_mask(); }
1357   static address vector_long_sizemask() { return StubRoutines::x86::vector_long_size_mask(); }
1358 #else
1359   static address float_signmask()  { return (address)float_signmask_pool; }
1360   static address float_signflip()  { return (address)float_signflip_pool; }
1361   static address double_signmask() { return (address)double_signmask_pool; }
1362   static address double_signflip() { return (address)double_signflip_pool; }
1363 #endif
1364 
1365 
1366 const bool Matcher::match_rule_supported(int opcode) {
1367   if (!has_match_rule(opcode))
1368     return false;
1369 
1370   bool ret_value = true;
1371   switch (opcode) {
1372     case Op_PopCountI:
1373     case Op_PopCountL:
1374       if (!UsePopCountInstruction)
1375         ret_value = false;
1376       break;
1377     case Op_PopCountVI:
1378       if (!UsePopCountInstruction || !VM_Version::supports_vpopcntdq())
1379         ret_value = false;
1380       break;
1381     case Op_MulVI:
1382     case Op_MulVL:
1383       if ((UseSSE < 4) && (UseAVX < 1)) // only with SSE4_1 or AVX
1384         ret_value = false;
1385       break;
1386     case Op_MulReductionVL:
1387       if (VM_Version::supports_avx512dq() == false)
1388         ret_value = false;
1389       break;
1390     case Op_AddReductionVL:
1391       if (UseAVX < 3) // only EVEX : vector connectivity becomes an issue here
1392         ret_value = false;
1393       break;
1394     case Op_MulReductionVI:
1395       if (UseSSE < 4) // requires at least SSE4
1396         ret_value = false;
1397       break;
1398     case Op_AddReductionVF:
1399     case Op_AddReductionVD:
1400     case Op_MulReductionVF:
1401     case Op_MulReductionVD:
1402       if (UseSSE < 1) // requires at least SSE
1403         ret_value = false;
1404       break;
1405     case Op_SqrtVD:
1406     case Op_SqrtVF:
1407       if (UseAVX < 1) // enabled for AVX only
1408         ret_value = false;
1409       break;
1410     case Op_CompareAndSwapL:
1411 #ifdef _LP64
1412     case Op_CompareAndSwapP:
1413 #endif
1414       if (!VM_Version::supports_cx8())
1415         ret_value = false;
1416       break;
1417     case Op_CMoveVF:
1418     case Op_CMoveVD:
1419       if (UseAVX < 1 || UseAVX > 2)
1420         ret_value = false;
1421       break;
1422     case Op_StrIndexOf:
1423       if (!UseSSE42Intrinsics)
1424         ret_value = false;
1425       break;
1426     case Op_StrIndexOfChar:
1427       if (!UseSSE42Intrinsics)
1428         ret_value = false;
1429       break;
1430     case Op_OnSpinWait:
1431       if (VM_Version::supports_on_spin_wait() == false)
1432         ret_value = false;
1433       break;
1434   }
1435 
1436   return ret_value;  // Per default match rules are supported.
1437 }
1438 
1439 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt, int op_arity) {
1440   // identify extra cases that we might want to provide match rules for
1441   // e.g. Op_ vector nodes and other intrinsics while guarding with vlen
1442   bool ret_value = match_rule_supported(opcode);
1443   if (ret_value) {
1444     int size_in_bits = vlen * type2aelembytes(bt) * BitsPerByte;
1445     if (!vector_size_supported(bt, vlen)) {
1446       ret_value = false;
1447     } else if (size_in_bits > 256 && UseAVX <= 2) {
1448       // Only AVX512 supports 512-bit vectors
1449       ret_value = false;
1450     } else if (UseAVX == 0 && size_in_bits > 128) {
1451       // Only AVX supports 256-bit vectors
1452       ret_value = false;
1453     } else if (is_subword_type(bt) && size_in_bits == 512 && VM_Version::supports_avx512bw() == false) {
1454       // Byte and Short types are not supported in AVX512 if AVX512BW is not true.
1455       ret_value = false;
1456     } else {
1457         switch (opcode) {
1458         case Op_AbsV:
1459           if (is_integral_type(bt) && UseSSE < 3) { ret_value = false; }
1460           else if (is_integral_type(bt) && size_in_bits == 256 && UseAVX < 2) { ret_value = false; }
1461           else if (bt == T_LONG && UseAVX <= 2) { ret_value = false; } // Implementation limitation
1462           break;
1463         case Op_AddVB:
1464         case Op_SubVB:
1465           if ((vlen == 64) && (VM_Version::supports_avx512bw() == false))
1466             ret_value = false;
1467           break;
1468         case Op_MaxV:
1469         case Op_MinV:
1470           if (UseSSE < 4 && (bt == T_BYTE || bt == T_INT || bt == T_LONG))
1471             ret_value = false;
1472           break;
1473         case Op_MulVB:
1474           if (size_in_bits <= 128 && UseSSE < 4) { ret_value = false; }
1475           else if (size_in_bits > 256 && UseAVX < 2) { ret_value = false; }
1476           break;
1477         case Op_LShiftVI:
1478         case Op_RShiftVI:
1479         case Op_URShiftVI:
1480           if (op_arity == 2 && UseAVX <= 1)
1481             ret_value  = false;
1482           break;
1483         case Op_LShiftVL:
1484         case Op_RShiftVL:
1485         case Op_URShiftVL:
1486           if (op_arity == 2 && UseAVX <= 1)
1487             ret_value  = false;
1488           break;
1489         case Op_URShiftVS:
1490         case Op_RShiftVS:
1491         case Op_LShiftVS:
1492         case Op_MulVS:
1493         case Op_AddVS:
1494         case Op_SubVS:
1495           if ((vlen == 32) && (VM_Version::supports_avx512bw() == false))
1496             ret_value = false;
1497           break;
1498         case Op_CMoveVF:
1499           if (vlen != 8)
1500             ret_value  = false;
1501           break;
1502         case Op_CMoveVD:
1503           if (vlen != 4)
1504             ret_value  = false;
1505           break;
1506         case Op_AddReductionVI:
1507           if (bt == T_INT && UseSSE < 3) { ret_value = false; }
1508           else if (is_subword_type(bt) && UseSSE <= 3) { ret_value = false; }
1509           break;
1510         case Op_AndReductionV:
1511         case Op_OrReductionV:
1512         case Op_XorReductionV:
1513           if (bt == T_BYTE && UseSSE <= 3) { ret_value = false; }
1514           break;
1515         case Op_VectorMaskCmp:
1516           if (UseAVX <= 0) { ret_value = false; }
1517           else if (is_integral_type(bt) && size_in_bits == 256 && UseAVX < 2) { ret_value = false; }
1518           break;
1519         case Op_MinReductionV:
1520         case Op_MaxReductionV:
1521           if ((bt == T_INT || bt == T_LONG || bt == T_BYTE) && UseSSE <= 3) { ret_value = false; }
1522           else if (is_integral_type(bt) && size_in_bits == 256 && UseAVX < 2) { ret_value = false; }
1523           break;
1524         case Op_VectorBlend:
1525           if (UseSSE <= 3 && UseAVX == 0) { ret_value = false; }
1526           else if (is_integral_type(bt) && size_in_bits == 256 && UseAVX < 2) { ret_value = false; }
1527           break;
1528         case Op_VectorTest:
1529           if (UseAVX <= 0) { ret_value = false; }
1530           else if (size_in_bits != 128 && size_in_bits != 256) { ret_value = false; } // Implementation limitation
1531           break;
1532         case Op_VectorLoadMask:
1533           if (UseSSE <= 3) { ret_value = false; }
1534           else if (vlen == 1 || vlen == 2) { ret_value = false; } // Implementation limitation
1535           else if (size_in_bits >= 256 && UseAVX < 2) { ret_value = false; } // Implementation limitation
1536           break;
1537         case Op_VectorLoadShuffle:
1538         case Op_VectorRearrange:
1539           if (vlen == 1 || vlen == 2) { ret_value = false; } // Implementation limitation due to how shuffle is loaded
1540           else if (size_in_bits >= 256 && UseAVX < 2) { ret_value = false; } // Implementation limitation
1541           else if (bt == T_BYTE && size_in_bits >= 256 && !VM_Version::supports_avx512vbmi())  { ret_value = false; } // Implementation limitation
1542           else if (bt == T_SHORT && size_in_bits >= 256 && !VM_Version::supports_avx512vlbw())  { ret_value = false; } // Implementation limitation
1543           break;
1544         case Op_VectorStoreMask:
1545           if (UseAVX < 0) { ret_value = false; } // Implementation limitation
1546           else if ((size_in_bits >= 256 || bt == T_LONG || bt == T_DOUBLE) && UseAVX < 2) { ret_value = false; } // Implementation limitation
1547           else if (vlen == 1 || vlen == 2) { ret_value = false; } // Implementation limitation
1548           else if (size_in_bits == 512 && !VM_Version::supports_avx512bw()) { ret_value = false; } // Implementation limitation
1549           break;
1550         case Op_VectorCastB2X:
1551           if (UseAVX <= 0) { ret_value = false; }
1552           else if (size_in_bits >= 256 && UseAVX < 2) { ret_value = false; }
1553           break;
1554         case Op_VectorCastS2X:
1555           if (UseAVX <= 0) { ret_value = false; }
1556           else if (is_integral_type(bt) && size_in_bits == 256 && UseAVX < 2) { ret_value = false; }
1557           else if (is_integral_type(bt) && vlen * type2aelembytes(T_SHORT) * BitsPerByte == 256 && UseAVX < 2) { ret_value = false; }
1558           break;
1559         case Op_VectorCastI2X:
1560           if (UseAVX <= 0) { ret_value = false; }
1561           else if (is_integral_type(bt) && size_in_bits == 256 && UseAVX < 2) { ret_value = false; }
1562           else if (is_integral_type(bt) && vlen * type2aelembytes(T_INT) * BitsPerByte == 256 && UseAVX < 2) { ret_value = false; }
1563           break;
1564         case Op_VectorCastL2X:
1565           if (UseAVX <= 0) { ret_value = false; }
1566           else if (is_integral_type(bt) && size_in_bits == 256 && UseAVX < 2) { ret_value = false; }
1567           else if (is_integral_type(bt) && vlen * type2aelembytes(T_LONG) * BitsPerByte == 256 && UseAVX < 2) { ret_value = false; }
1568           else if (!is_integral_type(bt) && !VM_Version::supports_avx512dq()) { ret_value = false; }
1569           break;
1570         case Op_VectorCastF2X:
1571           // Casts from FP to integral types require special fixup logic not easily
1572           // implementable with vectors.
1573           if (UseAVX <= 0) { ret_value = false; }
1574           else if (bt != T_DOUBLE) { ret_value = false; } // Implementation limitation
1575           break;
1576         case Op_VectorCastD2X:
1577           // Casts from FP to integral types require special fixup logic not easily
1578           // implementable with vectors.
1579           if (UseAVX <= 0) { ret_value = false; }
1580           else if (bt != T_FLOAT) { ret_value = false; } // Implementation limitation
1581           break;
1582         case Op_VectorReinterpret:
1583           if (size_in_bits >= 256 && UseAVX < 2) { ret_value = false; }
1584           break;
1585         case Op_MulReductionVI:
1586           if (bt ==T_BYTE && size_in_bits == 512 && !VM_Version::supports_avx512bw()) { ret_value = false; }
1587           break;
1588         default:
1589           break;
1590       }
1591     }
1592   }
1593   if (ret_value) {
1594     assert(is_java_primitive(bt) && (vlen > 0) && is_power_of_2(vlen) &&
1595            vector_size_supported(bt, vlen), "must be supported");
1596   }
1597 
1598   return ret_value;  // Per default match rules are supported.
1599 }
1600 
1601 const bool Matcher::has_predicated_vectors(void) {
1602   bool ret_value = false;
1603   if (UseAVX > 2) {
1604     ret_value = VM_Version::supports_avx512vl();
1605   }
1606 
1607   return ret_value;
1608 }
1609 
1610 const int Matcher::float_pressure(int default_pressure_threshold) {
1611   int float_pressure_threshold = default_pressure_threshold;
1612 #ifdef _LP64
1613   if (UseAVX > 2) {
1614     // Increase pressure threshold on machines with AVX3 which have
1615     // 2x more XMM registers.
1616     float_pressure_threshold = default_pressure_threshold * 2;
1617   }
1618 #endif
1619   return float_pressure_threshold;
1620 }
1621 
1622 // Max vector size in bytes. 0 if not supported.
1623 const int Matcher::vector_width_in_bytes(BasicType bt) {
1624   assert(is_java_primitive(bt), "only primitive type vectors");
1625   if (UseSSE < 2) return 0;
1626   // SSE2 supports 128bit vectors for all types.
1627   // AVX2 supports 256bit vectors for all types.
1628   // AVX2/EVEX supports 512bit vectors for all types.
1629   int size = (UseAVX > 1) ? (1 << UseAVX) * 8 : 16;
1630   // AVX1 supports 256bit vectors only for FLOAT and DOUBLE.
1631   if (UseAVX > 0 && (bt == T_FLOAT || bt == T_DOUBLE))
1632     size = (UseAVX > 2) ? 64 : 32;
1633   // Use flag to limit vector size.
1634   size = MIN2(size,(int)MaxVectorSize);
1635   // Minimum 2 values in vector (or 4 for bytes).
1636   switch (bt) {
1637   case T_DOUBLE:
1638   case T_LONG:
1639     if (size < 16) return 0;
1640     break;
1641   case T_FLOAT:
1642   case T_INT:
1643     if (size < 8) return 0;
1644     break;
1645   case T_BOOLEAN:
1646     if (size < 4) return 0;
1647     break;
1648   case T_CHAR:
1649     if (size < 4) return 0;
1650     break;
1651   case T_BYTE:
1652     if (size < 4) return 0;
1653     break;
1654   case T_SHORT:
1655     if (size < 4) return 0;
1656     break;
1657   default:
1658     ShouldNotReachHere();
1659   }
1660   return size;
1661 }
1662 
1663 // Limits on vector size (number of elements) loaded into vector.
1664 const int Matcher::max_vector_size(const BasicType bt) {
1665   return vector_width_in_bytes(bt)/type2aelembytes(bt);
1666 }
1667 const int Matcher::min_vector_size(const BasicType bt) {
1668   int max_size = max_vector_size(bt);
1669   // Min size which can be loaded into vector is 4 bytes.
1670   int size = (type2aelembytes(bt) == 1) ? 4 : 2;
1671   return MIN2(size,max_size);
1672 }
1673 
1674 // Vector ideal reg corresponding to specidied size in bytes
1675 const uint Matcher::vector_ideal_reg(int size) {
1676   assert(MaxVectorSize >= size, "");
1677   switch(size) {
1678     case  4: return Op_VecS;
1679     case  8: return Op_VecD;
1680     case 16: return Op_VecX;
1681     case 32: return Op_VecY;
1682     case 64: return Op_VecZ;
1683   }
1684   ShouldNotReachHere();
1685   return 0;
1686 }
1687 
1688 // Only lowest bits of xmm reg are used for vector shift count.
1689 const uint Matcher::vector_shift_count_ideal_reg(int size) {
1690   return Op_VecS;
1691 }
1692 
1693 // x86 supports misaligned vectors store/load.
1694 const bool Matcher::misaligned_vectors_ok() {
1695   return !AlignVector; // can be changed by flag
1696 }
1697 
1698 // x86 AES instructions are compatible with SunJCE expanded
1699 // keys, hence we do not need to pass the original key to stubs
1700 const bool Matcher::pass_original_key_for_aes() {
1701   return false;
1702 }
1703 
1704 
1705 const bool Matcher::convi2l_type_required = true;
1706 
1707 // Check for shift by small constant as well
1708 static bool clone_shift(Node* shift, Matcher* matcher, Matcher::MStack& mstack, VectorSet& address_visited) {
1709   if (shift->Opcode() == Op_LShiftX && shift->in(2)->is_Con() &&
1710       shift->in(2)->get_int() <= 3 &&
1711       // Are there other uses besides address expressions?
1712       !matcher->is_visited(shift)) {
1713     address_visited.set(shift->_idx); // Flag as address_visited
1714     mstack.push(shift->in(2), Matcher::Visit);
1715     Node *conv = shift->in(1);
1716 #ifdef _LP64
1717     // Allow Matcher to match the rule which bypass
1718     // ConvI2L operation for an array index on LP64
1719     // if the index value is positive.
1720     if (conv->Opcode() == Op_ConvI2L &&
1721         conv->as_Type()->type()->is_long()->_lo >= 0 &&
1722         // Are there other uses besides address expressions?
1723         !matcher->is_visited(conv)) {
1724       address_visited.set(conv->_idx); // Flag as address_visited
1725       mstack.push(conv->in(1), Matcher::Pre_Visit);
1726     } else
1727 #endif
1728       mstack.push(conv, Matcher::Pre_Visit);
1729     return true;
1730   }
1731   return false;
1732 }
1733 
1734 // Should the Matcher clone shifts on addressing modes, expecting them
1735 // to be subsumed into complex addressing expressions or compute them
1736 // into registers?
1737 bool Matcher::clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
1738   Node *off = m->in(AddPNode::Offset);
1739   if (off->is_Con()) {
1740     address_visited.test_set(m->_idx); // Flag as address_visited
1741     Node *adr = m->in(AddPNode::Address);
1742 
1743     // Intel can handle 2 adds in addressing mode
1744     // AtomicAdd is not an addressing expression.
1745     // Cheap to find it by looking for screwy base.
1746     if (adr->is_AddP() &&
1747         !adr->in(AddPNode::Base)->is_top() &&
1748         // Are there other uses besides address expressions?
1749         !is_visited(adr)) {
1750       address_visited.set(adr->_idx); // Flag as address_visited
1751       Node *shift = adr->in(AddPNode::Offset);
1752       if (!clone_shift(shift, this, mstack, address_visited)) {
1753         mstack.push(shift, Pre_Visit);
1754       }
1755       mstack.push(adr->in(AddPNode::Address), Pre_Visit);
1756       mstack.push(adr->in(AddPNode::Base), Pre_Visit);
1757     } else {
1758       mstack.push(adr, Pre_Visit);
1759     }
1760 
1761     // Clone X+offset as it also folds into most addressing expressions
1762     mstack.push(off, Visit);
1763     mstack.push(m->in(AddPNode::Base), Pre_Visit);
1764     return true;
1765   } else if (clone_shift(off, this, mstack, address_visited)) {
1766     address_visited.test_set(m->_idx); // Flag as address_visited
1767     mstack.push(m->in(AddPNode::Address), Pre_Visit);
1768     mstack.push(m->in(AddPNode::Base), Pre_Visit);
1769     return true;
1770   }
1771   return false;
1772 }
1773 
1774 void Compile::reshape_address(AddPNode* addp) {
1775 }
1776 
1777 // Helper methods for MachSpillCopyNode::implementation().
1778 static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
1779                           int src_hi, int dst_hi, uint ireg, outputStream* st) {
1780   // In 64-bit VM size calculation is very complex. Emitting instructions
1781   // into scratch buffer is used to get size in 64-bit VM.
1782   LP64_ONLY( assert(!do_size, "this method calculates size only for 32-bit VM"); )
1783   assert(ireg == Op_VecS || // 32bit vector
1784          (src_lo & 1) == 0 && (src_lo + 1) == src_hi &&
1785          (dst_lo & 1) == 0 && (dst_lo + 1) == dst_hi,
1786          "no non-adjacent vector moves" );
1787   if (cbuf) {
1788     MacroAssembler _masm(cbuf);
1789     int offset = __ offset();
1790     switch (ireg) {
1791     case Op_VecS: // copy whole register
1792     case Op_VecD:
1793     case Op_VecX:
1794       __ movdqu(as_XMMRegister(Matcher::_regEncode[dst_lo]), as_XMMRegister(Matcher::_regEncode[src_lo]));
1795       break;
1796     case Op_VecY:
1797       __ vmovdqu(as_XMMRegister(Matcher::_regEncode[dst_lo]), as_XMMRegister(Matcher::_regEncode[src_lo]));
1798       break;
1799     case Op_VecZ:
1800       __ evmovdquq(as_XMMRegister(Matcher::_regEncode[dst_lo]), as_XMMRegister(Matcher::_regEncode[src_lo]), 2);
1801       break;
1802     default:
1803       ShouldNotReachHere();
1804     }
1805     int size = __ offset() - offset;
1806 #ifdef ASSERT
1807     // VEX_2bytes prefix is used if UseAVX > 0, so it takes the same 2 bytes as SIMD prefix.
1808     assert(!do_size || size == 4, "incorrect size calculattion");
1809 #endif
1810     return size;
1811 #ifndef PRODUCT
1812   } else if (!do_size) {
1813     switch (ireg) {
1814     case Op_VecS:
1815     case Op_VecD:
1816     case Op_VecX:
1817       st->print("movdqu  %s,%s\t# spill",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
1818       break;
1819     case Op_VecY:
1820     case Op_VecZ:
1821       st->print("vmovdqu %s,%s\t# spill",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
1822       break;
1823     default:
1824       ShouldNotReachHere();
1825     }
1826 #endif
1827   }
1828   // VEX_2bytes prefix is used if UseAVX > 0, and it takes the same 2 bytes as SIMD prefix.
1829   return (UseAVX > 2) ? 6 : 4;
1830 }
1831 
1832 static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
1833                             int stack_offset, int reg, uint ireg, outputStream* st) {
1834   // In 64-bit VM size calculation is very complex. Emitting instructions
1835   // into scratch buffer is used to get size in 64-bit VM.
1836   LP64_ONLY( assert(!do_size, "this method calculates size only for 32-bit VM"); )
1837   if (cbuf) {
1838     MacroAssembler _masm(cbuf);
1839     int offset = __ offset();
1840     if (is_load) {
1841       switch (ireg) {
1842       case Op_VecS:
1843         __ movdl(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset));
1844         break;
1845       case Op_VecD:
1846         __ movq(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset));
1847         break;
1848       case Op_VecX:
1849         __ movdqu(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset));
1850         break;
1851       case Op_VecY:
1852         __ vmovdqu(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset));
1853         break;
1854       case Op_VecZ:
1855         __ evmovdquq(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset), 2);
1856         break;
1857       default:
1858         ShouldNotReachHere();
1859       }
1860     } else { // store
1861       switch (ireg) {
1862       case Op_VecS:
1863         __ movdl(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]));
1864         break;
1865       case Op_VecD:
1866         __ movq(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]));
1867         break;
1868       case Op_VecX:
1869         __ movdqu(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]));
1870         break;
1871       case Op_VecY:
1872         __ vmovdqu(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]));
1873         break;
1874       case Op_VecZ:
1875         __ evmovdquq(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]), 2);
1876         break;
1877       default:
1878         ShouldNotReachHere();
1879       }
1880     }
1881     int size = __ offset() - offset;
1882 #ifdef ASSERT
1883     int offset_size = (stack_offset == 0) ? 0 : ((stack_offset < 0x80) ? 1 : (UseAVX > 2) ? 6 : 4);
1884     // VEX_2bytes prefix is used if UseAVX > 0, so it takes the same 2 bytes as SIMD prefix.
1885     assert(!do_size || size == (5+offset_size), "incorrect size calculattion");
1886 #endif
1887     return size;
1888 #ifndef PRODUCT
1889   } else if (!do_size) {
1890     if (is_load) {
1891       switch (ireg) {
1892       case Op_VecS:
1893         st->print("movd    %s,[rsp + %d]\t# spill", Matcher::regName[reg], stack_offset);
1894         break;
1895       case Op_VecD:
1896         st->print("movq    %s,[rsp + %d]\t# spill", Matcher::regName[reg], stack_offset);
1897         break;
1898        case Op_VecX:
1899         st->print("movdqu  %s,[rsp + %d]\t# spill", Matcher::regName[reg], stack_offset);
1900         break;
1901       case Op_VecY:
1902       case Op_VecZ:
1903         st->print("vmovdqu %s,[rsp + %d]\t# spill", Matcher::regName[reg], stack_offset);
1904         break;
1905       default:
1906         ShouldNotReachHere();
1907       }
1908     } else { // store
1909       switch (ireg) {
1910       case Op_VecS:
1911         st->print("movd    [rsp + %d],%s\t# spill", stack_offset, Matcher::regName[reg]);
1912         break;
1913       case Op_VecD:
1914         st->print("movq    [rsp + %d],%s\t# spill", stack_offset, Matcher::regName[reg]);
1915         break;
1916        case Op_VecX:
1917         st->print("movdqu  [rsp + %d],%s\t# spill", stack_offset, Matcher::regName[reg]);
1918         break;
1919       case Op_VecY:
1920       case Op_VecZ:
1921         st->print("vmovdqu [rsp + %d],%s\t# spill", stack_offset, Matcher::regName[reg]);
1922         break;
1923       default:
1924         ShouldNotReachHere();
1925       }
1926     }
1927 #endif
1928   }
1929   bool is_single_byte = false;
1930   int vec_len = 0;
1931   if ((UseAVX > 2) && (stack_offset != 0)) {
1932     int tuple_type = Assembler::EVEX_FVM;
1933     int input_size = Assembler::EVEX_32bit;
1934     switch (ireg) {
1935     case Op_VecS:
1936       tuple_type = Assembler::EVEX_T1S;
1937       break;
1938     case Op_VecD:
1939       tuple_type = Assembler::EVEX_T1S;
1940       input_size = Assembler::EVEX_64bit;
1941       break;
1942     case Op_VecX:
1943       break;
1944     case Op_VecY:
1945       vec_len = 1;
1946       break;
1947     case Op_VecZ:
1948       vec_len = 2;
1949       break;
1950     }
1951     is_single_byte = Assembler::query_compressed_disp_byte(stack_offset, true, vec_len, tuple_type, input_size, 0);
1952   }
1953   int offset_size = 0;
1954   int size = 5;
1955   if (UseAVX > 2 ) {
1956     if (VM_Version::supports_avx512novl() && (vec_len == 2)) {
1957       offset_size = (stack_offset == 0) ? 0 : ((is_single_byte) ? 1 : 4);
1958       size += 2; // Need an additional two bytes for EVEX encoding
1959     } else if (VM_Version::supports_avx512novl() && (vec_len < 2)) {
1960       offset_size = (stack_offset == 0) ? 0 : ((stack_offset <= 127) ? 1 : 4);
1961     } else {
1962       offset_size = (stack_offset == 0) ? 0 : ((is_single_byte) ? 1 : 4);
1963       size += 2; // Need an additional two bytes for EVEX encodding
1964     }
1965   } else {
1966     offset_size = (stack_offset == 0) ? 0 : ((stack_offset <= 127) ? 1 : 4);
1967   }
1968   // VEX_2bytes prefix is used if UseAVX > 0, so it takes the same 2 bytes as SIMD prefix.
1969   return size+offset_size;
1970 }
1971 
1972 static inline jint replicate4_imm(int con, int width) {
1973   // Load a constant of "width" (in bytes) and replicate it to fill 32bit.
1974   assert(width == 1 || width == 2, "only byte or short types here");
1975   int bit_width = width * 8;
1976   jint val = con;
1977   val &= (1 << bit_width) - 1;  // mask off sign bits
1978   while(bit_width < 32) {
1979     val |= (val << bit_width);
1980     bit_width <<= 1;
1981   }
1982   return val;
1983 }
1984 
1985 static inline jlong replicate8_imm(int con, int width) {
1986   // Load a constant of "width" (in bytes) and replicate it to fill 64bit.
1987   assert(width == 1 || width == 2 || width == 4, "only byte, short or int types here");
1988   int bit_width = width * 8;
1989   jlong val = con;
1990   val &= (((jlong) 1) << bit_width) - 1;  // mask off sign bits
1991   while(bit_width < 64) {
1992     val |= (val << bit_width);
1993     bit_width <<= 1;
1994   }
1995   return val;
1996 }
1997 
1998 #ifndef PRODUCT
1999   void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
2000     st->print("nop \t# %d bytes pad for loops and calls", _count);
2001   }
2002 #endif
2003 
2004   void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
2005     MacroAssembler _masm(&cbuf);
2006     __ nop(_count);
2007   }
2008 
2009   uint MachNopNode::size(PhaseRegAlloc*) const {
2010     return _count;
2011   }
2012 
2013 #ifndef PRODUCT
2014   void MachBreakpointNode::format(PhaseRegAlloc*, outputStream* st) const {
2015     st->print("# breakpoint");
2016   }
2017 #endif
2018 
2019   void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc* ra_) const {
2020     MacroAssembler _masm(&cbuf);
2021     __ int3();
2022   }
2023 
2024   uint MachBreakpointNode::size(PhaseRegAlloc* ra_) const {
2025     return MachNode::size(ra_);
2026   }
2027 
2028 %}
2029 
2030 encode %{
2031 
2032   enc_class call_epilog %{
2033     if (VerifyStackAtCalls) {
2034       // Check that stack depth is unchanged: find majik cookie on stack
2035       int framesize = ra_->reg2offset_unchecked(OptoReg::add(ra_->_matcher._old_SP, -3*VMRegImpl::slots_per_word));
2036       MacroAssembler _masm(&cbuf);
2037       Label L;
2038       __ cmpptr(Address(rsp, framesize), (int32_t)0xbadb100d);
2039       __ jccb(Assembler::equal, L);
2040       // Die if stack mismatch
2041       __ int3();
2042       __ bind(L);
2043     }
2044   %}
2045 
2046 %}
2047 
2048 
2049 //----------OPERANDS-----------------------------------------------------------
2050 // Operand definitions must precede instruction definitions for correct parsing
2051 // in the ADLC because operands constitute user defined types which are used in
2052 // instruction definitions.
2053 
2054 operand immU1() %{
2055   predicate(n->get_int() >= 0 && n->get_int() < nth_bit(1));
2056   match(ConI);
2057 
2058   op_cost(0);
2059   format %{ %}
2060   interface(CONST_INTER);
2061 %}
2062 
2063 operand immU2() %{
2064   predicate(n->get_int() >= 0 && n->get_int() < nth_bit(2));
2065   match(ConI);
2066 
2067   op_cost(0);
2068   format %{ %}
2069   interface(CONST_INTER);
2070 %}
2071 
2072 operand immU3() %{
2073   predicate(n->get_int() >= 0 && n->get_int() < nth_bit(3));
2074   match(ConI);
2075 
2076   op_cost(0);
2077   format %{ %}
2078   interface(CONST_INTER);
2079 %}
2080 
2081 operand immU4() %{
2082   predicate(n->get_int() >= 0 && n->get_int() < nth_bit(4));
2083   match(ConI);
2084 
2085   op_cost(0);
2086   format %{ %}
2087   interface(CONST_INTER);
2088 %}
2089 
2090 operand immU5() %{
2091   predicate(n->get_int() >= 0 && n->get_int() < nth_bit(5));
2092   match(ConI);
2093 
2094   op_cost(0);
2095   format %{ %}
2096   interface(CONST_INTER);
2097 %}
2098 
2099 operand immU6() %{
2100   predicate(n->get_int() >= 0 && n->get_int() < nth_bit(6));
2101   match(ConI);
2102 
2103   op_cost(0);
2104   format %{ %}
2105   interface(CONST_INTER);
2106 %}
2107 
2108 // This one generically applies only for evex, so only one version
2109 operand vecZ() %{
2110   constraint(ALLOC_IN_RC(vectorz_reg));
2111   match(VecZ);
2112 
2113   format %{ %}
2114   interface(REG_INTER);
2115 %}
2116 
2117 // Comparison Code for FP conditional move
2118 operand cmpOp_vcmppd() %{
2119   match(Bool);
2120 
2121   predicate(n->as_Bool()->_test._test != BoolTest::overflow &&
2122             n->as_Bool()->_test._test != BoolTest::no_overflow);
2123   format %{ "" %}
2124   interface(COND_INTER) %{
2125     equal        (0x0, "eq");
2126     less         (0x1, "lt");
2127     less_equal   (0x2, "le");
2128     not_equal    (0xC, "ne");
2129     greater_equal(0xD, "ge");
2130     greater      (0xE, "gt");
2131     //TODO cannot compile (adlc breaks) without two next lines with error:
2132     // x86_64.ad(13987) Syntax Error: :In operand cmpOp_vcmppd: Do not support this encode constant: ' %{
2133     // equal' for overflow.
2134     overflow     (0x20, "o");  // not really supported by the instruction
2135     no_overflow  (0x21, "no"); // not really supported by the instruction
2136   %}
2137 %}
2138 
2139 
2140 // INSTRUCTIONS -- Platform independent definitions (same for 32- and 64-bit)
2141 
2142 // ============================================================================
2143 
2144 instruct ShouldNotReachHere() %{
2145   match(Halt);
2146   format %{ "ud2\t# ShouldNotReachHere" %}
2147   ins_encode %{
2148     __ ud2();
2149   %}
2150   ins_pipe(pipe_slow);
2151 %}
2152 
2153 // =================================EVEX special===============================
2154 
2155 instruct setMask(rRegI dst, rRegI src) %{
2156   predicate(Matcher::has_predicated_vectors());
2157   match(Set dst (SetVectMaskI  src));
2158   effect(TEMP dst);
2159   format %{ "setvectmask   $dst, $src" %}
2160   ins_encode %{
2161     __ setvectmask($dst$$Register, $src$$Register);
2162   %}
2163   ins_pipe(pipe_slow);
2164 %}
2165 
2166 // ============================================================================
2167 
2168 instruct addF_reg(regF dst, regF src) %{
2169   predicate((UseSSE>=1) && (UseAVX == 0));
2170   match(Set dst (AddF dst src));
2171 
2172   format %{ "addss   $dst, $src" %}
2173   ins_cost(150);
2174   ins_encode %{
2175     __ addss($dst$$XMMRegister, $src$$XMMRegister);
2176   %}
2177   ins_pipe(pipe_slow);
2178 %}
2179 
2180 instruct addF_mem(regF dst, memory src) %{
2181   predicate((UseSSE>=1) && (UseAVX == 0));
2182   match(Set dst (AddF dst (LoadF src)));
2183 
2184   format %{ "addss   $dst, $src" %}
2185   ins_cost(150);
2186   ins_encode %{
2187     __ addss($dst$$XMMRegister, $src$$Address);
2188   %}
2189   ins_pipe(pipe_slow);
2190 %}
2191 
2192 instruct addF_imm(regF dst, immF con) %{
2193   predicate((UseSSE>=1) && (UseAVX == 0));
2194   match(Set dst (AddF dst con));
2195   format %{ "addss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
2196   ins_cost(150);
2197   ins_encode %{
2198     __ addss($dst$$XMMRegister, $constantaddress($con));
2199   %}
2200   ins_pipe(pipe_slow);
2201 %}
2202 
2203 instruct addF_reg_reg(regF dst, regF src1, regF src2) %{
2204   predicate(UseAVX > 0);
2205   match(Set dst (AddF src1 src2));
2206 
2207   format %{ "vaddss  $dst, $src1, $src2" %}
2208   ins_cost(150);
2209   ins_encode %{
2210     __ vaddss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
2211   %}
2212   ins_pipe(pipe_slow);
2213 %}
2214 
2215 instruct addF_reg_mem(regF dst, regF src1, memory src2) %{
2216   predicate(UseAVX > 0);
2217   match(Set dst (AddF src1 (LoadF src2)));
2218 
2219   format %{ "vaddss  $dst, $src1, $src2" %}
2220   ins_cost(150);
2221   ins_encode %{
2222     __ vaddss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
2223   %}
2224   ins_pipe(pipe_slow);
2225 %}
2226 
2227 instruct addF_reg_imm(regF dst, regF src, immF con) %{
2228   predicate(UseAVX > 0);
2229   match(Set dst (AddF src con));
2230 
2231   format %{ "vaddss  $dst, $src, [$constantaddress]\t# load from constant table: float=$con" %}
2232   ins_cost(150);
2233   ins_encode %{
2234     __ vaddss($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
2235   %}
2236   ins_pipe(pipe_slow);
2237 %}
2238 
2239 instruct addD_reg(regD dst, regD src) %{
2240   predicate((UseSSE>=2) && (UseAVX == 0));
2241   match(Set dst (AddD dst src));
2242 
2243   format %{ "addsd   $dst, $src" %}
2244   ins_cost(150);
2245   ins_encode %{
2246     __ addsd($dst$$XMMRegister, $src$$XMMRegister);
2247   %}
2248   ins_pipe(pipe_slow);
2249 %}
2250 
2251 instruct addD_mem(regD dst, memory src) %{
2252   predicate((UseSSE>=2) && (UseAVX == 0));
2253   match(Set dst (AddD dst (LoadD src)));
2254 
2255   format %{ "addsd   $dst, $src" %}
2256   ins_cost(150);
2257   ins_encode %{
2258     __ addsd($dst$$XMMRegister, $src$$Address);
2259   %}
2260   ins_pipe(pipe_slow);
2261 %}
2262 
2263 instruct addD_imm(regD dst, immD con) %{
2264   predicate((UseSSE>=2) && (UseAVX == 0));
2265   match(Set dst (AddD dst con));
2266   format %{ "addsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
2267   ins_cost(150);
2268   ins_encode %{
2269     __ addsd($dst$$XMMRegister, $constantaddress($con));
2270   %}
2271   ins_pipe(pipe_slow);
2272 %}
2273 
2274 instruct addD_reg_reg(regD dst, regD src1, regD src2) %{
2275   predicate(UseAVX > 0);
2276   match(Set dst (AddD src1 src2));
2277 
2278   format %{ "vaddsd  $dst, $src1, $src2" %}
2279   ins_cost(150);
2280   ins_encode %{
2281     __ vaddsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
2282   %}
2283   ins_pipe(pipe_slow);
2284 %}
2285 
2286 instruct addD_reg_mem(regD dst, regD src1, memory src2) %{
2287   predicate(UseAVX > 0);
2288   match(Set dst (AddD src1 (LoadD src2)));
2289 
2290   format %{ "vaddsd  $dst, $src1, $src2" %}
2291   ins_cost(150);
2292   ins_encode %{
2293     __ vaddsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
2294   %}
2295   ins_pipe(pipe_slow);
2296 %}
2297 
2298 instruct addD_reg_imm(regD dst, regD src, immD con) %{
2299   predicate(UseAVX > 0);
2300   match(Set dst (AddD src con));
2301 
2302   format %{ "vaddsd  $dst, $src, [$constantaddress]\t# load from constant table: double=$con" %}
2303   ins_cost(150);
2304   ins_encode %{
2305     __ vaddsd($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
2306   %}
2307   ins_pipe(pipe_slow);
2308 %}
2309 
2310 instruct subF_reg(regF dst, regF src) %{
2311   predicate((UseSSE>=1) && (UseAVX == 0));
2312   match(Set dst (SubF dst src));
2313 
2314   format %{ "subss   $dst, $src" %}
2315   ins_cost(150);
2316   ins_encode %{
2317     __ subss($dst$$XMMRegister, $src$$XMMRegister);
2318   %}
2319   ins_pipe(pipe_slow);
2320 %}
2321 
2322 instruct subF_mem(regF dst, memory src) %{
2323   predicate((UseSSE>=1) && (UseAVX == 0));
2324   match(Set dst (SubF dst (LoadF src)));
2325 
2326   format %{ "subss   $dst, $src" %}
2327   ins_cost(150);
2328   ins_encode %{
2329     __ subss($dst$$XMMRegister, $src$$Address);
2330   %}
2331   ins_pipe(pipe_slow);
2332 %}
2333 
2334 instruct subF_imm(regF dst, immF con) %{
2335   predicate((UseSSE>=1) && (UseAVX == 0));
2336   match(Set dst (SubF dst con));
2337   format %{ "subss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
2338   ins_cost(150);
2339   ins_encode %{
2340     __ subss($dst$$XMMRegister, $constantaddress($con));
2341   %}
2342   ins_pipe(pipe_slow);
2343 %}
2344 
2345 instruct subF_reg_reg(regF dst, regF src1, regF src2) %{
2346   predicate(UseAVX > 0);
2347   match(Set dst (SubF src1 src2));
2348 
2349   format %{ "vsubss  $dst, $src1, $src2" %}
2350   ins_cost(150);
2351   ins_encode %{
2352     __ vsubss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
2353   %}
2354   ins_pipe(pipe_slow);
2355 %}
2356 
2357 instruct subF_reg_mem(regF dst, regF src1, memory src2) %{
2358   predicate(UseAVX > 0);
2359   match(Set dst (SubF src1 (LoadF src2)));
2360 
2361   format %{ "vsubss  $dst, $src1, $src2" %}
2362   ins_cost(150);
2363   ins_encode %{
2364     __ vsubss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
2365   %}
2366   ins_pipe(pipe_slow);
2367 %}
2368 
2369 instruct subF_reg_imm(regF dst, regF src, immF con) %{
2370   predicate(UseAVX > 0);
2371   match(Set dst (SubF src con));
2372 
2373   format %{ "vsubss  $dst, $src, [$constantaddress]\t# load from constant table: float=$con" %}
2374   ins_cost(150);
2375   ins_encode %{
2376     __ vsubss($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
2377   %}
2378   ins_pipe(pipe_slow);
2379 %}
2380 
2381 instruct subD_reg(regD dst, regD src) %{
2382   predicate((UseSSE>=2) && (UseAVX == 0));
2383   match(Set dst (SubD dst src));
2384 
2385   format %{ "subsd   $dst, $src" %}
2386   ins_cost(150);
2387   ins_encode %{
2388     __ subsd($dst$$XMMRegister, $src$$XMMRegister);
2389   %}
2390   ins_pipe(pipe_slow);
2391 %}
2392 
2393 instruct subD_mem(regD dst, memory src) %{
2394   predicate((UseSSE>=2) && (UseAVX == 0));
2395   match(Set dst (SubD dst (LoadD src)));
2396 
2397   format %{ "subsd   $dst, $src" %}
2398   ins_cost(150);
2399   ins_encode %{
2400     __ subsd($dst$$XMMRegister, $src$$Address);
2401   %}
2402   ins_pipe(pipe_slow);
2403 %}
2404 
2405 instruct subD_imm(regD dst, immD con) %{
2406   predicate((UseSSE>=2) && (UseAVX == 0));
2407   match(Set dst (SubD dst con));
2408   format %{ "subsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
2409   ins_cost(150);
2410   ins_encode %{
2411     __ subsd($dst$$XMMRegister, $constantaddress($con));
2412   %}
2413   ins_pipe(pipe_slow);
2414 %}
2415 
2416 instruct subD_reg_reg(regD dst, regD src1, regD src2) %{
2417   predicate(UseAVX > 0);
2418   match(Set dst (SubD src1 src2));
2419 
2420   format %{ "vsubsd  $dst, $src1, $src2" %}
2421   ins_cost(150);
2422   ins_encode %{
2423     __ vsubsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
2424   %}
2425   ins_pipe(pipe_slow);
2426 %}
2427 
2428 instruct subD_reg_mem(regD dst, regD src1, memory src2) %{
2429   predicate(UseAVX > 0);
2430   match(Set dst (SubD src1 (LoadD src2)));
2431 
2432   format %{ "vsubsd  $dst, $src1, $src2" %}
2433   ins_cost(150);
2434   ins_encode %{
2435     __ vsubsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
2436   %}
2437   ins_pipe(pipe_slow);
2438 %}
2439 
2440 instruct subD_reg_imm(regD dst, regD src, immD con) %{
2441   predicate(UseAVX > 0);
2442   match(Set dst (SubD src con));
2443 
2444   format %{ "vsubsd  $dst, $src, [$constantaddress]\t# load from constant table: double=$con" %}
2445   ins_cost(150);
2446   ins_encode %{
2447     __ vsubsd($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
2448   %}
2449   ins_pipe(pipe_slow);
2450 %}
2451 
2452 instruct mulF_reg(regF dst, regF src) %{
2453   predicate((UseSSE>=1) && (UseAVX == 0));
2454   match(Set dst (MulF dst src));
2455 
2456   format %{ "mulss   $dst, $src" %}
2457   ins_cost(150);
2458   ins_encode %{
2459     __ mulss($dst$$XMMRegister, $src$$XMMRegister);
2460   %}
2461   ins_pipe(pipe_slow);
2462 %}
2463 
2464 instruct mulF_mem(regF dst, memory src) %{
2465   predicate((UseSSE>=1) && (UseAVX == 0));
2466   match(Set dst (MulF dst (LoadF src)));
2467 
2468   format %{ "mulss   $dst, $src" %}
2469   ins_cost(150);
2470   ins_encode %{
2471     __ mulss($dst$$XMMRegister, $src$$Address);
2472   %}
2473   ins_pipe(pipe_slow);
2474 %}
2475 
2476 instruct mulF_imm(regF dst, immF con) %{
2477   predicate((UseSSE>=1) && (UseAVX == 0));
2478   match(Set dst (MulF dst con));
2479   format %{ "mulss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
2480   ins_cost(150);
2481   ins_encode %{
2482     __ mulss($dst$$XMMRegister, $constantaddress($con));
2483   %}
2484   ins_pipe(pipe_slow);
2485 %}
2486 
2487 instruct mulF_reg_reg(regF dst, regF src1, regF src2) %{
2488   predicate(UseAVX > 0);
2489   match(Set dst (MulF src1 src2));
2490 
2491   format %{ "vmulss  $dst, $src1, $src2" %}
2492   ins_cost(150);
2493   ins_encode %{
2494     __ vmulss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
2495   %}
2496   ins_pipe(pipe_slow);
2497 %}
2498 
2499 instruct mulF_reg_mem(regF dst, regF src1, memory src2) %{
2500   predicate(UseAVX > 0);
2501   match(Set dst (MulF src1 (LoadF src2)));
2502 
2503   format %{ "vmulss  $dst, $src1, $src2" %}
2504   ins_cost(150);
2505   ins_encode %{
2506     __ vmulss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
2507   %}
2508   ins_pipe(pipe_slow);
2509 %}
2510 
2511 instruct mulF_reg_imm(regF dst, regF src, immF con) %{
2512   predicate(UseAVX > 0);
2513   match(Set dst (MulF src con));
2514 
2515   format %{ "vmulss  $dst, $src, [$constantaddress]\t# load from constant table: float=$con" %}
2516   ins_cost(150);
2517   ins_encode %{
2518     __ vmulss($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
2519   %}
2520   ins_pipe(pipe_slow);
2521 %}
2522 
2523 instruct mulD_reg(regD dst, regD src) %{
2524   predicate((UseSSE>=2) && (UseAVX == 0));
2525   match(Set dst (MulD dst src));
2526 
2527   format %{ "mulsd   $dst, $src" %}
2528   ins_cost(150);
2529   ins_encode %{
2530     __ mulsd($dst$$XMMRegister, $src$$XMMRegister);
2531   %}
2532   ins_pipe(pipe_slow);
2533 %}
2534 
2535 instruct mulD_mem(regD dst, memory src) %{
2536   predicate((UseSSE>=2) && (UseAVX == 0));
2537   match(Set dst (MulD dst (LoadD src)));
2538 
2539   format %{ "mulsd   $dst, $src" %}
2540   ins_cost(150);
2541   ins_encode %{
2542     __ mulsd($dst$$XMMRegister, $src$$Address);
2543   %}
2544   ins_pipe(pipe_slow);
2545 %}
2546 
2547 instruct mulD_imm(regD dst, immD con) %{
2548   predicate((UseSSE>=2) && (UseAVX == 0));
2549   match(Set dst (MulD dst con));
2550   format %{ "mulsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
2551   ins_cost(150);
2552   ins_encode %{
2553     __ mulsd($dst$$XMMRegister, $constantaddress($con));
2554   %}
2555   ins_pipe(pipe_slow);
2556 %}
2557 
2558 instruct mulD_reg_reg(regD dst, regD src1, regD src2) %{
2559   predicate(UseAVX > 0);
2560   match(Set dst (MulD src1 src2));
2561 
2562   format %{ "vmulsd  $dst, $src1, $src2" %}
2563   ins_cost(150);
2564   ins_encode %{
2565     __ vmulsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
2566   %}
2567   ins_pipe(pipe_slow);
2568 %}
2569 
2570 instruct mulD_reg_mem(regD dst, regD src1, memory src2) %{
2571   predicate(UseAVX > 0);
2572   match(Set dst (MulD src1 (LoadD src2)));
2573 
2574   format %{ "vmulsd  $dst, $src1, $src2" %}
2575   ins_cost(150);
2576   ins_encode %{
2577     __ vmulsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
2578   %}
2579   ins_pipe(pipe_slow);
2580 %}
2581 
2582 instruct mulD_reg_imm(regD dst, regD src, immD con) %{
2583   predicate(UseAVX > 0);
2584   match(Set dst (MulD src con));
2585 
2586   format %{ "vmulsd  $dst, $src, [$constantaddress]\t# load from constant table: double=$con" %}
2587   ins_cost(150);
2588   ins_encode %{
2589     __ vmulsd($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
2590   %}
2591   ins_pipe(pipe_slow);
2592 %}
2593 
2594 instruct divF_reg(regF dst, regF src) %{
2595   predicate((UseSSE>=1) && (UseAVX == 0));
2596   match(Set dst (DivF dst src));
2597 
2598   format %{ "divss   $dst, $src" %}
2599   ins_cost(150);
2600   ins_encode %{
2601     __ divss($dst$$XMMRegister, $src$$XMMRegister);
2602   %}
2603   ins_pipe(pipe_slow);
2604 %}
2605 
2606 instruct divF_mem(regF dst, memory src) %{
2607   predicate((UseSSE>=1) && (UseAVX == 0));
2608   match(Set dst (DivF dst (LoadF src)));
2609 
2610   format %{ "divss   $dst, $src" %}
2611   ins_cost(150);
2612   ins_encode %{
2613     __ divss($dst$$XMMRegister, $src$$Address);
2614   %}
2615   ins_pipe(pipe_slow);
2616 %}
2617 
2618 instruct divF_imm(regF dst, immF con) %{
2619   predicate((UseSSE>=1) && (UseAVX == 0));
2620   match(Set dst (DivF dst con));
2621   format %{ "divss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
2622   ins_cost(150);
2623   ins_encode %{
2624     __ divss($dst$$XMMRegister, $constantaddress($con));
2625   %}
2626   ins_pipe(pipe_slow);
2627 %}
2628 
2629 instruct divF_reg_reg(regF dst, regF src1, regF src2) %{
2630   predicate(UseAVX > 0);
2631   match(Set dst (DivF src1 src2));
2632 
2633   format %{ "vdivss  $dst, $src1, $src2" %}
2634   ins_cost(150);
2635   ins_encode %{
2636     __ vdivss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
2637   %}
2638   ins_pipe(pipe_slow);
2639 %}
2640 
2641 instruct divF_reg_mem(regF dst, regF src1, memory src2) %{
2642   predicate(UseAVX > 0);
2643   match(Set dst (DivF src1 (LoadF src2)));
2644 
2645   format %{ "vdivss  $dst, $src1, $src2" %}
2646   ins_cost(150);
2647   ins_encode %{
2648     __ vdivss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
2649   %}
2650   ins_pipe(pipe_slow);
2651 %}
2652 
2653 instruct divF_reg_imm(regF dst, regF src, immF con) %{
2654   predicate(UseAVX > 0);
2655   match(Set dst (DivF src con));
2656 
2657   format %{ "vdivss  $dst, $src, [$constantaddress]\t# load from constant table: float=$con" %}
2658   ins_cost(150);
2659   ins_encode %{
2660     __ vdivss($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
2661   %}
2662   ins_pipe(pipe_slow);
2663 %}
2664 
2665 instruct divD_reg(regD dst, regD src) %{
2666   predicate((UseSSE>=2) && (UseAVX == 0));
2667   match(Set dst (DivD dst src));
2668 
2669   format %{ "divsd   $dst, $src" %}
2670   ins_cost(150);
2671   ins_encode %{
2672     __ divsd($dst$$XMMRegister, $src$$XMMRegister);
2673   %}
2674   ins_pipe(pipe_slow);
2675 %}
2676 
2677 instruct divD_mem(regD dst, memory src) %{
2678   predicate((UseSSE>=2) && (UseAVX == 0));
2679   match(Set dst (DivD dst (LoadD src)));
2680 
2681   format %{ "divsd   $dst, $src" %}
2682   ins_cost(150);
2683   ins_encode %{
2684     __ divsd($dst$$XMMRegister, $src$$Address);
2685   %}
2686   ins_pipe(pipe_slow);
2687 %}
2688 
2689 instruct divD_imm(regD dst, immD con) %{
2690   predicate((UseSSE>=2) && (UseAVX == 0));
2691   match(Set dst (DivD dst con));
2692   format %{ "divsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
2693   ins_cost(150);
2694   ins_encode %{
2695     __ divsd($dst$$XMMRegister, $constantaddress($con));
2696   %}
2697   ins_pipe(pipe_slow);
2698 %}
2699 
2700 instruct divD_reg_reg(regD dst, regD src1, regD src2) %{
2701   predicate(UseAVX > 0);
2702   match(Set dst (DivD src1 src2));
2703 
2704   format %{ "vdivsd  $dst, $src1, $src2" %}
2705   ins_cost(150);
2706   ins_encode %{
2707     __ vdivsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
2708   %}
2709   ins_pipe(pipe_slow);
2710 %}
2711 
2712 instruct divD_reg_mem(regD dst, regD src1, memory src2) %{
2713   predicate(UseAVX > 0);
2714   match(Set dst (DivD src1 (LoadD src2)));
2715 
2716   format %{ "vdivsd  $dst, $src1, $src2" %}
2717   ins_cost(150);
2718   ins_encode %{
2719     __ vdivsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
2720   %}
2721   ins_pipe(pipe_slow);
2722 %}
2723 
2724 instruct divD_reg_imm(regD dst, regD src, immD con) %{
2725   predicate(UseAVX > 0);
2726   match(Set dst (DivD src con));
2727 
2728   format %{ "vdivsd  $dst, $src, [$constantaddress]\t# load from constant table: double=$con" %}
2729   ins_cost(150);
2730   ins_encode %{
2731     __ vdivsd($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
2732   %}
2733   ins_pipe(pipe_slow);
2734 %}
2735 
2736 instruct absF_reg(regF dst) %{
2737   predicate((UseSSE>=1) && (UseAVX == 0));
2738   match(Set dst (AbsF dst));
2739   ins_cost(150);
2740   format %{ "andps   $dst, [0x7fffffff]\t# abs float by sign masking" %}
2741   ins_encode %{
2742     __ andps($dst$$XMMRegister, ExternalAddress(float_signmask()));
2743   %}
2744   ins_pipe(pipe_slow);
2745 %}
2746 
2747 instruct absF_reg_reg(regF dst, regF src) %{
2748   predicate(VM_Version::supports_avxonly());
2749   match(Set dst (AbsF src));
2750   ins_cost(150);
2751   format %{ "vandps  $dst, $src, [0x7fffffff]\t# abs float by sign masking" %}
2752   ins_encode %{
2753     int vector_len = 0;
2754     __ vandps($dst$$XMMRegister, $src$$XMMRegister,
2755               ExternalAddress(float_signmask()), vector_len);
2756   %}
2757   ins_pipe(pipe_slow);
2758 %}
2759 
2760 #ifdef _LP64
2761 instruct absF_reg_reg_evex(regF dst, regF src) %{
2762   predicate(UseAVX > 2 && VM_Version::supports_avx512vl());
2763   match(Set dst (AbsF src));
2764   ins_cost(150);
2765   format %{ "vandps  $dst, $src, [0x7fffffff]\t# abs float by sign masking" %}
2766   ins_encode %{
2767     int vector_len = 0;
2768     __ vandps($dst$$XMMRegister, $src$$XMMRegister,
2769               ExternalAddress(float_signmask()), vector_len);
2770   %}
2771   ins_pipe(pipe_slow);
2772 %}
2773 
2774 instruct absF_reg_reg_evex_special(regF dst, regF src1, regF src2) %{
2775   predicate(VM_Version::supports_avx512novl());
2776   match(Set dst (AbsF src1));
2777   effect(TEMP src2);
2778   ins_cost(150);
2779   format %{ "vabsss  $dst, $src1, $src2, [0x7fffffff]\t# abs float by sign masking" %}
2780   ins_encode %{
2781     int vector_len = 0;
2782     __ vabsss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister,
2783               ExternalAddress(float_signmask()), vector_len);
2784   %}
2785   ins_pipe(pipe_slow);
2786 %}
2787 #else // _LP64
2788 instruct absF_reg_reg_evex(regF dst, regF src) %{
2789   predicate(UseAVX > 2);
2790   match(Set dst (AbsF src));
2791   ins_cost(150);
2792   format %{ "vandps  $dst, $src, [0x7fffffff]\t# abs float by sign masking" %}
2793   ins_encode %{
2794     int vector_len = 0;
2795     __ vandps($dst$$XMMRegister, $src$$XMMRegister,
2796               ExternalAddress(float_signmask()), vector_len);
2797   %}
2798   ins_pipe(pipe_slow);
2799 %}
2800 #endif
2801 
2802 instruct absD_reg(regD dst) %{
2803   predicate((UseSSE>=2) && (UseAVX == 0));
2804   match(Set dst (AbsD dst));
2805   ins_cost(150);
2806   format %{ "andpd   $dst, [0x7fffffffffffffff]\t"
2807             "# abs double by sign masking" %}
2808   ins_encode %{
2809     __ andpd($dst$$XMMRegister, ExternalAddress(double_signmask()));
2810   %}
2811   ins_pipe(pipe_slow);
2812 %}
2813 
2814 instruct absD_reg_reg(regD dst, regD src) %{
2815   predicate(VM_Version::supports_avxonly());
2816   match(Set dst (AbsD src));
2817   ins_cost(150);
2818   format %{ "vandpd  $dst, $src, [0x7fffffffffffffff]\t"
2819             "# abs double by sign masking" %}
2820   ins_encode %{
2821     int vector_len = 0;
2822     __ vandpd($dst$$XMMRegister, $src$$XMMRegister,
2823               ExternalAddress(double_signmask()), vector_len);
2824   %}
2825   ins_pipe(pipe_slow);
2826 %}
2827 
2828 #ifdef _LP64
2829 instruct absD_reg_reg_evex(regD dst, regD src) %{
2830   predicate(UseAVX > 2 && VM_Version::supports_avx512vl());
2831   match(Set dst (AbsD src));
2832   ins_cost(150);
2833   format %{ "vandpd  $dst, $src, [0x7fffffffffffffff]\t"
2834             "# abs double by sign masking" %}
2835   ins_encode %{
2836     int vector_len = 0;
2837     __ vandpd($dst$$XMMRegister, $src$$XMMRegister,
2838               ExternalAddress(double_signmask()), vector_len);
2839   %}
2840   ins_pipe(pipe_slow);
2841 %}
2842 
2843 instruct absD_reg_reg_evex_special(regD dst, regD src1, regD src2) %{
2844   predicate(VM_Version::supports_avx512novl());
2845   match(Set dst (AbsD src1));
2846   effect(TEMP src2);
2847   ins_cost(150);
2848   format %{ "vabssd  $dst, $src1, $src2, [0x7fffffffffffffff]\t# abs float by sign masking" %}
2849   ins_encode %{
2850     int vector_len = 0;
2851     __ vabssd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister,
2852               ExternalAddress(double_signmask()), vector_len);
2853   %}
2854   ins_pipe(pipe_slow);
2855 %}
2856 #else // _LP64
2857 instruct absD_reg_reg_evex(regD dst, regD src) %{
2858   predicate(UseAVX > 2);
2859   match(Set dst (AbsD src));
2860   ins_cost(150);
2861   format %{ "vandpd  $dst, $src, [0x7fffffffffffffff]\t"
2862             "# abs double by sign masking" %}
2863   ins_encode %{
2864     int vector_len = 0;
2865     __ vandpd($dst$$XMMRegister, $src$$XMMRegister,
2866               ExternalAddress(double_signmask()), vector_len);
2867   %}
2868   ins_pipe(pipe_slow);
2869 %}
2870 #endif
2871 
2872 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
2873 %{
2874   match(Set dst (NegI dst));
2875   effect(KILL cr);
2876 
2877   format %{ "negl    $dst\t# int" %}
2878   ins_encode %{
2879     __ negl($dst$$Register);
2880   %}
2881   ins_pipe(ialu_reg);
2882 %}
2883 
2884 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
2885 %{
2886   match(Set dst (NegL dst));
2887   effect(KILL cr);
2888 
2889   format %{ "negq    $dst\t# int" %}
2890   ins_encode %{
2891     __ negq($dst$$Register);
2892   %}
2893   ins_pipe(ialu_reg);
2894 %}
2895 
2896 instruct negF_reg(regF dst) %{
2897   predicate((UseSSE>=1) && (UseAVX == 0));
2898   match(Set dst (NegF dst));
2899   ins_cost(150);
2900   format %{ "xorps   $dst, [0x80000000]\t# neg float by sign flipping" %}
2901   ins_encode %{
2902     __ xorps($dst$$XMMRegister, ExternalAddress(float_signflip()));
2903   %}
2904   ins_pipe(pipe_slow);
2905 %}
2906 
2907 instruct negF_reg_reg(regF dst, regF src) %{
2908   predicate(UseAVX > 0);
2909   match(Set dst (NegF src));
2910   ins_cost(150);
2911   format %{ "vnegatess  $dst, $src, [0x80000000]\t# neg float by sign flipping" %}
2912   ins_encode %{
2913     __ vnegatess($dst$$XMMRegister, $src$$XMMRegister,
2914                  ExternalAddress(float_signflip()));
2915   %}
2916   ins_pipe(pipe_slow);
2917 %}
2918 
2919 instruct negD_reg(regD dst) %{
2920   predicate((UseSSE>=2) && (UseAVX == 0));
2921   match(Set dst (NegD dst));
2922   ins_cost(150);
2923   format %{ "xorpd   $dst, [0x8000000000000000]\t"
2924             "# neg double by sign flipping" %}
2925   ins_encode %{
2926     __ xorpd($dst$$XMMRegister, ExternalAddress(double_signflip()));
2927   %}
2928   ins_pipe(pipe_slow);
2929 %}
2930 
2931 instruct negD_reg_reg(regD dst, regD src) %{
2932   predicate(UseAVX > 0);
2933   match(Set dst (NegD src));
2934   ins_cost(150);
2935   format %{ "vnegatess  $dst, $src, [0x8000000000000000]\t"
2936             "# neg double by sign flipping" %}
2937   ins_encode %{
2938     __ vnegatesd($dst$$XMMRegister, $src$$XMMRegister,
2939                  ExternalAddress(double_signflip()));
2940   %}
2941   ins_pipe(pipe_slow);
2942 %}
2943 
2944 instruct sqrtF_reg(regF dst, regF src) %{
2945   predicate(UseSSE>=1);
2946   match(Set dst (SqrtF src));
2947 
2948   format %{ "sqrtss  $dst, $src" %}
2949   ins_cost(150);
2950   ins_encode %{
2951     __ sqrtss($dst$$XMMRegister, $src$$XMMRegister);
2952   %}
2953   ins_pipe(pipe_slow);
2954 %}
2955 
2956 instruct sqrtF_mem(regF dst, memory src) %{
2957   predicate(UseSSE>=1);
2958   match(Set dst (SqrtF (LoadF src)));
2959 
2960   format %{ "sqrtss  $dst, $src" %}
2961   ins_cost(150);
2962   ins_encode %{
2963     __ sqrtss($dst$$XMMRegister, $src$$Address);
2964   %}
2965   ins_pipe(pipe_slow);
2966 %}
2967 
2968 instruct sqrtF_imm(regF dst, immF con) %{
2969   predicate(UseSSE>=1);
2970   match(Set dst (SqrtF con));
2971 
2972   format %{ "sqrtss  $dst, [$constantaddress]\t# load from constant table: float=$con" %}
2973   ins_cost(150);
2974   ins_encode %{
2975     __ sqrtss($dst$$XMMRegister, $constantaddress($con));
2976   %}
2977   ins_pipe(pipe_slow);
2978 %}
2979 
2980 instruct sqrtD_reg(regD dst, regD src) %{
2981   predicate(UseSSE>=2);
2982   match(Set dst (SqrtD src));
2983 
2984   format %{ "sqrtsd  $dst, $src" %}
2985   ins_cost(150);
2986   ins_encode %{
2987     __ sqrtsd($dst$$XMMRegister, $src$$XMMRegister);
2988   %}
2989   ins_pipe(pipe_slow);
2990 %}
2991 
2992 instruct sqrtD_mem(regD dst, memory src) %{
2993   predicate(UseSSE>=2);
2994   match(Set dst (SqrtD (LoadD src)));
2995 
2996   format %{ "sqrtsd  $dst, $src" %}
2997   ins_cost(150);
2998   ins_encode %{
2999     __ sqrtsd($dst$$XMMRegister, $src$$Address);
3000   %}
3001   ins_pipe(pipe_slow);
3002 %}
3003 
3004 instruct sqrtD_imm(regD dst, immD con) %{
3005   predicate(UseSSE>=2);
3006   match(Set dst (SqrtD con));
3007   format %{ "sqrtsd  $dst, [$constantaddress]\t# load from constant table: double=$con" %}
3008   ins_cost(150);
3009   ins_encode %{
3010     __ sqrtsd($dst$$XMMRegister, $constantaddress($con));
3011   %}
3012   ins_pipe(pipe_slow);
3013 %}
3014 
3015 instruct onspinwait() %{
3016   match(OnSpinWait);
3017   ins_cost(200);
3018 
3019   format %{
3020     $$template
3021     if (os::is_MP()) {
3022       $$emit$$"pause\t! membar_onspinwait"
3023     } else {
3024       $$emit$$"MEMBAR-onspinwait ! (empty encoding)"
3025     }
3026   %}
3027   ins_encode %{
3028     __ pause();
3029   %}
3030   ins_pipe(pipe_slow);
3031 %}
3032 
3033 // a * b + c
3034 instruct fmaD_reg(regD a, regD b, regD c) %{
3035   predicate(UseFMA);
3036   match(Set c (FmaD  c (Binary a b)));
3037   format %{ "fmasd $a,$b,$c\t# $c = $a * $b + $c" %}
3038   ins_cost(150);
3039   ins_encode %{
3040     __ fmad($c$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $c$$XMMRegister);
3041   %}
3042   ins_pipe( pipe_slow );
3043 %}
3044 
3045 // a * b + c
3046 instruct fmaF_reg(regF a, regF b, regF c) %{
3047   predicate(UseFMA);
3048   match(Set c (FmaF  c (Binary a b)));
3049   format %{ "fmass $a,$b,$c\t# $c = $a * $b + $c" %}
3050   ins_cost(150);
3051   ins_encode %{
3052     __ fmaf($c$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $c$$XMMRegister);
3053   %}
3054   ins_pipe( pipe_slow );
3055 %}
3056 
3057 // ====================VECTOR INSTRUCTIONS=====================================
3058 
3059 instruct reinterpretS(vecS dst) %{
3060   predicate(n->bottom_type()->is_vect()->length_in_bytes() == 4 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 4);
3061   match(Set dst (VectorReinterpret dst));
3062   ins_cost(125);
3063   format %{ " # reinterpret $dst" %}
3064   ins_encode %{
3065     // empty
3066   %}
3067   ins_pipe( pipe_slow );
3068 %}
3069 
3070 instruct reinterpretS2D(vecD dst, vecS src, rRegL scratch) %{
3071   predicate(UseAVX == 0 && n->bottom_type()->is_vect()->length_in_bytes() == 8 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 4);
3072   match(Set dst (VectorReinterpret src));
3073   ins_cost(125);
3074   effect(TEMP dst, TEMP scratch);
3075   format %{ " # reinterpret $dst,$src" %}
3076   ins_encode %{
3077     __ movdqu($dst$$XMMRegister, ExternalAddress(vector_32_bit_mask()), $scratch$$Register);
3078     __ pand($dst$$XMMRegister, $src$$XMMRegister);
3079   %}
3080   ins_pipe( pipe_slow );
3081 %}
3082 
3083 instruct reinterpretS2D_avx(vecD dst, vecS src, rRegL scratch) %{
3084   predicate(UseAVX > 0 && n->bottom_type()->is_vect()->length_in_bytes() == 8 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 4);
3085   match(Set dst (VectorReinterpret src));
3086   ins_cost(125);
3087   effect(TEMP dst, TEMP scratch);
3088   format %{ " # reinterpret $dst,$src" %}
3089   ins_encode %{
3090     int vector_len = 0;
3091     __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_32_bit_mask()), vector_len, $scratch$$Register);
3092   %}
3093   ins_pipe( pipe_slow );
3094 %}
3095 
3096 instruct reinterpretS2X(vecX dst, vecS src, rRegL scratch) %{
3097   predicate(UseAVX == 0 && n->bottom_type()->is_vect()->length_in_bytes() == 16 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 4);
3098   match(Set dst (VectorReinterpret src));
3099   ins_cost(125);
3100   effect(TEMP dst, TEMP scratch);
3101   format %{ " # reinterpret $dst,$src" %}
3102   ins_encode %{
3103     __ movdqu($dst$$XMMRegister, ExternalAddress(vector_32_bit_mask()), $scratch$$Register);
3104     __ pand($dst$$XMMRegister, $src$$XMMRegister);
3105   %}
3106   ins_pipe( pipe_slow );
3107 %}
3108 
3109 instruct reinterpretS2X_avx(vecX dst, vecS src, rRegL scratch) %{
3110   predicate(UseAVX > 0 && n->bottom_type()->is_vect()->length_in_bytes() == 16 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 4);
3111   match(Set dst (VectorReinterpret src));
3112   ins_cost(125);
3113   effect(TEMP scratch);
3114   format %{ " # reinterpret $dst,$src" %}
3115   ins_encode %{
3116     int vector_len = 0;
3117     __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_32_bit_mask()), vector_len, $scratch$$Register);
3118   %}
3119   ins_pipe( pipe_slow );
3120 %}
3121 
3122 instruct reinterpretS2Y(vecY dst, vecS src, rRegL scratch) %{
3123   predicate(UseAVX >= 2 && n->bottom_type()->is_vect()->length_in_bytes() == 32 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 4);
3124   match(Set dst (VectorReinterpret src));
3125   ins_cost(125);
3126   effect(TEMP scratch);
3127   format %{ " # reinterpret $dst,$src" %}
3128   ins_encode %{
3129     int vector_len = 1;
3130     __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_32_bit_mask()), vector_len, $scratch$$Register);
3131   %}
3132   ins_pipe( pipe_slow );
3133 %}
3134 
3135 instruct reinterpretS2Z(vecZ dst, vecS src, rRegL scratch) %{
3136   predicate(UseAVX > 2 && n->bottom_type()->is_vect()->length_in_bytes() == 64 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 4);
3137   match(Set dst (VectorReinterpret src));
3138   ins_cost(125);
3139   effect(TEMP scratch);
3140   format %{ " # reinterpret $dst,$src" %}
3141   ins_encode %{
3142     int vector_len = 2;
3143     __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_32_bit_mask()), vector_len, $scratch$$Register);
3144   %}
3145   ins_pipe( pipe_slow );
3146 %}
3147 
3148 instruct reinterpretD2S(vecS dst, vecD src) %{
3149   predicate(n->bottom_type()->is_vect()->length_in_bytes() == 4 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 8);
3150   match(Set dst (VectorReinterpret src));
3151   ins_cost(125);
3152   format %{ " # reinterpret $dst,$src" %}
3153   ins_encode %{
3154     // If register is the same, then move is not needed.
3155     if ($dst$$XMMRegister != $src$$XMMRegister) {
3156       __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
3157     }
3158   %}
3159   ins_pipe( pipe_slow );
3160 %}
3161 
3162 instruct reinterpretD(vecD dst) %{
3163   predicate(n->bottom_type()->is_vect()->length_in_bytes() == 8 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 8);
3164   match(Set dst (VectorReinterpret dst));
3165   ins_cost(125);
3166   format %{ " # reinterpret $dst" %}
3167   ins_encode %{
3168     // empty
3169   %}
3170   ins_pipe( pipe_slow );
3171 %}
3172 
3173 instruct reinterpretD2X(vecX dst, vecD src, rRegL scratch) %{
3174   predicate(UseAVX == 0 && n->bottom_type()->is_vect()->length_in_bytes() == 16 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 8);
3175   match(Set dst (VectorReinterpret src));
3176   ins_cost(125);
3177   effect(TEMP dst, TEMP scratch);
3178   format %{ " # reinterpret $dst,$src" %}
3179   ins_encode %{
3180     __ movdqu($dst$$XMMRegister, ExternalAddress(vector_64_bit_mask()), $scratch$$Register);
3181     __ pand($dst$$XMMRegister, $src$$XMMRegister);
3182   %}
3183   ins_pipe( pipe_slow );
3184 %}
3185 
3186 instruct reinterpretD2X_avx(vecX dst, vecD src, rRegL scratch) %{
3187   predicate(UseAVX > 0 && n->bottom_type()->is_vect()->length_in_bytes() == 16 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 8);
3188   match(Set dst (VectorReinterpret src));
3189   ins_cost(125);
3190   effect(TEMP dst, TEMP scratch);
3191   format %{ " # reinterpret $dst,$src" %}
3192   ins_encode %{
3193     int vector_len = 0;
3194     __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_64_bit_mask()), vector_len, $scratch$$Register);
3195   %}
3196   ins_pipe( pipe_slow );
3197 %}
3198 
3199 instruct reinterpretD2Y(vecY dst, vecD src, rRegL scratch) %{
3200   predicate(UseAVX >= 2 && n->bottom_type()->is_vect()->length_in_bytes() == 32 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 8);
3201   match(Set dst (VectorReinterpret src));
3202   ins_cost(125);
3203   effect(TEMP scratch);
3204   format %{ " # reinterpret $dst,$src" %}
3205   ins_encode %{
3206     int vector_len = 1;
3207     __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_64_bit_mask()), vector_len, $scratch$$Register);
3208   %}
3209   ins_pipe( pipe_slow );
3210 %}
3211 
3212 instruct reinterpretD2Z(vecZ dst, vecD src, rRegL scratch) %{
3213   predicate(UseAVX > 2 && n->bottom_type()->is_vect()->length_in_bytes() == 64 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 8);
3214   match(Set dst (VectorReinterpret src));
3215   ins_cost(125);
3216   effect(TEMP scratch);
3217   format %{ " # reinterpret $dst,$src" %}
3218   ins_encode %{
3219     int vector_len = 2;
3220     __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_64_bit_mask()), vector_len, $scratch$$Register);
3221   %}
3222   ins_pipe( pipe_slow );
3223 %}
3224 
3225 instruct reinterpretX2S(vecS dst, vecX src) %{
3226   predicate(n->bottom_type()->is_vect()->length_in_bytes() == 4 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 16);
3227   match(Set dst (VectorReinterpret src));
3228   ins_cost(125);
3229   format %{ " # reinterpret $dst,$src" %}
3230   ins_encode %{
3231     // If register is the same, then move is not needed.
3232     if ($dst$$XMMRegister != $src$$XMMRegister) {
3233       __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
3234     }
3235   %}
3236   ins_pipe( pipe_slow );
3237 %}
3238 
3239 instruct reinterpretX2D(vecD dst, vecX src) %{
3240   predicate(n->bottom_type()->is_vect()->length_in_bytes() == 8 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 16);
3241   match(Set dst (VectorReinterpret src));
3242   ins_cost(125);
3243   format %{ " # reinterpret $dst,$src" %}
3244   ins_encode %{
3245     // If register is the same, then move is not needed.
3246     if ($dst$$XMMRegister != $src$$XMMRegister) {
3247       __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
3248     }
3249   %}
3250   ins_pipe( pipe_slow );
3251 %}
3252 
3253 instruct reinterpretX(vecX dst) %{
3254   predicate(n->bottom_type()->is_vect()->length_in_bytes() == 16 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 16);
3255   match(Set dst (VectorReinterpret dst));
3256   ins_cost(125);
3257   format %{ " # reinterpret $dst" %}
3258   ins_encode %{
3259     // empty
3260   %}
3261   ins_pipe( pipe_slow );
3262 %}
3263 
3264 instruct reinterpretX2Y(vecY dst, vecX src) %{
3265   predicate(n->bottom_type()->is_vect()->length_in_bytes() == 32 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 16);
3266   match(Set dst (VectorReinterpret src));
3267   ins_cost(125);
3268   effect(TEMP dst);
3269   format %{ " # reinterpret $dst,$src" %}
3270   ins_encode %{
3271     int vector_len = 1;
3272     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vector_len);
3273     __ movdqu($dst$$XMMRegister, $src$$XMMRegister);  // just 128-bits need moved
3274   %}
3275   ins_pipe( pipe_slow );
3276 %}
3277 
3278 instruct reinterpretX2Z(vecZ dst, vecX src) %{
3279   predicate(n->bottom_type()->is_vect()->length_in_bytes() == 64 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 16);
3280   match(Set dst (VectorReinterpret src));
3281   ins_cost(125);
3282   effect(TEMP dst);
3283   format %{ " # reinterpret $dst,$src" %}
3284   ins_encode %{
3285     int vector_len = 2;
3286     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vector_len);
3287     __ movdqu($dst$$XMMRegister, $src$$XMMRegister);  // just 128-bits need moved
3288   %}
3289   ins_pipe( pipe_slow );
3290 %}
3291 
3292 instruct reinterpretY2S(vecS dst, vecY src) %{
3293   predicate(n->bottom_type()->is_vect()->length_in_bytes() == 4 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 32);
3294   match(Set dst (VectorReinterpret src));
3295   ins_cost(125);
3296   format %{ " # reinterpret $dst,$src" %}
3297   ins_encode %{
3298     // If register is the same, then move is not needed.
3299     if ($dst$$XMMRegister != $src$$XMMRegister) {
3300       __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
3301     }
3302   %}
3303   ins_pipe( pipe_slow );
3304 %}
3305 
3306 instruct reinterpretY2D(vecD dst, vecY src) %{
3307   predicate(n->bottom_type()->is_vect()->length_in_bytes() == 8 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 32);
3308   match(Set dst (VectorReinterpret src));
3309   ins_cost(125);
3310   format %{ " # reinterpret $dst,$src" %}
3311   ins_encode %{
3312     // If register is the same, then move is not needed.
3313     if ($dst$$XMMRegister != $src$$XMMRegister) {
3314       __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
3315     }
3316   %}
3317   ins_pipe( pipe_slow );
3318 %}
3319 
3320 instruct reinterpretY2X(vecX dst, vecY src) %{
3321   predicate(n->bottom_type()->is_vect()->length_in_bytes() == 16 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 32);
3322   match(Set dst (VectorReinterpret src));
3323   ins_cost(125);
3324   format %{ " # reinterpret $dst,$src" %}
3325   ins_encode %{
3326     // If register is the same, then move is not needed.
3327     if ($dst$$XMMRegister != $src$$XMMRegister) {
3328       __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
3329     }
3330   %}
3331   ins_pipe( pipe_slow );
3332 %}
3333 
3334 instruct reinterpretY(vecY dst) %{
3335   predicate(n->bottom_type()->is_vect()->length_in_bytes() == 32 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 32);
3336   match(Set dst (VectorReinterpret dst));
3337   ins_cost(125);
3338   format %{ " # reinterpret $dst" %}
3339   ins_encode %{
3340     // empty
3341   %}
3342   ins_pipe( pipe_slow );
3343 %}
3344 
3345 instruct reinterpretY2Z(vecZ dst, vecY src) %{
3346   predicate(n->bottom_type()->is_vect()->length_in_bytes() == 64 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 32);
3347   match(Set dst (VectorReinterpret src));
3348   ins_cost(125);
3349   effect(TEMP dst);
3350   format %{ " # reinterpret $dst,$src" %}
3351   ins_encode %{
3352     int vector_len = 2;
3353     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vector_len);
3354     __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister);
3355   %}
3356   ins_pipe( pipe_slow );
3357 %}
3358 
3359 instruct reinterpretZ2S(vecS dst, vecZ src) %{
3360   predicate(n->bottom_type()->is_vect()->length_in_bytes() == 4 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 64);
3361   match(Set dst (VectorReinterpret src));
3362   ins_cost(125);
3363   format %{ " # reinterpret $dst,$src" %}
3364   ins_encode %{
3365     // If register is the same, then move is not needed.
3366     if ($dst$$XMMRegister != $src$$XMMRegister) {
3367       __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
3368     }
3369   %}
3370   ins_pipe( pipe_slow );
3371 %}
3372 
3373 instruct reinterpretZ2D(vecD dst, vecZ src) %{
3374   predicate(n->bottom_type()->is_vect()->length_in_bytes() == 8 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 64);
3375   match(Set dst (VectorReinterpret src));
3376   ins_cost(125);
3377   format %{ " # reinterpret $dst,$src" %}
3378   ins_encode %{
3379     // If register is the same, then move is not needed.
3380     if ($dst$$XMMRegister != $src$$XMMRegister) {
3381       __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
3382     }
3383   %}
3384   ins_pipe( pipe_slow );
3385 %}
3386 
3387 instruct reinterpretZ2X(vecX dst, vecZ src) %{
3388   predicate(n->bottom_type()->is_vect()->length_in_bytes() == 16 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 64);
3389   match(Set dst (VectorReinterpret src));
3390   ins_cost(125);
3391   format %{ " # reinterpret $dst,$src" %}
3392   ins_encode %{
3393     // If register is the same, then move is not needed.
3394     if ($dst$$XMMRegister != $src$$XMMRegister) {
3395       __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
3396     }
3397   %}
3398   ins_pipe( pipe_slow );
3399 %}
3400 
3401 instruct reinterpretZ2Y(vecY dst, vecZ src) %{
3402   predicate(n->bottom_type()->is_vect()->length_in_bytes() == 32 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 64);
3403   match(Set dst (VectorReinterpret src));
3404   ins_cost(125);
3405   format %{ " # reinterpret $dst,$src" %}
3406   ins_encode %{
3407     // If register is the same, then move is not needed.
3408     if ($dst$$XMMRegister != $src$$XMMRegister) {
3409       __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister);
3410     }
3411   %}
3412   ins_pipe( pipe_slow );
3413 %}
3414 
3415 instruct reinterpretZ(vecZ dst) %{
3416   predicate(n->bottom_type()->is_vect()->length_in_bytes() == 64 && n->in(1)->bottom_type()->is_vect()->length_in_bytes() == 64);
3417   match(Set dst (VectorReinterpret dst));
3418   ins_cost(125);
3419   format %{ " # reinterpret $dst" %}
3420   ins_encode %{
3421     // empty
3422   %}
3423   ins_pipe( pipe_slow );
3424 %}
3425 
3426 // ==========
3427 
3428 // Load vectors (1 byte long)
3429 instruct loadV1(vecS dst, memory mem, rRegI tmp) %{
3430   predicate(n->as_LoadVector()->memory_size() == 1);
3431   match(Set dst (LoadVector mem));
3432   ins_cost(125);
3433   effect(TEMP tmp);
3434   format %{ "movzbl $tmp,$mem\n\t"
3435           "movd $dst,$tmp\t! load vector (1 byte)" %}
3436   ins_encode %{
3437     __ movzbl($tmp$$Register, $mem$$Address);
3438     __ movdl($dst$$XMMRegister, $tmp$$Register);
3439   %}
3440   ins_pipe( pipe_slow );
3441 %}
3442 
3443 // Load vectors (2 bytes long)
3444 instruct loadV2(vecS dst, memory mem, rRegI tmp) %{
3445   predicate(n->as_LoadVector()->memory_size() == 2);
3446   match(Set dst (LoadVector mem));
3447   ins_cost(125);
3448   effect(TEMP tmp);
3449   format %{ "movzwl $tmp,$mem\n\t"
3450           "movd $dst,$tmp\t! load vector (2 bytes)" %}
3451   ins_encode %{
3452     __ movzwl($tmp$$Register, $mem$$Address);
3453     __ movdl($dst$$XMMRegister, $tmp$$Register);
3454   %}
3455   ins_pipe( pipe_slow );
3456 %}
3457 
3458 // Load vectors (4 bytes long)
3459 instruct loadV4(vecS dst, memory mem) %{
3460   predicate(n->as_LoadVector()->memory_size() == 4);
3461   match(Set dst (LoadVector mem));
3462   ins_cost(125);
3463   format %{ "movd    $dst,$mem\t! load vector (4 bytes)" %}
3464   ins_encode %{
3465     __ movdl($dst$$XMMRegister, $mem$$Address);
3466   %}
3467   ins_pipe( pipe_slow );
3468 %}
3469 
3470 // Load vectors (8 bytes long)
3471 instruct loadV8(vecD dst, memory mem) %{
3472   predicate(n->as_LoadVector()->memory_size() == 8);
3473   match(Set dst (LoadVector mem));
3474   ins_cost(125);
3475   format %{ "movq    $dst,$mem\t! load vector (8 bytes)" %}
3476   ins_encode %{
3477     __ movq($dst$$XMMRegister, $mem$$Address);
3478   %}
3479   ins_pipe( pipe_slow );
3480 %}
3481 
3482 // Load vectors (16 bytes long)
3483 instruct loadV16(vecX dst, memory mem) %{
3484   predicate(n->as_LoadVector()->memory_size() == 16);
3485   match(Set dst (LoadVector mem));
3486   ins_cost(125);
3487   format %{ "movdqu  $dst,$mem\t! load vector (16 bytes)" %}
3488   ins_encode %{
3489     __ movdqu($dst$$XMMRegister, $mem$$Address);
3490   %}
3491   ins_pipe( pipe_slow );
3492 %}
3493 
3494 // Load vectors (32 bytes long)
3495 instruct loadV32(vecY dst, memory mem) %{
3496   predicate(n->as_LoadVector()->memory_size() == 32);
3497   match(Set dst (LoadVector mem));
3498   ins_cost(125);
3499   format %{ "vmovdqu $dst,$mem\t! load vector (32 bytes)" %}
3500   ins_encode %{
3501     __ vmovdqu($dst$$XMMRegister, $mem$$Address);
3502   %}
3503   ins_pipe( pipe_slow );
3504 %}
3505 
3506 // Load vectors (64 bytes long)
3507 instruct loadV64_dword(vecZ dst, memory mem) %{
3508   predicate(n->as_LoadVector()->memory_size() == 64 && n->as_LoadVector()->element_size() <= 4);
3509   match(Set dst (LoadVector mem));
3510   ins_cost(125);
3511   format %{ "vmovdqul $dst k0,$mem\t! load vector (64 bytes)" %}
3512   ins_encode %{
3513     int vector_len = 2;
3514     __ evmovdqul($dst$$XMMRegister, $mem$$Address, vector_len);
3515   %}
3516   ins_pipe( pipe_slow );
3517 %}
3518 
3519 // Load vectors (64 bytes long)
3520 instruct loadV64_qword(vecZ dst, memory mem) %{
3521   predicate(n->as_LoadVector()->memory_size() == 64 && n->as_LoadVector()->element_size() > 4);
3522   match(Set dst (LoadVector mem));
3523   ins_cost(125);
3524   format %{ "vmovdquq $dst k0,$mem\t! load vector (64 bytes)" %}
3525   ins_encode %{
3526     int vector_len = 2;
3527     __ evmovdquq($dst$$XMMRegister, $mem$$Address, vector_len);
3528   %}
3529   ins_pipe( pipe_slow );
3530 %}
3531 
3532 // Store vectors
3533 instruct storeV1(memory mem, vecS src, rRegI tmp) %{
3534   predicate(n->as_StoreVector()->memory_size() == 1);
3535   match(Set mem (StoreVector mem src));
3536   ins_cost(145);
3537   effect(TEMP tmp);
3538   format %{ "movd $tmp,$src\n\t"
3539           "movb $mem,$tmp\t! store vector (1 byte)" %}
3540   ins_encode %{
3541     __ movdl($tmp$$Register, $src$$XMMRegister);
3542     __ movb($mem$$Address, $tmp$$Register);
3543   %}
3544   ins_pipe( pipe_slow );
3545 %}
3546 
3547 instruct storeV2(memory mem, vecS src, rRegI tmp) %{
3548   predicate(n->as_StoreVector()->memory_size() == 2);
3549   match(Set mem (StoreVector mem src));
3550   ins_cost(145);
3551   effect(TEMP tmp);
3552   format %{ "movd $tmp,$src\n\t"
3553           "movw $mem,$tmp\t! store vector (2 bytes)" %}
3554   ins_encode %{
3555     __ movdl($tmp$$Register, $src$$XMMRegister);
3556     __ movw($mem$$Address, $tmp$$Register);
3557   %}
3558   ins_pipe( pipe_slow );
3559 %}
3560 
3561 instruct storeV4(memory mem, vecS src) %{
3562   predicate(n->as_StoreVector()->memory_size() == 4);
3563   match(Set mem (StoreVector mem src));
3564   ins_cost(145);
3565   format %{ "movd    $mem,$src\t! store vector (4 bytes)" %}
3566   ins_encode %{
3567     __ movdl($mem$$Address, $src$$XMMRegister);
3568   %}
3569   ins_pipe( pipe_slow );
3570 %}
3571 
3572 instruct storeV8(memory mem, vecD src) %{
3573   predicate(n->as_StoreVector()->memory_size() == 8);
3574   match(Set mem (StoreVector mem src));
3575   ins_cost(145);
3576   format %{ "movq    $mem,$src\t! store vector (8 bytes)" %}
3577   ins_encode %{
3578     __ movq($mem$$Address, $src$$XMMRegister);
3579   %}
3580   ins_pipe( pipe_slow );
3581 %}
3582 
3583 instruct storeV16(memory mem, vecX src) %{
3584   predicate(n->as_StoreVector()->memory_size() == 16);
3585   match(Set mem (StoreVector mem src));
3586   ins_cost(145);
3587   format %{ "movdqu  $mem,$src\t! store vector (16 bytes)" %}
3588   ins_encode %{
3589     __ movdqu($mem$$Address, $src$$XMMRegister);
3590   %}
3591   ins_pipe( pipe_slow );
3592 %}
3593 
3594 instruct storeV32(memory mem, vecY src) %{
3595   predicate(n->as_StoreVector()->memory_size() == 32);
3596   match(Set mem (StoreVector mem src));
3597   ins_cost(145);
3598   format %{ "vmovdqu $mem,$src\t! store vector (32 bytes)" %}
3599   ins_encode %{
3600     __ vmovdqu($mem$$Address, $src$$XMMRegister);
3601   %}
3602   ins_pipe( pipe_slow );
3603 %}
3604 
3605 instruct storeV64_dword(memory mem, vecZ src) %{
3606   predicate(n->as_StoreVector()->memory_size() == 64 && n->as_StoreVector()->element_size() <= 4);
3607   match(Set mem (StoreVector mem src));
3608   ins_cost(145);
3609   format %{ "vmovdqul $mem k0,$src\t! store vector (64 bytes)" %}
3610   ins_encode %{
3611     int vector_len = 2;
3612     __ evmovdqul($mem$$Address, $src$$XMMRegister, vector_len);
3613   %}
3614   ins_pipe( pipe_slow );
3615 %}
3616 
3617 instruct storeV64_qword(memory mem, vecZ src) %{
3618   predicate(n->as_StoreVector()->memory_size() == 64 && n->as_StoreVector()->element_size() > 4);
3619   match(Set mem (StoreVector mem src));
3620   ins_cost(145);
3621   format %{ "vmovdquq $mem k0,$src\t! store vector (64 bytes)" %}
3622   ins_encode %{
3623     int vector_len = 2;
3624     __ evmovdquq($mem$$Address, $src$$XMMRegister, vector_len);
3625   %}
3626   ins_pipe( pipe_slow );
3627 %}
3628 
3629 // ====================LEGACY REPLICATE=======================================
3630 
3631 instruct Repl4B_mem(vecS dst, memory mem) %{
3632   predicate(n->as_Vector()->length() == 4 && UseAVX > 0 && !VM_Version::supports_avx512vlbw());
3633   match(Set dst (ReplicateB (LoadB mem)));
3634   format %{ "punpcklbw $dst,$mem\n\t"
3635             "pshuflw $dst,$dst,0x00\t! replicate4B" %}
3636   ins_encode %{
3637     __ punpcklbw($dst$$XMMRegister, $mem$$Address);
3638     __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
3639   %}
3640   ins_pipe( pipe_slow );
3641 %}
3642 
3643 instruct Repl8B_mem(vecD dst, memory mem) %{
3644   predicate(n->as_Vector()->length() == 8 && UseAVX > 0 && !VM_Version::supports_avx512vlbw());
3645   match(Set dst (ReplicateB (LoadB mem)));
3646   format %{ "punpcklbw $dst,$mem\n\t"
3647             "pshuflw $dst,$dst,0x00\t! replicate8B" %}
3648   ins_encode %{
3649     __ punpcklbw($dst$$XMMRegister, $mem$$Address);
3650     __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
3651   %}
3652   ins_pipe( pipe_slow );
3653 %}
3654 
3655 instruct Repl16B(vecX dst, rRegI src) %{
3656   predicate(n->as_Vector()->length() == 16 && !VM_Version::supports_avx512vlbw());
3657   match(Set dst (ReplicateB src));
3658   format %{ "movd    $dst,$src\n\t"
3659             "punpcklbw $dst,$dst\n\t"
3660             "pshuflw $dst,$dst,0x00\n\t"
3661             "punpcklqdq $dst,$dst\t! replicate16B" %}
3662   ins_encode %{
3663     __ movdl($dst$$XMMRegister, $src$$Register);
3664     __ punpcklbw($dst$$XMMRegister, $dst$$XMMRegister);
3665     __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
3666     __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3667   %}
3668   ins_pipe( pipe_slow );
3669 %}
3670 
3671 instruct Repl16B_mem(vecX dst, memory mem) %{
3672   predicate(n->as_Vector()->length() == 16 && UseAVX > 0 && !VM_Version::supports_avx512vlbw());
3673   match(Set dst (ReplicateB (LoadB mem)));
3674   format %{ "punpcklbw $dst,$mem\n\t"
3675             "pshuflw $dst,$dst,0x00\n\t"
3676             "punpcklqdq $dst,$dst\t! replicate16B" %}
3677   ins_encode %{
3678     __ punpcklbw($dst$$XMMRegister, $mem$$Address);
3679     __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
3680     __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3681   %}
3682   ins_pipe( pipe_slow );
3683 %}
3684 
3685 instruct Repl32B(vecY dst, rRegI src) %{
3686   predicate(n->as_Vector()->length() == 32 && !VM_Version::supports_avx512vlbw());
3687   match(Set dst (ReplicateB src));
3688   format %{ "movd    $dst,$src\n\t"
3689             "punpcklbw $dst,$dst\n\t"
3690             "pshuflw $dst,$dst,0x00\n\t"
3691             "punpcklqdq $dst,$dst\n\t"
3692             "vinserti128_high $dst,$dst\t! replicate32B" %}
3693   ins_encode %{
3694     __ movdl($dst$$XMMRegister, $src$$Register);
3695     __ punpcklbw($dst$$XMMRegister, $dst$$XMMRegister);
3696     __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
3697     __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3698     __ vinserti128_high($dst$$XMMRegister, $dst$$XMMRegister);
3699   %}
3700   ins_pipe( pipe_slow );
3701 %}
3702 
3703 instruct Repl32B_mem(vecY dst, memory mem) %{
3704   predicate(n->as_Vector()->length() == 32 && !VM_Version::supports_avx512vlbw());
3705   match(Set dst (ReplicateB (LoadB mem)));
3706   format %{ "punpcklbw $dst,$mem\n\t"
3707             "pshuflw $dst,$dst,0x00\n\t"
3708             "punpcklqdq $dst,$dst\n\t"
3709             "vinserti128_high $dst,$dst\t! replicate32B" %}
3710   ins_encode %{
3711     __ punpcklbw($dst$$XMMRegister, $mem$$Address);
3712     __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
3713     __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3714     __ vinserti128_high($dst$$XMMRegister, $dst$$XMMRegister);
3715   %}
3716   ins_pipe( pipe_slow );
3717 %}
3718 
3719 instruct Repl16B_imm(vecX dst, immI con) %{
3720   predicate(n->as_Vector()->length() == 16 && !VM_Version::supports_avx512vlbw());
3721   match(Set dst (ReplicateB con));
3722   format %{ "movq    $dst,[$constantaddress]\n\t"
3723             "punpcklqdq $dst,$dst\t! replicate16B($con)" %}
3724   ins_encode %{
3725     __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 1)));
3726     __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3727   %}
3728   ins_pipe( pipe_slow );
3729 %}
3730 
3731 instruct Repl32B_imm(vecY dst, immI con) %{
3732   predicate(n->as_Vector()->length() == 32 && !VM_Version::supports_avx512vlbw());
3733   match(Set dst (ReplicateB con));
3734   format %{ "movq    $dst,[$constantaddress]\n\t"
3735             "punpcklqdq $dst,$dst\n\t"
3736             "vinserti128_high $dst,$dst\t! lreplicate32B($con)" %}
3737   ins_encode %{
3738     __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 1)));
3739     __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3740     __ vinserti128_high($dst$$XMMRegister, $dst$$XMMRegister);
3741   %}
3742   ins_pipe( pipe_slow );
3743 %}
3744 
3745 instruct Repl4S(vecD dst, rRegI src) %{
3746   predicate(n->as_Vector()->length() == 4 && !VM_Version::supports_avx512vlbw());
3747   match(Set dst (ReplicateS src));
3748   format %{ "movd    $dst,$src\n\t"
3749             "pshuflw $dst,$dst,0x00\t! replicate4S" %}
3750   ins_encode %{
3751     __ movdl($dst$$XMMRegister, $src$$Register);
3752     __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
3753   %}
3754   ins_pipe( pipe_slow );
3755 %}
3756 
3757 instruct Repl4S_mem(vecD dst, memory mem) %{
3758   predicate(n->as_Vector()->length() == 4 && UseAVX > 0 && !VM_Version::supports_avx512vlbw());
3759   match(Set dst (ReplicateS (LoadS mem)));
3760   format %{ "pshuflw $dst,$mem,0x00\t! replicate4S" %}
3761   ins_encode %{
3762     __ pshuflw($dst$$XMMRegister, $mem$$Address, 0x00);
3763   %}
3764   ins_pipe( pipe_slow );
3765 %}
3766 
3767 instruct Repl8S(vecX dst, rRegI src) %{
3768   predicate(n->as_Vector()->length() == 8 && !VM_Version::supports_avx512vlbw());
3769   match(Set dst (ReplicateS src));
3770   format %{ "movd    $dst,$src\n\t"
3771             "pshuflw $dst,$dst,0x00\n\t"
3772             "punpcklqdq $dst,$dst\t! replicate8S" %}
3773   ins_encode %{
3774     __ movdl($dst$$XMMRegister, $src$$Register);
3775     __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
3776     __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3777   %}
3778   ins_pipe( pipe_slow );
3779 %}
3780 
3781 instruct Repl8S_mem(vecX dst, memory mem) %{
3782   predicate(n->as_Vector()->length() == 8 && UseAVX > 0 && !VM_Version::supports_avx512vlbw());
3783   match(Set dst (ReplicateS (LoadS mem)));
3784   format %{ "pshuflw $dst,$mem,0x00\n\t"
3785             "punpcklqdq $dst,$dst\t! replicate8S" %}
3786   ins_encode %{
3787     __ pshuflw($dst$$XMMRegister, $mem$$Address, 0x00);
3788     __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3789   %}
3790   ins_pipe( pipe_slow );
3791 %}
3792 
3793 instruct Repl8S_imm(vecX dst, immI con) %{
3794   predicate(n->as_Vector()->length() == 8 && !VM_Version::supports_avx512vlbw());
3795   match(Set dst (ReplicateS con));
3796   format %{ "movq    $dst,[$constantaddress]\n\t"
3797             "punpcklqdq $dst,$dst\t! replicate8S($con)" %}
3798   ins_encode %{
3799     __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 2)));
3800     __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3801   %}
3802   ins_pipe( pipe_slow );
3803 %}
3804 
3805 instruct Repl16S(vecY dst, rRegI src) %{
3806   predicate(n->as_Vector()->length() == 16 && !VM_Version::supports_avx512vlbw());
3807   match(Set dst (ReplicateS src));
3808   format %{ "movd    $dst,$src\n\t"
3809             "pshuflw $dst,$dst,0x00\n\t"
3810             "punpcklqdq $dst,$dst\n\t"
3811             "vinserti128_high $dst,$dst\t! replicate16S" %}
3812   ins_encode %{
3813     __ movdl($dst$$XMMRegister, $src$$Register);
3814     __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
3815     __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3816     __ vinserti128_high($dst$$XMMRegister, $dst$$XMMRegister);
3817   %}
3818   ins_pipe( pipe_slow );
3819 %}
3820 
3821 instruct Repl16S_mem(vecY dst, memory mem) %{
3822   predicate(n->as_Vector()->length() == 16 && !VM_Version::supports_avx512vlbw());
3823   match(Set dst (ReplicateS (LoadS mem)));
3824   format %{ "pshuflw $dst,$mem,0x00\n\t"
3825             "punpcklqdq $dst,$dst\n\t"
3826             "vinserti128_high $dst,$dst\t! replicate16S" %}
3827   ins_encode %{
3828     __ pshuflw($dst$$XMMRegister, $mem$$Address, 0x00);
3829     __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3830     __ vinserti128_high($dst$$XMMRegister, $dst$$XMMRegister);
3831   %}
3832   ins_pipe( pipe_slow );
3833 %}
3834 
3835 instruct Repl16S_imm(vecY dst, immI con) %{
3836   predicate(n->as_Vector()->length() == 16 && !VM_Version::supports_avx512vlbw());
3837   match(Set dst (ReplicateS con));
3838   format %{ "movq    $dst,[$constantaddress]\n\t"
3839             "punpcklqdq $dst,$dst\n\t"
3840             "vinserti128_high $dst,$dst\t! replicate16S($con)" %}
3841   ins_encode %{
3842     __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 2)));
3843     __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3844     __ vinserti128_high($dst$$XMMRegister, $dst$$XMMRegister);
3845   %}
3846   ins_pipe( pipe_slow );
3847 %}
3848 
3849 instruct Repl4I(vecX dst, rRegI src) %{
3850   predicate(n->as_Vector()->length() == 4 && !VM_Version::supports_avx512vl());
3851   match(Set dst (ReplicateI src));
3852   format %{ "movd    $dst,$src\n\t"
3853             "pshufd  $dst,$dst,0x00\t! replicate4I" %}
3854   ins_encode %{
3855     __ movdl($dst$$XMMRegister, $src$$Register);
3856     __ pshufd($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
3857   %}
3858   ins_pipe( pipe_slow );
3859 %}
3860 
3861 instruct Repl4I_mem(vecX dst, memory mem) %{
3862   predicate(n->as_Vector()->length() == 4 && UseAVX > 0 && !VM_Version::supports_avx512vl());
3863   match(Set dst (ReplicateI (LoadI mem)));
3864   format %{ "pshufd  $dst,$mem,0x00\t! replicate4I" %}
3865   ins_encode %{
3866     __ pshufd($dst$$XMMRegister, $mem$$Address, 0x00);
3867   %}
3868   ins_pipe( pipe_slow );
3869 %}
3870 
3871 instruct Repl8I(vecY dst, rRegI src) %{
3872   predicate(n->as_Vector()->length() == 8 && !VM_Version::supports_avx512vl());
3873   match(Set dst (ReplicateI src));
3874   format %{ "movd    $dst,$src\n\t"
3875             "pshufd  $dst,$dst,0x00\n\t"
3876             "vinserti128_high $dst,$dst\t! replicate8I" %}
3877   ins_encode %{
3878     __ movdl($dst$$XMMRegister, $src$$Register);
3879     __ pshufd($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
3880     __ vinserti128_high($dst$$XMMRegister, $dst$$XMMRegister);
3881   %}
3882   ins_pipe( pipe_slow );
3883 %}
3884 
3885 instruct Repl8I_mem(vecY dst, memory mem) %{
3886   predicate(n->as_Vector()->length() == 8 && !VM_Version::supports_avx512vl());
3887   match(Set dst (ReplicateI (LoadI mem)));
3888   format %{ "pshufd  $dst,$mem,0x00\n\t"
3889             "vinserti128_high $dst,$dst\t! replicate8I" %}
3890   ins_encode %{
3891     __ pshufd($dst$$XMMRegister, $mem$$Address, 0x00);
3892     __ vinserti128_high($dst$$XMMRegister, $dst$$XMMRegister);
3893   %}
3894   ins_pipe( pipe_slow );
3895 %}
3896 
3897 instruct Repl4I_imm(vecX dst, immI con) %{
3898   predicate(n->as_Vector()->length() == 4 && !VM_Version::supports_avx512vl());
3899   match(Set dst (ReplicateI con));
3900   format %{ "movq    $dst,[$constantaddress]\t! replicate4I($con)\n\t"
3901             "punpcklqdq $dst,$dst" %}
3902   ins_encode %{
3903     __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 4)));
3904     __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3905   %}
3906   ins_pipe( pipe_slow );
3907 %}
3908 
3909 instruct Repl8I_imm(vecY dst, immI con) %{
3910   predicate(n->as_Vector()->length() == 8 && !VM_Version::supports_avx512vl());
3911   match(Set dst (ReplicateI con));
3912   format %{ "movq    $dst,[$constantaddress]\t! replicate8I($con)\n\t"
3913             "punpcklqdq $dst,$dst\n\t"
3914             "vinserti128_high $dst,$dst" %}
3915   ins_encode %{
3916     __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 4)));
3917     __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3918     __ vinserti128_high($dst$$XMMRegister, $dst$$XMMRegister);
3919   %}
3920   ins_pipe( pipe_slow );
3921 %}
3922 
3923 // Long could be loaded into xmm register directly from memory.
3924 instruct Repl2L_mem(vecX dst, memory mem) %{
3925   predicate(n->as_Vector()->length() == 2 && !VM_Version::supports_avx512vlbw());
3926   match(Set dst (ReplicateL (LoadL mem)));
3927   format %{ "movq    $dst,$mem\n\t"
3928             "punpcklqdq $dst,$dst\t! replicate2L" %}
3929   ins_encode %{
3930     __ movq($dst$$XMMRegister, $mem$$Address);
3931     __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3932   %}
3933   ins_pipe( pipe_slow );
3934 %}
3935 
3936 // Replicate long (8 byte) scalar to be vector
3937 #ifdef _LP64
3938 instruct Repl4L(vecY dst, rRegL src) %{
3939   predicate(n->as_Vector()->length() == 4 && !VM_Version::supports_avx512vl());
3940   match(Set dst (ReplicateL src));
3941   format %{ "movdq   $dst,$src\n\t"
3942             "punpcklqdq $dst,$dst\n\t"
3943             "vinserti128_high $dst,$dst\t! replicate4L" %}
3944   ins_encode %{
3945     __ movdq($dst$$XMMRegister, $src$$Register);
3946     __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3947     __ vinserti128_high($dst$$XMMRegister, $dst$$XMMRegister);
3948   %}
3949   ins_pipe( pipe_slow );
3950 %}
3951 #else // _LP64
3952 instruct Repl4L(vecY dst, eRegL src, regD tmp) %{
3953   predicate(n->as_Vector()->length() == 4 && !VM_Version::supports_avx512vl());
3954   match(Set dst (ReplicateL src));
3955   effect(TEMP dst, USE src, TEMP tmp);
3956   format %{ "movdl   $dst,$src.lo\n\t"
3957             "movdl   $tmp,$src.hi\n\t"
3958             "punpckldq $dst,$tmp\n\t"
3959             "punpcklqdq $dst,$dst\n\t"
3960             "vinserti128_high $dst,$dst\t! replicate4L" %}
3961   ins_encode %{
3962     __ movdl($dst$$XMMRegister, $src$$Register);
3963     __ movdl($tmp$$XMMRegister, HIGH_FROM_LOW($src$$Register));
3964     __ punpckldq($dst$$XMMRegister, $tmp$$XMMRegister);
3965     __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3966     __ vinserti128_high($dst$$XMMRegister, $dst$$XMMRegister);
3967   %}
3968   ins_pipe( pipe_slow );
3969 %}
3970 #endif // _LP64
3971 
3972 instruct Repl4L_imm(vecY dst, immL con) %{
3973   predicate(n->as_Vector()->length() == 4 && !VM_Version::supports_avx512vl());
3974   match(Set dst (ReplicateL con));
3975   format %{ "movq    $dst,[$constantaddress]\n\t"
3976             "punpcklqdq $dst,$dst\n\t"
3977             "vinserti128_high $dst,$dst\t! replicate4L($con)" %}
3978   ins_encode %{
3979     __ movq($dst$$XMMRegister, $constantaddress($con));
3980     __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3981     __ vinserti128_high($dst$$XMMRegister, $dst$$XMMRegister);
3982   %}
3983   ins_pipe( pipe_slow );
3984 %}
3985 
3986 instruct Repl4L_mem(vecY dst, memory mem) %{
3987   predicate(n->as_Vector()->length() == 4 && !VM_Version::supports_avx512vl());
3988   match(Set dst (ReplicateL (LoadL mem)));
3989   format %{ "movq    $dst,$mem\n\t"
3990             "punpcklqdq $dst,$dst\n\t"
3991             "vinserti128_high $dst,$dst\t! replicate4L" %}
3992   ins_encode %{
3993     __ movq($dst$$XMMRegister, $mem$$Address);
3994     __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
3995     __ vinserti128_high($dst$$XMMRegister, $dst$$XMMRegister);
3996   %}
3997   ins_pipe( pipe_slow );
3998 %}
3999 
4000 instruct Repl2F_mem(vecD dst, memory mem) %{
4001   predicate(n->as_Vector()->length() == 2 && UseAVX > 0 && !VM_Version::supports_avx512vl());
4002   match(Set dst (ReplicateF (LoadF mem)));
4003   format %{ "pshufd  $dst,$mem,0x00\t! replicate2F" %}
4004   ins_encode %{
4005     __ pshufd($dst$$XMMRegister, $mem$$Address, 0x00);
4006   %}
4007   ins_pipe( pipe_slow );
4008 %}
4009 
4010 instruct Repl4F_mem(vecX dst, memory mem) %{
4011   predicate(n->as_Vector()->length() == 4 && UseAVX > 0 && !VM_Version::supports_avx512vl());
4012   match(Set dst (ReplicateF (LoadF mem)));
4013   format %{ "pshufd  $dst,$mem,0x00\t! replicate4F" %}
4014   ins_encode %{
4015     __ pshufd($dst$$XMMRegister, $mem$$Address, 0x00);
4016   %}
4017   ins_pipe( pipe_slow );
4018 %}
4019 
4020 instruct Repl8F(vecY dst, regF src) %{
4021   predicate(n->as_Vector()->length() == 8 && !VM_Version::supports_avx512vl());
4022   match(Set dst (ReplicateF src));
4023   format %{ "pshufd  $dst,$src,0x00\n\t"
4024             "vinsertf128_high $dst,$dst\t! replicate8F" %}
4025   ins_encode %{
4026     __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0x00);
4027     __ vinsertf128_high($dst$$XMMRegister, $dst$$XMMRegister);
4028   %}
4029   ins_pipe( pipe_slow );
4030 %}
4031 
4032 instruct Repl8F_mem(vecY dst, memory mem) %{
4033   predicate(n->as_Vector()->length() == 8 && !VM_Version::supports_avx512vl());
4034   match(Set dst (ReplicateF (LoadF mem)));
4035   format %{ "pshufd  $dst,$mem,0x00\n\t"
4036             "vinsertf128_high $dst,$dst\t! replicate8F" %}
4037   ins_encode %{
4038     __ pshufd($dst$$XMMRegister, $mem$$Address, 0x00);
4039     __ vinsertf128_high($dst$$XMMRegister, $dst$$XMMRegister);
4040   %}
4041   ins_pipe( pipe_slow );
4042 %}
4043 
4044 instruct Repl2F_zero(vecD dst, immF0 zero) %{
4045   predicate(n->as_Vector()->length() == 2 && UseAVX < 3);
4046   match(Set dst (ReplicateF zero));
4047   format %{ "xorps   $dst,$dst\t! replicate2F zero" %}
4048   ins_encode %{
4049     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
4050   %}
4051   ins_pipe( fpu_reg_reg );
4052 %}
4053 
4054 instruct Repl4F_zero(vecX dst, immF0 zero) %{
4055   predicate(n->as_Vector()->length() == 4 && UseAVX < 3);
4056   match(Set dst (ReplicateF zero));
4057   format %{ "xorps   $dst,$dst\t! replicate4F zero" %}
4058   ins_encode %{
4059     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
4060   %}
4061   ins_pipe( fpu_reg_reg );
4062 %}
4063 
4064 instruct Repl8F_zero(vecY dst, immF0 zero) %{
4065   predicate(n->as_Vector()->length() == 8 && UseAVX < 3);
4066   match(Set dst (ReplicateF zero));
4067   format %{ "vxorps  $dst,$dst,$dst\t! replicate8F zero" %}
4068   ins_encode %{
4069     int vector_len = 1;
4070     __ vxorps($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4071   %}
4072   ins_pipe( fpu_reg_reg );
4073 %}
4074 
4075 instruct Repl2D_mem(vecX dst, memory mem) %{
4076   predicate(n->as_Vector()->length() == 2 && UseAVX > 0 && !VM_Version::supports_avx512vl());
4077   match(Set dst (ReplicateD (LoadD mem)));
4078   format %{ "pshufd  $dst,$mem,0x44\t! replicate2D" %}
4079   ins_encode %{
4080     __ pshufd($dst$$XMMRegister, $mem$$Address, 0x44);
4081   %}
4082   ins_pipe( pipe_slow );
4083 %}
4084 
4085 instruct Repl4D(vecY dst, regD src) %{
4086   predicate(n->as_Vector()->length() == 4 && !VM_Version::supports_avx512vl());
4087   match(Set dst (ReplicateD src));
4088   format %{ "pshufd  $dst,$src,0x44\n\t"
4089             "vinsertf128_high $dst,$dst\t! replicate4D" %}
4090   ins_encode %{
4091     __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0x44);
4092     __ vinsertf128_high($dst$$XMMRegister, $dst$$XMMRegister);
4093   %}
4094   ins_pipe( pipe_slow );
4095 %}
4096 
4097 instruct Repl4D_mem(vecY dst, memory mem) %{
4098   predicate(n->as_Vector()->length() == 4 && !VM_Version::supports_avx512vl());
4099   match(Set dst (ReplicateD (LoadD mem)));
4100   format %{ "pshufd  $dst,$mem,0x44\n\t"
4101             "vinsertf128_high $dst,$dst\t! replicate4D" %}
4102   ins_encode %{
4103     __ pshufd($dst$$XMMRegister, $mem$$Address, 0x44);
4104     __ vinsertf128_high($dst$$XMMRegister, $dst$$XMMRegister);
4105   %}
4106   ins_pipe( pipe_slow );
4107 %}
4108 
4109 // Replicate double (8 byte) scalar zero to be vector
4110 instruct Repl2D_zero(vecX dst, immD0 zero) %{
4111   predicate(n->as_Vector()->length() == 2 && UseAVX < 3);
4112   match(Set dst (ReplicateD zero));
4113   format %{ "xorpd   $dst,$dst\t! replicate2D zero" %}
4114   ins_encode %{
4115     __ xorpd($dst$$XMMRegister, $dst$$XMMRegister);
4116   %}
4117   ins_pipe( fpu_reg_reg );
4118 %}
4119 
4120 instruct Repl4D_zero(vecY dst, immD0 zero) %{
4121   predicate(n->as_Vector()->length() == 4 && UseAVX < 3);
4122   match(Set dst (ReplicateD zero));
4123   format %{ "vxorpd  $dst,$dst,$dst,vect256\t! replicate4D zero" %}
4124   ins_encode %{
4125     int vector_len = 1;
4126     __ vxorpd($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4127   %}
4128   ins_pipe( fpu_reg_reg );
4129 %}
4130 
4131 // ====================GENERIC REPLICATE==========================================
4132 
4133 // Replicate byte scalar to be vector
4134 instruct Repl4B(vecS dst, rRegI src) %{
4135   predicate(n->as_Vector()->length() == 4);
4136   match(Set dst (ReplicateB src));
4137   format %{ "movd    $dst,$src\n\t"
4138             "punpcklbw $dst,$dst\n\t"
4139             "pshuflw $dst,$dst,0x00\t! replicate4B" %}
4140   ins_encode %{
4141     __ movdl($dst$$XMMRegister, $src$$Register);
4142     __ punpcklbw($dst$$XMMRegister, $dst$$XMMRegister);
4143     __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
4144   %}
4145   ins_pipe( pipe_slow );
4146 %}
4147 
4148 instruct Repl8B(vecD dst, rRegI src) %{
4149   predicate(n->as_Vector()->length() == 8);
4150   match(Set dst (ReplicateB src));
4151   format %{ "movd    $dst,$src\n\t"
4152             "punpcklbw $dst,$dst\n\t"
4153             "pshuflw $dst,$dst,0x00\t! replicate8B" %}
4154   ins_encode %{
4155     __ movdl($dst$$XMMRegister, $src$$Register);
4156     __ punpcklbw($dst$$XMMRegister, $dst$$XMMRegister);
4157     __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
4158   %}
4159   ins_pipe( pipe_slow );
4160 %}
4161 
4162 // Replicate byte scalar immediate to be vector by loading from const table.
4163 instruct Repl4B_imm(vecS dst, immI con) %{
4164   predicate(n->as_Vector()->length() == 4);
4165   match(Set dst (ReplicateB con));
4166   format %{ "movdl   $dst,[$constantaddress]\t! replicate4B($con)" %}
4167   ins_encode %{
4168     __ movdl($dst$$XMMRegister, $constantaddress(replicate4_imm($con$$constant, 1)));
4169   %}
4170   ins_pipe( pipe_slow );
4171 %}
4172 
4173 instruct Repl8B_imm(vecD dst, immI con) %{
4174   predicate(n->as_Vector()->length() == 8);
4175   match(Set dst (ReplicateB con));
4176   format %{ "movq    $dst,[$constantaddress]\t! replicate8B($con)" %}
4177   ins_encode %{
4178     __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 1)));
4179   %}
4180   ins_pipe( pipe_slow );
4181 %}
4182 
4183 // Replicate byte scalar zero to be vector
4184 instruct Repl4B_zero(vecS dst, immI0 zero) %{
4185   predicate(n->as_Vector()->length() == 4);
4186   match(Set dst (ReplicateB zero));
4187   format %{ "pxor    $dst,$dst\t! replicate4B zero" %}
4188   ins_encode %{
4189     __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
4190   %}
4191   ins_pipe( fpu_reg_reg );
4192 %}
4193 
4194 instruct Repl8B_zero(vecD dst, immI0 zero) %{
4195   predicate(n->as_Vector()->length() == 8);
4196   match(Set dst (ReplicateB zero));
4197   format %{ "pxor    $dst,$dst\t! replicate8B zero" %}
4198   ins_encode %{
4199     __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
4200   %}
4201   ins_pipe( fpu_reg_reg );
4202 %}
4203 
4204 instruct Repl16B_zero(vecX dst, immI0 zero) %{
4205   predicate(n->as_Vector()->length() == 16);
4206   match(Set dst (ReplicateB zero));
4207   format %{ "pxor    $dst,$dst\t! replicate16B zero" %}
4208   ins_encode %{
4209     __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
4210   %}
4211   ins_pipe( fpu_reg_reg );
4212 %}
4213 
4214 instruct Repl32B_zero(vecY dst, immI0 zero) %{
4215   predicate(n->as_Vector()->length() == 32);
4216   match(Set dst (ReplicateB zero));
4217   format %{ "vpxor   $dst,$dst,$dst\t! replicate32B zero" %}
4218   ins_encode %{
4219     // Use vxorpd since AVX does not have vpxor for 256-bit (AVX2 will have it).
4220     int vector_len = 1;
4221     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4222   %}
4223   ins_pipe( fpu_reg_reg );
4224 %}
4225 
4226 // Replicate char/short (2 byte) scalar to be vector
4227 instruct Repl2S(vecS dst, rRegI src) %{
4228   predicate(n->as_Vector()->length() == 2);
4229   match(Set dst (ReplicateS src));
4230   format %{ "movd    $dst,$src\n\t"
4231             "pshuflw $dst,$dst,0x00\t! replicate2S" %}
4232   ins_encode %{
4233     __ movdl($dst$$XMMRegister, $src$$Register);
4234     __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
4235   %}
4236   ins_pipe( fpu_reg_reg );
4237 %}
4238 
4239 // Replicate char/short (2 byte) scalar immediate to be vector by loading from const table.
4240 instruct Repl2S_imm(vecS dst, immI con) %{
4241   predicate(n->as_Vector()->length() == 2);
4242   match(Set dst (ReplicateS con));
4243   format %{ "movdl   $dst,[$constantaddress]\t! replicate2S($con)" %}
4244   ins_encode %{
4245     __ movdl($dst$$XMMRegister, $constantaddress(replicate4_imm($con$$constant, 2)));
4246   %}
4247   ins_pipe( fpu_reg_reg );
4248 %}
4249 
4250 instruct Repl4S_imm(vecD dst, immI con) %{
4251   predicate(n->as_Vector()->length() == 4);
4252   match(Set dst (ReplicateS con));
4253   format %{ "movq    $dst,[$constantaddress]\t! replicate4S($con)" %}
4254   ins_encode %{
4255     __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 2)));
4256   %}
4257   ins_pipe( fpu_reg_reg );
4258 %}
4259 
4260 // Replicate char/short (2 byte) scalar zero to be vector
4261 instruct Repl2S_zero(vecS dst, immI0 zero) %{
4262   predicate(n->as_Vector()->length() == 2);
4263   match(Set dst (ReplicateS zero));
4264   format %{ "pxor    $dst,$dst\t! replicate2S zero" %}
4265   ins_encode %{
4266     __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
4267   %}
4268   ins_pipe( fpu_reg_reg );
4269 %}
4270 
4271 instruct Repl4S_zero(vecD dst, immI0 zero) %{
4272   predicate(n->as_Vector()->length() == 4);
4273   match(Set dst (ReplicateS zero));
4274   format %{ "pxor    $dst,$dst\t! replicate4S zero" %}
4275   ins_encode %{
4276     __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
4277   %}
4278   ins_pipe( fpu_reg_reg );
4279 %}
4280 
4281 instruct Repl8S_zero(vecX dst, immI0 zero) %{
4282   predicate(n->as_Vector()->length() == 8);
4283   match(Set dst (ReplicateS zero));
4284   format %{ "pxor    $dst,$dst\t! replicate8S zero" %}
4285   ins_encode %{
4286     __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
4287   %}
4288   ins_pipe( fpu_reg_reg );
4289 %}
4290 
4291 instruct Repl16S_zero(vecY dst, immI0 zero) %{
4292   predicate(n->as_Vector()->length() == 16);
4293   match(Set dst (ReplicateS zero));
4294   format %{ "vpxor   $dst,$dst,$dst\t! replicate16S zero" %}
4295   ins_encode %{
4296     // Use vxorpd since AVX does not have vpxor for 256-bit (AVX2 will have it).
4297     int vector_len = 1;
4298     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4299   %}
4300   ins_pipe( fpu_reg_reg );
4301 %}
4302 
4303 // Replicate integer (4 byte) scalar to be vector
4304 instruct Repl2I(vecD dst, rRegI src) %{
4305   predicate(n->as_Vector()->length() == 2);
4306   match(Set dst (ReplicateI src));
4307   format %{ "movd    $dst,$src\n\t"
4308             "pshufd  $dst,$dst,0x00\t! replicate2I" %}
4309   ins_encode %{
4310     __ movdl($dst$$XMMRegister, $src$$Register);
4311     __ pshufd($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
4312   %}
4313   ins_pipe( fpu_reg_reg );
4314 %}
4315 
4316 // Integer could be loaded into xmm register directly from memory.
4317 instruct Repl2I_mem(vecD dst, memory mem) %{
4318   predicate(n->as_Vector()->length() == 2);
4319   match(Set dst (ReplicateI (LoadI mem)));
4320   format %{ "movd    $dst,$mem\n\t"
4321             "pshufd  $dst,$dst,0x00\t! replicate2I" %}
4322   ins_encode %{
4323     __ movdl($dst$$XMMRegister, $mem$$Address);
4324     __ pshufd($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
4325   %}
4326   ins_pipe( fpu_reg_reg );
4327 %}
4328 
4329 // Replicate integer (4 byte) scalar immediate to be vector by loading from const table.
4330 instruct Repl2I_imm(vecD dst, immI con) %{
4331   predicate(n->as_Vector()->length() == 2);
4332   match(Set dst (ReplicateI con));
4333   format %{ "movq    $dst,[$constantaddress]\t! replicate2I($con)" %}
4334   ins_encode %{
4335     __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 4)));
4336   %}
4337   ins_pipe( fpu_reg_reg );
4338 %}
4339 
4340 // Replicate integer (4 byte) scalar zero to be vector
4341 instruct Repl2I_zero(vecD dst, immI0 zero) %{
4342   predicate(n->as_Vector()->length() == 2);
4343   match(Set dst (ReplicateI zero));
4344   format %{ "pxor    $dst,$dst\t! replicate2I" %}
4345   ins_encode %{
4346     __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
4347   %}
4348   ins_pipe( fpu_reg_reg );
4349 %}
4350 
4351 instruct Repl4I_zero(vecX dst, immI0 zero) %{
4352   predicate(n->as_Vector()->length() == 4);
4353   match(Set dst (ReplicateI zero));
4354   format %{ "pxor    $dst,$dst\t! replicate4I zero)" %}
4355   ins_encode %{
4356     __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
4357   %}
4358   ins_pipe( fpu_reg_reg );
4359 %}
4360 
4361 instruct Repl8I_zero(vecY dst, immI0 zero) %{
4362   predicate(n->as_Vector()->length() == 8);
4363   match(Set dst (ReplicateI zero));
4364   format %{ "vpxor   $dst,$dst,$dst\t! replicate8I zero" %}
4365   ins_encode %{
4366     // Use vxorpd since AVX does not have vpxor for 256-bit (AVX2 will have it).
4367     int vector_len = 1;
4368     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4369   %}
4370   ins_pipe( fpu_reg_reg );
4371 %}
4372 
4373 // Replicate long (8 byte) scalar to be vector
4374 #ifdef _LP64
4375 instruct Repl2L(vecX dst, rRegL src) %{
4376   predicate(n->as_Vector()->length() == 2);
4377   match(Set dst (ReplicateL src));
4378   format %{ "movdq   $dst,$src\n\t"
4379             "punpcklqdq $dst,$dst\t! replicate2L" %}
4380   ins_encode %{
4381     __ movdq($dst$$XMMRegister, $src$$Register);
4382     __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
4383   %}
4384   ins_pipe( pipe_slow );
4385 %}
4386 #else // _LP64
4387 instruct Repl2L(vecX dst, eRegL src, regD tmp) %{
4388   predicate(n->as_Vector()->length() == 2);
4389   match(Set dst (ReplicateL src));
4390   effect(TEMP dst, USE src, TEMP tmp);
4391   format %{ "movdl   $dst,$src.lo\n\t"
4392             "movdl   $tmp,$src.hi\n\t"
4393             "punpckldq $dst,$tmp\n\t"
4394             "punpcklqdq $dst,$dst\t! replicate2L"%}
4395   ins_encode %{
4396     __ movdl($dst$$XMMRegister, $src$$Register);
4397     __ movdl($tmp$$XMMRegister, HIGH_FROM_LOW($src$$Register));
4398     __ punpckldq($dst$$XMMRegister, $tmp$$XMMRegister);
4399     __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
4400   %}
4401   ins_pipe( pipe_slow );
4402 %}
4403 #endif // _LP64
4404 
4405 // Replicate long (8 byte) scalar immediate to be vector by loading from const table.
4406 instruct Repl2L_imm(vecX dst, immL con) %{
4407   predicate(n->as_Vector()->length() == 2);
4408   match(Set dst (ReplicateL con));
4409   format %{ "movq    $dst,[$constantaddress]\n\t"
4410             "punpcklqdq $dst,$dst\t! replicate2L($con)" %}
4411   ins_encode %{
4412     __ movq($dst$$XMMRegister, $constantaddress($con));
4413     __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
4414   %}
4415   ins_pipe( pipe_slow );
4416 %}
4417 
4418 // Replicate long (8 byte) scalar zero to be vector
4419 instruct Repl2L_zero(vecX dst, immL0 zero) %{
4420   predicate(n->as_Vector()->length() == 2);
4421   match(Set dst (ReplicateL zero));
4422   format %{ "pxor    $dst,$dst\t! replicate2L zero" %}
4423   ins_encode %{
4424     __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
4425   %}
4426   ins_pipe( fpu_reg_reg );
4427 %}
4428 
4429 instruct Repl4L_zero(vecY dst, immL0 zero) %{
4430   predicate(n->as_Vector()->length() == 4);
4431   match(Set dst (ReplicateL zero));
4432   format %{ "vpxor   $dst,$dst,$dst\t! replicate4L zero" %}
4433   ins_encode %{
4434     // Use vxorpd since AVX does not have vpxor for 256-bit (AVX2 will have it).
4435     int vector_len = 1;
4436     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4437   %}
4438   ins_pipe( fpu_reg_reg );
4439 %}
4440 
4441 // Replicate float (4 byte) scalar to be vector
4442 instruct Repl2F(vecD dst, regF src) %{
4443   predicate(n->as_Vector()->length() == 2);
4444   match(Set dst (ReplicateF src));
4445   format %{ "pshufd  $dst,$dst,0x00\t! replicate2F" %}
4446   ins_encode %{
4447     __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0x00);
4448   %}
4449   ins_pipe( fpu_reg_reg );
4450 %}
4451 
4452 instruct Repl4F(vecX dst, regF src) %{
4453   predicate(n->as_Vector()->length() == 4);
4454   match(Set dst (ReplicateF src));
4455   format %{ "pshufd  $dst,$dst,0x00\t! replicate4F" %}
4456   ins_encode %{
4457     __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0x00);
4458   %}
4459   ins_pipe( pipe_slow );
4460 %}
4461 
4462 // Replicate double (8 bytes) scalar to be vector
4463 instruct Repl2D(vecX dst, regD src) %{
4464   predicate(n->as_Vector()->length() == 2);
4465   match(Set dst (ReplicateD src));
4466   format %{ "pshufd  $dst,$src,0x44\t! replicate2D" %}
4467   ins_encode %{
4468     __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0x44);
4469   %}
4470   ins_pipe( pipe_slow );
4471 %}
4472 
4473 // ====================EVEX REPLICATE=============================================
4474 
4475 instruct Repl4B_mem_evex(vecS dst, memory mem) %{
4476   predicate(n->as_Vector()->length() == 4 && VM_Version::supports_avx512vlbw());
4477   match(Set dst (ReplicateB (LoadB mem)));
4478   format %{ "vpbroadcastb  $dst,$mem\t! replicate4B" %}
4479   ins_encode %{
4480     int vector_len = 0;
4481     __ evpbroadcastb($dst$$XMMRegister, $mem$$Address, vector_len);
4482   %}
4483   ins_pipe( pipe_slow );
4484 %}
4485 
4486 instruct Repl8B_mem_evex(vecD dst, memory mem) %{
4487   predicate(n->as_Vector()->length() == 8 && VM_Version::supports_avx512vlbw());
4488   match(Set dst (ReplicateB (LoadB mem)));
4489   format %{ "vpbroadcastb  $dst,$mem\t! replicate8B" %}
4490   ins_encode %{
4491     int vector_len = 0;
4492     __ evpbroadcastb($dst$$XMMRegister, $mem$$Address, vector_len);
4493   %}
4494   ins_pipe( pipe_slow );
4495 %}
4496 
4497 instruct Repl16B_evex(vecX dst, rRegI src) %{
4498   predicate(n->as_Vector()->length() == 16 && VM_Version::supports_avx512vlbw());
4499   match(Set dst (ReplicateB src));
4500   format %{ "vpbroadcastb $dst,$src\t! replicate16B" %}
4501   ins_encode %{
4502    int vector_len = 0;
4503     __ evpbroadcastb($dst$$XMMRegister, $src$$Register, vector_len);
4504   %}
4505   ins_pipe( pipe_slow );
4506 %}
4507 
4508 instruct Repl16B_mem_evex(vecX dst, memory mem) %{
4509   predicate(n->as_Vector()->length() == 16 && VM_Version::supports_avx512vlbw());
4510   match(Set dst (ReplicateB (LoadB mem)));
4511   format %{ "vpbroadcastb  $dst,$mem\t! replicate16B" %}
4512   ins_encode %{
4513     int vector_len = 0;
4514     __ evpbroadcastb($dst$$XMMRegister, $mem$$Address, vector_len);
4515   %}
4516   ins_pipe( pipe_slow );
4517 %}
4518 
4519 instruct Repl32B_evex(vecY dst, rRegI src) %{
4520   predicate(n->as_Vector()->length() == 32 && VM_Version::supports_avx512vlbw());
4521   match(Set dst (ReplicateB src));
4522   format %{ "vpbroadcastb $dst,$src\t! replicate32B" %}
4523   ins_encode %{
4524    int vector_len = 1;
4525     __ evpbroadcastb($dst$$XMMRegister, $src$$Register, vector_len);
4526   %}
4527   ins_pipe( pipe_slow );
4528 %}
4529 
4530 instruct Repl32B_mem_evex(vecY dst, memory mem) %{
4531   predicate(n->as_Vector()->length() == 32 && VM_Version::supports_avx512vlbw());
4532   match(Set dst (ReplicateB (LoadB mem)));
4533   format %{ "vpbroadcastb  $dst,$mem\t! replicate32B" %}
4534   ins_encode %{
4535     int vector_len = 1;
4536     __ evpbroadcastb($dst$$XMMRegister, $mem$$Address, vector_len);
4537   %}
4538   ins_pipe( pipe_slow );
4539 %}
4540 
4541 instruct Repl64B_evex(vecZ dst, rRegI src) %{
4542   predicate(n->as_Vector()->length() == 64 && VM_Version::supports_avx512bw());
4543   match(Set dst (ReplicateB src));
4544   format %{ "vpbroadcastb $dst,$src\t! upper replicate64B" %}
4545   ins_encode %{
4546    int vector_len = 2;
4547     __ evpbroadcastb($dst$$XMMRegister, $src$$Register, vector_len);
4548   %}
4549   ins_pipe( pipe_slow );
4550 %}
4551 
4552 instruct Repl64B_mem_evex(vecZ dst, memory mem) %{
4553   predicate(n->as_Vector()->length() == 64 && VM_Version::supports_avx512bw());
4554   match(Set dst (ReplicateB (LoadB mem)));
4555   format %{ "vpbroadcastb  $dst,$mem\t! replicate64B" %}
4556   ins_encode %{
4557     int vector_len = 2;
4558     __ evpbroadcastb($dst$$XMMRegister, $mem$$Address, vector_len);
4559   %}
4560   ins_pipe( pipe_slow );
4561 %}
4562 
4563 instruct Repl16B_imm_evex(vecX dst, immI con) %{
4564   predicate(n->as_Vector()->length() == 16 && VM_Version::supports_avx512vlbw());
4565   match(Set dst (ReplicateB con));
4566   format %{ "movq    $dst,[$constantaddress]\n\t"
4567             "vpbroadcastb $dst,$dst\t! replicate16B" %}
4568   ins_encode %{
4569    int vector_len = 0;
4570     __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 1)));
4571     __ evpbroadcastb($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4572   %}
4573   ins_pipe( pipe_slow );
4574 %}
4575 
4576 instruct Repl32B_imm_evex(vecY dst, immI con) %{
4577   predicate(n->as_Vector()->length() == 32 && VM_Version::supports_avx512vlbw());
4578   match(Set dst (ReplicateB con));
4579   format %{ "movq    $dst,[$constantaddress]\n\t"
4580             "vpbroadcastb $dst,$dst\t! replicate32B" %}
4581   ins_encode %{
4582    int vector_len = 1;
4583     __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 1)));
4584     __ evpbroadcastb($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4585   %}
4586   ins_pipe( pipe_slow );
4587 %}
4588 
4589 instruct Repl64B_imm_evex(vecZ dst, immI con) %{
4590   predicate(n->as_Vector()->length() == 64 && VM_Version::supports_avx512bw());
4591   match(Set dst (ReplicateB con));
4592   format %{ "movq    $dst,[$constantaddress]\n\t"
4593             "vpbroadcastb $dst,$dst\t! upper replicate64B" %}
4594   ins_encode %{
4595    int vector_len = 2;
4596     __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 1)));
4597     __ evpbroadcastb($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4598   %}
4599   ins_pipe( pipe_slow );
4600 %}
4601 
4602 instruct Repl64B_zero_evex(vecZ dst, immI0 zero) %{
4603   predicate(n->as_Vector()->length() == 64 && UseAVX > 2);
4604   match(Set dst (ReplicateB zero));
4605   format %{ "vpxor   $dst k0,$dst,$dst\t! replicate64B zero" %}
4606   ins_encode %{
4607     // Use vxorpd since AVX does not have vpxor for 512-bit (EVEX will have it).
4608     int vector_len = 2;
4609     __ vpxor($dst$$XMMRegister,$dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4610   %}
4611   ins_pipe( fpu_reg_reg );
4612 %}
4613 
4614 instruct Repl4S_evex(vecD dst, rRegI src) %{
4615   predicate(n->as_Vector()->length() == 4 && VM_Version::supports_avx512vlbw());
4616   match(Set dst (ReplicateS src));
4617   format %{ "vpbroadcastw $dst,$src\t! replicate4S" %}
4618   ins_encode %{
4619    int vector_len = 0;
4620     __ evpbroadcastw($dst$$XMMRegister, $src$$Register, vector_len);
4621   %}
4622   ins_pipe( pipe_slow );
4623 %}
4624 
4625 instruct Repl4S_mem_evex(vecD dst, memory mem) %{
4626   predicate(n->as_Vector()->length() == 4 && VM_Version::supports_avx512vlbw());
4627   match(Set dst (ReplicateS (LoadS mem)));
4628   format %{ "vpbroadcastw  $dst,$mem\t! replicate4S" %}
4629   ins_encode %{
4630     int vector_len = 0;
4631     __ evpbroadcastw($dst$$XMMRegister, $mem$$Address, vector_len);
4632   %}
4633   ins_pipe( pipe_slow );
4634 %}
4635 
4636 instruct Repl8S_evex(vecX dst, rRegI src) %{
4637   predicate(n->as_Vector()->length() == 8 && VM_Version::supports_avx512vlbw());
4638   match(Set dst (ReplicateS src));
4639   format %{ "vpbroadcastw $dst,$src\t! replicate8S" %}
4640   ins_encode %{
4641    int vector_len = 0;
4642     __ evpbroadcastw($dst$$XMMRegister, $src$$Register, vector_len);
4643   %}
4644   ins_pipe( pipe_slow );
4645 %}
4646 
4647 instruct Repl8S_mem_evex(vecX dst, memory mem) %{
4648   predicate(n->as_Vector()->length() == 8 && VM_Version::supports_avx512vlbw());
4649   match(Set dst (ReplicateS (LoadS mem)));
4650   format %{ "vpbroadcastw  $dst,$mem\t! replicate8S" %}
4651   ins_encode %{
4652     int vector_len = 0;
4653     __ evpbroadcastw($dst$$XMMRegister, $mem$$Address, vector_len);
4654   %}
4655   ins_pipe( pipe_slow );
4656 %}
4657 
4658 instruct Repl16S_evex(vecY dst, rRegI src) %{
4659   predicate(n->as_Vector()->length() == 16 && VM_Version::supports_avx512vlbw());
4660   match(Set dst (ReplicateS src));
4661   format %{ "vpbroadcastw $dst,$src\t! replicate16S" %}
4662   ins_encode %{
4663    int vector_len = 1;
4664     __ evpbroadcastw($dst$$XMMRegister, $src$$Register, vector_len);
4665   %}
4666   ins_pipe( pipe_slow );
4667 %}
4668 
4669 instruct Repl16S_mem_evex(vecY dst, memory mem) %{
4670   predicate(n->as_Vector()->length() == 16 && VM_Version::supports_avx512vlbw());
4671   match(Set dst (ReplicateS (LoadS mem)));
4672   format %{ "vpbroadcastw  $dst,$mem\t! replicate16S" %}
4673   ins_encode %{
4674     int vector_len = 1;
4675     __ evpbroadcastw($dst$$XMMRegister, $mem$$Address, vector_len);
4676   %}
4677   ins_pipe( pipe_slow );
4678 %}
4679 
4680 instruct Repl32S_evex(vecZ dst, rRegI src) %{
4681   predicate(n->as_Vector()->length() == 32 && VM_Version::supports_avx512bw());
4682   match(Set dst (ReplicateS src));
4683   format %{ "vpbroadcastw $dst,$src\t! replicate32S" %}
4684   ins_encode %{
4685    int vector_len = 2;
4686     __ evpbroadcastw($dst$$XMMRegister, $src$$Register, vector_len);
4687   %}
4688   ins_pipe( pipe_slow );
4689 %}
4690 
4691 instruct Repl32S_mem_evex(vecZ dst, memory mem) %{
4692   predicate(n->as_Vector()->length() == 32 && VM_Version::supports_avx512bw());
4693   match(Set dst (ReplicateS (LoadS mem)));
4694   format %{ "vpbroadcastw  $dst,$mem\t! replicate32S" %}
4695   ins_encode %{
4696     int vector_len = 2;
4697     __ evpbroadcastw($dst$$XMMRegister, $mem$$Address, vector_len);
4698   %}
4699   ins_pipe( pipe_slow );
4700 %}
4701 
4702 instruct Repl8S_imm_evex(vecX dst, immI con) %{
4703   predicate(n->as_Vector()->length() == 8 && VM_Version::supports_avx512vlbw());
4704   match(Set dst (ReplicateS con));
4705   format %{ "movq    $dst,[$constantaddress]\n\t"
4706             "vpbroadcastw $dst,$dst\t! replicate8S" %}
4707   ins_encode %{
4708    int vector_len = 0;
4709     __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 2)));
4710     __ evpbroadcastw($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4711   %}
4712   ins_pipe( pipe_slow );
4713 %}
4714 
4715 instruct Repl16S_imm_evex(vecY dst, immI con) %{
4716   predicate(n->as_Vector()->length() == 16 && VM_Version::supports_avx512vlbw());
4717   match(Set dst (ReplicateS con));
4718   format %{ "movq    $dst,[$constantaddress]\n\t"
4719             "vpbroadcastw $dst,$dst\t! replicate16S" %}
4720   ins_encode %{
4721    int vector_len = 1;
4722     __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 2)));
4723     __ evpbroadcastw($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4724   %}
4725   ins_pipe( pipe_slow );
4726 %}
4727 
4728 instruct Repl32S_imm_evex(vecZ dst, immI con) %{
4729   predicate(n->as_Vector()->length() == 32 && VM_Version::supports_avx512bw());
4730   match(Set dst (ReplicateS con));
4731   format %{ "movq    $dst,[$constantaddress]\n\t"
4732             "vpbroadcastw $dst,$dst\t! replicate32S" %}
4733   ins_encode %{
4734    int vector_len = 2;
4735     __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 2)));
4736     __ evpbroadcastw($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4737   %}
4738   ins_pipe( pipe_slow );
4739 %}
4740 
4741 instruct Repl32S_zero_evex(vecZ dst, immI0 zero) %{
4742   predicate(n->as_Vector()->length() == 32 && UseAVX > 2);
4743   match(Set dst (ReplicateS zero));
4744   format %{ "vpxor   $dst k0,$dst,$dst\t! replicate32S zero" %}
4745   ins_encode %{
4746     // Use vxorpd since AVX does not have vpxor for 512-bit (EVEX will have it).
4747     int vector_len = 2;
4748     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4749   %}
4750   ins_pipe( fpu_reg_reg );
4751 %}
4752 
4753 instruct Repl4I_evex(vecX dst, rRegI src) %{
4754   predicate(n->as_Vector()->length() == 4 && VM_Version::supports_avx512vl());
4755   match(Set dst (ReplicateI src));
4756   format %{ "vpbroadcastd  $dst,$src\t! replicate4I" %}
4757   ins_encode %{
4758     int vector_len = 0;
4759     __ evpbroadcastd($dst$$XMMRegister, $src$$Register, vector_len);
4760   %}
4761   ins_pipe( pipe_slow );
4762 %}
4763 
4764 instruct Repl4I_mem_evex(vecX dst, memory mem) %{
4765   predicate(n->as_Vector()->length() == 4 && VM_Version::supports_avx512vl());
4766   match(Set dst (ReplicateI (LoadI mem)));
4767   format %{ "vpbroadcastd  $dst,$mem\t! replicate4I" %}
4768   ins_encode %{
4769     int vector_len = 0;
4770     __ evpbroadcastd($dst$$XMMRegister, $mem$$Address, vector_len);
4771   %}
4772   ins_pipe( pipe_slow );
4773 %}
4774 
4775 instruct Repl8I_evex(vecY dst, rRegI src) %{
4776   predicate(n->as_Vector()->length() == 8 && VM_Version::supports_avx512vl());
4777   match(Set dst (ReplicateI src));
4778   format %{ "vpbroadcastd  $dst,$src\t! replicate8I" %}
4779   ins_encode %{
4780     int vector_len = 1;
4781     __ evpbroadcastd($dst$$XMMRegister, $src$$Register, vector_len);
4782   %}
4783   ins_pipe( pipe_slow );
4784 %}
4785 
4786 instruct Repl8I_mem_evex(vecY dst, memory mem) %{
4787   predicate(n->as_Vector()->length() == 8 && VM_Version::supports_avx512vl());
4788   match(Set dst (ReplicateI (LoadI mem)));
4789   format %{ "vpbroadcastd  $dst,$mem\t! replicate8I" %}
4790   ins_encode %{
4791     int vector_len = 1;
4792     __ evpbroadcastd($dst$$XMMRegister, $mem$$Address, vector_len);
4793   %}
4794   ins_pipe( pipe_slow );
4795 %}
4796 
4797 instruct Repl16I_evex(vecZ dst, rRegI src) %{
4798   predicate(n->as_Vector()->length() == 16 && UseAVX > 2);
4799   match(Set dst (ReplicateI src));
4800   format %{ "vpbroadcastd  $dst,$src\t! replicate16I" %}
4801   ins_encode %{
4802     int vector_len = 2;
4803     __ evpbroadcastd($dst$$XMMRegister, $src$$Register, vector_len);
4804   %}
4805   ins_pipe( pipe_slow );
4806 %}
4807 
4808 instruct Repl16I_mem_evex(vecZ dst, memory mem) %{
4809   predicate(n->as_Vector()->length() == 16 && UseAVX > 2);
4810   match(Set dst (ReplicateI (LoadI mem)));
4811   format %{ "vpbroadcastd  $dst,$mem\t! replicate16I" %}
4812   ins_encode %{
4813     int vector_len = 2;
4814     __ evpbroadcastd($dst$$XMMRegister, $mem$$Address, vector_len);
4815   %}
4816   ins_pipe( pipe_slow );
4817 %}
4818 
4819 instruct Repl4I_imm_evex(vecX dst, immI con) %{
4820   predicate(n->as_Vector()->length() == 4 && VM_Version::supports_avx512vl());
4821   match(Set dst (ReplicateI con));
4822   format %{ "movq    $dst,[$constantaddress]\t! replicate8I($con)\n\t"
4823             "vpbroadcastd  $dst,$dst\t! replicate4I" %}
4824   ins_encode %{
4825     int vector_len = 0;
4826     __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 4)));
4827     __ evpbroadcastd($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4828   %}
4829   ins_pipe( pipe_slow );
4830 %}
4831 
4832 instruct Repl8I_imm_evex(vecY dst, immI con) %{
4833   predicate(n->as_Vector()->length() == 8 && VM_Version::supports_avx512vl());
4834   match(Set dst (ReplicateI con));
4835   format %{ "movq    $dst,[$constantaddress]\t! replicate8I($con)\n\t"
4836             "vpbroadcastd  $dst,$dst\t! replicate8I" %}
4837   ins_encode %{
4838     int vector_len = 1;
4839     __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 4)));
4840     __ evpbroadcastd($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4841   %}
4842   ins_pipe( pipe_slow );
4843 %}
4844 
4845 instruct Repl16I_imm_evex(vecZ dst, immI con) %{
4846   predicate(n->as_Vector()->length() == 16 && UseAVX > 2);
4847   match(Set dst (ReplicateI con));
4848   format %{ "movq    $dst,[$constantaddress]\t! replicate16I($con)\n\t"
4849             "vpbroadcastd  $dst,$dst\t! replicate16I" %}
4850   ins_encode %{
4851     int vector_len = 2;
4852     __ movq($dst$$XMMRegister, $constantaddress(replicate8_imm($con$$constant, 4)));
4853     __ evpbroadcastd($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4854   %}
4855   ins_pipe( pipe_slow );
4856 %}
4857 
4858 instruct Repl16I_zero_evex(vecZ dst, immI0 zero) %{
4859   predicate(n->as_Vector()->length() == 16 && UseAVX > 2);
4860   match(Set dst (ReplicateI zero));
4861   format %{ "vpxor   $dst k0,$dst,$dst\t! replicate16I zero" %}
4862   ins_encode %{
4863     // Use vxorpd since AVX does not have vpxor for 512-bit (AVX2 will have it).
4864     int vector_len = 2;
4865     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4866   %}
4867   ins_pipe( fpu_reg_reg );
4868 %}
4869 
4870 // Replicate long (8 byte) scalar to be vector
4871 #ifdef _LP64
4872 instruct Repl4L_evex(vecY dst, rRegL src) %{
4873   predicate(n->as_Vector()->length() == 4 && VM_Version::supports_avx512vl());
4874   match(Set dst (ReplicateL src));
4875   format %{ "vpbroadcastq  $dst,$src\t! replicate4L" %}
4876   ins_encode %{
4877     int vector_len = 1;
4878     __ evpbroadcastq($dst$$XMMRegister, $src$$Register, vector_len);
4879   %}
4880   ins_pipe( pipe_slow );
4881 %}
4882 
4883 instruct Repl8L_evex(vecZ dst, rRegL src) %{
4884   predicate(n->as_Vector()->length() == 8 && UseAVX > 2);
4885   match(Set dst (ReplicateL src));
4886   format %{ "vpbroadcastq  $dst,$src\t! replicate8L" %}
4887   ins_encode %{
4888     int vector_len = 2;
4889     __ evpbroadcastq($dst$$XMMRegister, $src$$Register, vector_len);
4890   %}
4891   ins_pipe( pipe_slow );
4892 %}
4893 #else // _LP64
4894 instruct Repl4L_evex(vecY dst, eRegL src, regD tmp) %{
4895   predicate(n->as_Vector()->length() == 4 && VM_Version::supports_avx512vl());
4896   match(Set dst (ReplicateL src));
4897   effect(TEMP dst, USE src, TEMP tmp);
4898   format %{ "movdl   $dst,$src.lo\n\t"
4899             "movdl   $tmp,$src.hi\n\t"
4900             "punpckldq $dst,$tmp\n\t"
4901             "vpbroadcastq  $dst,$dst\t! replicate4L" %}
4902   ins_encode %{
4903     int vector_len = 1;
4904     __ movdl($dst$$XMMRegister, $src$$Register);
4905     __ movdl($tmp$$XMMRegister, HIGH_FROM_LOW($src$$Register));
4906     __ punpckldq($dst$$XMMRegister, $tmp$$XMMRegister);
4907     __ evpbroadcastq($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4908   %}
4909   ins_pipe( pipe_slow );
4910 %}
4911 
4912 instruct Repl8L_evex(vecZ dst, eRegL src, regD tmp) %{
4913   predicate(n->as_Vector()->length() == 8 && UseAVX > 2);
4914   match(Set dst (ReplicateL src));
4915   effect(TEMP dst, USE src, TEMP tmp);
4916   format %{ "movdl   $dst,$src.lo\n\t"
4917             "movdl   $tmp,$src.hi\n\t"
4918             "punpckldq $dst,$tmp\n\t"
4919             "vpbroadcastq  $dst,$dst\t! replicate8L" %}
4920   ins_encode %{
4921     int vector_len = 2;
4922     __ movdl($dst$$XMMRegister, $src$$Register);
4923     __ movdl($tmp$$XMMRegister, HIGH_FROM_LOW($src$$Register));
4924     __ punpckldq($dst$$XMMRegister, $tmp$$XMMRegister);
4925     __ evpbroadcastq($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4926   %}
4927   ins_pipe( pipe_slow );
4928 %}
4929 #endif // _LP64
4930 
4931 instruct Repl4L_imm_evex(vecY dst, immL con) %{
4932   predicate(n->as_Vector()->length() == 4 && VM_Version::supports_avx512vl());
4933   match(Set dst (ReplicateL con));
4934   format %{ "movq    $dst,[$constantaddress]\n\t"
4935             "vpbroadcastq  $dst,$dst\t! replicate4L" %}
4936   ins_encode %{
4937     int vector_len = 1;
4938     __ movq($dst$$XMMRegister, $constantaddress($con));
4939     __ evpbroadcastq($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4940   %}
4941   ins_pipe( pipe_slow );
4942 %}
4943 
4944 instruct Repl8L_imm_evex(vecZ dst, immL con) %{
4945   predicate(n->as_Vector()->length() == 8 && UseAVX > 2);
4946   match(Set dst (ReplicateL con));
4947   format %{ "movq    $dst,[$constantaddress]\n\t"
4948             "vpbroadcastq  $dst,$dst\t! replicate8L" %}
4949   ins_encode %{
4950     int vector_len = 2;
4951     __ movq($dst$$XMMRegister, $constantaddress($con));
4952     __ evpbroadcastq($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4953   %}
4954   ins_pipe( pipe_slow );
4955 %}
4956 
4957 instruct Repl2L_mem_evex(vecX dst, memory mem) %{
4958   predicate(n->as_Vector()->length() == 2 && VM_Version::supports_avx512vl());
4959   match(Set dst (ReplicateL (LoadL mem)));
4960   format %{ "vpbroadcastd  $dst,$mem\t! replicate2L" %}
4961   ins_encode %{
4962     int vector_len = 0;
4963     __ evpbroadcastq($dst$$XMMRegister, $mem$$Address, vector_len);
4964   %}
4965   ins_pipe( pipe_slow );
4966 %}
4967 
4968 instruct Repl4L_mem_evex(vecY dst, memory mem) %{
4969   predicate(n->as_Vector()->length() == 4 && VM_Version::supports_avx512vl());
4970   match(Set dst (ReplicateL (LoadL mem)));
4971   format %{ "vpbroadcastd  $dst,$mem\t! replicate4L" %}
4972   ins_encode %{
4973     int vector_len = 1;
4974     __ evpbroadcastq($dst$$XMMRegister, $mem$$Address, vector_len);
4975   %}
4976   ins_pipe( pipe_slow );
4977 %}
4978 
4979 instruct Repl8L_mem_evex(vecZ dst, memory mem) %{
4980   predicate(n->as_Vector()->length() == 8 && UseAVX > 2);
4981   match(Set dst (ReplicateL (LoadL mem)));
4982   format %{ "vpbroadcastd  $dst,$mem\t! replicate8L" %}
4983   ins_encode %{
4984     int vector_len = 2;
4985     __ evpbroadcastq($dst$$XMMRegister, $mem$$Address, vector_len);
4986   %}
4987   ins_pipe( pipe_slow );
4988 %}
4989 
4990 instruct Repl8L_zero_evex(vecZ dst, immL0 zero) %{
4991   predicate(n->as_Vector()->length() == 8 && UseAVX > 2);
4992   match(Set dst (ReplicateL zero));
4993   format %{ "vpxor   $dst k0,$dst,$dst\t! replicate8L zero" %}
4994   ins_encode %{
4995     // Use vxorpd since AVX does not have vpxor for 512-bit (EVEX will have it).
4996     int vector_len = 2;
4997     __ vpxor($dst$$XMMRegister,$dst$$XMMRegister, $dst$$XMMRegister, vector_len);
4998   %}
4999   ins_pipe( fpu_reg_reg );
5000 %}
5001 
5002 instruct Repl8F_evex(vecY dst, regF src) %{
5003   predicate(n->as_Vector()->length() == 8 && VM_Version::supports_avx512vl());
5004   match(Set dst (ReplicateF src));
5005   format %{ "vbroadcastss $dst,$src\t! replicate8F" %}
5006   ins_encode %{
5007     int vector_len = 1;
5008     __ evpbroadcastss($dst$$XMMRegister, $src$$XMMRegister, vector_len);
5009   %}
5010   ins_pipe( pipe_slow );
5011 %}
5012 
5013 instruct Repl8F_mem_evex(vecY dst, memory mem) %{
5014   predicate(n->as_Vector()->length() == 8 && VM_Version::supports_avx512vl());
5015   match(Set dst (ReplicateF (LoadF mem)));
5016   format %{ "vbroadcastss  $dst,$mem\t! replicate8F" %}
5017   ins_encode %{
5018     int vector_len = 1;
5019     __ evpbroadcastss($dst$$XMMRegister, $mem$$Address, vector_len);
5020   %}
5021   ins_pipe( pipe_slow );
5022 %}
5023 
5024 instruct Repl16F_evex(vecZ dst, regF src) %{
5025   predicate(n->as_Vector()->length() == 16 && UseAVX > 2);
5026   match(Set dst (ReplicateF src));
5027   format %{ "vbroadcastss $dst,$src\t! replicate16F" %}
5028   ins_encode %{
5029     int vector_len = 2;
5030     __ evpbroadcastss($dst$$XMMRegister, $src$$XMMRegister, vector_len);
5031   %}
5032   ins_pipe( pipe_slow );
5033 %}
5034 
5035 instruct Repl16F_mem_evex(vecZ dst, memory mem) %{
5036   predicate(n->as_Vector()->length() == 16 && UseAVX > 2);
5037   match(Set dst (ReplicateF (LoadF mem)));
5038   format %{ "vbroadcastss  $dst,$mem\t! replicate16F" %}
5039   ins_encode %{
5040     int vector_len = 2;
5041     __ evpbroadcastss($dst$$XMMRegister, $mem$$Address, vector_len);
5042   %}
5043   ins_pipe( pipe_slow );
5044 %}
5045 
5046 instruct Repl2F_zero_evex(vecD dst, immF0 zero) %{
5047   predicate(n->as_Vector()->length() == 2 && UseAVX > 2);
5048   match(Set dst (ReplicateF zero));
5049   format %{ "vpxor  $dst k0,$dst,$dst\t! replicate2F zero" %}
5050   ins_encode %{
5051     // Use vpxor in place of vxorps since EVEX has a constriant on dq for vxorps: this is a 512-bit operation
5052     int vector_len = 2;
5053     __ vpxor($dst$$XMMRegister,$dst$$XMMRegister, $dst$$XMMRegister, vector_len);
5054   %}
5055   ins_pipe( fpu_reg_reg );
5056 %}
5057 
5058 instruct Repl4F_zero_evex(vecX dst, immF0 zero) %{
5059   predicate(n->as_Vector()->length() == 4 && UseAVX > 2);
5060   match(Set dst (ReplicateF zero));
5061   format %{ "vpxor  $dst k0,$dst,$dst\t! replicate4F zero" %}
5062   ins_encode %{
5063     // Use vpxor in place of vxorps since EVEX has a constriant on dq for vxorps: this is a 512-bit operation
5064     int vector_len = 2;
5065     __ vpxor($dst$$XMMRegister,$dst$$XMMRegister, $dst$$XMMRegister, vector_len);
5066   %}
5067   ins_pipe( fpu_reg_reg );
5068 %}
5069 
5070 instruct Repl8F_zero_evex(vecY dst, immF0 zero) %{
5071   predicate(n->as_Vector()->length() == 8 && UseAVX > 2);
5072   match(Set dst (ReplicateF zero));
5073   format %{ "vpxor  $dst k0,$dst,$dst\t! replicate8F zero" %}
5074   ins_encode %{
5075     // Use vpxor in place of vxorps since EVEX has a constriant on dq for vxorps: this is a 512-bit operation
5076     int vector_len = 2;
5077     __ vpxor($dst$$XMMRegister,$dst$$XMMRegister, $dst$$XMMRegister, vector_len);
5078   %}
5079   ins_pipe( fpu_reg_reg );
5080 %}
5081 
5082 instruct Repl16F_zero_evex(vecZ dst, immF0 zero) %{
5083   predicate(n->as_Vector()->length() == 16 && UseAVX > 2);
5084   match(Set dst (ReplicateF zero));
5085   format %{ "vpxor  $dst k0,$dst,$dst\t! replicate16F zero" %}
5086   ins_encode %{
5087     // Use vpxor in place of vxorps since EVEX has a constriant on dq for vxorps: this is a 512-bit operation
5088     int vector_len = 2;
5089     __ vpxor($dst$$XMMRegister,$dst$$XMMRegister, $dst$$XMMRegister, vector_len);
5090   %}
5091   ins_pipe( fpu_reg_reg );
5092 %}
5093 
5094 instruct Repl4D_evex(vecY dst, regD src) %{
5095   predicate(n->as_Vector()->length() == 4 && VM_Version::supports_avx512vl());
5096   match(Set dst (ReplicateD src));
5097   format %{ "vbroadcastsd $dst,$src\t! replicate4D" %}
5098   ins_encode %{
5099     int vector_len = 1;
5100     __ evpbroadcastsd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
5101   %}
5102   ins_pipe( pipe_slow );
5103 %}
5104 
5105 instruct Repl4D_mem_evex(vecY dst, memory mem) %{
5106   predicate(n->as_Vector()->length() == 4 && VM_Version::supports_avx512vl());
5107   match(Set dst (ReplicateD (LoadD mem)));
5108   format %{ "vbroadcastsd  $dst,$mem\t! replicate4D" %}
5109   ins_encode %{
5110     int vector_len = 1;
5111     __ evpbroadcastsd($dst$$XMMRegister, $mem$$Address, vector_len);
5112   %}
5113   ins_pipe( pipe_slow );
5114 %}
5115 
5116 instruct Repl8D_evex(vecZ dst, regD src) %{
5117   predicate(n->as_Vector()->length() == 8 && UseAVX > 2);
5118   match(Set dst (ReplicateD src));
5119   format %{ "vbroadcastsd $dst,$src\t! replicate8D" %}
5120   ins_encode %{
5121     int vector_len = 2;
5122     __ evpbroadcastsd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
5123   %}
5124   ins_pipe( pipe_slow );
5125 %}
5126 
5127 instruct Repl8D_mem_evex(vecZ dst, memory mem) %{
5128   predicate(n->as_Vector()->length() == 8 && UseAVX > 2);
5129   match(Set dst (ReplicateD (LoadD mem)));
5130   format %{ "vbroadcastsd  $dst,$mem\t! replicate8D" %}
5131   ins_encode %{
5132     int vector_len = 2;
5133     __ evpbroadcastsd($dst$$XMMRegister, $mem$$Address, vector_len);
5134   %}
5135   ins_pipe( pipe_slow );
5136 %}
5137 
5138 instruct Repl2D_zero_evex(vecX dst, immD0 zero) %{
5139   predicate(n->as_Vector()->length() == 2 && UseAVX > 2);
5140   match(Set dst (ReplicateD zero));
5141   format %{ "vpxor  $dst k0,$dst,$dst\t! replicate2D zero" %}
5142   ins_encode %{
5143     // Use vpxor in place of vxorpd since EVEX has a constriant on dq for vxorpd: this is a 512-bit operation
5144     int vector_len = 2;
5145     __ vpxor($dst$$XMMRegister,$dst$$XMMRegister, $dst$$XMMRegister, vector_len);
5146   %}
5147   ins_pipe( fpu_reg_reg );
5148 %}
5149 
5150 instruct Repl4D_zero_evex(vecY dst, immD0 zero) %{
5151   predicate(n->as_Vector()->length() == 4 && UseAVX > 2);
5152   match(Set dst (ReplicateD zero));
5153   format %{ "vpxor  $dst k0,$dst,$dst\t! replicate4D zero" %}
5154   ins_encode %{
5155     // Use vpxor in place of vxorpd since EVEX has a constriant on dq for vxorpd: this is a 512-bit operation
5156     int vector_len = 2;
5157     __ vpxor($dst$$XMMRegister,$dst$$XMMRegister, $dst$$XMMRegister, vector_len);
5158   %}
5159   ins_pipe( fpu_reg_reg );
5160 %}
5161 
5162 instruct Repl8D_zero_evex(vecZ dst, immD0 zero) %{
5163   predicate(n->as_Vector()->length() == 8 && UseAVX > 2);
5164   match(Set dst (ReplicateD zero));
5165   format %{ "vpxor  $dst k0,$dst,$dst,vect512\t! replicate8D zero" %}
5166   ins_encode %{
5167     // Use vpxor in place of vxorpd since EVEX has a constriant on dq for vxorpd: this is a 512-bit operation
5168     int vector_len = 2;
5169     __ vpxor($dst$$XMMRegister,$dst$$XMMRegister, $dst$$XMMRegister, vector_len);
5170   %}
5171   ins_pipe( fpu_reg_reg );
5172 %}
5173 
5174 // ====================VECTOR INSERT=======================================
5175 
5176 instruct rvinsert8B(vecD dst, vecD src, rRegI val, immU3 idx) %{
5177   predicate(UseSSE > 3 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
5178   match(Set dst (VectorInsert (Binary src val) idx));
5179   effect(TEMP dst);
5180   format %{ "movdqu  $dst,$src\n\t"
5181             "pinsrb  $dst,$val\t! Insert 8B" %}
5182   ins_encode %{
5183     if ($dst$$XMMRegister != $src$$XMMRegister) {
5184       __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
5185     }
5186     __ pinsrb($dst$$XMMRegister, $val$$Register, $idx$$constant);
5187   %}
5188   ins_pipe( pipe_slow );
5189 %}
5190 
5191 instruct rvinsert16B(vecX dst, vecX src, rRegI val, immU4 idx) %{
5192   predicate(UseSSE > 3 && UseAVX == 0 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
5193   match(Set dst (VectorInsert (Binary src val) idx));
5194   effect(TEMP dst);
5195   format %{ "movdqu  $dst,$src\n\t"
5196             "pinsrb  $dst,$val\t! Insert 16B" %}
5197   ins_encode %{
5198     if ($dst$$XMMRegister != $src$$XMMRegister) {
5199       __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
5200     }
5201     __ pinsrb($dst$$XMMRegister, $val$$Register, $idx$$constant);
5202   %}
5203   ins_pipe( pipe_slow );
5204 %}
5205 
5206 instruct rvinsert16B_avx(vecX dst, vecX src, rRegI val, immU4 idx) %{
5207   predicate(UseAVX > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
5208   match(Set dst (VectorInsert (Binary src val) idx));
5209   effect(TEMP dst);
5210   format %{ "vmovdqu  $dst,$src\n\t"
5211             "vpinsrb  $dst,$dst,$val\t! Insert 16B" %}
5212   ins_encode %{
5213     if ($dst$$XMMRegister != $src$$XMMRegister) {
5214       __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister);
5215     }
5216     __ vpinsrb($dst$$XMMRegister, $dst$$XMMRegister, $val$$Register, $idx$$constant);
5217   %}
5218   ins_pipe( pipe_slow );
5219 %}
5220 
5221 instruct rvinsert32B(vecY dst, vecY src, vecY tmp, rRegI val, immU5 idx) %{
5222   predicate(UseAVX > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
5223   match(Set dst (VectorInsert (Binary src val) idx));
5224   effect(TEMP dst, TEMP tmp);
5225   format %{"vmovdqu  $dst,$src\n\t"
5226            "vextracti128  $tmp,$src\n\t"
5227            "vpinsrb  $tmp,$tmp,$val\n\t"
5228            "vinserti128  $dst,$tmp\t! Insert 32B" %}
5229   ins_encode %{
5230     uint x_idx = $idx$$constant & right_n_bits(4);
5231     uint y_idx = ($idx$$constant >> 4) & 1;
5232 
5233     if ($dst$$XMMRegister != $src$$XMMRegister) {
5234       __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister);
5235     }
5236     __ vextracti128($tmp$$XMMRegister, $src$$XMMRegister, y_idx);
5237     __ vpinsrb($tmp$$XMMRegister, $tmp$$XMMRegister, $val$$Register, x_idx);
5238     __ vinserti128($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, y_idx);
5239   %}
5240   ins_pipe( pipe_slow );
5241 %}
5242 
5243 instruct rvinsert64B(vecZ dst, vecZ src, vecZ tmp, vecX tmp1, rRegI val, immU6 idx) %{
5244   predicate(UseAVX > 2 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
5245   match(Set dst (VectorInsert (Binary src val) idx));
5246   effect(TEMP dst, TEMP tmp, TEMP tmp1);
5247   format %{ "evmovdquq  $dst,$src\n\t"
5248             "vextracti64x4  $tmp,$src\n\t"
5249             "vextracti128  $tmp1,$tmp\n\t"
5250             "vpinsrb  $tmp1,$tmp1,$val\n\t"
5251             "vinserti128  $tmp,$tmp,$tmp1\n\t"
5252             "vinserti64x4  $dst,$dst,$tmp\t! Insert 64B" %}
5253   ins_encode %{
5254     uint x_idx = $idx$$constant & right_n_bits(4);
5255     uint y_idx = ($idx$$constant >> 4) & 1;
5256     uint z_idx = ($idx$$constant >> 5) & 1;
5257 
5258     if ($dst$$XMMRegister != $src$$XMMRegister) {
5259       __ evmovdquq($dst$$XMMRegister, $src$$XMMRegister, 2);
5260     }
5261     __ vextracti64x4($tmp$$XMMRegister, $src$$XMMRegister, z_idx);
5262     __ vextracti128($tmp1$$XMMRegister, $tmp$$XMMRegister, y_idx);
5263     __ vpinsrb($tmp1$$XMMRegister, $tmp1$$XMMRegister, $val$$Register, x_idx);
5264     __ vinserti128($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp1$$XMMRegister, y_idx);
5265     __ vinserti64x4($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, z_idx);
5266   %}
5267   ins_pipe( pipe_slow );
5268 %}
5269 
5270 instruct rvinsert4S(vecD dst, vecD src, rRegI val, immU2 idx) %{
5271   predicate(UseSSE > 3 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
5272   match(Set dst (VectorInsert (Binary src val) idx));
5273   effect(TEMP dst);
5274   format %{ "movdqu  $dst,$src\n\t"
5275             "pinsrw  $dst,$val\t! Insert 4S" %}
5276   ins_encode %{
5277     if ($dst$$XMMRegister != $src$$XMMRegister) {
5278       __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
5279     }
5280     __ pinsrw($dst$$XMMRegister, $val$$Register, $idx$$constant);
5281   %}
5282   ins_pipe( pipe_slow );
5283 %}
5284 
5285 instruct rvinsert8S(vecX dst, vecX src, rRegI val, immU3 idx) %{
5286   predicate(UseSSE > 3 && UseAVX == 0 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
5287   match(Set dst (VectorInsert (Binary src val) idx));
5288   effect(TEMP dst);
5289   format %{ "movdqu  $dst,$src\n\t"
5290             "pinsrw  $dst,$val\t! Insert 8S" %}
5291   ins_encode %{
5292     if ($dst$$XMMRegister != $src$$XMMRegister) {
5293       __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
5294     }
5295     __ pinsrw($dst$$XMMRegister, $val$$Register, $idx$$constant);
5296   %}
5297   ins_pipe( pipe_slow );
5298 %}
5299 
5300 instruct rvinsert8S_avx(vecX dst, vecX src, rRegI val, immU3 idx) %{
5301   predicate(UseAVX > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
5302   match(Set dst (VectorInsert (Binary src val) idx));
5303   effect(TEMP dst);
5304   format %{ "vmovdqu  $dst,$src\n\t"
5305             "vpinsrw  $dst,$dst,$val\t! Insert 8S" %}
5306   ins_encode %{
5307     if ($dst$$XMMRegister != $src$$XMMRegister) {
5308       __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister);
5309     }
5310     __ vpinsrw($dst$$XMMRegister, $dst$$XMMRegister, $val$$Register, $idx$$constant);
5311   %}
5312   ins_pipe( pipe_slow );
5313 %}
5314 
5315 
5316 instruct rvinsert16S(vecY dst, vecY src, vecX tmp, rRegI val, immU4 idx) %{
5317   predicate(UseAVX > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
5318   match(Set dst (VectorInsert (Binary src val) idx));
5319   effect(TEMP dst, TEMP tmp);
5320   format %{ "vmovdqu  $dst,$src\n\t"
5321             "vextracti128  $tmp,$src\n\t"
5322             "vpinsrw  $tmp,$tmp,$val\n\t"
5323             "vinserti128  $dst,$dst,$tmp\t! Insert 16S" %}
5324   ins_encode %{
5325     uint x_idx = $idx$$constant & right_n_bits(3);
5326     uint y_idx = ($idx$$constant >> 3) & 1;
5327 
5328     if ($dst$$XMMRegister != $src$$XMMRegister) {
5329       __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister);
5330     }
5331     __ vextracti128($tmp$$XMMRegister, $src$$XMMRegister, y_idx);
5332     __ vpinsrw($tmp$$XMMRegister, $tmp$$XMMRegister, $val$$Register, x_idx);
5333     __ vinserti128($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, y_idx);
5334   %}
5335   ins_pipe( pipe_slow );
5336 %}
5337 
5338 instruct rvinsert32S(vecZ dst, vecZ src, vecZ tmp, vecX tmp1, rRegI val, immU5 idx) %{
5339   predicate(UseAVX > 2 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
5340   match(Set dst (VectorInsert (Binary src val) idx));
5341   effect(TEMP dst, TEMP tmp, TEMP tmp1);
5342   format %{ "evmovdquq  $dst,$src\n\t"
5343             "vextracti64x4  $tmp,$src\n\t"
5344             "vextracti128  $tmp1,$tmp\n\t"
5345             "vpinsrw  $tmp1,$tmp1,$val\n\t"
5346             "vinserti128  $tmp,$tmp,$tmp1\n\t"
5347             "vinserti64x4  $dst,$dst,$tmp\t! Insert 32S" %}
5348   ins_encode %{
5349     uint x_idx = $idx$$constant & right_n_bits(3);
5350     uint y_idx = ($idx$$constant >> 3) & 1;
5351     uint z_idx = ($idx$$constant >> 4) & 1;
5352 
5353     if ($dst$$XMMRegister != $src$$XMMRegister) {
5354       __ evmovdquq($dst$$XMMRegister, $src$$XMMRegister, 2);
5355     }
5356     __ vextracti64x4($tmp$$XMMRegister, $src$$XMMRegister, z_idx);
5357     __ vextracti128($tmp1$$XMMRegister, $tmp$$XMMRegister, y_idx);
5358     __ vpinsrw($tmp1$$XMMRegister, $tmp1$$XMMRegister, $val$$Register, x_idx);
5359     __ vinserti128($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp1$$XMMRegister, y_idx);
5360     __ vinserti64x4($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, z_idx);
5361   %}
5362   ins_pipe( pipe_slow );
5363 %}
5364 
5365 instruct rvinsert2I(vecD dst, vecD src, rRegI val, immU1 idx) %{
5366   predicate(UseSSE > 3 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
5367   match(Set dst (VectorInsert (Binary src val) idx));
5368   effect(TEMP dst);
5369   format %{ "movdqu  $dst,$src\n\t"
5370             "pinsrd  $dst,$val\t! Insert 2I" %}
5371   ins_encode %{
5372     if ($dst$$XMMRegister != $src$$XMMRegister) {
5373       __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
5374     }
5375     __ pinsrd($dst$$XMMRegister, $val$$Register, $idx$$constant);
5376   %}
5377   ins_pipe( pipe_slow );
5378 %}
5379 
5380 instruct rvinsert4I(vecX dst, vecX src, rRegI val, immU2 idx) %{
5381   predicate(UseSSE > 3 && UseAVX == 0 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
5382   match(Set dst (VectorInsert (Binary src val) idx));
5383   effect(TEMP dst);
5384   format %{ "movdqu  $dst,$src\n\t"
5385             "pinsrd  $dst,$val\t! Insert 4I" %}
5386   ins_encode %{
5387     if ($dst$$XMMRegister != $src$$XMMRegister) {
5388       __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
5389     }
5390     __ pinsrd($dst$$XMMRegister, $val$$Register, $idx$$constant);
5391   %}
5392   ins_pipe( pipe_slow );
5393 %}
5394 
5395 instruct rvinsert4I_avx(vecX dst, vecX src, rRegI val, immU2 idx) %{
5396   predicate(UseAVX > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
5397   match(Set dst (VectorInsert (Binary src val) idx));
5398   effect(TEMP dst);
5399   format %{ "vmovdqu  $dst,$src\n\t"
5400             "vpinsrd  $dst,$val\t! Insert 4I" %}
5401   ins_encode %{
5402     if ($dst$$XMMRegister != $src$$XMMRegister) {
5403       __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister);
5404     }
5405     __ vpinsrd($dst$$XMMRegister, $dst$$XMMRegister, $val$$Register, $idx$$constant);
5406   %}
5407   ins_pipe( pipe_slow );
5408 %}
5409 
5410 instruct rvinsert8I(vecY dst, vecY src, vecY tmp, rRegI val, immU3 idx) %{
5411   predicate(UseAVX > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
5412   match(Set dst (VectorInsert (Binary src val) idx));
5413   effect(TEMP dst, TEMP tmp);
5414   format %{ "vmovdqu  $dst,$src\n\t"
5415             "vextracti128  $tmp,$src\n\t"
5416             "vpinsrd  $tmp,$tmp,$val\n\t"
5417             "vinserti128  $dst,$dst,$tmp\t! Insert 8I" %}
5418   ins_encode %{
5419     uint x_idx = $idx$$constant & right_n_bits(2);
5420     uint y_idx = ($idx$$constant >> 2) & 1;
5421 
5422     if ($dst$$XMMRegister != $src$$XMMRegister) {
5423       __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister);
5424     }
5425     __ vextracti128($tmp$$XMMRegister, $src$$XMMRegister, y_idx);
5426     __ vpinsrd($tmp$$XMMRegister, $tmp$$XMMRegister, $val$$Register, x_idx);
5427     __ vinserti128($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, y_idx);
5428   %}
5429   ins_pipe( pipe_slow );
5430 %}
5431 
5432 instruct rvinsert16I(vecZ dst, vecZ src, vecZ tmp, vecX tmp1, rRegI val, immU4 idx) %{
5433   predicate(UseAVX > 2 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
5434   match(Set dst (VectorInsert (Binary src val) idx));
5435   effect(TEMP dst, TEMP tmp, TEMP tmp1);
5436   format %{ "evmovdquq  $dst,$src\n\t"
5437             "vextracti64x4  $tmp,$src\n\t"
5438             "vextracti128  $tmp,$tmp\n\t"
5439             "vpinsrd  $tmp,$tmp,$val\n\t"
5440             "vinserti128  $tmp,$tmp,$tmp\n\t"
5441             "vinserti64x4  $dst,$dst,$tmp\t! Insert 16I" %}
5442   ins_encode %{
5443     uint x_idx = $idx$$constant & right_n_bits(2);
5444     uint y_idx = ($idx$$constant >> 2) & 1;
5445     uint z_idx = ($idx$$constant >> 3) & 1;
5446 
5447     if ($dst$$XMMRegister != $src$$XMMRegister) {
5448       __ evmovdquq($dst$$XMMRegister, $src$$XMMRegister, 2);
5449     }
5450     __ vextracti64x4($tmp$$XMMRegister, $src$$XMMRegister, z_idx);
5451     __ vextracti128($tmp1$$XMMRegister, $tmp$$XMMRegister, y_idx);
5452     __ vpinsrd($tmp1$$XMMRegister, $tmp1$$XMMRegister, $val$$Register, x_idx);
5453     __ vinserti128($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp1$$XMMRegister, y_idx);
5454     __ vinserti64x4($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, z_idx);
5455   %}
5456   ins_pipe( pipe_slow );
5457 %}
5458 
5459 instruct rvinsert1L(vecD dst, vecD src, rRegL val, immI0 idx) %{
5460   predicate(UseSSE > 3 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
5461   match(Set dst (VectorInsert (Binary src val) idx));
5462   effect(TEMP dst);
5463   format %{ "movdqu  $dst,$src\n\t"
5464             "pinsrq  $dst,$val\t! Insert 1L" %}
5465   ins_encode %{
5466     if ($dst$$XMMRegister != $src$$XMMRegister) {
5467       __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
5468     }
5469     __ pinsrq($dst$$XMMRegister, $val$$Register, 0);
5470   %}
5471   ins_pipe( pipe_slow );
5472 %}
5473 
5474 instruct rvinsert2L(vecX dst, vecX src, rRegL val, immU1 idx) %{
5475   predicate(UseSSE > 3 && UseAVX == 0 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
5476   match(Set dst (VectorInsert (Binary src val) idx));
5477   effect(TEMP dst);
5478   format %{ "movdqu  $dst,$src\n\t"
5479             "pinsrq  $dst,$dst\t! Insert 2L" %}
5480   ins_encode %{
5481     if ($dst$$XMMRegister != $src$$XMMRegister) {
5482       __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
5483     }
5484     __ pinsrq($dst$$XMMRegister, $val$$Register, $idx$$constant);
5485   %}
5486   ins_pipe( pipe_slow );
5487 %}
5488 
5489 instruct rvinsert2L_avx(vecX dst, vecX src, rRegL val, immU1 idx) %{
5490   predicate(UseAVX > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
5491   match(Set dst (VectorInsert (Binary src val) idx));
5492   effect(TEMP dst);
5493   format %{ "vmovdqu  $dst,$src\n\t"
5494             "vpinsrq  $dst,$dst,$val\t! Insert 2L" %}
5495   ins_encode %{
5496     if ($dst$$XMMRegister != $src$$XMMRegister) {
5497       __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister);
5498     }
5499     __ vpinsrq($dst$$XMMRegister, $dst$$XMMRegister, $val$$Register, $idx$$constant);
5500   %}
5501   ins_pipe( pipe_slow );
5502 %}
5503 
5504 instruct rvinsert4L(vecY dst, vecY src, vecY tmp, rRegL val, immU2 idx) %{
5505   predicate(UseAVX > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
5506   match(Set dst (VectorInsert (Binary src val) idx));
5507   effect(TEMP dst, TEMP tmp);
5508   format %{ "vmovdqu  $dst,$src\n\t"
5509             "vextracti128  $tmp,$src\n\t"
5510             "vpinsrq  $tmp,$tmp,$val\n\t"
5511             "vinserti128  $dst,$dst,$tmp\t! Insert 4L" %}
5512   ins_encode %{
5513     uint x_idx = $idx$$constant & 1;
5514     uint y_idx = ($idx$$constant >> 1) & 1;
5515 
5516     if ($dst$$XMMRegister != $src$$XMMRegister) {
5517       __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister);
5518     }
5519     __ vextracti128($tmp$$XMMRegister, $src$$XMMRegister, y_idx);
5520     __ vpinsrq($tmp$$XMMRegister, $tmp$$XMMRegister, $val$$Register, x_idx);
5521     __ vinserti128($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, y_idx);
5522   %}
5523   ins_pipe( pipe_slow );
5524 %}
5525 
5526 instruct rvinsert8L(vecZ dst, vecZ src, vecZ tmp, vecX tmp1, rRegL val, immU3 idx) %{
5527   predicate(UseAVX > 2 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
5528   match(Set dst (VectorInsert (Binary src val) idx));
5529   effect(TEMP dst, TEMP tmp, TEMP tmp1);
5530   format %{ "evmovdquq  $dst,$src\n\t"
5531             "vextracti64x4  $tmp,$src\n\t"
5532             "vextracti128  $tmp,$tmp\n\t"
5533             "vpinsrq  $tmp,$tmp,$val\n\t"
5534             "vinserti128  $tmp,$tmp,$tmp\n\t"
5535             "vinserti64x4  $dst,$dst,$tmp\t! Insert 8L" %}
5536   ins_encode %{
5537     uint x_idx = $idx$$constant & 1;
5538     uint y_idx = ($idx$$constant >> 1) & 1;
5539     uint z_idx = ($idx$$constant >> 2) & 1;
5540 
5541     if ($dst$$XMMRegister != $src$$XMMRegister) {
5542       __ evmovdquq($dst$$XMMRegister, $src$$XMMRegister, 2);
5543     }
5544     __ vextracti64x4($tmp$$XMMRegister, $src$$XMMRegister, z_idx);
5545     __ vextracti128($tmp1$$XMMRegister, $tmp$$XMMRegister, y_idx);
5546     __ vpinsrq($tmp1$$XMMRegister, $tmp1$$XMMRegister, $val$$Register, x_idx);
5547     __ vinserti128($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp1$$XMMRegister, y_idx);
5548     __ vinserti64x4($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, z_idx);
5549   %}
5550   ins_pipe( pipe_slow );
5551 %}
5552 
5553 instruct rvinsert2F(vecD dst, vecD src, regF val, immU1 idx) %{
5554   predicate(UseSSE > 3 && UseAVX == 0 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
5555   match(Set dst (VectorInsert (Binary src val) idx));
5556   effect(TEMP dst);
5557   format %{ "movdqu  $dst,$src\n\t"
5558             "insertps  $dst,$dst,$val\t! Insert 2F" %}
5559   ins_encode %{
5560     if ($dst$$XMMRegister != $src$$XMMRegister) {
5561       __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
5562     }
5563     __ insertps($dst$$XMMRegister, $val$$XMMRegister, $idx$$constant);
5564   %}
5565   ins_pipe( pipe_slow );
5566 %}
5567 
5568 instruct rvinsert2F_avx(vecD dst, vecD src, regF val, immU1 idx) %{
5569   predicate(UseAVX > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
5570   match(Set dst (VectorInsert (Binary src val) idx));
5571   effect(TEMP dst);
5572   format %{ "movdqu  $dst,$src\n\t"
5573             "insertps  $dst,$dst,$val\t! Insert 2F" %}
5574   ins_encode %{
5575     if ($dst$$XMMRegister != $src$$XMMRegister) {
5576       __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
5577     }
5578     __ vinsertps($dst$$XMMRegister, $dst$$XMMRegister, $val$$XMMRegister, $idx$$constant);
5579   %}
5580   ins_pipe( pipe_slow );
5581 %}
5582 
5583 instruct rvinsert4F(vecX dst, vecX src, regF val, immU2 idx) %{
5584   predicate(UseSSE > 3 && UseAVX == 0 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
5585   match(Set dst (VectorInsert (Binary src val) idx));
5586   effect(TEMP dst);
5587   format %{ "movdqu  $dst,$src\n\t"
5588             "insertps  $dst,$dst,$val\t! Insert 4F" %}
5589   ins_encode %{
5590     if ($dst$$XMMRegister != $src$$XMMRegister) {
5591       __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
5592     }
5593     __ insertps($dst$$XMMRegister, $val$$XMMRegister, $idx$$constant);
5594   %}
5595   ins_pipe( pipe_slow );
5596 %}
5597 
5598 instruct rvinsert4F_avx(vecX dst, vecX src, regF val, immU2 idx) %{
5599   predicate(UseAVX > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
5600   match(Set dst (VectorInsert (Binary src val) idx));
5601   effect(TEMP dst);
5602   format %{ "vmovdqu  $dst,$src\n\t"
5603             "vinsertps  $dst,$dst,$val\t! Insert 4F" %}
5604   ins_encode %{
5605     if ($dst$$XMMRegister != $src$$XMMRegister) {
5606       __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister);
5607     }
5608     __ vinsertps($dst$$XMMRegister, $dst$$XMMRegister, $val$$XMMRegister, $idx$$constant);
5609   %}
5610   ins_pipe( pipe_slow );
5611 %}
5612 
5613 instruct rvinsert8F(vecY dst, vecY src, vecY tmp, regF val, immU3 idx) %{
5614   predicate(UseAVX > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
5615   match(Set dst (VectorInsert (Binary src val) idx));
5616   effect(TEMP dst, TEMP tmp);
5617   format %{ "vmovdqu  $dst,$src\n\t"
5618             "vextractf128  $tmp,$src\n\t"
5619             "vinsertps  $tmp,$tmp,$val\n\t"
5620             "vinsertf128  $dst,$dst,$tmp\t! Insert 8F" %}
5621   ins_encode %{
5622     uint x_idx = $idx$$constant & right_n_bits(2);
5623     uint y_idx = ($idx$$constant >> 2) & 1;
5624 
5625     if ($dst$$XMMRegister != $src$$XMMRegister) {
5626       __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister);
5627     }
5628     __ vextractf128($tmp$$XMMRegister, $src$$XMMRegister, y_idx);
5629     __ vinsertps($tmp$$XMMRegister, $tmp$$XMMRegister, $val$$XMMRegister, x_idx);
5630     __ vinsertf128($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, y_idx);
5631   %}
5632   ins_pipe( pipe_slow );
5633 %}
5634 
5635 instruct rvinsert16F(vecZ dst, vecZ src, vecZ tmp, vecX tmp1, regF val, immU4 idx) %{
5636   predicate(UseAVX > 2 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
5637   match(Set dst (VectorInsert (Binary src val) idx));
5638   effect(TEMP dst, TEMP tmp, TEMP tmp1);
5639   format %{ "evmovdquq  $dst,$src\n\t"
5640             "vextractf128  $tmp,$src\n\t"
5641             "vinsertps  $tmp,$tmp,$val\n\t"
5642             "movsbl  $dst,$dst\t! Insert 4I" %}
5643   ins_encode %{
5644     uint x_idx = $idx$$constant & right_n_bits(2);
5645     uint y_idx = ($idx$$constant >> 2) & 1;
5646     uint z_idx = ($idx$$constant >> 3) & 1;
5647 
5648     if ($dst$$XMMRegister != $src$$XMMRegister) {
5649       __ evmovdquq($dst$$XMMRegister, $src$$XMMRegister, 2);
5650     }
5651     __ vextractf64x4($tmp$$XMMRegister, $src$$XMMRegister, z_idx);
5652     __ vextractf128($tmp1$$XMMRegister, $tmp$$XMMRegister, y_idx);
5653     __ vinsertps($tmp1$$XMMRegister, $tmp1$$XMMRegister, $val$$XMMRegister, x_idx);
5654     __ vinsertf128($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp1$$XMMRegister, y_idx);
5655     __ vinsertf64x4($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, z_idx);
5656   %}
5657   ins_pipe( pipe_slow );
5658 %}
5659 
5660 instruct rvinsert1D(vecD dst, vecD src, regD val, rRegL tmp, immI0 idx) %{
5661   predicate(UseSSE > 3 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
5662   match(Set dst (VectorInsert (Binary src val) idx));
5663   effect(TEMP dst, TEMP tmp);
5664   format %{ "movdqu  $dst,$src\n\t"
5665             "movq $tmp,$val\n\t"
5666             "pinsrq  $dst,$tmp\t! Insert 1D" %}
5667   ins_encode %{
5668     if ($dst$$XMMRegister != $src$$XMMRegister) {
5669       __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
5670     }
5671     __ movq($tmp$$Register, $val$$XMMRegister);
5672     __ pinsrq($dst$$XMMRegister, $tmp$$Register, 0);
5673   %}
5674   ins_pipe( pipe_slow );
5675 %}
5676 
5677 instruct rvinsert2D(vecX dst, vecX src, regD val, rRegL tmp, immU1 idx) %{
5678   predicate(UseSSE > 3 && UseAVX == 0 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
5679   match(Set dst (VectorInsert (Binary src val) idx));
5680   effect(TEMP dst, TEMP tmp);
5681   format %{ "movdqu  $dst,$src\n\t"
5682             "movq  $dst,$src\n\t"
5683             "pinsrq  $dst,$dst\t! Insert 2D" %}
5684   ins_encode %{
5685     if ($dst$$XMMRegister != $src$$XMMRegister) {
5686       __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
5687     }
5688     __ movq($tmp$$Register, $val$$XMMRegister);
5689     __ pinsrq($dst$$XMMRegister, $tmp$$Register, $idx$$constant);
5690   %}
5691   ins_pipe( pipe_slow );
5692 %}
5693 
5694 instruct rvinsert2D_avx(vecX dst, vecX src, regD val, rRegL tmp, immU1 idx) %{
5695   predicate(UseAVX > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
5696   match(Set dst (VectorInsert (Binary src val) idx));
5697   effect(TEMP dst, TEMP tmp);
5698   format %{ "vmovdqu  $dst,$src\n\t"
5699             "movq  $tmp,$val\n\t"
5700             "vpinsrq  $dst,$dst,$tmp\t! Insert 2D" %}
5701   ins_encode %{
5702     if ($dst$$XMMRegister != $src$$XMMRegister) {
5703       __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister);
5704     }
5705     __ movq($tmp$$Register, $val$$XMMRegister);
5706     __ vpinsrq($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$Register, $idx$$constant);
5707   %}
5708   ins_pipe( pipe_slow );
5709 %}
5710 
5711 instruct rvinsert4D(vecY dst, vecY src, vecY tmp, regD val, rRegL tmp1, immU2 idx) %{
5712   predicate(UseAVX > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
5713   match(Set dst (VectorInsert (Binary src val) idx));
5714   effect(TEMP dst, TEMP tmp, TEMP tmp1);
5715   format %{ "vmovdqu  $dst,$src\n\t"
5716             "vextracti128  $tmp,$src\n\t"
5717             "movq $tmp1,$val\n\t"
5718             "vpinsrq  $tmp,$tmp,$tmp1\n\t"
5719             "vinserti128  $dst,$dst,$tmp\t! Insert 4D" %}
5720   ins_encode %{
5721     uint x_idx = $idx$$constant & 1;
5722     uint y_idx = ($idx$$constant >> 1) & 1;
5723 
5724     if ($dst$$XMMRegister != $src$$XMMRegister) {
5725       __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister);
5726     }
5727     __ vextractf128($tmp$$XMMRegister, $src$$XMMRegister, y_idx);
5728     __ movq($tmp1$$Register, $val$$XMMRegister);
5729     __ vpinsrq($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp1$$Register, x_idx);
5730     __ vinsertf128($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, y_idx);
5731   %}
5732   ins_pipe( pipe_slow );
5733 %}
5734 
5735 instruct rvinsert8D(vecZ dst, vecZ src, vecZ tmp, vecY tmp2, regD val, rRegL tmp1, immU3 idx) %{
5736   predicate(UseAVX > 2 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
5737   match(Set dst (VectorInsert (Binary src val) idx));
5738   effect(TEMP dst, TEMP tmp, TEMP tmp1, TEMP tmp2);
5739   format %{ "evmovdquq  $dst,$src\n\t"
5740             "vextractf64x4  $tmp,$src\n\t"
5741             "vextractf128  $tmp,$tmp\n\t"
5742             "movq $tmp1,$val\n\t"
5743             "vpinsrq  $tmp,$tmp,$val\n\t"
5744             "vinsertf128  $tmp,$tmp,$tmp\n\t"
5745             "vinsertf64x4  $dst,$dst,$tmp\t! Insert 8D" %}
5746   ins_encode %{
5747     uint x_idx = $idx$$constant & 1;
5748     uint y_idx = ($idx$$constant >> 1) & 1;
5749     uint z_idx = ($idx$$constant >> 2) & 1;
5750 
5751     if ($dst$$XMMRegister != $src$$XMMRegister) {
5752       __ evmovdquq($dst$$XMMRegister, $src$$XMMRegister, 2);
5753     }
5754     __ vextractf64x4($tmp2$$XMMRegister, $src$$XMMRegister, z_idx);
5755     __ vextractf128($tmp$$XMMRegister, $tmp2$$XMMRegister, y_idx);
5756     __ movq($tmp1$$Register, $val$$XMMRegister);
5757     __ vpinsrq($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp1$$Register, x_idx);
5758     __ vinsertf128($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, y_idx);
5759     __ vinsertf64x4($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister, z_idx);
5760   %}
5761   ins_pipe( pipe_slow );
5762 %}
5763 
5764 // ====================REDUCTION ARITHMETIC=======================================
5765 
5766 instruct rsadd8B_reduction_reg(rRegI dst, rRegI src1, vecD src2, regF tmp, rRegI tmp2) %{
5767   predicate(UseSSE > 3 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
5768   match(Set dst (AddReductionVI src1 src2));
5769   effect(TEMP tmp, TEMP tmp2, TEMP dst);
5770   format %{
5771             "pshufd  $tmp,$src2,0x1\n\t"
5772             "paddb   $tmp,$src2\n\t"
5773             "movzbl  $dst,$src1\n\t"
5774             "pextrb  $tmp2,$tmp, 0x0\n\t"
5775             "addl    $dst,$tmp2\n\t"
5776             "pextrb  $tmp2,$tmp, 0x1\n\t"
5777             "addl    $dst,$tmp2\n\t"
5778             "pextrb  $tmp2,$tmp, 0x2\n\t"
5779             "addl    $dst,$tmp2\n\t"
5780             "pextrb  $tmp2,$tmp, 0x3\n\t"
5781             "addl    $dst,$tmp2\n\t"
5782             "movsbl  $dst,$dst\t! add reduction8B" %}
5783   ins_encode %{
5784     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x1);
5785     __ paddb($tmp$$XMMRegister, $src2$$XMMRegister);
5786     __ movzbl($dst$$Register, $src1$$Register);
5787     __ pextrb($tmp2$$Register, $tmp$$XMMRegister, 0x0);
5788     __ addl($dst$$Register, $tmp2$$Register);
5789     __ pextrb($tmp2$$Register, $tmp$$XMMRegister, 0x1);
5790     __ addl($dst$$Register, $tmp2$$Register);
5791     __ pextrb($tmp2$$Register, $tmp$$XMMRegister, 0x2);
5792     __ addl($dst$$Register, $tmp2$$Register);
5793     __ pextrb($tmp2$$Register, $tmp$$XMMRegister, 0x3);
5794     __ addl($dst$$Register, $tmp2$$Register);
5795     __ movsbl($dst$$Register, $dst$$Register);
5796   %}
5797   ins_pipe( pipe_slow );
5798 %}
5799 
5800 instruct rsadd16B_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp2, rRegI tmp3) %{
5801   predicate(UseSSE > 3 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
5802   match(Set dst (AddReductionVI src1 src2));
5803   effect(TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP dst);
5804   format %{ "pshufd  $tmp,$src2,0xE\n\t"
5805             "paddb   $tmp,$src2\n\t"
5806             "pshufd  $tmp2,$tmp,0x1\n\t"
5807             "paddb   $tmp,$tmp,$tmp2\n\t"
5808             "movzbl  $dst,$src1\n\t"
5809             "pextrb  $tmp3,$tmp, 0x0\n\t"
5810             "addl    $dst,$tmp3\n\t"
5811             "pextrb  $tmp3,$tmp, 0x1\n\t"
5812             "addl    $dst,$tmp3\n\t"
5813             "pextrb  $tmp3,$tmp, 0x2\n\t"
5814             "addl    $dst,$tmp3\n\t"
5815             "pextrb  $tmp3,$tmp, 0x3\n\t"
5816             "addl    $dst,$tmp3\n\t"
5817             "movsbl  $dst,$dst\t! add reduction16B" %}
5818   ins_encode %{
5819     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
5820     __ paddb($tmp$$XMMRegister, $src2$$XMMRegister);
5821     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1);
5822     __ paddb($tmp$$XMMRegister, $tmp2$$XMMRegister);
5823     __ movzbl($dst$$Register, $src1$$Register);
5824     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x0);
5825     __ addl($dst$$Register, $tmp3$$Register);
5826     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x1);
5827     __ addl($dst$$Register, $tmp3$$Register);
5828     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x2);
5829     __ addl($dst$$Register, $tmp3$$Register);
5830     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x3);
5831     __ addl($dst$$Register, $tmp3$$Register);
5832     __ movsbl($dst$$Register, $dst$$Register);
5833   %}
5834   ins_pipe( pipe_slow );
5835 %}
5836 
5837 instruct rvadd32B_reduction_reg(rRegI dst, rRegI src1, vecY src2, regF tmp, regF tmp2, rRegI tmp3) %{
5838   predicate(UseAVX > 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
5839   match(Set dst (AddReductionVI src1 src2));
5840   effect(TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP dst);
5841   format %{ "vextracti128_high  $tmp,$src2\n\t"
5842             "vpaddb  $tmp,$tmp,$src2\n\t"
5843             "pshufd  $tmp2,$tmp,0xE\n\t"
5844             "vpaddb  $tmp,$tmp,$tmp2\n\t"
5845             "pshufd  $tmp2,$tmp,0x1\n\t"
5846             "vpaddb  $tmp,$tmp,$tmp2\n\t"
5847             "movzbl  $dst,$src1\n\t"
5848             "pextrb  $tmp3,$tmp, 0x0\n\t"
5849             "addl    $dst,$tmp3\n\t"
5850             "pextrb  $tmp3,$tmp, 0x1\n\t"
5851             "addl    $dst,$tmp3\n\t"
5852             "pextrb  $tmp3,$tmp, 0x2\n\t"
5853             "addl    $dst,$tmp3\n\t"
5854             "pextrb  $tmp3,$tmp, 0x3\n\t"
5855             "addl    $dst,$tmp3\n\t"
5856             "movsbl  $dst,$dst\t! add reduction32B" %}
5857   ins_encode %{
5858     int vector_len = 0;
5859     __ vextracti128_high($tmp$$XMMRegister, $src2$$XMMRegister);
5860     __ vpaddb($tmp$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, vector_len);
5861     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE);
5862     __ vpaddb($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
5863     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1);
5864     __ vpaddb($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
5865     __ movzbl($dst$$Register, $src1$$Register);
5866     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x0);
5867     __ addl($dst$$Register, $tmp3$$Register);
5868     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x1);
5869     __ addl($dst$$Register, $tmp3$$Register);
5870     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x2);
5871     __ addl($dst$$Register, $tmp3$$Register);
5872     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x3);
5873     __ addl($dst$$Register, $tmp3$$Register);
5874     __ movsbl($dst$$Register, $dst$$Register);
5875   %}
5876   ins_pipe( pipe_slow );
5877 %}
5878 
5879 instruct rvadd64B_reduction_reg(rRegI dst, rRegI src1, vecZ src2, regF tmp, regF tmp2, rRegI tmp3) %{
5880   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
5881   match(Set dst (AddReductionVI src1 src2));
5882   effect(TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP dst);
5883   format %{ "vextracti64x4_high  $tmp2,$src2\n\t"
5884             "vpaddb  $tmp2,$tmp2,$src2\n\t"
5885             "vextracti128_high  $tmp,$tmp2\n\t"
5886             "vpaddb  $tmp,$tmp,$tmp2\n\t"
5887             "pshufd  $tmp2,$tmp,0xE\n\t"
5888             "vpaddb  $tmp,$tmp,$tmp2\n\t"
5889             "pshufd  $tmp2,$tmp,0x1\n\t"
5890             "vpaddb  $tmp,$tmp,$tmp2\n\t"
5891             "movzbl  $dst,$src1\n\t"
5892             "movdl   $tmp3,$tmp\n\t"
5893             "addl    $dst,$tmp3\n\t"
5894             "shrl    $tmp3,0x8\n\t"
5895             "addl    $dst,$tmp3\n\t"
5896             "shrl    $tmp3,0x8\n\t"
5897             "addl    $dst,$tmp3\n\t"
5898             "shrl    $tmp3,0x8\n\t"
5899             "addl    $dst,$tmp3\n\t"
5900             "movsbl  $dst,$dst\t! add reduction64B" %}
5901   ins_encode %{
5902     int vector_len = 0;
5903     __ vextracti64x4_high($tmp2$$XMMRegister, $src2$$XMMRegister);
5904     __ vpaddb($tmp2$$XMMRegister, $tmp2$$XMMRegister, $src2$$XMMRegister, 1);
5905     __ vextracti128_high($tmp$$XMMRegister, $tmp2$$XMMRegister);
5906     __ vpaddb($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
5907     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE);
5908     __ vpaddb($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
5909     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1);
5910     __ vpaddb($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
5911     __ movzbl($dst$$Register, $src1$$Register);
5912     __ movdl($tmp3$$Register, $tmp$$XMMRegister);
5913     __ addl($dst$$Register, $tmp3$$Register);
5914     __ shrl($tmp3$$Register, 8);
5915     __ addl($dst$$Register, $tmp3$$Register);
5916     __ shrl($tmp3$$Register, 8);
5917     __ addl($dst$$Register, $tmp3$$Register);
5918     __ shrl($tmp3$$Register, 8);
5919     __ addl($dst$$Register, $tmp3$$Register);
5920     __ movsbl($dst$$Register, $dst$$Register);
5921   %}
5922   ins_pipe( pipe_slow );
5923 %}
5924 
5925 instruct rsadd4S_reduction_reg(rRegI dst, rRegI src1, vecD src2, regF tmp, rRegI tmp2) %{
5926   predicate(UseSSE > 3 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
5927   match(Set dst (AddReductionVI src1 src2));
5928   effect(TEMP tmp, TEMP tmp2, TEMP dst);
5929   format %{
5930             "movdqu   $tmp,$src2\n\t"
5931             "phaddw   $tmp,$tmp\n\t"
5932             "phaddw   $tmp,$tmp\n\t"
5933             "movzwl   $dst,$src1\n\t"
5934             "pextrw   $tmp2,$tmp, 0x0\n\t"
5935             "addw     $dst,$tmp2\n\t"
5936             "movswl  $dst,$dst\t! add reduction4S" %}
5937   ins_encode %{
5938     __ movdqu($tmp$$XMMRegister, $src2$$XMMRegister);
5939     __ phaddw($tmp$$XMMRegister, $tmp$$XMMRegister);
5940     __ phaddw($tmp$$XMMRegister, $tmp$$XMMRegister);
5941     __ movzwl($dst$$Register, $src1$$Register);
5942     __ pextrw($tmp2$$Register, $tmp$$XMMRegister, 0x0);
5943     __ addw($dst$$Register, $tmp2$$Register);
5944     __ movswl($dst$$Register, $dst$$Register);
5945   %}
5946   ins_pipe( pipe_slow );
5947 %}
5948 
5949 instruct rvadd4S_reduction_reg(rRegI dst, rRegI src1, vecD src2, regF tmp, rRegI tmp2) %{
5950   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
5951   match(Set dst (AddReductionVI src1 src2));
5952   effect(TEMP tmp, TEMP tmp2, TEMP dst);
5953   format %{ "vphaddw  $tmp,$src2,$src2\n\t"
5954             "vphaddw  $tmp,$tmp,$tmp\n\t"
5955             "movzwl   $dst,$src1\n\t"
5956             "pextrw   $tmp2,$tmp, 0x0\n\t"
5957             "addw     $dst,$tmp2\n\t"
5958             "movswl  $dst,$dst\t! add reduction4S" %}
5959   ins_encode %{
5960     int vector_len = 0;
5961     __ vphaddw($tmp$$XMMRegister, $src2$$XMMRegister, $src2$$XMMRegister, vector_len);
5962     __ vphaddw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp$$XMMRegister, vector_len);
5963     __ movzwl($dst$$Register, $src1$$Register);
5964     __ pextrw($tmp2$$Register, $tmp$$XMMRegister, 0x0);
5965     __ addw($dst$$Register, $tmp2$$Register);
5966     __ movswl($dst$$Register, $dst$$Register);
5967   %}
5968   ins_pipe( pipe_slow );
5969 %}
5970 
5971 instruct rsadd8S_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, rRegI tmp2) %{
5972   predicate(UseSSE > 3 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
5973   match(Set dst (AddReductionVI src1 src2));
5974   effect(TEMP tmp, TEMP tmp2, TEMP dst);
5975   format %{
5976             "movdqu   $tmp,$src2\n\t"
5977             "phaddw  $tmp,$tmp\n\t"
5978             "phaddw  $tmp,$tmp\n\t"
5979             "phaddw  $tmp,$tmp\n\t"
5980             "movzwl   $dst,$src1\n\t"
5981             "pextrw   $tmp2,$tmp, 0x0\n\t"
5982             "addw     $dst,$tmp2\n\t"
5983             "movswl  $dst,$dst\t! add reduction8S" %}
5984   ins_encode %{
5985     __ movdqu($tmp$$XMMRegister, $src2$$XMMRegister);
5986     __ phaddw($tmp$$XMMRegister, $tmp$$XMMRegister);
5987     __ phaddw($tmp$$XMMRegister, $tmp$$XMMRegister);
5988     __ phaddw($tmp$$XMMRegister, $tmp$$XMMRegister);
5989     __ movzwl($dst$$Register, $src1$$Register);
5990     __ pextrw($tmp2$$Register, $tmp$$XMMRegister, 0x0);
5991     __ addw($dst$$Register, $tmp2$$Register);
5992     __ movswl($dst$$Register, $dst$$Register);
5993   %}
5994   ins_pipe( pipe_slow );
5995 %}
5996 
5997 instruct rvadd8S_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, rRegI tmp2) %{
5998   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
5999   match(Set dst (AddReductionVI src1 src2));
6000   effect(TEMP tmp, TEMP tmp2, TEMP dst);
6001   format %{ "vphaddw  $tmp,$src2,$src2\n\t"
6002             "vphaddw  $tmp,$tmp,$tmp\n\t"
6003             "vphaddw  $tmp,$tmp,$tmp\n\t"
6004             "movzwl   $dst,$src1\n\t"
6005             "pextrw   $tmp2,$tmp, 0x0\n\t"
6006             "addw     $dst,$tmp2\n\t"
6007             "movswl  $dst,$dst\t! add reduction8S" %}
6008   ins_encode %{
6009     int vector_len = 0;
6010     __ vphaddw($tmp$$XMMRegister, $src2$$XMMRegister, $src2$$XMMRegister, vector_len);
6011     __ vphaddw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp$$XMMRegister, vector_len);
6012     __ vphaddw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp$$XMMRegister, vector_len);
6013     __ movzwl($dst$$Register, $src1$$Register);
6014     __ pextrw($tmp2$$Register, $tmp$$XMMRegister, 0x0);
6015     __ addw($dst$$Register, $tmp2$$Register);
6016     __ movswl($dst$$Register, $dst$$Register);
6017   %}
6018   ins_pipe( pipe_slow );
6019 %}
6020 
6021 instruct rvadd16S_reduction_reg(rRegI dst, rRegI src1, vecY src2, regF tmp, rRegI tmp2) %{
6022   predicate(UseAVX > 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
6023   match(Set dst (AddReductionVI src1 src2));
6024   effect(TEMP tmp, TEMP tmp2, TEMP dst);
6025   format %{ "vphaddw  $tmp,$src2,$src2\n\t"
6026             "vphaddw  $tmp,$tmp,$tmp\n\t"
6027             "vphaddw  $tmp,$tmp,$tmp\n\t"
6028             "vphaddw  $tmp,$tmp,$tmp\n\t"
6029             "movzwl   $dst,$src1\n\t"
6030             "pextrw   $tmp2,$tmp, 0x0\n\t"
6031             "addw     $dst,$tmp2\n\t"
6032             "movswl  $dst,$dst\t! add reduction16S" %}
6033   ins_encode %{
6034     int vector_len = 1;
6035     __ vphaddw($tmp$$XMMRegister, $src2$$XMMRegister, $src2$$XMMRegister, vector_len);
6036     __ vpermq($tmp$$XMMRegister, $tmp$$XMMRegister, 0xD8, vector_len);
6037     __ vphaddw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp$$XMMRegister, vector_len);
6038     __ vphaddw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp$$XMMRegister, vector_len);
6039     __ vphaddw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp$$XMMRegister, vector_len);
6040     __ movzwl($dst$$Register, $src1$$Register);
6041     __ pextrw($tmp2$$Register, $tmp$$XMMRegister, 0x0);
6042     __ addw($dst$$Register, $tmp2$$Register);
6043     __ movswl($dst$$Register, $dst$$Register);
6044   %}
6045   ins_pipe( pipe_slow );
6046 %}
6047 
6048 instruct rvadd32S_reduction_reg_evex(rRegI dst, rRegI src1, vecZ src2, regF tmp, regF tmp2, rRegI tmp3) %{
6049   predicate(UseAVX > 2  && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
6050   match(Set dst (AddReductionVI src1 src2));
6051   effect(TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP dst);
6052   format %{ "vextracti64x4_high  $tmp2,$src2\n\t"
6053             "vpaddw  $tmp2,$tmp2,$src2\n\t"
6054             "vextracti128_high  $tmp,$tmp2\n\t"
6055             "vpaddw  $tmp,$tmp,$tmp2\n\t"
6056             "pshufd  $tmp2,$tmp,0xE\n\t"
6057             "vpaddw  $tmp,$tmp,$tmp2\n\t"
6058             "pshufd  $tmp2,$tmp,0x1\n\t"
6059             "vpaddw  $tmp,$tmp,$tmp2\n\t"
6060             "movdl   $tmp3,$tmp\n\t"
6061             "addw    $dst,$tmp3\n\t"
6062             "shrl    $tmp3,0x16\n\t"
6063             "addw     $dst,$tmp3\n\t"
6064             "movswl  $dst,$dst\t! add reduction32S" %}
6065   ins_encode %{
6066     __ vextracti64x4_high($tmp2$$XMMRegister, $src2$$XMMRegister);
6067     __ vpaddw($tmp2$$XMMRegister, $tmp2$$XMMRegister, $src2$$XMMRegister, 1);
6068     __ vextracti128_high($tmp$$XMMRegister, $tmp2$$XMMRegister);
6069     __ vpaddw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, 0);
6070     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE);
6071     __ vpaddw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, 0);
6072     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1);
6073     __ vpaddw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, 0);
6074     __ movdl($tmp3$$Register, $tmp$$XMMRegister);
6075     __ movzwl($dst$$Register, $src1$$Register);
6076     __ addw($dst$$Register, $tmp3$$Register);
6077     __ shrl($tmp3$$Register, 16);
6078     __ addw($dst$$Register, $tmp3$$Register);
6079     __ movswl($dst$$Register, $dst$$Register);
6080   %}
6081   ins_pipe( pipe_slow );
6082 %}
6083 
6084 instruct rsadd2I_reduction_reg(rRegI dst, rRegI src1, vecD src2, regF tmp, regF tmp2) %{
6085   predicate(UseSSE > 2 && UseAVX == 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
6086   match(Set dst (AddReductionVI src1 src2));
6087   effect(TEMP tmp2, TEMP tmp);
6088   format %{ "movdqu  $tmp2,$src2\n\t"
6089             "phaddd  $tmp2,$tmp2\n\t"
6090             "movd    $tmp,$src1\n\t"
6091             "paddd   $tmp,$tmp2\n\t"
6092             "movd    $dst,$tmp\t! add reduction2I" %}
6093   ins_encode %{
6094     __ movdqu($tmp2$$XMMRegister, $src2$$XMMRegister);
6095     __ phaddd($tmp2$$XMMRegister, $tmp2$$XMMRegister);
6096     __ movdl($tmp$$XMMRegister, $src1$$Register);
6097     __ paddd($tmp$$XMMRegister, $tmp2$$XMMRegister);
6098     __ movdl($dst$$Register, $tmp$$XMMRegister);
6099   %}
6100   ins_pipe( pipe_slow );
6101 %}
6102 
6103 instruct rvadd2I_reduction_reg(rRegI dst, rRegI src1, vecD src2, regF tmp, regF tmp2) %{
6104   predicate(VM_Version::supports_avxonly()  && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
6105   match(Set dst (AddReductionVI src1 src2));
6106   effect(TEMP tmp, TEMP tmp2);
6107   format %{ "vphaddd  $tmp,$src2,$src2\n\t"
6108             "movd     $tmp2,$src1\n\t"
6109             "vpaddd   $tmp2,$tmp2,$tmp\n\t"
6110             "movd     $dst,$tmp2\t! add reduction2I" %}
6111   ins_encode %{
6112     int vector_len = 0;
6113     __ vphaddd($tmp$$XMMRegister, $src2$$XMMRegister, $src2$$XMMRegister, vector_len);
6114     __ movdl($tmp2$$XMMRegister, $src1$$Register);
6115     __ vpaddd($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, vector_len);
6116     __ movdl($dst$$Register, $tmp2$$XMMRegister);
6117   %}
6118   ins_pipe( pipe_slow );
6119 %}
6120 
6121 instruct rvadd2I_reduction_reg_evex(rRegI dst, rRegI src1, vecD src2, regF tmp, regF tmp2) %{
6122   predicate(UseAVX > 2  && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
6123   match(Set dst (AddReductionVI src1 src2));
6124   effect(TEMP tmp, TEMP tmp2);
6125   format %{ "pshufd  $tmp2,$src2,0x1\n\t"
6126             "vpaddd  $tmp,$src2,$tmp2\n\t"
6127             "movd    $tmp2,$src1\n\t"
6128             "vpaddd  $tmp2,$tmp,$tmp2\n\t"
6129             "movd    $dst,$tmp2\t! add reduction2I" %}
6130   ins_encode %{
6131     int vector_len = 0;
6132     __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0x1);
6133     __ vpaddd($tmp$$XMMRegister, $src2$$XMMRegister, $tmp2$$XMMRegister, vector_len);
6134     __ movdl($tmp2$$XMMRegister, $src1$$Register);
6135     __ vpaddd($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
6136     __ movdl($dst$$Register, $tmp2$$XMMRegister);
6137   %}
6138   ins_pipe( pipe_slow );
6139 %}
6140 
6141 instruct rsadd4I_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp2) %{
6142   predicate(UseSSE > 2 && UseAVX == 0  && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
6143   match(Set dst (AddReductionVI src1 src2));
6144   effect(TEMP tmp, TEMP tmp2);
6145   format %{ "movdqu  $tmp,$src2\n\t"
6146             "phaddd  $tmp,$tmp\n\t"
6147             "phaddd  $tmp,$tmp\n\t"
6148             "movd    $tmp2,$src1\n\t"
6149             "paddd   $tmp2,$tmp\n\t"
6150             "movd    $dst,$tmp2\t! add reduction4I" %}
6151   ins_encode %{
6152     __ movdqu($tmp$$XMMRegister, $src2$$XMMRegister);
6153     __ phaddd($tmp$$XMMRegister, $tmp$$XMMRegister);
6154     __ phaddd($tmp$$XMMRegister, $tmp$$XMMRegister);
6155     __ movdl($tmp2$$XMMRegister, $src1$$Register);
6156     __ paddd($tmp2$$XMMRegister, $tmp$$XMMRegister);
6157     __ movdl($dst$$Register, $tmp2$$XMMRegister);
6158   %}
6159   ins_pipe( pipe_slow );
6160 %}
6161 
6162 instruct rvadd4I_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp2) %{
6163   predicate(VM_Version::supports_avxonly() && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
6164   match(Set dst (AddReductionVI src1 src2));
6165   effect(TEMP tmp, TEMP tmp2);
6166   format %{ "vphaddd  $tmp,$src2,$src2\n\t"
6167             "vphaddd  $tmp,$tmp,$tmp\n\t"
6168             "movd     $tmp2,$src1\n\t"
6169             "vpaddd   $tmp2,$tmp2,$tmp\n\t"
6170             "movd     $dst,$tmp2\t! add reduction4I" %}
6171   ins_encode %{
6172     int vector_len = 0;
6173     __ vphaddd($tmp$$XMMRegister, $src2$$XMMRegister, $src2$$XMMRegister, vector_len);
6174     __ vphaddd($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp$$XMMRegister, vector_len);
6175     __ movdl($tmp2$$XMMRegister, $src1$$Register);
6176     __ vpaddd($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, vector_len);
6177     __ movdl($dst$$Register, $tmp2$$XMMRegister);
6178   %}
6179   ins_pipe( pipe_slow );
6180 %}
6181 
6182 instruct rvadd4I_reduction_reg_evex(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp2) %{
6183   predicate(UseAVX > 2  && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
6184   match(Set dst (AddReductionVI src1 src2));
6185   effect(TEMP tmp, TEMP tmp2);
6186   format %{ "pshufd  $tmp2,$src2,0xE\n\t"
6187             "vpaddd  $tmp,$src2,$tmp2\n\t"
6188             "pshufd  $tmp2,$tmp,0x1\n\t"
6189             "vpaddd  $tmp,$tmp,$tmp2\n\t"
6190             "movd    $tmp2,$src1\n\t"
6191             "vpaddd  $tmp2,$tmp,$tmp2\n\t"
6192             "movd    $dst,$tmp2\t! add reduction4I" %}
6193   ins_encode %{
6194     int vector_len = 0;
6195     __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0xE);
6196     __ vpaddd($tmp$$XMMRegister, $src2$$XMMRegister, $tmp2$$XMMRegister, vector_len);
6197     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1);
6198     __ vpaddd($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
6199     __ movdl($tmp2$$XMMRegister, $src1$$Register);
6200     __ vpaddd($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
6201     __ movdl($dst$$Register, $tmp2$$XMMRegister);
6202   %}
6203   ins_pipe( pipe_slow );
6204 %}
6205 
6206 instruct rvadd8I_reduction_reg(rRegI dst, rRegI src1, vecY src2, regF tmp, regF tmp2) %{
6207   predicate(VM_Version::supports_avxonly()  && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
6208   match(Set dst (AddReductionVI src1 src2));
6209   effect(TEMP tmp, TEMP tmp2);
6210   format %{ "vphaddd  $tmp,$src2,$src2\n\t"
6211             "vphaddd  $tmp,$tmp,$tmp2\n\t"
6212             "vextracti128_high  $tmp2,$tmp\n\t"
6213             "vpaddd   $tmp,$tmp,$tmp2\n\t"
6214             "movd     $tmp2,$src1\n\t"
6215             "vpaddd   $tmp2,$tmp2,$tmp\n\t"
6216             "movd     $dst,$tmp2\t! add reduction8I" %}
6217   ins_encode %{
6218     int vector_len = 1;
6219     __ vphaddd($tmp$$XMMRegister, $src2$$XMMRegister, $src2$$XMMRegister, vector_len);
6220     __ vphaddd($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
6221     __ vextracti128_high($tmp2$$XMMRegister, $tmp$$XMMRegister);
6222     __ vpaddd($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, 0);
6223     __ movdl($tmp2$$XMMRegister, $src1$$Register);
6224     __ vpaddd($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, 0);
6225     __ movdl($dst$$Register, $tmp2$$XMMRegister);
6226   %}
6227   ins_pipe( pipe_slow );
6228 %}
6229 
6230 instruct rvadd8I_reduction_reg_evex(rRegI dst, rRegI src1, vecY src2, regF tmp, regF tmp2) %{
6231   predicate(UseAVX > 2  && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
6232   match(Set dst (AddReductionVI src1 src2));
6233   effect(TEMP tmp, TEMP tmp2);
6234   format %{ "vextracti128_high  $tmp,$src2\n\t"
6235             "vpaddd  $tmp,$tmp,$src2\n\t"
6236             "pshufd  $tmp2,$tmp,0xE\n\t"
6237             "vpaddd  $tmp,$tmp,$tmp2\n\t"
6238             "pshufd  $tmp2,$tmp,0x1\n\t"
6239             "vpaddd  $tmp,$tmp,$tmp2\n\t"
6240             "movd    $tmp2,$src1\n\t"
6241             "vpaddd  $tmp2,$tmp,$tmp2\n\t"
6242             "movd    $dst,$tmp2\t! add reduction8I" %}
6243   ins_encode %{
6244     int vector_len = 0;
6245     __ vextracti128_high($tmp$$XMMRegister, $src2$$XMMRegister);
6246     __ vpaddd($tmp$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, vector_len);
6247     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE);
6248     __ vpaddd($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
6249     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1);
6250     __ vpaddd($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
6251     __ movdl($tmp2$$XMMRegister, $src1$$Register);
6252     __ vpaddd($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
6253     __ movdl($dst$$Register, $tmp2$$XMMRegister);
6254   %}
6255   ins_pipe( pipe_slow );
6256 %}
6257 
6258 instruct rvadd16I_reduction_reg_evex(rRegI dst, rRegI src1, vecZ src2, regF tmp, regF tmp2, regF tmp3) %{
6259   predicate(UseAVX > 2  && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
6260   match(Set dst (AddReductionVI src1 src2));
6261   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
6262   format %{ "vextracti64x4_high  $tmp3,$src2\n\t"
6263             "vpaddd  $tmp3,$tmp3,$src2\n\t"
6264             "vextracti128_high  $tmp,$tmp3\n\t"
6265             "vpaddd  $tmp,$tmp,$tmp3\n\t"
6266             "pshufd  $tmp2,$tmp,0xE\n\t"
6267             "vpaddd  $tmp,$tmp,$tmp2\n\t"
6268             "pshufd  $tmp2,$tmp,0x1\n\t"
6269             "vpaddd  $tmp,$tmp,$tmp2\n\t"
6270             "movd    $tmp2,$src1\n\t"
6271             "vpaddd  $tmp2,$tmp,$tmp2\n\t"
6272             "movd    $dst,$tmp2\t! mul reduction16I" %}
6273   ins_encode %{
6274     __ vextracti64x4_high($tmp3$$XMMRegister, $src2$$XMMRegister);
6275     __ vpaddd($tmp3$$XMMRegister, $tmp3$$XMMRegister, $src2$$XMMRegister, 1);
6276     __ vextracti128_high($tmp$$XMMRegister, $tmp3$$XMMRegister);
6277     __ vpaddd($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp3$$XMMRegister, 0);
6278     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE);
6279     __ vpaddd($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, 0);
6280     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1);
6281     __ vpaddd($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, 0);
6282     __ movdl($tmp2$$XMMRegister, $src1$$Register);
6283     __ vpaddd($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, 0);
6284     __ movdl($dst$$Register, $tmp2$$XMMRegister);
6285   %}
6286   ins_pipe( pipe_slow );
6287 %}
6288 
6289 #ifdef _LP64
6290 instruct rvadd2L_reduction_reg(rRegL dst, rRegL src1, vecX src2, regF tmp, regF tmp2) %{
6291   predicate(UseAVX > 2);
6292   match(Set dst (AddReductionVL src1 src2));
6293   effect(TEMP tmp, TEMP tmp2);
6294   format %{ "pshufd  $tmp2,$src2,0xE\n\t"
6295             "vpaddq  $tmp,$src2,$tmp2\n\t"
6296             "movdq   $tmp2,$src1\n\t"
6297             "vpaddq  $tmp2,$tmp,$tmp2\n\t"
6298             "movdq   $dst,$tmp2\t! add reduction2L" %}
6299   ins_encode %{
6300     __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0xE);
6301     __ vpaddq($tmp$$XMMRegister, $src2$$XMMRegister, $tmp2$$XMMRegister, 0);
6302     __ movdq($tmp2$$XMMRegister, $src1$$Register);
6303     __ vpaddq($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, 0);
6304     __ movdq($dst$$Register, $tmp2$$XMMRegister);
6305   %}
6306   ins_pipe( pipe_slow );
6307 %}
6308 
6309 instruct rvadd4L_reduction_reg(rRegL dst, rRegL src1, vecY src2, regF tmp, regF tmp2) %{
6310   predicate(UseAVX > 2);
6311   match(Set dst (AddReductionVL src1 src2));
6312   effect(TEMP tmp, TEMP tmp2);
6313   format %{ "vextracti128_high  $tmp,$src2\n\t"
6314             "vpaddq  $tmp2,$tmp,$src2\n\t"
6315             "pshufd  $tmp,$tmp2,0xE\n\t"
6316             "vpaddq  $tmp2,$tmp2,$tmp\n\t"
6317             "movdq   $tmp,$src1\n\t"
6318             "vpaddq  $tmp2,$tmp2,$tmp\n\t"
6319             "movdq   $dst,$tmp2\t! add reduction4L" %}
6320   ins_encode %{
6321     __ vextracti128_high($tmp$$XMMRegister, $src2$$XMMRegister);
6322     __ vpaddq($tmp2$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, 0);
6323     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE);
6324     __ vpaddq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, 0);
6325     __ movdq($tmp$$XMMRegister, $src1$$Register);
6326     __ vpaddq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, 0);
6327     __ movdq($dst$$Register, $tmp2$$XMMRegister);
6328   %}
6329   ins_pipe( pipe_slow );
6330 %}
6331 
6332 instruct rvadd8L_reduction_reg(rRegL dst, rRegL src1, vecZ src2, regF tmp, regF tmp2) %{
6333   predicate(UseAVX > 2);
6334   match(Set dst (AddReductionVL src1 src2));
6335   effect(TEMP tmp, TEMP tmp2);
6336   format %{ "vextracti64x4_high  $tmp2,$src2\n\t"
6337             "vpaddq  $tmp2,$tmp2,$src2\n\t"
6338             "vextracti128_high  $tmp,$tmp2\n\t"
6339             "vpaddq  $tmp2,$tmp2,$tmp\n\t"
6340             "pshufd  $tmp,$tmp2,0xE\n\t"
6341             "vpaddq  $tmp2,$tmp2,$tmp\n\t"
6342             "movdq   $tmp,$src1\n\t"
6343             "vpaddq  $tmp2,$tmp2,$tmp\n\t"
6344             "movdq   $dst,$tmp2\t! add reduction8L" %}
6345   ins_encode %{
6346     __ vextracti64x4_high($tmp2$$XMMRegister, $src2$$XMMRegister);
6347     __ vpaddq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $src2$$XMMRegister, 1);
6348     __ vextracti128_high($tmp$$XMMRegister, $tmp2$$XMMRegister);
6349     __ vpaddq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, 0);
6350     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE);
6351     __ vpaddq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, 0);
6352     __ movdq($tmp$$XMMRegister, $src1$$Register);
6353     __ vpaddq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, 0);
6354     __ movdq($dst$$Register, $tmp2$$XMMRegister);
6355   %}
6356   ins_pipe( pipe_slow );
6357 %}
6358 #endif
6359 
6360 instruct rsadd2F_reduction_reg(regF dst, vecD src2, regF tmp) %{
6361   predicate(UseSSE >= 1 && UseAVX == 0);
6362   match(Set dst (AddReductionVF dst src2));
6363   effect(TEMP dst, TEMP tmp);
6364   format %{ "addss   $dst,$src2\n\t"
6365             "pshufd  $tmp,$src2,0x01\n\t"
6366             "addss   $dst,$tmp\t! add reduction2F" %}
6367   ins_encode %{
6368     __ addss($dst$$XMMRegister, $src2$$XMMRegister);
6369     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x01);
6370     __ addss($dst$$XMMRegister, $tmp$$XMMRegister);
6371   %}
6372   ins_pipe( pipe_slow );
6373 %}
6374 
6375 instruct rvadd2F_reduction_reg(regF dst, vecD src2, regF tmp) %{
6376   predicate(UseAVX > 0);
6377   match(Set dst (AddReductionVF dst src2));
6378   effect(TEMP dst, TEMP tmp);
6379   format %{ "vaddss  $dst,$dst,$src2\n\t"
6380             "pshufd  $tmp,$src2,0x01\n\t"
6381             "vaddss  $dst,$dst,$tmp\t! add reduction2F" %}
6382   ins_encode %{
6383     __ vaddss($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister);
6384     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x01);
6385     __ vaddss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6386   %}
6387   ins_pipe( pipe_slow );
6388 %}
6389 
6390 instruct rsadd4F_reduction_reg(regF dst, vecX src2, regF tmp) %{
6391   predicate(UseSSE >= 1 && UseAVX == 0);
6392   match(Set dst (AddReductionVF dst src2));
6393   effect(TEMP dst, TEMP tmp);
6394   format %{ "addss   $dst,$src2\n\t"
6395             "pshufd  $tmp,$src2,0x01\n\t"
6396             "addss   $dst,$tmp\n\t"
6397             "pshufd  $tmp,$src2,0x02\n\t"
6398             "addss   $dst,$tmp\n\t"
6399             "pshufd  $tmp,$src2,0x03\n\t"
6400             "addss   $dst,$tmp\t! add reduction4F" %}
6401   ins_encode %{
6402     __ addss($dst$$XMMRegister, $src2$$XMMRegister);
6403     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x01);
6404     __ addss($dst$$XMMRegister, $tmp$$XMMRegister);
6405     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x02);
6406     __ addss($dst$$XMMRegister, $tmp$$XMMRegister);
6407     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x03);
6408     __ addss($dst$$XMMRegister, $tmp$$XMMRegister);
6409   %}
6410   ins_pipe( pipe_slow );
6411 %}
6412 
6413 instruct rvadd4F_reduction_reg(regF dst, vecX src2, regF tmp) %{
6414   predicate(UseAVX > 0);
6415   match(Set dst (AddReductionVF dst src2));
6416   effect(TEMP tmp, TEMP dst);
6417   format %{ "vaddss  $dst,dst,$src2\n\t"
6418             "pshufd  $tmp,$src2,0x01\n\t"
6419             "vaddss  $dst,$dst,$tmp\n\t"
6420             "pshufd  $tmp,$src2,0x02\n\t"
6421             "vaddss  $dst,$dst,$tmp\n\t"
6422             "pshufd  $tmp,$src2,0x03\n\t"
6423             "vaddss  $dst,$dst,$tmp\t! add reduction4F" %}
6424   ins_encode %{
6425     __ vaddss($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister);
6426     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x01);
6427     __ vaddss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6428     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x02);
6429     __ vaddss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6430     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x03);
6431     __ vaddss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6432   %}
6433   ins_pipe( pipe_slow );
6434 %}
6435 
6436 instruct radd8F_reduction_reg(regF dst, vecY src2, regF tmp, regF tmp2) %{
6437   predicate(UseAVX > 0);
6438   match(Set dst (AddReductionVF dst src2));
6439   effect(TEMP tmp, TEMP dst, TEMP tmp2);
6440   format %{ "vaddss  $dst,$dst,$src2\n\t"
6441             "pshufd  $tmp,$src2,0x01\n\t"
6442             "vaddss  $dst,$dst,$tmp\n\t"
6443             "pshufd  $tmp,$src2,0x02\n\t"
6444             "vaddss  $dst,$dst,$tmp\n\t"
6445             "pshufd  $tmp,$src2,0x03\n\t"
6446             "vaddss  $dst,$dst,$tmp\n\t"
6447             "vextractf128_high  $tmp2,$src2\n\t"
6448             "vaddss  $dst,$dst,$tmp2\n\t"
6449             "pshufd  $tmp,$tmp2,0x01\n\t"
6450             "vaddss  $dst,$dst,$tmp\n\t"
6451             "pshufd  $tmp,$tmp2,0x02\n\t"
6452             "vaddss  $dst,$dst,$tmp\n\t"
6453             "pshufd  $tmp,$tmp2,0x03\n\t"
6454             "vaddss  $dst,$dst,$tmp\t! add reduction8F" %}
6455   ins_encode %{
6456     __ vaddss($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister);
6457     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x01);
6458     __ vaddss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6459     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x02);
6460     __ vaddss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6461     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x03);
6462     __ vaddss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6463     __ vextractf128_high($tmp2$$XMMRegister, $src2$$XMMRegister);
6464     __ vaddss($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister);
6465     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x01);
6466     __ vaddss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6467     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x02);
6468     __ vaddss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6469     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x03);
6470     __ vaddss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6471   %}
6472   ins_pipe( pipe_slow );
6473 %}
6474 
6475 instruct radd16F_reduction_reg(regF dst, vecZ src2, regF tmp, regF tmp2) %{
6476   predicate(UseAVX > 2);
6477   match(Set dst (AddReductionVF dst src2));
6478   effect(TEMP tmp, TEMP dst, TEMP tmp2);
6479   format %{ "vaddss  $dst,$dst,$src2\n\t"
6480             "pshufd  $tmp,$src2,0x01\n\t"
6481             "vaddss  $dst,$dst,$tmp\n\t"
6482             "pshufd  $tmp,$src2,0x02\n\t"
6483             "vaddss  $dst,$dst,$tmp\n\t"
6484             "pshufd  $tmp,$src2,0x03\n\t"
6485             "vaddss  $dst,$dst,$tmp\n\t"
6486             "vextractf32x4  $tmp2,$src2,0x1\n\t"
6487             "vaddss  $dst,$dst,$tmp2\n\t"
6488             "pshufd  $tmp,$tmp2,0x01\n\t"
6489             "vaddss  $dst,$dst,$tmp\n\t"
6490             "pshufd  $tmp,$tmp2,0x02\n\t"
6491             "vaddss  $dst,$dst,$tmp\n\t"
6492             "pshufd  $tmp,$tmp2,0x03\n\t"
6493             "vaddss  $dst,$dst,$tmp\n\t"
6494             "vextractf32x4  $tmp2,$src2,0x2\n\t"
6495             "vaddss  $dst,$dst,$tmp2\n\t"
6496             "pshufd  $tmp,$tmp2,0x01\n\t"
6497             "vaddss  $dst,$dst,$tmp\n\t"
6498             "pshufd  $tmp,$tmp2,0x02\n\t"
6499             "vaddss  $dst,$dst,$tmp\n\t"
6500             "pshufd  $tmp,$tmp2,0x03\n\t"
6501             "vaddss  $dst,$dst,$tmp\n\t"
6502             "vextractf32x4  $tmp2,$src2,0x3\n\t"
6503             "vaddss  $dst,$dst,$tmp2\n\t"
6504             "pshufd  $tmp,$tmp2,0x01\n\t"
6505             "vaddss  $dst,$dst,$tmp\n\t"
6506             "pshufd  $tmp,$tmp2,0x02\n\t"
6507             "vaddss  $dst,$dst,$tmp\n\t"
6508             "pshufd  $tmp,$tmp2,0x03\n\t"
6509             "vaddss  $dst,$dst,$tmp\t! add reduction16F" %}
6510   ins_encode %{
6511     __ vaddss($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister);
6512     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x01);
6513     __ vaddss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6514     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x02);
6515     __ vaddss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6516     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x03);
6517     __ vaddss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6518     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x1);
6519     __ vaddss($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister);
6520     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x01);
6521     __ vaddss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6522     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x02);
6523     __ vaddss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6524     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x03);
6525     __ vaddss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6526     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x2);
6527     __ vaddss($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister);
6528     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x01);
6529     __ vaddss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6530     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x02);
6531     __ vaddss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6532     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x03);
6533     __ vaddss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6534     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x3);
6535     __ vaddss($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister);
6536     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x01);
6537     __ vaddss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6538     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x02);
6539     __ vaddss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6540     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x03);
6541     __ vaddss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6542   %}
6543   ins_pipe( pipe_slow );
6544 %}
6545 
6546 instruct rsadd2D_reduction_reg(regD dst, vecX src2, regD tmp) %{
6547   predicate(UseSSE >= 1 && UseAVX == 0);
6548   match(Set dst (AddReductionVD dst src2));
6549   effect(TEMP tmp, TEMP dst);
6550   format %{ "addsd   $dst,$src2\n\t"
6551             "pshufd  $tmp,$src2,0xE\n\t"
6552             "addsd   $dst,$tmp\t! add reduction2D" %}
6553   ins_encode %{
6554     __ addsd($dst$$XMMRegister, $src2$$XMMRegister);
6555     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
6556     __ addsd($dst$$XMMRegister, $tmp$$XMMRegister);
6557   %}
6558   ins_pipe( pipe_slow );
6559 %}
6560 
6561 instruct rvadd2D_reduction_reg(regD dst, vecX src2, regD tmp) %{
6562   predicate(UseAVX > 0);
6563   match(Set dst (AddReductionVD dst src2));
6564   effect(TEMP tmp, TEMP dst);
6565   format %{ "vaddsd  $dst,$dst,$src2\n\t"
6566             "pshufd  $tmp,$src2,0xE\n\t"
6567             "vaddsd  $dst,$dst,$tmp\t! add reduction2D" %}
6568   ins_encode %{
6569     __ vaddsd($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister);
6570     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
6571     __ vaddsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6572   %}
6573   ins_pipe( pipe_slow );
6574 %}
6575 
6576 instruct rvadd4D_reduction_reg(regD dst, vecY src2, regD tmp, regD tmp2) %{
6577   predicate(UseAVX > 0);
6578   match(Set dst (AddReductionVD dst src2));
6579   effect(TEMP tmp, TEMP dst, TEMP tmp2);
6580   format %{ "vaddsd  $dst,$dst,$src2\n\t"
6581             "pshufd  $tmp,$src2,0xE\n\t"
6582             "vaddsd  $dst,$dst,$tmp\n\t"
6583             "vextractf32x4  $tmp2,$src2,0x1\n\t"
6584             "vaddsd  $dst,$dst,$tmp2\n\t"
6585             "pshufd  $tmp,$tmp2,0xE\n\t"
6586             "vaddsd  $dst,$dst,$tmp\t! add reduction4D" %}
6587   ins_encode %{
6588     __ vaddsd($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister);
6589     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
6590     __ vaddsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6591     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x1);
6592     __ vaddsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister);
6593     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE);
6594     __ vaddsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6595   %}
6596   ins_pipe( pipe_slow );
6597 %}
6598 
6599 instruct rvadd8D_reduction_reg(regD dst, vecZ src2, regD tmp, regD tmp2) %{
6600   predicate(UseAVX > 2);
6601   match(Set dst (AddReductionVD dst src2));
6602   effect(TEMP tmp, TEMP dst, TEMP tmp2);
6603   format %{ "vaddsd  $dst,$dst,$src2\n\t"
6604             "pshufd  $tmp,$src2,0xE\n\t"
6605             "vaddsd  $dst,$dst,$tmp\n\t"
6606             "vextractf32x4  $tmp2,$src2,0x1\n\t"
6607             "vaddsd  $dst,$dst,$tmp2\n\t"
6608             "pshufd  $tmp,$tmp2,0xE\n\t"
6609             "vaddsd  $dst,$dst,$tmp\n\t"
6610             "vextractf32x4  $tmp2,$src2,0x2\n\t"
6611             "vaddsd  $dst,$dst,$tmp2\n\t"
6612             "pshufd  $tmp,$tmp2,0xE\n\t"
6613             "vaddsd  $dst,$dst,$tmp\n\t"
6614             "vextractf32x4  $tmp2,$src2,0x3\n\t"
6615             "vaddsd  $dst,$dst,$tmp2\n\t"
6616             "pshufd  $tmp,$tmp2,0xE\n\t"
6617             "vaddsd  $dst,$dst,$tmp\t! add reduction8D" %}
6618   ins_encode %{
6619     __ vaddsd($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister);
6620     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
6621     __ vaddsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6622     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x1);
6623     __ vaddsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister);
6624     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE);
6625     __ vaddsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6626     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x2);
6627     __ vaddsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister);
6628     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE);
6629     __ vaddsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6630     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x3);
6631     __ vaddsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister);
6632     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE);
6633     __ vaddsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6634   %}
6635   ins_pipe( pipe_slow );
6636 %}
6637 
6638 instruct rssub2F_reduction_reg(regF dst, vecD src2, regF tmp) %{
6639   predicate(UseSSE >= 1 && UseAVX == 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
6640   match(Set dst (SubReductionVFP dst src2));
6641   effect(TEMP dst, TEMP tmp);
6642   format %{ "subss  $dst,$src2\n\t"
6643             "pshufd  $tmp,$src2,0x01\n\t"
6644             "subss  $dst,$dst,$tmp\t! sub reduction2F" %}
6645   ins_encode %{
6646     __ subss($dst$$XMMRegister, $src2$$XMMRegister);
6647     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x01);
6648     __ subss($dst$$XMMRegister, $tmp$$XMMRegister);
6649   %}
6650   ins_pipe( pipe_slow );
6651 %}
6652 
6653 instruct rvsub2F_reduction_reg(regF dst, vecD src2, regF tmp) %{
6654   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
6655   match(Set dst (SubReductionVFP dst src2));
6656   effect(TEMP dst, TEMP tmp);
6657   format %{ "vsubss  $dst,$dst,$src2\n\t"
6658             "pshufd  $tmp,$src2,0x01\n\t"
6659             "vsubss  $dst,$dst,$tmp\t! sub reduction2F" %}
6660   ins_encode %{
6661     __ vsubss($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister);
6662     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x01);
6663     __ vsubss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6664   %}
6665   ins_pipe( pipe_slow );
6666 %}
6667 
6668 instruct rssub4F_reduction_reg(regF dst, vecX src2, regF tmp) %{
6669   predicate(UseSSE >= 1 && UseAVX == 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
6670   match(Set dst (SubReductionVFP dst src2));
6671   effect(TEMP dst, TEMP tmp);
6672   format %{ "subss   $dst,$src2\n\t"
6673             "pshufd  $tmp,$src2,0x01\n\t"
6674             "subss   $dst,$tmp\n\t"
6675             "pshufd  $tmp,$src2,0x02\n\t"
6676             "subss   $dst,$tmp\n\t"
6677             "pshufd  $tmp,$src2,0x03\n\t"
6678             "subss   $dst,$tmp\t! sub reduction4F" %}
6679   ins_encode %{
6680     __ subss($dst$$XMMRegister, $src2$$XMMRegister);
6681     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x01);
6682     __ subss($dst$$XMMRegister, $tmp$$XMMRegister);
6683     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x02);
6684     __ subss($dst$$XMMRegister, $tmp$$XMMRegister);
6685     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x03);
6686     __ subss($dst$$XMMRegister, $tmp$$XMMRegister);
6687   %}
6688   ins_pipe( pipe_slow );
6689 %}
6690 
6691 instruct rvsub4F_reduction_reg(regF dst, vecX src2, regF tmp) %{
6692   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
6693   match(Set dst (SubReductionVFP dst src2));
6694   effect(TEMP tmp, TEMP dst);
6695   format %{ "vsubss  $dst,dst,$src2\n\t"
6696             "pshufd  $tmp,$src2,0x01\n\t"
6697             "vsubss  $dst,$dst,$tmp\n\t"
6698             "pshufd  $tmp,$src2,0x02\n\t"
6699             "vsubss  $dst,$dst,$tmp\n\t"
6700             "pshufd  $tmp,$src2,0x03\n\t"
6701             "vsubss  $dst,$dst,$tmp\t! sub reduction4F" %}
6702   ins_encode %{
6703     __ vsubss($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister);
6704     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x01);
6705     __ vsubss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6706     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x02);
6707     __ vsubss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6708     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x03);
6709     __ vsubss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6710   %}
6711   ins_pipe( pipe_slow );
6712 %}
6713 
6714 instruct rsub8F_reduction_reg(regF dst, vecY src2, regF tmp, regF tmp2) %{
6715   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
6716   match(Set dst (SubReductionVFP dst src2));
6717   effect(TEMP tmp, TEMP dst, TEMP tmp2);
6718   format %{ "vsubss  $dst,$dst,$src2\n\t"
6719             "pshufd  $tmp,$src2,0x01\n\t"
6720             "vsubss  $dst,$dst,$tmp\n\t"
6721             "pshufd  $tmp,$src2,0x02\n\t"
6722             "vsubss  $dst,$dst,$tmp\n\t"
6723             "pshufd  $tmp,$src2,0x03\n\t"
6724             "vsubss  $dst,$dst,$tmp\n\t"
6725             "vextractf128_high  $tmp2,$src2\n\t"
6726             "vsubss  $dst,$dst,$tmp2\n\t"
6727             "pshufd  $tmp,$tmp2,0x01\n\t"
6728             "vsubss  $dst,$dst,$tmp\n\t"
6729             "pshufd  $tmp,$tmp2,0x02\n\t"
6730             "vsubss  $dst,$dst,$tmp\n\t"
6731             "pshufd  $tmp,$tmp2,0x03\n\t"
6732             "vsubss  $dst,$dst,$tmp\t! sub reduction8F" %}
6733   ins_encode %{
6734     __ vsubss($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister);
6735     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x01);
6736     __ vsubss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6737     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x02);
6738     __ vsubss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6739     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x03);
6740     __ vsubss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6741     __ vextractf128_high($tmp2$$XMMRegister, $src2$$XMMRegister);
6742     __ vsubss($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister);
6743     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x01);
6744     __ vsubss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6745     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x02);
6746     __ vsubss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6747     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x03);
6748     __ vsubss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6749   %}
6750   ins_pipe( pipe_slow );
6751 %}
6752 
6753 instruct rsub16F_reduction_reg(regF dst, vecZ src2, regF tmp, regF tmp2) %{
6754   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
6755   match(Set dst (SubReductionVFP dst src2));
6756   effect(TEMP tmp, TEMP dst, TEMP tmp2);
6757   format %{ "vsubss  $dst,$dst,$src2\n\t"
6758             "pshufd  $tmp,$src2,0x01\n\t"
6759             "vsubss  $dst,$dst,$tmp\n\t"
6760             "pshufd  $tmp,$src2,0x02\n\t"
6761             "vsubss  $dst,$dst,$tmp\n\t"
6762             "pshufd  $tmp,$src2,0x03\n\t"
6763             "vsubss  $dst,$dst,$tmp\n\t"
6764             "vextractf32x4  $tmp2,$src2,0x1\n\t"
6765             "vsubss  $dst,$dst,$tmp2\n\t"
6766             "pshufd  $tmp,$tmp2,0x01\n\t"
6767             "vsubss  $dst,$dst,$tmp\n\t"
6768             "pshufd  $tmp,$tmp2,0x02\n\t"
6769             "vsubss  $dst,$dst,$tmp\n\t"
6770             "pshufd  $tmp,$tmp2,0x03\n\t"
6771             "vsubss  $dst,$dst,$tmp\n\t"
6772             "vextractf32x4  $tmp2,$src2,0x2\n\t"
6773             "vsubss  $dst,$dst,$tmp2\n\t"
6774             "pshufd  $tmp,$tmp2,0x01\n\t"
6775             "vsubss  $dst,$dst,$tmp\n\t"
6776             "pshufd  $tmp,$tmp2,0x02\n\t"
6777             "vsubss  $dst,$dst,$tmp\n\t"
6778             "pshufd  $tmp,$tmp2,0x03\n\t"
6779             "vsubss  $dst,$dst,$tmp\n\t"
6780             "vextractf32x4  $tmp2,$src2,0x3\n\t"
6781             "vsubss  $dst,$dst,$tmp2\n\t"
6782             "pshufd  $tmp,$tmp2,0x01\n\t"
6783             "vsubss  $dst,$dst,$tmp\n\t"
6784             "pshufd  $tmp,$tmp2,0x02\n\t"
6785             "vsubss  $dst,$dst,$tmp\n\t"
6786             "pshufd  $tmp,$tmp2,0x03\n\t"
6787             "vsubss  $dst,$dst,$tmp\t! sub reduction16F" %}
6788   ins_encode %{
6789     __ vsubss($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister);
6790     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x01);
6791     __ vsubss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6792     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x02);
6793     __ vsubss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6794     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x03);
6795     __ vsubss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6796     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x1);
6797     __ vsubss($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister);
6798     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x01);
6799     __ vsubss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6800     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x02);
6801     __ vsubss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6802     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x03);
6803     __ vsubss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6804     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x2);
6805     __ vsubss($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister);
6806     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x01);
6807     __ vsubss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6808     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x02);
6809     __ vsubss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6810     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x03);
6811     __ vsubss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6812     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x3);
6813     __ vsubss($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister);
6814     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x01);
6815     __ vsubss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6816     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x02);
6817     __ vsubss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6818     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x03);
6819     __ vsubss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6820   %}
6821   ins_pipe( pipe_slow );
6822 %}
6823 
6824 instruct rssub2D_reduction_reg(regD dst, vecX src2, regD tmp) %{
6825   predicate(UseSSE >= 1 && UseAVX == 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
6826   match(Set dst (SubReductionVFP dst src2));
6827   effect(TEMP tmp, TEMP dst);
6828   format %{ "subsd   $dst,$src2\n\t"
6829             "pshufd  $tmp,$src2,0xE\n\t"
6830             "subsd   $dst,$tmp\t! sub reduction2D" %}
6831   ins_encode %{
6832     __ subsd($dst$$XMMRegister, $src2$$XMMRegister);
6833     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
6834     __ subsd($dst$$XMMRegister, $tmp$$XMMRegister);
6835   %}
6836   ins_pipe( pipe_slow );
6837 %}
6838 
6839 instruct rvsub2D_reduction_reg(regD dst, vecX src2, regD tmp) %{
6840   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
6841   match(Set dst (SubReductionVFP dst src2));
6842   effect(TEMP tmp, TEMP dst);
6843   format %{ "vsubsd  $dst,$dst,$src2\n\t"
6844             "pshufd  $tmp,$src2,0xE\n\t"
6845             "vsubsd  $dst,$dst,$tmp\t! sub reduction2D" %}
6846   ins_encode %{
6847     __ vsubsd($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister);
6848     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
6849     __ vsubsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6850   %}
6851   ins_pipe( pipe_slow );
6852 %}
6853 
6854 instruct rvsub4D_reduction_reg(regD dst, vecY src2, regD tmp, regD tmp2) %{
6855   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
6856   match(Set dst (SubReductionVFP dst src2));
6857   effect(TEMP tmp, TEMP dst, TEMP tmp2);
6858   format %{ "vsubsd  $dst,$dst,$src2\n\t"
6859             "pshufd  $tmp,$src2,0xE\n\t"
6860             "vsubsd  $dst,$dst,$tmp\n\t"
6861             "vextractf32x4  $tmp2,$src2,0x1\n\t"
6862             "vsubsd  $dst,$dst,$tmp2\n\t"
6863             "pshufd  $tmp,$tmp2,0xE\n\t"
6864             "vsubsd  $dst,$dst,$tmp\t! sub reduction4D" %}
6865   ins_encode %{
6866     __ vsubsd($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister);
6867     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
6868     __ vsubsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6869     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x1);
6870     __ vsubsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister);
6871     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE);
6872     __ vsubsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6873   %}
6874   ins_pipe( pipe_slow );
6875 %}
6876 
6877 instruct rvsub8D_reduction_reg(regD dst, vecZ src2, regD tmp, regD tmp2) %{
6878   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
6879   match(Set dst (SubReductionVFP dst src2));
6880   effect(TEMP tmp, TEMP dst, TEMP tmp2);
6881   format %{ "vsubsd  $dst,$dst,$src2\n\t"
6882             "pshufd  $tmp,$src2,0xE\n\t"
6883             "vsubsd  $dst,$dst,$tmp\n\t"
6884             "vextractf32x4  $tmp2,$src2,0x1\n\t"
6885             "vsubsd  $dst,$dst,$tmp2\n\t"
6886             "pshufd  $tmp,$tmp2,0xE\n\t"
6887             "vsubsd  $dst,$dst,$tmp\n\t"
6888             "vextractf32x4  $tmp2,$src2,0x2\n\t"
6889             "vsubsd  $dst,$dst,$tmp2\n\t"
6890             "pshufd  $tmp,$tmp2,0xE\n\t"
6891             "vsubsd  $dst,$dst,$tmp\n\t"
6892             "vextractf32x4  $tmp2,$src2,0x3\n\t"
6893             "vsubsd  $dst,$dst,$tmp2\n\t"
6894             "pshufd  $tmp,$tmp2,0xE\n\t"
6895             "vsubsd  $dst,$dst,$tmp\t! sub reduction8D" %}
6896   ins_encode %{
6897     __ vsubsd($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister);
6898     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
6899     __ vsubsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6900     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x1);
6901     __ vsubsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister);
6902     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE);
6903     __ vsubsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6904     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x2);
6905     __ vsubsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister);
6906     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE);
6907     __ vsubsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6908     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x3);
6909     __ vsubsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister);
6910     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE);
6911     __ vsubsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
6912   %}
6913   ins_pipe( pipe_slow );
6914 %}
6915 
6916 instruct rsmul8B_reduction_reg(rRegI dst, rRegI src1, vecD src2, regF tmp, regF tmp1, rRegI tmp2, rRegI tmp3) %{
6917   predicate(UseSSE > 3 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
6918   match(Set dst (MulReductionVI src1 src2));
6919   effect(TEMP dst, TEMP tmp, TEMP tmp1, TEMP tmp2, TEMP tmp3);
6920   format %{ "pmovsxbw $tmp,$src2\n\t"
6921             "pshufd   $tmp1,$tmp,0xE\n\t"
6922             "pmullw   $tmp,$tmp1\n\t"
6923             "pshufd   $tmp1,$tmp,0x1\n\t"
6924             "pmullw   $tmp,$tmp1\n\t"
6925             "pextrw   $tmp2,$tmp, 0x1\n\t"
6926             "pextrw   $tmp3,$tmp, 0x0\n\t"
6927             "imul     $tmp2,$tmp3 \n\t"
6928             "movsbl   $dst,$src1\n\t"
6929             "imull    $dst,$tmp2\n\t"
6930             "movsbl   $dst,$dst\t! mul reduction8B" %}
6931   ins_encode %{
6932     __ pmovsxbw($tmp$$XMMRegister, $src2$$XMMRegister);
6933     __ pshufd($tmp1$$XMMRegister, $tmp$$XMMRegister, 0xE);
6934     __ pmullw($tmp$$XMMRegister, $tmp1$$XMMRegister);
6935     __ pshufd($tmp1$$XMMRegister, $tmp$$XMMRegister, 0x1);
6936     __ pmullw($tmp$$XMMRegister, $tmp1$$XMMRegister);
6937     __ pextrw($tmp2$$Register, $tmp$$XMMRegister, 0x1);
6938     __ pextrw($tmp3$$Register, $tmp$$XMMRegister, 0x0);
6939     __ imull($tmp2$$Register, $tmp3$$Register);
6940     __ movsbl($dst$$Register, $src1$$Register);
6941     __ imull($dst$$Register, $tmp2$$Register);
6942     __ movsbl($dst$$Register, $dst$$Register);
6943   %}
6944   ins_pipe( pipe_slow );
6945 %}
6946 
6947 instruct rsmul16B_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp1, rRegI tmp2, rRegI tmp3) %{
6948   predicate(UseSSE > 3 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
6949   match(Set dst (MulReductionVI src1 src2));
6950   effect(TEMP dst, TEMP tmp, TEMP tmp1, TEMP tmp2, TEMP tmp3);
6951   format %{ "pmovsxbw $tmp,$src2\n\t"
6952             "pshufd   $tmp1,$src2,0xEE\n\t"
6953             "pmovsxbw $tmp1,$tmp1\n\t"
6954             "pmullw   $tmp,$tmp1\n\t"
6955             "pshufd   $tmp1,$tmp,0xE\n\t"
6956             "pmullw   $tmp,$tmp1\n\t"
6957             "pshufd   $tmp1,$tmp,0x1\n\t"
6958             "pmullw   $tmp,$tmp1\n\t"
6959             "pextrw   $tmp2,$tmp, 0x1\n\t"
6960             "pextrw   $tmp3,$tmp, 0x0\n\t"
6961             "imull    $tmp2,$tmp3 \n\t"
6962             "movsbl   $dst,$src1\n\t"
6963             "imull    $dst,$tmp2\n\t"
6964             "movsbl   $dst,$dst\t! mul reduction16B" %}
6965   ins_encode %{
6966     int vector_len = 0;
6967     __ pmovsxbw($tmp$$XMMRegister, $src2$$XMMRegister);
6968     __ pshufd($tmp1$$XMMRegister, $src2$$XMMRegister, 0xEE);
6969     __ pmovsxbw($tmp1$$XMMRegister, $tmp1$$XMMRegister);
6970     __ pmullw($tmp$$XMMRegister, $tmp1$$XMMRegister);
6971     __ pshufd($tmp1$$XMMRegister, $tmp$$XMMRegister, 0xE);
6972     __ pmullw($tmp$$XMMRegister, $tmp1$$XMMRegister);
6973     __ pshufd($tmp1$$XMMRegister, $tmp$$XMMRegister, 0x1);
6974     __ pmullw($tmp$$XMMRegister, $tmp1$$XMMRegister);
6975     __ pextrw($tmp2$$Register, $tmp$$XMMRegister, 0x1);
6976     __ pextrw($tmp3$$Register, $tmp$$XMMRegister, 0x0);
6977     __ imull($tmp2$$Register, $tmp3$$Register);
6978     __ movsbl($dst$$Register, $src1$$Register);
6979     __ imull($dst$$Register, $tmp2$$Register);
6980     __ movsbl($dst$$Register, $dst$$Register);
6981   %}
6982   ins_pipe( pipe_slow );
6983 %}
6984 
6985 instruct rvmul32B_reduction_reg(rRegI dst, rRegI src1, vecY src2, regF tmp, regF tmp1, rRegI tmp2, rRegI tmp3) %{
6986   predicate(UseAVX > 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
6987   match(Set dst (MulReductionVI src1 src2));
6988   effect(TEMP dst, TEMP tmp, TEMP tmp1, TEMP tmp2, TEMP tmp3);
6989   format %{ "vextracti128_high  $tmp,$src2\n\t"
6990             "pmovsxbw $tmp,$tmp\n\t"
6991             "pmovsxbw $tmp1,$src2\n\t"
6992             "vpmullw  $tmp,$tmp,$tmp1\n\t"
6993             "vextracti128_high  $tmp1,$tmp\n\t"
6994             "vpmullw  $tmp,$tmp,$tmp1\n\t"
6995             "pshufd   $tmp1,$tmp,0xE\n\t"
6996             "vpmullw  $tmp,$tmp,$tmp1\n\t"
6997             "pshufd   $tmp1,$tmp,0x1\n\t"
6998             "vpmullw  $tmp,$tmp,$tmp1\n\t"
6999             "pextrw   $tmp2,$tmp, 0x1\n\t"
7000             "pextrw   $tmp3,$tmp, 0x0\n\t"
7001             "imull    $tmp2,$tmp3 \n\t"
7002             "movsbl   $dst,$src1\n\t"
7003             "imull    $dst,$tmp2\n\t"
7004             "movsbl   $dst,$dst\t! mul reduction32B" %}
7005   ins_encode %{
7006     int vector_len = 1;
7007     __ vextracti128_high($tmp$$XMMRegister, $src2$$XMMRegister);
7008     __ vpmovsxbw($tmp$$XMMRegister, $tmp$$XMMRegister, vector_len);
7009     __ vpmovsxbw($tmp1$$XMMRegister, $src2$$XMMRegister, vector_len);
7010     __ vpmullw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp1$$XMMRegister, vector_len);
7011     __ vextracti128_high($tmp1$$XMMRegister, $tmp$$XMMRegister);
7012     __ vpmullw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp1$$XMMRegister, 0);
7013     __ pshufd($tmp1$$XMMRegister, $tmp$$XMMRegister, 0xE);
7014     __ vpmullw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp1$$XMMRegister, 0);
7015     __ pshufd($tmp1$$XMMRegister, $tmp$$XMMRegister, 0x1);
7016     __ vpmullw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp1$$XMMRegister, 0);
7017     __ pextrw($tmp2$$Register, $tmp$$XMMRegister, 0x1);
7018     __ pextrw($tmp3$$Register, $tmp$$XMMRegister, 0x0);
7019     __ imull($tmp2$$Register, $tmp3$$Register);
7020     __ movsbl($dst$$Register, $src1$$Register);
7021     __ imull($dst$$Register, $tmp2$$Register);
7022     __ movsbl($dst$$Register, $dst$$Register);
7023   %}
7024   ins_pipe( pipe_slow );
7025 %}
7026 
7027 instruct rvmul64B_reduction_reg(rRegI dst, rRegI src1, vecZ src2, regF tmp, regF tmp1, rRegI tmp2, rRegI tmp3) %{
7028   predicate(UseAVX > 2 && VM_Version::supports_avx512bw() && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
7029   match(Set dst (MulReductionVI src1 src2));
7030   effect(TEMP dst, TEMP tmp, TEMP tmp1, TEMP tmp2, TEMP tmp3);
7031   format %{ "vextracti64x4_high  $tmp,$src2\n\t"
7032             "vpmovsxbw $tmp,$tmp\n\t"
7033             "vpmovsxbw $tmp1,$src2\n\t"
7034             "vpmullw  $tmp,$tmp,$tmp1\n\t"
7035             "vextracti64x4_high  $tmp1,$tmp\n\t"
7036             "vpmullw  $tmp,$tmp,$tmp1\n\t"
7037             "vextracti128_high  $tmp1,$tmp\n\t"
7038             "vpmullw  $tmp,$tmp,$tmp1\n\t"
7039             "pshufd   $tmp1,$tmp,0xE\n\t"
7040             "vpmullw  $tmp,$tmp,$tmp1\n\t"
7041             "pshufd   $tmp1,$tmp,0x1\n\t"
7042             "vpmullw  $tmp,$tmp,$tmp1\n\t"
7043             "pextrw   $tmp2,$tmp, 0x1\n\t"
7044             "pextrw   $tmp3,$tmp, 0x0\n\t"
7045             "imull    $tmp2,$tmp3 \n\t"
7046             "movsbl   $dst,$src1\n\t"
7047             "imull    $dst,$tmp2\n\t"
7048             "movsbl   $dst,$dst\t! mul reduction64B" %}
7049   ins_encode %{
7050     int vector_len = 2;
7051     __ vextracti64x4_high($tmp$$XMMRegister, $src2$$XMMRegister);
7052     __ vpmovsxbw($tmp$$XMMRegister, $tmp$$XMMRegister, vector_len);
7053     __ vpmovsxbw($tmp1$$XMMRegister, $src2$$XMMRegister, vector_len);
7054     __ vpmullw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp1$$XMMRegister, vector_len);
7055     __ vextracti64x4_high($tmp1$$XMMRegister, $tmp$$XMMRegister);
7056     __ vpmullw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp1$$XMMRegister, 1);
7057     __ vextracti128_high($tmp1$$XMMRegister, $tmp$$XMMRegister);
7058     __ vpmullw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp1$$XMMRegister, 0);
7059     __ pshufd($tmp1$$XMMRegister, $tmp$$XMMRegister, 0xE);
7060     __ vpmullw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp1$$XMMRegister, 0);
7061     __ pshufd($tmp1$$XMMRegister, $tmp$$XMMRegister, 0x1);
7062     __ vpmullw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp1$$XMMRegister, 0);
7063     __ pextrw($tmp2$$Register, $tmp$$XMMRegister, 0x1);
7064     __ pextrw($tmp3$$Register, $tmp$$XMMRegister, 0x0);
7065     __ imull($tmp2$$Register, $tmp3$$Register);
7066     __ movsbl($dst$$Register, $src1$$Register);
7067     __ imull($dst$$Register, $tmp2$$Register);
7068     __ movsbl($dst$$Register, $dst$$Register);
7069   %}
7070   ins_pipe( pipe_slow );
7071 %}
7072 
7073 instruct rsmul4S_reduction_reg(rRegI dst, rRegI src1, vecD src2, regF tmp, rRegI tmp2, rRegI tmp3) %{
7074   predicate(UseSSE > 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
7075   match(Set dst (MulReductionVI src1 src2));
7076   effect(TEMP dst, TEMP tmp, TEMP tmp2, TEMP tmp3);
7077   format %{ "pshufd  $tmp,$src2,0x1\n\t"
7078             "pmullw  $tmp,$src2\n\t"
7079             "pextrw  $tmp2,$tmp, 0x1\n\t"
7080             "pextrw  $tmp3,$tmp, 0x0\n\t"
7081             "imull    $tmp2,$tmp3 \n\t"
7082             "movswl   $dst,$src1\n\t"
7083             "imull    $dst,$tmp2\n\t"
7084             "movswl   $dst,$dst\t! mul reduction4S" %}
7085   ins_encode %{
7086     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x1);
7087     __ pmullw($tmp$$XMMRegister, $src2$$XMMRegister);
7088     __ pextrw($tmp2$$Register, $tmp$$XMMRegister, 0x1);
7089     __ pextrw($tmp3$$Register, $tmp$$XMMRegister, 0x0);
7090     __ imull($tmp2$$Register, $tmp3$$Register);
7091     __ movswl($dst$$Register, $src1$$Register);
7092     __ imull($dst$$Register, $tmp2$$Register);
7093     __ movswl($dst$$Register, $dst$$Register);
7094   %}
7095   ins_pipe( pipe_slow );
7096 %}
7097 
7098 instruct rsmul8S_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp1, rRegI tmp2, rRegI tmp3) %{
7099   predicate(UseSSE > 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
7100   match(Set dst (MulReductionVI src1 src2));
7101   effect(TEMP dst, TEMP tmp, TEMP tmp1, TEMP tmp2, TEMP tmp3);
7102   format %{ "pshufd  $tmp,$src2,0xE\n\t"
7103             "pmullw  $tmp,$src2\n\t"
7104             "pshufd  $tmp1,$tmp,0x1\n\t"
7105             "pmullw  $tmp,$tmp1\n\t"
7106             "pextrw  $tmp2,$tmp, 0x1\n\t"
7107             "pextrw  $tmp3,$tmp, 0x0\n\t"
7108             "imul    $tmp2,$tmp3 \n\t"
7109             "movswl   $dst,$src1\n\t"
7110             "imull    $dst,$tmp2\n\t"
7111             "movswl   $dst,$dst\t! mul reduction8S" %}
7112   ins_encode %{
7113     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
7114     __ pmullw($tmp$$XMMRegister, $src2$$XMMRegister);
7115     __ pshufd($tmp1$$XMMRegister, $tmp$$XMMRegister, 0x1);
7116     __ pmullw($tmp$$XMMRegister, $tmp1$$XMMRegister);
7117     __ pextrw($tmp2$$Register, $tmp$$XMMRegister, 0x1);
7118     __ pextrw($tmp3$$Register, $tmp$$XMMRegister, 0x0);
7119     __ imull($tmp2$$Register, $tmp3$$Register);
7120     __ movswl($dst$$Register, $src1$$Register);
7121     __ imull($dst$$Register, $tmp2$$Register);
7122     __ movswl($dst$$Register, $dst$$Register);
7123   %}
7124   ins_pipe( pipe_slow );
7125 %}
7126 
7127 instruct rvmul16S_reduction_reg(rRegI dst, rRegI src1, vecY src2, regF tmp, regF tmp1, rRegI tmp2, rRegI tmp3) %{
7128   predicate(UseAVX > 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
7129   match(Set dst (MulReductionVI src1 src2));
7130   effect(TEMP dst, TEMP tmp, TEMP tmp1, TEMP tmp2, TEMP tmp3);
7131   format %{ "vextracti128_high  $tmp,$src2\n\t"
7132             "vpmullw  $tmp,$tmp,$src2\n\t"
7133             "pshufd  $tmp1,$tmp,0xE\n\t"
7134             "pmullw  $tmp,$tmp1\n\t"
7135             "pshufd  $tmp1,$tmp,0x1\n\t"
7136             "pmullw  $tmp,$tmp1\n\t"
7137             "pextrw  $tmp2,$tmp, 0x1\n\t"
7138             "pextrw  $tmp3,$tmp, 0x0\n\t"
7139             "imul    $tmp2,$tmp3 \n\t"
7140             "movswl   $dst,$src1\n\t"
7141             "imull    $dst,$tmp2\n\t"
7142             "movswl   $dst,$dst\t! mul reduction16S" %}
7143   ins_encode %{
7144     __ vextracti128_high($tmp$$XMMRegister, $src2$$XMMRegister);
7145     __ vpmullw($tmp$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, 1);
7146     __ pshufd($tmp1$$XMMRegister, $tmp$$XMMRegister, 0xE);
7147     __ vpmullw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp1$$XMMRegister, 0);
7148     __ pshufd($tmp1$$XMMRegister, $tmp$$XMMRegister, 0x1);
7149     __ vpmullw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp1$$XMMRegister, 0);
7150     __ pextrw($tmp2$$Register, $tmp$$XMMRegister, 0x1);
7151     __ pextrw($tmp3$$Register, $tmp$$XMMRegister, 0x0);
7152     __ imull($tmp2$$Register, $tmp3$$Register);
7153     __ movswl($dst$$Register, $src1$$Register);
7154     __ imull($dst$$Register, $tmp2$$Register);
7155     __ movswl($dst$$Register, $dst$$Register);
7156   %}
7157   ins_pipe( pipe_slow );
7158 %}
7159 
7160 instruct rvmul32S_reduction_reg(rRegI dst, rRegI src1, vecZ src2, regF tmp, regF tmp1, rRegI tmp2, rRegI tmp3) %{
7161   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
7162   match(Set dst (MulReductionVI src1 src2));
7163   effect(TEMP dst, TEMP tmp, TEMP tmp1, TEMP tmp2, TEMP tmp3);
7164   format %{ "vextracti64x4_high  $tmp1,$src2\n\t"
7165             "vpmullw  $tmp1,$tmp1,$src2\n\t"
7166             "vextracti128_high  $tmp,$tmp1\n\t"
7167             "vpmullw  $tmp,$tmp,$tmp1\n\t"
7168             "pshufd  $tmp1,$tmp,0xE\n\t"
7169             "pmullw  $tmp,$tmp1\n\t"
7170             "pshufd  $tmp1,$tmp,0x1\n\t"
7171             "pmullw  $tmp,$tmp1\n\t"
7172             "pextrw  $tmp2,$tmp, 0x1\n\t"
7173             "pextrw  $tmp3,$tmp, 0x0\n\t"
7174             "imul    $tmp2,$tmp3 \n\t"
7175             "movswl   $dst,$src1\n\t"
7176             "imull    $dst,$tmp2\n\t"
7177             "movswl   $dst,$dst\t! mul reduction32S" %}
7178   ins_encode %{
7179     int vector_len = 0;
7180     __ vextracti64x4_high($tmp1$$XMMRegister, $src2$$XMMRegister);
7181     __ vpmullw($tmp1$$XMMRegister, $tmp1$$XMMRegister, $src2$$XMMRegister, 1);
7182     __ vextracti128_high($tmp$$XMMRegister, $tmp1$$XMMRegister);
7183     __ vpmullw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp1$$XMMRegister, vector_len);
7184     __ pshufd($tmp1$$XMMRegister, $tmp$$XMMRegister, 0xE);
7185     __ vpmullw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp1$$XMMRegister, vector_len);
7186     __ pshufd($tmp1$$XMMRegister, $tmp$$XMMRegister, 0x1);
7187     __ vpmullw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp1$$XMMRegister, vector_len);
7188     __ pextrw($tmp2$$Register, $tmp$$XMMRegister, 0x1);
7189     __ pextrw($tmp3$$Register, $tmp$$XMMRegister, 0x0);
7190     __ imull($tmp2$$Register, $tmp3$$Register);
7191     __ movswl($dst$$Register, $src1$$Register);
7192     __ imull($dst$$Register, $tmp2$$Register);
7193     __ movswl($dst$$Register, $dst$$Register);
7194   %}
7195   ins_pipe( pipe_slow );
7196 %}
7197 
7198 
7199 instruct rsmul2I_reduction_reg(rRegI dst, rRegI src1, vecD src2, regF tmp, regF tmp2) %{
7200   predicate(UseSSE > 3 && UseAVX == 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
7201   match(Set dst (MulReductionVI src1 src2));
7202   effect(TEMP tmp, TEMP tmp2);
7203   format %{ "pshufd  $tmp2,$src2,0x1\n\t"
7204             "pmulld  $tmp2,$src2\n\t"
7205             "movd    $tmp,$src1\n\t"
7206             "pmulld  $tmp2,$tmp\n\t"
7207             "movd    $dst,$tmp2\t! mul reduction2I" %}
7208   ins_encode %{
7209     __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0x1);
7210     __ pmulld($tmp2$$XMMRegister, $src2$$XMMRegister);
7211     __ movdl($tmp$$XMMRegister, $src1$$Register);
7212     __ pmulld($tmp2$$XMMRegister, $tmp$$XMMRegister);
7213     __ movdl($dst$$Register, $tmp2$$XMMRegister);
7214   %}
7215   ins_pipe( pipe_slow );
7216 %}
7217 
7218 instruct rvmul2I_reduction_reg(rRegI dst, rRegI src1, vecD src2, regF tmp, regF tmp2) %{
7219   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
7220   match(Set dst (MulReductionVI src1 src2));
7221   effect(TEMP tmp, TEMP tmp2);
7222   format %{ "pshufd   $tmp2,$src2,0x1\n\t"
7223             "vpmulld  $tmp,$src2,$tmp2\n\t"
7224             "movd     $tmp2,$src1\n\t"
7225             "vpmulld  $tmp2,$tmp,$tmp2\n\t"
7226             "movd     $dst,$tmp2\t! mul reduction2I" %}
7227   ins_encode %{
7228     int vector_len = 0;
7229     __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0x1);
7230     __ vpmulld($tmp$$XMMRegister, $src2$$XMMRegister, $tmp2$$XMMRegister, vector_len);
7231     __ movdl($tmp2$$XMMRegister, $src1$$Register);
7232     __ vpmulld($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
7233     __ movdl($dst$$Register, $tmp2$$XMMRegister);
7234   %}
7235   ins_pipe( pipe_slow );
7236 %}
7237 
7238 instruct rsmul4I_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp2) %{
7239   predicate(UseSSE > 3 && UseAVX == 0 && UseAVX == 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
7240   match(Set dst (MulReductionVI src1 src2));
7241   effect(TEMP tmp, TEMP tmp2);
7242   format %{ "pshufd  $tmp2,$src2,0xE\n\t"
7243             "pmulld  $tmp2,$src2\n\t"
7244             "pshufd  $tmp,$tmp2,0x1\n\t"
7245             "pmulld  $tmp2,$tmp\n\t"
7246             "movd    $tmp,$src1\n\t"
7247             "pmulld  $tmp2,$tmp\n\t"
7248             "movd    $dst,$tmp2\t! mul reduction4I" %}
7249   ins_encode %{
7250     __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0xE);
7251     __ pmulld($tmp2$$XMMRegister, $src2$$XMMRegister);
7252     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x1);
7253     __ pmulld($tmp2$$XMMRegister, $tmp$$XMMRegister);
7254     __ movdl($tmp$$XMMRegister, $src1$$Register);
7255     __ pmulld($tmp2$$XMMRegister, $tmp$$XMMRegister);
7256     __ movdl($dst$$Register, $tmp2$$XMMRegister);
7257   %}
7258   ins_pipe( pipe_slow );
7259 %}
7260 
7261 instruct rvmul4I_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp2) %{
7262   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
7263   match(Set dst (MulReductionVI src1 src2));
7264   effect(TEMP tmp, TEMP tmp2);
7265   format %{ "pshufd   $tmp2,$src2,0xE\n\t"
7266             "vpmulld  $tmp,$src2,$tmp2\n\t"
7267             "pshufd   $tmp2,$tmp,0x1\n\t"
7268             "vpmulld  $tmp,$tmp,$tmp2\n\t"
7269             "movd     $tmp2,$src1\n\t"
7270             "vpmulld  $tmp2,$tmp,$tmp2\n\t"
7271             "movd     $dst,$tmp2\t! mul reduction4I" %}
7272   ins_encode %{
7273     int vector_len = 0;
7274     __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0xE);
7275     __ vpmulld($tmp$$XMMRegister, $src2$$XMMRegister, $tmp2$$XMMRegister, vector_len);
7276     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1);
7277     __ vpmulld($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
7278     __ movdl($tmp2$$XMMRegister, $src1$$Register);
7279     __ vpmulld($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
7280     __ movdl($dst$$Register, $tmp2$$XMMRegister);
7281   %}
7282   ins_pipe( pipe_slow );
7283 %}
7284 
7285 instruct rvmul8I_reduction_reg(rRegI dst, rRegI src1, vecY src2, regF tmp, regF tmp2) %{
7286   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
7287   match(Set dst (MulReductionVI src1 src2));
7288   effect(TEMP tmp, TEMP tmp2);
7289   format %{ "vextracti128_high  $tmp,$src2\n\t"
7290             "vpmulld  $tmp,$tmp,$src2\n\t"
7291             "pshufd   $tmp2,$tmp,0xE\n\t"
7292             "vpmulld  $tmp,$tmp,$tmp2\n\t"
7293             "pshufd   $tmp2,$tmp,0x1\n\t"
7294             "vpmulld  $tmp,$tmp,$tmp2\n\t"
7295             "movd     $tmp2,$src1\n\t"
7296             "vpmulld  $tmp2,$tmp,$tmp2\n\t"
7297             "movd     $dst,$tmp2\t! mul reduction8I" %}
7298   ins_encode %{
7299     int vector_len = 0;
7300     __ vextracti128_high($tmp$$XMMRegister, $src2$$XMMRegister);
7301     __ vpmulld($tmp$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, vector_len);
7302     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE);
7303     __ vpmulld($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
7304     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1);
7305     __ vpmulld($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
7306     __ movdl($tmp2$$XMMRegister, $src1$$Register);
7307     __ vpmulld($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
7308     __ movdl($dst$$Register, $tmp2$$XMMRegister);
7309   %}
7310   ins_pipe( pipe_slow );
7311 %}
7312 
7313 instruct rvmul16I_reduction_reg(rRegI dst, rRegI src1, vecZ src2, regF tmp, regF tmp2, regF tmp3) %{
7314   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
7315   match(Set dst (MulReductionVI src1 src2));
7316   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
7317   format %{ "vextracti64x4_high  $tmp3,$src2\n\t"
7318             "vpmulld  $tmp3,$tmp3,$src2\n\t"
7319             "vextracti128_high  $tmp,$tmp3\n\t"
7320             "vpmulld  $tmp,$tmp,$src2\n\t"
7321             "pshufd   $tmp2,$tmp,0xE\n\t"
7322             "vpmulld  $tmp,$tmp,$tmp2\n\t"
7323             "pshufd   $tmp2,$tmp,0x1\n\t"
7324             "vpmulld  $tmp,$tmp,$tmp2\n\t"
7325             "movd     $tmp2,$src1\n\t"
7326             "vpmulld  $tmp2,$tmp,$tmp2\n\t"
7327             "movd     $dst,$tmp2\t! mul reduction16I" %}
7328   ins_encode %{
7329     __ vextracti64x4_high($tmp3$$XMMRegister, $src2$$XMMRegister);
7330     __ vpmulld($tmp3$$XMMRegister, $tmp3$$XMMRegister, $src2$$XMMRegister, 1);
7331     __ vextracti128_high($tmp$$XMMRegister, $tmp3$$XMMRegister);
7332     __ vpmulld($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp3$$XMMRegister, 0);
7333     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE);
7334     __ vpmulld($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, 0);
7335     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1);
7336     __ vpmulld($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, 0);
7337     __ movdl($tmp2$$XMMRegister, $src1$$Register);
7338     __ vpmulld($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, 0);
7339     __ movdl($dst$$Register, $tmp2$$XMMRegister);
7340   %}
7341   ins_pipe( pipe_slow );
7342 %}
7343 
7344 #ifdef _LP64
7345 instruct rvmul2L_reduction_reg(rRegL dst, rRegL src1, vecX src2, regF tmp, regF tmp2) %{
7346   predicate(UseAVX > 2 && VM_Version::supports_avx512dq());
7347   match(Set dst (MulReductionVL src1 src2));
7348   effect(TEMP tmp, TEMP tmp2);
7349   format %{ "pshufd   $tmp2,$src2,0xE\n\t"
7350             "vpmullq  $tmp,$src2,$tmp2\n\t"
7351             "movdq    $tmp2,$src1\n\t"
7352             "vpmullq  $tmp2,$tmp,$tmp2\n\t"
7353             "movdq    $dst,$tmp2\t! mul reduction2L" %}
7354   ins_encode %{
7355     __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0xE);
7356     __ vpmullq($tmp$$XMMRegister, $src2$$XMMRegister, $tmp2$$XMMRegister, 0);
7357     __ movdq($tmp2$$XMMRegister, $src1$$Register);
7358     __ vpmullq($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, 0);
7359     __ movdq($dst$$Register, $tmp2$$XMMRegister);
7360   %}
7361   ins_pipe( pipe_slow );
7362 %}
7363 
7364 instruct rvmul4L_reduction_reg(rRegL dst, rRegL src1, vecY src2, regF tmp, regF tmp2) %{
7365   predicate(UseAVX > 2 && VM_Version::supports_avx512dq());
7366   match(Set dst (MulReductionVL src1 src2));
7367   effect(TEMP tmp, TEMP tmp2);
7368   format %{ "vextracti128_high  $tmp,$src2\n\t"
7369             "vpmullq  $tmp2,$tmp,$src2\n\t"
7370             "pshufd   $tmp,$tmp2,0xE\n\t"
7371             "vpmullq  $tmp2,$tmp2,$tmp\n\t"
7372             "movdq    $tmp,$src1\n\t"
7373             "vpmullq  $tmp2,$tmp2,$tmp\n\t"
7374             "movdq    $dst,$tmp2\t! mul reduction4L" %}
7375   ins_encode %{
7376     __ vextracti128_high($tmp$$XMMRegister, $src2$$XMMRegister);
7377     __ vpmullq($tmp2$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, 0);
7378     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE);
7379     __ vpmullq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, 0);
7380     __ movdq($tmp$$XMMRegister, $src1$$Register);
7381     __ vpmullq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, 0);
7382     __ movdq($dst$$Register, $tmp2$$XMMRegister);
7383   %}
7384   ins_pipe( pipe_slow );
7385 %}
7386 
7387 instruct rvmul8L_reduction_reg(rRegL dst, rRegL src1, vecZ src2, regF tmp, regF tmp2) %{
7388   predicate(UseAVX > 2 && VM_Version::supports_avx512dq());
7389   match(Set dst (MulReductionVL src1 src2));
7390   effect(TEMP tmp, TEMP tmp2);
7391   format %{ "vextracti64x4_high  $tmp2,$src2\n\t"
7392             "vpmullq  $tmp2,$tmp2,$src2\n\t"
7393             "vextracti128_high  $tmp,$tmp2\n\t"
7394             "vpmullq  $tmp2,$tmp2,$tmp\n\t"
7395             "pshufd   $tmp,$tmp2,0xE\n\t"
7396             "vpmullq  $tmp2,$tmp2,$tmp\n\t"
7397             "movdq    $tmp,$src1\n\t"
7398             "vpmullq  $tmp2,$tmp2,$tmp\n\t"
7399             "movdq    $dst,$tmp2\t! mul reduction8L" %}
7400   ins_encode %{
7401     __ vextracti64x4_high($tmp2$$XMMRegister, $src2$$XMMRegister);
7402     __ vpmullq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $src2$$XMMRegister, 1);
7403     __ vextracti128_high($tmp$$XMMRegister, $tmp2$$XMMRegister);
7404     __ vpmullq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, 0);
7405     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE);
7406     __ vpmullq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, 0);
7407     __ movdq($tmp$$XMMRegister, $src1$$Register);
7408     __ vpmullq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, 0);
7409     __ movdq($dst$$Register, $tmp2$$XMMRegister);
7410   %}
7411   ins_pipe( pipe_slow );
7412 %}
7413 #endif
7414 
7415 instruct rsmul2F_reduction(regF dst, vecD src2, regF tmp) %{
7416   predicate(UseSSE >= 1 && UseAVX == 0);
7417   match(Set dst (MulReductionVF dst src2));
7418   effect(TEMP dst, TEMP tmp);
7419   format %{ "mulss   $dst,$src2\n\t"
7420             "pshufd  $tmp,$src2,0x01\n\t"
7421             "mulss   $dst,$tmp\t! mul reduction2F" %}
7422   ins_encode %{
7423     __ mulss($dst$$XMMRegister, $src2$$XMMRegister);
7424     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x01);
7425     __ mulss($dst$$XMMRegister, $tmp$$XMMRegister);
7426   %}
7427   ins_pipe( pipe_slow );
7428 %}
7429 
7430 instruct rvmul2F_reduction_reg(regF dst, vecD src2, regF tmp) %{
7431   predicate(UseAVX > 0);
7432   match(Set dst (MulReductionVF dst src2));
7433   effect(TEMP tmp, TEMP dst);
7434   format %{ "vmulss  $dst,$dst,$src2\n\t"
7435             "pshufd  $tmp,$src2,0x01\n\t"
7436             "vmulss  $dst,$dst,$tmp\t! mul reduction2F" %}
7437   ins_encode %{
7438     __ vmulss($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister);
7439     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x01);
7440     __ vmulss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
7441   %}
7442   ins_pipe( pipe_slow );
7443 %}
7444 
7445 instruct rsmul4F_reduction_reg(regF dst, vecX src2, regF tmp) %{
7446   predicate(UseSSE >= 1 && UseAVX == 0);
7447   match(Set dst (MulReductionVF dst src2));
7448   effect(TEMP dst, TEMP tmp);
7449   format %{ "mulss   $dst,$src2\n\t"
7450             "pshufd  $tmp,$src2,0x01\n\t"
7451             "mulss   $dst,$tmp\n\t"
7452             "pshufd  $tmp,$src2,0x02\n\t"
7453             "mulss   $dst,$tmp\n\t"
7454             "pshufd  $tmp,$src2,0x03\n\t"
7455             "mulss   $dst,$tmp\t! mul reduction4F" %}
7456   ins_encode %{
7457     __ mulss($dst$$XMMRegister, $src2$$XMMRegister);
7458     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x01);
7459     __ mulss($dst$$XMMRegister, $tmp$$XMMRegister);
7460     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x02);
7461     __ mulss($dst$$XMMRegister, $tmp$$XMMRegister);
7462     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x03);
7463     __ mulss($dst$$XMMRegister, $tmp$$XMMRegister);
7464   %}
7465   ins_pipe( pipe_slow );
7466 %}
7467 
7468 instruct rvmul4F_reduction_reg(regF dst, vecX src2, regF tmp) %{
7469   predicate(UseAVX > 0);
7470   match(Set dst (MulReductionVF dst src2));
7471   effect(TEMP tmp, TEMP dst);
7472   format %{ "vmulss  $dst,$dst,$src2\n\t"
7473             "pshufd  $tmp,$src2,0x01\n\t"
7474             "vmulss  $dst,$dst,$tmp\n\t"
7475             "pshufd  $tmp,$src2,0x02\n\t"
7476             "vmulss  $dst,$dst,$tmp\n\t"
7477             "pshufd  $tmp,$src2,0x03\n\t"
7478             "vmulss  $dst,$dst,$tmp\t! mul reduction4F" %}
7479   ins_encode %{
7480     __ vmulss($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister);
7481     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x01);
7482     __ vmulss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
7483     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x02);
7484     __ vmulss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
7485     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x03);
7486     __ vmulss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
7487   %}
7488   ins_pipe( pipe_slow );
7489 %}
7490 
7491 instruct rvmul8F_reduction_reg(regF dst, vecY src2, regF tmp, regF tmp2) %{
7492   predicate(UseAVX > 0);
7493   match(Set dst (MulReductionVF dst src2));
7494   effect(TEMP tmp, TEMP dst, TEMP tmp2);
7495   format %{ "vmulss  $dst,$dst,$src2\n\t"
7496             "pshufd  $tmp,$src2,0x01\n\t"
7497             "vmulss  $dst,$dst,$tmp\n\t"
7498             "pshufd  $tmp,$src2,0x02\n\t"
7499             "vmulss  $dst,$dst,$tmp\n\t"
7500             "pshufd  $tmp,$src2,0x03\n\t"
7501             "vmulss  $dst,$dst,$tmp\n\t"
7502             "vextractf128_high  $tmp2,$src2\n\t"
7503             "vmulss  $dst,$dst,$tmp2\n\t"
7504             "pshufd  $tmp,$tmp2,0x01\n\t"
7505             "vmulss  $dst,$dst,$tmp\n\t"
7506             "pshufd  $tmp,$tmp2,0x02\n\t"
7507             "vmulss  $dst,$dst,$tmp\n\t"
7508             "pshufd  $tmp,$tmp2,0x03\n\t"
7509             "vmulss  $dst,$dst,$tmp\t! mul reduction8F" %}
7510   ins_encode %{
7511     __ vmulss($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister);
7512     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x01);
7513     __ vmulss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
7514     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x02);
7515     __ vmulss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
7516     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x03);
7517     __ vmulss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
7518     __ vextractf128_high($tmp2$$XMMRegister, $src2$$XMMRegister);
7519     __ vmulss($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister);
7520     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x01);
7521     __ vmulss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
7522     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x02);
7523     __ vmulss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
7524     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x03);
7525     __ vmulss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
7526   %}
7527   ins_pipe( pipe_slow );
7528 %}
7529 
7530 instruct rvmul16F_reduction_reg(regF dst, vecZ src2, regF tmp, regF tmp2) %{
7531   predicate(UseAVX > 2);
7532   match(Set dst (MulReductionVF dst src2));
7533   effect(TEMP tmp, TEMP dst, TEMP tmp2);
7534   format %{ "vmulss  $dst,$dst,$src2\n\t"
7535             "pshufd  $tmp,$src2,0x01\n\t"
7536             "vmulss  $dst,$dst,$tmp\n\t"
7537             "pshufd  $tmp,$src2,0x02\n\t"
7538             "vmulss  $dst,$dst,$tmp\n\t"
7539             "pshufd  $tmp,$src2,0x03\n\t"
7540             "vmulss  $dst,$dst,$tmp\n\t"
7541             "vextractf32x4  $tmp2,$src2,0x1\n\t"
7542             "vmulss  $dst,$dst,$tmp2\n\t"
7543             "pshufd  $tmp,$tmp2,0x01\n\t"
7544             "vmulss  $dst,$dst,$tmp\n\t"
7545             "pshufd  $tmp,$tmp2,0x02\n\t"
7546             "vmulss  $dst,$dst,$tmp\n\t"
7547             "pshufd  $tmp,$tmp2,0x03\n\t"
7548             "vmulss  $dst,$dst,$tmp\n\t"
7549             "vextractf32x4  $tmp2,$src2,0x2\n\t"
7550             "vmulss  $dst,$dst,$tmp2\n\t"
7551             "pshufd  $tmp,$tmp2,0x01\n\t"
7552             "vmulss  $dst,$dst,$tmp\n\t"
7553             "pshufd  $tmp,$tmp2,0x02\n\t"
7554             "vmulss  $dst,$dst,$tmp\n\t"
7555             "pshufd  $tmp,$tmp2,0x03\n\t"
7556             "vmulss  $dst,$dst,$tmp\n\t"
7557             "vextractf32x4  $tmp2,$src2,0x3\n\t"
7558             "vmulss  $dst,$dst,$tmp2\n\t"
7559             "pshufd  $tmp,$tmp2,0x01\n\t"
7560             "vmulss  $dst,$dst,$tmp\n\t"
7561             "pshufd  $tmp,$tmp2,0x02\n\t"
7562             "vmulss  $dst,$dst,$tmp\n\t"
7563             "pshufd  $tmp,$tmp2,0x03\n\t"
7564             "vmulss  $dst,$dst,$tmp\t! mul reduction16F" %}
7565   ins_encode %{
7566     __ vmulss($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister);
7567     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x01);
7568     __ vmulss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
7569     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x02);
7570     __ vmulss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
7571     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x03);
7572     __ vmulss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
7573     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x1);
7574     __ vmulss($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister);
7575     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x01);
7576     __ vmulss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
7577     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x02);
7578     __ vmulss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
7579     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x03);
7580     __ vmulss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
7581     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x2);
7582     __ vmulss($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister);
7583     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x01);
7584     __ vmulss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
7585     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x02);
7586     __ vmulss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
7587     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x03);
7588     __ vmulss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
7589     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x3);
7590     __ vmulss($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister);
7591     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x01);
7592     __ vmulss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
7593     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x02);
7594     __ vmulss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
7595     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x03);
7596     __ vmulss($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
7597   %}
7598   ins_pipe( pipe_slow );
7599 %}
7600 
7601 instruct rsmul2D_reduction_reg(regD dst, vecX src2, regD tmp) %{
7602   predicate(UseSSE >= 1 && UseAVX == 0);
7603   match(Set dst (MulReductionVD dst src2));
7604   effect(TEMP dst, TEMP tmp);
7605   format %{ "mulsd   $dst,$src2\n\t"
7606             "pshufd  $tmp,$src2,0xE\n\t"
7607             "mulsd   $dst,$tmp\t! mul reduction2D" %}
7608   ins_encode %{
7609     __ mulsd($dst$$XMMRegister, $src2$$XMMRegister);
7610     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
7611     __ mulsd($dst$$XMMRegister, $tmp$$XMMRegister);
7612   %}
7613   ins_pipe( pipe_slow );
7614 %}
7615 
7616 instruct rvmul2D_reduction_reg(regD dst, vecX src2, regD tmp) %{
7617   predicate(UseAVX > 0);
7618   match(Set dst (MulReductionVD dst src2));
7619   effect(TEMP tmp, TEMP dst);
7620   format %{ "vmulsd  $dst,$dst,$src2\n\t"
7621             "pshufd  $tmp,$src2,0xE\n\t"
7622             "vmulsd  $dst,$dst,$tmp\t! mul reduction2D" %}
7623   ins_encode %{
7624     __ vmulsd($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister);
7625     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
7626     __ vmulsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
7627   %}
7628   ins_pipe( pipe_slow );
7629 %}
7630 
7631 instruct rvmul4D_reduction_reg(regD dst, vecY src2, regD tmp, regD tmp2) %{
7632   predicate(UseAVX > 0);
7633   match(Set dst (MulReductionVD dst src2));
7634   effect(TEMP tmp, TEMP dst, TEMP tmp2);
7635   format %{ "vmulsd  $dst,$dst,$src2\n\t"
7636             "pshufd  $tmp,$src2,0xE\n\t"
7637             "vmulsd  $dst,$dst,$tmp\n\t"
7638             "vextractf128_high  $tmp2,$src2\n\t"
7639             "vmulsd  $dst,$dst,$tmp2\n\t"
7640             "pshufd  $tmp,$tmp2,0xE\n\t"
7641             "vmulsd  $dst,$dst,$tmp\t! mul reduction4D" %}
7642   ins_encode %{
7643     __ vmulsd($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister);
7644     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
7645     __ vmulsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
7646     __ vextractf128_high($tmp2$$XMMRegister, $src2$$XMMRegister);
7647     __ vmulsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister);
7648     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE);
7649     __ vmulsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
7650   %}
7651   ins_pipe( pipe_slow );
7652 %}
7653 
7654 instruct rvmul8D_reduction_reg(regD dst, vecZ src2, regD tmp, regD tmp2) %{
7655   predicate(UseAVX > 2);
7656   match(Set dst (MulReductionVD dst src2));
7657   effect(TEMP tmp, TEMP dst, TEMP tmp2);
7658   format %{ "vmulsd  $dst,$dst,$src2\n\t"
7659             "pshufd  $tmp,$src2,0xE\n\t"
7660             "vmulsd  $dst,$dst,$tmp\n\t"
7661             "vextractf32x4  $tmp2,$src2,0x1\n\t"
7662             "vmulsd  $dst,$dst,$tmp2\n\t"
7663             "pshufd  $tmp,$src2,0xE\n\t"
7664             "vmulsd  $dst,$dst,$tmp\n\t"
7665             "vextractf32x4  $tmp2,$src2,0x2\n\t"
7666             "vmulsd  $dst,$dst,$tmp2\n\t"
7667             "pshufd  $tmp,$tmp2,0xE\n\t"
7668             "vmulsd  $dst,$dst,$tmp\n\t"
7669             "vextractf32x4  $tmp2,$src2,0x3\n\t"
7670             "vmulsd  $dst,$dst,$tmp2\n\t"
7671             "pshufd  $tmp,$tmp2,0xE\n\t"
7672             "vmulsd  $dst,$dst,$tmp\t! mul reduction8D" %}
7673   ins_encode %{
7674     __ vmulsd($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister);
7675     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
7676     __ vmulsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
7677     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x1);
7678     __ vmulsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister);
7679     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE);
7680     __ vmulsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
7681     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x2);
7682     __ vmulsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister);
7683     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE);
7684     __ vmulsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
7685     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x3);
7686     __ vmulsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister);
7687     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE);
7688     __ vmulsd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister);
7689   %}
7690   ins_pipe( pipe_slow );
7691 %}
7692 
7693 //--------------------Min Reduction --------------------
7694 instruct rsmin8B_reduction_reg(rRegI dst, rRegI src1, vecD src2, regF tmp, rRegI tmp2, rRegI tmp3, regF tmp4) %{
7695   predicate(UseSSE > 3 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
7696   match(Set dst (MinReductionV src1 src2));
7697   effect(TEMP dst, TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP tmp4);
7698   format %{ "pshufd  $tmp,$src2,0x1\n\t"
7699             "pminsb  $tmp,$src2\n\t"
7700             "pextrb  $tmp2,$tmp, 0x1\n\t"
7701             "movsbl  $tmp2,$tmp2\n\t"
7702             "pextrb  $tmp3,$tmp,0x0\n\t"
7703             "movsbl  $tmp3,$tmp3\n\t"
7704             "cmpl  $tmp2,$tmp3\n\t"
7705             "cmovl  $tmp3,$tmp2\n\t"
7706             "cmpl  $src1,$tmp3\n\t"
7707             "cmovl  $tmp3,$src1, 0x0\n\t"
7708             "movl  $dst,$tmp2\n\t"
7709             "pextrb  $tmp2,$tmp\n\t"
7710             "movsbl  $tmp2,$tmp2\n\t"
7711             "pextrb  $tmp3,$tmp\n\t"
7712             "movsbl  $tmp3,$tmp3\n\t"
7713             "cmpl  $tmp2,$tmp3\n\t"
7714             "cmovl  $tmp3,$tmp2\n\t"
7715             "cmpl  $tmp3,$dst\n\t"
7716             "cmovl  $dst,$tmp3\t! min reduction4S" %}
7717   ins_encode %{
7718     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister,0x1);
7719     __ pminsb($tmp$$XMMRegister, $src2$$XMMRegister);
7720     __ pextrb($tmp2$$Register, $tmp$$XMMRegister,0x1);
7721     __ movsbl($tmp2$$Register, $tmp2$$Register);
7722     __ pextrb($tmp3$$Register, $tmp$$XMMRegister,0x0);
7723     __ movsbl($tmp3$$Register, $tmp3$$Register);
7724     __ cmpl($tmp2$$Register, $tmp3$$Register);
7725     __ cmovl(Assembler::less, $tmp3$$Register, $tmp2$$Register);
7726     __ cmpl($src1$$Register, $tmp3$$Register);
7727     __ cmovl(Assembler::less, $tmp3$$Register, $src1$$Register);
7728     __ movl($dst$$Register, $tmp3$$Register);
7729     __ pextrb($tmp2$$Register, $tmp$$XMMRegister,0x3);
7730     __ movsbl($tmp2$$Register, $tmp2$$Register);
7731     __ pextrb($tmp3$$Register, $tmp$$XMMRegister,0x2);
7732     __ movsbl($tmp3$$Register, $tmp3$$Register);
7733     __ cmpl($tmp2$$Register, $tmp3$$Register);
7734     __ cmovl(Assembler::less, $tmp3$$Register, $tmp2$$Register);
7735     __ cmpl($tmp3$$Register, $dst$$Register);
7736     __ cmovl(Assembler::less, $dst$$Register, $tmp3$$Register);
7737     __ movsbl($dst$$Register, $dst$$Register);
7738   %}
7739   ins_pipe( pipe_slow );
7740 %}
7741 
7742 instruct rsmin16B_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, rRegI tmp2, rRegI tmp3, regF tmp4) %{
7743   predicate(UseSSE > 3 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
7744   match(Set dst (MinReductionV src1 src2));
7745   effect(TEMP dst, TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP tmp4);
7746   format %{ "pshufd  $tmp4,$src2,0xE\n\t"
7747             "pminsb  $tmp4,$src2\n\t"
7748             "pshufd  $tmp,$tmp4,0x1\n\t"
7749             "pminsb  $tmp,$tmp4\n\t"
7750             "pextrb  $tmp2,$tmp, 0x1\n\t"
7751             "movsbl  $tmp2,$tmp2\n\t"
7752             "pextrb  $tmp3,$tmp,0x0\n\t"
7753             "movsbl  $tmp3,$tmp3\n\t"
7754             "cmpl  $tmp2,$tmp3\n\t"
7755             "cmovl  $tmp3,$tmp2\n\t"
7756             "cmpl  $src1,$tmp3\n\t"
7757             "cmovl  $tmp3,$src1, 0x0\n\t"
7758             "movl  $dst,$tmp2\n\t"
7759             "pextrb  $tmp2,$tmp\n\t"
7760             "movsbl  $tmp2,$tmp2\n\t"
7761             "pextrb  $tmp3,$tmp\n\t"
7762             "movsbl  $tmp3,$tmp3\n\t"
7763             "cmpl  $tmp2,$tmp3\n\t"
7764             "cmovl  $tmp3,$tmp2\n\t"
7765             "cmpl  $tmp3,$dst\n\t"
7766             "cmovl  $dst,$tmp3\t! min reduction4S" %}
7767   ins_encode %{
7768     __ pshufd($tmp4$$XMMRegister, $src2$$XMMRegister, 0xE);
7769     __ pminsb($tmp4$$XMMRegister, $src2$$XMMRegister);
7770     __ pshufd($tmp$$XMMRegister, $tmp4$$XMMRegister,0x1);
7771     __ pminsb($tmp$$XMMRegister, $tmp4$$XMMRegister);
7772     __ pextrb($tmp2$$Register, $tmp$$XMMRegister,0x1);
7773     __ movsbl($tmp2$$Register, $tmp2$$Register);
7774     __ pextrb($tmp3$$Register, $tmp$$XMMRegister,0x0);
7775     __ movsbl($tmp3$$Register, $tmp3$$Register);
7776     __ cmpl($tmp2$$Register, $tmp3$$Register);
7777     __ cmovl(Assembler::less, $tmp3$$Register, $tmp2$$Register);
7778     __ cmpl($src1$$Register, $tmp3$$Register);
7779     __ cmovl(Assembler::less, $tmp3$$Register, $src1$$Register);
7780     __ movl($dst$$Register, $tmp3$$Register);
7781     __ pextrb($tmp2$$Register, $tmp$$XMMRegister,0x3);
7782     __ movsbl($tmp2$$Register, $tmp2$$Register);
7783     __ pextrb($tmp3$$Register, $tmp$$XMMRegister,0x2);
7784     __ movsbl($tmp3$$Register, $tmp3$$Register);
7785     __ cmpl($tmp2$$Register, $tmp3$$Register);
7786     __ cmovl(Assembler::less, $tmp3$$Register, $tmp2$$Register);
7787     __ cmpl($tmp3$$Register, $dst$$Register);
7788     __ cmovl(Assembler::less, $dst$$Register, $tmp3$$Register);
7789     __ movsbl($dst$$Register, $dst$$Register);
7790   %}
7791   ins_pipe( pipe_slow );
7792 %}
7793 
7794 instruct rvmin16B_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, rRegI tmp2, rRegI tmp3, regF tmp4) %{
7795   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
7796   match(Set dst (MinReductionV src1 src2));
7797   effect(TEMP dst, TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP tmp4);
7798   format %{ "pshufd  $tmp4,$src2,0xE\n\t"
7799             "vpminsb  $tmp,$tmp4,$src2\n\t"
7800             "pshufd  $tmp,$tmp4,0x1\n\t"
7801             "vpminsb  $tmp,$tmp,$tmp4\n\t"
7802             "pextrb  $tmp2,$tmp, 0x1\n\t"
7803             "movsbl  $tmp2,$tmp2\n\t"
7804             "pextrb  $tmp3,$tmp,0x0\n\t"
7805             "movsbl  $tmp3,$tmp3\n\t"
7806             "cmpl  $tmp2,$tmp3\n\t"
7807             "cmovl  $tmp3,$tmp2\n\t"
7808             "cmpl  $src1,$tmp3\n\t"
7809             "cmovl  $tmp3,$src1, 0x0\n\t"
7810             "movl  $dst,$tmp2\n\t"
7811             "pextrb  $tmp2,$tmp\n\t"
7812             "movsbl  $tmp2,$tmp2\n\t"
7813             "pextrb  $tmp3,$tmp\n\t"
7814             "movsbl  $tmp3,$tmp3\n\t"
7815             "cmpl  $tmp2,$tmp3\n\t"
7816             "cmovl  $tmp3,$tmp2\n\t"
7817             "cmpl  $tmp3,$dst\n\t"
7818             "cmovl  $dst,$tmp3\t! min reduction4S" %}
7819   ins_encode %{
7820     int vector_len = 0;
7821     __ pshufd($tmp4$$XMMRegister, $src2$$XMMRegister, 0xE);
7822     __ vpminsb($tmp4$$XMMRegister, $tmp4$$XMMRegister, $src2$$XMMRegister, 0);
7823     __ pshufd($tmp$$XMMRegister, $tmp4$$XMMRegister,0x1);
7824     __ vpminsb($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp4$$XMMRegister, 0);
7825     __ pextrb($tmp2$$Register, $tmp$$XMMRegister,0x1);
7826     __ movsbl($tmp2$$Register, $tmp2$$Register);
7827     __ pextrb($tmp3$$Register, $tmp$$XMMRegister,0x0);
7828     __ movsbl($tmp3$$Register, $tmp3$$Register);
7829     __ cmpl($tmp2$$Register, $tmp3$$Register);
7830     __ cmovl(Assembler::less, $tmp3$$Register, $tmp2$$Register);
7831     __ cmpl($src1$$Register, $tmp3$$Register);
7832     __ cmovl(Assembler::less, $tmp3$$Register, $src1$$Register);
7833     __ movl($dst$$Register, $tmp3$$Register);
7834     __ pextrb($tmp2$$Register, $tmp$$XMMRegister,0x3);
7835     __ movsbl($tmp2$$Register, $tmp2$$Register);
7836     __ pextrb($tmp3$$Register, $tmp$$XMMRegister,0x2);
7837     __ movsbl($tmp3$$Register, $tmp3$$Register);
7838     __ cmpl($tmp2$$Register, $tmp3$$Register);
7839     __ cmovl(Assembler::less, $tmp3$$Register, $tmp2$$Register);
7840     __ cmpl($tmp3$$Register, $dst$$Register);
7841     __ cmovl(Assembler::less, $dst$$Register, $tmp3$$Register);
7842     __ movsbl($dst$$Register, $dst$$Register);
7843   %}
7844   ins_pipe( pipe_slow );
7845 %}
7846 
7847 instruct rvmin32B_reduction_reg(rRegI dst, rRegI src1, vecY src2, regF tmp, rRegI tmp2, rRegI tmp3, regF tmp4) %{
7848   predicate(UseAVX > 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
7849   match(Set dst (MinReductionV src1 src2));
7850   effect(TEMP dst, TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP tmp4);
7851   format %{ "vextracti128_high  $tmp,$src2\n\t"
7852             "vpminsb  $tmp,$tmp,$src2\n\t"
7853             "pshufd  $tmp4,$tmp,0xE\n\t"
7854             "vpminsb  $tmp4,$tmp4,$tmp\n\t"
7855             "pshufd  $tmp,$tmp4,0x1\n\t"
7856             "vpminsb  $tmp,$tmp,$tmp4\n\t"
7857             "pextrb  $tmp2,$tmp, 0x1\n\t"
7858             "movsbl  $tmp2,$tmp2\n\t"
7859             "pextrb  $tmp3,$tmp,0x0\n\t"
7860             "movsbl  $tmp3,$tmp3\n\t"
7861             "cmpl  $tmp2,$tmp3\n\t"
7862             "cmovl  $tmp3,$tmp2\n\t"
7863             "cmpl  $src1,$tmp3\n\t"
7864             "cmovl  $tmp3,$src1, 0x0\n\t"
7865             "movl  $dst,$tmp2\n\t"
7866             "pextrb  $tmp2,$tmp\n\t"
7867             "movsbl  $tmp2,$tmp2\n\t"
7868             "pextrb  $tmp3,$tmp\n\t"
7869             "movsbl  $tmp3,$tmp3\n\t"
7870             "cmpl  $tmp2,$tmp3\n\t"
7871             "cmovl  $tmp3,$tmp2\n\t"
7872             "cmpl  $tmp3,$dst\n\t"
7873             "cmovl  $dst,$tmp3\t! min reduction4S" %}
7874   ins_encode %{
7875     int vector_len = 1;
7876     __ vextracti128_high($tmp$$XMMRegister, $src2$$XMMRegister);
7877     __ vpminsb($tmp$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, vector_len);
7878     __ pshufd($tmp4$$XMMRegister, $tmp$$XMMRegister, 0xE);
7879     __ vpminsb($tmp4$$XMMRegister, $tmp4$$XMMRegister, $tmp$$XMMRegister, 0);
7880     __ pshufd($tmp$$XMMRegister, $tmp4$$XMMRegister,0x1);
7881     __ vpminsb($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp4$$XMMRegister, 0);
7882     __ pextrb($tmp2$$Register, $tmp$$XMMRegister,0x1);
7883     __ movsbl($tmp2$$Register, $tmp2$$Register);
7884     __ pextrb($tmp3$$Register, $tmp$$XMMRegister,0x0);
7885     __ movsbl($tmp3$$Register, $tmp3$$Register);
7886     __ cmpl($tmp2$$Register, $tmp3$$Register);
7887     __ cmovl(Assembler::less, $tmp3$$Register, $tmp2$$Register);
7888     __ cmpl($src1$$Register, $tmp3$$Register);
7889     __ cmovl(Assembler::less, $tmp3$$Register, $src1$$Register);
7890     __ movl($dst$$Register, $tmp3$$Register);
7891     __ pextrb($tmp2$$Register, $tmp$$XMMRegister,0x3);
7892     __ movsbl($tmp2$$Register, $tmp2$$Register);
7893     __ pextrb($tmp3$$Register, $tmp$$XMMRegister,0x2);
7894     __ movsbl($tmp3$$Register, $tmp3$$Register);
7895     __ cmpl($tmp2$$Register, $tmp3$$Register);
7896     __ cmovl(Assembler::less, $tmp3$$Register, $tmp2$$Register);
7897     __ cmpl($tmp3$$Register, $dst$$Register);
7898     __ cmovl(Assembler::less, $dst$$Register, $tmp3$$Register);
7899     __ movsbl($dst$$Register, $dst$$Register);
7900   %}
7901   ins_pipe( pipe_slow );
7902 %}
7903 
7904 instruct rvmin64B_reduction_reg(rRegI dst, rRegI src1, vecZ src2, regF tmp, rRegI tmp2, rRegI tmp3, regF tmp4) %{
7905   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
7906   match(Set dst (MinReductionV src1 src2));
7907   effect(TEMP dst, TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP tmp4);
7908   format %{ "vextracti64x4_high  $tmp4,$src2\n\t"
7909             "vpminsb  $tmp4,$tmp4,$src2\n\t"
7910             "vextracti128_high  $tmp,$tmp4\n\t"
7911             "vpminsb  $tmp,$tmp,$tmp4\n\t"
7912             "pshufd  $tmp4,$tmp,0xE\n\t"
7913             "vpminsb  $tmp4,$tmp4,$tmp\n\t"
7914             "pshufd  $tmp,$tmp4,0x1\n\t"
7915             "vpminsb  $tmp,$tmp,$tmp4\n\t"
7916             "pextrb  $tmp2,$tmp, 0x1\n\t"
7917             "movsbl  $tmp2,$tmp2\n\t"
7918             "pextrb  $tmp3,$tmp,0x0\n\t"
7919             "movsbl  $tmp3,$tmp3\n\t"
7920             "cmpl  $tmp2,$tmp3\n\t"
7921             "cmovl  $tmp3,$tmp2\n\t"
7922             "cmpl  $src1,$tmp3\n\t"
7923             "cmovl  $tmp3,$src1, 0x0\n\t"
7924             "movl  $dst,$tmp2\n\t"
7925             "pextrb  $tmp2,$tmp\n\t"
7926             "movsbl  $tmp2,$tmp2\n\t"
7927             "pextrb  $tmp3,$tmp\n\t"
7928             "movsbl  $tmp3,$tmp3\n\t"
7929             "cmpl  $tmp2,$tmp3\n\t"
7930             "cmovl  $tmp3,$tmp2\n\t"
7931             "cmpl  $tmp3,$dst\n\t"
7932             "cmovl  $dst,$tmp3\t! min reduction4S" %}
7933   ins_encode %{
7934     __ vextracti64x4_high($tmp4$$XMMRegister, $src2$$XMMRegister);
7935     __ vpminsb($tmp4$$XMMRegister, $tmp4$$XMMRegister, $src2$$XMMRegister, 2);
7936     __ vextracti128_high($tmp$$XMMRegister, $tmp4$$XMMRegister);
7937     __ vpminsb($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp4$$XMMRegister, 1);
7938     __ pshufd($tmp4$$XMMRegister, $tmp$$XMMRegister, 0xE);
7939     __ vpminsb($tmp4$$XMMRegister, $tmp4$$XMMRegister, $tmp$$XMMRegister, 0);
7940     __ pshufd($tmp$$XMMRegister, $tmp4$$XMMRegister,0x1);
7941     __ vpminsb($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp4$$XMMRegister, 0);
7942     __ pextrb($tmp2$$Register, $tmp$$XMMRegister,0x1);
7943     __ movsbl($tmp2$$Register, $tmp2$$Register);
7944     __ pextrb($tmp3$$Register, $tmp$$XMMRegister,0x0);
7945     __ movsbl($tmp3$$Register, $tmp3$$Register);
7946     __ cmpl($tmp2$$Register, $tmp3$$Register);
7947     __ cmovl(Assembler::less, $tmp3$$Register, $tmp2$$Register);
7948     __ cmpl($src1$$Register, $tmp3$$Register);
7949     __ cmovl(Assembler::less, $tmp3$$Register, $src1$$Register);
7950     __ movl($dst$$Register, $tmp3$$Register);
7951     __ pextrb($tmp2$$Register, $tmp$$XMMRegister,0x3);
7952     __ movsbl($tmp2$$Register, $tmp2$$Register);
7953     __ pextrb($tmp3$$Register, $tmp$$XMMRegister,0x2);
7954     __ movsbl($tmp3$$Register, $tmp3$$Register);
7955     __ cmpl($tmp2$$Register, $tmp3$$Register);
7956     __ cmovl(Assembler::less, $tmp3$$Register, $tmp2$$Register);
7957     __ cmpl($tmp3$$Register, $dst$$Register);
7958     __ cmovl(Assembler::less, $dst$$Register, $tmp3$$Register);
7959     __ movsbl($dst$$Register, $dst$$Register);
7960   %}
7961   ins_pipe( pipe_slow );
7962 %}
7963 
7964 instruct rsmin4S_reduction_reg(rRegI dst, rRegI src1, vecD src2, regF tmp, rRegI tmp2, rRegI tmp3) %{
7965   predicate(UseSSE > 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
7966   match(Set dst (MinReductionV src1 src2));
7967   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
7968   format %{ "pshufd  $tmp,$src2,0x1\n\t"
7969             "pminsw  $tmp,$src2\n\t"
7970             "pextrw  $tmp2,$tmp, 0x1\n\t"
7971             "movswl  $tmp2,$tmp2\n\t"
7972             "pextrb  $tmp3,$tmp, 0x0\n\t"
7973             "movswl  $tmp3,$tmp3,0x1\n\t"
7974             "cmpl  $tmp2,$tmp3\n\t"
7975             "cmovl  $tmp3,tmp2\n\t"
7976             "cmpl  $src1,$tmp3\n\t"
7977             "cmovl  $tmp3,$src1\n\t"
7978             "movswl  $dst,$tmp3\t! min reduction4S" %}
7979   ins_encode %{
7980     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister,0x1);
7981     __ pminsw($tmp$$XMMRegister, $src2$$XMMRegister);
7982     __ pextrw($tmp2$$Register, $tmp$$XMMRegister,0x1);
7983     __ movswl($tmp2$$Register, $tmp2$$Register);
7984     __ pextrw($tmp3$$Register, $tmp$$XMMRegister,0x0);
7985     __ movswl($tmp3$$Register, $tmp3$$Register);
7986     __ cmpl($tmp2$$Register, $tmp3$$Register);
7987     __ cmovl(Assembler::less, $tmp3$$Register, $tmp2$$Register);
7988     __ cmpl($src1$$Register, $tmp3$$Register);
7989     __ cmovl(Assembler::less, $tmp3$$Register, $src1$$Register);
7990     __ movl($dst$$Register, $tmp3$$Register);
7991   %}
7992   ins_pipe( pipe_slow );
7993 %}
7994 
7995 instruct rsmin8S_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp2, rRegI tmp3, rRegI tmp4) %{
7996   predicate(UseSSE > 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
7997   match(Set dst (MinReductionV src1 src2));
7998   effect(TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP tmp4);
7999   format %{ "pshufd  $tmp2,$src2,0xE\n\t"
8000             "pminsw  $tmp2,$src2\n\t"
8001             "pshufd  $tmp,$tmp2,0x1\n\t"
8002             "pminsw  $tmp,$tmp2\n\t"
8003             "pextrw  $tmp2,$tmp\n\t"
8004             "movswl  $tmp2,$tmp2\n\t"
8005             "pextrw  $tmp3,$tmp, 0x0\n\t"
8006             "movswl  $tmp3,$tmp3\n\t"
8007             "cmpl    $tmp2,$tmp3\n\t"
8008             "cmovl  $tmp3,$tmp2\n\t"
8009             "cmpl  $src1,$tmp3\n\t"
8010             "cmovl  $tmp3,$src1\n\t"
8011             "movl  $dst,$tmp3\t! min reduction8S" %}
8012   ins_encode %{
8013     __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister,0xE);
8014     __ pminsw($tmp2$$XMMRegister, $src2$$XMMRegister);
8015     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister,0x1);
8016     __ pminsw($tmp$$XMMRegister, $tmp2$$XMMRegister);
8017     __ pextrw($tmp4$$Register, $tmp$$XMMRegister,0x1);
8018     __ movswl($tmp4$$Register, $tmp4$$Register);
8019     __ pextrw($tmp3$$Register, $tmp$$XMMRegister,0x0);
8020     __ movswl($tmp3$$Register, $tmp3$$Register);
8021     __ cmpl($tmp4$$Register, $tmp3$$Register);
8022     __ cmovl(Assembler::less, $tmp3$$Register, $tmp4$$Register);
8023     __ cmpl($src1$$Register, $tmp3$$Register);
8024     __ cmovl(Assembler::less, $tmp3$$Register, $src1$$Register);
8025     __ movl($dst$$Register, $tmp3$$Register);
8026   %}
8027   ins_pipe( pipe_slow );
8028 %}
8029 
8030 instruct rvmin8S_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp2, rRegI tmp3, rRegI tmp4) %{
8031   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
8032   match(Set dst (MinReductionV src1 src2));
8033   effect(TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP tmp4);
8034   format %{ "pshufd   $tmp,$src2,0xE\n\t"
8035             "vpminsw  $tmp,$tmp,$src2\n\t"
8036             "pshufd   $tmp2,$tmp,0x1\n\t"
8037             "vpminsw  $tmp,$tmp,$tmp2\n\t"
8038             "movzwl   $dst,$src1\n\t"
8039             "pextrw   $tmp3,$tmp, 0x0\n\t"
8040             "vpminsw  $dst,$dst,$tmp3\n\t"
8041             "pextrw   $tmp3,$tmp, 0x1\n\t"
8042             "vpminsw  $dst,$dst,$tmp3\n\t"
8043             "movswl   $dst,$dst\t! min reduction8S" %}
8044   ins_encode %{
8045     int vector_len = 0;
8046     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
8047     __ vpminsw($tmp$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, vector_len);
8048     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister,0x1);
8049     __ vpminsw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
8050     __ pextrw($tmp4$$Register, $tmp$$XMMRegister,0x1);
8051     __ movswl($tmp4$$Register, $tmp4$$Register);
8052     __ pextrw($tmp3$$Register, $tmp$$XMMRegister,0x0);
8053     __ movswl($tmp3$$Register, $tmp3$$Register);
8054     __ cmpl($tmp4$$Register, $tmp3$$Register);
8055     __ cmovl(Assembler::less, $tmp3$$Register, $tmp4$$Register);
8056     __ cmpl($src1$$Register, $tmp3$$Register);
8057     __ cmovl(Assembler::less, $tmp3$$Register, $src1$$Register);
8058     __ movl($dst$$Register, $tmp3$$Register);
8059   %}
8060   ins_pipe( pipe_slow );
8061 %}
8062 
8063 instruct rvmin16S_reduction_reg(rRegI dst, rRegI src1, vecY src2, regF tmp, regF tmp2, rRegI tmp3, rRegI tmp4) %{
8064   predicate(UseAVX > 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
8065   match(Set dst (MinReductionV src1 src2));
8066   effect(TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP tmp4);
8067   format %{ "vextracti128_high  $tmp,$src2\n\t"
8068             "vpminsw  $tmp,$tmp,$src2\n\t"
8069             "pshufd  $tmp2,$tmp,0xE\n\t"
8070             "vpminsw  $tmp,$tmp,$tmp2\n\t"
8071             "pshufd  $tmp2,$tmp,0x1\n\t"
8072             "vpminsw  $tmp,$tmp,$tmp2\n\t"
8073             "pextrw  $tmp2,$tmp, 0x1\n\t"
8074             "movswl  $tmp2,$tmp2\n\t"
8075             "pextrw  $tmp3,$tmp, 0x0\n\t"
8076             "movswl  $tmp3,$tmp3\n\t"
8077             "cmpl  $tmp2$tmp3\n\t"
8078             "cmovl  $tmp3,$tmp2\n\t"
8079             "cmpl  $src1,$tmp3\n\t"
8080             "cmovl  $tmp3,$src1\n\t"
8081             "movl  $dst,$tmp3\t! min reduction16S" %}
8082   ins_encode %{
8083     int vector_len = 1;
8084     __ vextracti128_high($tmp$$XMMRegister, $src2$$XMMRegister);
8085     __ vpminsw($tmp$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, vector_len);
8086     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE);
8087     __ vpminsw($tmp$$XMMRegister,$tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
8088     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister,0x1);
8089     __ vpminsw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
8090     __ pextrw($tmp4$$Register, $tmp$$XMMRegister,0x1);
8091     __ movswl($tmp4$$Register, $tmp4$$Register);
8092     __ pextrw($tmp3$$Register, $tmp$$XMMRegister,0x0);
8093     __ movswl($tmp3$$Register, $tmp3$$Register);
8094     __ cmpl($tmp4$$Register, $tmp3$$Register);
8095     __ cmovl(Assembler::less, $tmp3$$Register, $tmp4$$Register);
8096     __ cmpl($src1$$Register, $tmp3$$Register);
8097     __ cmovl(Assembler::less, $tmp3$$Register, $src1$$Register);
8098     __ movl($dst$$Register, $tmp3$$Register);
8099   %}
8100   ins_pipe( pipe_slow );
8101 %}
8102 
8103 instruct rvmin32S_reduction_reg(rRegI dst, rRegI src1, vecZ src2, regF tmp, regF tmp2, rRegI tmp3, rRegI tmp4) %{
8104   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
8105   match(Set dst (MinReductionV src1 src2));
8106   effect(TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP tmp4);
8107   format %{ "vextracti64x4_high  $tmp2,$src2\n\t"
8108             "vpminsw  $tmp2,$tmp2,$src2\n\t"
8109             "vextracti128_high  $tmp,$tmp2\n\t"
8110             "vpminsw  $tmp,$tmp,$tmp2\n\t"
8111             "pshufd  $tmp2,$tmp,0xE\n\t"
8112             "vpminsw  $tmp,$tmp,$tmp2\n\t"
8113             "pshufd  $tmp2,$tmp,0x1\n\t"
8114             "vpminsw  $tmp,$tmp,$tmp2\n\t"
8115             "pextrw  $tmp3,$tmp, 0x0\n\t"
8116             "movswl  $dst,$src1\n\t"
8117             "pextrw  $tmp3,$tmp, 0x0\n\t"
8118             "movswl  $dst,$src1\n\t"
8119             "cmpl  $tmp2$tmp3\n\t"
8120             "cmovl  $tmp3,$tmp2\n\t"
8121             "cmpl  $src1,$tmp3\n\t"
8122             "cmovl  $tmp3,$src1\n\t"
8123             "movl  $dst,$dst\t! min reduction32S" %}
8124   ins_encode %{
8125     int vector_len = 2;
8126     __ vextracti64x4_high($tmp2$$XMMRegister, $src2$$XMMRegister);
8127     __ vpminsw($tmp2$$XMMRegister, $tmp2$$XMMRegister, $src2$$XMMRegister, vector_len);
8128     __ vextracti128_high($tmp$$XMMRegister, $tmp2$$XMMRegister);
8129     __ vpminsw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
8130     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE);
8131     __ vpminsw($tmp$$XMMRegister,$tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
8132     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister,0x1);
8133     __ vpminsw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
8134     __ pextrw($tmp4$$Register, $tmp$$XMMRegister,0x1);
8135     __ movswl($tmp4$$Register, $tmp4$$Register);
8136     __ pextrw($tmp3$$Register, $tmp$$XMMRegister,0x0);
8137     __ movswl($tmp3$$Register, $tmp3$$Register);
8138     __ cmpl($tmp4$$Register, $tmp3$$Register);
8139     __ cmovl(Assembler::less, $tmp3$$Register, $tmp4$$Register);
8140     __ cmpl($src1$$Register, $tmp3$$Register);
8141     __ cmovl(Assembler::less, $tmp3$$Register, $src1$$Register);
8142     __ movl($dst$$Register, $tmp3$$Register);
8143   %}
8144   ins_pipe( pipe_slow );
8145 %}
8146 
8147 instruct rsmin2I_reduction_reg(rRegI dst, rRegI src1, vecD src2, regF tmp, regF tmp2) %{
8148   predicate(UseSSE > 3 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
8149   match(Set dst (MinReductionV src1 src2));
8150   effect(TEMP tmp, TEMP tmp2);
8151   format %{ "pshufd  $tmp,$src2,0x1\n\t"
8152             "pminsd  $tmp,$src2\n\t"
8153             "movd    $tmp2,$src1\n\t"
8154             "pminsd  $tmp2,$tmp\n\t"
8155             "movd    $dst,$tmp2\t! min reduction2I" %}
8156   ins_encode %{
8157     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x1);
8158     __ pminsd($tmp$$XMMRegister, $src2$$XMMRegister);
8159     __ movdl($tmp2$$XMMRegister, $src1$$Register);
8160     __ pminsd($tmp2$$XMMRegister, $tmp$$XMMRegister);
8161     __ movdl($dst$$Register, $tmp2$$XMMRegister);
8162   %}
8163   ins_pipe( pipe_slow );
8164 %}
8165 
8166 instruct rvmin2I_reduction_reg(rRegI dst, rRegI src1, vecD src2, regF tmp, regF tmp2) %{
8167   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
8168   match(Set dst (MinReductionV src1 src2));
8169   effect(TEMP tmp, TEMP tmp2);
8170   format %{ "pshufd   $tmp,$src2,0x1\n\t"
8171             "vpminsd  $tmp2,$tmp,$src2\n\t"
8172             "movd     $tmp,$src1\n\t"
8173             "vpminsd  $tmp2,$tmp,$tmp2\n\t"
8174             "movd     $dst,$tmp2\t! min reduction2I" %}
8175   ins_encode %{
8176     int vector_len = 0;
8177     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x1);
8178     __ vpminsd($tmp$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, vector_len);
8179     __ movdl($tmp2$$XMMRegister, $src1$$Register);
8180     __ vpminsd($tmp2$$XMMRegister,$tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
8181     __ movdl($dst$$Register, $tmp2$$XMMRegister);
8182   %}
8183   ins_pipe( pipe_slow );
8184 %}
8185 
8186 instruct rsmin4I_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp2) %{
8187   predicate(UseSSE > 3 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
8188   match(Set dst (MinReductionV src1 src2));
8189   effect(TEMP tmp, TEMP tmp2);
8190   format %{ "pshufd  $tmp,$src2,0xE\n\t"
8191             "pminsd  $tmp,$src2\n\t"
8192             "pshufd  $tmp2,$tmp,0x1\n\t"
8193             "pminsd  $tmp2,$tmp\n\t"
8194             "movd    $tmp,$src1\n\t"
8195             "pminsd  $tmp2,$tmp\n\t"
8196             "movd    $dst,$tmp2\t! min reduction4I" %}
8197   ins_encode %{
8198     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
8199     __ pminsd($tmp$$XMMRegister, $src2$$XMMRegister);
8200     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1);
8201     __ pminsd($tmp2$$XMMRegister,$tmp$$XMMRegister);
8202     __ movdl($tmp$$XMMRegister, $src1$$Register);
8203     __ pminsd($tmp2$$XMMRegister,$tmp$$XMMRegister);
8204     __ movdl($dst$$Register, $tmp2$$XMMRegister);
8205   %}
8206   ins_pipe( pipe_slow );
8207 %}
8208 
8209 instruct rvmin4I_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp2) %{
8210   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
8211   match(Set dst (MinReductionV src1 src2));
8212   effect(TEMP tmp, TEMP tmp2);
8213   format %{ "pshufd   $tmp,$src2,0xE\n\t"
8214             "vpminsd  $tmp2,$tmp,$src2\n\t"
8215             "pshufd   $tmp,$tmp2,0x1\n\t"
8216             "vpminsd  $tmp2,$tmp2,$tmp\n\t"
8217             "movd     $tmp,$src1\n\t"
8218             "vpminsd  $tmp2,$tmp,$tmp2\n\t"
8219             "movd     $dst,$tmp2\t! min reduction4I" %}
8220   ins_encode %{
8221     int vector_len = 0;
8222     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
8223     __ vpminsd($tmp2$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, vector_len);
8224     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister,0x1);
8225     __ vpminsd($tmp2$$XMMRegister,$tmp2$$XMMRegister, $tmp$$XMMRegister, vector_len);
8226     __ movdl($tmp$$XMMRegister, $src1$$Register);
8227     __ vpminsd($tmp2$$XMMRegister,$tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
8228     __ movdl($dst$$Register, $tmp2$$XMMRegister);
8229   %}
8230   ins_pipe( pipe_slow );
8231 %}
8232 
8233 instruct rvmin4I_reduction_evex(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp2) %{
8234   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
8235   match(Set dst (MinReductionV src1 src2));
8236   effect(TEMP tmp, TEMP tmp2);
8237   format %{ "pshufd   $tmp,$src2,0xE\n\t"
8238             "vpminsd  $tmp2,$tmp,$src2\n\t"
8239             "pshufd   $tmp,$tmp2,0x1\n\t"
8240             "vpminsd  $tmp2,$tmp2,$tmp\n\t"
8241             "movd     $tmp,$src1\n\t"
8242             "vpminsd  $tmp2,$tmp,$tmp2\n\t"
8243             "movd     $dst,$tmp2\t! min reduction4I" %}
8244   ins_encode %{
8245     int vector_len = 0;
8246     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
8247     __ vpminsd($tmp2$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, vector_len);
8248     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister,0x1);
8249     __ vpminsd($tmp2$$XMMRegister,$tmp2$$XMMRegister, $tmp$$XMMRegister, vector_len);
8250     __ movdl($tmp$$XMMRegister, $src1$$Register);
8251     __ vpminsd($tmp2$$XMMRegister,$tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
8252     __ movdl($dst$$Register, $tmp2$$XMMRegister);
8253   %}
8254   ins_pipe( pipe_slow );
8255 %}
8256 
8257 instruct rvmin8I_reduction_reg(rRegI dst, rRegI src1, vecY src2, regF tmp, regF tmp2) %{
8258   predicate(UseAVX > 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
8259   match(Set dst (MinReductionV src1 src2));
8260   effect(TEMP tmp, TEMP tmp2);
8261   format %{ "vextracti128_high   $tmp,$src2\n\t"
8262             "vpminsd  $tmp,$tmp,$src2\n\t"
8263             "pshufd   $tmp2,$tmp,0xE\n\t"
8264             "vpminsd  $tmp2,$tmp,$tmp2\n\t"
8265             "pshufd   $tmp,$tmp2,0x1\n\t"
8266             "vpminsd  $tmp2,$tmp2,$tmp\n\t"
8267             "movd     $tmp,$src1\n\t"
8268             "vpminsd  $tmp2,$tmp,$tmp2\n\t"
8269             "movd     $dst,$tmp2\t! min reduction8I" %}
8270   ins_encode %{
8271     int vector_len = 1;
8272     __ vextracti128_high($tmp$$XMMRegister, $src2$$XMMRegister);
8273     __ vpminsd($tmp$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, vector_len);
8274     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE);
8275     __ vpminsd($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
8276     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister,0x1);
8277     __ vpminsd($tmp2$$XMMRegister,$tmp2$$XMMRegister, $tmp$$XMMRegister, vector_len);
8278     __ movdl($tmp$$XMMRegister, $src1$$Register);
8279     __ vpminsd($tmp2$$XMMRegister,$tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
8280     __ movdl($dst$$Register, $tmp2$$XMMRegister);
8281   %}
8282   ins_pipe( pipe_slow );
8283 %}
8284 
8285 instruct rvmin8I_reduction_evex(rRegI dst, rRegI src1, vecY src2, regF tmp, regF tmp2) %{
8286   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
8287   match(Set dst (MinReductionV src1 src2));
8288   effect(TEMP tmp, TEMP tmp2);
8289   format %{ "vextracti128_high   $tmp,$src2\n\t"
8290             "vpminsd  $tmp,$tmp,$src2\n\t"
8291             "pshufd   $tmp2,$tmp,0xE\n\t"
8292             "vpminsd  $tmp2,$tmp,$tmp2\n\t"
8293             "pshufd   $tmp,$tmp2,0x1\n\t"
8294             "vpminsd  $tmp2,$tmp2,$tmp\n\t"
8295             "movd     $tmp,$src1\n\t"
8296             "vpminsd  $tmp2,$tmp,$tmp2\n\t"
8297             "movd     $dst,$tmp2\t! min reduction8I" %}
8298   ins_encode %{
8299     int vector_len = 1;
8300     __ vextracti128_high($tmp$$XMMRegister, $src2$$XMMRegister);
8301     __ vpminsd($tmp$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, vector_len);
8302     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE);
8303     __ vpminsd($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
8304     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister,0x1);
8305     __ vpminsd($tmp2$$XMMRegister,$tmp2$$XMMRegister, $tmp$$XMMRegister, vector_len);
8306     __ movdl($tmp$$XMMRegister, $src1$$Register);
8307     __ vpminsd($tmp2$$XMMRegister,$tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
8308     __ movdl($dst$$Register, $tmp2$$XMMRegister);
8309   %}
8310   ins_pipe( pipe_slow );
8311 %}
8312 
8313 instruct rvmin16I_reduction_evex(rRegI dst, rRegI src1, vecZ src2, regF tmp, regF tmp2, regF tmp3) %{
8314   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
8315   match(Set dst (MinReductionV src1 src2));
8316   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
8317   format %{ "vextracti64x4_high  $tmp3,$src2\n\t"
8318             "vpminsd  $tmp3,$tmp3,$src2\n\t"
8319             "vextracti128_high   $tmp,$tmp3\n\t"
8320             "vpminsd  $tmp,$tmp,$tmp3\n\t"
8321             "pshufd   $tmp2,$tmp,0xE\n\t"
8322             "vpminsd  $tmp2,$tmp,$tmp2\n\t"
8323             "pshufd   $tmp,$tmp2,0x1\n\t"
8324             "vpminsd  $tmp2,$tmp2,$tmp\n\t"
8325             "movd     $tmp,$src1\n\t"
8326             "vpminsd  $tmp2,$tmp,$tmp2\n\t"
8327             "movd     $dst,$tmp2\t! min reduction16I" %}
8328   ins_encode %{
8329     int vector_len = 2;
8330     __ vextracti64x4_high($tmp3$$XMMRegister, $src2$$XMMRegister);
8331     __ vpminsd($tmp3$$XMMRegister, $tmp3$$XMMRegister, $src2$$XMMRegister, vector_len);
8332     __ vextracti128_high($tmp$$XMMRegister, $tmp3$$XMMRegister);
8333     __ vpminsd($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp3$$XMMRegister, vector_len);
8334     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE);
8335     __ vpminsd($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
8336     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister,0x1);
8337     __ vpminsd($tmp2$$XMMRegister,$tmp2$$XMMRegister, $tmp$$XMMRegister, vector_len);
8338     __ movdl($tmp$$XMMRegister, $src1$$Register);
8339     __ vpminsd($tmp2$$XMMRegister,$tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
8340     __ movdl($dst$$Register, $tmp2$$XMMRegister);
8341   %}
8342   ins_pipe( pipe_slow );
8343 %}
8344 
8345 // Long Min Reduction
8346 instruct rsmin1L_reduction_reg(rRegL dst, rRegL src1, vecD src2, rxmm0 tmp, regF tmp2) %{
8347   predicate(UseSSE > 3 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
8348   match(Set dst (MinReductionV src1 src2));
8349   effect(TEMP tmp, TEMP tmp2);
8350   format %{ "movdq      $tmp,$src1\n\t"
8351             "movdq      $tmp2,$src1\n\t"
8352             "pcmpgtq   $tmp,$src2\n\t"
8353             "blendvpd  $tmp2,$src2\n\t"
8354             "movdq      $dst,$tmp2\t! min reduction1L" %}
8355   ins_encode %{
8356     __ movdq($tmp$$XMMRegister,$src1$$Register);
8357     __ movdq($tmp2$$XMMRegister,$src1$$Register);
8358     __ pcmpgtq($tmp$$XMMRegister, $src2$$XMMRegister);
8359     __ blendvpd($tmp2$$XMMRegister,$src2$$XMMRegister);
8360     __ movdq($dst$$Register, $tmp2$$XMMRegister);
8361   %}
8362   ins_pipe( pipe_slow );
8363 %}
8364 
8365 instruct rsmin2L_reduction_reg(rRegL dst, rRegL src1, vecX src2, rxmm0 xmm_0, regF tmp2, regF tmp3) %{
8366   predicate(UseSSE > 3 && UseAVX == 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
8367   match(Set dst (MinReductionV src1 src2));
8368   effect(TEMP xmm_0, TEMP tmp2, TEMP tmp3);
8369   format %{ "pshufd  $tmp3,$src2,0xE\n\t"
8370             "movdqu  $xmm_0,$src2\n\t"
8371             "movdqu  $tmp2,$src2\n\t"
8372             "pcmpgtq  $xmm_0,$tmp3\n\t"
8373             "blendvpd  $tmp2,$tmp3\n\t"
8374             "movdqu  $xmm_0,$tmp2\n\t"
8375             "movdq  $tmp3,$src1\n\t"
8376             "pcmpgtq  $xmm_0,$tmp3\n\t"
8377             "blendvpd  $tmp2,$tmp3\n\t"
8378             "movq  $dst,$tmp2\t! min reduction2L" %}
8379   ins_encode %{
8380     __ pshufd($tmp3$$XMMRegister, $src2$$XMMRegister, 0xE);
8381     __ movdqu($xmm_0$$XMMRegister, $src2$$XMMRegister);
8382     __ movdqu($tmp2$$XMMRegister, $src2$$XMMRegister);
8383     __ pcmpgtq($xmm_0$$XMMRegister, $tmp3$$XMMRegister);
8384     __ blendvpd($tmp2$$XMMRegister, $tmp3$$XMMRegister);
8385     __ movdqu($xmm_0$$XMMRegister, $tmp2$$XMMRegister);
8386     __ movdq($tmp3$$XMMRegister, $src1$$Register);
8387     __ pcmpgtq($xmm_0$$XMMRegister, $tmp3$$XMMRegister);
8388     __ blendvpd($tmp2$$XMMRegister,$tmp3$$XMMRegister);
8389     __ movdq($dst$$Register, $tmp2$$XMMRegister);
8390   %}
8391   ins_pipe( pipe_slow );
8392 %}
8393 
8394 instruct rvmin2L_reduction_reg(rRegL dst, rRegL src1, vecX src2, regF tmp, regF tmp2, regF tmp3) %{
8395   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
8396   match(Set dst (MinReductionV src1 src2));
8397   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
8398   format %{ "pshufd   $tmp2,$src2,0xE\n\t"
8399             "vpcmpgtq  $tmp,$tmp2,$src2\n\t"
8400             "vblendvpd   $tmp2,$tmp2,$src2,$tmp\n\t"
8401             "movq     $tmp,$src1\n\t"
8402             "vpcmpgtq  $tmp3,$tmp2,$tmp\n\t"
8403             "blendvpd   $tmp2,$tmp2,$src1,$tmp3\n\t"
8404             "movq     $dst,$tmp2\t! min reduction2L" %}
8405   ins_encode %{
8406     int vector_len = 0;
8407     __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0xE);
8408     __ vpcmpgtq($tmp$$XMMRegister, $tmp2$$XMMRegister, $src2$$XMMRegister, vector_len);
8409     __ vblendvpd($tmp2$$XMMRegister, $tmp2$$XMMRegister, $src2$$XMMRegister, $tmp$$XMMRegister, vector_len);
8410     __ movdq($tmp$$XMMRegister,$src1$$Register);
8411     __ vpcmpgtq($tmp3$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, vector_len);
8412     __ vblendvpd($tmp2$$XMMRegister, $tmp2$$XMMRegister,$src1$$XMMRegister,$tmp3$$XMMRegister, vector_len);
8413     __ movdq($dst$$Register, $tmp2$$XMMRegister);
8414   %}
8415   ins_pipe( pipe_slow );
8416 %}
8417 
8418 instruct rvmin4L_reduction_reg(rRegL dst, rRegL src1, vecY src2, regF tmp, regF tmp2, regF tmp3) %{
8419   predicate(UseAVX > 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
8420   match(Set dst (MinReductionV src1 src2));
8421   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
8422   format %{ "vextracti128_high   $tmp2,$src2\n\t"
8423             "vpcmpgtq  $tmp,$tmp2,$src2\n\t"
8424             "vblendvpd   $tmp2,$tmp2,$src2,$tmp\n\t"
8425             "vpshufd   $tmp3, $tmp2,0x1\n\t"
8426             "vpcmpgtq  $tmp, $tmp3,$tmp\n\t2"
8427             "vblendvpd $tmp3,$tmp3,$tmp2,$tmp\n\t"
8428             "movq     $tmp2,$src1\n\t"
8429             "vpcmpgtq  $tmp,$tmp2,$tmp3\n\t"
8430             "blendvpd   $tmp2,$tmp2,$tmp3,$tmp\n\t"
8431             "movq     $dst,$tmp2\t! min reduction2L" %}
8432   ins_encode %{
8433     int vector_len = 1;
8434     __ vextracti128_high($tmp2$$XMMRegister, $src2$$XMMRegister);
8435     __ vpcmpgtq($tmp$$XMMRegister, $tmp2$$XMMRegister, $src2$$XMMRegister, vector_len);
8436     __ vblendvpd($tmp2$$XMMRegister, $tmp2$$XMMRegister, $src2$$XMMRegister, $tmp$$XMMRegister, vector_len);
8437     __ pshufd($tmp3$$XMMRegister, $tmp2$$XMMRegister, 0xE);
8438     __ vpcmpgtq($tmp$$XMMRegister, $tmp3$$XMMRegister, $tmp2$$XMMRegister, vector_len);
8439     __ vblendvpd($tmp3$$XMMRegister,$tmp3$$XMMRegister,$tmp2$$XMMRegister, $tmp$$XMMRegister, vector_len);
8440     __ movdq($tmp$$XMMRegister,$src1$$Register);
8441     __ vpcmpgtq($tmp2$$XMMRegister, $tmp3$$XMMRegister, $tmp$$XMMRegister, vector_len);
8442     __ vblendvpd($tmp2$$XMMRegister, $tmp3$$XMMRegister,$tmp$$XMMRegister,$tmp2$$XMMRegister, vector_len);
8443     __ movdq($dst$$Register, $tmp2$$XMMRegister);
8444   %}
8445   ins_pipe( pipe_slow );
8446 %}
8447 
8448 instruct rvmin8L_reduction_reg(rRegL dst, rRegL src1, vecZ src2, regF tmp, regF tmp2, regF tmp3) %{
8449   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
8450   match(Set dst (MinReductionV src1 src2));
8451   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
8452   format %{ "vextracti64x4_high  $tmp3,$src2\n\t"
8453             "vpcmpgtq  $tmp,$tmp3,$src2\n\t"
8454             "vblendvpd   $tmp3,$tmp3,$src2,$tmp\n\t"
8455             "vextracti128_high   $tmp2,$tmp3\n\t"
8456             "vpcmpgtq  $tmp,$tmp2,$tmp3\n\t"
8457             "vblendvpd   $tmp2,$tmp2,$tmp3,$tmp\n\t"
8458             "vpshufd  $tmp3,$tmp2,0x1\n\t"
8459             "vpcmpgtq   $tmp,$tmp3,$tmp2\n\t"
8460             "vblendvpd  $tmp3,$tmp3,$tmp2,$tmp\n\t"
8461             "movq     $tmp2,$src1\n\t"
8462             "vpcmpgtq  $tmp,$tmp2,$tmp3\n\t"
8463             "vblendvpd  $tmp2,$tmp2,$tmp3,$tmp\n\t"
8464             "movq     $dst,$tmp2\t! min reduction4I" %}
8465   ins_encode %{
8466     __ vextracti64x4_high($tmp3$$XMMRegister, $src2$$XMMRegister);
8467     __ vpcmpgtq($tmp$$XMMRegister, $tmp3$$XMMRegister, $src2$$XMMRegister, 1);
8468     __ vblendvpd($tmp3$$XMMRegister, $tmp3$$XMMRegister, $src2$$XMMRegister, $tmp$$XMMRegister, 1);
8469     __ vextracti128_high($tmp2$$XMMRegister, $tmp3$$XMMRegister);
8470     __ vpcmpgtq($tmp$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister, 1);
8471     __ vblendvpd($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister, $tmp$$XMMRegister, 1);
8472     __ pshufd($tmp3$$XMMRegister, $tmp2$$XMMRegister, 0xE);
8473     __ vpcmpgtq($tmp$$XMMRegister, $tmp3$$XMMRegister, $tmp2$$XMMRegister, 1);
8474     __ vblendvpd($tmp3$$XMMRegister,$tmp3$$XMMRegister,$tmp2$$XMMRegister, $tmp$$XMMRegister, 1);
8475     __ movdq($tmp2$$XMMRegister, $src1$$Register);
8476     __ vpcmpgtq($tmp$$XMMRegister,$tmp2$$XMMRegister, $tmp3$$XMMRegister, 1);
8477     __ vblendvpd($tmp2$$XMMRegister,$tmp2$$XMMRegister, $tmp3$$XMMRegister, $tmp$$XMMRegister, 1);
8478     __ movdq($dst$$Register, $tmp2$$XMMRegister);
8479   %}
8480   ins_pipe( pipe_slow );
8481 %}
8482 
8483 // Float Min Reduction
8484 instruct rsmin2F_reduction_reg(regF dst, vecD src2, regF tmp) %{
8485   predicate(UseSSE > 0 && UseAVX == 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
8486   match(Set dst (MinReductionV dst src2));
8487   effect(TEMP dst, TEMP tmp);
8488   format %{ "minps   $dst, $src2\n\t"
8489             "pshufd  $tmp,$src2,0x1\n\t"
8490             "minps   $dst,$tmp\t! min reduction2F" %}
8491   ins_encode %{
8492     __ minps($dst$$XMMRegister, $src2$$XMMRegister);
8493     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x1);
8494     __ minps($dst$$XMMRegister, $tmp$$XMMRegister);
8495   %}
8496   ins_pipe( pipe_slow );
8497 %}
8498 
8499 instruct rvmin2F_reduction_reg(regF dst, vecD src2, regF tmp) %{
8500   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
8501   match(Set dst (MinReductionV dst src2));
8502   effect(TEMP dst, TEMP tmp);
8503   format %{ "vminps  $dst,$dst,$src2\n\t"
8504             "pshufd  $tmp,$src2,0x1\n\t"
8505             "vminps  $dst,$dst,$tmp\t! min reduction2F" %}
8506   ins_encode %{
8507     int vector_len = 0;
8508     __ vminps($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister, vector_len);
8509     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x1);
8510     __ vminps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
8511   %}
8512   ins_pipe( pipe_slow );
8513 %}
8514 
8515 instruct rsmin4F_reduction_reg(regF dst, vecX src2, regF tmp) %{
8516   predicate(UseSSE > 0 && UseAVX == 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
8517   match(Set dst (MinReductionV dst src2));
8518   effect(TEMP tmp, TEMP dst);
8519   format %{ "minps  $dst,$src2\n\t"
8520             "pshufd  $tmp,$src2,0x1\n\t"
8521             "minps  $dst,tmp\n\t"
8522             "pshufd  $tmp,$src2,0x2\n\t"
8523             "minps  $dst,tmp\n\t"
8524             "pshufd  $tmp,$src2,0x3\n\t"
8525             "minps  $dst,$tmp\t! min reduction4F" %}
8526   ins_encode %{
8527     int vector_len = 0;
8528     __ minps($dst$$XMMRegister, $src2$$XMMRegister);
8529     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x1);
8530     __ minps($dst$$XMMRegister, $tmp$$XMMRegister);
8531     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x2);
8532     __ minps($dst$$XMMRegister, $tmp$$XMMRegister);
8533     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x3);
8534     __ minps($dst$$XMMRegister, $tmp$$XMMRegister);
8535   %}
8536   ins_pipe( pipe_slow );
8537 %}
8538 
8539 instruct rvmin4F_reduction_reg(regF dst, vecX src2, regF tmp) %{
8540   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
8541   match(Set dst (MinReductionV dst src2));
8542   effect(TEMP tmp, TEMP dst);
8543   format %{ "vminps  $dst,$dst,$src2\n\t"
8544             "pshufd  $tmp,$src2,0x1\n\t"
8545             "vminps  $dst,$dst,tmp\n\t"
8546             "pshufd  $tmp,$src2,0x2\n\t"
8547             "vminps  $dst,$dst,tmp\n\t"
8548             "pshufd  $tmp,$src2,0x3\n\t"
8549             "vminps  $dst,$dst,$tmp\t! min reduction4F" %}
8550   ins_encode %{
8551     int vector_len = 0;
8552     __ vminps($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister, vector_len);
8553     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x1);
8554     __ vminps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
8555     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x2);
8556     __ vminps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
8557     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x3);
8558     __ vminps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
8559   %}
8560   ins_pipe( pipe_slow );
8561 %}
8562 
8563 instruct rvmin8F_reduction_reg(regF dst, vecY src2, regF tmp, regF tmp2) %{
8564   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
8565   match(Set dst (MinReductionV dst src2));
8566   effect(TEMP tmp, TEMP dst, TEMP tmp2);
8567   format %{ "vminps  $dst,$dst,$src2\n\t"
8568             "pshufd  $tmp,$src2,0x01\n\t"
8569             "vminps  $dst,$dst,$tmp\n\t"
8570             "pshufd  $tmp,$src2,0x02\n\t"
8571             "vminps  $dst,$dst,$tmp\n\t"
8572             "pshufd  $tmp,$src2,0x03\n\t"
8573             "vminps  $dst,$dst,$tmp\n\t"
8574             "vextractf128_high  $tmp2,$src2\n\t"
8575             "vminps  $dst,$dst,$tmp2\n\t"
8576             "pshufd  $tmp,$tmp2,0x01\n\t"
8577             "vminps  $dst,$dst,$tmp\n\t"
8578             "pshufd  $tmp,$tmp2,0x02\n\t"
8579             "vminps  $dst,$dst,$tmp\n\t"
8580             "pshufd  $tmp,$tmp2,0x03\n\t"
8581             "vminps  $dst,$dst,$tmp\t! sub reduction8F" %}
8582   ins_encode %{
8583     int vector_len = 1;
8584     __ vminps($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister, vector_len);
8585     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x01);
8586     __ vminps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
8587     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x02);
8588     __ vminps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
8589     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x03);
8590     __ vminps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
8591     __ vextractf128_high($tmp2$$XMMRegister, $src2$$XMMRegister);
8592     __ vminps($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister, vector_len);
8593     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x01);
8594     __ vminps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
8595     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x02);
8596     __ vminps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
8597     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x03);
8598     __ vminps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
8599   %}
8600   ins_pipe( pipe_slow );
8601 %}
8602 
8603 instruct rvmin16F_reduction_reg(regF dst, vecZ src2, regF tmp, regF tmp2) %{
8604   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
8605   match(Set dst (MinReductionV dst src2));
8606   effect(TEMP tmp, TEMP dst, TEMP tmp2);
8607   format %{ "vminps  $dst,$dst,$src2\n\t"
8608             "pshufd  $tmp,$src2,0x01\n\t"
8609             "vminps  $dst,$dst,$tmp\n\t"
8610             "pshufd  $tmp,$src2,0x02\n\t"
8611             "vminps  $dst,$dst,$tmp\n\t"
8612             "pshufd  $tmp,$src2,0x03\n\t"
8613             "vminps  $dst,$dst,$tmp\n\t"
8614             "vextractf32x4  $tmp2,$src2,0x1\n\t"
8615             "vminps  $dst,$dst,$tmp2\n\t"
8616             "pshufd  $tmp,$tmp2,0x01\n\t"
8617             "vminps  $dst,$dst,$tmp\n\t"
8618             "pshufd  $tmp,$tmp2,0x02\n\t"
8619             "vminps  $dst,$dst,$tmp\n\t"
8620             "pshufd  $tmp,$tmp2,0x03\n\t"
8621             "vminps  $dst,$dst,$tmp\n\t"
8622             "vextractf32x4  $tmp2,$src2,0x2\n\t"
8623             "vminps  $dst,$dst,$tmp2\n\t"
8624             "pshufd  $tmp,$tmp2,0x01\n\t"
8625             "vminps  $dst,$dst,$tmp\n\t"
8626             "pshufd  $tmp,$tmp2,0x02\n\t"
8627             "vminps  $dst,$dst,$tmp\n\t"
8628             "pshufd  $tmp,$tmp2,0x03\n\t"
8629             "vminps  $dst,$dst,$tmp\n\t"
8630             "vextractf32x4  $tmp2,$src2,0x3\n\t"
8631             "vminps  $dst,$dst,$tmp2\n\t"
8632             "pshufd  $tmp,$tmp2,0x01\n\t"
8633             "vminps  $dst,$dst,$tmp\n\t"
8634             "pshufd  $tmp,$tmp2,0x02\n\t"
8635             "vminps  $dst,$dst,$tmp\n\t"
8636             "pshufd  $tmp,$tmp2,0x03\n\t"
8637             "vminps  $dst,$dst,$tmp\t! sub reduction16F" %}
8638   ins_encode %{
8639     int vector_len = 2;
8640     __ vminps($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister, vector_len);
8641     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x01);
8642     __ vminps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
8643     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x02);
8644     __ vminps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
8645     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x03);
8646     __ vminps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
8647     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x1);
8648     __ vminps($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister, vector_len);
8649     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x01);
8650     __ vminps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
8651     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x02);
8652     __ vminps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
8653     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x03);
8654     __ vminps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
8655     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x2);
8656     __ vminps($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister, vector_len);
8657     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x01);
8658     __ vminps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
8659     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x02);
8660     __ vminps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
8661     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x03);
8662     __ vminps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
8663     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x3);
8664     __ vminps($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister, vector_len);
8665     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x01);
8666     __ vminps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
8667     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x02);
8668     __ vminps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
8669     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x03);
8670     __ vminps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
8671   %}
8672   ins_pipe( pipe_slow );
8673 %}
8674 
8675 instruct rsmin2D_reduction_reg(regD dst, vecX src2, regD tmp) %{
8676   predicate(UseSSE >= 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
8677   match(Set dst (MinReductionV dst src2));
8678   effect(TEMP tmp, TEMP dst);
8679   format %{ "minpd   $dst,$src2\n\t"
8680             "pshufd  $tmp,$src2,0xE\n\t"
8681             "minpd   $dst,$tmp\t! min reduction2D" %}
8682   ins_encode %{
8683     __ minpd($dst$$XMMRegister, $src2$$XMMRegister);
8684     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
8685     __ minpd($dst$$XMMRegister, $tmp$$XMMRegister);
8686   %}
8687   ins_pipe( pipe_slow );
8688 %}
8689 
8690 instruct rvmin2D_reduction_reg(regD dst, vecX src2, regD tmp) %{
8691   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
8692   match(Set dst (MinReductionV dst src2));
8693   effect(TEMP tmp, TEMP dst);
8694   format %{ "vminpd  $dst,$dst,$src2\n\t"
8695             "pshufd  $tmp,$src2,0xE\n\t"
8696             "vminpd  $dst,$dst,$tmp\t! min reduction2D" %}
8697   ins_encode %{
8698     int vector_len = 0;
8699     __ vminpd($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister, vector_len);
8700     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
8701     __ vminpd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
8702   %}
8703   ins_pipe( pipe_slow );
8704 %}
8705 
8706 instruct rvmin4D_reduction_reg(regD dst, vecY src2, regD tmp, regD tmp2) %{
8707   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
8708   match(Set dst (MinReductionV dst src2));
8709   effect(TEMP tmp, TEMP dst, TEMP tmp2);
8710   format %{ "vminpd  $dst,$dst,$src2\n\t"
8711             "pshufd  $tmp,$src2,0xE\n\t"
8712             "vminpd  $dst,$dst,$tmp\n\t"
8713             "vextractf32x4  $tmp2,$src2,0x1\n\t"
8714             "vminpd  $dst,$dst,$tmp2\n\t"
8715             "pshufd  $tmp,$tmp2,0xE\n\t"
8716             "vminpd  $dst,$dst,$tmp\t! min reduction4D" %}
8717   ins_encode %{
8718     int vector_len = 1;
8719     __ vminpd($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister, vector_len);
8720     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
8721     __ vminpd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
8722     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x1);
8723     __ vminpd($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister, vector_len);
8724     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE);
8725     __ vminpd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
8726   %}
8727   ins_pipe( pipe_slow );
8728 %}
8729 
8730 instruct rvmin8D_reduction_reg(regD dst, vecZ src2, regD tmp, regD tmp2) %{
8731   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
8732   match(Set dst (MinReductionV dst src2));
8733   effect(TEMP tmp, TEMP dst, TEMP tmp2);
8734   format %{ "vminpd  $dst,$dst,$src2\n\t"
8735             "pshufd  $tmp,$src2,0xE\n\t"
8736             "vminpd  $dst,$dst,$tmp\n\t"
8737             "vextractf32x4  $tmp2,$src2,0x1\n\t"
8738             "vminpd  $dst,$dst,$tmp2\n\t"
8739             "pshufd  $tmp,$tmp2,0xE\n\t"
8740             "vminpd  $dst,$dst,$tmp\n\t"
8741             "vextractf32x4  $tmp2,$src2,0x2\n\t"
8742             "vminpd  $dst,$dst,$tmp2\n\t"
8743             "pshufd  $tmp,$tmp2,0xE\n\t"
8744             "vminpd  $dst,$dst,$tmp\n\t"
8745             "vextractf32x4  $tmp2,$src2,0x3\n\t"
8746             "vminpd  $dst,$dst,$tmp2\n\t"
8747             "pshufd  $tmp,$tmp2,0xE\n\t"
8748             "vminpd  $dst,$dst,$tmp\t! min reduction8D" %}
8749   ins_encode %{
8750     int vector_len = 2;
8751     __ vminpd($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister, vector_len);
8752     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
8753     __ vminpd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
8754     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x1);
8755     __ vminpd($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister, vector_len);
8756     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE);
8757     __ vminpd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
8758     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x2);
8759     __ vminpd($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister, vector_len);
8760     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE);
8761     __ vminpd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
8762     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x3);
8763     __ vminpd($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister, vector_len);
8764     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE);
8765     __ vminpd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
8766   %}
8767   ins_pipe( pipe_slow );
8768 %}
8769 
8770 // ------- Max Reduction ------------
8771 
8772 instruct rsmax8B_reduction_reg(rRegI dst, rRegI src1, vecD src2, regF tmp, rRegI tmp2, rRegI tmp3, regF tmp4) %{
8773   predicate(UseSSE > 3 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
8774   match(Set dst (MaxReductionV src1 src2));
8775   effect(TEMP dst, TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP tmp4);
8776   format %{ "pshufd  $tmp,$src2,0x1\n\t"
8777             "pminsb  $tmp,$src2\n\t"
8778             "pextrb  $tmp2,$tmp, 0x1\n\t"
8779             "movsbl  $tmp2,$tmp2\n\t"
8780             "pextrb  $tmp3,$tmp,0x0\n\t"
8781             "movsbl  $tmp3,$tmp3\n\t"
8782             "cmpl  $tmp2,$tmp3\n\t"
8783             "cmovl  $tmp3,$tmp2\n\t"
8784             "cmpl  $src1,$tmp3\n\t"
8785             "cmovl  $tmp3,$src1, 0x0\n\t"
8786             "movl  $dst,$tmp2\n\t"
8787             "pextrb  $tmp2,$tmp\n\t"
8788             "movsbl  $tmp2,$tmp2\n\t"
8789             "pextrb  $tmp3,$tmp\n\t"
8790             "movsbl  $tmp3,$tmp3\n\t"
8791             "cmpl  $tmp2,$tmp3\n\t"
8792             "cmovl  $tmp3,$tmp2\n\t"
8793             "cmpl  $tmp3,$dst\n\t"
8794             "cmovl  $dst,$tmp3\t! min reduction4S" %}
8795   ins_encode %{
8796     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister,0x1);
8797     __ pmaxsb($tmp$$XMMRegister, $src2$$XMMRegister);
8798     __ pextrb($tmp2$$Register, $tmp$$XMMRegister,0x1);
8799     __ movsbl($tmp2$$Register, $tmp2$$Register);
8800     __ pextrb($tmp3$$Register, $tmp$$XMMRegister,0x0);
8801     __ movsbl($tmp3$$Register, $tmp3$$Register);
8802     __ cmpl($tmp2$$Register, $tmp3$$Register);
8803     __ cmovl(Assembler::greater, $tmp3$$Register, $tmp2$$Register);
8804     __ cmpl($src1$$Register, $tmp3$$Register);
8805     __ cmovl(Assembler::greater, $tmp3$$Register, $src1$$Register);
8806     __ movl($dst$$Register, $tmp3$$Register);
8807     __ pextrb($tmp2$$Register, $tmp$$XMMRegister,0x3);
8808     __ movsbl($tmp2$$Register, $tmp2$$Register);
8809     __ pextrb($tmp3$$Register, $tmp$$XMMRegister,0x2);
8810     __ movsbl($tmp3$$Register, $tmp3$$Register);
8811     __ cmpl($tmp2$$Register, $tmp3$$Register);
8812     __ cmovl(Assembler::greater, $tmp3$$Register, $tmp2$$Register);
8813     __ cmpl($tmp3$$Register, $dst$$Register);
8814     __ cmovl(Assembler::greater, $dst$$Register, $tmp3$$Register);
8815     __ movsbl($dst$$Register, $dst$$Register);
8816   %}
8817   ins_pipe( pipe_slow );
8818 %}
8819 
8820 instruct rsmax16B_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, rRegI tmp2, rRegI tmp3, regF tmp4) %{
8821   predicate(UseSSE > 3 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
8822   match(Set dst (MaxReductionV src1 src2));
8823   effect(TEMP dst, TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP tmp4);
8824   format %{ "pshufd  $tmp4,$src2,0xE\n\t"
8825             "pmaxsb  $tmp4,$src2\n\t"
8826             "pshufd  $tmp,$tmp4,0x1\n\t"
8827             "pmaxsb  $tmp,$tmp4\n\t"
8828             "pextrb  $tmp2,$tmp, 0x1\n\t"
8829             "movsbl  $tmp2,$tmp2\n\t"
8830             "pextrb  $tmp3,$tmp,0x0\n\t"
8831             "movsbl  $tmp3,$tmp3\n\t"
8832             "cmpl  $tmp2,$tmp3\n\t"
8833             "cmovl  $tmp3,$tmp2\n\t"
8834             "cmpl  $src1,$tmp3\n\t"
8835             "cmovl  $tmp3,$src1, 0x0\n\t"
8836             "movl  $dst,$tmp2\n\t"
8837             "pextrb  $tmp2,$tmp\n\t"
8838             "movsbl  $tmp2,$tmp2\n\t"
8839             "pextrb  $tmp3,$tmp\n\t"
8840             "movsbl  $tmp3,$tmp3\n\t"
8841             "cmpl  $tmp2,$tmp3\n\t"
8842             "cmovl  $tmp3,$tmp2\n\t"
8843             "cmpl  $tmp3,$dst\n\t"
8844             "cmovl  $dst,$tmp3\t! max reduction4S" %}
8845   ins_encode %{
8846     __ pshufd($tmp4$$XMMRegister, $src2$$XMMRegister, 0xE);
8847     __ pmaxsb($tmp4$$XMMRegister, $src2$$XMMRegister);
8848     __ pshufd($tmp$$XMMRegister, $tmp4$$XMMRegister,0x1);
8849     __ pmaxsb($tmp$$XMMRegister, $tmp4$$XMMRegister);
8850     __ pextrb($tmp2$$Register, $tmp$$XMMRegister,0x1);
8851     __ movsbl($tmp2$$Register, $tmp2$$Register);
8852     __ pextrb($tmp3$$Register, $tmp$$XMMRegister,0x0);
8853     __ movsbl($tmp3$$Register, $tmp3$$Register);
8854     __ cmpl($tmp2$$Register, $tmp3$$Register);
8855     __ cmovl(Assembler::greater, $tmp3$$Register, $tmp2$$Register);
8856     __ cmpl($src1$$Register, $tmp3$$Register);
8857     __ cmovl(Assembler::greater, $tmp3$$Register, $src1$$Register);
8858     __ movl($dst$$Register, $tmp3$$Register);
8859     __ pextrb($tmp2$$Register, $tmp$$XMMRegister,0x3);
8860     __ movsbl($tmp2$$Register, $tmp2$$Register);
8861     __ pextrb($tmp3$$Register, $tmp$$XMMRegister,0x2);
8862     __ movsbl($tmp3$$Register, $tmp3$$Register);
8863     __ cmpl($tmp2$$Register, $tmp3$$Register);
8864     __ cmovl(Assembler::greater, $tmp3$$Register, $tmp2$$Register);
8865     __ cmpl($tmp3$$Register, $dst$$Register);
8866     __ cmovl(Assembler::greater, $dst$$Register, $tmp3$$Register);
8867     __ movsbl($dst$$Register, $dst$$Register);
8868   %}
8869   ins_pipe( pipe_slow );
8870 %}
8871 
8872 instruct rvmax16B_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, rRegI tmp2, rRegI tmp3, regF tmp4) %{
8873   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
8874   match(Set dst (MaxReductionV src1 src2));
8875   effect(TEMP dst, TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP tmp4);
8876   format %{ "pshufd  $tmp4,$src2,0xE\n\t"
8877             "vpmaxsb  $tmp,$tmp4,$src2\n\t"
8878             "pshufd  $tmp,$tmp4,0x1\n\t"
8879             "vpmaxsb  $tmp,$tmp,$tmp4\n\t"
8880             "pextrb  $tmp2,$tmp, 0x1\n\t"
8881             "movsbl  $tmp2,$tmp2\n\t"
8882             "pextrb  $tmp3,$tmp,0x0\n\t"
8883             "movsbl  $tmp3,$tmp3\n\t"
8884             "cmpl  $tmp2,$tmp3\n\t"
8885             "cmovl  $tmp3,$tmp2\n\t"
8886             "cmpl  $src1,$tmp3\n\t"
8887             "cmovl  $tmp3,$src1, 0x0\n\t"
8888             "movl  $dst,$tmp2\n\t"
8889             "pextrb  $tmp2,$tmp\n\t"
8890             "movsbl  $tmp2,$tmp2\n\t"
8891             "pextrb  $tmp3,$tmp\n\t"
8892             "movsbl  $tmp3,$tmp3\n\t"
8893             "cmpl  $tmp2,$tmp3\n\t"
8894             "cmovl  $tmp3,$tmp2\n\t"
8895             "cmpl  $tmp3,$dst\n\t"
8896             "cmovl  $dst,$tmp3\t! max reduction4S" %}
8897   ins_encode %{
8898     int vector_len = 0;
8899     __ pshufd($tmp4$$XMMRegister, $src2$$XMMRegister, 0xE);
8900     __ vpmaxsb($tmp4$$XMMRegister, $tmp4$$XMMRegister, $src2$$XMMRegister, 0);
8901     __ pshufd($tmp$$XMMRegister, $tmp4$$XMMRegister,0x1);
8902     __ vpmaxsb($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp4$$XMMRegister, 0);
8903     __ pextrb($tmp2$$Register, $tmp$$XMMRegister,0x1);
8904     __ movsbl($tmp2$$Register, $tmp2$$Register);
8905     __ pextrb($tmp3$$Register, $tmp$$XMMRegister,0x0);
8906     __ movsbl($tmp3$$Register, $tmp3$$Register);
8907     __ cmpl($tmp2$$Register, $tmp3$$Register);
8908     __ cmovl(Assembler::greater, $tmp3$$Register, $tmp2$$Register);
8909     __ cmpl($src1$$Register, $tmp3$$Register);
8910     __ cmovl(Assembler::greater, $tmp3$$Register, $src1$$Register);
8911     __ movl($dst$$Register, $tmp3$$Register);
8912     __ pextrb($tmp2$$Register, $tmp$$XMMRegister,0x3);
8913     __ movsbl($tmp2$$Register, $tmp2$$Register);
8914     __ pextrb($tmp3$$Register, $tmp$$XMMRegister,0x2);
8915     __ movsbl($tmp3$$Register, $tmp3$$Register);
8916     __ cmpl($tmp2$$Register, $tmp3$$Register);
8917     __ cmovl(Assembler::greater, $tmp3$$Register, $tmp2$$Register);
8918     __ cmpl($tmp3$$Register, $dst$$Register);
8919     __ cmovl(Assembler::greater, $dst$$Register, $tmp3$$Register);
8920     __ movsbl($dst$$Register, $dst$$Register);
8921   %}
8922   ins_pipe( pipe_slow );
8923 %}
8924 
8925 instruct rvmax32B_reduction_reg(rRegI dst, rRegI src1, vecY src2, regF tmp, rRegI tmp2, rRegI tmp3, regF tmp4) %{
8926   predicate(UseAVX > 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
8927   match(Set dst (MaxReductionV src1 src2));
8928   effect(TEMP dst, TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP tmp4);
8929   format %{ "vextracti128_high  $tmp,$src2\n\t"
8930             "vpmaxsb  $tmp,$tmp,$src2\n\t"
8931             "pshufd  $tmp4,$tmp,0xE\n\t"
8932             "vpmaxsb  $tmp4,$tmp4,$tmp\n\t"
8933             "pshufd  $tmp,$tmp4,0x1\n\t"
8934             "vpmaxsb  $tmp,$tmp,$tmp4\n\t"
8935             "pextrb  $tmp2,$tmp, 0x1\n\t"
8936             "movsbl  $tmp2,$tmp2\n\t"
8937             "pextrb  $tmp3,$tmp,0x0\n\t"
8938             "movsbl  $tmp3,$tmp3\n\t"
8939             "cmpl  $tmp2,$tmp3\n\t"
8940             "cmovl  $tmp3,$tmp2\n\t"
8941             "cmpl  $src1,$tmp3\n\t"
8942             "cmovl  $tmp3,$src1, 0x0\n\t"
8943             "movl  $dst,$tmp2\n\t"
8944             "pextrb  $tmp2,$tmp\n\t"
8945             "movsbl  $tmp2,$tmp2\n\t"
8946             "pextrb  $tmp3,$tmp\n\t"
8947             "movsbl  $tmp3,$tmp3\n\t"
8948             "cmpl  $tmp2,$tmp3\n\t"
8949             "cmovl  $tmp3,$tmp2\n\t"
8950             "cmpl  $tmp3,$dst\n\t"
8951             "cmovl  $dst,$tmp3\t! min reduction4S" %}
8952   ins_encode %{
8953     __ vextracti128_high($tmp$$XMMRegister, $src2$$XMMRegister);
8954     __ vpmaxsb($tmp$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, 0);
8955     __ pshufd($tmp4$$XMMRegister, $tmp$$XMMRegister, 0xE);
8956     __ vpmaxsb($tmp4$$XMMRegister, $tmp4$$XMMRegister, $tmp$$XMMRegister, 0);
8957     __ pshufd($tmp$$XMMRegister, $tmp4$$XMMRegister,0x1);
8958     __ vpmaxsb($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp4$$XMMRegister, 0);
8959     __ pextrb($tmp2$$Register, $tmp$$XMMRegister,0x1);
8960     __ movsbl($tmp2$$Register, $tmp2$$Register);
8961     __ pextrb($tmp3$$Register, $tmp$$XMMRegister,0x0);
8962     __ movsbl($tmp3$$Register, $tmp3$$Register);
8963     __ cmpl($tmp2$$Register, $tmp3$$Register);
8964     __ cmovl(Assembler::greater, $tmp3$$Register, $tmp2$$Register);
8965     __ cmpl($src1$$Register, $tmp3$$Register);
8966     __ cmovl(Assembler::greater, $tmp3$$Register, $src1$$Register);
8967     __ movl($dst$$Register, $tmp3$$Register);
8968     __ pextrb($tmp2$$Register, $tmp$$XMMRegister,0x3);
8969     __ movsbl($tmp2$$Register, $tmp2$$Register);
8970     __ pextrb($tmp3$$Register, $tmp$$XMMRegister,0x2);
8971     __ movsbl($tmp3$$Register, $tmp3$$Register);
8972     __ cmpl($tmp2$$Register, $tmp3$$Register);
8973     __ cmovl(Assembler::greater, $tmp3$$Register, $tmp2$$Register);
8974     __ cmpl($tmp3$$Register, $dst$$Register);
8975     __ cmovl(Assembler::greater, $dst$$Register, $tmp3$$Register);
8976     __ movsbl($dst$$Register, $dst$$Register);
8977   %}
8978   ins_pipe( pipe_slow );
8979 %}
8980 
8981 instruct rvmax64B_reduction_reg(rRegI dst, rRegI src1, vecZ src2, regF tmp, rRegI tmp2, rRegI tmp3, regF tmp4) %{
8982   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
8983   match(Set dst (MaxReductionV src1 src2));
8984   effect(TEMP dst, TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP tmp4);
8985   format %{ "vextracti64x4_high  $tmp4,$src2\n\t"
8986             "vpmaxsb  $tmp4,$tmp4,$src2\n\t"
8987             "vextracti128_high  $tmp,$tmp4\n\t"
8988             "vpmaxsb  $tmp,$tmp,$tmp4\n\t"
8989             "pshufd  $tmp4,$tmp,0xE\n\t"
8990             "vpmaxsb  $tmp,$tmp4,$tmp\n\t"
8991             "pshufd  $tmp4,$src2,0xE\n\t"
8992             "vpmaxsb  $tmp,$tmp4,$src2\n\t"
8993             "pshufd  $tmp,$tmp4,0x1\n\t"
8994             "vpmaxsb  $tmp,$tmp,$tmp4\n\t"
8995             "pextrb  $tmp2,$tmp, 0x1\n\t"
8996             "movsbl  $tmp2,$tmp2\n\t"
8997             "pextrb  $tmp3,$tmp,0x0\n\t"
8998             "movsbl  $tmp3,$tmp3\n\t"
8999             "cmpl  $tmp2,$tmp3\n\t"
9000             "cmovl  $tmp3,$tmp2\n\t"
9001             "cmpl  $src1,$tmp3\n\t"
9002             "cmovl  $tmp3,$src1, 0x0\n\t"
9003             "movl  $dst,$tmp2\n\t"
9004             "pextrb  $tmp2,$tmp\n\t"
9005             "movsbl  $tmp2,$tmp2\n\t"
9006             "pextrb  $tmp3,$tmp\n\t"
9007             "movsbl  $tmp3,$tmp3\n\t"
9008             "cmpl  $tmp2,$tmp3\n\t"
9009             "cmovl  $tmp3,$tmp2\n\t"
9010             "cmpl  $tmp3,$dst\n\t"
9011             "cmovl  $dst,$tmp3\t! max reduction32B" %}
9012   ins_encode %{
9013     __ vextracti64x4_high($tmp4$$XMMRegister, $src2$$XMMRegister);
9014     __ vpmaxsb($tmp4$$XMMRegister, $tmp4$$XMMRegister, $src2$$XMMRegister, 1);
9015     __ vextracti128_high($tmp$$XMMRegister, $tmp4$$XMMRegister);
9016     __ vpmaxsb($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp4$$XMMRegister, 0);
9017     __ pshufd($tmp4$$XMMRegister, $tmp$$XMMRegister, 0xE);
9018     __ vpmaxsb($tmp4$$XMMRegister, $tmp4$$XMMRegister, $tmp$$XMMRegister, 0);
9019     __ pshufd($tmp$$XMMRegister, $tmp4$$XMMRegister,0x1);
9020     __ vpmaxsb($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp4$$XMMRegister, 0);
9021     __ pextrb($tmp2$$Register, $tmp$$XMMRegister,0x1);
9022     __ movsbl($tmp2$$Register, $tmp2$$Register);
9023     __ pextrb($tmp3$$Register, $tmp$$XMMRegister,0x0);
9024     __ movsbl($tmp3$$Register, $tmp3$$Register);
9025     __ cmpl($tmp2$$Register, $tmp3$$Register);
9026     __ cmovl(Assembler::greater, $tmp3$$Register, $tmp2$$Register);
9027     __ cmpl($src1$$Register, $tmp3$$Register);
9028     __ cmovl(Assembler::greater, $tmp3$$Register, $src1$$Register);
9029     __ movl($dst$$Register, $tmp3$$Register);
9030     __ pextrb($tmp2$$Register, $tmp$$XMMRegister,0x3);
9031     __ movsbl($tmp2$$Register, $tmp2$$Register);
9032     __ pextrb($tmp3$$Register, $tmp$$XMMRegister,0x2);
9033     __ movsbl($tmp3$$Register, $tmp3$$Register);
9034     __ cmpl($tmp2$$Register, $tmp3$$Register);
9035     __ cmovl(Assembler::greater, $tmp3$$Register, $tmp2$$Register);
9036     __ cmpl($tmp3$$Register, $dst$$Register);
9037     __ cmovl(Assembler::greater, $dst$$Register, $tmp3$$Register);
9038     __ movsbl($dst$$Register, $dst$$Register);
9039   %}
9040   ins_pipe( pipe_slow );
9041 %}
9042 
9043 instruct rsmax4S_reduction_reg(rRegI dst, rRegI src1, vecD src2, regF tmp, rRegI tmp2, rRegI tmp3) %{
9044   predicate(UseSSE > 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
9045   match(Set dst (MaxReductionV src1 src2));
9046   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
9047   format %{ "pshufd  $tmp,$src2,0x1\n\t"
9048             "pminsw  $tmp,$src2\n\t"
9049             "movzwl  $dst,$src1\n\t"
9050             "pextrw  $tmp2,$tmp, 0x0\n\t"
9051             "pminsw  $dst,$tmp2\n\t"
9052             "pminsw  $dst,$tmp2\n\t"
9053             "movswl  $dst,$dst\t! min reduction4S" %}
9054   ins_encode %{
9055     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister,0x1);
9056     __ pmaxsw($tmp$$XMMRegister, $src2$$XMMRegister);
9057     __ pextrw($tmp2$$Register, $tmp$$XMMRegister,0x1);
9058     __ movswl($tmp2$$Register, $tmp2$$Register);
9059     __ pextrw($tmp3$$Register, $tmp$$XMMRegister,0x0);
9060     __ movswl($tmp3$$Register, $tmp3$$Register);
9061     __ cmpl($tmp2$$Register, $tmp3$$Register);
9062     __ cmovl(Assembler::greater, $tmp3$$Register, $tmp2$$Register);
9063     __ cmpl($src1$$Register, $tmp3$$Register);
9064     __ cmovl(Assembler::greater, $tmp3$$Register, $src1$$Register);
9065     __ movl($dst$$Register, $tmp3$$Register);
9066   %}
9067   ins_pipe( pipe_slow );
9068 %}
9069 
9070 instruct rvmax4S_reduction_reg(rRegI dst, rRegI src1, vecD src2, regF tmp, rRegI tmp2, rRegI tmp3) %{
9071   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
9072   match(Set dst (MaxReductionV src1 src2));
9073   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
9074   format %{ "pshufd  $tmp,$src2,0x1\n\t"
9075             "pminsw  $tmp,$src2\n\t"
9076             "movzwl  $dst,$src1\n\t"
9077             "pextrw  $tmp2,$tmp, 0x0\n\t"
9078             "pminsw  $dst,$tmp2\n\t"
9079             "pminsw  $dst,$tmp2\n\t"
9080             "movswl  $dst,$dst\t! min reduction4S" %}
9081   ins_encode %{
9082     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister,0x1);
9083     __ vpmaxsw($tmp$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, 0);
9084     __ pextrw($tmp2$$Register, $tmp$$XMMRegister,0x1);
9085     __ movswl($tmp2$$Register, $tmp2$$Register);
9086     __ pextrw($tmp3$$Register, $tmp$$XMMRegister,0x0);
9087     __ movswl($tmp3$$Register, $tmp3$$Register);
9088     __ cmpl($tmp2$$Register, $tmp3$$Register);
9089     __ cmovl(Assembler::greater, $tmp3$$Register, $tmp2$$Register);
9090     __ cmpl($src1$$Register, $tmp3$$Register);
9091     __ cmovl(Assembler::greater, $tmp3$$Register, $src1$$Register);
9092     __ movl($dst$$Register, $tmp3$$Register);
9093   %}
9094   ins_pipe( pipe_slow );
9095 %}
9096 
9097 instruct rsmax8S_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp2, rRegI tmp3, rRegI tmp4) %{
9098   predicate(UseSSE > 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
9099   match(Set dst (MaxReductionV src1 src2));
9100   effect(TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP tmp4);
9101   format %{ "pshufd  $tmp2,$src2,0xE\n\t"
9102             "pmaxsw  $tmp2,$src2\n\t"
9103             "pshufd  $tmp,$tmp2,0x1\n\t"
9104             "pmaxsw  $tmp,$tmp2\n\t"
9105             "pextrw  $tmp2,$tmp\n\t"
9106             "movswl  $tmp2,$tmp2\n\t"
9107             "pextrw  $tmp3,$tmp, 0x0\n\t"
9108             "movswl  $tmp3,$tmp3\n\t"
9109             "cmpl    $tmp2,$tmp3\n\t"
9110             "cmovl  $tmp3,$tmp2\n\t"
9111             "cmpl  $src1,$tmp3\n\t"
9112             "cmovl  $tmp3,$src1\n\t"
9113             "movl  $dst,$tmp3\t! max reduction8S" %}
9114   ins_encode %{
9115     __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister,0xE);
9116     __ pmaxsw($tmp2$$XMMRegister, $src2$$XMMRegister);
9117     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister,0x1);
9118     __ pmaxsw($tmp$$XMMRegister, $tmp2$$XMMRegister);
9119     __ pextrw($tmp4$$Register, $tmp$$XMMRegister,0x1);
9120     __ movswl($tmp4$$Register, $tmp4$$Register);
9121     __ pextrw($tmp3$$Register, $tmp$$XMMRegister,0x0);
9122     __ movswl($tmp3$$Register, $tmp3$$Register);
9123     __ cmpl($tmp4$$Register, $tmp3$$Register);
9124     __ cmovl(Assembler::greater, $tmp3$$Register, $tmp4$$Register);
9125     __ cmpl($src1$$Register, $tmp3$$Register);
9126     __ cmovl(Assembler::greater, $tmp3$$Register, $src1$$Register);
9127     __ movl($dst$$Register, $tmp3$$Register);
9128   %}
9129   ins_pipe( pipe_slow );
9130 %}
9131 
9132 instruct rvmax8S_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp2, rRegI tmp3, rRegI tmp4) %{
9133   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
9134   match(Set dst (MaxReductionV src1 src2));
9135   effect(TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP tmp4);
9136   format %{ "pshufd   $tmp,$src2,0xE\n\t"
9137             "vpmaxsw  $tmp,$tmp,$src2\n\t"
9138             "pshufd   $tmp2,$tmp,0x1\n\t"
9139             "vpmaxsw  $tmp,$tmp,$tmp2\n\t"
9140             "movzwl   $dst,$src1\n\t"
9141             "pextrw   $tmp3,$tmp, 0x0\n\t"
9142             "vpmaxsw  $dst,$dst,$tmp3\n\t"
9143             "pextrw   $tmp3,$tmp, 0x1\n\t"
9144             "vpmaxsw  $dst,$dst,$tmp3\n\t"
9145             "movswl   $dst,$dst\t! max reduction8S" %}
9146   ins_encode %{
9147     int vector_len = 0;
9148     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
9149     __ vpmaxsw($tmp$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, vector_len);
9150     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister,0x1);
9151     __ vpmaxsw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
9152     __ pextrw($tmp4$$Register, $tmp$$XMMRegister,0x1);
9153     __ movswl($tmp4$$Register, $tmp4$$Register);
9154     __ pextrw($tmp3$$Register, $tmp$$XMMRegister,0x0);
9155     __ movswl($tmp3$$Register, $tmp3$$Register);
9156     __ cmpl($tmp4$$Register, $tmp3$$Register);
9157     __ cmovl(Assembler::greater, $tmp3$$Register, $tmp4$$Register);
9158     __ cmpl($src1$$Register, $tmp3$$Register);
9159     __ cmovl(Assembler::greater, $tmp3$$Register, $src1$$Register);
9160     __ movl($dst$$Register, $tmp3$$Register);
9161   %}
9162   ins_pipe( pipe_slow );
9163 %}
9164 
9165 instruct rvmax16S_reduction_reg(rRegI dst, rRegI src1, vecY src2, regF tmp, regF tmp2, rRegI tmp3, rRegI tmp4) %{
9166   predicate(UseAVX > 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
9167   match(Set dst (MaxReductionV src1 src2));
9168   effect(TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP tmp4);
9169   format %{ "vextracti128_high  $tmp,$src2\n\t"
9170             "vpmaxsw  $tmp,$tmp,$src2\n\t"
9171             "pshufd  $tmp2,$tmp,0xE\n\t"
9172             "vpmaxsw  $tmp,$tmp,$tmp2\n\t"
9173             "pshufd  $tmp2,$tmp,0x1\n\t"
9174             "vpmaxsw  $tmp,$tmp,$tmp2\n\t"
9175             "pextrw  $tmp2,$tmp, 0x1\n\t"
9176             "movswl  $tmp2,$tmp2\n\t"
9177             "pextrw  $tmp3,$tmp, 0x0\n\t"
9178             "movswl  $tmp3,$tmp3\n\t"
9179             "cmpl  $tmp2$tmp3\n\t"
9180             "cmovl  $tmp3,$tmp2\n\t"
9181             "cmpl  $src1,$tmp3\n\t"
9182             "cmovl  $tmp3,$src1\n\t"
9183             "movl  $dst,$tmp3\t! max reduction16S" %}
9184   ins_encode %{
9185     int vector_len = 1;
9186     __ vextracti128_high($tmp$$XMMRegister, $src2$$XMMRegister);
9187     __ vpmaxsw($tmp$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, vector_len);
9188     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE);
9189     __ vpmaxsw($tmp$$XMMRegister,$tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
9190     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister,0x1);
9191     __ vpmaxsw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
9192     __ pextrw($tmp4$$Register, $tmp$$XMMRegister,0x1);
9193     __ movswl($tmp4$$Register, $tmp4$$Register);
9194     __ pextrw($tmp3$$Register, $tmp$$XMMRegister,0x0);
9195     __ movswl($tmp3$$Register, $tmp3$$Register);
9196     __ cmpl($tmp4$$Register, $tmp3$$Register);
9197     __ cmovl(Assembler::greater, $tmp3$$Register, $tmp4$$Register);
9198     __ cmpl($src1$$Register, $tmp3$$Register);
9199     __ cmovl(Assembler::greater, $tmp3$$Register, $src1$$Register);
9200     __ movl($dst$$Register, $tmp3$$Register);
9201   %}
9202   ins_pipe( pipe_slow );
9203 %}
9204 
9205 instruct rvmax32S_reduction_reg(rRegI dst, rRegI src1, vecZ src2, regF tmp, regF tmp2, rRegI tmp3, rRegI tmp4) %{
9206   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
9207   match(Set dst (MaxReductionV src1 src2));
9208   effect(TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP tmp4);
9209   format %{ "vextracti64x4_high  $tmp2,$src2\n\t"
9210             "vpmaxsw  $tmp2,$tmp2,$src2\n\t"
9211             "vextracti128_high  $tmp,$tmp2\n\t"
9212             "vpmaxsw  $tmp,$tmp,$tmp2\n\t"
9213             "pshufd  $tmp2,$tmp,0xE\n\t"
9214             "vpmaxsw  $tmp,$tmp,$tmp2\n\t"
9215             "pshufd  $tmp2,$tmp,0x1\n\t"
9216             "vpmaxsw  $tmp,$tmp,$tmp2\n\t"
9217             "pextrw  $tmp3,$tmp, 0x0\n\t"
9218             "movswl  $dst,$src1\n\t"
9219             "pextrw  $tmp3,$tmp, 0x0\n\t"
9220             "movswl  $dst,$src1\n\t"
9221             "cmpl  $tmp2$tmp3\n\t"
9222             "cmovl  $tmp3,$tmp2\n\t"
9223             "cmpl  $src1,$tmp3\n\t"
9224             "cmovl  $tmp3,$src1\n\t"
9225             "movl  $dst,$dst\t! max reduction32S" %}
9226   ins_encode %{
9227     int vector_len = 2;
9228     __ vextracti64x4_high($tmp2$$XMMRegister, $src2$$XMMRegister);
9229     __ vpmaxsw($tmp2$$XMMRegister, $tmp2$$XMMRegister, $src2$$XMMRegister, vector_len);
9230     __ vextracti128_high($tmp$$XMMRegister, $tmp2$$XMMRegister);
9231     __ vpmaxsw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
9232     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE);
9233     __ vpmaxsw($tmp$$XMMRegister,$tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
9234     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister,0x1);
9235     __ vpmaxsw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
9236     __ pextrw($tmp4$$Register, $tmp$$XMMRegister,0x1);
9237     __ movswl($tmp4$$Register, $tmp4$$Register);
9238     __ pextrw($tmp3$$Register, $tmp$$XMMRegister,0x0);
9239     __ movswl($tmp3$$Register, $tmp3$$Register);
9240     __ cmpl($tmp4$$Register, $tmp3$$Register);
9241     __ cmovl(Assembler::greater, $tmp3$$Register, $tmp4$$Register);
9242     __ cmpl($src1$$Register, $tmp3$$Register);
9243     __ cmovl(Assembler::greater, $tmp3$$Register, $src1$$Register);
9244     __ movl($dst$$Register, $tmp3$$Register);
9245   %}
9246   ins_pipe( pipe_slow );
9247 %}
9248 
9249 instruct rsmax2I_reduction_reg(rRegI dst, rRegI src1, vecD src2, regF tmp, regF tmp2) %{
9250   predicate(UseSSE > 3 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
9251   match(Set dst (MaxReductionV src1 src2));
9252   effect(TEMP tmp, TEMP tmp2);
9253   format %{ "pshufd  $tmp,$src2,0x1\n\t"
9254             "pmaxsd  $tmp,$src2\n\t"
9255             "movd    $tmp2,$src1\n\t"
9256             "pmaxsd  $tmp2,$tmp\n\t"
9257             "movd    $dst,$tmp2\t! max reduction2I" %}
9258   ins_encode %{
9259     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x1);
9260     __ pmaxsd($tmp$$XMMRegister, $src2$$XMMRegister);
9261     __ movdl($tmp2$$XMMRegister, $src1$$Register);
9262     __ pmaxsd($tmp2$$XMMRegister, $tmp$$XMMRegister);
9263     __ movdl($dst$$Register, $tmp2$$XMMRegister);
9264   %}
9265   ins_pipe( pipe_slow );
9266 %}
9267 
9268 instruct rvmax2I_reduction_reg(rRegI dst, rRegI src1, vecD src2, regF tmp, regF tmp2) %{
9269   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
9270   match(Set dst (MaxReductionV src1 src2));
9271   effect(TEMP tmp, TEMP tmp2);
9272   format %{ "pshufd   $tmp,$src2,0x1\n\t"
9273             "vpmaxsd  $tmp2,$tmp,$src2\n\t"
9274             "movd     $tmp,$src1\n\t"
9275             "vpmaxsd  $tmp2,$tmp,$tmp2\n\t"
9276             "movd     $dst,$tmp2\t! max reduction2I" %}
9277   ins_encode %{
9278     int vector_len = 0;
9279     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x1);
9280     __ vpmaxsd($tmp2$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, vector_len);
9281     __ movdl($tmp$$XMMRegister, $src1$$Register);
9282     __ vpmaxsd($tmp2$$XMMRegister,$tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
9283     __ movdl($dst$$Register, $tmp2$$XMMRegister);
9284   %}
9285   ins_pipe( pipe_slow );
9286 %}
9287 
9288 instruct rsmax4I_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp2) %{
9289   predicate(UseSSE > 3 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
9290   match(Set dst (MaxReductionV src1 src2));
9291   effect(TEMP tmp, TEMP tmp2);
9292   format %{ "pshufd  $tmp,$src2,0xE\n\t"
9293             "pmaxsd  $tmp,$src2\n\t"
9294             "pshufd  $tmp2,$tmp,0x1\n\t"
9295             "pmaxsd  $tmp2,$tmp\n\t"
9296             "movd    $tmp,$src1\n\t"
9297             "pmaxsd  $tmp2,$tmp\n\t"
9298             "movd    $dst,$tmp2\t! max reduction4I" %}
9299   ins_encode %{
9300     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
9301     __ pmaxsd($tmp$$XMMRegister, $src2$$XMMRegister);
9302     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1);
9303     __ pmaxsd($tmp2$$XMMRegister,$tmp$$XMMRegister);
9304     __ movdl($tmp$$XMMRegister, $src1$$Register);
9305     __ pmaxsd($tmp2$$XMMRegister,$tmp$$XMMRegister);
9306     __ movdl($dst$$Register, $tmp2$$XMMRegister);
9307   %}
9308   ins_pipe( pipe_slow );
9309 %}
9310 
9311 instruct rvmax4I_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp2) %{
9312   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
9313   match(Set dst (MaxReductionV src1 src2));
9314   effect(TEMP tmp, TEMP tmp2);
9315   format %{ "pshufd   $tmp,$src2,0xE\n\t"
9316             "vpmaxsd  $tmp2,$tmp,$src2\n\t"
9317             "pshufd   $tmp,$tmp2,0x1\n\t"
9318             "vpmaxsd  $tmp2,$tmp2,$tmp\n\t"
9319             "movd     $tmp,$src1\n\t"
9320             "vpmaxsd  $tmp2,$tmp,$tmp2\n\t"
9321             "movd     $dst,$tmp2\t! max reduction4I" %}
9322   ins_encode %{
9323     int vector_len = 0;
9324     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
9325     __ vpmaxsd($tmp2$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, vector_len);
9326     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister,0x1);
9327     __ vpmaxsd($tmp2$$XMMRegister,$tmp2$$XMMRegister, $tmp$$XMMRegister, vector_len);
9328     __ movdl($tmp$$XMMRegister, $src1$$Register);
9329     __ vpmaxsd($tmp2$$XMMRegister,$tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
9330     __ movdl($dst$$Register, $tmp2$$XMMRegister);
9331   %}
9332   ins_pipe( pipe_slow );
9333 %}
9334 
9335 instruct rvmax4I_reduction_evex(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp2) %{
9336   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
9337   match(Set dst (MaxReductionV src1 src2));
9338   effect(TEMP tmp, TEMP tmp2);
9339   format %{ "pshufd   $tmp,$src2,0xE\n\t"
9340             "vpmaxsd  $tmp2,$tmp,$src2\n\t"
9341             "pshufd   $tmp,$tmp2,0x1\n\t"
9342             "vpmaxsd  $tmp2,$tmp2,$tmp\n\t"
9343             "movd     $tmp,$src1\n\t"
9344             "vpmaxsd  $tmp2,$tmp,$tmp2\n\t"
9345             "movd     $dst,$tmp2\t! max reduction4I" %}
9346   ins_encode %{
9347     int vector_len = 0;
9348     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
9349     __ vpmaxsd($tmp2$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, vector_len);
9350     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister,0x1);
9351     __ vpmaxsd($tmp2$$XMMRegister,$tmp2$$XMMRegister, $tmp$$XMMRegister, vector_len);
9352     __ movdl($tmp$$XMMRegister, $src1$$Register);
9353     __ vpmaxsd($tmp2$$XMMRegister,$tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
9354     __ movdl($dst$$Register, $tmp2$$XMMRegister);
9355   %}
9356   ins_pipe( pipe_slow );
9357 %}
9358 
9359 instruct rvmax8I_reduction_reg(rRegI dst, rRegI src1, vecY src2, regF tmp, regF tmp2) %{
9360   predicate(UseAVX > 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
9361   match(Set dst (MaxReductionV src1 src2));
9362   effect(TEMP tmp, TEMP tmp2);
9363   format %{ "vextracti128_high   $tmp,$src2\n\t"
9364             "vpmaxsd  $tmp,$tmp,$src2\n\t"
9365             "pshufd   $tmp2,$tmp,0xE\n\t"
9366             "vpmaxsd  $tmp2,$tmp,$tmp2\n\t"
9367             "pshufd   $tmp,$tmp2,0x1\n\t"
9368             "vpmaxsd  $tmp2,$tmp2,$tmp\n\t"
9369             "movd     $tmp,$src1\n\t"
9370             "vpmaxsd  $tmp2,$tmp,$tmp2\n\t"
9371             "movd     $dst,$tmp2\t! max reduction8I" %}
9372   ins_encode %{
9373     int vector_len = 1;
9374     __ vextracti128_high($tmp$$XMMRegister, $src2$$XMMRegister);
9375     __ vpmaxsd($tmp$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, vector_len);
9376     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE);
9377     __ vpmaxsd($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
9378     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister,0x1);
9379     __ vpmaxsd($tmp2$$XMMRegister,$tmp2$$XMMRegister, $tmp$$XMMRegister, vector_len);
9380     __ movdl($tmp$$XMMRegister, $src1$$Register);
9381     __ vpmaxsd($tmp2$$XMMRegister,$tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
9382     __ movdl($dst$$Register, $tmp2$$XMMRegister);
9383   %}
9384   ins_pipe( pipe_slow );
9385 %}
9386 
9387 instruct rvmax8I_reduction_evex(rRegI dst, rRegI src1, vecY src2, regF tmp, regF tmp2) %{
9388   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
9389   match(Set dst (MaxReductionV src1 src2));
9390   effect(TEMP tmp, TEMP tmp2);
9391   format %{ "vextracti128_high   $tmp,$src2\n\t"
9392             "vpmaxsd  $tmp,$tmp,$src2\n\t"
9393             "pshufd   $tmp2,$tmp,0xE\n\t"
9394             "vpmaxsd  $tmp2,$tmp,$tmp2\n\t"
9395             "pshufd   $tmp,$tmp2,0x1\n\t"
9396             "vpmaxsd  $tmp2,$tmp2,$tmp\n\t"
9397             "movd     $tmp,$src1\n\t"
9398             "vpmaxsd  $tmp2,$tmp,$tmp2\n\t"
9399             "movd     $dst,$tmp2\t! max reduction8I" %}
9400   ins_encode %{
9401     int vector_len = 1;
9402     __ vextracti128_high($tmp$$XMMRegister, $src2$$XMMRegister);
9403     __ vpmaxsd($tmp$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, vector_len);
9404     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE);
9405     __ vpmaxsd($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
9406     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister,0x1);
9407     __ vpmaxsd($tmp2$$XMMRegister,$tmp2$$XMMRegister, $tmp$$XMMRegister, vector_len);
9408     __ movdl($tmp$$XMMRegister, $src1$$Register);
9409     __ vpmaxsd($tmp2$$XMMRegister,$tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
9410     __ movdl($dst$$Register, $tmp2$$XMMRegister);
9411   %}
9412   ins_pipe( pipe_slow );
9413 %}
9414 
9415 instruct rvmax16I_reduction_evex(rRegI dst, rRegI src1, vecZ src2, regF tmp, regF tmp2, regF tmp3) %{
9416   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
9417   match(Set dst (MaxReductionV src1 src2));
9418   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
9419   format %{ "vextracti64x4_high  $tmp3,$src2\n\t"
9420             "vpmaxsd  $tmp3,$tmp3,$src2\n\t"
9421             "vextracti128_high   $tmp,$tmp3\n\t"
9422             "vpmaxsd  $tmp,$tmp,$tmp3\n\t"
9423             "pshufd   $tmp2,$tmp,0xE\n\t"
9424             "vpmaxsd  $tmp2,$tmp,$tmp2\n\t"
9425             "pshufd   $tmp,$tmp2,0x1\n\t"
9426             "vpmaxsd  $tmp2,$tmp2,$tmp\n\t"
9427             "movd     $tmp,$src1\n\t"
9428             "vpmaxsd  $tmp2,$tmp,$tmp2\n\t"
9429             "movd     $dst,$tmp2\t! max reduction16I" %}
9430   ins_encode %{
9431     int vector_len = 2;
9432     __ vextracti64x4_high($tmp3$$XMMRegister, $src2$$XMMRegister);
9433     __ vpmaxsd($tmp3$$XMMRegister, $tmp3$$XMMRegister, $src2$$XMMRegister, vector_len);
9434     __ vextracti128_high($tmp$$XMMRegister, $tmp3$$XMMRegister);
9435     __ vpmaxsd($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp3$$XMMRegister, vector_len);
9436     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE);
9437     __ vpmaxsd($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
9438     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister,0x1);
9439     __ vpmaxsd($tmp2$$XMMRegister,$tmp2$$XMMRegister, $tmp$$XMMRegister, vector_len);
9440     __ movdl($tmp$$XMMRegister, $src1$$Register);
9441     __ vpmaxsd($tmp2$$XMMRegister,$tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
9442     __ movdl($dst$$Register, $tmp2$$XMMRegister);
9443   %}
9444   ins_pipe( pipe_slow );
9445 %}
9446 
9447 // Long Max Reduction
9448 instruct rsmax1L_reduction_reg(rRegL dst, rRegL src1, vecD src2, rxmm0 xmm_0, regF tmp2, regF tmp3) %{
9449   predicate(UseSSE > 3 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
9450   match(Set dst (MaxReductionV src1 src2));
9451   effect(TEMP xmm_0, TEMP tmp2, TEMP tmp3);
9452   format %{ "movdq      $xmm_0,$src1\n\t"
9453             "movdq      $tmp2,$src1\n\t"
9454             "pcmpgtq   $xmm_0,$src2\n\t"
9455             "blendvpd  $tmp2,$src2\n\t"
9456             "movdq      $dst,$tmp2\t! max reduction1L" %}
9457   ins_encode %{
9458     __ movdq($xmm_0$$XMMRegister,$src1$$Register);
9459     __ movdq($tmp2$$XMMRegister,$src1$$Register);
9460     __ movdq($tmp3$$XMMRegister,$src2$$Register);
9461     __ pcmpgtq($xmm_0$$XMMRegister, $tmp3$$XMMRegister);
9462     __ blendvpd($tmp3$$XMMRegister,$tmp2$$XMMRegister);
9463     __ movdq($dst$$Register, $tmp3$$XMMRegister);
9464   %}
9465   ins_pipe( pipe_slow );
9466 %}
9467 
9468 instruct rsmax2L_reduction_reg(rRegL dst, rRegL src1, vecX src2, rxmm0 xmm_0, regF tmp2, regF tmp3) %{
9469   predicate(UseSSE > 3 && UseAVX == 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
9470   match(Set dst (MaxReductionV src1 src2));
9471   effect(TEMP xmm_0, TEMP tmp2, TEMP tmp3);
9472   format %{ "pshufd   $tmp3,$src2,0xE\n\t"
9473             "movdqu  $xmm_0,$src2\n\t"
9474             "pcmpgtq  $xmm_0,$tmp3\n\t"
9475             "blendvpd  $tmp3,$src2\n\t"
9476             "movdqu  $xmm_0,$tmp3\n\t"
9477             "movdq  $tmp2,$src1\n\t"
9478             "pcmpgtq  $xmm_0,$tmp2\n\t"
9479             "blendvpd  $tmp2,$tmp3\n\t"
9480             "movq     $dst,$tmp2\t! max reduction2L" %}
9481   ins_encode %{
9482     __ pshufd($tmp3$$XMMRegister, $src2$$XMMRegister, 0xE);
9483     __ movdqu($xmm_0$$XMMRegister, $src2$$XMMRegister);
9484     __ pcmpgtq($xmm_0$$XMMRegister, $tmp3$$XMMRegister);
9485     __ blendvpd($tmp3$$XMMRegister, $src2$$XMMRegister);
9486     __ movdqu($xmm_0$$XMMRegister, $tmp3$$XMMRegister);
9487     __ movdq($tmp2$$XMMRegister, $src1$$Register);
9488     __ pcmpgtq($xmm_0$$XMMRegister, $tmp2$$XMMRegister);
9489     __ blendvpd($tmp2$$XMMRegister,$tmp3$$XMMRegister);
9490     __ movdq($dst$$Register, $tmp2$$XMMRegister);
9491   %}
9492   ins_pipe( pipe_slow );
9493 %}
9494 
9495 instruct rvmax2L_reduction_reg(rRegL dst, rRegL src1, vecX src2, regF tmp, regF tmp2, regF tmp3) %{
9496   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
9497   match(Set dst (MaxReductionV src1 src2));
9498   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
9499   format %{ "pshufd   $tmp2,$src2,0xE\n\t"
9500             "vpcmpgtq  $tmp,$tmp2,$src2\n\t"
9501             "vblendvpd   $tmp2,$tmp2,$src2,$tmp\n\t"
9502             "movq     $tmp,$src1\n\t"
9503             "vpcmpgtq  $tmp3,$tmp2,$tmp\n\t"
9504             "blendvpd   $tmp2,$tmp2,$src1,$tmp3\n\t"
9505             "movq     $dst,$tmp2\t! max reduction2L" %}
9506   ins_encode %{
9507     int vector_len = 0;
9508     __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0xE);
9509     __ vpcmpgtq($tmp$$XMMRegister, $tmp2$$XMMRegister, $src2$$XMMRegister, vector_len);
9510     __ vblendvpd($tmp2$$XMMRegister, $src2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, vector_len);
9511     __ movdq($tmp$$XMMRegister,$src1$$Register);
9512     __ vpcmpgtq($tmp3$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, vector_len);
9513     __ vblendvpd($tmp2$$XMMRegister, $tmp$$XMMRegister,$tmp2$$XMMRegister,$tmp3$$XMMRegister, vector_len);
9514     __ movdq($dst$$Register, $tmp2$$XMMRegister);
9515   %}
9516   ins_pipe( pipe_slow );
9517 %}
9518 
9519 instruct rvmax4L_reduction_reg(rRegL dst, rRegL src1, vecY src2, regF tmp, regF tmp2, regF tmp3) %{
9520   predicate(UseAVX > 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
9521   match(Set dst (MaxReductionV src1 src2));
9522   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
9523   format %{ "vextracti128_high   $tmp2,$src2\n\t"
9524             "vpcmpgtq  $tmp,$tmp2,$src2\n\t"
9525             "vblendvpd   $tmp2,$tmp2,$src2,$tmp\n\t"
9526             "vpshufd   $tmp3, $tmp2,0x1\n\t"
9527             "vpcmpgtq  $tmp, $tmp3,$tmp\n\t2"
9528             "vblendvpd $tmp3,$tmp3,$tmp2,$tmp\n\t"
9529             "movq     $tmp2,$src1\n\t"
9530             "vpcmpgtq  $tmp,$tmp2,$tmp3\n\t"
9531             "blendvpd   $tmp2,$tmp2,$tmp3,$tmp\n\t"
9532             "movq     $dst,$tmp2\t! max reduction2L" %}
9533   ins_encode %{
9534     int vector_len = 1;
9535     __ vextracti128_high($tmp2$$XMMRegister, $src2$$XMMRegister);
9536     __ vpcmpgtq($tmp$$XMMRegister, $tmp2$$XMMRegister, $src2$$XMMRegister, vector_len);
9537     __ vblendvpd($tmp2$$XMMRegister, $src2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, vector_len);
9538     __ pshufd($tmp3$$XMMRegister, $tmp2$$XMMRegister, 0xE);
9539     __ vpcmpgtq($tmp$$XMMRegister, $tmp3$$XMMRegister, $tmp2$$XMMRegister, vector_len);
9540     __ vblendvpd($tmp3$$XMMRegister,$tmp2$$XMMRegister,$tmp3$$XMMRegister, $tmp$$XMMRegister, vector_len);
9541     __ movdq($tmp$$XMMRegister,$src1$$Register);
9542     __ vpcmpgtq($tmp2$$XMMRegister, $tmp3$$XMMRegister, $tmp$$XMMRegister, vector_len);
9543     __ vblendvpd($tmp2$$XMMRegister, $tmp$$XMMRegister,$tmp3$$XMMRegister,$tmp2$$XMMRegister, vector_len);
9544     __ movdq($dst$$Register, $tmp2$$XMMRegister);
9545   %}
9546   ins_pipe( pipe_slow );
9547 %}
9548 
9549 instruct rvmax8L_reduction_reg(rRegL dst, rRegL src1, vecZ src2, regF tmp, regF tmp2, regF tmp3) %{
9550   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
9551   match(Set dst (MaxReductionV src1 src2));
9552   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
9553   format %{ "vextracti64x4_high  $tmp3,$src2\n\t"
9554             "vpcmpgtq  $tmp,$tmp3,$src2\n\t"
9555             "vblendvpd   $tmp3,$tmp3,$src2,$tmp\n\t"
9556             "vextracti128_high   $tmp2,$tmp3\n\t"
9557             "vpcmpgtq  $tmp,$tmp2,$tmp3\n\t"
9558             "vblendvpd   $tmp2,$tmp2,$tmp3,$tmp\n\t"
9559             "vpshufd  $tmp3,$tmp2,0x1\n\t"
9560             "vpcmpgtq   $tmp,$tmp3,$tmp2\n\t"
9561             "vblendvpd  $tmp3,$tmp3,$tmp2,$tmp\n\t"
9562             "movq     $tmp2,$src1\n\t"
9563             "vpcmpgtq  $tmp,$tmp2,$tmp3\n\t"
9564             "vblendvpd  $tmp2,$tmp2,$tmp3,$tmp\n\t"
9565             "movq     $dst,$tmp2\t! max reduction4I" %}
9566   ins_encode %{
9567     int vector_len = 1;
9568     __ vextracti64x4_high($tmp3$$XMMRegister, $src2$$XMMRegister);
9569     __ vpcmpgtq($tmp$$XMMRegister, $tmp3$$XMMRegister, $src2$$XMMRegister, vector_len);
9570     __ vblendvpd($tmp3$$XMMRegister, $src2$$XMMRegister, $tmp3$$XMMRegister, $tmp$$XMMRegister, vector_len);
9571     __ vextracti128_high($tmp2$$XMMRegister, $tmp3$$XMMRegister);
9572     __ vpcmpgtq($tmp$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister, vector_len);
9573     __ vblendvpd($tmp2$$XMMRegister, $tmp3$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, vector_len);
9574     __ pshufd($tmp3$$XMMRegister, $tmp2$$XMMRegister, 0xE);
9575     __ vpcmpgtq($tmp$$XMMRegister, $tmp3$$XMMRegister, $tmp2$$XMMRegister, vector_len);
9576     __ vblendvpd($tmp3$$XMMRegister,$tmp2$$XMMRegister,$tmp3$$XMMRegister, $tmp$$XMMRegister, vector_len);
9577     __ movdq($tmp2$$XMMRegister, $src1$$Register);
9578     __ vpcmpgtq($tmp$$XMMRegister,$tmp2$$XMMRegister, $tmp3$$XMMRegister, vector_len);
9579     __ vblendvpd($tmp2$$XMMRegister,$tmp3$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, vector_len);
9580     __ movdq($dst$$Register, $tmp2$$XMMRegister);
9581   %}
9582   ins_pipe( pipe_slow );
9583 %}
9584 
9585 // Float max Reduction
9586 instruct rsmax2F_reduction_reg(regF dst, vecD src2, regF tmp) %{
9587   predicate(UseSSE > 0 && UseAVX == 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
9588   match(Set dst (MaxReductionV dst src2));
9589   effect(TEMP dst, TEMP tmp);
9590   format %{ "maxps   $dst, $src2\n\t"
9591             "pshufd  $tmp,$src2,0x1\n\t"
9592             "maxps   $dst,$tmp\t! max reduction2F" %}
9593   ins_encode %{
9594     __ maxps($dst$$XMMRegister, $src2$$XMMRegister);
9595     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x1);
9596     __ maxps($dst$$XMMRegister, $tmp$$XMMRegister);
9597   %}
9598   ins_pipe( pipe_slow );
9599 %}
9600 
9601 instruct rvmax2F_reduction_reg(regF dst, vecD src2, regF tmp) %{
9602   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
9603   match(Set dst (MaxReductionV dst src2));
9604   effect(TEMP dst, TEMP tmp);
9605   format %{ "vmaxps  $dst,$dst,$src2\n\t"
9606             "pshufd  $tmp,$src2,0x1\n\t"
9607             "vmaxps  $dst,$dst,$tmp\t! max reduction2F" %}
9608   ins_encode %{
9609     int vector_len = 0;
9610     __ vmaxps($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister, vector_len);
9611     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x1);
9612     __ vmaxps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
9613   %}
9614   ins_pipe( pipe_slow );
9615 %}
9616 
9617 instruct rsmax4F_reduction_reg(regF dst, vecX src2, regF tmp) %{
9618   predicate(UseSSE > 0 && UseAVX == 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
9619   match(Set dst (MaxReductionV dst src2));
9620   effect(TEMP tmp, TEMP dst);
9621   format %{ "maxps  $dst,$src2\n\t"
9622             "pshufd  $tmp,$src2,0x1\n\t"
9623             "maxps  $dst,tmp\n\t"
9624             "pshufd  $tmp,$src2,0x2\n\t"
9625             "maxps  $dst,tmp\n\t"
9626             "pshufd  $tmp,$src2,0x3\n\t"
9627             "maxps  $dst,$tmp\t! max reduction4F" %}
9628   ins_encode %{
9629     __ maxps($dst$$XMMRegister, $src2$$XMMRegister);
9630     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x1);
9631     __ maxps($dst$$XMMRegister, $tmp$$XMMRegister);
9632     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x2);
9633     __ maxps($dst$$XMMRegister, $tmp$$XMMRegister);
9634     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x3);
9635     __ maxps($dst$$XMMRegister, $tmp$$XMMRegister);
9636   %}
9637   ins_pipe( pipe_slow );
9638 %}
9639 
9640 instruct rvmax4F_reduction_reg(regF dst, vecX src2, regF tmp) %{
9641   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
9642   match(Set dst (MaxReductionV dst src2));
9643   effect(TEMP tmp, TEMP dst);
9644   format %{ "vmaxps  $dst,$dst,$src2\n\t"
9645             "pshufd  $tmp,$src2,0x1\n\t"
9646             "vmaxps  $dst,$dst,tmp\n\t"
9647             "pshufd  $tmp,$src2,0x2\n\t"
9648             "vmaxps  $dst,$dst,tmp\n\t"
9649             "pshufd  $tmp,$src2,0x3\n\t"
9650             "vmaxps  $dst,$dst,$tmp\t! max reduction4F" %}
9651   ins_encode %{
9652     int vector_len = 0;
9653     __ vmaxps($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister, vector_len);
9654     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x1);
9655     __ vmaxps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
9656     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x2);
9657     __ vmaxps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
9658     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x3);
9659     __ vmaxps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
9660   %}
9661   ins_pipe( pipe_slow );
9662 %}
9663 
9664 instruct rvmax8F_reduction_reg(regF dst, vecY src2, regF tmp, regF tmp2) %{
9665   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
9666   match(Set dst (MaxReductionV dst src2));
9667   effect(TEMP tmp, TEMP dst, TEMP tmp2);
9668   format %{ "vmaxps  $dst,$dst,$src2\n\t"
9669             "pshufd  $tmp,$src2,0x01\n\t"
9670             "vmaxps  $dst,$dst,$tmp\n\t"
9671             "pshufd  $tmp,$src2,0x02\n\t"
9672             "vmaxps  $dst,$dst,$tmp\n\t"
9673             "pshufd  $tmp,$src2,0x03\n\t"
9674             "vmaxps  $dst,$dst,$tmp\n\t"
9675             "vextractf128_high  $tmp2,$src2\n\t"
9676             "vmaxps  $dst,$dst,$tmp2\n\t"
9677             "pshufd  $tmp,$tmp2,0x01\n\t"
9678             "vmaxps  $dst,$dst,$tmp\n\t"
9679             "pshufd  $tmp,$tmp2,0x02\n\t"
9680             "vmaxps  $dst,$dst,$tmp\n\t"
9681             "pshufd  $tmp,$tmp2,0x03\n\t"
9682             "vmaxps  $dst,$dst,$tmp\t! sub reduction8F" %}
9683   ins_encode %{
9684     int vector_len = 1;
9685     __ vmaxps($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister, vector_len);
9686     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x01);
9687     __ vmaxps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
9688     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x02);
9689     __ vmaxps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
9690     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x03);
9691     __ vmaxps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
9692     __ vextractf128_high($tmp2$$XMMRegister, $src2$$XMMRegister);
9693     __ vmaxps($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister, vector_len);
9694     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x01);
9695     __ vmaxps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
9696     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x02);
9697     __ vmaxps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
9698     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x03);
9699     __ vmaxps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
9700   %}
9701   ins_pipe( pipe_slow );
9702 %}
9703 
9704 instruct rvmax16F_reduction_reg(regF dst, vecZ src2, regF tmp, regF tmp2) %{
9705   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
9706   match(Set dst (MaxReductionV dst src2));
9707   effect(TEMP tmp, TEMP dst, TEMP tmp2);
9708   format %{ "vmaxps  $dst,$dst,$src2\n\t"
9709             "pshufd  $tmp,$src2,0x01\n\t"
9710             "vmaxps  $dst,$dst,$tmp\n\t"
9711             "pshufd  $tmp,$src2,0x02\n\t"
9712             "vmaxps  $dst,$dst,$tmp\n\t"
9713             "pshufd  $tmp,$src2,0x03\n\t"
9714             "vmaxps  $dst,$dst,$tmp\n\t"
9715             "vextractf32x4  $tmp2,$src2,0x1\n\t"
9716             "vmaxps  $dst,$dst,$tmp2\n\t"
9717             "pshufd  $tmp,$tmp2,0x01\n\t"
9718             "vmaxps  $dst,$dst,$tmp\n\t"
9719             "pshufd  $tmp,$tmp2,0x02\n\t"
9720             "vmaxps  $dst,$dst,$tmp\n\t"
9721             "pshufd  $tmp,$tmp2,0x03\n\t"
9722             "vmaxps  $dst,$dst,$tmp\n\t"
9723             "vextractf32x4  $tmp2,$src2,0x2\n\t"
9724             "vmaxps  $dst,$dst,$tmp2\n\t"
9725             "pshufd  $tmp,$tmp2,0x01\n\t"
9726             "vmaxps  $dst,$dst,$tmp\n\t"
9727             "pshufd  $tmp,$tmp2,0x02\n\t"
9728             "vmaxps  $dst,$dst,$tmp\n\t"
9729             "pshufd  $tmp,$tmp2,0x03\n\t"
9730             "vmaxps  $dst,$dst,$tmp\n\t"
9731             "vextractf32x4  $tmp2,$src2,0x3\n\t"
9732             "vmaxps  $dst,$dst,$tmp2\n\t"
9733             "pshufd  $tmp,$tmp2,0x01\n\t"
9734             "vmaxps  $dst,$dst,$tmp\n\t"
9735             "pshufd  $tmp,$tmp2,0x02\n\t"
9736             "vmaxps  $dst,$dst,$tmp\n\t"
9737             "pshufd  $tmp,$tmp2,0x03\n\t"
9738             "vmaxps  $dst,$dst,$tmp\t! sub reduction16F" %}
9739   ins_encode %{
9740     int vector_len = 2;
9741     __ vmaxps($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister, vector_len);
9742     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x01);
9743     __ vmaxps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
9744     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x02);
9745     __ vmaxps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
9746     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x03);
9747     __ vmaxps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
9748     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x1);
9749     __ vmaxps($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister, vector_len);
9750     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x01);
9751     __ vmaxps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
9752     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x02);
9753     __ vmaxps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
9754     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x03);
9755     __ vmaxps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
9756     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x2);
9757     __ vmaxps($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister, vector_len);
9758     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x01);
9759     __ vmaxps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
9760     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x02);
9761     __ vmaxps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
9762     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x03);
9763     __ vmaxps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
9764     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x3);
9765     __ vmaxps($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister, vector_len);
9766     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x01);
9767     __ vmaxps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
9768     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x02);
9769     __ vmaxps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
9770     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x03);
9771     __ vmaxps($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
9772   %}
9773   ins_pipe( pipe_slow );
9774 %}
9775 
9776 instruct rsmax2D_reduction_reg(regD dst, vecX src2, regD tmp) %{
9777   predicate(UseSSE >= 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
9778   match(Set dst (MaxReductionV dst src2));
9779   effect(TEMP tmp, TEMP dst);
9780   format %{ "maxpd   $dst,$src2\n\t"
9781             "pshufd  $tmp,$src2,0xE\n\t"
9782             "maxpd   $dst,$tmp\t! max reduction2D" %}
9783   ins_encode %{
9784     __ maxpd($dst$$XMMRegister, $src2$$XMMRegister);
9785     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
9786     __ maxpd($dst$$XMMRegister, $tmp$$XMMRegister);
9787   %}
9788   ins_pipe( pipe_slow );
9789 %}
9790 
9791 instruct rvmax2D_reduction_reg(regD dst, vecX src2, regD tmp) %{
9792   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
9793   match(Set dst (MaxReductionV dst src2));
9794   effect(TEMP tmp, TEMP dst);
9795   format %{ "vmaxpd  $dst,$dst,$src2\n\t"
9796             "pshufd  $tmp,$src2,0xE\n\t"
9797             "vmaxpd  $dst,$dst,$tmp\t! max reduction2D" %}
9798   ins_encode %{
9799     int vector_len = 0;
9800     __ vmaxpd($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister, vector_len);
9801     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
9802     __ vmaxpd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
9803   %}
9804   ins_pipe( pipe_slow );
9805 %}
9806 
9807 instruct rvmax4D_reduction_reg(regD dst, vecY src2, regD tmp, regD tmp2) %{
9808   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
9809   match(Set dst (MaxReductionV dst src2));
9810   effect(TEMP tmp, TEMP dst, TEMP tmp2);
9811   format %{ "vmaxpd  $dst,$dst,$src2\n\t"
9812             "pshufd  $tmp,$src2,0xE\n\t"
9813             "vmaxpd  $dst,$dst,$tmp\n\t"
9814             "vextractf32x4  $tmp2,$src2,0x1\n\t"
9815             "vmaxpd  $dst,$dst,$tmp2\n\t"
9816             "pshufd  $tmp,$tmp2,0xE\n\t"
9817             "vmaxpd  $dst,$dst,$tmp\t! max reduction4D" %}
9818   ins_encode %{
9819     int vector_len = 1;
9820     __ vmaxpd($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister, vector_len);
9821     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
9822     __ vmaxpd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
9823     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x1);
9824     __ vmaxpd($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister, vector_len);
9825     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE);
9826     __ vmaxpd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
9827   %}
9828   ins_pipe( pipe_slow );
9829 %}
9830 
9831 instruct rvmax8D_reduction_reg(regD dst, vecZ src2, regD tmp, regD tmp2) %{
9832   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
9833   match(Set dst (MaxReductionV dst src2));
9834   effect(TEMP tmp, TEMP dst, TEMP tmp2);
9835   format %{ "vmaxpd  $dst,$dst,$src2\n\t"
9836             "pshufd  $tmp,$src2,0xE\n\t"
9837             "vmaxpd  $dst,$dst,$tmp\n\t"
9838             "vextractf32x4  $tmp2,$src2,0x1\n\t"
9839             "vmaxpd  $dst,$dst,$tmp2\n\t"
9840             "pshufd  $tmp,$tmp2,0xE\n\t"
9841             "vmaxpd  $dst,$dst,$tmp\n\t"
9842             "vextractf32x4  $tmp2,$src2,0x2\n\t"
9843             "vmaxpd  $dst,$dst,$tmp2\n\t"
9844             "pshufd  $tmp,$tmp2,0xE\n\t"
9845             "vmaxpd  $dst,$dst,$tmp\n\t"
9846             "vextractf32x4  $tmp2,$src2,0x3\n\t"
9847             "vmaxpd  $dst,$dst,$tmp2\n\t"
9848             "pshufd  $tmp,$tmp2,0xE\n\t"
9849             "vmaxpd  $dst,$dst,$tmp\t! max reduction8D" %}
9850   ins_encode %{
9851     int vector_len = 2;
9852     __ vmaxpd($dst$$XMMRegister, $dst$$XMMRegister, $src2$$XMMRegister, vector_len);
9853     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
9854     __ vmaxpd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
9855     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x1);
9856     __ vmaxpd($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister, vector_len);
9857     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE);
9858     __ vmaxpd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
9859     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x2);
9860     __ vmaxpd($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister, vector_len);
9861     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE);
9862     __ vmaxpd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
9863     __ vextractf32x4($tmp2$$XMMRegister, $src2$$XMMRegister, 0x3);
9864     __ vmaxpd($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister, vector_len);
9865     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE);
9866     __ vmaxpd($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
9867   %}
9868   ins_pipe( pipe_slow );
9869 %}
9870 
9871 instruct rsand8B_reduction_reg(rRegI dst, rRegI src1, vecD src2, regF tmp, rRegI tmp2) %{
9872   predicate(UseSSE > 3 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
9873   match(Set dst (AndReductionV src1 src2));
9874   effect(TEMP tmp, TEMP tmp2, TEMP dst);
9875   format %{
9876             "pshufd  $tmp,$src2,0x1\n\t"
9877             "pand    $tmp,$src2\n\t"
9878             "movzbl  $dst,$src1\n\t"
9879             "pextrb  $tmp2,$tmp, 0x0\n\t"
9880             "andl    $dst,$tmp2\n\t"
9881             "pextrb  $tmp2,$tmp, 0x1\n\t"
9882             "andl    $dst,$tmp2\n\t"
9883             "pextrb  $tmp2,$tmp, 0x2\n\t"
9884             "andl    $dst,$tmp2\n\t"
9885             "pextrb  $tmp2,$tmp, 0x3\n\t"
9886             "andl    $dst,$tmp2\n\t"
9887             "movsbl  $dst,$dst\t! and reduction8B" %}
9888   ins_encode %{
9889     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x1);
9890     __ pand($tmp$$XMMRegister, $src2$$XMMRegister);
9891     __ movzbl($dst$$Register, $src1$$Register);
9892     __ pextrb($tmp2$$Register, $tmp$$XMMRegister, 0x0);
9893     __ andl($dst$$Register, $tmp2$$Register);
9894     __ pextrb($tmp2$$Register, $tmp$$XMMRegister, 0x1);
9895     __ andl($dst$$Register, $tmp2$$Register);
9896     __ pextrb($tmp2$$Register, $tmp$$XMMRegister, 0x2);
9897     __ andl($dst$$Register, $tmp2$$Register);
9898     __ pextrb($tmp2$$Register, $tmp$$XMMRegister, 0x3);
9899     __ andl($dst$$Register, $tmp2$$Register);
9900     __ movsbl($dst$$Register, $dst$$Register);
9901   %}
9902   ins_pipe( pipe_slow );
9903 %}
9904 
9905 instruct rsand16B_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp2, rRegI tmp3) %{
9906   predicate(UseSSE > 3 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
9907   match(Set dst (AndReductionV src1 src2));
9908   effect(TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP dst);
9909   format %{ "pshufd  $tmp,$src2,0xE\n\t"
9910             "pand    $tmp,$src2\n\t"
9911             "pshufd  $tmp2,$tmp,0x1\n\t"
9912             "pand    $tmp,$tmp,$tmp2\n\t"
9913             "movzbl  $dst,$src1\n\t"
9914             "pextrb  $tmp3,$tmp, 0x0\n\t"
9915             "andl    $dst,$tmp3\n\t"
9916             "pextrb  $tmp3,$tmp, 0x1\n\t"
9917             "andl    $dst,$tmp3\n\t"
9918             "pextrb  $tmp3,$tmp, 0x2\n\t"
9919             "andl    $dst,$tmp3\n\t"
9920             "pextrb  $tmp3,$tmp, 0x3\n\t"
9921             "andl    $dst,$tmp3\n\t"
9922             "movsbl  $dst,$dst\t! and reduction16B" %}
9923   ins_encode %{
9924     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
9925     __ pand($tmp$$XMMRegister, $src2$$XMMRegister);
9926     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1);
9927     __ pand($tmp$$XMMRegister, $tmp2$$XMMRegister);
9928     __ movzbl($dst$$Register, $src1$$Register);
9929     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x0);
9930     __ andl($dst$$Register, $tmp3$$Register);
9931     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x1);
9932     __ andl($dst$$Register, $tmp3$$Register);
9933     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x2);
9934     __ andl($dst$$Register, $tmp3$$Register);
9935     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x3);
9936     __ andl($dst$$Register, $tmp3$$Register);
9937     __ movsbl($dst$$Register, $dst$$Register);
9938   %}
9939   ins_pipe( pipe_slow );
9940 %}
9941 
9942 instruct rvand32B_reduction_reg(rRegI dst, rRegI src1, vecY src2, regF tmp, regF tmp2, rRegI tmp3) %{
9943   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
9944   match(Set dst (AndReductionV src1 src2));
9945   effect(TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP dst);
9946    format %{ "vextracti128_high  $tmp,$src2\n\t"
9947             "vpand   $tmp,$tmp,$src2\n\t"
9948             "pshufd  $tmp2,$tmp,0xE\n\t"
9949             "vpand   $tmp,$tmp,$tmp2\n\t"
9950             "pshufd  $tmp2,$tmp,0x1\n\t"
9951             "vpand   $tmp,$tmp,$tmp2\n\t"
9952             "movzbl  $dst,$src1\n\t"
9953             "pextrb  $tmp3,$tmp, 0x0\n\t"
9954             "andl    $dst,$tmp3\n\t"
9955             "pextrb  $tmp3,$tmp, 0x1\n\t"
9956             "andl    $dst,$tmp3\n\t"
9957             "pextrb  $tmp3,$tmp, 0x2\n\t"
9958             "andl    $dst,$tmp3\n\t"
9959             "pextrb  $tmp3,$tmp, 0x3\n\t"
9960             "andl    $dst,$tmp3\n\t"
9961             "movsbl  $dst,$dst\t! and reduction32B" %}
9962   ins_encode %{
9963     int vector_len = 0;
9964     __ vextracti128_high($tmp$$XMMRegister, $src2$$XMMRegister);
9965     __ vpand($tmp$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, vector_len);
9966     __ vpshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE, vector_len);
9967     __ vpand($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
9968     __ vpshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1, vector_len);
9969     __ vpand($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
9970     __ movzbl($dst$$Register, $src1$$Register);
9971     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x0);
9972     __ andl($dst$$Register, $tmp3$$Register);
9973     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x1);
9974     __ andl($dst$$Register, $tmp3$$Register);
9975     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x2);
9976     __ andl($dst$$Register, $tmp3$$Register);
9977     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x3);
9978     __ andl($dst$$Register, $tmp3$$Register);
9979     __ movsbl($dst$$Register, $dst$$Register);
9980   %}
9981   ins_pipe( pipe_slow );
9982 %}
9983 
9984 instruct rvand64B_reduction_reg(rRegI dst, rRegI src1, vecZ src2, regF tmp, regF tmp2, rRegI tmp3) %{
9985   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
9986   match(Set dst (AndReductionV src1 src2));
9987   effect(TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP dst);
9988   format %{ "vextracti64x4_high  $tmp2,$src2\n\t"
9989             "vpand   $tmp2,$tmp2,$src2\n\t"
9990             "vextracti128_high  $tmp,$tmp2\n\t"
9991             "vpand   $tmp,$tmp,$tmp2\n\t"
9992             "pshufd  $tmp2,$tmp,0xE\n\t"
9993             "vpand   $tmp,$tmp,$tmp2\n\t"
9994             "pshufd  $tmp2,$tmp,0x1\n\t"
9995             "vpand   $tmp,$tmp,$tmp2\n\t"
9996             "movzbl  $dst,$src1\n\t"
9997             "movdl   $tmp3,$tmp\n\t"
9998             "andl    $dst,$tmp3\n\t"
9999             "shrl    $tmp3,0x8\n\t"
10000             "andl    $dst,$tmp3\n\t"
10001             "shrl    $tmp3,0x8\n\t"
10002             "andl    $dst,$tmp3\n\t"
10003             "shrl    $tmp3,0x8\n\t"
10004             "andl    $dst,$tmp3\n\t"
10005             "movsbl  $dst,$dst\t! and reduction64B" %}
10006   ins_encode %{
10007     int vector_len = 0;
10008     __ vextracti64x4_high($tmp2$$XMMRegister, $src2$$XMMRegister);
10009     __ vpand($tmp2$$XMMRegister, $tmp2$$XMMRegister, $src2$$XMMRegister, 1);
10010     __ vextracti128_high($tmp$$XMMRegister, $tmp2$$XMMRegister);
10011     __ vpand($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10012     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE);
10013     __ vpand($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10014     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1);
10015     __ vpand($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10016     __ movzbl($dst$$Register, $src1$$Register);
10017     __ movdl($tmp3$$Register, $tmp$$XMMRegister);
10018     __ andl($dst$$Register, $tmp3$$Register);
10019     __ shrl($tmp3$$Register, 8);
10020     __ andl($dst$$Register, $tmp3$$Register);
10021     __ shrl($tmp3$$Register, 8);
10022     __ andl($dst$$Register, $tmp3$$Register);
10023     __ shrl($tmp3$$Register, 8);
10024     __ andl($dst$$Register, $tmp3$$Register);
10025     __ movsbl($dst$$Register, $dst$$Register);
10026   %}
10027   ins_pipe( pipe_slow );
10028 %}
10029 
10030 instruct rsand4S_reduction_reg(rRegI dst, rRegI src1, vecD src2, regF tmp, rRegI tmp2) %{
10031   predicate(UseSSE > 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
10032   match(Set dst (AndReductionV src1 src2));
10033   effect(TEMP tmp, TEMP tmp2, TEMP dst);
10034   format %{
10035             "pshufd  $tmp,$src2,0x1\n\t"
10036             "pand    $tmp,$src2\n\t"
10037             "movzwl  $dst,$src1\n\t"
10038             "pextrw  $tmp2,$tmp, 0x0\n\t"
10039             "andw    $dst,$tmp2\n\t"
10040             "pextrw  $tmp2,$tmp, 0x1\n\t"
10041             "andw    $dst,$tmp2\n\t"
10042             "movswl  $dst,$dst\t! and reduction4S" %}
10043   ins_encode %{
10044     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x1);
10045     __ pand($tmp$$XMMRegister, $src2$$XMMRegister);
10046     __ movzwl($dst$$Register, $src1$$Register);
10047     __ pextrw($tmp2$$Register, $tmp$$XMMRegister, 0x0);
10048     __ andw($dst$$Register, $tmp2$$Register);
10049     __ pextrw($tmp2$$Register, $tmp$$XMMRegister, 0x1);
10050     __ andw($dst$$Register, $tmp2$$Register);
10051     __ movswl($dst$$Register, $dst$$Register);
10052   %}
10053   ins_pipe( pipe_slow );
10054 %}
10055 
10056 instruct rsand8S_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp2, rRegI tmp3) %{
10057   predicate(UseSSE > 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
10058   match(Set dst (AndReductionV src1 src2));
10059   effect(TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP dst);
10060   format %{ "pshufd  $tmp,$src2,0xE\n\t"
10061             "pand    $tmp,$src2\n\t"
10062             "pshufd  $tmp2,$tmp,0x1\n\t"
10063             "pand    $tmp,$tmp,$tmp2\n\t"
10064             "movzwl  $dst,$src1\n\t"
10065             "pextrw  $tmp3,$tmp, 0x0\n\t"
10066             "andw    $dst,$tmp3\n\t"
10067             "pextrw  $tmp3,$tmp, 0x1\n\t"
10068             "andw    $dst,$tmp3\n\t"
10069             "movswl  $dst,$dst\t! and reduction8S" %}
10070   ins_encode %{
10071     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
10072     __ pand($tmp$$XMMRegister, $src2$$XMMRegister);
10073     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1);
10074     __ pand($tmp$$XMMRegister, $tmp2$$XMMRegister);
10075     __ movzwl($dst$$Register, $src1$$Register);
10076     __ pextrw($tmp3$$Register, $tmp$$XMMRegister, 0x0);
10077     __ andw($dst$$Register, $tmp3$$Register);
10078     __ pextrw($tmp3$$Register, $tmp$$XMMRegister, 0x1);
10079     __ andw($dst$$Register, $tmp3$$Register);
10080     __ movswl($dst$$Register, $dst$$Register);
10081   %}
10082   ins_pipe( pipe_slow );
10083 %}
10084 
10085 instruct rvand16S_reduction_reg(rRegI dst, rRegI src1, vecY src2, regF tmp, regF tmp2, rRegI tmp3) %{
10086   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
10087   match(Set dst (AndReductionV src1 src2));
10088   effect(TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP dst);
10089    format %{ "vextracti128_high  $tmp,$src2\n\t"
10090             "vpand   $tmp,$tmp,$src2\n\t"
10091             "pshufd  $tmp2,$tmp,0xE\n\t"
10092             "vpand   $tmp,$tmp,$tmp2\n\t"
10093             "pshufd  $tmp2,$tmp,0x1\n\t"
10094             "vpand   $tmp,$tmp,$tmp2\n\t"
10095             "movzwl  $dst,$src1\n\t"
10096             "pextrw  $tmp3,$tmp, 0x0\n\t"
10097             "andw    $dst,$tmp3\n\t"
10098             "pextrw  $tmp3,$tmp, 0x1\n\t"
10099             "andw    $dst,$tmp3\n\t"
10100             "movswl  $dst,$dst\t! and reduction16S" %}
10101   ins_encode %{
10102     int vector_len = 0;
10103     __ vextracti128_high($tmp$$XMMRegister, $src2$$XMMRegister);
10104     __ vpand($tmp$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, vector_len);
10105     __ vpshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE, vector_len);
10106     __ vpand($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10107     __ vpshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1, vector_len);
10108     __ vpand($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10109     __ movzwl($dst$$Register, $src1$$Register);
10110     __ pextrw($tmp3$$Register, $tmp$$XMMRegister, 0x0);
10111     __ andw($dst$$Register, $tmp3$$Register);
10112     __ pextrw($tmp3$$Register, $tmp$$XMMRegister, 0x1);
10113     __ andw($dst$$Register, $tmp3$$Register);
10114     __ movswl($dst$$Register, $dst$$Register);
10115   %}
10116   ins_pipe( pipe_slow );
10117 %}
10118 
10119 instruct rvand32S_reduction_reg(rRegI dst, rRegI src1, vecZ src2, regF tmp, regF tmp2, rRegI tmp3) %{
10120   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
10121   match(Set dst (AndReductionV src1 src2));
10122   effect(TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP dst);
10123   format %{ "vextracti64x4_high  $tmp2,$src2\n\t"
10124             "vpand   $tmp2,$tmp2,$src2\n\t"
10125             "vextracti128_high  $tmp,$tmp2\n\t"
10126             "vpand   $tmp,$tmp,$tmp2\n\t"
10127             "pshufd  $tmp2,$tmp,0xE\n\t"
10128             "vpand   $tmp,$tmp,$tmp2\n\t"
10129             "pshufd  $tmp2,$tmp,0x1\n\t"
10130             "vpand   $tmp,$tmp,$tmp2\n\t"
10131             "movzwl  $dst,$src1\n\t"
10132             "movdl   $tmp3,$tmp\n\t"
10133             "andw    $dst,$tmp3\n\t"
10134             "shrl    $tmp3,0x16\n\t"
10135             "andw    $dst,$tmp3\n\t"
10136             "movswl  $dst,$dst\t! and reduction32S" %}
10137   ins_encode %{
10138     int vector_len = 0;
10139     __ vextracti64x4_high($tmp2$$XMMRegister, $src2$$XMMRegister);
10140     __ vpand($tmp2$$XMMRegister, $tmp2$$XMMRegister, $src2$$XMMRegister, 1);
10141     __ vextracti128_high($tmp$$XMMRegister, $tmp2$$XMMRegister);
10142     __ vpand($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10143     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE);
10144     __ vpand($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10145     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1);
10146     __ vpand($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10147     __ movzwl($dst$$Register, $src1$$Register);
10148     __ movdl($tmp3$$Register, $tmp$$XMMRegister);
10149     __ andw($dst$$Register, $tmp3$$Register);
10150     __ shrl($tmp3$$Register, 16);
10151     __ andw($dst$$Register, $tmp3$$Register);
10152     __ movswl($dst$$Register, $dst$$Register);
10153   %}
10154   ins_pipe( pipe_slow );
10155 %}
10156 
10157 instruct rsand2I_reduction_reg(rRegI dst, rRegI src1, vecD src2, regF tmp, regF tmp2) %{
10158   predicate(UseSSE > 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
10159   match(Set dst (AndReductionV src1 src2));
10160   effect(TEMP tmp, TEMP tmp2);
10161   format %{ "pshufd  $tmp2,$src2,0x1\n\t"
10162             "pand    $tmp2,$src2\n\t"
10163             "movd    $tmp,$src1\n\t"
10164             "pand    $tmp2,$tmp\n\t"
10165             "movd    $dst,$tmp2\t! and reduction2I" %}
10166   ins_encode %{
10167     __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0x1);
10168     __ pand($tmp2$$XMMRegister, $src2$$XMMRegister);
10169     __ movdl($tmp$$XMMRegister, $src1$$Register);
10170     __ pand($tmp2$$XMMRegister, $tmp$$XMMRegister);
10171     __ movdl($dst$$Register, $tmp2$$XMMRegister);
10172   %}
10173   ins_pipe( pipe_slow );
10174 %}
10175 
10176 instruct rsand4I_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp2) %{
10177   predicate(UseSSE > 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
10178   match(Set dst (AndReductionV src1 src2));
10179   effect(TEMP tmp, TEMP tmp2);
10180   format %{ "pshufd  $tmp2,$src2,0xE\n\t"
10181             "pand    $tmp2,$src2\n\t"
10182             "pshufd  $tmp,$tmp2,0x1\n\t"
10183             "pand    $tmp2,$tmp\n\t"
10184             "movd    $tmp,$src1\n\t"
10185             "pand    $tmp2,$tmp\n\t"
10186             "movd    $dst,$tmp2\t! and reduction4I" %}
10187   ins_encode %{
10188     __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0xE);
10189     __ pand($tmp2$$XMMRegister, $src2$$XMMRegister);
10190     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x1);
10191     __ pand($tmp2$$XMMRegister, $tmp$$XMMRegister);
10192     __ movdl($tmp$$XMMRegister, $src1$$Register);
10193     __ pand($tmp2$$XMMRegister, $tmp$$XMMRegister);
10194     __ movdl($dst$$Register, $tmp2$$XMMRegister);
10195   %}
10196   ins_pipe( pipe_slow );
10197 %}
10198 
10199 instruct rvand8I_reduction_reg(rRegI dst, rRegI src1, vecY src2, regF tmp, regF tmp2) %{
10200   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
10201   match(Set dst (AndReductionV src1 src2));
10202   effect(TEMP tmp, TEMP tmp2);
10203   format %{ "vextracti128_high  $tmp,$src2\n\t"
10204             "vpand    $tmp,$tmp,$src2\n\t"
10205             "vpshufd   $tmp2,$tmp,0xE\n\t"
10206             "vpand    $tmp,$tmp,$tmp2\n\t"
10207             "vpshufd   $tmp2,$tmp,0x1\n\t"
10208             "vpand    $tmp,$tmp,$tmp2\n\t"
10209             "movd     $tmp2,$src1\n\t"
10210             "vpand    $tmp2,$tmp,$tmp2\n\t"
10211             "movd     $dst,$tmp2\t! and reduction8I" %}
10212   ins_encode %{
10213     int vector_len = 0;
10214     __ vextracti128_high($tmp$$XMMRegister, $src2$$XMMRegister);
10215     __ vpand($tmp$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, vector_len);
10216     __ vpshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE, vector_len);
10217     __ vpand($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10218     __ vpshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1, vector_len);
10219     __ vpand($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10220     __ movdl($tmp2$$XMMRegister, $src1$$Register);
10221     __ vpand($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10222     __ movdl($dst$$Register, $tmp2$$XMMRegister);
10223   %}
10224   ins_pipe( pipe_slow );
10225 %}
10226 
10227 instruct rvand16I_reduction_reg(rRegI dst, rRegI src1, vecZ src2, regF tmp, regF tmp2, regF tmp3) %{
10228   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
10229   match(Set dst (AndReductionV src1 src2));
10230   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
10231   format %{ "vextracti64x4_high  $tmp3,$src2\n\t"
10232             "vpand  $tmp3,$tmp3,$src2\n\t"
10233             "vextracti128_high  $tmp,$tmp3\n\t"
10234             "vpand    $tmp,$tmp,$src2\n\t"
10235             "vpshufd   $tmp2,$tmp,0xE\n\t"
10236             "vpand    $tmp,$tmp,$tmp2\n\t"
10237             "vpshufd   $tmp2,$tmp,0x1\n\t"
10238             "vpand    $tmp,$tmp,$tmp2\n\t"
10239             "movd     $tmp2,$src1\n\t"
10240             "vpand    $tmp2,$tmp,$tmp2\n\t"
10241             "movd     $dst,$tmp2\t! and reduction16I" %}
10242   ins_encode %{
10243     __ vextracti64x4_high($tmp3$$XMMRegister, $src2$$XMMRegister);
10244     __ vpand($tmp3$$XMMRegister, $tmp3$$XMMRegister, $src2$$XMMRegister, 1);
10245     __ vextracti128_high($tmp$$XMMRegister, $tmp3$$XMMRegister);
10246     __ vpand($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp3$$XMMRegister, 0);
10247     __ vpshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE, 0);
10248     __ vpand($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, 0);
10249     __ vpshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1, 0);
10250     __ vpand($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, 0);
10251     __ movdl($tmp2$$XMMRegister, $src1$$Register);
10252     __ vpand($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, 0);
10253     __ movdl($dst$$Register, $tmp2$$XMMRegister);
10254   %}
10255   ins_pipe( pipe_slow );
10256 %}
10257 
10258 #ifdef _LP64
10259 instruct rsand2L_reduction_reg(rRegL dst, rRegL src1, vecX src2, regF tmp, regF tmp2) %{
10260   predicate(UseSSE >= 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
10261   match(Set dst (AndReductionV src1 src2));
10262   effect(TEMP tmp, TEMP tmp2);
10263   format %{ "pshufd  $tmp2,$src2,0xE\n\t"
10264             "pand    $tmp2,$src2\n\t"
10265             "movdq   $tmp,$src1\n\t"
10266             "pand    $tmp2,$tmp\n\t"
10267             "movq   $dst,$tmp2\t! and reduction2L" %}
10268   ins_encode %{
10269     __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0xE);
10270     __ pand($tmp2$$XMMRegister, $src2$$XMMRegister);
10271     __ movdq($tmp$$XMMRegister, $src1$$Register);
10272     __ pand($tmp2$$XMMRegister, $tmp$$XMMRegister);
10273     __ movq($dst$$Register, $tmp2$$XMMRegister);
10274   %}
10275   ins_pipe( pipe_slow );
10276 %}
10277 
10278 instruct rvand4L_reduction_reg_avx(rRegL dst, rRegL src1, vecY src2, regF tmp, regF tmp2) %{
10279   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
10280   match(Set dst (AndReductionV src1 src2));
10281   effect(TEMP tmp, TEMP tmp2);
10282   format %{ "vextracti128_high  $tmp,$src2\n\t"
10283             "vpand  $tmp2,$tmp,$src2\n\t"
10284             "vpshufd  $tmp,$tmp2,0xE\n\t"
10285             "vpand  $tmp2,$tmp2,$tmp\n\t"
10286             "movq   $tmp,$src1\n\t"
10287             "vpand  $tmp2,$tmp2,$tmp\n\t"
10288             "movq   $dst,$tmp2\t! and reduction4L" %}
10289   ins_encode %{
10290     __ vextracti128_high($tmp$$XMMRegister, $src2$$XMMRegister);
10291     __ vpand($tmp2$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, 0);
10292     __ vpshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE, 0);
10293     __ vpand($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, 0);
10294     __ movq($tmp$$XMMRegister, $src1$$Register);
10295     __ vpand($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, 0);
10296     __ movq($dst$$Register, $tmp2$$XMMRegister);
10297   %}
10298   ins_pipe( pipe_slow );
10299 %}
10300 
10301 instruct rvand8L_reduction_reg(rRegL dst, rRegL src1, vecZ src2, regF tmp, regF tmp2) %{
10302   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
10303   match(Set dst (AndReductionV src1 src2));
10304   effect(TEMP tmp, TEMP tmp2);
10305   format %{ "vextracti64x4_high  $tmp2,$src2\n\t"
10306             "vpandq  $tmp2,$tmp2,$src2\n\t"
10307             "vextracti128_high  $tmp,$tmp2\n\t"
10308             "vpandq  $tmp2,$tmp2,$tmp\n\t"
10309             "vpshufd  $tmp,$tmp2,0xE\n\t"
10310             "vpandq  $tmp2,$tmp2,$tmp\n\t"
10311             "movdq   $tmp,$src1\n\t"
10312             "vpandq  $tmp2,$tmp2,$tmp\n\t"
10313             "movdq   $dst,$tmp2\t! and reduction8L" %}
10314   ins_encode %{
10315     __ vextracti64x4_high($tmp2$$XMMRegister, $src2$$XMMRegister);
10316     __ vpandq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $src2$$XMMRegister, 1);
10317     __ vextracti128_high($tmp$$XMMRegister, $tmp2$$XMMRegister);
10318     __ vpandq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, 0);
10319     __ vpshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE, 0);
10320     __ vpandq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, 0);
10321     __ movdq($tmp$$XMMRegister, $src1$$Register);
10322     __ vpandq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, 0);
10323     __ movdq($dst$$Register, $tmp2$$XMMRegister);
10324   %}
10325   ins_pipe( pipe_slow );
10326 %}
10327 #endif
10328 
10329 instruct rsor8B_reduction_reg(rRegI dst, rRegI src1, vecD src2, regF tmp, rRegI tmp2) %{
10330   predicate(UseSSE > 3 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
10331   match(Set dst (OrReductionV src1 src2));
10332   effect(TEMP tmp, TEMP tmp2, TEMP dst);
10333   format %{
10334             "pshufd  $tmp,$src2,0x1\n\t"
10335             "por    $tmp,$src2\n\t"
10336             "movzbl  $dst,$src1\n\t"
10337             "pextrb  $tmp2,$tmp, 0x0\n\t"
10338             "orl    $dst,$tmp2\n\t"
10339             "pextrb  $tmp2,$tmp, 0x1\n\t"
10340             "orl    $dst,$tmp2\n\t"
10341             "pextrb  $tmp2,$tmp, 0x2\n\t"
10342             "orl    $dst,$tmp2\n\t"
10343             "pextrb  $tmp2,$tmp, 0x3\n\t"
10344             "orl    $dst,$tmp2\n\t"
10345             "movsbl  $dst,$dst\t! or reduction8B" %}
10346   ins_encode %{
10347     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x1);
10348     __ por($tmp$$XMMRegister, $src2$$XMMRegister);
10349     __ movzbl($dst$$Register, $src1$$Register);
10350     __ pextrb($tmp2$$Register, $tmp$$XMMRegister, 0x0);
10351     __ orl($dst$$Register, $tmp2$$Register);
10352     __ pextrb($tmp2$$Register, $tmp$$XMMRegister, 0x1);
10353     __ orl($dst$$Register, $tmp2$$Register);
10354     __ pextrb($tmp2$$Register, $tmp$$XMMRegister, 0x2);
10355     __ orl($dst$$Register, $tmp2$$Register);
10356     __ pextrb($tmp2$$Register, $tmp$$XMMRegister, 0x3);
10357     __ orl($dst$$Register, $tmp2$$Register);
10358     __ movsbl($dst$$Register, $dst$$Register);
10359   %}
10360   ins_pipe( pipe_slow );
10361 %}
10362 
10363 instruct rsor16B_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp2, rRegI tmp3) %{
10364   predicate(UseSSE > 3 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
10365   match(Set dst (OrReductionV src1 src2));
10366   effect(TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP dst);
10367   format %{ "pshufd  $tmp,$src2,0xE\n\t"
10368             "por    $tmp,$src2\n\t"
10369             "pshufd  $tmp2,$tmp,0x1\n\t"
10370             "por    $tmp,$tmp,$tmp2\n\t"
10371             "movzbl  $dst,$src1\n\t"
10372             "pextrb  $tmp3,$tmp, 0x0\n\t"
10373             "orl    $dst,$tmp3\n\t"
10374             "pextrb  $tmp3,$tmp, 0x1\n\t"
10375             "orl    $dst,$tmp3\n\t"
10376             "pextrb  $tmp3,$tmp, 0x2\n\t"
10377             "orl    $dst,$tmp3\n\t"
10378             "pextrb  $tmp3,$tmp, 0x3\n\t"
10379             "orl    $dst,$tmp3\n\t"
10380             "movsbl  $dst,$dst\t! or reduction16B" %}
10381   ins_encode %{
10382     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
10383     __ por($tmp$$XMMRegister, $src2$$XMMRegister);
10384     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1);
10385     __ por($tmp$$XMMRegister, $tmp2$$XMMRegister);
10386     __ movzbl($dst$$Register, $src1$$Register);
10387     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x0);
10388     __ orl($dst$$Register, $tmp3$$Register);
10389     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x1);
10390     __ orl($dst$$Register, $tmp3$$Register);
10391     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x2);
10392     __ orl($dst$$Register, $tmp3$$Register);
10393     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x3);
10394     __ orl($dst$$Register, $tmp3$$Register);
10395     __ movsbl($dst$$Register, $dst$$Register);
10396   %}
10397   ins_pipe( pipe_slow );
10398 %}
10399 
10400 instruct rvor32B_reduction_reg(rRegI dst, rRegI src1, vecY src2, regF tmp, regF tmp2, rRegI tmp3) %{
10401   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
10402   match(Set dst (OrReductionV src1 src2));
10403   effect(TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP dst);
10404    format %{ "vextracti128_high  $tmp,$src2\n\t"
10405             "vpor   $tmp,$tmp,$src2\n\t"
10406             "pshufd  $tmp2,$tmp,0xE\n\t"
10407             "vpor   $tmp,$tmp,$tmp2\n\t"
10408             "pshufd  $tmp2,$tmp,0x1\n\t"
10409             "vpor   $tmp,$tmp,$tmp2\n\t"
10410             "movzbl  $dst,$src1\n\t"
10411             "pextrb  $tmp3,$tmp, 0x0\n\t"
10412             "orl    $dst,$tmp3\n\t"
10413             "pextrb  $tmp3,$tmp, 0x1\n\t"
10414             "orl    $dst,$tmp3\n\t"
10415             "pextrb  $tmp3,$tmp, 0x2\n\t"
10416             "orl    $dst,$tmp3\n\t"
10417             "pextrb  $tmp3,$tmp, 0x3\n\t"
10418             "orl    $dst,$tmp3\n\t"
10419             "movsbl  $dst,$dst\t! or reduction32B" %}
10420   ins_encode %{
10421     int vector_len = 0;
10422     __ vextracti128_high($tmp$$XMMRegister, $src2$$XMMRegister);
10423     __ vpor($tmp$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, vector_len);
10424     __ vpshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE, vector_len);
10425     __ vpor($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10426     __ vpshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1, vector_len);
10427     __ vpor($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10428     __ movzbl($dst$$Register, $src1$$Register);
10429     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x0);
10430     __ orl($dst$$Register, $tmp3$$Register);
10431     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x1);
10432     __ orl($dst$$Register, $tmp3$$Register);
10433     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x2);
10434     __ orl($dst$$Register, $tmp3$$Register);
10435     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x3);
10436     __ orl($dst$$Register, $tmp3$$Register);
10437     __ movsbl($dst$$Register, $dst$$Register);
10438   %}
10439   ins_pipe( pipe_slow );
10440 %}
10441 
10442 instruct rvor64B_reduction_reg(rRegI dst, rRegI src1, vecZ src2, regF tmp, regF tmp2, rRegI tmp3) %{
10443   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
10444   match(Set dst (OrReductionV src1 src2));
10445   effect(TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP dst);
10446   format %{ "vextracti64x4_high  $tmp2,$src2\n\t"
10447             "vpor   $tmp2,$tmp2,$src2\n\t"
10448             "vextracti128_high  $tmp,$tmp2\n\t"
10449             "vpor   $tmp,$tmp,$tmp2\n\t"
10450             "pshufd  $tmp2,$tmp,0xE\n\t"
10451             "vpor   $tmp,$tmp,$tmp2\n\t"
10452             "pshufd  $tmp2,$tmp,0x1\n\t"
10453             "vpor   $tmp,$tmp,$tmp2\n\t"
10454             "movzbl  $dst,$src1\n\t"
10455             "movdl   $tmp3,$tmp\n\t"
10456             "orl    $dst,$tmp3\n\t"
10457             "shrl    $tmp3,0x8\n\t"
10458             "orl    $dst,$tmp3\n\t"
10459             "shrl    $tmp3,0x8\n\t"
10460             "orl    $dst,$tmp3\n\t"
10461             "shrl    $tmp3,0x8\n\t"
10462             "orl    $dst,$tmp3\n\t"
10463             "movsbl  $dst,$dst\t! or reduction64B" %}
10464   ins_encode %{
10465     int vector_len = 0;
10466     __ vextracti64x4_high($tmp2$$XMMRegister, $src2$$XMMRegister);
10467     __ vpor($tmp2$$XMMRegister, $tmp2$$XMMRegister, $src2$$XMMRegister, 1);
10468     __ vextracti128_high($tmp$$XMMRegister, $tmp2$$XMMRegister);
10469     __ vpor($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10470     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE);
10471     __ vpor($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10472     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1);
10473     __ vpor($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10474     __ movzbl($dst$$Register, $src1$$Register);
10475     __ movdl($tmp3$$Register, $tmp$$XMMRegister);
10476     __ orl($dst$$Register, $tmp3$$Register);
10477     __ shrl($tmp3$$Register, 8);
10478     __ orl($dst$$Register, $tmp3$$Register);
10479     __ shrl($tmp3$$Register, 8);
10480     __ orl($dst$$Register, $tmp3$$Register);
10481     __ shrl($tmp3$$Register, 8);
10482     __ orl($dst$$Register, $tmp3$$Register);
10483     __ movsbl($dst$$Register, $dst$$Register);
10484   %}
10485   ins_pipe( pipe_slow );
10486 %}
10487 
10488 instruct rsor4S_reduction_reg(rRegI dst, rRegI src1, vecD src2, regF tmp, rRegI tmp2) %{
10489   predicate(UseSSE > 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
10490   match(Set dst (OrReductionV src1 src2));
10491   effect(TEMP tmp, TEMP tmp2, TEMP dst);
10492   format %{
10493             "pshufd  $tmp,$src2,0x1\n\t"
10494             "por    $tmp,$src2\n\t"
10495             "movzwl  $dst,$src1\n\t"
10496             "pextrw  $tmp2,$tmp, 0x0\n\t"
10497             "orw    $dst,$tmp2\n\t"
10498             "pextrw  $tmp2,$tmp, 0x1\n\t"
10499             "orw    $dst,$tmp2\n\t"
10500             "movswl  $dst,$dst\t! or reduction4S" %}
10501   ins_encode %{
10502     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x1);
10503     __ por($tmp$$XMMRegister, $src2$$XMMRegister);
10504     __ movzwl($dst$$Register, $src1$$Register);
10505     __ pextrw($tmp2$$Register, $tmp$$XMMRegister, 0x0);
10506     __ orw($dst$$Register, $tmp2$$Register);
10507     __ pextrw($tmp2$$Register, $tmp$$XMMRegister, 0x1);
10508     __ orw($dst$$Register, $tmp2$$Register);
10509     __ movswl($dst$$Register, $dst$$Register);
10510   %}
10511   ins_pipe( pipe_slow );
10512 %}
10513 
10514 instruct rsor8S_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp2, rRegI tmp3) %{
10515   predicate(UseSSE > 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
10516   match(Set dst (OrReductionV src1 src2));
10517   effect(TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP dst);
10518   format %{ "pshufd  $tmp,$src2,0xE\n\t"
10519             "por    $tmp,$src2\n\t"
10520             "pshufd  $tmp2,$tmp,0x1\n\t"
10521             "por    $tmp,$tmp,$tmp2\n\t"
10522             "movzwl  $dst,$src1\n\t"
10523             "pextrw  $tmp3,$tmp, 0x0\n\t"
10524             "orw    $dst,$tmp3\n\t"
10525             "pextrw  $tmp3,$tmp, 0x1\n\t"
10526             "orw    $dst,$tmp3\n\t"
10527             "movswl  $dst,$dst\t! or reduction8S" %}
10528   ins_encode %{
10529     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
10530     __ por($tmp$$XMMRegister, $src2$$XMMRegister);
10531     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1);
10532     __ por($tmp$$XMMRegister, $tmp2$$XMMRegister);
10533     __ movzwl($dst$$Register, $src1$$Register);
10534     __ pextrw($tmp3$$Register, $tmp$$XMMRegister, 0x0);
10535     __ orw($dst$$Register, $tmp3$$Register);
10536     __ pextrw($tmp3$$Register, $tmp$$XMMRegister, 0x1);
10537     __ orw($dst$$Register, $tmp3$$Register);
10538     __ movswl($dst$$Register, $dst$$Register);
10539   %}
10540   ins_pipe( pipe_slow );
10541 %}
10542 
10543 instruct rvor16S_reduction_reg(rRegI dst, rRegI src1, vecY src2, regF tmp, regF tmp2, rRegI tmp3) %{
10544   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
10545   match(Set dst (OrReductionV src1 src2));
10546   effect(TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP dst);
10547    format %{ "vextracti128_high  $tmp,$src2\n\t"
10548             "vpor   $tmp,$tmp,$src2\n\t"
10549             "pshufd  $tmp2,$tmp,0xE\n\t"
10550             "vpor   $tmp,$tmp,$tmp2\n\t"
10551             "pshufd  $tmp2,$tmp,0x1\n\t"
10552             "vpor   $tmp,$tmp,$tmp2\n\t"
10553             "movzwl  $dst,$src1\n\t"
10554             "pextrw  $tmp3,$tmp, 0x0\n\t"
10555             "orw    $dst,$tmp3\n\t"
10556             "pextrw  $tmp3,$tmp, 0x1\n\t"
10557             "orw    $dst,$tmp3\n\t"
10558             "movswl  $dst,$dst\t! or reduction16S" %}
10559   ins_encode %{
10560     int vector_len = 0;
10561     __ vextracti128_high($tmp$$XMMRegister, $src2$$XMMRegister);
10562     __ vpor($tmp$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, vector_len);
10563     __ vpshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE, vector_len);
10564     __ vpor($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10565     __ vpshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1, vector_len);
10566     __ vpor($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10567     __ movzwl($dst$$Register, $src1$$Register);
10568     __ pextrw($tmp3$$Register, $tmp$$XMMRegister, 0x0);
10569     __ orw($dst$$Register, $tmp3$$Register);
10570     __ pextrw($tmp3$$Register, $tmp$$XMMRegister, 0x1);
10571     __ orw($dst$$Register, $tmp3$$Register);
10572     __ movswl($dst$$Register, $dst$$Register);
10573   %}
10574   ins_pipe( pipe_slow );
10575 %}
10576 
10577 instruct rvor32S_reduction_reg(rRegI dst, rRegI src1, vecZ src2, regF tmp, regF tmp2, rRegI tmp3) %{
10578   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
10579   match(Set dst (OrReductionV src1 src2));
10580   effect(TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP dst);
10581   format %{ "vextracti64x4_high  $tmp2,$src2\n\t"
10582             "vpor   $tmp2,$tmp2,$src2\n\t"
10583             "vextracti128_high  $tmp,$tmp2\n\t"
10584             "vpor   $tmp,$tmp,$tmp2\n\t"
10585             "pshufd  $tmp2,$tmp,0xE\n\t"
10586             "vpor   $tmp,$tmp,$tmp2\n\t"
10587             "pshufd  $tmp2,$tmp,0x1\n\t"
10588             "vpor   $tmp,$tmp,$tmp2\n\t"
10589             "movzwl  $dst,$src1\n\t"
10590             "movdl   $tmp3,$tmp\n\t"
10591             "orw    $dst,$tmp3\n\t"
10592             "shrl    $tmp3,0x16\n\t"
10593             "orw    $dst,$tmp3\n\t"
10594             "movswl  $dst,$dst\t! or reduction32S" %}
10595   ins_encode %{
10596     int vector_len = 0;
10597     __ vextracti64x4_high($tmp2$$XMMRegister, $src2$$XMMRegister);
10598     __ vpor($tmp2$$XMMRegister, $tmp2$$XMMRegister, $src2$$XMMRegister, 1);
10599     __ vextracti128_high($tmp$$XMMRegister, $tmp2$$XMMRegister);
10600     __ vpor($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10601     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE);
10602     __ vpor($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10603     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1);
10604     __ vpor($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10605     __ movzwl($dst$$Register, $src1$$Register);
10606     __ movdl($tmp3$$Register, $tmp$$XMMRegister);
10607     __ orw($dst$$Register, $tmp3$$Register);
10608     __ shrl($tmp3$$Register, 16);
10609     __ orw($dst$$Register, $tmp3$$Register);
10610     __ movswl($dst$$Register, $dst$$Register);
10611   %}
10612   ins_pipe( pipe_slow );
10613 %}
10614 
10615 instruct rsor2I_reduction_reg(rRegI dst, rRegI src1, vecD src2, regF tmp, regF tmp2) %{
10616   predicate(UseSSE > 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
10617   match(Set dst (OrReductionV src1 src2));
10618   effect(TEMP tmp, TEMP tmp2);
10619   format %{ "pshufd  $tmp2,$src2,0x1\n\t"
10620             "por    $tmp2,$src2\n\t"
10621             "movd    $tmp,$src1\n\t"
10622             "por    $tmp2,$tmp\n\t"
10623             "movd    $dst,$tmp2\t! or reduction2I" %}
10624   ins_encode %{
10625     __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0x1);
10626     __ por($tmp2$$XMMRegister, $src2$$XMMRegister);
10627     __ movdl($tmp$$XMMRegister, $src1$$Register);
10628     __ por($tmp2$$XMMRegister, $tmp$$XMMRegister);
10629     __ movdl($dst$$Register, $tmp2$$XMMRegister);
10630   %}
10631   ins_pipe( pipe_slow );
10632 %}
10633 
10634 instruct rsor4I_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp2) %{
10635   predicate(UseSSE > 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
10636   match(Set dst (OrReductionV src1 src2));
10637   effect(TEMP tmp, TEMP tmp2);
10638   format %{ "pshufd  $tmp2,$src2,0xE\n\t"
10639             "por    $tmp2,$src2\n\t"
10640             "pshufd  $tmp,$tmp2,0x1\n\t"
10641             "por    $tmp2,$tmp\n\t"
10642             "movd    $tmp,$src1\n\t"
10643             "por    $tmp2,$tmp\n\t"
10644             "movd    $dst,$tmp2\t! or reduction4I" %}
10645   ins_encode %{
10646     __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0xE);
10647     __ por($tmp2$$XMMRegister, $src2$$XMMRegister);
10648     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x1);
10649     __ por($tmp2$$XMMRegister, $tmp$$XMMRegister);
10650     __ movdl($tmp$$XMMRegister, $src1$$Register);
10651     __ por($tmp2$$XMMRegister, $tmp$$XMMRegister);
10652     __ movdl($dst$$Register, $tmp2$$XMMRegister);
10653   %}
10654   ins_pipe( pipe_slow );
10655 %}
10656 
10657 instruct rvor8I_reduction_reg(rRegI dst, rRegI src1, vecY src2, regF tmp, regF tmp2) %{
10658   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
10659   match(Set dst (OrReductionV src1 src2));
10660   effect(TEMP tmp, TEMP tmp2);
10661   format %{ "vextracti128_high  $tmp,$src2\n\t"
10662             "vpor    $tmp,$tmp,$src2\n\t"
10663             "vpshufd   $tmp2,$tmp,0xE\t"
10664             "vpor    $tmp,$tmp,$tmp2\n\t"
10665             "vpshufd   $tmp2,$tmp,0x1\t"
10666             "vpor    $tmp,$tmp,$tmp2\n\t"
10667             "movd     $tmp2,$src1\n\t"
10668             "vpor    $tmp2,$tmp,$tmp2\n\t"
10669             "movd     $dst,$tmp2\t! or reduction8I" %}
10670   ins_encode %{
10671     int vector_len = 0;
10672     __ vextracti128_high($tmp$$XMMRegister, $src2$$XMMRegister);
10673     __ vpor($tmp$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, vector_len);
10674     __ vpshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE, vector_len);
10675     __ vpor($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10676     __ vpshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1, vector_len);
10677     __ vpor($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10678     __ movdl($tmp2$$XMMRegister, $src1$$Register);
10679     __ vpor($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10680     __ movdl($dst$$Register, $tmp2$$XMMRegister);
10681   %}
10682   ins_pipe( pipe_slow );
10683 %}
10684 
10685 instruct rvor16I_reduction_reg(rRegI dst, rRegI src1, vecZ src2, regF tmp, regF tmp2, regF tmp3) %{
10686   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
10687   match(Set dst (OrReductionV src1 src2));
10688   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
10689   format %{ "vextracti64x4_high  $tmp3,$src2\n\t"
10690             "vpor  $tmp3,$tmp3,$src2\n\t"
10691             "vextracti128_high  $tmp,$tmp3\n\t"
10692             "vpor    $tmp,$tmp,$src2\n\t"
10693             "vpshufd   $tmp2,$tmp,0xE\t"
10694             "vpor    $tmp,$tmp,$tmp2\n\t"
10695             "vpshufd   $tmp2,$tmp,0x1\t"
10696             "vpor    $tmp,$tmp,$tmp2\n\t"
10697             "movd     $tmp2,$src1\n\t"
10698             "vpor    $tmp2,$tmp,$tmp2\n\t"
10699             "movd     $dst,$tmp2\t! or reduction16I" %}
10700   ins_encode %{
10701     int vector_len = 0;
10702     __ vextracti64x4_high($tmp3$$XMMRegister, $src2$$XMMRegister);
10703     __ vpor($tmp3$$XMMRegister, $tmp3$$XMMRegister, $src2$$XMMRegister, 1);
10704     __ vextracti128_high($tmp$$XMMRegister, $tmp3$$XMMRegister);
10705     __ vpor($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp3$$XMMRegister, vector_len);
10706     __ vpshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE, vector_len);
10707     __ vpor($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10708     __ vpshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1, vector_len);
10709     __ vpor($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10710     __ movdl($tmp2$$XMMRegister, $src1$$Register);
10711     __ vpor($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10712     __ movdl($dst$$Register, $tmp2$$XMMRegister);
10713   %}
10714   ins_pipe( pipe_slow );
10715 %}
10716 
10717 instruct rsor2L_reduction_reg(rRegL dst, rRegL src1, vecX src2, regF tmp, regF tmp2) %{
10718   predicate(UseSSE >= 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
10719   match(Set dst (OrReductionV src1 src2));
10720   effect(TEMP tmp, TEMP tmp2);
10721   format %{ "pshufd  $tmp2,$src2,0xE\n\t"
10722             "por    $tmp2,$src2\n\t"
10723             "movdq   $tmp,$src1\n\t"
10724             "por    $tmp2,$tmp\n\t"
10725             "movq   $dst,$tmp2\t! or reduction2L" %}
10726   ins_encode %{
10727     __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0xE);
10728     __ por($tmp2$$XMMRegister, $src2$$XMMRegister);
10729     __ movdq($tmp$$XMMRegister, $src1$$Register);
10730     __ por($tmp2$$XMMRegister, $tmp$$XMMRegister);
10731     __ movq($dst$$Register, $tmp2$$XMMRegister);
10732   %}
10733   ins_pipe( pipe_slow );
10734 %}
10735 
10736 instruct rvor4L_reduction_reg_avx(rRegL dst, rRegL src1, vecY src2, regF tmp, regF tmp2) %{
10737   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
10738   match(Set dst (OrReductionV src1 src2));
10739   effect(TEMP tmp, TEMP tmp2);
10740   format %{ "vextracti128_high  $tmp,$src2\n\t"
10741             "vpor  $tmp2,$tmp,$src2\n\t"
10742             "vpshufd  $tmp,$tmp2,0xE\t"
10743             "vpor  $tmp2,$tmp2,$tmp\n\t"
10744             "movq   $tmp,$src1\n\t"
10745             "vpor  $tmp2,$tmp2,$tmp\n\t"
10746             "movq   $dst,$tmp2\t! or reduction4L" %}
10747   ins_encode %{
10748     int vector_len = 0;
10749     __ vextracti128_high($tmp$$XMMRegister, $src2$$XMMRegister);
10750     __ vpor($tmp2$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, vector_len);
10751     __ vpshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE, vector_len);
10752     __ vpor($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, vector_len);
10753     __ movq($tmp$$XMMRegister, $src1$$Register);
10754     __ vpor($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, vector_len);
10755     __ movq($dst$$Register, $tmp2$$XMMRegister);
10756   %}
10757   ins_pipe( pipe_slow );
10758 %}
10759 
10760 #ifdef _LP64
10761 instruct rvor8L_reduction_reg(rRegL dst, rRegL src1, vecZ src2, regF tmp, regF tmp2) %{
10762   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
10763   match(Set dst (OrReductionV src1 src2));
10764   effect(TEMP tmp, TEMP tmp2);
10765   format %{ "vextracti64x4_high  $tmp2,$src2\n\t"
10766             "vporq  $tmp2,$tmp2,$src2\n\t"
10767             "vextracti128_high  $tmp,$tmp2\n\t"
10768             "vporq  $tmp2,$tmp2,$tmp\n\t"
10769             "vpshufd  $tmp,$tmp2,0xE\t"
10770             "vporq  $tmp2,$tmp2,$tmp\n\t"
10771             "movdq   $tmp,$src1\n\t"
10772             "vporq  $tmp2,$tmp2,$tmp\n\t"
10773             "movdq   $dst,$tmp2\t! or reduction8L" %}
10774   ins_encode %{
10775     int vector_len = 0;
10776     __ vextracti64x4_high($tmp2$$XMMRegister, $src2$$XMMRegister);
10777     __ vporq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $src2$$XMMRegister, 1);
10778     __ vextracti128_high($tmp$$XMMRegister, $tmp2$$XMMRegister);
10779     __ vporq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, vector_len);
10780     __ vpshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE, vector_len);
10781     __ vporq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, vector_len);
10782     __ movdq($tmp$$XMMRegister, $src1$$Register);
10783     __ vporq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, vector_len);
10784     __ movdq($dst$$Register, $tmp2$$XMMRegister);
10785   %}
10786   ins_pipe( pipe_slow );
10787 %}
10788 #endif
10789 
10790 instruct rsxor8B_reduction_reg(rRegI dst, rRegI src1, vecD src2, regF tmp, rRegI tmp2) %{
10791   predicate(UseSSE > 3 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
10792   match(Set dst (XorReductionV src1 src2));
10793   effect(TEMP tmp, TEMP tmp2, TEMP dst);
10794   format %{
10795             "pshufd  $tmp,$src2,0x1\n\t"
10796             "pxor    $tmp,$src2\n\t"
10797             "movzbl  $dst,$src1\n\t"
10798             "pextrb  $tmp2,$tmp, 0x0\n\t"
10799             "xorl    $dst,$tmp2\n\t"
10800             "pextrb  $tmp2,$tmp, 0x1\n\t"
10801             "xorl    $dst,$tmp2\n\t"
10802             "pextrb  $tmp2,$tmp, 0x2\n\t"
10803             "xorl    $dst,$tmp2\n\t"
10804             "pextrb  $tmp2,$tmp, 0x3\n\t"
10805             "xorl    $dst,$tmp2\n\t"
10806             "movsbl  $dst,$dst\t! xor reduction8B" %}
10807   ins_encode %{
10808     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x1);
10809     __ pxor($tmp$$XMMRegister, $src2$$XMMRegister);
10810     __ movzbl($dst$$Register, $src1$$Register);
10811     __ pextrb($tmp2$$Register, $tmp$$XMMRegister, 0x0);
10812     __ xorl($dst$$Register, $tmp2$$Register);
10813     __ pextrb($tmp2$$Register, $tmp$$XMMRegister, 0x1);
10814     __ xorl($dst$$Register, $tmp2$$Register);
10815     __ pextrb($tmp2$$Register, $tmp$$XMMRegister, 0x2);
10816     __ xorl($dst$$Register, $tmp2$$Register);
10817     __ pextrb($tmp2$$Register, $tmp$$XMMRegister, 0x3);
10818     __ xorl($dst$$Register, $tmp2$$Register);
10819     __ movsbl($dst$$Register, $dst$$Register);
10820   %}
10821   ins_pipe( pipe_slow );
10822 %}
10823 
10824 instruct rsxor16B_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp2, rRegI tmp3) %{
10825   predicate(UseSSE > 3 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
10826   match(Set dst (XorReductionV src1 src2));
10827   effect(TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP dst);
10828   format %{ "pshufd  $tmp,$src2,0xE\n\t"
10829             "pxor    $tmp,$src2\n\t"
10830             "pshufd  $tmp2,$tmp,0x1\n\t"
10831             "pxor    $tmp,$tmp,$tmp2\n\t"
10832             "movzbl  $dst,$src1\n\t"
10833             "pextrb  $tmp3,$tmp, 0x0\n\t"
10834             "xorl    $dst,$tmp3\n\t"
10835             "pextrb  $tmp3,$tmp, 0x1\n\t"
10836             "xorl    $dst,$tmp3\n\t"
10837             "pextrb  $tmp3,$tmp, 0x2\n\t"
10838             "xorl    $dst,$tmp3\n\t"
10839             "pextrb  $tmp3,$tmp, 0x3\n\t"
10840             "xorl    $dst,$tmp3\n\t"
10841             "movsbl  $dst,$dst\t! xor reduction16B" %}
10842   ins_encode %{
10843     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
10844     __ pxor($tmp$$XMMRegister, $src2$$XMMRegister);
10845     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1);
10846     __ pxor($tmp$$XMMRegister, $tmp2$$XMMRegister);
10847     __ movzbl($dst$$Register, $src1$$Register);
10848     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x0);
10849     __ xorl($dst$$Register, $tmp3$$Register);
10850     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x1);
10851     __ xorl($dst$$Register, $tmp3$$Register);
10852     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x2);
10853     __ xorl($dst$$Register, $tmp3$$Register);
10854     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x3);
10855     __ xorl($dst$$Register, $tmp3$$Register);
10856     __ movsbl($dst$$Register, $dst$$Register);
10857   %}
10858   ins_pipe( pipe_slow );
10859 %}
10860 
10861 instruct rvxor32B_reduction_reg(rRegI dst, rRegI src1, vecY src2, regF tmp, regF tmp2, rRegI tmp3) %{
10862   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
10863   match(Set dst (XorReductionV src1 src2));
10864   effect(TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP dst);
10865    format %{ "vextracti128_high  $tmp,$src2\n\t"
10866             "vpxor   $tmp,$tmp,$src2\n\t"
10867             "pshufd  $tmp2,$tmp,0xE\n\t"
10868             "vpxor   $tmp,$tmp,$tmp2\n\t"
10869             "pshufd  $tmp2,$tmp,0x1\n\t"
10870             "vpxor   $tmp,$tmp,$tmp2\n\t"
10871             "movzbl  $dst,$src1\n\t"
10872             "pextrb  $tmp3,$tmp, 0x0\n\t"
10873             "xorl    $dst,$tmp3\n\t"
10874             "pextrb  $tmp3,$tmp, 0x1\n\t"
10875             "xorl    $dst,$tmp3\n\t"
10876             "pextrb  $tmp3,$tmp, 0x2\n\t"
10877             "xorl    $dst,$tmp3\n\t"
10878             "pextrb  $tmp3,$tmp, 0x3\n\t"
10879             "xorl    $dst,$tmp3\n\t"
10880             "movsbl  $dst,$dst\t! xor reduction32B" %}
10881   ins_encode %{
10882     int vector_len = 0;
10883     __ vextracti128_high($tmp$$XMMRegister, $src2$$XMMRegister);
10884     __ vpxor($tmp$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, vector_len);
10885     __ vpshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE, vector_len);
10886     __ vpxor($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10887     __ vpshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1, vector_len);
10888     __ vpxor($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10889     __ movzbl($dst$$Register, $src1$$Register);
10890     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x0);
10891     __ xorl($dst$$Register, $tmp3$$Register);
10892     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x1);
10893     __ xorl($dst$$Register, $tmp3$$Register);
10894     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x2);
10895     __ xorl($dst$$Register, $tmp3$$Register);
10896     __ pextrb($tmp3$$Register, $tmp$$XMMRegister, 0x3);
10897     __ xorl($dst$$Register, $tmp3$$Register);
10898     __ movsbl($dst$$Register, $dst$$Register);
10899   %}
10900   ins_pipe( pipe_slow );
10901 %}
10902 
10903 instruct rvxor64B_reduction_reg(rRegI dst, rRegI src1, vecZ src2, regF tmp, regF tmp2, rRegI tmp3) %{
10904   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
10905   match(Set dst (XorReductionV src1 src2));
10906   effect(TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP dst);
10907   format %{ "vextracti64x4_high  $tmp2,$src2\n\t"
10908             "vpxor   $tmp2,$tmp2,$src2\n\t"
10909             "vextracti128_high  $tmp,$tmp2\n\t"
10910             "vpxor   $tmp,$tmp,$tmp2\n\t"
10911             "pshufd  $tmp2,$tmp,0xE\n\t"
10912             "vpxor   $tmp,$tmp,$tmp2\n\t"
10913             "pshufd  $tmp2,$tmp,0x1\n\t"
10914             "vpxor   $tmp,$tmp,$tmp2\n\t"
10915             "movzbl  $dst,$src1\n\t"
10916             "movdl   $tmp3,$tmp\n\t"
10917             "xorl    $dst,$tmp3\n\t"
10918             "shrl    $tmp3,0x8\n\t"
10919             "xorl    $dst,$tmp3\n\t"
10920             "shrl    $tmp3,0x8\n\t"
10921             "xorl    $dst,$tmp3\n\t"
10922             "shrl    $tmp3,0x8\n\t"
10923             "xorl    $dst,$tmp3\n\t"
10924             "movsbl  $dst,$dst\t! xor reduction64B" %}
10925   ins_encode %{
10926     int vector_len = 0;
10927     __ vextracti64x4_high($tmp2$$XMMRegister, $src2$$XMMRegister);
10928     __ vpxor($tmp2$$XMMRegister, $tmp2$$XMMRegister, $src2$$XMMRegister, 1);
10929     __ vextracti128_high($tmp$$XMMRegister, $tmp2$$XMMRegister);
10930     __ vpxor($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10931     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE);
10932     __ vpxor($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10933     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1);
10934     __ vpxor($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
10935     __ movzbl($dst$$Register, $src1$$Register);
10936     __ movdl($tmp3$$Register, $tmp$$XMMRegister);
10937     __ xorl($dst$$Register, $tmp3$$Register);
10938     __ shrl($tmp3$$Register, 8);
10939     __ xorl($dst$$Register, $tmp3$$Register);
10940     __ shrl($tmp3$$Register, 8);
10941     __ xorl($dst$$Register, $tmp3$$Register);
10942     __ shrl($tmp3$$Register, 8);
10943     __ xorl($dst$$Register, $tmp3$$Register);
10944     __ movsbl($dst$$Register, $dst$$Register);
10945   %}
10946   ins_pipe( pipe_slow );
10947 %}
10948 
10949 instruct rsxor4S_reduction_reg(rRegI dst, rRegI src1, vecD src2, regF tmp, rRegI tmp2) %{
10950   predicate(UseSSE > 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
10951   match(Set dst (XorReductionV src1 src2));
10952   effect(TEMP tmp, TEMP tmp2, TEMP dst);
10953   format %{
10954             "pshufd  $tmp,$src2,0x1\n\t"
10955             "pxor    $tmp,$src2\n\t"
10956             "movzwl  $dst,$src1\n\t"
10957             "pextrw  $tmp2,$tmp, 0x0\n\t"
10958             "xorw    $dst,$tmp2\n\t"
10959             "pextrw  $tmp2,$tmp, 0x1\n\t"
10960             "xorw    $dst,$tmp2\n\t"
10961             "movswl  $dst,$dst\t! xor reduction4S" %}
10962   ins_encode %{
10963     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0x1);
10964     __ pxor($tmp$$XMMRegister, $src2$$XMMRegister);
10965     __ movzwl($dst$$Register, $src1$$Register);
10966     __ pextrw($tmp2$$Register, $tmp$$XMMRegister, 0x0);
10967     __ xorw($dst$$Register, $tmp2$$Register);
10968     __ pextrw($tmp2$$Register, $tmp$$XMMRegister, 0x1);
10969     __ xorw($dst$$Register, $tmp2$$Register);
10970     __ movswl($dst$$Register, $dst$$Register);
10971   %}
10972   ins_pipe( pipe_slow );
10973 %}
10974 
10975 instruct rsxor8S_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp2, rRegI tmp3) %{
10976   predicate(UseSSE > 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
10977   match(Set dst (XorReductionV src1 src2));
10978   effect(TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP dst);
10979   format %{ "pshufd  $tmp,$src2,0xE\n\t"
10980             "pxor    $tmp,$src2\n\t"
10981             "pshufd  $tmp2,$tmp,0x1\n\t"
10982             "pxor    $tmp,$tmp,$tmp2\n\t"
10983             "movzwl  $dst,$src1\n\t"
10984             "pextrw  $tmp3,$tmp, 0x0\n\t"
10985             "xorw    $dst,$tmp3\n\t"
10986             "pextrw  $tmp3,$tmp, 0x1\n\t"
10987             "xorw    $dst,$tmp3\n\t"
10988             "movswl  $dst,$dst\t! xor reduction8S" %}
10989   ins_encode %{
10990     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 0xE);
10991     __ pxor($tmp$$XMMRegister, $src2$$XMMRegister);
10992     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1);
10993     __ pxor($tmp$$XMMRegister, $tmp2$$XMMRegister);
10994     __ movzwl($dst$$Register, $src1$$Register);
10995     __ pextrw($tmp3$$Register, $tmp$$XMMRegister, 0x0);
10996     __ xorw($dst$$Register, $tmp3$$Register);
10997     __ pextrw($tmp3$$Register, $tmp$$XMMRegister, 0x1);
10998     __ xorw($dst$$Register, $tmp3$$Register);
10999     __ movswl($dst$$Register, $dst$$Register);
11000   %}
11001   ins_pipe( pipe_slow );
11002 %}
11003 
11004 instruct rvxor16S_reduction_reg(rRegI dst, rRegI src1, vecY src2, regF tmp, regF tmp2, rRegI tmp3) %{
11005   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
11006   match(Set dst (XorReductionV src1 src2));
11007   effect(TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP dst);
11008    format %{ "vextracti128_high  $tmp,$src2\n\t"
11009             "vpxor   $tmp,$tmp,$src2\n\t"
11010             "pshufd  $tmp2,$tmp,0xE\n\t"
11011             "vpxor   $tmp,$tmp,$tmp2\n\t"
11012             "pshufd  $tmp2,$tmp,0x1\n\t"
11013             "vpxor   $tmp,$tmp,$tmp2\n\t"
11014             "movzwl  $dst,$src1\n\t"
11015             "pextrw  $tmp3,$tmp, 0x0\n\t"
11016             "xorw    $dst,$tmp3\n\t"
11017             "pextrw  $tmp3,$tmp, 0x1\n\t"
11018             "xorw    $dst,$tmp3\n\t"
11019             "movswl  $dst,$dst\t! xor reduction16S" %}
11020   ins_encode %{
11021     int vector_len = 0;
11022     __ vextracti128_high($tmp$$XMMRegister, $src2$$XMMRegister);
11023     __ vpxor($tmp$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, vector_len);
11024     __ vpshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE, vector_len);
11025     __ vpxor($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
11026     __ vpshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1, vector_len);
11027     __ vpxor($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
11028     __ movzwl($dst$$Register, $src1$$Register);
11029     __ pextrw($tmp3$$Register, $tmp$$XMMRegister, 0x0);
11030     __ xorw($dst$$Register, $tmp3$$Register);
11031     __ pextrw($tmp3$$Register, $tmp$$XMMRegister, 0x1);
11032     __ xorw($dst$$Register, $tmp3$$Register);
11033     __ movswl($dst$$Register, $dst$$Register);
11034   %}
11035   ins_pipe( pipe_slow );
11036 %}
11037 
11038 instruct rvxor32S_reduction_reg(rRegI dst, rRegI src1, vecZ src2, regF tmp, regF tmp2, rRegI tmp3) %{
11039   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
11040   match(Set dst (XorReductionV src1 src2));
11041   effect(TEMP tmp, TEMP tmp2, TEMP tmp3, TEMP dst);
11042   format %{ "vextracti64x4_high  $tmp2,$src2\n\t"
11043             "vpxor   $tmp2,$tmp2,$src2\n\t"
11044             "vextracti128_high  $tmp,$tmp2\n\t"
11045             "vpxor   $tmp,$tmp,$tmp2\n\t"
11046             "pshufd  $tmp2,$tmp,0xE\n\t"
11047             "vpxor   $tmp,$tmp,$tmp2\n\t"
11048             "pshufd  $tmp2,$tmp,0x1\n\t"
11049             "vpxor   $tmp,$tmp,$tmp2\n\t"
11050             "movzwl  $dst,$src1\n\t"
11051             "movdl   $tmp3,$tmp\n\t"
11052             "xorw    $dst,$tmp3\n\t"
11053             "shrl    $tmp3,0x16\n\t"
11054             "xorw    $dst,$tmp3\n\t"
11055             "movswl  $dst,$dst\t! xor reduction32S" %}
11056   ins_encode %{
11057     int vector_len = 0;
11058     __ vextracti64x4_high($tmp2$$XMMRegister, $src2$$XMMRegister);
11059     __ vpxor($tmp2$$XMMRegister, $tmp2$$XMMRegister, $src2$$XMMRegister, 1);
11060     __ vextracti128_high($tmp$$XMMRegister, $tmp2$$XMMRegister);
11061     __ vpxor($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
11062     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE);
11063     __ vpxor($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
11064     __ pshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1);
11065     __ vpxor($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
11066     __ movzwl($dst$$Register, $src1$$Register);
11067     __ movdl($tmp3$$Register, $tmp$$XMMRegister);
11068     __ xorw($dst$$Register, $tmp3$$Register);
11069     __ shrl($tmp3$$Register, 16);
11070     __ xorw($dst$$Register, $tmp3$$Register);
11071     __ movswl($dst$$Register, $dst$$Register);
11072   %}
11073   ins_pipe( pipe_slow );
11074 %}
11075 
11076 instruct rsxor2I_reduction_reg(rRegI dst, rRegI src1, vecD src2, regF tmp, regF tmp2) %{
11077   predicate(UseSSE > 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
11078   match(Set dst (XorReductionV src1 src2));
11079   effect(TEMP tmp, TEMP tmp2);
11080   format %{ "pshufd  $tmp2,$src2,0x1\n\t"
11081             "pxor    $tmp2,$src2\n\t"
11082             "movd    $tmp,$src1\n\t"
11083             "pxor    $tmp2,$tmp\n\t"
11084             "movd    $dst,$tmp2\t! xor reduction2I" %}
11085   ins_encode %{
11086     __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0x1);
11087     __ pxor($tmp2$$XMMRegister, $src2$$XMMRegister);
11088     __ movdl($tmp$$XMMRegister, $src1$$Register);
11089     __ pxor($tmp2$$XMMRegister, $tmp$$XMMRegister);
11090     __ movdl($dst$$Register, $tmp2$$XMMRegister);
11091   %}
11092   ins_pipe( pipe_slow );
11093 %}
11094 
11095 instruct rsxor4I_reduction_reg(rRegI dst, rRegI src1, vecX src2, regF tmp, regF tmp2) %{
11096   predicate(UseSSE > 1 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
11097   match(Set dst (XorReductionV src1 src2));
11098   effect(TEMP tmp, TEMP tmp2);
11099   format %{ "pshufd  $tmp2,$src2,0xE\n\t"
11100             "pxor    $tmp2,$src2\n\t"
11101             "pshufd  $tmp,$tmp2,0x1\n\t"
11102             "pxor    $tmp2,$tmp\n\t"
11103             "movd    $tmp,$src1\n\t"
11104             "pxor    $tmp2,$tmp\n\t"
11105             "movd    $dst,$tmp2\t! xor reduction4I" %}
11106   ins_encode %{
11107     __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0xE);
11108     __ pxor($tmp2$$XMMRegister, $src2$$XMMRegister);
11109     __ pshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0x1);
11110     __ pxor($tmp2$$XMMRegister, $tmp$$XMMRegister);
11111     __ movdl($tmp$$XMMRegister, $src1$$Register);
11112     __ pxor($tmp2$$XMMRegister, $tmp$$XMMRegister);
11113     __ movdl($dst$$Register, $tmp2$$XMMRegister);
11114   %}
11115   ins_pipe( pipe_slow );
11116 %}
11117 
11118 instruct rvxor8I_reduction_reg(rRegI dst, rRegI src1, vecY src2, regF tmp, regF tmp2) %{
11119   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
11120   match(Set dst (XorReductionV src1 src2));
11121   effect(TEMP tmp, TEMP tmp2);
11122   format %{ "vextracti128_high  $tmp,$src2\n\t"
11123             "vpxor    $tmp,$tmp,$src2\n\t"
11124             "vpshufd   $tmp2,$tmp,0xE\t"
11125             "vpxor    $tmp,$tmp,$tmp2\n\t"
11126             "vpshufd   $tmp2,$tmp,0x1\t"
11127             "vpxor    $tmp,$tmp,$tmp2\n\t"
11128             "movd     $tmp2,$src1\n\t"
11129             "vpxor    $tmp2,$tmp,$tmp2\n\t"
11130             "movd     $dst,$tmp2\t! xor reduction8I" %}
11131   ins_encode %{
11132     int vector_len = 0;
11133     __ vextracti128_high($tmp$$XMMRegister, $src2$$XMMRegister);
11134     __ vpxor($tmp$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, vector_len);
11135     __ vpshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE, vector_len);
11136     __ vpxor($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
11137     __ vpshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1, vector_len);
11138     __ vpxor($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
11139     __ movdl($tmp2$$XMMRegister, $src1$$Register);
11140     __ vpxor($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
11141     __ movdl($dst$$Register, $tmp2$$XMMRegister);
11142   %}
11143   ins_pipe( pipe_slow );
11144 %}
11145 
11146 instruct rvxor16I_reduction_reg(rRegI dst, rRegI src1, vecZ src2, regF tmp, regF tmp2, regF tmp3) %{
11147   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
11148   match(Set dst (XorReductionV src1 src2));
11149   effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
11150   format %{ "vextracti64x4_high  $tmp3,$src2\n\t"
11151             "vpxor  $tmp3,$tmp3,$src2\n\t"
11152             "vextracti128_high  $tmp,$tmp3\n\t"
11153             "vpxor    $tmp,$tmp,$src2\n\t"
11154             "vpshufd   $tmp2,$tmp,0xE\t"
11155             "vpxor    $tmp,$tmp,$tmp2\n\t"
11156             "vpshufd   $tmp2,$tmp,0x1\t"
11157             "vpxor    $tmp,$tmp,$tmp2\n\t"
11158             "movd     $tmp2,$src1\n\t"
11159             "vpxor    $tmp2,$tmp,$tmp2\n\t"
11160             "movd     $dst,$tmp2\t! xor reduction16I" %}
11161   ins_encode %{
11162     int vector_len = 0;
11163     __ vextracti64x4_high($tmp3$$XMMRegister, $src2$$XMMRegister);
11164     __ vpxor($tmp3$$XMMRegister, $tmp3$$XMMRegister, $src2$$XMMRegister, 1);
11165     __ vextracti128_high($tmp$$XMMRegister, $tmp3$$XMMRegister);
11166     __ vpxor($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp3$$XMMRegister, vector_len);
11167     __ vpshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0xE, vector_len);
11168     __ vpxor($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
11169     __ vpshufd($tmp2$$XMMRegister, $tmp$$XMMRegister, 0x1, vector_len);
11170     __ vpxor($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
11171     __ movdl($tmp2$$XMMRegister, $src1$$Register);
11172     __ vpxor($tmp2$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
11173     __ movdl($dst$$Register, $tmp2$$XMMRegister);
11174   %}
11175   ins_pipe( pipe_slow );
11176 %}
11177 
11178 instruct rsxor2L_reduction_reg(rRegL dst, rRegL src1, vecX src2, regF tmp, regF tmp2) %{
11179   predicate(UseSSE >= 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
11180   match(Set dst (XorReductionV src1 src2));
11181   effect(TEMP tmp, TEMP tmp2);
11182   format %{ "pshufd  $tmp2,$src2,0xE\n\t"
11183             "pxor    $tmp2,$src2\n\t"
11184             "movdq   $tmp,$src1\n\t"
11185             "pxor    $tmp2,$tmp\n\t"
11186             "movq   $dst,$tmp2\t! xor reduction2L" %}
11187   ins_encode %{
11188     __ pshufd($tmp2$$XMMRegister, $src2$$XMMRegister, 0xE);
11189     __ pxor($tmp2$$XMMRegister, $src2$$XMMRegister);
11190     __ movdq($tmp$$XMMRegister, $src1$$Register);
11191     __ pxor($tmp2$$XMMRegister, $tmp$$XMMRegister);
11192     __ movq($dst$$Register, $tmp2$$XMMRegister);
11193   %}
11194   ins_pipe( pipe_slow );
11195 %}
11196 
11197 instruct rvxor4L_reduction_reg_avx(rRegL dst, rRegL src1, vecY src2, regF tmp, regF tmp2) %{
11198   predicate(UseAVX > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
11199   match(Set dst (XorReductionV src1 src2));
11200   effect(TEMP tmp, TEMP tmp2);
11201   format %{ "vextracti128_high  $tmp,$src2\n\t"
11202             "vpxor  $tmp2,$tmp,$src2\n\t"
11203             "vpshufd  $tmp,$tmp2,0xE\t"
11204             "vpxor  $tmp2,$tmp2,$tmp\n\t"
11205             "movq   $tmp,$src1\n\t"
11206             "vpxor  $tmp2,$tmp2,$tmp\n\t"
11207             "movq   $dst,$tmp2\t! xor reduction4L" %}
11208   ins_encode %{
11209     int vector_len = 0;
11210     __ vextracti128_high($tmp$$XMMRegister, $src2$$XMMRegister);
11211     __ vpxor($tmp2$$XMMRegister, $tmp$$XMMRegister, $src2$$XMMRegister, vector_len);
11212     __ vpshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE, vector_len);
11213     __ vpxor($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, vector_len);
11214     __ movq($tmp$$XMMRegister, $src1$$Register);
11215     __ vpxor($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, vector_len);
11216     __ movq($dst$$Register, $tmp2$$XMMRegister);
11217   %}
11218   ins_pipe( pipe_slow );
11219 %}
11220 
11221 #ifdef _LP64
11222 instruct rvxor8L_reduction_reg(rRegL dst, rRegL src1, vecZ src2, regF tmp, regF tmp2) %{
11223   predicate(UseAVX > 2 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
11224   match(Set dst (XorReductionV src1 src2));
11225   effect(TEMP tmp, TEMP tmp2);
11226   format %{ "vextracti64x4_high  $tmp2,$src2\n\t"
11227             "vpxorq  $tmp2,$tmp2,$src2\n\t"
11228             "vextracti128_high  $tmp,$tmp2\n\t"
11229             "vpxorq  $tmp2,$tmp2,$tmp\n\t"
11230             "vpshufd  $tmp,$tmp2,0xE\t"
11231             "vpxorq  $tmp2,$tmp2,$tmp\n\t"
11232             "movdq   $tmp,$src1\n\t"
11233             "vpxorq  $tmp2,$tmp2,$tmp\n\t"
11234             "movdq   $dst,$tmp2\t! xor reduction8L" %}
11235   ins_encode %{
11236     int vector_len = 0;
11237     __ vextracti64x4_high($tmp2$$XMMRegister, $src2$$XMMRegister);
11238     __ vpxorq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $src2$$XMMRegister, 1);
11239     __ vextracti128_high($tmp$$XMMRegister, $tmp2$$XMMRegister);
11240     __ vpxorq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, vector_len);
11241     __ vpshufd($tmp$$XMMRegister, $tmp2$$XMMRegister, 0xE, vector_len);
11242     __ vpxorq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, vector_len);
11243     __ movdq($tmp$$XMMRegister, $src1$$Register);
11244     __ vpxorq($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp$$XMMRegister, vector_len);
11245     __ movdq($dst$$Register, $tmp2$$XMMRegister);
11246   %}
11247   ins_pipe( pipe_slow );
11248 %}
11249 #endif
11250 
11251 // ====================VECTOR ARITHMETIC=======================================
11252 
11253 // --------------------------------- ADD --------------------------------------
11254 
11255 // Bytes vector add
11256 instruct vadd4B(vecS dst, vecS src) %{
11257   predicate(UseAVX == 0 && n->as_Vector()->length() == 4);
11258   match(Set dst (AddVB dst src));
11259   format %{ "paddb   $dst,$src\t! add packed4B" %}
11260   ins_encode %{
11261     __ paddb($dst$$XMMRegister, $src$$XMMRegister);
11262   %}
11263   ins_pipe( pipe_slow );
11264 %}
11265 
11266 instruct vadd4B_reg_avx(vecS dst, vecS src1, vecS src2) %{
11267   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 4);
11268   match(Set dst (AddVB src1 src2));
11269   format %{ "vpaddb  $dst,$src1,$src2\t! add packed4B" %}
11270   ins_encode %{
11271     int vector_len = 0;
11272     __ vpaddb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
11273   %}
11274   ins_pipe( pipe_slow );
11275 %}
11276 
11277 instruct vadd4B_reg_evex(vecS dst, vecS src1, vecS src2) %{
11278   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 4);
11279   match(Set dst (AddVB src1 src2));
11280   format %{ "vpaddb  $dst,$src1,$src2\t! add packed4B" %}
11281   ins_encode %{
11282     int vector_len = 0;
11283     __ vpaddb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
11284   %}
11285   ins_pipe( pipe_slow );
11286 %}
11287 
11288 instruct vadd4B_reg_evex_special(vecS dst, vecS src1, vecS src2) %{
11289   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 4);
11290   match(Set dst (AddVB dst src2));
11291   effect(TEMP src1);
11292   format %{ "vpaddb  $dst,$dst,$src2\t! add packed4B" %}
11293   ins_encode %{
11294     int vector_len = 0;
11295     __ vpaddb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
11296   %}
11297   ins_pipe( pipe_slow );
11298 %}
11299 
11300 instruct vadd4B_mem_avx(vecS dst, vecS src, memory mem) %{
11301   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 4);
11302   match(Set dst (AddVB src (LoadVector mem)));
11303   format %{ "vpaddb  $dst,$src,$mem\t! add packed4B" %}
11304   ins_encode %{
11305     int vector_len = 0;
11306     __ vpaddb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
11307   %}
11308   ins_pipe( pipe_slow );
11309 %}
11310 
11311 instruct vadd4B_mem_evex(vecS dst, vecS src, memory mem) %{
11312   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 4);
11313   match(Set dst (AddVB src (LoadVector mem)));
11314   format %{ "vpaddb  $dst,$src,$mem\t! add packed4B" %}
11315   ins_encode %{
11316     int vector_len = 0;
11317     __ vpaddb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
11318   %}
11319   ins_pipe( pipe_slow );
11320 %}
11321 
11322 instruct vadd4B_mem_evex_special(vecS dst, vecS src, memory mem) %{
11323   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 4);
11324   match(Set dst (AddVB dst (LoadVector mem)));
11325   effect(TEMP src);
11326   format %{ "vpaddb  $dst,$src,$mem\t! add packed4B" %}
11327   ins_encode %{
11328     int vector_len = 0;
11329     __ vpaddb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
11330   %}
11331   ins_pipe( pipe_slow );
11332 %}
11333 
11334 instruct vadd8B(vecD dst, vecD src) %{
11335   predicate(UseAVX == 0 && n->as_Vector()->length() == 8);
11336   match(Set dst (AddVB dst src));
11337   format %{ "paddb   $dst,$src\t! add packed8B" %}
11338   ins_encode %{
11339     __ paddb($dst$$XMMRegister, $src$$XMMRegister);
11340   %}
11341   ins_pipe( pipe_slow );
11342 %}
11343 
11344 instruct vadd8B_reg_avx(vecD dst, vecD src1, vecD src2) %{
11345   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 8);
11346   match(Set dst (AddVB src1 src2));
11347   format %{ "vpaddb  $dst,$src1,$src2\t! add packed8B" %}
11348   ins_encode %{
11349     int vector_len = 0;
11350     __ vpaddb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
11351   %}
11352   ins_pipe( pipe_slow );
11353 %}
11354 
11355 instruct vadd8B_reg_evex(vecD dst, vecD src1, vecD src2) %{
11356   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 8);
11357   match(Set dst (AddVB src1 src2));
11358   format %{ "vpaddb  $dst,$src1,$src2\t! add packed8B" %}
11359   ins_encode %{
11360     int vector_len = 0;
11361     __ vpaddb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
11362   %}
11363   ins_pipe( pipe_slow );
11364 %}
11365 
11366 instruct vadd8B_reg_evex_special(vecD dst, vecD src1, vecD src2) %{
11367   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 8);
11368   match(Set dst (AddVB dst src2));
11369   effect(TEMP src1);
11370   format %{ "vpaddb  $dst,$dst,$src2\t! add packed8B" %}
11371   ins_encode %{
11372     int vector_len = 0;
11373     __ vpaddb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
11374   %}
11375   ins_pipe( pipe_slow );
11376 %}
11377 
11378 instruct vadd8B_mem_avx(vecD dst, vecD src, memory mem) %{
11379   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 8);
11380   match(Set dst (AddVB src (LoadVector mem)));
11381   format %{ "vpaddb  $dst,$src,$mem\t! add packed8B" %}
11382   ins_encode %{
11383     int vector_len = 0;
11384     __ vpaddb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
11385   %}
11386   ins_pipe( pipe_slow );
11387 %}
11388 
11389 instruct vadd8B_mem_evex(vecD dst, vecD src, memory mem) %{
11390   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 8);
11391   match(Set dst (AddVB src (LoadVector mem)));
11392   format %{ "vpaddb  $dst,$src,$mem\t! add packed8B" %}
11393   ins_encode %{
11394     int vector_len = 0;
11395     __ vpaddb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
11396   %}
11397   ins_pipe( pipe_slow );
11398 %}
11399 
11400 instruct vadd8B_mem_evex_special(vecD dst, vecD src, memory mem) %{
11401   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 8);
11402   match(Set dst (AddVB dst (LoadVector mem)));
11403   effect(TEMP src);
11404   format %{ "vpaddb  $dst,$src,$mem\t! add packed8B" %}
11405   ins_encode %{
11406     int vector_len = 0;
11407     __ vpaddb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
11408   %}
11409   ins_pipe( pipe_slow );
11410 %}
11411 
11412 instruct vadd16B(vecX dst, vecX src) %{
11413   predicate(UseAVX == 0 && n->as_Vector()->length() == 16);
11414   match(Set dst (AddVB dst src));
11415   format %{ "paddb   $dst,$src\t! add packed16B" %}
11416   ins_encode %{
11417     __ paddb($dst$$XMMRegister, $src$$XMMRegister);
11418   %}
11419   ins_pipe( pipe_slow );
11420 %}
11421 
11422 instruct vadd16B_reg_avx(vecX dst, vecX src1, vecX src2) %{
11423   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 16);
11424   match(Set dst (AddVB src1 src2));
11425   format %{ "vpaddb  $dst,$src1,$src2\t! add packed16B" %}
11426   ins_encode %{
11427     int vector_len = 0;
11428     __ vpaddb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
11429   %}
11430   ins_pipe( pipe_slow );
11431 %}
11432 
11433 instruct vadd16B_reg_evex(vecX dst, vecX src1, vecX src2) %{
11434   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 16);
11435   match(Set dst (AddVB src1 src2));
11436   format %{ "vpaddb  $dst,$src1,$src2\t! add packed16B" %}
11437   ins_encode %{
11438     int vector_len = 0;
11439     __ vpaddb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
11440   %}
11441   ins_pipe( pipe_slow );
11442 %}
11443 
11444 instruct vadd16B_reg_evex_special(vecX dst, vecX src1, vecX src2) %{
11445   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 16);
11446   match(Set dst (AddVB dst src2));
11447   effect(TEMP src1);
11448   format %{ "vpaddb  $dst,$dst,$src2\t! add packed16B" %}
11449   ins_encode %{
11450     int vector_len = 0;
11451     __ vpaddb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
11452   %}
11453   ins_pipe( pipe_slow );
11454 %}
11455 
11456 instruct vadd16B_mem_avx(vecX dst, vecX src, memory mem) %{
11457   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 16);
11458   match(Set dst (AddVB src (LoadVector mem)));
11459   format %{ "vpaddb  $dst,$src,$mem\t! add packed16B" %}
11460   ins_encode %{
11461     int vector_len = 0;
11462     __ vpaddb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
11463   %}
11464   ins_pipe( pipe_slow );
11465 %}
11466 
11467 instruct vadd16B_mem_evex(vecX dst, vecX src, memory mem) %{
11468   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 16);
11469   match(Set dst (AddVB src (LoadVector mem)));
11470   format %{ "vpaddb  $dst,$src,$mem\t! add packed16B" %}
11471   ins_encode %{
11472     int vector_len = 0;
11473     __ vpaddb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
11474   %}
11475   ins_pipe( pipe_slow );
11476 %}
11477 
11478 instruct vadd16B_mem_evex_special(vecX dst, vecX src, memory mem) %{
11479   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 16);
11480   match(Set dst (AddVB dst (LoadVector mem)));
11481   effect(TEMP src);
11482   format %{ "vpaddb  $dst,$src,$mem\t! add packed16B" %}
11483   ins_encode %{
11484     int vector_len = 0;
11485     __ vpaddb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
11486   %}
11487   ins_pipe( pipe_slow );
11488 %}
11489 
11490 instruct vadd32B_reg_avx(vecY dst, vecY src1, vecY src2) %{
11491   predicate(VM_Version::supports_avx256only() && n->as_Vector()->length() == 32);
11492   match(Set dst (AddVB src1 src2));
11493   format %{ "vpaddb  $dst,$src1,$src2\t! add packed32B" %}
11494   ins_encode %{
11495     int vector_len = 1;
11496     __ vpaddb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
11497   %}
11498   ins_pipe( pipe_slow );
11499 %}
11500 
11501 instruct vadd32B_reg_evex(vecY dst, vecY src1, vecY src2) %{
11502   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 32);
11503   match(Set dst (AddVB src1 src2));
11504   format %{ "vpaddb  $dst,$src1,$src2\t! add packed32B" %}
11505   ins_encode %{
11506     int vector_len = 1;
11507     __ vpaddb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
11508   %}
11509   ins_pipe( pipe_slow );
11510 %}
11511 
11512 instruct vadd32B_reg_evex_special(vecY dst, vecY src1, vecY src2) %{
11513   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 32);
11514   match(Set dst (AddVB dst src2));
11515   effect(TEMP src1);
11516   format %{ "vpaddb  $dst,$dst,$src2\t! add packed32B" %}
11517   ins_encode %{
11518     int vector_len = 1;
11519     __ vpaddb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
11520   %}
11521   ins_pipe( pipe_slow );
11522 %}
11523 
11524 instruct vadd32B_mem_avx(vecY dst, vecY src, memory mem) %{
11525   predicate(VM_Version::supports_avx256only() && n->as_Vector()->length() == 32);
11526   match(Set dst (AddVB src (LoadVector mem)));
11527   format %{ "vpaddb  $dst,$src,$mem\t! add packed32B" %}
11528   ins_encode %{
11529     int vector_len = 1;
11530     __ vpaddb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
11531   %}
11532   ins_pipe( pipe_slow );
11533 %}
11534 
11535 instruct vadd32B_mem_evex(vecY dst, vecY src, memory mem) %{
11536   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 32);
11537   match(Set dst (AddVB src (LoadVector mem)));
11538   format %{ "vpaddb  $dst,$src,$mem\t! add packed32B" %}
11539   ins_encode %{
11540     int vector_len = 1;
11541     __ vpaddb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
11542   %}
11543   ins_pipe( pipe_slow );
11544 %}
11545 
11546 instruct vadd32B_mem_evex_special(vecY dst, vecY src, memory mem) %{
11547   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 32);
11548   match(Set dst (AddVB dst (LoadVector mem)));
11549   effect(TEMP src);
11550   format %{ "vpaddb  $dst,$src,$mem\t! add packed32B" %}
11551   ins_encode %{
11552     int vector_len = 1;
11553     __ vpaddb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
11554   %}
11555   ins_pipe( pipe_slow );
11556 %}
11557 
11558 instruct vadd64B_reg(vecZ dst, vecZ src1, vecZ src2) %{
11559   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 64);
11560   match(Set dst (AddVB src1 src2));
11561   format %{ "vpaddb  $dst,$src1,$src2\t! add packed64B" %}
11562   ins_encode %{
11563     int vector_len = 2;
11564     __ vpaddb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
11565   %}
11566   ins_pipe( pipe_slow );
11567 %}
11568 
11569 instruct vadd64B_mem(vecZ dst, vecZ src, memory mem) %{
11570   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 64);
11571   match(Set dst (AddVB src (LoadVector mem)));
11572   format %{ "vpaddb  $dst,$src,$mem\t! add packed64B" %}
11573   ins_encode %{
11574     int vector_len = 2;
11575     __ vpaddb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
11576   %}
11577   ins_pipe( pipe_slow );
11578 %}
11579 
11580 // Shorts/Chars vector add
11581 instruct vadd2S(vecS dst, vecS src) %{
11582   predicate(UseAVX == 0 && n->as_Vector()->length() == 2);
11583   match(Set dst (AddVS dst src));
11584   format %{ "paddw   $dst,$src\t! add packed2S" %}
11585   ins_encode %{
11586     __ paddw($dst$$XMMRegister, $src$$XMMRegister);
11587   %}
11588   ins_pipe( pipe_slow );
11589 %}
11590 
11591 instruct vadd2S_reg_avx(vecS dst, vecS src1, vecS src2) %{
11592   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 2);
11593   match(Set dst (AddVS src1 src2));
11594   format %{ "vpaddw  $dst,$src1,$src2\t! add packed2S" %}
11595   ins_encode %{
11596     int vector_len = 0;
11597     __ vpaddw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
11598   %}
11599   ins_pipe( pipe_slow );
11600 %}
11601 
11602 instruct vadd2S_reg_evex(vecS dst, vecS src1, vecS src2) %{
11603   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 2);
11604   match(Set dst (AddVS src1 src2));
11605   format %{ "vpaddw  $dst,$src1,$src2\t! add packed2S" %}
11606   ins_encode %{
11607     int vector_len = 0;
11608     __ vpaddw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
11609   %}
11610   ins_pipe( pipe_slow );
11611 %}
11612 
11613 instruct vadd2S_reg_evex_special(vecS dst, vecS src1, vecS src2) %{
11614   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 2);
11615   match(Set dst (AddVS dst src2));
11616   effect(TEMP src1);
11617   format %{ "vpaddw  $dst,$dst,$src2\t! add packed2S" %}
11618   ins_encode %{
11619     int vector_len = 0;
11620     __ vpaddw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
11621   %}
11622   ins_pipe( pipe_slow );
11623 %}
11624 
11625 instruct vadd2S_mem_avx(vecS dst, vecS src, memory mem) %{
11626   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 2);
11627   match(Set dst (AddVS src (LoadVector mem)));
11628   format %{ "vpaddw  $dst,$src,$mem\t! add packed2S" %}
11629   ins_encode %{
11630     int vector_len = 0;
11631     __ vpaddw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
11632   %}
11633   ins_pipe( pipe_slow );
11634 %}
11635 
11636 instruct vadd2S_mem_evex(vecS dst, vecS src, memory mem) %{
11637   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 2);
11638   match(Set dst (AddVS src (LoadVector mem)));
11639   format %{ "vpaddw  $dst,$src,$mem\t! add packed2S" %}
11640   ins_encode %{
11641     int vector_len = 0;
11642     __ vpaddw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
11643   %}
11644   ins_pipe( pipe_slow );
11645 %}
11646 
11647 instruct vadd2S_mem_evex_special(vecS dst, vecS src, memory mem) %{
11648   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 2);
11649   match(Set dst (AddVS dst (LoadVector mem)));
11650   effect(TEMP src);
11651   format %{ "vpaddw  $dst,$src,$mem\t! add packed2S" %}
11652   ins_encode %{
11653     int vector_len = 0;
11654     __ vpaddw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
11655   %}
11656   ins_pipe( pipe_slow );
11657 %}
11658 
11659 instruct vadd4S(vecD dst, vecD src) %{
11660   predicate(UseAVX == 0 && n->as_Vector()->length() == 4);
11661   match(Set dst (AddVS dst src));
11662   format %{ "paddw   $dst,$src\t! add packed4S" %}
11663   ins_encode %{
11664     __ paddw($dst$$XMMRegister, $src$$XMMRegister);
11665   %}
11666   ins_pipe( pipe_slow );
11667 %}
11668 
11669 instruct vadd4S_reg_avx(vecD dst, vecD src1, vecD src2) %{
11670   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 4);
11671   match(Set dst (AddVS src1 src2));
11672   format %{ "vpaddw  $dst,$src1,$src2\t! add packed4S" %}
11673   ins_encode %{
11674     int vector_len = 0;
11675     __ vpaddw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
11676   %}
11677   ins_pipe( pipe_slow );
11678 %}
11679 
11680 instruct vadd4S_reg_evex(vecD dst, vecD src1, vecD src2) %{
11681   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 4);
11682   match(Set dst (AddVS src1 src2));
11683   format %{ "vpaddw  $dst,$src1,$src2\t! add packed4S" %}
11684   ins_encode %{
11685     int vector_len = 0;
11686     __ vpaddw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
11687   %}
11688   ins_pipe( pipe_slow );
11689 %}
11690 
11691 instruct vadd4S_reg_evex_special(vecD dst, vecD src1, vecD src2) %{
11692   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 4);
11693   match(Set dst (AddVS dst src2));
11694   effect(TEMP src1);
11695   format %{ "vpaddw  $dst,$dst,$src2\t! add packed4S" %}
11696   ins_encode %{
11697     int vector_len = 0;
11698     __ vpaddw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
11699   %}
11700   ins_pipe( pipe_slow );
11701 %}
11702 
11703 instruct vadd4S_mem_avx(vecD dst, vecD src, memory mem) %{
11704   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 4);
11705   match(Set dst (AddVS src (LoadVector mem)));
11706   format %{ "vpaddw  $dst,$src,$mem\t! add packed4S" %}
11707   ins_encode %{
11708     int vector_len = 0;
11709     __ vpaddw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
11710   %}
11711   ins_pipe( pipe_slow );
11712 %}
11713 
11714 instruct vadd4S_mem_evex(vecD dst, vecD src, memory mem) %{
11715   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 4);
11716   match(Set dst (AddVS src (LoadVector mem)));
11717   format %{ "vpaddw  $dst,$src,$mem\t! add packed4S" %}
11718   ins_encode %{
11719     int vector_len = 0;
11720     __ vpaddw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
11721   %}
11722   ins_pipe( pipe_slow );
11723 %}
11724 
11725 instruct vadd4S_mem_evex_special(vecD dst, vecD src, memory mem) %{
11726   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 4);
11727   match(Set dst (AddVS dst (LoadVector mem)));
11728   effect(TEMP src);
11729   format %{ "vpaddw  $dst,$src,$mem\t! add packed4S" %}
11730   ins_encode %{
11731     int vector_len = 0;
11732     __ vpaddw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
11733   %}
11734   ins_pipe( pipe_slow );
11735 %}
11736 
11737 instruct vadd8S(vecX dst, vecX src) %{
11738   predicate(UseAVX == 0 && n->as_Vector()->length() == 8);
11739   match(Set dst (AddVS dst src));
11740   format %{ "paddw   $dst,$src\t! add packed8S" %}
11741   ins_encode %{
11742     __ paddw($dst$$XMMRegister, $src$$XMMRegister);
11743   %}
11744   ins_pipe( pipe_slow );
11745 %}
11746 
11747 instruct vadd8S_reg_avx(vecX dst, vecX src1, vecX src2) %{
11748   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 8);
11749   match(Set dst (AddVS src1 src2));
11750   format %{ "vpaddw  $dst,$src1,$src2\t! add packed8S" %}
11751   ins_encode %{
11752     int vector_len = 0;
11753     __ vpaddw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
11754   %}
11755   ins_pipe( pipe_slow );
11756 %}
11757 
11758 instruct vadd8S_reg_evex(vecX dst, vecX src1, vecX src2) %{
11759   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 8);
11760   match(Set dst (AddVS src1 src2));
11761   format %{ "vpaddw  $dst,$src1,$src2\t! add packed8S" %}
11762   ins_encode %{
11763     int vector_len = 0;
11764     __ vpaddw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
11765   %}
11766   ins_pipe( pipe_slow );
11767 %}
11768 
11769 instruct vadd8S_reg_evex_special(vecX dst, vecX src1, vecX src2) %{
11770   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 8);
11771   match(Set dst (AddVS dst src2));
11772   effect(TEMP src1);
11773   format %{ "vpaddw  $dst,$dst,$src2\t! add packed8S" %}
11774   ins_encode %{
11775     int vector_len = 0;
11776     __ vpaddw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
11777   %}
11778   ins_pipe( pipe_slow );
11779 %}
11780 
11781 instruct vadd8S_mem_avx(vecX dst, vecX src, memory mem) %{
11782   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 8);
11783   match(Set dst (AddVS src (LoadVector mem)));
11784   format %{ "vpaddw  $dst,$src,$mem\t! add packed8S" %}
11785   ins_encode %{
11786     int vector_len = 0;
11787     __ vpaddw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
11788   %}
11789   ins_pipe( pipe_slow );
11790 %}
11791 
11792 instruct vadd8S_mem_evex(vecX dst, vecX src, memory mem) %{
11793   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 8);
11794   match(Set dst (AddVS src (LoadVector mem)));
11795   format %{ "vpaddw  $dst,$src,$mem\t! add packed8S" %}
11796   ins_encode %{
11797     int vector_len = 0;
11798     __ vpaddw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
11799   %}
11800   ins_pipe( pipe_slow );
11801 %}
11802 
11803 instruct vadd8S_mem_evex_special(vecX dst, vecX src, memory mem) %{
11804   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 8);
11805   match(Set dst (AddVS dst (LoadVector mem)));
11806   effect(TEMP src);
11807   format %{ "vpaddw  $dst,$src,$mem\t! add packed8S" %}
11808   ins_encode %{
11809     int vector_len = 0;
11810     __ vpaddw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
11811   %}
11812   ins_pipe( pipe_slow );
11813 %}
11814 
11815 instruct vadd16S_reg_avx(vecY dst, vecY src1, vecY src2) %{
11816   predicate(VM_Version::supports_avx256only() && n->as_Vector()->length() == 16);
11817   match(Set dst (AddVS src1 src2));
11818   format %{ "vpaddw  $dst,$src1,$src2\t! add packed16S" %}
11819   ins_encode %{
11820     int vector_len = 1;
11821     __ vpaddw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
11822   %}
11823   ins_pipe( pipe_slow );
11824 %}
11825 
11826 instruct vadd16S_reg_evex(vecY dst, vecY src1, vecY src2) %{
11827   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 16);
11828   match(Set dst (AddVS src1 src2));
11829   format %{ "vpaddw  $dst,$src1,$src2\t! add packed16S" %}
11830   ins_encode %{
11831     int vector_len = 1;
11832     __ vpaddw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
11833   %}
11834   ins_pipe( pipe_slow );
11835 %}
11836 
11837 instruct vadd16S_reg_evex_special(vecY dst, vecY src1, vecY src2) %{
11838   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 16);
11839   match(Set dst (AddVS dst src2));
11840   effect(TEMP src1);
11841   format %{ "vpaddw  $dst,$dst,$src2\t! add packed16S" %}
11842   ins_encode %{
11843     int vector_len = 1;
11844     __ vpaddw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
11845   %}
11846   ins_pipe( pipe_slow );
11847 %}
11848 
11849 instruct vadd16S_mem_avx(vecY dst, vecY src, memory mem) %{
11850   predicate(VM_Version::supports_avx256only() && n->as_Vector()->length() == 16);
11851   match(Set dst (AddVS src (LoadVector mem)));
11852   format %{ "vpaddw  $dst,$src,$mem\t! add packed16S" %}
11853   ins_encode %{
11854     int vector_len = 1;
11855     __ vpaddw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
11856   %}
11857   ins_pipe( pipe_slow );
11858 %}
11859 
11860 instruct vadd16S_mem_evex(vecY dst, vecY src, memory mem) %{
11861   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 16);
11862   match(Set dst (AddVS src (LoadVector mem)));
11863   format %{ "vpaddw  $dst,$src,$mem\t! add packed16S" %}
11864   ins_encode %{
11865     int vector_len = 1;
11866     __ vpaddw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
11867   %}
11868   ins_pipe( pipe_slow );
11869 %}
11870 
11871 instruct vadd16S_mem_evex_special(vecY dst, vecY src, memory mem) %{
11872   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 16);
11873   match(Set dst (AddVS dst (LoadVector mem)));
11874   effect(TEMP src);
11875   format %{ "vpaddw  $dst,$src,$mem\t! add packed16S" %}
11876   ins_encode %{
11877     int vector_len = 1;
11878     __ vpaddw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
11879   %}
11880   ins_pipe( pipe_slow );
11881 %}
11882 
11883 instruct vadd32S_reg(vecZ dst, vecZ src1, vecZ src2) %{
11884   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 32);
11885   match(Set dst (AddVS src1 src2));
11886   format %{ "vpaddw  $dst,$src1,$src2\t! add packed32S" %}
11887   ins_encode %{
11888     int vector_len = 2;
11889     __ vpaddw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
11890   %}
11891   ins_pipe( pipe_slow );
11892 %}
11893 
11894 instruct vadd32S_mem(vecZ dst, vecZ src, memory mem) %{
11895   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 32);
11896   match(Set dst (AddVS src (LoadVector mem)));
11897   format %{ "vpaddw  $dst,$src,$mem\t! add packed32S" %}
11898   ins_encode %{
11899     int vector_len = 2;
11900     __ vpaddw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
11901   %}
11902   ins_pipe( pipe_slow );
11903 %}
11904 
11905 // Integers vector add
11906 instruct vadd2I(vecD dst, vecD src) %{
11907   predicate(n->as_Vector()->length() == 2);
11908   match(Set dst (AddVI dst src));
11909   format %{ "paddd   $dst,$src\t! add packed2I" %}
11910   ins_encode %{
11911     __ paddd($dst$$XMMRegister, $src$$XMMRegister);
11912   %}
11913   ins_pipe( pipe_slow );
11914 %}
11915 
11916 instruct vadd2I_reg(vecD dst, vecD src1, vecD src2) %{
11917   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
11918   match(Set dst (AddVI src1 src2));
11919   format %{ "vpaddd  $dst,$src1,$src2\t! add packed2I" %}
11920   ins_encode %{
11921     int vector_len = 0;
11922     __ vpaddd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
11923   %}
11924   ins_pipe( pipe_slow );
11925 %}
11926 
11927 instruct vadd2I_mem(vecD dst, vecD src, memory mem) %{
11928   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
11929   match(Set dst (AddVI src (LoadVector mem)));
11930   format %{ "vpaddd  $dst,$src,$mem\t! add packed2I" %}
11931   ins_encode %{
11932     int vector_len = 0;
11933     __ vpaddd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
11934   %}
11935   ins_pipe( pipe_slow );
11936 %}
11937 
11938 instruct vadd4I(vecX dst, vecX src) %{
11939   predicate(n->as_Vector()->length() == 4);
11940   match(Set dst (AddVI dst src));
11941   format %{ "paddd   $dst,$src\t! add packed4I" %}
11942   ins_encode %{
11943     __ paddd($dst$$XMMRegister, $src$$XMMRegister);
11944   %}
11945   ins_pipe( pipe_slow );
11946 %}
11947 
11948 instruct vadd4I_reg(vecX dst, vecX src1, vecX src2) %{
11949   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
11950   match(Set dst (AddVI src1 src2));
11951   format %{ "vpaddd  $dst,$src1,$src2\t! add packed4I" %}
11952   ins_encode %{
11953     int vector_len = 0;
11954     __ vpaddd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
11955   %}
11956   ins_pipe( pipe_slow );
11957 %}
11958 
11959 instruct vadd4I_mem(vecX dst, vecX src, memory mem) %{
11960   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
11961   match(Set dst (AddVI src (LoadVector mem)));
11962   format %{ "vpaddd  $dst,$src,$mem\t! add packed4I" %}
11963   ins_encode %{
11964     int vector_len = 0;
11965     __ vpaddd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
11966   %}
11967   ins_pipe( pipe_slow );
11968 %}
11969 
11970 instruct vadd8I_reg(vecY dst, vecY src1, vecY src2) %{
11971   predicate(UseAVX > 1 && n->as_Vector()->length() == 8);
11972   match(Set dst (AddVI src1 src2));
11973   format %{ "vpaddd  $dst,$src1,$src2\t! add packed8I" %}
11974   ins_encode %{
11975     int vector_len = 1;
11976     __ vpaddd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
11977   %}
11978   ins_pipe( pipe_slow );
11979 %}
11980 
11981 instruct vadd8I_mem(vecY dst, vecY src, memory mem) %{
11982   predicate(UseAVX > 1 && n->as_Vector()->length() == 8);
11983   match(Set dst (AddVI src (LoadVector mem)));
11984   format %{ "vpaddd  $dst,$src,$mem\t! add packed8I" %}
11985   ins_encode %{
11986     int vector_len = 1;
11987     __ vpaddd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
11988   %}
11989   ins_pipe( pipe_slow );
11990 %}
11991 
11992 instruct vadd16I_reg(vecZ dst, vecZ src1, vecZ src2) %{
11993   predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
11994   match(Set dst (AddVI src1 src2));
11995   format %{ "vpaddd  $dst,$src1,$src2\t! add packed16I" %}
11996   ins_encode %{
11997     int vector_len = 2;
11998     __ vpaddd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
11999   %}
12000   ins_pipe( pipe_slow );
12001 %}
12002 
12003 instruct vadd16I_mem(vecZ dst, vecZ src, memory mem) %{
12004   predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
12005   match(Set dst (AddVI src (LoadVector mem)));
12006   format %{ "vpaddd  $dst,$src,$mem\t! add packed16I" %}
12007   ins_encode %{
12008     int vector_len = 2;
12009     __ vpaddd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12010   %}
12011   ins_pipe( pipe_slow );
12012 %}
12013 
12014 // Longs vector add
12015 instruct vadd2L(vecX dst, vecX src) %{
12016   predicate(n->as_Vector()->length() == 2);
12017   match(Set dst (AddVL dst src));
12018   format %{ "paddq   $dst,$src\t! add packed2L" %}
12019   ins_encode %{
12020     __ paddq($dst$$XMMRegister, $src$$XMMRegister);
12021   %}
12022   ins_pipe( pipe_slow );
12023 %}
12024 
12025 instruct vadd2L_reg(vecX dst, vecX src1, vecX src2) %{
12026   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
12027   match(Set dst (AddVL src1 src2));
12028   format %{ "vpaddq  $dst,$src1,$src2\t! add packed2L" %}
12029   ins_encode %{
12030     int vector_len = 0;
12031     __ vpaddq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12032   %}
12033   ins_pipe( pipe_slow );
12034 %}
12035 
12036 instruct vadd2L_mem(vecX dst, vecX src, memory mem) %{
12037   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
12038   match(Set dst (AddVL src (LoadVector mem)));
12039   format %{ "vpaddq  $dst,$src,$mem\t! add packed2L" %}
12040   ins_encode %{
12041     int vector_len = 0;
12042     __ vpaddq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12043   %}
12044   ins_pipe( pipe_slow );
12045 %}
12046 
12047 instruct vadd4L_reg(vecY dst, vecY src1, vecY src2) %{
12048   predicate(UseAVX > 1 && n->as_Vector()->length() == 4);
12049   match(Set dst (AddVL src1 src2));
12050   format %{ "vpaddq  $dst,$src1,$src2\t! add packed4L" %}
12051   ins_encode %{
12052     int vector_len = 1;
12053     __ vpaddq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12054   %}
12055   ins_pipe( pipe_slow );
12056 %}
12057 
12058 instruct vadd4L_mem(vecY dst, vecY src, memory mem) %{
12059   predicate(UseAVX > 1 && n->as_Vector()->length() == 4);
12060   match(Set dst (AddVL src (LoadVector mem)));
12061   format %{ "vpaddq  $dst,$src,$mem\t! add packed4L" %}
12062   ins_encode %{
12063     int vector_len = 1;
12064     __ vpaddq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12065   %}
12066   ins_pipe( pipe_slow );
12067 %}
12068 
12069 instruct vadd8L_reg(vecZ dst, vecZ src1, vecZ src2) %{
12070   predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
12071   match(Set dst (AddVL src1 src2));
12072   format %{ "vpaddq  $dst,$src1,$src2\t! add packed8L" %}
12073   ins_encode %{
12074     int vector_len = 2;
12075     __ vpaddq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12076   %}
12077   ins_pipe( pipe_slow );
12078 %}
12079 
12080 instruct vadd8L_mem(vecZ dst, vecZ src, memory mem) %{
12081   predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
12082   match(Set dst (AddVL src (LoadVector mem)));
12083   format %{ "vpaddq  $dst,$src,$mem\t! add packed8L" %}
12084   ins_encode %{
12085     int vector_len = 2;
12086     __ vpaddq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12087   %}
12088   ins_pipe( pipe_slow );
12089 %}
12090 
12091 // Floats vector add
12092 instruct vadd2F(vecD dst, vecD src) %{
12093   predicate(n->as_Vector()->length() == 2);
12094   match(Set dst (AddVF dst src));
12095   format %{ "addps   $dst,$src\t! add packed2F" %}
12096   ins_encode %{
12097     __ addps($dst$$XMMRegister, $src$$XMMRegister);
12098   %}
12099   ins_pipe( pipe_slow );
12100 %}
12101 
12102 instruct vadd2F_reg(vecD dst, vecD src1, vecD src2) %{
12103   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
12104   match(Set dst (AddVF src1 src2));
12105   format %{ "vaddps  $dst,$src1,$src2\t! add packed2F" %}
12106   ins_encode %{
12107     int vector_len = 0;
12108     __ vaddps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12109   %}
12110   ins_pipe( pipe_slow );
12111 %}
12112 
12113 instruct vadd2F_mem(vecD dst, vecD src, memory mem) %{
12114   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
12115   match(Set dst (AddVF src (LoadVector mem)));
12116   format %{ "vaddps  $dst,$src,$mem\t! add packed2F" %}
12117   ins_encode %{
12118     int vector_len = 0;
12119     __ vaddps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12120   %}
12121   ins_pipe( pipe_slow );
12122 %}
12123 
12124 instruct vadd4F(vecX dst, vecX src) %{
12125   predicate(n->as_Vector()->length() == 4);
12126   match(Set dst (AddVF dst src));
12127   format %{ "addps   $dst,$src\t! add packed4F" %}
12128   ins_encode %{
12129     __ addps($dst$$XMMRegister, $src$$XMMRegister);
12130   %}
12131   ins_pipe( pipe_slow );
12132 %}
12133 
12134 instruct vadd4F_reg(vecX dst, vecX src1, vecX src2) %{
12135   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
12136   match(Set dst (AddVF src1 src2));
12137   format %{ "vaddps  $dst,$src1,$src2\t! add packed4F" %}
12138   ins_encode %{
12139     int vector_len = 0;
12140     __ vaddps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12141   %}
12142   ins_pipe( pipe_slow );
12143 %}
12144 
12145 instruct vadd4F_mem(vecX dst, vecX src, memory mem) %{
12146   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
12147   match(Set dst (AddVF src (LoadVector mem)));
12148   format %{ "vaddps  $dst,$src,$mem\t! add packed4F" %}
12149   ins_encode %{
12150     int vector_len = 0;
12151     __ vaddps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12152   %}
12153   ins_pipe( pipe_slow );
12154 %}
12155 
12156 instruct vadd8F_reg(vecY dst, vecY src1, vecY src2) %{
12157   predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
12158   match(Set dst (AddVF src1 src2));
12159   format %{ "vaddps  $dst,$src1,$src2\t! add packed8F" %}
12160   ins_encode %{
12161     int vector_len = 1;
12162     __ vaddps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12163   %}
12164   ins_pipe( pipe_slow );
12165 %}
12166 
12167 instruct vadd8F_mem(vecY dst, vecY src, memory mem) %{
12168   predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
12169   match(Set dst (AddVF src (LoadVector mem)));
12170   format %{ "vaddps  $dst,$src,$mem\t! add packed8F" %}
12171   ins_encode %{
12172     int vector_len = 1;
12173     __ vaddps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12174   %}
12175   ins_pipe( pipe_slow );
12176 %}
12177 
12178 instruct vadd16F_reg(vecZ dst, vecZ src1, vecZ src2) %{
12179   predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
12180   match(Set dst (AddVF src1 src2));
12181   format %{ "vaddps  $dst,$src1,$src2\t! add packed16F" %}
12182   ins_encode %{
12183     int vector_len = 2;
12184     __ vaddps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12185   %}
12186   ins_pipe( pipe_slow );
12187 %}
12188 
12189 instruct vadd16F_mem(vecZ dst, vecZ src, memory mem) %{
12190   predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
12191   match(Set dst (AddVF src (LoadVector mem)));
12192   format %{ "vaddps  $dst,$src,$mem\t! add packed16F" %}
12193   ins_encode %{
12194     int vector_len = 2;
12195     __ vaddps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12196   %}
12197   ins_pipe( pipe_slow );
12198 %}
12199 
12200 // Doubles vector add
12201 instruct vadd2D(vecX dst, vecX src) %{
12202   predicate(n->as_Vector()->length() == 2);
12203   match(Set dst (AddVD dst src));
12204   format %{ "addpd   $dst,$src\t! add packed2D" %}
12205   ins_encode %{
12206     __ addpd($dst$$XMMRegister, $src$$XMMRegister);
12207   %}
12208   ins_pipe( pipe_slow );
12209 %}
12210 
12211 instruct vadd2D_reg(vecX dst, vecX src1, vecX src2) %{
12212   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
12213   match(Set dst (AddVD src1 src2));
12214   format %{ "vaddpd  $dst,$src1,$src2\t! add packed2D" %}
12215   ins_encode %{
12216     int vector_len = 0;
12217     __ vaddpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12218   %}
12219   ins_pipe( pipe_slow );
12220 %}
12221 
12222 instruct vadd2D_mem(vecX dst, vecX src, memory mem) %{
12223   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
12224   match(Set dst (AddVD src (LoadVector mem)));
12225   format %{ "vaddpd  $dst,$src,$mem\t! add packed2D" %}
12226   ins_encode %{
12227     int vector_len = 0;
12228     __ vaddpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12229   %}
12230   ins_pipe( pipe_slow );
12231 %}
12232 
12233 instruct vadd4D_reg(vecY dst, vecY src1, vecY src2) %{
12234   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
12235   match(Set dst (AddVD src1 src2));
12236   format %{ "vaddpd  $dst,$src1,$src2\t! add packed4D" %}
12237   ins_encode %{
12238     int vector_len = 1;
12239     __ vaddpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12240   %}
12241   ins_pipe( pipe_slow );
12242 %}
12243 
12244 instruct vadd4D_mem(vecY dst, vecY src, memory mem) %{
12245   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
12246   match(Set dst (AddVD src (LoadVector mem)));
12247   format %{ "vaddpd  $dst,$src,$mem\t! add packed4D" %}
12248   ins_encode %{
12249     int vector_len = 1;
12250     __ vaddpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12251   %}
12252   ins_pipe( pipe_slow );
12253 %}
12254 
12255 instruct vadd8D_reg(vecZ dst, vecZ src1, vecZ src2) %{
12256   predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
12257   match(Set dst (AddVD src1 src2));
12258   format %{ "vaddpd  $dst,$src1,$src2\t! add packed8D" %}
12259   ins_encode %{
12260     int vector_len = 2;
12261     __ vaddpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12262   %}
12263   ins_pipe( pipe_slow );
12264 %}
12265 
12266 instruct vadd8D_mem(vecZ dst, vecZ src, memory mem) %{
12267   predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
12268   match(Set dst (AddVD src (LoadVector mem)));
12269   format %{ "vaddpd  $dst,$src,$mem\t! add packed8D" %}
12270   ins_encode %{
12271     int vector_len = 2;
12272     __ vaddpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12273   %}
12274   ins_pipe( pipe_slow );
12275 %}
12276 
12277 // --------------------------------- SUB --------------------------------------
12278 
12279 // Bytes vector sub
12280 instruct vsub4B(vecS dst, vecS src) %{
12281   predicate(UseAVX == 0 && n->as_Vector()->length() == 4);
12282   match(Set dst (SubVB dst src));
12283   format %{ "psubb   $dst,$src\t! sub packed4B" %}
12284   ins_encode %{
12285     __ psubb($dst$$XMMRegister, $src$$XMMRegister);
12286   %}
12287   ins_pipe( pipe_slow );
12288 %}
12289 
12290 instruct vsub4B_reg_avx(vecS dst, vecS src1, vecS src2) %{
12291   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 4);
12292   match(Set dst (SubVB src1 src2));
12293   format %{ "vpsubb  $dst,$src1,$src2\t! sub packed4B" %}
12294   ins_encode %{
12295     int vector_len = 0;
12296     __ vpsubb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12297   %}
12298   ins_pipe( pipe_slow );
12299 %}
12300 
12301 instruct vsub4B_reg_evex(vecS dst, vecS src1, vecS src2) %{
12302   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 4);
12303   match(Set dst (SubVB src1 src2));
12304   format %{ "vpsubb  $dst,$src1,$src2\t! sub packed4B" %}
12305   ins_encode %{
12306     int vector_len = 0;
12307     __ vpsubb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12308   %}
12309   ins_pipe( pipe_slow );
12310 %}
12311 
12312 instruct vsub4B_reg_exex_special(vecS dst, vecS src1, vecS src2) %{
12313   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 4);
12314   match(Set dst (SubVB dst src2));
12315   effect(TEMP src1);
12316   format %{ "vpsubb  $dst,$src1,$src2\t! sub packed4B" %}
12317   ins_encode %{
12318     int vector_len = 0;
12319     __ vpsubb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12320   %}
12321   ins_pipe( pipe_slow );
12322 %}
12323 
12324 instruct vsub4B_mem_avx(vecS dst, vecS src, memory mem) %{
12325   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 4);
12326   match(Set dst (SubVB src (LoadVector mem)));
12327   format %{ "vpsubb  $dst,$src,$mem\t! sub packed4B" %}
12328   ins_encode %{
12329     int vector_len = 0;
12330     __ vpsubb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12331   %}
12332   ins_pipe( pipe_slow );
12333 %}
12334 
12335 instruct vsub4B_mem_evex(vecS dst, vecS src, memory mem) %{
12336   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 4);
12337   match(Set dst (SubVB src (LoadVector mem)));
12338   format %{ "vpsubb  $dst,$src,$mem\t! sub packed4B" %}
12339   ins_encode %{
12340     int vector_len = 0;
12341     __ vpsubb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12342   %}
12343   ins_pipe( pipe_slow );
12344 %}
12345 
12346 instruct vsub4B_mem_evex_special(vecS dst, vecS src, memory mem) %{
12347   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 4);
12348   match(Set dst (SubVB dst (LoadVector mem)));
12349   effect(TEMP src);
12350   format %{ "vpsubb  $dst,$src,$mem\t! sub packed4B" %}
12351   ins_encode %{
12352     int vector_len = 0;
12353     __ vpsubb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12354   %}
12355   ins_pipe( pipe_slow );
12356 %}
12357 
12358 instruct vsub8B(vecD dst, vecD src) %{
12359   predicate(UseAVX == 0 && n->as_Vector()->length() == 8);
12360   match(Set dst (SubVB dst src));
12361   format %{ "psubb   $dst,$src\t! sub packed8B" %}
12362   ins_encode %{
12363     __ psubb($dst$$XMMRegister, $src$$XMMRegister);
12364   %}
12365   ins_pipe( pipe_slow );
12366 %}
12367 
12368 instruct vsub8B_reg_avx(vecD dst, vecD src1, vecD src2) %{
12369   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 8);
12370   match(Set dst (SubVB src1 src2));
12371   format %{ "vpsubb  $dst,$src1,$src2\t! sub packed8B" %}
12372   ins_encode %{
12373     int vector_len = 0;
12374     __ vpsubb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12375   %}
12376   ins_pipe( pipe_slow );
12377 %}
12378 
12379 instruct vsub8B_reg_evex(vecD dst, vecD src1, vecD src2) %{
12380   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 8);
12381   match(Set dst (SubVB src1 src2));
12382   format %{ "vpsubb  $dst,$src1,$src2\t! sub packed8B" %}
12383   ins_encode %{
12384     int vector_len = 0;
12385     __ vpsubb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12386   %}
12387   ins_pipe( pipe_slow );
12388 %}
12389 
12390 instruct vsub8B_reg_evex_special(vecD dst, vecD src1, vecD src2) %{
12391   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 8);
12392   match(Set dst (SubVB dst src2));
12393   effect(TEMP src1);
12394   format %{ "vpsubb  $dst,$src1,$src2\t! sub packed8B" %}
12395   ins_encode %{
12396     int vector_len = 0;
12397     __ vpsubb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12398   %}
12399   ins_pipe( pipe_slow );
12400 %}
12401 
12402 instruct vsub8B_mem_avx(vecD dst, vecD src, memory mem) %{
12403   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 8);
12404   match(Set dst (SubVB src (LoadVector mem)));
12405   format %{ "vpsubb  $dst,$src,$mem\t! sub packed8B" %}
12406   ins_encode %{
12407     int vector_len = 0;
12408     __ vpsubb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12409   %}
12410   ins_pipe( pipe_slow );
12411 %}
12412 
12413 instruct vsub8B_mem_evex(vecD dst, vecD src, memory mem) %{
12414   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 8);
12415   match(Set dst (SubVB src (LoadVector mem)));
12416   format %{ "vpsubb  $dst,$src,$mem\t! sub packed8B" %}
12417   ins_encode %{
12418     int vector_len = 0;
12419     __ vpsubb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12420   %}
12421   ins_pipe( pipe_slow );
12422 %}
12423 
12424 instruct vsub8B_mem_evex_special(vecD dst, vecD src, memory mem) %{
12425   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 8);
12426   match(Set dst (SubVB dst (LoadVector mem)));
12427   effect(TEMP src);
12428   format %{ "vpsubb  $dst,$src,$mem\t! sub packed8B" %}
12429   ins_encode %{
12430     int vector_len = 0;
12431     __ vpsubb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12432   %}
12433   ins_pipe( pipe_slow );
12434 %}
12435 
12436 instruct vsub16B(vecX dst, vecX src) %{
12437   predicate(UseAVX == 0 && n->as_Vector()->length() == 16);
12438   match(Set dst (SubVB dst src));
12439   format %{ "psubb   $dst,$src\t! sub packed16B" %}
12440   ins_encode %{
12441     __ psubb($dst$$XMMRegister, $src$$XMMRegister);
12442   %}
12443   ins_pipe( pipe_slow );
12444 %}
12445 
12446 instruct vsub16B_reg_avx(vecX dst, vecX src1, vecX src2) %{
12447   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 16);
12448   match(Set dst (SubVB src1 src2));
12449   format %{ "vpsubb  $dst,$src1,$src2\t! sub packed16B" %}
12450   ins_encode %{
12451     int vector_len = 0;
12452     __ vpsubb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12453   %}
12454   ins_pipe( pipe_slow );
12455 %}
12456 
12457 instruct vsub16B_reg_evex(vecX dst, vecX src1, vecX src2) %{
12458   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 16);
12459   match(Set dst (SubVB src1 src2));
12460   format %{ "vpsubb  $dst,$src1,$src2\t! sub packed16B" %}
12461   ins_encode %{
12462     int vector_len = 0;
12463     __ vpsubb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12464   %}
12465   ins_pipe( pipe_slow );
12466 %}
12467 
12468 instruct vsub16B_reg_evex_special(vecX dst, vecX src1, vecX src2) %{
12469   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 16);
12470   match(Set dst (SubVB dst src2));
12471   effect(TEMP src1);
12472   format %{ "vpsubb  $dst,$src1,$src2\t! sub packed16B" %}
12473   ins_encode %{
12474     int vector_len = 0;
12475     __ vpsubb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12476   %}
12477   ins_pipe( pipe_slow );
12478 %}
12479 
12480 instruct vsub16B_mem_avx(vecX dst, vecX src, memory mem) %{
12481   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 16);
12482   match(Set dst (SubVB src (LoadVector mem)));
12483   format %{ "vpsubb  $dst,$src,$mem\t! sub packed16B" %}
12484   ins_encode %{
12485     int vector_len = 0;
12486     __ vpsubb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12487   %}
12488   ins_pipe( pipe_slow );
12489 %}
12490 
12491 instruct vsub16B_mem_evex(vecX dst, vecX src, memory mem) %{
12492   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 16);
12493   match(Set dst (SubVB src (LoadVector mem)));
12494   format %{ "vpsubb  $dst,$src,$mem\t! sub packed16B" %}
12495   ins_encode %{
12496     int vector_len = 0;
12497     __ vpsubb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12498   %}
12499   ins_pipe( pipe_slow );
12500 %}
12501 
12502 instruct vsub16B_mem_evex_special(vecX dst, vecX src, memory mem) %{
12503   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 16);
12504   match(Set dst (SubVB dst (LoadVector mem)));
12505   effect(TEMP src);
12506   format %{ "vpsubb  $dst,$src,$mem\t! sub packed16B" %}
12507   ins_encode %{
12508     int vector_len = 0;
12509     __ vpsubb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12510   %}
12511   ins_pipe( pipe_slow );
12512 %}
12513 
12514 instruct vsub32B_reg_avx(vecY dst, vecY src1, vecY src2) %{
12515   predicate(VM_Version::supports_avx256only() && n->as_Vector()->length() == 32);
12516   match(Set dst (SubVB src1 src2));
12517   format %{ "vpsubb  $dst,$src1,$src2\t! sub packed32B" %}
12518   ins_encode %{
12519     int vector_len = 1;
12520     __ vpsubb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12521   %}
12522   ins_pipe( pipe_slow );
12523 %}
12524 
12525 instruct vsub32B_reg_evex(vecY dst, vecY src1, vecY src2) %{
12526   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 32);
12527   match(Set dst (SubVB src1 src2));
12528   format %{ "vpsubb  $dst,$src1,$src2\t! sub packed32B" %}
12529   ins_encode %{
12530     int vector_len = 1;
12531     __ vpsubb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12532   %}
12533   ins_pipe( pipe_slow );
12534 %}
12535 
12536 instruct vsub32B_reg_evex_special(vecY dst, vecY src1, vecY src2) %{
12537   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 32);
12538   match(Set dst (SubVB dst src2));
12539   effect(TEMP src1);
12540   format %{ "vpsubb  $dst,$src1,$src2\t! sub packed32B" %}
12541   ins_encode %{
12542     int vector_len = 1;
12543     __ vpsubb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12544   %}
12545   ins_pipe( pipe_slow );
12546 %}
12547 
12548 instruct vsub32B_mem_avx(vecY dst, vecY src, memory mem) %{
12549   predicate(VM_Version::supports_avx256only() && n->as_Vector()->length() == 32);
12550   match(Set dst (SubVB src (LoadVector mem)));
12551   format %{ "vpsubb  $dst,$src,$mem\t! sub packed32B" %}
12552   ins_encode %{
12553     int vector_len = 1;
12554     __ vpsubb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12555   %}
12556   ins_pipe( pipe_slow );
12557 %}
12558 
12559 instruct vsub32B_mem_evex(vecY dst, vecY src, memory mem) %{
12560   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 32);
12561   match(Set dst (SubVB src (LoadVector mem)));
12562   format %{ "vpsubb  $dst,$src,$mem\t! sub packed32B" %}
12563   ins_encode %{
12564     int vector_len = 1;
12565     __ vpsubb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12566   %}
12567   ins_pipe( pipe_slow );
12568 %}
12569 
12570 instruct vsub32B_mem_evex_special(vecY dst, vecY src, memory mem) %{
12571   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 32);
12572   match(Set dst (SubVB dst (LoadVector mem)));
12573   effect(TEMP src);
12574   format %{ "vpsubb  $dst,$src,$mem\t! sub packed32B" %}
12575   ins_encode %{
12576     int vector_len = 1;
12577     __ vpsubb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12578   %}
12579   ins_pipe( pipe_slow );
12580 %}
12581 
12582 instruct vsub64B_reg(vecZ dst, vecZ src1, vecZ src2) %{
12583   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 64);
12584   match(Set dst (SubVB src1 src2));
12585   format %{ "vpsubb  $dst,$src1,$src2\t! sub packed64B" %}
12586   ins_encode %{
12587     int vector_len = 2;
12588     __ vpsubb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12589   %}
12590   ins_pipe( pipe_slow );
12591 %}
12592 
12593 instruct vsub64B_mem(vecZ dst, vecZ src, memory mem) %{
12594   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 64);
12595   match(Set dst (SubVB src (LoadVector mem)));
12596   format %{ "vpsubb  $dst,$src,$mem\t! sub packed64B" %}
12597   ins_encode %{
12598     int vector_len = 2;
12599     __ vpsubb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12600   %}
12601   ins_pipe( pipe_slow );
12602 %}
12603 
12604 // Shorts/Chars vector sub
12605 instruct vsub2S(vecS dst, vecS src) %{
12606   predicate(UseAVX == 0 && n->as_Vector()->length() == 2);
12607   match(Set dst (SubVS dst src));
12608   format %{ "psubw   $dst,$src\t! sub packed2S" %}
12609   ins_encode %{
12610     __ psubw($dst$$XMMRegister, $src$$XMMRegister);
12611   %}
12612   ins_pipe( pipe_slow );
12613 %}
12614 
12615 instruct vsub2S_reg_avx(vecS dst, vecS src1, vecS src2) %{
12616   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 2);
12617   match(Set dst (SubVS src1 src2));
12618   format %{ "vpsubw  $dst,$src1,$src2\t! sub packed2S" %}
12619   ins_encode %{
12620     int vector_len = 0;
12621     __ vpsubw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12622   %}
12623   ins_pipe( pipe_slow );
12624 %}
12625 
12626 instruct vsub2S_reg_evex(vecS dst, vecS src1, vecS src2) %{
12627   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 2);
12628   match(Set dst (SubVS src1 src2));
12629   format %{ "vpsubw  $dst,$src1,$src2\t! sub packed2S" %}
12630   ins_encode %{
12631     int vector_len = 0;
12632     __ vpsubw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12633   %}
12634   ins_pipe( pipe_slow );
12635 %}
12636 
12637 instruct vsub2S_reg_evex_special(vecS dst, vecS src1, vecS src2) %{
12638   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 2);
12639   match(Set dst (SubVS dst src2));
12640   effect(TEMP src1);
12641   format %{ "vpsubw  $dst,$src1,$src2\t! sub packed2S" %}
12642   ins_encode %{
12643     int vector_len = 0;
12644     __ vpsubw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12645   %}
12646   ins_pipe( pipe_slow );
12647 %}
12648 
12649 instruct vsub2S_mem_avx(vecS dst, vecS src, memory mem) %{
12650   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 2);
12651   match(Set dst (SubVS src (LoadVector mem)));
12652   format %{ "vpsubw  $dst,$src,$mem\t! sub packed2S" %}
12653   ins_encode %{
12654     int vector_len = 0;
12655     __ vpsubw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12656   %}
12657   ins_pipe( pipe_slow );
12658 %}
12659 
12660 instruct vsub2S_mem_evex(vecS dst, vecS src, memory mem) %{
12661   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 2);
12662   match(Set dst (SubVS src (LoadVector mem)));
12663   format %{ "vpsubw  $dst,$src,$mem\t! sub packed2S" %}
12664   ins_encode %{
12665     int vector_len = 0;
12666     __ vpsubw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12667   %}
12668   ins_pipe( pipe_slow );
12669 %}
12670 
12671 instruct vsub2S_mem_evex_special(vecS dst, vecS src, memory mem) %{
12672   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 2);
12673   match(Set dst (SubVS dst (LoadVector mem)));
12674   effect(TEMP src);
12675   format %{ "vpsubw  $dst,$src,$mem\t! sub packed2S" %}
12676   ins_encode %{
12677     int vector_len = 0;
12678     __ vpsubw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12679   %}
12680   ins_pipe( pipe_slow );
12681 %}
12682 
12683 instruct vsub4S(vecD dst, vecD src) %{
12684   predicate(UseAVX == 0 && n->as_Vector()->length() == 4);
12685   match(Set dst (SubVS dst src));
12686   format %{ "psubw   $dst,$src\t! sub packed4S" %}
12687   ins_encode %{
12688     __ psubw($dst$$XMMRegister, $src$$XMMRegister);
12689   %}
12690   ins_pipe( pipe_slow );
12691 %}
12692 
12693 instruct vsub4S_reg_avx(vecD dst, vecD src1, vecD src2) %{
12694   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 4);
12695   match(Set dst (SubVS src1 src2));
12696   format %{ "vpsubw  $dst,$src1,$src2\t! sub packed4S" %}
12697   ins_encode %{
12698     int vector_len = 0;
12699     __ vpsubw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12700   %}
12701   ins_pipe( pipe_slow );
12702 %}
12703 
12704 instruct vsub4S_reg_evex(vecD dst, vecD src1, vecD src2) %{
12705   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 4);
12706   match(Set dst (SubVS src1 src2));
12707   format %{ "vpsubw  $dst,$src1,$src2\t! sub packed4S" %}
12708   ins_encode %{
12709     int vector_len = 0;
12710     __ vpsubw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12711   %}
12712   ins_pipe( pipe_slow );
12713 %}
12714 
12715 instruct vsub4S_reg_evex_special(vecD dst, vecD src1, vecD src2) %{
12716   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 4);
12717   match(Set dst (SubVS dst src2));
12718   effect(TEMP src1);
12719   format %{ "vpsubw  $dst,$src1,$src2\t! sub packed4S" %}
12720   ins_encode %{
12721     int vector_len = 0;
12722     __ vpsubw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12723   %}
12724   ins_pipe( pipe_slow );
12725 %}
12726 
12727 instruct vsub4S_mem_avx(vecD dst, vecD src, memory mem) %{
12728   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 4);
12729   match(Set dst (SubVS src (LoadVector mem)));
12730   format %{ "vpsubw  $dst,$src,$mem\t! sub packed4S" %}
12731   ins_encode %{
12732     int vector_len = 0;
12733     __ vpsubw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12734   %}
12735   ins_pipe( pipe_slow );
12736 %}
12737 
12738 instruct vsub4S_mem_evex(vecD dst, vecD src, memory mem) %{
12739   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 4);
12740   match(Set dst (SubVS src (LoadVector mem)));
12741   format %{ "vpsubw  $dst,$src,$mem\t! sub packed4S" %}
12742   ins_encode %{
12743     int vector_len = 0;
12744     __ vpsubw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12745   %}
12746   ins_pipe( pipe_slow );
12747 %}
12748 
12749 instruct vsub4S_mem_evex_special(vecD dst, vecD src, memory mem) %{
12750   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 4);
12751   match(Set dst (SubVS dst (LoadVector mem)));
12752   effect(TEMP src);
12753   format %{ "vpsubw  $dst,$src,$mem\t! sub packed4S" %}
12754   ins_encode %{
12755     int vector_len = 0;
12756     __ vpsubw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12757   %}
12758   ins_pipe( pipe_slow );
12759 %}
12760 
12761 instruct vsub8S(vecX dst, vecX src) %{
12762   predicate(UseAVX == 0 && n->as_Vector()->length() == 8);
12763   match(Set dst (SubVS dst src));
12764   format %{ "psubw   $dst,$src\t! sub packed8S" %}
12765   ins_encode %{
12766     __ psubw($dst$$XMMRegister, $src$$XMMRegister);
12767   %}
12768   ins_pipe( pipe_slow );
12769 %}
12770 
12771 instruct vsub8S_reg_avx(vecX dst, vecX src1, vecX src2) %{
12772   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 8);
12773   match(Set dst (SubVS src1 src2));
12774   format %{ "vpsubw  $dst,$src1,$src2\t! sub packed8S" %}
12775   ins_encode %{
12776     int vector_len = 0;
12777     __ vpsubw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12778   %}
12779   ins_pipe( pipe_slow );
12780 %}
12781 
12782 instruct vsub8S_reg_evex(vecX dst, vecX src1, vecX src2) %{
12783   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 8);
12784   match(Set dst (SubVS src1 src2));
12785   format %{ "vpsubw  $dst,$src1,$src2\t! sub packed8S" %}
12786   ins_encode %{
12787     int vector_len = 0;
12788     __ vpsubw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12789   %}
12790   ins_pipe( pipe_slow );
12791 %}
12792 
12793 instruct vsub8S_reg_evex_special(vecX dst, vecX src1, vecX src2) %{
12794   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 8);
12795   match(Set dst (SubVS dst src2));
12796   effect(TEMP src1);
12797   format %{ "vpsubw  $dst,$src1,$src2\t! sub packed8S" %}
12798   ins_encode %{
12799     int vector_len = 0;
12800     __ vpsubw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12801   %}
12802   ins_pipe( pipe_slow );
12803 %}
12804 
12805 instruct vsub8S_mem_avx(vecX dst, vecX src, memory mem) %{
12806   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 8);
12807   match(Set dst (SubVS src (LoadVector mem)));
12808   format %{ "vpsubw  $dst,$src,$mem\t! sub packed8S" %}
12809   ins_encode %{
12810     int vector_len = 0;
12811     __ vpsubw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12812   %}
12813   ins_pipe( pipe_slow );
12814 %}
12815 
12816 instruct vsub8S_mem_evex(vecX dst, vecX src, memory mem) %{
12817   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 8);
12818   match(Set dst (SubVS src (LoadVector mem)));
12819   format %{ "vpsubw  $dst,$src,$mem\t! sub packed8S" %}
12820   ins_encode %{
12821     int vector_len = 0;
12822     __ vpsubw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12823   %}
12824   ins_pipe( pipe_slow );
12825 %}
12826 
12827 instruct vsub8S_mem_evex_special(vecX dst, vecX src, memory mem) %{
12828   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 8);
12829   match(Set dst (SubVS dst (LoadVector mem)));
12830   effect(TEMP src);
12831   format %{ "vpsubw  $dst,$src,$mem\t! sub packed8S" %}
12832   ins_encode %{
12833     int vector_len = 0;
12834     __ vpsubw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12835   %}
12836   ins_pipe( pipe_slow );
12837 %}
12838 
12839 instruct vsub16S_reg_avx(vecY dst, vecY src1, vecY src2) %{
12840   predicate(VM_Version::supports_avx256only() && n->as_Vector()->length() == 16);
12841   match(Set dst (SubVS src1 src2));
12842   format %{ "vpsubw  $dst,$src1,$src2\t! sub packed16S" %}
12843   ins_encode %{
12844     int vector_len = 1;
12845     __ vpsubw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12846   %}
12847   ins_pipe( pipe_slow );
12848 %}
12849 
12850 instruct vsub16S_reg_evex(vecY dst, vecY src1, vecY src2) %{
12851   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 16);
12852   match(Set dst (SubVS src1 src2));
12853   format %{ "vpsubw  $dst,$src1,$src2\t! sub packed16S" %}
12854   ins_encode %{
12855     int vector_len = 1;
12856     __ vpsubw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12857   %}
12858   ins_pipe( pipe_slow );
12859 %}
12860 
12861 instruct vsub16S_reg_evex_special(vecY dst, vecY src1, vecY src2) %{
12862   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 16);
12863   match(Set dst (SubVS dst src2));
12864   effect(TEMP src1);
12865   format %{ "vpsubw  $dst,$src1,$src2\t! sub packed16S" %}
12866   ins_encode %{
12867     int vector_len = 1;
12868     __ vpsubw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12869   %}
12870   ins_pipe( pipe_slow );
12871 %}
12872 
12873 instruct vsub16S_mem_avx(vecY dst, vecY src, memory mem) %{
12874   predicate(VM_Version::supports_avx256only() && n->as_Vector()->length() == 16);
12875   match(Set dst (SubVS src (LoadVector mem)));
12876   format %{ "vpsubw  $dst,$src,$mem\t! sub packed16S" %}
12877   ins_encode %{
12878     int vector_len = 1;
12879     __ vpsubw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12880   %}
12881   ins_pipe( pipe_slow );
12882 %}
12883 
12884 instruct vsub16S_mem_evex(vecY dst, vecY src, memory mem) %{
12885   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 16);
12886   match(Set dst (SubVS src (LoadVector mem)));
12887   format %{ "vpsubw  $dst,$src,$mem\t! sub packed16S" %}
12888   ins_encode %{
12889     int vector_len = 1;
12890     __ vpsubw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12891   %}
12892   ins_pipe( pipe_slow );
12893 %}
12894 
12895 instruct vsub16S_mem_evex_special(vecY dst, vecY src, memory mem) %{
12896   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 16);
12897   match(Set dst (SubVS dst (LoadVector mem)));
12898    effect(TEMP src);
12899   format %{ "vpsubw  $dst,$src,$mem\t! sub packed16S" %}
12900   ins_encode %{
12901     int vector_len = 1;
12902     __ vpsubw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12903   %}
12904   ins_pipe( pipe_slow );
12905 %}
12906 
12907 instruct vsub32S_reg(vecZ dst, vecZ src1, vecZ src2) %{
12908   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 32);
12909   match(Set dst (SubVS src1 src2));
12910   format %{ "vpsubw  $dst,$src1,$src2\t! sub packed32S" %}
12911   ins_encode %{
12912     int vector_len = 2;
12913     __ vpsubw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12914   %}
12915   ins_pipe( pipe_slow );
12916 %}
12917 
12918 instruct vsub32S_mem(vecZ dst, vecZ src, memory mem) %{
12919   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 32);
12920   match(Set dst (SubVS src (LoadVector mem)));
12921   format %{ "vpsubw  $dst,$src,$mem\t! sub packed32S" %}
12922   ins_encode %{
12923     int vector_len = 2;
12924     __ vpsubw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12925   %}
12926   ins_pipe( pipe_slow );
12927 %}
12928 
12929 // Integers vector sub
12930 instruct vsub2I(vecD dst, vecD src) %{
12931   predicate(n->as_Vector()->length() == 2);
12932   match(Set dst (SubVI dst src));
12933   format %{ "psubd   $dst,$src\t! sub packed2I" %}
12934   ins_encode %{
12935     __ psubd($dst$$XMMRegister, $src$$XMMRegister);
12936   %}
12937   ins_pipe( pipe_slow );
12938 %}
12939 
12940 instruct vsub2I_reg(vecD dst, vecD src1, vecD src2) %{
12941   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
12942   match(Set dst (SubVI src1 src2));
12943   format %{ "vpsubd  $dst,$src1,$src2\t! sub packed2I" %}
12944   ins_encode %{
12945     int vector_len = 0;
12946     __ vpsubd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12947   %}
12948   ins_pipe( pipe_slow );
12949 %}
12950 
12951 instruct vsub2I_mem(vecD dst, vecD src, memory mem) %{
12952   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
12953   match(Set dst (SubVI src (LoadVector mem)));
12954   format %{ "vpsubd  $dst,$src,$mem\t! sub packed2I" %}
12955   ins_encode %{
12956     int vector_len = 0;
12957     __ vpsubd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12958   %}
12959   ins_pipe( pipe_slow );
12960 %}
12961 
12962 instruct vsub4I(vecX dst, vecX src) %{
12963   predicate(n->as_Vector()->length() == 4);
12964   match(Set dst (SubVI dst src));
12965   format %{ "psubd   $dst,$src\t! sub packed4I" %}
12966   ins_encode %{
12967     __ psubd($dst$$XMMRegister, $src$$XMMRegister);
12968   %}
12969   ins_pipe( pipe_slow );
12970 %}
12971 
12972 instruct vsub4I_reg(vecX dst, vecX src1, vecX src2) %{
12973   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
12974   match(Set dst (SubVI src1 src2));
12975   format %{ "vpsubd  $dst,$src1,$src2\t! sub packed4I" %}
12976   ins_encode %{
12977     int vector_len = 0;
12978     __ vpsubd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
12979   %}
12980   ins_pipe( pipe_slow );
12981 %}
12982 
12983 instruct vsub4I_mem(vecX dst, vecX src, memory mem) %{
12984   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
12985   match(Set dst (SubVI src (LoadVector mem)));
12986   format %{ "vpsubd  $dst,$src,$mem\t! sub packed4I" %}
12987   ins_encode %{
12988     int vector_len = 0;
12989     __ vpsubd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
12990   %}
12991   ins_pipe( pipe_slow );
12992 %}
12993 
12994 instruct vsub8I_reg(vecY dst, vecY src1, vecY src2) %{
12995   predicate(UseAVX > 1 && n->as_Vector()->length() == 8);
12996   match(Set dst (SubVI src1 src2));
12997   format %{ "vpsubd  $dst,$src1,$src2\t! sub packed8I" %}
12998   ins_encode %{
12999     int vector_len = 1;
13000     __ vpsubd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
13001   %}
13002   ins_pipe( pipe_slow );
13003 %}
13004 
13005 instruct vsub8I_mem(vecY dst, vecY src, memory mem) %{
13006   predicate(UseAVX > 1 && n->as_Vector()->length() == 8);
13007   match(Set dst (SubVI src (LoadVector mem)));
13008   format %{ "vpsubd  $dst,$src,$mem\t! sub packed8I" %}
13009   ins_encode %{
13010     int vector_len = 1;
13011     __ vpsubd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
13012   %}
13013   ins_pipe( pipe_slow );
13014 %}
13015 
13016 instruct vsub16I_reg(vecZ dst, vecZ src1, vecZ src2) %{
13017   predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
13018   match(Set dst (SubVI src1 src2));
13019   format %{ "vpsubd  $dst,$src1,$src2\t! sub packed16I" %}
13020   ins_encode %{
13021     int vector_len = 2;
13022     __ vpsubd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
13023   %}
13024   ins_pipe( pipe_slow );
13025 %}
13026 
13027 instruct vsub16I_mem(vecZ dst, vecZ src, memory mem) %{
13028   predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
13029   match(Set dst (SubVI src (LoadVector mem)));
13030   format %{ "vpsubd  $dst,$src,$mem\t! sub packed16I" %}
13031   ins_encode %{
13032     int vector_len = 2;
13033     __ vpsubd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
13034   %}
13035   ins_pipe( pipe_slow );
13036 %}
13037 
13038 // Longs vector sub
13039 instruct vsub2L(vecX dst, vecX src) %{
13040   predicate(n->as_Vector()->length() == 2);
13041   match(Set dst (SubVL dst src));
13042   format %{ "psubq   $dst,$src\t! sub packed2L" %}
13043   ins_encode %{
13044     __ psubq($dst$$XMMRegister, $src$$XMMRegister);
13045   %}
13046   ins_pipe( pipe_slow );
13047 %}
13048 
13049 instruct vsub2L_reg(vecX dst, vecX src1, vecX src2) %{
13050   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
13051   match(Set dst (SubVL src1 src2));
13052   format %{ "vpsubq  $dst,$src1,$src2\t! sub packed2L" %}
13053   ins_encode %{
13054     int vector_len = 0;
13055     __ vpsubq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
13056   %}
13057   ins_pipe( pipe_slow );
13058 %}
13059 
13060 instruct vsub2L_mem(vecX dst, vecX src, memory mem) %{
13061   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
13062   match(Set dst (SubVL src (LoadVector mem)));
13063   format %{ "vpsubq  $dst,$src,$mem\t! sub packed2L" %}
13064   ins_encode %{
13065     int vector_len = 0;
13066     __ vpsubq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
13067   %}
13068   ins_pipe( pipe_slow );
13069 %}
13070 
13071 instruct vsub4L_reg(vecY dst, vecY src1, vecY src2) %{
13072   predicate(UseAVX > 1 && n->as_Vector()->length() == 4);
13073   match(Set dst (SubVL src1 src2));
13074   format %{ "vpsubq  $dst,$src1,$src2\t! sub packed4L" %}
13075   ins_encode %{
13076     int vector_len = 1;
13077     __ vpsubq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
13078   %}
13079   ins_pipe( pipe_slow );
13080 %}
13081 
13082 instruct vsub4L_mem(vecY dst, vecY src, memory mem) %{
13083   predicate(UseAVX > 1 && n->as_Vector()->length() == 4);
13084   match(Set dst (SubVL src (LoadVector mem)));
13085   format %{ "vpsubq  $dst,$src,$mem\t! sub packed4L" %}
13086   ins_encode %{
13087     int vector_len = 1;
13088     __ vpsubq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
13089   %}
13090   ins_pipe( pipe_slow );
13091 %}
13092 
13093 instruct vsub8L_reg(vecZ dst, vecZ src1, vecZ src2) %{
13094   predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
13095   match(Set dst (SubVL src1 src2));
13096   format %{ "vpsubq  $dst,$src1,$src2\t! sub packed8L" %}
13097   ins_encode %{
13098     int vector_len = 2;
13099     __ vpsubq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
13100   %}
13101   ins_pipe( pipe_slow );
13102 %}
13103 
13104 instruct vsub8L_mem(vecZ dst, vecZ src, memory mem) %{
13105   predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
13106   match(Set dst (SubVL src (LoadVector mem)));
13107   format %{ "vpsubq  $dst,$src,$mem\t! sub packed8L" %}
13108   ins_encode %{
13109     int vector_len = 2;
13110     __ vpsubq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
13111   %}
13112   ins_pipe( pipe_slow );
13113 %}
13114 
13115 // Floats vector sub
13116 instruct vsub2F(vecD dst, vecD src) %{
13117   predicate(n->as_Vector()->length() == 2);
13118   match(Set dst (SubVF dst src));
13119   format %{ "subps   $dst,$src\t! sub packed2F" %}
13120   ins_encode %{
13121     __ subps($dst$$XMMRegister, $src$$XMMRegister);
13122   %}
13123   ins_pipe( pipe_slow );
13124 %}
13125 
13126 instruct vsub2F_reg(vecD dst, vecD src1, vecD src2) %{
13127   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
13128   match(Set dst (SubVF src1 src2));
13129   format %{ "vsubps  $dst,$src1,$src2\t! sub packed2F" %}
13130   ins_encode %{
13131     int vector_len = 0;
13132     __ vsubps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
13133   %}
13134   ins_pipe( pipe_slow );
13135 %}
13136 
13137 instruct vsub2F_mem(vecD dst, vecD src, memory mem) %{
13138   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
13139   match(Set dst (SubVF src (LoadVector mem)));
13140   format %{ "vsubps  $dst,$src,$mem\t! sub packed2F" %}
13141   ins_encode %{
13142     int vector_len = 0;
13143     __ vsubps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
13144   %}
13145   ins_pipe( pipe_slow );
13146 %}
13147 
13148 instruct vsub4F(vecX dst, vecX src) %{
13149   predicate(n->as_Vector()->length() == 4);
13150   match(Set dst (SubVF dst src));
13151   format %{ "subps   $dst,$src\t! sub packed4F" %}
13152   ins_encode %{
13153     __ subps($dst$$XMMRegister, $src$$XMMRegister);
13154   %}
13155   ins_pipe( pipe_slow );
13156 %}
13157 
13158 instruct vsub4F_reg(vecX dst, vecX src1, vecX src2) %{
13159   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
13160   match(Set dst (SubVF src1 src2));
13161   format %{ "vsubps  $dst,$src1,$src2\t! sub packed4F" %}
13162   ins_encode %{
13163     int vector_len = 0;
13164     __ vsubps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
13165   %}
13166   ins_pipe( pipe_slow );
13167 %}
13168 
13169 instruct vsub4F_mem(vecX dst, vecX src, memory mem) %{
13170   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
13171   match(Set dst (SubVF src (LoadVector mem)));
13172   format %{ "vsubps  $dst,$src,$mem\t! sub packed4F" %}
13173   ins_encode %{
13174     int vector_len = 0;
13175     __ vsubps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
13176   %}
13177   ins_pipe( pipe_slow );
13178 %}
13179 
13180 instruct vsub8F_reg(vecY dst, vecY src1, vecY src2) %{
13181   predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
13182   match(Set dst (SubVF src1 src2));
13183   format %{ "vsubps  $dst,$src1,$src2\t! sub packed8F" %}
13184   ins_encode %{
13185     int vector_len = 1;
13186     __ vsubps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
13187   %}
13188   ins_pipe( pipe_slow );
13189 %}
13190 
13191 instruct vsub8F_mem(vecY dst, vecY src, memory mem) %{
13192   predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
13193   match(Set dst (SubVF src (LoadVector mem)));
13194   format %{ "vsubps  $dst,$src,$mem\t! sub packed8F" %}
13195   ins_encode %{
13196     int vector_len = 1;
13197     __ vsubps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
13198   %}
13199   ins_pipe( pipe_slow );
13200 %}
13201 
13202 instruct vsub16F_reg(vecZ dst, vecZ src1, vecZ src2) %{
13203   predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
13204   match(Set dst (SubVF src1 src2));
13205   format %{ "vsubps  $dst,$src1,$src2\t! sub packed16F" %}
13206   ins_encode %{
13207     int vector_len = 2;
13208     __ vsubps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
13209   %}
13210   ins_pipe( pipe_slow );
13211 %}
13212 
13213 instruct vsub16F_mem(vecZ dst, vecZ src, memory mem) %{
13214   predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
13215   match(Set dst (SubVF src (LoadVector mem)));
13216   format %{ "vsubps  $dst,$src,$mem\t! sub packed16F" %}
13217   ins_encode %{
13218     int vector_len = 2;
13219     __ vsubps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
13220   %}
13221   ins_pipe( pipe_slow );
13222 %}
13223 
13224 // Doubles vector sub
13225 instruct vsub2D(vecX dst, vecX src) %{
13226   predicate(n->as_Vector()->length() == 2);
13227   match(Set dst (SubVD dst src));
13228   format %{ "subpd   $dst,$src\t! sub packed2D" %}
13229   ins_encode %{
13230     __ subpd($dst$$XMMRegister, $src$$XMMRegister);
13231   %}
13232   ins_pipe( pipe_slow );
13233 %}
13234 
13235 instruct vsub2D_reg(vecX dst, vecX src1, vecX src2) %{
13236   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
13237   match(Set dst (SubVD src1 src2));
13238   format %{ "vsubpd  $dst,$src1,$src2\t! sub packed2D" %}
13239   ins_encode %{
13240     int vector_len = 0;
13241     __ vsubpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
13242   %}
13243   ins_pipe( pipe_slow );
13244 %}
13245 
13246 instruct vsub2D_mem(vecX dst, vecX src, memory mem) %{
13247   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
13248   match(Set dst (SubVD src (LoadVector mem)));
13249   format %{ "vsubpd  $dst,$src,$mem\t! sub packed2D" %}
13250   ins_encode %{
13251     int vector_len = 0;
13252     __ vsubpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
13253   %}
13254   ins_pipe( pipe_slow );
13255 %}
13256 
13257 instruct vsub4D_reg(vecY dst, vecY src1, vecY src2) %{
13258   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
13259   match(Set dst (SubVD src1 src2));
13260   format %{ "vsubpd  $dst,$src1,$src2\t! sub packed4D" %}
13261   ins_encode %{
13262     int vector_len = 1;
13263     __ vsubpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
13264   %}
13265   ins_pipe( pipe_slow );
13266 %}
13267 
13268 instruct vsub4D_mem(vecY dst, vecY src, memory mem) %{
13269   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
13270   match(Set dst (SubVD src (LoadVector mem)));
13271   format %{ "vsubpd  $dst,$src,$mem\t! sub packed4D" %}
13272   ins_encode %{
13273     int vector_len = 1;
13274     __ vsubpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
13275   %}
13276   ins_pipe( pipe_slow );
13277 %}
13278 
13279 instruct vsub8D_reg(vecZ dst, vecZ src1, vecZ src2) %{
13280   predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
13281   match(Set dst (SubVD src1 src2));
13282   format %{ "vsubpd  $dst,$src1,$src2\t! sub packed8D" %}
13283   ins_encode %{
13284     int vector_len = 2;
13285     __ vsubpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
13286   %}
13287   ins_pipe( pipe_slow );
13288 %}
13289 
13290 instruct vsub8D_mem(vecZ dst, vecZ src, memory mem) %{
13291   predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
13292   match(Set dst (SubVD src (LoadVector mem)));
13293   format %{ "vsubpd  $dst,$src,$mem\t! sub packed8D" %}
13294   ins_encode %{
13295     int vector_len = 2;
13296     __ vsubpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
13297   %}
13298   ins_pipe( pipe_slow );
13299 %}
13300 
13301 // --------------------------------- MUL --------------------------------------
13302 
13303 // Byte vector mul
13304 
13305 instruct mul4B_reg(vecS dst, vecS src1, vecS src2, vecS tmp2, vecS tmp) %{
13306   predicate(UseSSE > 3 && n->as_Vector()->length() == 4);
13307   match(Set dst (MulVB src1 src2));
13308   effect(TEMP dst, TEMP tmp2, TEMP tmp);
13309   format %{"pmovsxbw  $tmp,$src1\n\t"
13310            "pmovsxbw  $tmp2,$src2\n\t"
13311            "pmullw    $tmp,$tmp2\n\t"
13312            "movdqu    $tmp2,[0x00ff00ff0x00ff00ff]\n\t"
13313            "pand      $tmp,$tmp2\n\t"
13314            "packuswb  $tmp,$tmp\n\t"
13315            "movss     $dst,$tmp\t! mul packed4B" %}
13316   ins_encode %{
13317     __ pmovsxbw($tmp$$XMMRegister, $src1$$XMMRegister);
13318     __ pmovsxbw($tmp2$$XMMRegister, $src2$$XMMRegister);
13319     __ pmullw($tmp$$XMMRegister, $tmp2$$XMMRegister);
13320     __ movdqu($tmp2$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()));
13321     __ pand($tmp$$XMMRegister, $tmp2$$XMMRegister);
13322     __ packuswb($tmp$$XMMRegister, $tmp$$XMMRegister);
13323     __ movss($dst$$XMMRegister, $tmp$$XMMRegister);
13324   %}
13325   ins_pipe( pipe_slow );
13326 %}
13327 
13328 instruct mul8B_reg(vecD dst, vecD src1, vecD src2, vecD tmp2, vecD tmp) %{
13329   predicate(UseSSE > 3 && n->as_Vector()->length() == 8);
13330   match(Set dst (MulVB src1 src2));
13331   effect(TEMP dst, TEMP tmp2, TEMP tmp);
13332   format %{"pmovsxbw  $tmp,$src1\n\t"
13333            "pmovsxbw  $tmp2,$src2\n\t"
13334            "pmullw    $tmp,$tmp2\n\t"
13335            "movdqu    $tmp2,[0x00ff00ff0x00ff00ff]\n\t"
13336            "pand      $tmp,$tmp2\n\t"
13337            "packuswb  $tmp,$tmp\n\t"
13338            "movsd     $dst,$tmp\t! mul packed8B" %}
13339   ins_encode %{
13340     __ pmovsxbw($tmp$$XMMRegister, $src1$$XMMRegister);
13341     __ pmovsxbw($tmp2$$XMMRegister, $src2$$XMMRegister);
13342     __ pmullw($tmp$$XMMRegister, $tmp2$$XMMRegister);
13343     __ movdqu($tmp2$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()));
13344     __ pand($tmp$$XMMRegister, $tmp2$$XMMRegister);
13345     __ packuswb($tmp$$XMMRegister, $tmp$$XMMRegister);
13346     __ movsd($dst$$XMMRegister, $tmp$$XMMRegister);
13347   %}
13348   ins_pipe( pipe_slow );
13349 %}
13350 
13351 instruct mul16B_reg(vecX dst, vecX src1, vecX src2, vecX tmp3, vecX tmp2, vecX tmp) %{
13352   predicate(UseSSE > 3 && n->as_Vector()->length() == 16);
13353   match(Set dst (MulVB src1 src2));
13354   effect(TEMP tmp3, TEMP tmp2, TEMP tmp);
13355   format %{"pmovsxbw  $tmp,$src1\n\t"
13356            "pmovsxbw  $tmp2,$src2\n\t"
13357            "pmullw    $tmp,$tmp2\n\t"
13358            "pshufd    $tmp2,$src1\n\t"
13359            "pshufd    $tmp3,$src2\n\t"
13360            "pmovsxbw  $tmp2,$tmp2\n\t"
13361            "pmovsxbw  $tmp3,$tmp3\n\t"
13362            "pmullw    $tmp2,$tmp3\n\t"
13363            "movdqu    $tmp3,[0x00ff00ff0x00ff00ff]\n\t"
13364            "pand      $tmp,$tmp3\n\t"
13365            "pand      $tmp2,$tmp3\n\t"
13366            "packuswb  $tmp,$tmp2\n\t"
13367            "movdqu    $dst,$tmp \n\t! mul packed16B" %}
13368   ins_encode %{
13369     __ pmovsxbw($tmp$$XMMRegister, $src1$$XMMRegister);
13370     __ pmovsxbw($tmp2$$XMMRegister, $src2$$XMMRegister);
13371     __ pmullw($tmp$$XMMRegister, $tmp2$$XMMRegister);
13372     __ pshufd($tmp2$$XMMRegister, $src1$$XMMRegister, 238);
13373     __ pshufd($tmp3$$XMMRegister, $src2$$XMMRegister, 238);
13374     __ pmovsxbw($tmp2$$XMMRegister, $tmp2$$XMMRegister);
13375     __ pmovsxbw($tmp3$$XMMRegister, $tmp3$$XMMRegister);
13376     __ pmullw($tmp2$$XMMRegister, $tmp3$$XMMRegister);
13377     __ movdqu($tmp3$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()));
13378     __ pand($tmp$$XMMRegister, $tmp3$$XMMRegister);
13379     __ pand($tmp2$$XMMRegister, $tmp3$$XMMRegister);
13380     __ packuswb($tmp$$XMMRegister, $tmp2$$XMMRegister);
13381     __ movdqu($dst$$XMMRegister, $tmp$$XMMRegister);
13382   %}
13383   ins_pipe( pipe_slow );
13384 %}
13385 
13386 instruct vmul16B_reg_avx(vecX dst, vecX src1, vecX src2, vecY tmp2, vecY tmp) %{
13387   predicate(UseAVX > 1 && n->as_Vector()->length() == 16);
13388   match(Set dst (MulVB src1 src2));
13389   effect(TEMP dst, TEMP tmp2, TEMP tmp);
13390   format %{"vpmovsxbw  $tmp,$src1\n\t"
13391            "vpmovsxbw  $tmp2,$src2\n\t"
13392            "vpmullw    $tmp,$tmp2\n\t"
13393            "vmovdqu    $tmp2,[0x00ff00ff0x00ff00ff]\n\t"
13394            "vpand      $tmp,$tmp2\n\t"
13395            "vextracti128_high  $tmp2,$tmp\n\t"
13396            "vpackuswb  $dst,$tmp, $tmp2\n\t! mul packed16B" %}
13397   ins_encode %{
13398   int vector_len = 1;
13399     __ vpmovsxbw($tmp$$XMMRegister, $src1$$XMMRegister, vector_len);
13400     __ vpmovsxbw($tmp2$$XMMRegister, $src2$$XMMRegister, vector_len);
13401     __ vpmullw($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
13402     __ vmovdqu($tmp2$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()));
13403     __ vpand($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, vector_len);
13404     __ vextracti128_high($tmp2$$XMMRegister, $tmp$$XMMRegister);
13405     __ vpackuswb($dst$$XMMRegister, $tmp$$XMMRegister, $tmp2$$XMMRegister, 0);
13406   %}
13407   ins_pipe( pipe_slow );
13408 %}
13409 
13410 instruct vmul32B_reg_avx(vecY dst, vecY src1, vecY src2, vecY tmp1, vecY tmp2, vecY tmp3) %{
13411   predicate(UseAVX > 1 && n->as_Vector()->length() == 32);
13412   match(Set dst (MulVB src1 src2));
13413   effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3);
13414   format %{"vextracti128_high  $tmp1,$src1\n\t"
13415            "vextracti128_high  $tmp3,$src2\n\t"
13416            "vpmovsxbw $tmp1,$tmp1\n\t"
13417            "vpmovsxbw $tmp3,$tmp3\n\t"
13418            "vpmullw $tmp1,$tmp1,$tmp3\n\t"
13419            "vpmovsxbw $tmp2,$src1\n\t"
13420            "vpmovsxbw $tmp3,$src2\n\t"
13421            "vpmullw $tmp2,$tmp2,$tmp3\n\t"
13422            "vmovdqu $tmp3, [0x00ff00ff0x00ff00ff]\n\t"
13423            "vpbroadcastd $tmp3, $tmp3\n\t"
13424            "vpand $tmp2,$tmp2,$tmp3\n\t"
13425            "vpand $tmp1,$tmp1,$tmp3\n\t"
13426            "vpackuswb $dst,$tmp2,$tmp1\n\t"
13427            "vpermq $dst, $dst, 0xD8\t! mul packed32B" %}
13428   ins_encode %{
13429     int vector_len = 1;
13430     __ vextracti128_high($tmp1$$XMMRegister, $src1$$XMMRegister);
13431     __ vextracti128_high($tmp3$$XMMRegister, $src2$$XMMRegister);
13432     __ vpmovsxbw($tmp1$$XMMRegister, $tmp1$$XMMRegister, vector_len);
13433     __ vpmovsxbw($tmp3$$XMMRegister, $tmp3$$XMMRegister, vector_len);
13434     __ vpmullw($tmp1$$XMMRegister, $tmp1$$XMMRegister, $tmp3$$XMMRegister, vector_len);
13435     __ vpmovsxbw($tmp2$$XMMRegister, $src1$$XMMRegister, vector_len);
13436     __ vpmovsxbw($tmp3$$XMMRegister, $src2$$XMMRegister, vector_len);
13437     __ vpmullw($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister, vector_len);
13438     __ vmovdqu($tmp3$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()));
13439     __ vpbroadcastd($tmp3$$XMMRegister, $tmp3$$XMMRegister);
13440     __ vpand($tmp1$$XMMRegister, $tmp1$$XMMRegister, $tmp3$$XMMRegister, vector_len);
13441     __ vpand($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister, vector_len);
13442     __ vpackuswb($dst$$XMMRegister, $tmp2$$XMMRegister, $tmp1$$XMMRegister, vector_len);
13443     __ vpermq($dst$$XMMRegister, $dst$$XMMRegister, 0xD8, vector_len);
13444   %}
13445   ins_pipe( pipe_slow );
13446 %}
13447 
13448 instruct vmul64B_reg_avx(vecZ dst, vecZ src1, vecZ src2, vecZ tmp1, vecZ tmp2, vecZ tmp3, vecZ tmp4, vecZ tmp5, vecZ tmp6) %{
13449   predicate(UseAVX > 2 && n->as_Vector()->length() == 64);
13450   match(Set dst (MulVB src1 src2));
13451   effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4);
13452   format %{"vextracti64x4_high  $tmp1,$src1\n\t"
13453            "vextracti64x4_high  $tmp3,$src2\n\t"
13454            "vpmovsxbw $tmp1,$tmp1\n\t"
13455            "vpmovsxbw $tmp3,$tmp3\n\t"
13456            "vpmullw $tmp1,$tmp1,$tmp3\n\t"
13457            "vpmovsxbw $tmp2,$src1\n\t"
13458            "vpmovsxbw $tmp3,$src2\n\t"
13459            "vpmullw $tmp2,$tmp2,$tmp3\n\t"
13460            "vmovdqu $tmp3, [0x00ff00ff0x00ff00ff]\n\t"
13461            "evpbroadcastd $tmp3, $tmp3\n\t"
13462            "evpandd $tmp1,$tmp1,$tmp3\n\t"
13463            "evpandd $tmp2,$tmp2,$tmp3\n\t"
13464            "vpackuswb $tmp1,$tmp2,$tmp1\n\t"
13465            "vextracti64x4_high  $tmp3,$tmp1\n\t"
13466            "vpermq $tmp3, $tmp3, 0x8D\n\t"
13467            "vpermq $tmp1, $tmp1, 0xD8\n\t"
13468            "vmovdqu  $tmp4,$tmp3\n\t"
13469            "vmovdqu  $tmp2,$tmp1\n\t"
13470            "vpblendd  $tmp3,$tmp3,$tmp1\n\t"
13471            "vpblendd  $tmp2,$tmp2,$tmp4\n\t"
13472            "vpermq $tmp2,$tmp2,0x4E\n\t"
13473            "vinserti64x4 $dst,$dst,$tmp3,0x00\n\t"
13474            "vinserti64x4 $dst,$dst,$tmp2,0x01\t! mul packed64B" %}
13475   ins_encode %{
13476     int vector_len = 2;
13477     KRegister ktmp = k1;
13478     __ vextracti64x4_high($tmp1$$XMMRegister, $src1$$XMMRegister);
13479     __ vextracti64x4_high($tmp3$$XMMRegister, $src2$$XMMRegister);
13480     __ vpmovsxbw($tmp1$$XMMRegister, $tmp1$$XMMRegister, vector_len);
13481     __ vpmovsxbw($tmp3$$XMMRegister, $tmp3$$XMMRegister, vector_len);
13482     __ vpmullw($tmp1$$XMMRegister, $tmp1$$XMMRegister, $tmp3$$XMMRegister, vector_len);
13483     __ vpmovsxbw($tmp2$$XMMRegister, $src1$$XMMRegister, vector_len);
13484     __ vpmovsxbw($tmp3$$XMMRegister, $src2$$XMMRegister, vector_len);
13485     __ vpmullw($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister, vector_len);
13486     __ vmovdqu($tmp3$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()));
13487     __ evpbroadcastd($tmp3$$XMMRegister, $tmp3$$XMMRegister, vector_len);
13488     __ evpandd($tmp1$$XMMRegister, ktmp, $tmp1$$XMMRegister, $tmp3$$XMMRegister, false, vector_len);
13489     __ evpandd($tmp2$$XMMRegister, ktmp, $tmp2$$XMMRegister, $tmp3$$XMMRegister, false, vector_len);
13490     __ vpackuswb($tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp1$$XMMRegister, vector_len);
13491     __ vextracti64x4_high($tmp3$$XMMRegister, $tmp1$$XMMRegister);
13492     __ vpermq($tmp3$$XMMRegister, $tmp3$$XMMRegister, 0x8D, 1);
13493     __ vpermq($tmp1$$XMMRegister, $tmp1$$XMMRegister, 0xD8, 1);
13494     __ vmovdqu($tmp4$$XMMRegister, $tmp3$$XMMRegister);
13495     __ vmovdqu($tmp2$$XMMRegister, $tmp1$$XMMRegister);
13496     __ vpblendd($tmp3$$XMMRegister, $tmp3$$XMMRegister, $tmp1$$XMMRegister, 0x0F, 1);
13497     __ vpblendd($tmp2$$XMMRegister, $tmp2$$XMMRegister, $tmp4$$XMMRegister, 0x0F, 1);
13498     __ vpermq($tmp2$$XMMRegister, $tmp2$$XMMRegister, 0x4E, 1);
13499     __ vinserti64x4($dst$$XMMRegister, $dst$$XMMRegister, $tmp3$$XMMRegister, 0x00);
13500     __ vinserti64x4($dst$$XMMRegister, $dst$$XMMRegister, $tmp2$$XMMRegister, 0x01);
13501   %}
13502   ins_pipe( pipe_slow );
13503 %}
13504 
13505 // Shorts/Chars vector mul
13506 instruct vmul2S(vecS dst, vecS src) %{
13507   predicate(UseAVX == 0 && n->as_Vector()->length() == 2);
13508   match(Set dst (MulVS dst src));
13509   format %{ "pmullw $dst,$src\t! mul packed2S" %}
13510   ins_encode %{
13511     __ pmullw($dst$$XMMRegister, $src$$XMMRegister);
13512   %}
13513   ins_pipe( pipe_slow );
13514 %}
13515 
13516 instruct vmul2S_reg_avx(vecS dst, vecS src1, vecS src2) %{
13517   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 2);
13518   match(Set dst (MulVS src1 src2));
13519   format %{ "vpmullw $dst,$src1,$src2\t! mul packed2S" %}
13520   ins_encode %{
13521     int vector_len = 0;
13522     __ vpmullw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
13523   %}
13524   ins_pipe( pipe_slow );
13525 %}
13526 
13527 instruct vmul2S_reg_evex(vecS dst, vecS src1, vecS src2) %{
13528   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 2);
13529   match(Set dst (MulVS src1 src2));
13530   format %{ "vpmullw $dst,$src1,$src2\t! mul packed2S" %}
13531   ins_encode %{
13532     int vector_len = 0;
13533     __ vpmullw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
13534   %}
13535   ins_pipe( pipe_slow );
13536 %}
13537 
13538 instruct vmul2S_evex_special(vecS dst, vecS src1, vecS src2) %{
13539   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 2);
13540   match(Set dst (MulVS dst src2));
13541   effect(TEMP src1);
13542   format %{ "vpmullw $dst,$src1,$src2\t! mul packed2S" %}
13543   ins_encode %{
13544     int vector_len = 0;
13545     __ vpmullw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
13546   %}
13547   ins_pipe( pipe_slow );
13548 %}
13549 
13550 instruct vmul2S_mem_avx(vecS dst, vecS src, memory mem) %{
13551   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 2);
13552   match(Set dst (MulVS src (LoadVector mem)));
13553   format %{ "vpmullw $dst,$src,$mem\t! mul packed2S" %}
13554   ins_encode %{
13555     int vector_len = 0;
13556     __ vpmullw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
13557   %}
13558   ins_pipe( pipe_slow );
13559 %}
13560 
13561 instruct vmul2S_mem_evex(vecS dst, vecS src, memory mem) %{
13562   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 2);
13563   match(Set dst (MulVS src (LoadVector mem)));
13564   format %{ "vpmullw $dst,$src,$mem\t! mul packed2S" %}
13565   ins_encode %{
13566     int vector_len = 0;
13567     __ vpmullw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
13568   %}
13569   ins_pipe( pipe_slow );
13570 %}
13571 
13572 instruct vmul2S_mem_evex_special(vecS dst, vecS src, memory mem) %{
13573   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 2);
13574   match(Set dst (MulVS dst (LoadVector mem)));
13575   effect(TEMP src);
13576   format %{ "vpmullw $dst,$src,$mem\t! mul packed2S" %}
13577   ins_encode %{
13578     int vector_len = 0;
13579     __ vpmullw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
13580   %}
13581   ins_pipe( pipe_slow );
13582 %}
13583 
13584 instruct vmul4S(vecD dst, vecD src) %{
13585   predicate(UseAVX == 0 && n->as_Vector()->length() == 4);
13586   match(Set dst (MulVS dst src));
13587   format %{ "pmullw  $dst,$src\t! mul packed4S" %}
13588   ins_encode %{
13589     __ pmullw($dst$$XMMRegister, $src$$XMMRegister);
13590   %}
13591   ins_pipe( pipe_slow );
13592 %}
13593 
13594 instruct vmul4S_reg_avx(vecD dst, vecD src1, vecD src2) %{
13595   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 4);
13596   match(Set dst (MulVS src1 src2));
13597   format %{ "vpmullw $dst,$src1,$src2\t! mul packed4S" %}
13598   ins_encode %{
13599     int vector_len = 0;
13600     __ vpmullw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
13601   %}
13602   ins_pipe( pipe_slow );
13603 %}
13604 
13605 instruct vmul4S_reg_evex(vecD dst, vecD src1, vecD src2) %{
13606   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 4);
13607   match(Set dst (MulVS src1 src2));
13608   format %{ "vpmullw $dst,$src1,$src2\t! mul packed4S" %}
13609   ins_encode %{
13610     int vector_len = 0;
13611     __ vpmullw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
13612   %}
13613   ins_pipe( pipe_slow );
13614 %}
13615 
13616 instruct vmul4S_reg_evex_special(vecD dst, vecD src1, vecD src2) %{
13617   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 4);
13618   match(Set dst (MulVS dst src2));
13619   effect(TEMP src1);
13620   format %{ "vpmullw $dst,$src1,$src2\t! mul packed4S" %}
13621   ins_encode %{
13622     int vector_len = 0;
13623     __ vpmullw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
13624   %}
13625   ins_pipe( pipe_slow );
13626 %}
13627 
13628 instruct vmul4S_mem_avx(vecD dst, vecD src, memory mem) %{
13629   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 4);
13630   match(Set dst (MulVS src (LoadVector mem)));
13631   format %{ "vpmullw $dst,$src,$mem\t! mul packed4S" %}
13632   ins_encode %{
13633     int vector_len = 0;
13634     __ vpmullw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
13635   %}
13636   ins_pipe( pipe_slow );
13637 %}
13638 
13639 instruct vmul4S_mem_evex(vecD dst, vecD src, memory mem) %{
13640   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 4);
13641   match(Set dst (MulVS src (LoadVector mem)));
13642   format %{ "vpmullw $dst,$src,$mem\t! mul packed4S" %}
13643   ins_encode %{
13644     int vector_len = 0;
13645     __ vpmullw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
13646   %}
13647   ins_pipe( pipe_slow );
13648 %}
13649 
13650 instruct vmul4S_mem_evex_special(vecD dst, vecD src, memory mem) %{
13651   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 4);
13652   match(Set dst (MulVS dst (LoadVector mem)));
13653   effect(TEMP src);
13654   format %{ "vpmullw $dst,$src,$mem\t! mul packed4S" %}
13655   ins_encode %{
13656     int vector_len = 0;
13657     __ vpmullw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
13658   %}
13659   ins_pipe( pipe_slow );
13660 %}
13661 
13662 instruct vmul8S(vecX dst, vecX src) %{
13663   predicate(UseAVX == 0 && n->as_Vector()->length() == 8);
13664   match(Set dst (MulVS dst src));
13665   format %{ "pmullw  $dst,$src\t! mul packed8S" %}
13666   ins_encode %{
13667     __ pmullw($dst$$XMMRegister, $src$$XMMRegister);
13668   %}
13669   ins_pipe( pipe_slow );
13670 %}
13671 
13672 instruct vmul8S_reg_avx(vecX dst, vecX src1, vecX src2) %{
13673   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 8);
13674   match(Set dst (MulVS src1 src2));
13675   format %{ "vpmullw $dst,$src1,$src2\t! mul packed8S" %}
13676   ins_encode %{
13677     int vector_len = 0;
13678     __ vpmullw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
13679   %}
13680   ins_pipe( pipe_slow );
13681 %}
13682 
13683 instruct vmul8S_reg_evex(vecX dst, vecX src1, vecX src2) %{
13684   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 8);
13685   match(Set dst (MulVS src1 src2));
13686   format %{ "vpmullw $dst,$src1,$src2\t! mul packed8S" %}
13687   ins_encode %{
13688     int vector_len = 0;
13689     __ vpmullw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
13690   %}
13691   ins_pipe( pipe_slow );
13692 %}
13693 
13694 instruct vmul8S_reg_evex_special(vecX dst, vecX src1, vecX src2) %{
13695   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 8);
13696   match(Set dst (MulVS dst src2));
13697   effect(TEMP src1);
13698   format %{ "vpmullw $dst,$src1,$src2\t! mul packed8S" %}
13699   ins_encode %{
13700     int vector_len = 0;
13701     __ vpmullw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
13702   %}
13703   ins_pipe( pipe_slow );
13704 %}
13705 
13706 instruct vmul8S_mem_avx(vecX dst, vecX src, memory mem) %{
13707   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 8);
13708   match(Set dst (MulVS src (LoadVector mem)));
13709   format %{ "vpmullw $dst,$src,$mem\t! mul packed8S" %}
13710   ins_encode %{
13711     int vector_len = 0;
13712     __ vpmullw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
13713   %}
13714   ins_pipe( pipe_slow );
13715 %}
13716 
13717 instruct vmul8S_mem_evex(vecX dst, vecX src, memory mem) %{
13718   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 8);
13719   match(Set dst (MulVS src (LoadVector mem)));
13720   format %{ "vpmullw $dst,$src,$mem\t! mul packed8S" %}
13721   ins_encode %{
13722     int vector_len = 0;
13723     __ vpmullw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
13724   %}
13725   ins_pipe( pipe_slow );
13726 %}
13727 
13728 instruct vmul8S_mem_evex_special(vecX dst, vecX src, memory mem) %{
13729   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 8);
13730   match(Set dst (MulVS dst (LoadVector mem)));
13731   effect(TEMP src);
13732   format %{ "vpmullw $dst,$src,$mem\t! mul packed8S" %}
13733   ins_encode %{
13734     int vector_len = 0;
13735     __ vpmullw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
13736   %}
13737   ins_pipe( pipe_slow );
13738 %}
13739 
13740 instruct vmul16S_reg_avx(vecY dst, vecY src1, vecY src2) %{
13741   predicate(VM_Version::supports_avx256only() && n->as_Vector()->length() == 16);
13742   match(Set dst (MulVS src1 src2));
13743   format %{ "vpmullw $dst,$src1,$src2\t! mul packed16S" %}
13744   ins_encode %{
13745     int vector_len = 1;
13746     __ vpmullw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
13747   %}
13748   ins_pipe( pipe_slow );
13749 %}
13750 
13751 instruct vmul16S_reg_evex(vecY dst, vecY src1, vecY src2) %{
13752   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 16);
13753   match(Set dst (MulVS src1 src2));
13754   format %{ "vpmullw $dst,$src1,$src2\t! mul packed16S" %}
13755   ins_encode %{
13756     int vector_len = 1;
13757     __ vpmullw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
13758   %}
13759   ins_pipe( pipe_slow );
13760 %}
13761 
13762 instruct vmul16S_reg_evex_special(vecY dst, vecY src1, vecY src2) %{
13763   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 16);
13764   match(Set dst (MulVS dst src2));
13765   effect(TEMP src1);
13766   format %{ "vpmullw $dst,$src1,$src2\t! mul packed16S" %}
13767   ins_encode %{
13768     int vector_len = 1;
13769     __ vpmullw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
13770   %}
13771   ins_pipe( pipe_slow );
13772 %}
13773 
13774 instruct vmul16S_mem_avx(vecY dst, vecY src, memory mem) %{
13775   predicate(VM_Version::supports_avx256only() && n->as_Vector()->length() == 16);
13776   match(Set dst (MulVS src (LoadVector mem)));
13777   format %{ "vpmullw $dst,$src,$mem\t! mul packed16S" %}
13778   ins_encode %{
13779     int vector_len = 1;
13780     __ vpmullw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
13781   %}
13782   ins_pipe( pipe_slow );
13783 %}
13784 
13785 instruct vmul16S_mem_evex(vecY dst, vecY src, memory mem) %{
13786   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 16);
13787   match(Set dst (MulVS src (LoadVector mem)));
13788   format %{ "vpmullw $dst,$src,$mem\t! mul packed16S" %}
13789   ins_encode %{
13790     int vector_len = 1;
13791     __ vpmullw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
13792   %}
13793   ins_pipe( pipe_slow );
13794 %}
13795 
13796 instruct vmul16S_mem_evex_special(vecY dst, vecY src, memory mem) %{
13797   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 16);
13798   match(Set dst (MulVS dst (LoadVector mem)));
13799   effect(TEMP src);
13800   format %{ "vpmullw $dst,$src,$mem\t! mul packed16S" %}
13801   ins_encode %{
13802     int vector_len = 1;
13803     __ vpmullw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
13804   %}
13805   ins_pipe( pipe_slow );
13806 %}
13807 
13808 instruct vmul32S_reg(vecZ dst, vecZ src1, vecZ src2) %{
13809   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 32);
13810   match(Set dst (MulVS src1 src2));
13811   format %{ "vpmullw $dst,$src1,$src2\t! mul packed32S" %}
13812   ins_encode %{
13813     int vector_len = 2;
13814     __ vpmullw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
13815   %}
13816   ins_pipe( pipe_slow );
13817 %}
13818 
13819 instruct vmul32S_mem(vecZ dst, vecZ src, memory mem) %{
13820   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 32);
13821   match(Set dst (MulVS src (LoadVector mem)));
13822   format %{ "vpmullw $dst,$src,$mem\t! mul packed32S" %}
13823   ins_encode %{
13824     int vector_len = 2;
13825     __ vpmullw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
13826   %}
13827   ins_pipe( pipe_slow );
13828 %}
13829 
13830 // Integers vector mul (sse4_1)
13831 instruct vmul2I(vecD dst, vecD src) %{
13832   predicate(UseSSE > 3 && n->as_Vector()->length() == 2);
13833   match(Set dst (MulVI dst src));
13834   format %{ "pmulld  $dst,$src\t! mul packed2I" %}
13835   ins_encode %{
13836     __ pmulld($dst$$XMMRegister, $src$$XMMRegister);
13837   %}
13838   ins_pipe( pipe_slow );
13839 %}
13840 
13841 instruct vmul2I_reg(vecD dst, vecD src1, vecD src2) %{
13842   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
13843   match(Set dst (MulVI src1 src2));
13844   format %{ "vpmulld $dst,$src1,$src2\t! mul packed2I" %}
13845   ins_encode %{
13846     int vector_len = 0;
13847     __ vpmulld($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
13848   %}
13849   ins_pipe( pipe_slow );
13850 %}
13851 
13852 instruct vmul2I_mem(vecD dst, vecD src, memory mem) %{
13853   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
13854   match(Set dst (MulVI src (LoadVector mem)));
13855   format %{ "vpmulld $dst,$src,$mem\t! mul packed2I" %}
13856   ins_encode %{
13857     int vector_len = 0;
13858     __ vpmulld($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
13859   %}
13860   ins_pipe( pipe_slow );
13861 %}
13862 
13863 instruct vmul4I(vecX dst, vecX src) %{
13864   predicate(UseSSE > 3 && n->as_Vector()->length() == 4);
13865   match(Set dst (MulVI dst src));
13866   format %{ "pmulld  $dst,$src\t! mul packed4I" %}
13867   ins_encode %{
13868     __ pmulld($dst$$XMMRegister, $src$$XMMRegister);
13869   %}
13870   ins_pipe( pipe_slow );
13871 %}
13872 
13873 instruct vmul4I_reg(vecX dst, vecX src1, vecX src2) %{
13874   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
13875   match(Set dst (MulVI src1 src2));
13876   format %{ "vpmulld $dst,$src1,$src2\t! mul packed4I" %}
13877   ins_encode %{
13878     int vector_len = 0;
13879     __ vpmulld($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
13880   %}
13881   ins_pipe( pipe_slow );
13882 %}
13883 
13884 instruct vmul4I_mem(vecX dst, vecX src, memory mem) %{
13885   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
13886   match(Set dst (MulVI src (LoadVector mem)));
13887   format %{ "vpmulld $dst,$src,$mem\t! mul packed4I" %}
13888   ins_encode %{
13889     int vector_len = 0;
13890     __ vpmulld($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
13891   %}
13892   ins_pipe( pipe_slow );
13893 %}
13894 
13895 // Long vector mul
13896 
13897 instruct mul2L_reg(vecX dst, vecX src2, vecX tmp) %{
13898   predicate(UseSSE > 3 && n->as_Vector()->length() == 2 && VM_Version::supports_sse4_1());
13899   match(Set dst (MulVL dst src2));
13900   effect(TEMP dst, TEMP tmp);
13901   format %{ "pshufd $tmp,$src2, 177\n\t"
13902             "pmulld $tmp,$dst\n\t"
13903             "phaddd $tmp,$tmp\n\t"
13904             "pmovzxdq $tmp,$tmp\n\t"
13905             "psllq $tmp, 32\n\t"
13906             "pmuludq $dst,$src2\n\t"
13907             "paddq $dst,$tmp\n\t! mul packed2L" %}
13908 
13909   ins_encode %{
13910     int vector_len = 0;
13911     __ pshufd($tmp$$XMMRegister, $src2$$XMMRegister, 177);
13912     __ pmulld($tmp$$XMMRegister, $dst$$XMMRegister);
13913     __ phaddd($tmp$$XMMRegister, $tmp$$XMMRegister);
13914     __ pmovzxdq($tmp$$XMMRegister, $tmp$$XMMRegister);
13915     __ psllq($tmp$$XMMRegister, 32);
13916     __ pmuludq($dst$$XMMRegister, $src2$$XMMRegister);
13917     __ paddq($dst$$XMMRegister, $tmp$$XMMRegister);
13918   %}
13919   ins_pipe( pipe_slow );
13920 %}
13921 
13922 instruct vmul2L_reg_avx(vecX dst, vecX src1, vecX src2, vecX tmp1, vecX tmp) %{
13923   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 && VM_Version::supports_avx());
13924   match(Set dst (MulVL src1 src2));
13925   effect(TEMP tmp1, TEMP tmp);
13926   format %{ "vpshufd $tmp,$src2\n\t"
13927             "vpmulld $tmp,$src1,$tmp\n\t"
13928             "vphaddd $tmp,$tmp,$tmp\n\t"
13929             "vpmovzxdq $tmp,$tmp\n\t"
13930             "vpsllq $tmp,$tmp\n\t"
13931             "vpmuludq $tmp1,$src1,$src2\n\t"
13932             "vpaddq $dst,$tmp,$tmp1\t! mul packed2L" %}
13933   ins_encode %{
13934     int vector_len = 0;
13935     __ vpshufd($tmp$$XMMRegister, $src2$$XMMRegister, 177, vector_len);
13936     __ vpmulld($tmp$$XMMRegister, $src1$$XMMRegister, $tmp$$XMMRegister, vector_len);
13937     __ vphaddd($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp$$XMMRegister, vector_len);
13938     __ vpmovzxdq($tmp$$XMMRegister, $tmp$$XMMRegister, vector_len);
13939     __ vpsllq($tmp$$XMMRegister, $tmp$$XMMRegister, 32, vector_len);
13940     __ vpmuludq($tmp1$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
13941     __ vpaddq($dst$$XMMRegister, $tmp$$XMMRegister, $tmp1$$XMMRegister, vector_len);
13942   %}
13943   ins_pipe( pipe_slow );
13944 %}
13945 
13946 instruct vmul2L_reg(vecX dst, vecX src1, vecX src2) %{
13947   predicate(UseAVX > 2 && n->as_Vector()->length() == 2 && VM_Version::supports_avx512dq());
13948   match(Set dst (MulVL src1 src2));
13949   format %{ "vpmullq $dst,$src1,$src2\t! mul packed2L" %}
13950   ins_encode %{
13951     int vector_len = 0;
13952     __ vpmullq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
13953   %}
13954   ins_pipe( pipe_slow );
13955 %}
13956 
13957 instruct vmul2L_mem(vecX dst, vecX src, memory mem) %{
13958   predicate(UseAVX > 2 && n->as_Vector()->length() == 2 && VM_Version::supports_avx512dq());
13959   match(Set dst (MulVL src (LoadVector mem)));
13960   format %{ "vpmullq $dst,$src,$mem\t! mul packed2L" %}
13961   ins_encode %{
13962     int vector_len = 0;
13963     __ vpmullq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
13964   %}
13965   ins_pipe( pipe_slow );
13966 %}
13967 
13968 instruct vmul4L_reg_avx(vecY dst, vecY src1, vecY src2, vecY tmp, vecY tmp1,) %{
13969   predicate(UseAVX > 1 && n->as_Vector()->length() == 4 && VM_Version::supports_avx2());
13970   match(Set dst (MulVL src1 src2));
13971   effect(TEMP tmp1, TEMP tmp);
13972   format %{ "vpshufd $tmp,$src2\n\t"
13973             "vpmulld $tmp,$src1,$tmp\n\t"
13974             "vphaddd $tmp,$tmp,$tmp\n\t"
13975             "vpmovzxdq $tmp,$tmp\n\t"
13976             "vpsllq $tmp,$tmp\n\t"
13977             "vpmuludq $tmp1,$src1,$src2\n\t"
13978             "vpaddq $dst,$tmp,$tmp1\t! mul packed4L" %}
13979   ins_encode %{
13980     int vector_len = 1;
13981     __ vpshufd($tmp$$XMMRegister, $src2$$XMMRegister, 177, vector_len);
13982     __ vpmulld($tmp$$XMMRegister, $src1$$XMMRegister, $tmp$$XMMRegister, vector_len);
13983     __ vextracti128_high($tmp1$$XMMRegister, $tmp$$XMMRegister);
13984     __ vphaddd($tmp$$XMMRegister, $tmp$$XMMRegister, $tmp1$$XMMRegister, vector_len);
13985     __ vpmovzxdq($tmp$$XMMRegister, $tmp$$XMMRegister, vector_len);
13986     __ vpsllq($tmp$$XMMRegister, $tmp$$XMMRegister, 32, vector_len);
13987     __ vpmuludq($tmp1$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
13988     __ vpaddq($dst$$XMMRegister, $tmp$$XMMRegister, $tmp1$$XMMRegister, vector_len);
13989   %}
13990   ins_pipe( pipe_slow );
13991 %}
13992 
13993 instruct vmul4L_reg(vecY dst, vecY src1, vecY src2) %{
13994   predicate(UseAVX > 2 && n->as_Vector()->length() == 4 && VM_Version::supports_avx512dq());
13995   match(Set dst (MulVL src1 src2));
13996   format %{ "vpmullq $dst,$src1,$src2\t! mul packed4L" %}
13997   ins_encode %{
13998     int vector_len = 1;
13999     __ vpmullq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14000   %}
14001   ins_pipe( pipe_slow );
14002 %}
14003 
14004 instruct vmul4L_mem(vecY dst, vecY src, memory mem) %{
14005   predicate(UseAVX > 2 && n->as_Vector()->length() == 4 && VM_Version::supports_avx512dq());
14006   match(Set dst (MulVL src (LoadVector mem)));
14007   format %{ "vpmullq $dst,$src,$mem\t! mul packed4L" %}
14008   ins_encode %{
14009     int vector_len = 1;
14010     __ vpmullq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
14011   %}
14012   ins_pipe( pipe_slow );
14013 %}
14014 
14015 instruct vmul8L_reg(vecZ dst, vecZ src1, vecZ src2) %{
14016   predicate(UseAVX > 2 && n->as_Vector()->length() == 8 && VM_Version::supports_avx512dq());
14017   match(Set dst (MulVL src1 src2));
14018   format %{ "vpmullq $dst,$src1,$src2\t! mul packed8L" %}
14019   ins_encode %{
14020     int vector_len = 2;
14021     __ vpmullq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14022   %}
14023   ins_pipe( pipe_slow );
14024 %}
14025 
14026 instruct vmul8L_mem(vecZ dst, vecZ src, memory mem) %{
14027   predicate(UseAVX > 2 && n->as_Vector()->length() == 8 && VM_Version::supports_avx512dq());
14028   match(Set dst (MulVL src (LoadVector mem)));
14029   format %{ "vpmullq $dst,$src,$mem\t! mul packed8L" %}
14030   ins_encode %{
14031     int vector_len = 2;
14032     __ vpmullq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
14033   %}
14034   ins_pipe( pipe_slow );
14035 %}
14036 
14037 instruct vmul8I_reg(vecY dst, vecY src1, vecY src2) %{
14038   predicate(UseAVX > 1 && n->as_Vector()->length() == 8);
14039   match(Set dst (MulVI src1 src2));
14040   format %{ "vpmulld $dst,$src1,$src2\t! mul packed8I" %}
14041   ins_encode %{
14042     int vector_len = 1;
14043     __ vpmulld($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14044   %}
14045   ins_pipe( pipe_slow );
14046 %}
14047 
14048 instruct vmul8I_mem(vecY dst, vecY src, memory mem) %{
14049   predicate(UseAVX > 1 && n->as_Vector()->length() == 8);
14050   match(Set dst (MulVI src (LoadVector mem)));
14051   format %{ "vpmulld $dst,$src,$mem\t! mul packed8I" %}
14052   ins_encode %{
14053     int vector_len = 1;
14054     __ vpmulld($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
14055   %}
14056   ins_pipe( pipe_slow );
14057 %}
14058 
14059 instruct vmul16I_reg(vecZ dst, vecZ src1, vecZ src2) %{
14060   predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
14061   match(Set dst (MulVI src1 src2));
14062   format %{ "vpmulld $dst,$src1,$src2\t! mul packed16I" %}
14063   ins_encode %{
14064     int vector_len = 2;
14065     __ vpmulld($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14066   %}
14067   ins_pipe( pipe_slow );
14068 %}
14069 
14070 instruct vmul16I_mem(vecZ dst, vecZ src, memory mem) %{
14071   predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
14072   match(Set dst (MulVI src (LoadVector mem)));
14073   format %{ "vpmulld $dst,$src,$mem\t! mul packed16I" %}
14074   ins_encode %{
14075     int vector_len = 2;
14076     __ vpmulld($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
14077   %}
14078   ins_pipe( pipe_slow );
14079 %}
14080 
14081 // Floats vector mul
14082 instruct vmul2F(vecD dst, vecD src) %{
14083   predicate(n->as_Vector()->length() == 2);
14084   match(Set dst (MulVF dst src));
14085   format %{ "mulps   $dst,$src\t! mul packed2F" %}
14086   ins_encode %{
14087     __ mulps($dst$$XMMRegister, $src$$XMMRegister);
14088   %}
14089   ins_pipe( pipe_slow );
14090 %}
14091 
14092 instruct vmul2F_reg(vecD dst, vecD src1, vecD src2) %{
14093   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
14094   match(Set dst (MulVF src1 src2));
14095   format %{ "vmulps  $dst,$src1,$src2\t! mul packed2F" %}
14096   ins_encode %{
14097     int vector_len = 0;
14098     __ vmulps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14099   %}
14100   ins_pipe( pipe_slow );
14101 %}
14102 
14103 instruct vmul2F_mem(vecD dst, vecD src, memory mem) %{
14104   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
14105   match(Set dst (MulVF src (LoadVector mem)));
14106   format %{ "vmulps  $dst,$src,$mem\t! mul packed2F" %}
14107   ins_encode %{
14108     int vector_len = 0;
14109     __ vmulps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
14110   %}
14111   ins_pipe( pipe_slow );
14112 %}
14113 
14114 instruct vmul4F(vecX dst, vecX src) %{
14115   predicate(n->as_Vector()->length() == 4);
14116   match(Set dst (MulVF dst src));
14117   format %{ "mulps   $dst,$src\t! mul packed4F" %}
14118   ins_encode %{
14119     __ mulps($dst$$XMMRegister, $src$$XMMRegister);
14120   %}
14121   ins_pipe( pipe_slow );
14122 %}
14123 
14124 instruct vmul4F_reg(vecX dst, vecX src1, vecX src2) %{
14125   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
14126   match(Set dst (MulVF src1 src2));
14127   format %{ "vmulps  $dst,$src1,$src2\t! mul packed4F" %}
14128   ins_encode %{
14129     int vector_len = 0;
14130     __ vmulps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14131   %}
14132   ins_pipe( pipe_slow );
14133 %}
14134 
14135 instruct vmul4F_mem(vecX dst, vecX src, memory mem) %{
14136   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
14137   match(Set dst (MulVF src (LoadVector mem)));
14138   format %{ "vmulps  $dst,$src,$mem\t! mul packed4F" %}
14139   ins_encode %{
14140     int vector_len = 0;
14141     __ vmulps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
14142   %}
14143   ins_pipe( pipe_slow );
14144 %}
14145 
14146 instruct vmul8F_reg(vecY dst, vecY src1, vecY src2) %{
14147   predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
14148   match(Set dst (MulVF src1 src2));
14149   format %{ "vmulps  $dst,$src1,$src2\t! mul packed8F" %}
14150   ins_encode %{
14151     int vector_len = 1;
14152     __ vmulps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14153   %}
14154   ins_pipe( pipe_slow );
14155 %}
14156 
14157 instruct vmul8F_mem(vecY dst, vecY src, memory mem) %{
14158   predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
14159   match(Set dst (MulVF src (LoadVector mem)));
14160   format %{ "vmulps  $dst,$src,$mem\t! mul packed8F" %}
14161   ins_encode %{
14162     int vector_len = 1;
14163     __ vmulps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
14164   %}
14165   ins_pipe( pipe_slow );
14166 %}
14167 
14168 instruct vmul16F_reg(vecZ dst, vecZ src1, vecZ src2) %{
14169   predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
14170   match(Set dst (MulVF src1 src2));
14171   format %{ "vmulps  $dst,$src1,$src2\t! mul packed16F" %}
14172   ins_encode %{
14173     int vector_len = 2;
14174     __ vmulps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14175   %}
14176   ins_pipe( pipe_slow );
14177 %}
14178 
14179 instruct vmul16F_mem(vecZ dst, vecZ src, memory mem) %{
14180   predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
14181   match(Set dst (MulVF src (LoadVector mem)));
14182   format %{ "vmulps  $dst,$src,$mem\t! mul packed16F" %}
14183   ins_encode %{
14184     int vector_len = 2;
14185     __ vmulps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
14186   %}
14187   ins_pipe( pipe_slow );
14188 %}
14189 
14190 // Doubles vector mul
14191 instruct vmul2D(vecX dst, vecX src) %{
14192   predicate(n->as_Vector()->length() == 2);
14193   match(Set dst (MulVD dst src));
14194   format %{ "mulpd   $dst,$src\t! mul packed2D" %}
14195   ins_encode %{
14196     __ mulpd($dst$$XMMRegister, $src$$XMMRegister);
14197   %}
14198   ins_pipe( pipe_slow );
14199 %}
14200 
14201 instruct vmul2D_reg(vecX dst, vecX src1, vecX src2) %{
14202   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
14203   match(Set dst (MulVD src1 src2));
14204   format %{ "vmulpd  $dst,$src1,$src2\t! mul packed2D" %}
14205   ins_encode %{
14206     int vector_len = 0;
14207     __ vmulpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14208   %}
14209   ins_pipe( pipe_slow );
14210 %}
14211 
14212 instruct vmul2D_mem(vecX dst, vecX src, memory mem) %{
14213   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
14214   match(Set dst (MulVD src (LoadVector mem)));
14215   format %{ "vmulpd  $dst,$src,$mem\t! mul packed2D" %}
14216   ins_encode %{
14217     int vector_len = 0;
14218     __ vmulpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
14219   %}
14220   ins_pipe( pipe_slow );
14221 %}
14222 
14223 instruct vmul4D_reg(vecY dst, vecY src1, vecY src2) %{
14224   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
14225   match(Set dst (MulVD src1 src2));
14226   format %{ "vmulpd  $dst,$src1,$src2\t! mul packed4D" %}
14227   ins_encode %{
14228     int vector_len = 1;
14229     __ vmulpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14230   %}
14231   ins_pipe( pipe_slow );
14232 %}
14233 
14234 instruct vmul4D_mem(vecY dst, vecY src, memory mem) %{
14235   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
14236   match(Set dst (MulVD src (LoadVector mem)));
14237   format %{ "vmulpd  $dst,$src,$mem\t! mul packed4D" %}
14238   ins_encode %{
14239     int vector_len = 1;
14240     __ vmulpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
14241   %}
14242   ins_pipe( pipe_slow );
14243 %}
14244 
14245 instruct vmul8D_reg(vecZ dst, vecZ src1, vecZ src2) %{
14246   predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
14247   match(Set dst (MulVD src1 src2));
14248   format %{ "vmulpd  $dst k0,$src1,$src2\t! mul packed8D" %}
14249   ins_encode %{
14250     int vector_len = 2;
14251     __ vmulpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14252   %}
14253   ins_pipe( pipe_slow );
14254 %}
14255 
14256 instruct vmul8D_mem(vecZ dst, vecZ src, memory mem) %{
14257   predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
14258   match(Set dst (MulVD src (LoadVector mem)));
14259   format %{ "vmulpd  $dst k0,$src,$mem\t! mul packed8D" %}
14260   ins_encode %{
14261     int vector_len = 2;
14262     __ vmulpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
14263   %}
14264   ins_pipe( pipe_slow );
14265 %}
14266 
14267 instruct vcmov8F_reg(vecY dst, vecY src1, vecY src2, immI8 cop, cmpOp_vcmppd copnd) %{
14268   predicate(UseAVX > 0 && UseAVX < 3 && n->as_Vector()->length() == 8);
14269   match(Set dst (CMoveVF (Binary copnd cop) (Binary src1 src2)));
14270   effect(TEMP dst, USE src1, USE src2);
14271   format %{ "cmpps.$copnd  $dst, $src1, $src2  ! vcmovevf, cond=$cop\n\t"
14272             "blendvps $dst,$src1,$src2,$dst ! vcmovevf\n\t"
14273          %}
14274   ins_encode %{
14275     int vector_len = 1;
14276     int cond = (Assembler::Condition)($copnd$$cmpcode);
14277     __ vcmpps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cond, vector_len);
14278     __ vblendvps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $dst$$XMMRegister, vector_len);
14279   %}
14280   ins_pipe( pipe_slow );
14281 %}
14282 
14283 instruct vcmov4D_reg(vecY dst, vecY src1, vecY src2, immI8 cop, cmpOp_vcmppd copnd) %{
14284   predicate(UseAVX > 0 && UseAVX < 3 && n->as_Vector()->length() == 4);
14285   match(Set dst (CMoveVD (Binary copnd cop) (Binary src1 src2)));
14286   effect(TEMP dst, USE src1, USE src2);
14287   format %{ "cmppd.$copnd  $dst, $src1, $src2  ! vcmovevd, cond=$cop\n\t"
14288             "vblendvpd $dst,$src1,$src2,$dst ! vcmovevd\n\t"
14289          %}
14290   ins_encode %{
14291     int vector_len = 1;
14292     int cond = (Assembler::Condition)($copnd$$cmpcode);
14293     __ vcmppd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cond, vector_len);
14294     __ vblendvpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $dst$$XMMRegister, vector_len);
14295   %}
14296   ins_pipe( pipe_slow );
14297 %}
14298 
14299 // --------------------------------- DIV --------------------------------------
14300 
14301 // Floats vector div
14302 instruct vdiv2F(vecD dst, vecD src) %{
14303   predicate(n->as_Vector()->length() == 2);
14304   match(Set dst (DivVF dst src));
14305   format %{ "divps   $dst,$src\t! div packed2F" %}
14306   ins_encode %{
14307     __ divps($dst$$XMMRegister, $src$$XMMRegister);
14308   %}
14309   ins_pipe( pipe_slow );
14310 %}
14311 
14312 instruct vdiv2F_reg(vecD dst, vecD src1, vecD src2) %{
14313   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
14314   match(Set dst (DivVF src1 src2));
14315   format %{ "vdivps  $dst,$src1,$src2\t! div packed2F" %}
14316   ins_encode %{
14317     int vector_len = 0;
14318     __ vdivps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14319   %}
14320   ins_pipe( pipe_slow );
14321 %}
14322 
14323 instruct vdiv2F_mem(vecD dst, vecD src, memory mem) %{
14324   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
14325   match(Set dst (DivVF src (LoadVector mem)));
14326   format %{ "vdivps  $dst,$src,$mem\t! div packed2F" %}
14327   ins_encode %{
14328     int vector_len = 0;
14329     __ vdivps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
14330   %}
14331   ins_pipe( pipe_slow );
14332 %}
14333 
14334 instruct vdiv4F(vecX dst, vecX src) %{
14335   predicate(n->as_Vector()->length() == 4);
14336   match(Set dst (DivVF dst src));
14337   format %{ "divps   $dst,$src\t! div packed4F" %}
14338   ins_encode %{
14339     __ divps($dst$$XMMRegister, $src$$XMMRegister);
14340   %}
14341   ins_pipe( pipe_slow );
14342 %}
14343 
14344 instruct vdiv4F_reg(vecX dst, vecX src1, vecX src2) %{
14345   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
14346   match(Set dst (DivVF src1 src2));
14347   format %{ "vdivps  $dst,$src1,$src2\t! div packed4F" %}
14348   ins_encode %{
14349     int vector_len = 0;
14350     __ vdivps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14351   %}
14352   ins_pipe( pipe_slow );
14353 %}
14354 
14355 instruct vdiv4F_mem(vecX dst, vecX src, memory mem) %{
14356   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
14357   match(Set dst (DivVF src (LoadVector mem)));
14358   format %{ "vdivps  $dst,$src,$mem\t! div packed4F" %}
14359   ins_encode %{
14360     int vector_len = 0;
14361     __ vdivps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
14362   %}
14363   ins_pipe( pipe_slow );
14364 %}
14365 
14366 instruct vdiv8F_reg(vecY dst, vecY src1, vecY src2) %{
14367   predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
14368   match(Set dst (DivVF src1 src2));
14369   format %{ "vdivps  $dst,$src1,$src2\t! div packed8F" %}
14370   ins_encode %{
14371     int vector_len = 1;
14372     __ vdivps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14373   %}
14374   ins_pipe( pipe_slow );
14375 %}
14376 
14377 instruct vdiv8F_mem(vecY dst, vecY src, memory mem) %{
14378   predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
14379   match(Set dst (DivVF src (LoadVector mem)));
14380   format %{ "vdivps  $dst,$src,$mem\t! div packed8F" %}
14381   ins_encode %{
14382     int vector_len = 1;
14383     __ vdivps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
14384   %}
14385   ins_pipe( pipe_slow );
14386 %}
14387 
14388 instruct vdiv16F_reg(vecZ dst, vecZ src1, vecZ src2) %{
14389   predicate(UseAVX > 0 && n->as_Vector()->length() == 16);
14390   match(Set dst (DivVF src1 src2));
14391   format %{ "vdivps  $dst,$src1,$src2\t! div packed16F" %}
14392   ins_encode %{
14393     int vector_len = 2;
14394     __ vdivps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14395   %}
14396   ins_pipe( pipe_slow );
14397 %}
14398 
14399 instruct vdiv16F_mem(vecZ dst, vecZ src, memory mem) %{
14400   predicate(UseAVX > 0 && n->as_Vector()->length() == 16);
14401   match(Set dst (DivVF src (LoadVector mem)));
14402   format %{ "vdivps  $dst,$src,$mem\t! div packed16F" %}
14403   ins_encode %{
14404     int vector_len = 2;
14405     __ vdivps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
14406   %}
14407   ins_pipe( pipe_slow );
14408 %}
14409 
14410 // Doubles vector div
14411 instruct vdiv2D(vecX dst, vecX src) %{
14412   predicate(n->as_Vector()->length() == 2);
14413   match(Set dst (DivVD dst src));
14414   format %{ "divpd   $dst,$src\t! div packed2D" %}
14415   ins_encode %{
14416     __ divpd($dst$$XMMRegister, $src$$XMMRegister);
14417   %}
14418   ins_pipe( pipe_slow );
14419 %}
14420 
14421 instruct vdiv2D_reg(vecX dst, vecX src1, vecX src2) %{
14422   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
14423   match(Set dst (DivVD src1 src2));
14424   format %{ "vdivpd  $dst,$src1,$src2\t! div packed2D" %}
14425   ins_encode %{
14426     int vector_len = 0;
14427     __ vdivpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14428   %}
14429   ins_pipe( pipe_slow );
14430 %}
14431 
14432 instruct vdiv2D_mem(vecX dst, vecX src, memory mem) %{
14433   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
14434   match(Set dst (DivVD src (LoadVector mem)));
14435   format %{ "vdivpd  $dst,$src,$mem\t! div packed2D" %}
14436   ins_encode %{
14437     int vector_len = 0;
14438     __ vdivpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
14439   %}
14440   ins_pipe( pipe_slow );
14441 %}
14442 
14443 instruct vdiv4D_reg(vecY dst, vecY src1, vecY src2) %{
14444   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
14445   match(Set dst (DivVD src1 src2));
14446   format %{ "vdivpd  $dst,$src1,$src2\t! div packed4D" %}
14447   ins_encode %{
14448     int vector_len = 1;
14449     __ vdivpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14450   %}
14451   ins_pipe( pipe_slow );
14452 %}
14453 
14454 instruct vdiv4D_mem(vecY dst, vecY src, memory mem) %{
14455   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
14456   match(Set dst (DivVD src (LoadVector mem)));
14457   format %{ "vdivpd  $dst,$src,$mem\t! div packed4D" %}
14458   ins_encode %{
14459     int vector_len = 1;
14460     __ vdivpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
14461   %}
14462   ins_pipe( pipe_slow );
14463 %}
14464 
14465 instruct vdiv8D_reg(vecZ dst, vecZ src1, vecZ src2) %{
14466   predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
14467   match(Set dst (DivVD src1 src2));
14468   format %{ "vdivpd  $dst,$src1,$src2\t! div packed8D" %}
14469   ins_encode %{
14470     int vector_len = 2;
14471     __ vdivpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14472   %}
14473   ins_pipe( pipe_slow );
14474 %}
14475 
14476 instruct vdiv8D_mem(vecZ dst, vecZ src, memory mem) %{
14477   predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
14478   match(Set dst (DivVD src (LoadVector mem)));
14479   format %{ "vdivpd  $dst,$src,$mem\t! div packed8D" %}
14480   ins_encode %{
14481     int vector_len = 2;
14482     __ vdivpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
14483   %}
14484   ins_pipe( pipe_slow );
14485 %}
14486 
14487 // ------------------------------ Min ---------------------------------------
14488 // Byte vector Min
14489 instruct min8B_reg(vecD dst, vecD src1, vecD src2) %{
14490   predicate(UseSSE > 3 && UseAVX == 0 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
14491   match(Set dst (MinV src1 src2));
14492   effect(TEMP dst);
14493   format %{ "movdqu  $dst,$src1\n\t"
14494             "pminsb  $dst,$src2\t!  " %}
14495   ins_encode %{
14496     __ movdqu($dst$$XMMRegister, $src1$$XMMRegister);
14497     __ pminsb($dst$$XMMRegister, $src2$$XMMRegister);
14498   %}
14499   ins_pipe( pipe_slow );
14500 %}
14501 
14502 instruct min8B_reg_avx(vecD dst, vecD src1, vecD src2) %{
14503   predicate(UseAVX > 0 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
14504   match(Set dst (MinV src1 src2));
14505   format %{ "vpminsb  $dst,$src1,$src2\t!  " %}
14506   ins_encode %{
14507     __ vpminsb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, 0);
14508   %}
14509   ins_pipe( pipe_slow );
14510 %}
14511 
14512 instruct min16B_reg(vecX dst, vecX src1, vecX src2) %{
14513   predicate(UseSSE > 3 && UseAVX == 0 && n->as_Vector()->length() == 16 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
14514   match(Set dst (MinV src1 src2));
14515   effect(TEMP dst);
14516   format %{ "movdqu  $dst,$src1\n\t"
14517             "pminsb  $dst,$src2\t!  " %}
14518   ins_encode %{
14519     __ movdqu($dst$$XMMRegister, $src1$$XMMRegister);
14520     __ pminsb($dst$$XMMRegister, $src2$$XMMRegister);
14521   %}
14522   ins_pipe( pipe_slow );
14523 %}
14524 
14525 instruct min16B_reg_avx(vecX dst, vecX src1, vecX src2) %{
14526   predicate(UseAVX > 0 && n->as_Vector()->length() == 16 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
14527   match(Set dst (MinV src1 src2));
14528   format %{ "vpminsb    $dst,$src1,$src2\t! " %}
14529   ins_encode %{
14530     int vector_len = 0;
14531     __ vpminsb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14532   %}
14533   ins_pipe( pipe_slow );
14534 %}
14535 
14536 instruct min32B_reg(vecY dst, vecY src1, vecY src2) %{
14537   predicate(UseAVX > 1 && n->as_Vector()->length() == 32 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
14538   match(Set dst (MinV src1 src2));
14539   format %{ "vpminsb    $dst,$src1,$src2\t! " %}
14540   ins_encode %{
14541     int vector_len = 1;
14542     __ vpminsb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14543   %}
14544   ins_pipe( pipe_slow );
14545 %}
14546 
14547 instruct min64B_reg_evex(vecZ dst, vecZ src1, vecZ src2) %{
14548   predicate(UseAVX > 2 && VM_Version::supports_avx512bw() && n->as_Vector()->length() == 64 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
14549   match(Set dst (MinV src1 src2));
14550   format %{ "vpminsb  $dst,$src1,$src2\t! " %}
14551   ins_encode %{
14552     int vector_len = 2;
14553     __ vpminsb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14554   %}
14555   ins_pipe( pipe_slow );
14556 %}
14557 
14558 //Short vector Min
14559 instruct min4S_reg(vecD dst, vecD src1, vecD src2) %{
14560   predicate(UseSSE >= 2 && UseAVX == 0 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
14561   match(Set dst (MinV src1 src2));
14562   effect(TEMP dst);
14563   format %{ "movsd   $dst,$src1\n\t"
14564             "pminsw  $dst,$src2\t! " %}
14565   ins_encode %{
14566     __ movsd($dst$$XMMRegister, $src1$$XMMRegister);
14567     __ pminsw($dst$$XMMRegister, $src2$$XMMRegister);
14568   %}
14569   ins_pipe( pipe_slow );
14570 %}
14571 
14572 instruct min4S_reg_avx(vecD dst, vecD src1, vecD src2) %{
14573   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
14574   match(Set dst (MinV src1 src2));
14575   effect(TEMP dst);
14576   format %{ "vpminsw  $dst,$src1,$src2\t! " %}
14577   ins_encode %{
14578     __ vpminsw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, 0);
14579   %}
14580   ins_pipe( pipe_slow );
14581 %}
14582 
14583 instruct min8S_reg(vecX dst, vecX src1, vecX src2) %{
14584   predicate(UseSSE >= 2 && UseAVX == 0 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
14585   match(Set dst (MinV src1 src2));
14586   effect(TEMP dst);
14587   format %{ "movdqu   $dst,$src1\n\t"
14588             "pminsw  $dst,$src2\t! " %}
14589   ins_encode %{
14590     __ movdqu($dst$$XMMRegister, $src1$$XMMRegister);
14591     __ pminsw($dst$$XMMRegister, $src2$$XMMRegister);
14592   %}
14593   ins_pipe( pipe_slow );
14594 %}
14595 
14596 instruct min8S_reg_avx(vecX dst, vecX src1, vecX src2) %{
14597   predicate(UseAVX > 0 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
14598   match(Set dst (MinV src1 src2));
14599   format %{ "vpminsw    $dst,$src1,$src2\t! " %}
14600   ins_encode %{
14601     int vector_len = 0;
14602     __ vpminsw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14603   %}
14604   ins_pipe( pipe_slow );
14605 %}
14606 
14607 instruct min16S_reg(vecY dst, vecY src1, vecY src2) %{
14608   predicate(UseAVX > 1 && n->as_Vector()->length() == 16 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
14609   match(Set dst (MinV src1 src2));
14610   format %{ "vpminsw    $dst,$src1,$src2\t! " %}
14611   ins_encode %{
14612     int vector_len = 1;
14613     __ vpminsw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14614   %}
14615   ins_pipe( pipe_slow );
14616 %}
14617 
14618 instruct min32S_reg_evex(vecZ dst, vecZ src1, vecZ src2) %{
14619   predicate(UseAVX > 2 && VM_Version::supports_avx512bw() && n->as_Vector()->length() == 32 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
14620   match(Set dst (MinV src1 src2));
14621   format %{ "vpminsw  $dst,$src1,$src2\t! " %}
14622   ins_encode %{
14623     int vector_len = 2;
14624     __ vpminsw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14625   %}
14626   ins_pipe( pipe_slow );
14627 %}
14628 
14629 // Int vector Min
14630 instruct min2I_reg(vecD dst, vecD src1, vecD src2) %{
14631   predicate(UseSSE > 3 && UseAVX == 0 && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
14632   match(Set dst (MinV src1 src2));
14633   effect(TEMP dst);
14634   format %{ "movsd   $dst,$src1\n\t"
14635             "pminsd  $dst,$src2\t! " %}
14636   ins_encode %{
14637     __ movsd($dst$$XMMRegister, $src1$$XMMRegister);
14638     __ pminsd($dst$$XMMRegister, $src2$$XMMRegister);
14639   %}
14640   ins_pipe( pipe_slow );
14641 %}
14642 
14643 instruct min2I_reg_avx(vecD dst, vecD src1, vecD src2) %{
14644   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
14645   match(Set dst (MinV src1 src2));
14646   format %{ "vpminsd    $dst,$src1,$src2\t! " %}
14647   ins_encode %{
14648     int vector_len = 0;
14649     __ vpminsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14650   %}
14651   ins_pipe( pipe_slow );
14652 %}
14653 
14654 instruct min4I_reg(vecX dst, vecX src1, vecX src2) %{
14655   predicate(UseSSE > 3 && UseAVX == 0 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
14656   match(Set dst (MinV src1 src2));
14657   effect(TEMP dst);
14658   format %{ "movdqu   $dst,$src1\n\t"
14659             "pminsd   $dst,$src2\t! " %}
14660   ins_encode %{
14661     __ movdqu($dst$$XMMRegister, $src1$$XMMRegister);
14662     __ pminsd($dst$$XMMRegister, $src2$$XMMRegister);
14663   %}
14664   ins_pipe( pipe_slow );
14665 %}
14666 
14667 instruct min4I_reg_avx(vecX dst, vecX src1, vecX src2) %{
14668   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
14669   match(Set dst (MinV src1 src2));
14670   format %{ "vpminsd    $dst,$src1,$src2\t! " %}
14671   ins_encode %{
14672     int vector_len = 0;
14673     __ vpminsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14674   %}
14675   ins_pipe( pipe_slow );
14676 %}
14677 
14678 instruct min4I_reg_evex(vecX dst, vecX src1, vecX src2) %{
14679   predicate(UseAVX > 2 && VM_Version::supports_avx512vl() && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
14680   match(Set dst (MinV src1 src2));
14681   format %{ "vpminsd  $dst,$src1,$src2\t! " %}
14682   ins_encode %{
14683     int vector_len = 0;
14684     __ vpminsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14685   %}
14686   ins_pipe( pipe_slow );
14687 %}
14688 
14689 instruct min8I_reg_avx(vecY dst, vecY src1, vecY src2) %{
14690   predicate(UseAVX > 1 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
14691   match(Set dst (MinV src1 src2));
14692   format %{ "vpminsd    $dst,$src1,$src2\t! " %}
14693   ins_encode %{
14694     int vector_len = 1;
14695     __ vpminsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14696   %}
14697   ins_pipe( pipe_slow );
14698 %}
14699 
14700 instruct min8I_reg_evex(vecY dst, vecY src1, vecY src2) %{
14701   predicate(UseAVX > 2 && VM_Version::supports_avx512vl() && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
14702   match(Set dst (MinV src1 src2));
14703   format %{ "vpminsd  $dst,$src1,$src2\t! " %}
14704   ins_encode %{
14705     int vector_len = 1;
14706     __ vpminsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14707   %}
14708   ins_pipe( pipe_slow );
14709 %}
14710 
14711 instruct min16I_reg_evex(vecZ dst, vecZ src1, vecZ src2) %{
14712   predicate(UseAVX > 2 && VM_Version::supports_avx512vl() && n->as_Vector()->length() == 16 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
14713   match(Set dst (MinV src1 src2));
14714   format %{ "vpminsd  $dst,$src1,$src2\t! " %}
14715   ins_encode %{
14716     int vector_len = 2;
14717     __ vpminsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14718   %}
14719   ins_pipe( pipe_slow );
14720 %}
14721 
14722 // Long vector Min
14723 instruct minL_reg(vecD dst, vecD src1, vecD src2, rxmm0 tmp) %{
14724   predicate(UseAVX == 0 && UseSSE >= 4 && n->as_Vector()->length() == 1 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
14725   match(Set dst (MinV src1 src2));
14726   effect(TEMP dst, TEMP tmp);
14727   format %{ "movsd     $tmp,$src1\n\t"
14728             "movsd     $dst,$src1\n\t"
14729             "pcmpgtq   $tmp,$src2\n\t"
14730             "blendvpd  $dst,$src2\t! " %}
14731   ins_encode %{
14732     __ movsd($tmp$$XMMRegister, $src1$$XMMRegister);
14733     __ movsd($dst$$XMMRegister, $src1$$XMMRegister);
14734     __ pcmpgtq($tmp$$XMMRegister, $src2$$XMMRegister);
14735     __ blendvpd($dst$$XMMRegister, $src2$$XMMRegister);
14736   %}
14737   ins_pipe( pipe_slow );
14738 %}
14739 
14740 instruct min1L_reg_avx(vecD dst, vecD src1, vecD src2) %{
14741   predicate(UseAVX > 0 && n->as_Vector()->length() == 1 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
14742   match(Set dst (MinV src1 src2));
14743   effect(TEMP dst);
14744   format %{ "vpcmpgtq   $dst,$src1,$src2\n\t"
14745             "vblendvpd  $dst,$src1,$src2,$dst\t! " %}
14746   ins_encode %{
14747     int vector_len = 0;
14748     __ vpcmpgtq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14749     __ vblendvpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $dst$$XMMRegister, vector_len);
14750   %}
14751   ins_pipe( pipe_slow );
14752 %}
14753 
14754 instruct min2L_reg(vecX dst, vecX src1, vecX src2, rxmm0 tmp) %{
14755   predicate(UseAVX == 0 && UseSSE >= 4 && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
14756   match(Set dst (MinV src1 src2));
14757   effect(TEMP dst, TEMP tmp);
14758   format %{ "movdqu    $tmp,$src1\n\t"
14759             "movdqu    $dst,$src1\n\t"
14760             "pcmpgtq   $tmp,$src2\n\t"
14761             "blendvpd  $dst,$src2\t! " %}
14762   ins_encode %{
14763     __ movdqu($tmp$$XMMRegister, $src1$$XMMRegister);
14764     __ movdqu($dst$$XMMRegister, $src1$$XMMRegister);
14765     __ pcmpgtq($tmp$$XMMRegister, $src2$$XMMRegister);
14766     __ blendvpd($dst$$XMMRegister, $src2$$XMMRegister);
14767   %}
14768   ins_pipe( pipe_slow );
14769 %}
14770 
14771 instruct min2L_reg_avx(vecX dst, vecX src1, vecX src2) %{
14772   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
14773   match(Set dst (MinV src1 src2));
14774   effect(TEMP dst);
14775   format %{ "vpcmpgtq   $dst,$src1,$src2\n\t"
14776             "vblendvpd  $dst,$src1,$src2,$dst\t! " %}
14777   ins_encode %{
14778     int vector_len = 0;
14779     __ vpcmpgtq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14780     __ vblendvpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $dst$$XMMRegister, vector_len);
14781   %}
14782   ins_pipe( pipe_slow );
14783 %}
14784 
14785 instruct min4L_reg_avx(vecY dst, vecY src1, vecY src2) %{
14786   predicate(UseAVX > 1 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
14787   match(Set dst (MinV src1 src2));
14788   effect(TEMP dst);
14789   format %{ "vpcmpgtq   $dst,$src1,$src2\n\t"
14790             "vblendvpd  $dst,$src1,$src2,$dst\t! " %}
14791   ins_encode %{
14792     int vector_len = 1;
14793     __ vpcmpgtq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14794     __ vblendvpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $dst$$XMMRegister, vector_len);
14795   %}
14796   ins_pipe( pipe_slow );
14797 %}
14798 
14799 instruct min2L_reg_evex(vecX dst, vecX src1, vecX src2) %{
14800   predicate(UseAVX > 2 && VM_Version::supports_avx512vl() && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
14801   match(Set dst (MinV src1 src2));
14802   format %{ "vpminsq  $dst,$src1,src2\t! " %}
14803   ins_encode %{
14804     __ vpminsq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, 0);
14805   %}
14806   ins_pipe( pipe_slow );
14807 %}
14808 
14809 instruct min4L_reg_evex(vecY dst, vecY src1, vecY src2) %{
14810   predicate(UseAVX > 2 && VM_Version::supports_avx512vl() && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
14811   match(Set dst (MinV src1 src2));
14812   format %{ "vpminsq  $dst,$src1,src2\t! " %}
14813   ins_encode %{
14814     __ vpminsq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, 1);
14815   %}
14816   ins_pipe( pipe_slow );
14817 %}
14818 
14819 instruct min8L_reg_evex(vecZ dst, vecZ src1, vecZ src2) %{
14820   predicate(UseAVX > 2 && VM_Version::supports_avx512vl() && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
14821   match(Set dst (MinV src1 src2));
14822   format %{ "vpminsq  $dst,$src1,src2\t! " %}
14823   ins_encode %{
14824     __ vpminsq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, 2);
14825   %}
14826   ins_pipe( pipe_slow );
14827 %}
14828 
14829 // Float vector Min
14830 instruct min2F_reg(vecD dst, vecD src1, vecD src2) %{
14831   predicate(UseSSE > 0 && UseAVX == 0 && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
14832   match(Set dst (MinV src1 src2));
14833   effect(TEMP dst);
14834   format %{ "movsd  $dst,$src1\n\t"
14835             "minps  $dst,$src2\t! " %}
14836   ins_encode %{
14837     __ movsd($dst$$XMMRegister, $src1$$XMMRegister);
14838     __ minps($dst$$XMMRegister, $src2$$XMMRegister);
14839   %}
14840   ins_pipe( pipe_slow );
14841 %}
14842 
14843 instruct min2F_reg_avx(vecD dst, vecD src1, vecD src2) %{
14844   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
14845   match(Set dst (MinV src1 src2));
14846   format %{ "vminps  $dst,$src1,$src2\t! " %}
14847   ins_encode %{
14848     int vector_len = 0;
14849     __ vminps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14850   %}
14851   ins_pipe( pipe_slow );
14852 %}
14853 
14854 instruct min4F_reg(vecX dst, vecX src1, vecX src2) %{
14855   predicate(UseSSE > 0 && UseAVX == 0 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
14856   match(Set dst (MinV src1 src2));
14857   effect(TEMP dst);
14858   format %{ "movdqu  $dst,$src1\n\t"
14859             "minps   $dst,$src2\t! " %}
14860   ins_encode %{
14861     __ movdqu($dst$$XMMRegister, $src1$$XMMRegister);
14862     __ minps($dst$$XMMRegister, $src2$$XMMRegister);
14863   %}
14864   ins_pipe( pipe_slow );
14865 %}
14866 
14867 instruct min4F_reg_avx(vecX dst, vecX src1, vecX src2) %{
14868   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
14869   match(Set dst (MinV src1 src2));
14870   format %{ "vminps  $dst,$src1,$src2\t! " %}
14871   ins_encode %{
14872     int vector_len = 0;
14873     __ vminps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14874   %}
14875   ins_pipe( pipe_slow );
14876 %}
14877 
14878 instruct min4F_reg_evex(vecX dst, vecX src1, vecX src2) %{
14879   predicate(UseAVX > 2 && VM_Version::supports_avx512vl() && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
14880   match(Set dst (MinV src1 src2));
14881   format %{ "vminps  $dst,$src1,$src2\t! " %}
14882   ins_encode %{
14883     int vector_len = 0;
14884     __ vminps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14885   %}
14886   ins_pipe( pipe_slow );
14887 %}
14888 
14889 instruct min8F_reg_avx(vecY dst, vecY src1, vecY src2) %{
14890   predicate(UseAVX > 0 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
14891   match(Set dst (MinV src1 src2));
14892   format %{ "vminps  $dst,$src1,$src2\t! " %}
14893   ins_encode %{
14894     int vector_len = 1;
14895     __ vminps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14896   %}
14897   ins_pipe( pipe_slow );
14898 %}
14899 
14900 instruct min8F_reg_evex(vecY dst, vecY src1, vecY src2) %{
14901   predicate(UseAVX > 2 && VM_Version::supports_avx512vl() && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
14902   match(Set dst (MinV src1 src2));
14903   format %{ "vminps  $dst,$src1,$src2\t! " %}
14904   ins_encode %{
14905     int vector_len = 1;
14906     __ vminps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14907   %}
14908   ins_pipe( pipe_slow );
14909 %}
14910 
14911 instruct min16F_reg_evex(vecZ dst, vecZ src1, vecZ src2) %{
14912   predicate(UseAVX > 2 && VM_Version::supports_avx512vl() && n->as_Vector()->length() == 16 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
14913   match(Set dst (MinV src1 src2));
14914   format %{ "vminps  $dst,$src1,$src2\t! " %}
14915   ins_encode %{
14916     int vector_len = 2;
14917     __ vminps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14918   %}
14919   ins_pipe( pipe_slow );
14920 %}
14921 
14922 // Double vector Min
14923 instruct minD_reg(vecD dst, vecD src1, vecD src2) %{
14924   predicate(UseSSE > 0 && UseAVX == 0 && n->as_Vector()->length() == 1 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
14925   match(Set dst (MinV src1 src2));
14926   effect(TEMP dst);
14927   format %{ "movsd  $dst,$src1\n\t"
14928             "minpd  $dst,$src2\t! " %}
14929   ins_encode %{
14930     int vector_len = 0;
14931     __ movsd($dst$$XMMRegister, $src1$$XMMRegister);
14932     __ minpd($dst$$XMMRegister, $src2$$XMMRegister);
14933   %}
14934   ins_pipe( pipe_slow );
14935 %}
14936 
14937 instruct min1D_reg_avx(vecD dst, vecD src1, vecD src2) %{
14938   predicate(UseAVX > 0 && n->as_Vector()->length() == 1 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
14939   match(Set dst (MinV src1 src2));
14940   format %{ "vminpd  $dst,$src1,$src2\t! " %}
14941   ins_encode %{
14942     int vector_len = 0;
14943     __ vminpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14944   %}
14945   ins_pipe( pipe_slow );
14946 %}
14947 
14948 instruct min2D_reg(vecX dst, vecX src1, vecX src2) %{
14949   predicate(UseSSE > 0 && UseAVX == 0 && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
14950   match(Set dst (MinV src1 src2));
14951   effect(TEMP dst);
14952   format %{ "movdqu  $dst,$src1\n\t"
14953             "minpd   $dst,$src2\t! " %}
14954   ins_encode %{
14955     int vector_len = 0;
14956     __ movdqu($dst$$XMMRegister, $src1$$XMMRegister);
14957     __ minpd($dst$$XMMRegister, $src2$$XMMRegister);
14958   %}
14959   ins_pipe( pipe_slow );
14960 %}
14961 
14962 instruct min2D_reg_avx(vecX dst, vecX src1, vecX src2) %{
14963   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
14964   match(Set dst (MinV src1 src2));
14965   format %{ "vminpd  $dst,$src1,$src2\t! " %}
14966   ins_encode %{
14967     int vector_len = 0;
14968     __ vminpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14969   %}
14970   ins_pipe( pipe_slow );
14971 %}
14972 
14973 instruct min2D_reg_evex(vecX dst, vecX src1, vecX src2) %{
14974   predicate(UseAVX > 2 && VM_Version::supports_avx512vl() && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
14975   match(Set dst (MinV src1 src2));
14976   format %{ "vminpd  $dst,$src1,$src2\t! " %}
14977   ins_encode %{
14978     int vector_len = 0;
14979     __ vminpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14980   %}
14981   ins_pipe( pipe_slow );
14982 %}
14983 
14984 instruct min4D_reg_avx(vecY dst, vecY src1, vecY src2) %{
14985   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
14986   match(Set dst (MinV src1 src2));
14987   format %{ "vminpd  $dst,$src1,$src2\t! " %}
14988   ins_encode %{
14989     int vector_len = 1;
14990     __ vminpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
14991   %}
14992   ins_pipe( pipe_slow );
14993 %}
14994 
14995 instruct min4D_reg_evex(vecY dst, vecY src1, vecY src2) %{
14996   predicate(UseAVX > 2 && VM_Version::supports_avx512vl() && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
14997   match(Set dst (MinV src1 src2));
14998   format %{ "vminpd  $dst,$src1,$src2\t! " %}
14999   ins_encode %{
15000     int vector_len = 1;
15001     __ vminpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
15002   %}
15003   ins_pipe( pipe_slow );
15004 %}
15005 
15006 instruct min8D_reg_evex(vecZ dst, vecZ src1, vecZ src2) %{
15007   predicate(UseAVX > 2 && VM_Version::supports_avx512vl() && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
15008   match(Set dst (MinV src1 src2));
15009   format %{ "vminpd  $dst,$src1,$src2\t! " %}
15010   ins_encode %{
15011     int vector_len = 2;
15012     __ vminpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
15013   %}
15014   ins_pipe( pipe_slow );
15015 %}
15016 
15017 // ------------------------------ Max ---------------------------------------
15018 // Byte vector Max
15019 instruct max8B_reg(vecD dst, vecD src1, vecD src2) %{
15020   predicate(UseSSE > 3 && UseAVX == 0 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
15021   match(Set dst (MaxV src1 src2));
15022   effect(TEMP dst);
15023   format %{ "movsd   $dst,$src1\n\t"
15024             "pmaxsb  $dst,$src2\t! " %}
15025   ins_encode %{
15026     __ movsd($dst$$XMMRegister, $src1$$XMMRegister);
15027     __ pmaxsb($dst$$XMMRegister, $src2$$XMMRegister);
15028   %}
15029   ins_pipe( pipe_slow );
15030 %}
15031 
15032 instruct max8B_reg_avx(vecD dst, vecD src1, vecD src2) %{
15033   predicate(UseAVX > 0 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
15034   match(Set dst (MaxV src1 src2));
15035   format %{ "vpmaxsb    $dst,$src1,$src2\t! " %}
15036   ins_encode %{
15037     int vector_len = 0;
15038     __ vpmaxsb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
15039   %}
15040   ins_pipe( pipe_slow );
15041 %}
15042 
15043 instruct max16B_reg(vecX dst, vecX src1, vecX src2) %{
15044   predicate(UseSSE > 3 && UseAVX == 0 && n->as_Vector()->length() == 16 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
15045   match(Set dst (MaxV src1 src2));
15046   effect(TEMP dst);
15047   format %{ "movdqu  $dst,$src1\n\t"
15048             "pmaxsb  $dst,$src2\t! " %}
15049   ins_encode %{
15050     __ movdqu($dst$$XMMRegister, $src1$$XMMRegister);
15051     __ pmaxsb($dst$$XMMRegister, $src2$$XMMRegister);
15052   %}
15053   ins_pipe( pipe_slow );
15054 %}
15055 
15056 instruct max16B_reg_avx(vecX dst, vecX src1, vecX src2) %{
15057   predicate(UseAVX > 0 && n->as_Vector()->length() == 16 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
15058   match(Set dst (MaxV src1 src2));
15059   format %{ "vpmaxsb    $dst,$src1,$src2\t! " %}
15060   ins_encode %{
15061     int vector_len = 0;
15062     __ vpmaxsb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
15063   %}
15064   ins_pipe( pipe_slow );
15065 %}
15066 
15067 instruct max32B_reg(vecY dst, vecY src1, vecY src2) %{
15068   predicate(UseAVX > 1 && n->as_Vector()->length() == 32 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
15069   match(Set dst (MaxV src1 src2));
15070   format %{ "vpmaxsb    $dst,$src1,$src2\t! " %}
15071   ins_encode %{
15072     int vector_len = 1;
15073     __ vpmaxsb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
15074   %}
15075   ins_pipe( pipe_slow );
15076 %}
15077 
15078 instruct max64B_reg_evex(vecZ dst, vecZ src1, vecZ src2) %{
15079   predicate(UseAVX > 2 && VM_Version::supports_avx512bw() && n->as_Vector()->length() == 64 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
15080   match(Set dst (MaxV src1 src2));
15081   format %{ "vpmaxsb  $dst,$src1,$src2\t! " %}
15082   ins_encode %{
15083     int vector_len = 2;
15084     __ vpmaxsb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
15085   %}
15086   ins_pipe( pipe_slow );
15087 %}
15088 
15089 //Short vector Max
15090 instruct max4S_reg(vecD dst, vecD src1, vecD src2) %{
15091   predicate(UseSSE > 1 && UseAVX == 0 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
15092   match(Set dst (MaxV src1 src2));
15093   effect(TEMP dst);
15094   format %{ "movsd   $dst,$src1\n\t"
15095             "pmaxsw  $dst,$src2\t! " %}
15096   ins_encode %{
15097     __ movsd($dst$$XMMRegister, $src1$$XMMRegister);
15098     __ pmaxsw($dst$$XMMRegister, $src2$$XMMRegister);
15099   %}
15100   ins_pipe( pipe_slow );
15101 %}
15102 
15103 instruct max4S_reg_avx(vecD dst, vecD src1, vecD src2) %{
15104   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
15105   match(Set dst (MaxV src1 src2));
15106   format %{ "vpmaxsw    $dst,$src1,$src2\t! " %}
15107   ins_encode %{
15108     int vector_len = 0;
15109     __ vpmaxsw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
15110   %}
15111   ins_pipe( pipe_slow );
15112 %}
15113 
15114 instruct max8S_reg(vecX dst, vecX src1, vecX src2) %{
15115   predicate(UseSSE > 1 && UseAVX == 0 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
15116   match(Set dst (MaxV src1 src2));
15117   effect(TEMP dst);
15118   format %{ "movdqu  $dst,$src1\n\t"
15119             "pmaxsw  $dst,$src2\t! " %}
15120   ins_encode %{
15121     __ movdqu($dst$$XMMRegister, $src1$$XMMRegister);
15122     __ pmaxsw($dst$$XMMRegister, $src2$$XMMRegister);
15123   %}
15124   ins_pipe( pipe_slow );
15125 %}
15126 
15127 instruct max8S_reg_avx(vecX dst, vecX src1, vecX src2) %{
15128   predicate(UseAVX > 0 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
15129   match(Set dst (MaxV src1 src2));
15130   format %{ "vpmaxsw    $dst,$src1,$src2\t! " %}
15131   ins_encode %{
15132     int vector_len = 0;
15133     __ vpmaxsw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
15134   %}
15135   ins_pipe( pipe_slow );
15136 %}
15137 
15138 instruct max16S_reg(vecY dst, vecY src1, vecY src2) %{
15139   predicate(UseAVX > 1 && n->as_Vector()->length() == 16 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
15140   match(Set dst (MaxV src1 src2));
15141   format %{ "vpmaxsw    $dst,$src1,$src2\t! " %}
15142   ins_encode %{
15143     int vector_len = 1;
15144     __ vpmaxsw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
15145   %}
15146   ins_pipe( pipe_slow );
15147 %}
15148 
15149 instruct max32S_reg_evex(vecZ dst, vecZ src1, vecZ src2) %{
15150   predicate(UseAVX > 2 && VM_Version::supports_avx512bw() && n->as_Vector()->length() == 32 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
15151   match(Set dst (MaxV src1 src2));
15152   format %{ "vpmaxsw  $dst,$src1,$src2\t! " %}
15153   ins_encode %{
15154     int vector_len = 2;
15155     __ vpmaxsw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
15156   %}
15157   ins_pipe( pipe_slow );
15158 %}
15159 
15160 // Int vector Max
15161 instruct max2I_reg(vecD dst, vecD src1, vecD src2) %{
15162   predicate(UseSSE > 3 && UseAVX == 0 && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
15163   match(Set dst (MaxV src1 src2));
15164   effect(TEMP dst);
15165   format %{ "movdqu  $dst,$src1\n\t"
15166             "pmaxsd  $dst,$src2\t! " %}
15167   ins_encode %{
15168     __ movdqu($dst$$XMMRegister, $src1$$XMMRegister);
15169     __ pmaxsd($dst$$XMMRegister, $src2$$XMMRegister);
15170   %}
15171   ins_pipe( pipe_slow );
15172 %}
15173 
15174 instruct max2I_reg_avx(vecD dst, vecD src1, vecD src2) %{
15175   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
15176   match(Set dst (MaxV src1 src2));
15177   format %{ "vpmaxsd    $dst,$src1,$src2\t! " %}
15178   ins_encode %{
15179     int vector_len = 0;
15180     __ vpmaxsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
15181   %}
15182   ins_pipe( pipe_slow );
15183 %}
15184 
15185 instruct max4I_reg(vecX dst, vecX src1, vecX src2) %{
15186   predicate(UseSSE > 3 && UseAVX == 0 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
15187   match(Set dst (MaxV src1 src2));
15188   effect(TEMP dst);
15189   format %{ "movdqu  $dst,$src1\n\t"
15190             "pmaxsd  $dst,$src2\t! " %}
15191   ins_encode %{
15192     __ movdqu($dst$$XMMRegister, $src1$$XMMRegister);
15193     __ pmaxsd($dst$$XMMRegister, $src2$$XMMRegister);
15194   %}
15195   ins_pipe( pipe_slow );
15196 %}
15197 
15198 instruct max4I_reg_avx(vecX dst, vecX src1, vecX src2) %{
15199   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
15200   match(Set dst (MaxV src1 src2));
15201   format %{ "vpmaxsd    $dst,$src1,$src2\t! " %}
15202   ins_encode %{
15203     int vector_len = 0;
15204     __ vpmaxsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
15205   %}
15206   ins_pipe( pipe_slow );
15207 %}
15208 
15209 instruct max4I_reg_evex(vecX dst, vecX src1, vecX src2) %{
15210   predicate(UseAVX > 2 && VM_Version::supports_avx512vl() && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
15211   match(Set dst (MaxV src1 src2));
15212   format %{ "vpmaxsd  $dst,$src1,$src2\t! " %}
15213   ins_encode %{
15214     int vector_len = 0;
15215     __ vpmaxsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
15216   %}
15217   ins_pipe( pipe_slow );
15218 %}
15219 
15220 instruct max8I_reg_avx(vecY dst, vecY src1, vecY src2) %{
15221   predicate(UseAVX > 1 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
15222   match(Set dst (MaxV src1 src2));
15223   format %{ "vpmaxsd    $dst,$src1,$src2\t! " %}
15224   ins_encode %{
15225     int vector_len = 1;
15226     __ vpmaxsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
15227   %}
15228   ins_pipe( pipe_slow );
15229 %}
15230 
15231 instruct max8I_reg_evex(vecY dst, vecY src1, vecY src2) %{
15232   predicate(UseAVX > 2 && VM_Version::supports_avx512vl() && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
15233   match(Set dst (MaxV src1 src2));
15234   format %{ "vpmaxsd  $dst,$src1,$src2\t! " %}
15235   ins_encode %{
15236     int vector_len = 1;
15237     __ vpmaxsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
15238   %}
15239   ins_pipe( pipe_slow );
15240 %}
15241 
15242 instruct max16I_reg_evex(vecZ dst, vecZ src1, vecZ src2) %{
15243   predicate(UseAVX > 2 && VM_Version::supports_avx512vl() && n->as_Vector()->length() == 16 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
15244   match(Set dst (MaxV src1 src2));
15245   format %{ "vpmaxsd  $dst,$src1,$src2\t! " %}
15246   ins_encode %{
15247     int vector_len = 2;
15248     __ vpmaxsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
15249   %}
15250   ins_pipe( pipe_slow );
15251 %}
15252 
15253 // Long Vector Max
15254 instruct maxL_reg(vecD dst, vecD src1, vecD src2, rxmm0 tmp) %{
15255   predicate(UseAVX == 0 && UseSSE >= 4 && n->as_Vector()->length() == 1 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
15256   match(Set dst (MaxV src1 src2));
15257   effect(TEMP dst, TEMP tmp);
15258   format %{ "movsd     $tmp,$src1\n\t"
15259             "movsd     $dst,$src1\n\t"
15260             "pcmpgtq   $tmp,$src2\n\t"
15261             "blendvpd  $dst,$src2\t! " %}
15262   ins_encode %{
15263     __ movsd($tmp$$XMMRegister, $src1$$XMMRegister);
15264     __ movsd($dst$$XMMRegister, $src2$$XMMRegister);
15265     __ pcmpgtq($tmp$$XMMRegister, $src2$$XMMRegister);
15266     __ blendvpd($dst$$XMMRegister, $src1$$XMMRegister);
15267   %}
15268   ins_pipe( pipe_slow );
15269 %}
15270 
15271 instruct max1L_reg_avx(vecD dst, vecD src1, vecD src2) %{
15272   predicate(UseAVX > 0 && n->as_Vector()->length() == 1 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
15273   match(Set dst (MaxV src1 src2));
15274   effect(TEMP dst);
15275   format %{ "vpcmpgtq   $dst,$src1,$src2\n\t"
15276             "vblendvpd  $dst,$src2,$src1,$dst\t! " %}
15277   ins_encode %{
15278     int vector_len = 0;
15279     __ vpcmpgtq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
15280     __ vblendvpd($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, $dst$$XMMRegister, vector_len);
15281   %}
15282   ins_pipe( pipe_slow );
15283 %}
15284 
15285 instruct max2L_reg(vecX dst, vecX src1, vecX src2, rxmm0 tmp) %{
15286   predicate(UseAVX == 0 && UseSSE >= 4 && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
15287   match(Set dst (MaxV src1 src2));
15288   effect(TEMP dst, TEMP tmp);
15289   format %{ "movdqu    $tmp,$src2\n\t"
15290             "movdqu    $dst,$src1\n\t"
15291             "pcmpgtq   $tmp,$src1\n\t"
15292             "blendvpd  $dst,$src2\t! " %}
15293   ins_encode %{
15294     __ movdqu($tmp$$XMMRegister, $src2$$XMMRegister);
15295     __ movdqu($dst$$XMMRegister, $src1$$XMMRegister);
15296     __ pcmpgtq($tmp$$XMMRegister, $src1$$XMMRegister);
15297     __ blendvpd($dst$$XMMRegister, $src2$$XMMRegister);
15298   %}
15299   ins_pipe( pipe_slow );
15300 %}
15301 
15302 instruct max2L_reg_avx(vecX dst, vecX src1, vecX src2) %{
15303   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
15304   match(Set dst (MaxV src1 src2));
15305   effect(TEMP dst);
15306   format %{ "vpcmpgtq   $dst,$src1,$src2\n\t"
15307             "vblendvpd  $dst,$src2,$src1,$dst\t! " %}
15308   ins_encode %{
15309     int vector_len = 0;
15310     __ vpcmpgtq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
15311     __ vblendvpd($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, $dst$$XMMRegister, vector_len);
15312   %}
15313   ins_pipe( pipe_slow );
15314 %}
15315 
15316 instruct max2L_reg_evex(vecX dst, vecX src1, vecX src2) %{
15317   predicate(UseAVX > 2 && VM_Version::supports_avx512vl() && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
15318   match(Set dst (MaxV src1 src2));
15319   format %{ "vpmaxsq  $dst,$src1,src2\t! " %}
15320   ins_encode %{
15321     __ vpmaxsq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, 0);
15322   %}
15323   ins_pipe( pipe_slow );
15324 %}
15325 
15326 instruct max4L_reg_avx(vecY dst, vecY src1, vecY src2) %{
15327   predicate(UseAVX > 1 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
15328   match(Set dst (MaxV src1 src2));
15329   effect(TEMP dst);
15330   format %{ "vpcmpgtq   $dst,$src1,$src2\n\t"
15331             "vblendvpd  $dst,$src2,$src1,$dst\t! " %}
15332   ins_encode %{
15333     int vector_len = 1;
15334     __ vpcmpgtq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
15335     __ vblendvpd($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, $dst$$XMMRegister, vector_len);
15336   %}
15337   ins_pipe( pipe_slow );
15338 %}
15339 
15340 instruct max4L_reg_evex(vecY dst, vecY src1, vecY src2) %{
15341   predicate(UseAVX > 2 && VM_Version::supports_avx512vl() && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
15342   match(Set dst (MaxV src1 src2));
15343   format %{ "vpmaxsq  $dst,$src1,src2\t! " %}
15344   ins_encode %{
15345     __ vpmaxsq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, 1);
15346   %}
15347   ins_pipe( pipe_slow );
15348 %}
15349 
15350 instruct max8L_reg_evex(vecZ dst, vecZ src1, vecZ src2) %{
15351   predicate(UseAVX > 2 && VM_Version::supports_avx512vl() && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
15352   match(Set dst (MaxV src1 src2));
15353   format %{ "vpmaxsq  $dst,$src1,src2\t! " %}
15354   ins_encode %{
15355     __ vpmaxsq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, 2);
15356   %}
15357   ins_pipe( pipe_slow );
15358 %}
15359 
15360 // Float Vector Max
15361 instruct max2F_reg(vecD dst, vecD src1, vecD src2) %{
15362   predicate(UseSSE > 0 && UseAVX == 0 && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
15363   match(Set dst (MaxV src1 src2));
15364   effect(TEMP dst);
15365   format %{ "movsd  $dst,$src1\n\t"
15366             "maxps  $dst,$src2\t! " %}
15367   ins_encode %{
15368     __ movsd($dst$$XMMRegister, $src1$$XMMRegister);
15369     __ maxps($dst$$XMMRegister, $src2$$XMMRegister);
15370   %}
15371   ins_pipe( pipe_slow );
15372 %}
15373 
15374 instruct max2F_reg_avx(vecD dst, vecD src1, vecD src2) %{
15375   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
15376   match(Set dst (MaxV src1 src2));
15377   format %{ "vmaxps  $dst,$src1,$src2\t! " %}
15378   ins_encode %{
15379     int vector_len = 0;
15380     __ vmaxps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
15381   %}
15382   ins_pipe( pipe_slow );
15383 %}
15384 
15385 instruct max4F_reg(vecX dst, vecX src1, vecX src2) %{
15386   predicate(UseSSE > 0 && UseAVX == 0 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
15387   match(Set dst (MaxV src1 src2));
15388   effect(TEMP dst);
15389   format %{ "movdqu  $dst,$src1\n\t"
15390             "maxps   $dst,$src2\t! " %}
15391   ins_encode %{
15392     __ movdqu($dst$$XMMRegister, $src1$$XMMRegister);
15393     __ maxps($dst$$XMMRegister, $src2$$XMMRegister);
15394   %}
15395   ins_pipe( pipe_slow );
15396 %}
15397 
15398 instruct max4F_reg_avx(vecX dst, vecX src1, vecX src2) %{
15399   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
15400   match(Set dst (MaxV src1 src2));
15401   format %{ "vmaxps  $dst,$src1,$src2\t! " %}
15402   ins_encode %{
15403     int vector_len = 0;
15404     __ vmaxps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
15405   %}
15406   ins_pipe( pipe_slow );
15407 %}
15408 
15409 instruct max4F_reg_evex(vecX dst, vecX src1, vecX src2) %{
15410   predicate(UseAVX > 2 && VM_Version::supports_avx512vl() && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
15411   match(Set dst (MaxV src1 src2));
15412   format %{ "vmaxps  $dst,$src1,$src2\t! " %}
15413   ins_encode %{
15414     int vector_len = 0;
15415     __ vmaxps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
15416   %}
15417   ins_pipe( pipe_slow );
15418 %}
15419 
15420 instruct max8F_reg_avx(vecY dst, vecY src1, vecY src2) %{
15421   predicate(UseAVX > 0 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
15422   match(Set dst (MaxV src1 src2));
15423   format %{ "vmaxps  $dst,$src1,$src2\t! " %}
15424   ins_encode %{
15425     int vector_len = 1;
15426     __ vmaxps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
15427   %}
15428   ins_pipe( pipe_slow );
15429 %}
15430 
15431 instruct max8F_reg_evex(vecY dst, vecY src1, vecY src2) %{
15432   predicate(UseAVX > 2 && VM_Version::supports_avx512vl() && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
15433   match(Set dst (MaxV src1 src2));
15434   format %{ "vmaxps  $dst,$src1,$src2\t! " %}
15435   ins_encode %{
15436     int vector_len = 1;
15437     __ vmaxps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
15438   %}
15439   ins_pipe( pipe_slow );
15440 %}
15441 
15442 instruct max16F_reg_evex(vecZ dst, vecZ src1, vecZ src2) %{
15443   predicate(UseAVX > 2 && VM_Version::supports_avx512vl() && n->as_Vector()->length() == 16 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
15444   match(Set dst (MaxV src1 src2));
15445   format %{ "vmaxps  $dst,$src1,$src2\t! " %}
15446   ins_encode %{
15447     int vector_len = 2;
15448     __ vmaxps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
15449   %}
15450   ins_pipe( pipe_slow );
15451 %}
15452 
15453 // Double Vector Max
15454 instruct maxD_reg(vecD dst, vecD src1, vecD src2) %{
15455   predicate(UseSSE > 0 && UseAVX == 0 && n->as_Vector()->length() == 1 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
15456   match(Set dst (MaxV src1 src2));
15457   effect(TEMP dst);
15458   format %{ "movsd  $dst,$src1\n\t"
15459             "maxpd  $dst,$src2\t! " %}
15460   ins_encode %{
15461     __ movsd($dst$$XMMRegister, $src1$$XMMRegister);
15462     __ maxpd($dst$$XMMRegister, $src2$$XMMRegister);
15463   %}
15464   ins_pipe( pipe_slow );
15465 %}
15466 
15467 instruct max1D_reg_avx(vecD dst, vecD src1, vecD src2) %{
15468   predicate(UseAVX > 0 && n->as_Vector()->length() == 1 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
15469   match(Set dst (MaxV src1 src2));
15470   format %{ "vmaxpd  $dst,$src1,$src2\t! " %}
15471   ins_encode %{
15472     int vector_len = 0;
15473     __ vmaxpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
15474   %}
15475   ins_pipe( pipe_slow );
15476 %}
15477 
15478 instruct max2D_reg(vecX dst, vecX src1, vecX src2) %{
15479   predicate(UseSSE > 0 && UseAVX == 0 && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
15480   match(Set dst (MaxV src1 src2));
15481   effect(TEMP dst);
15482   format %{ "movdqu  $dst,$src1\n\t"
15483             "maxpd   $dst,$src2\t! " %}
15484   ins_encode %{
15485     __ movdqu($dst$$XMMRegister, $src1$$XMMRegister);
15486     __ maxpd($dst$$XMMRegister, $src2$$XMMRegister);
15487   %}
15488   ins_pipe( pipe_slow );
15489 %}
15490 
15491 instruct max2D_reg_avx(vecX dst, vecX src1, vecX src2) %{
15492   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
15493   match(Set dst (MaxV src1 src2));
15494   format %{ "vmaxpd  $dst,$src1,$src2\t! " %}
15495   ins_encode %{
15496     int vector_len = 0;
15497     __ vmaxpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
15498   %}
15499   ins_pipe( pipe_slow );
15500 %}
15501 
15502 instruct max2D_reg_evex(vecX dst, vecX src1, vecX src2) %{
15503   predicate(UseAVX > 2 && VM_Version::supports_avx512vl() && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
15504   match(Set dst (MaxV src1 src2));
15505   format %{ "vmaxpd  $dst,$src1,$src2\t! " %}
15506   ins_encode %{
15507     int vector_len = 0;
15508     __ vmaxpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
15509   %}
15510   ins_pipe( pipe_slow );
15511 %}
15512 
15513 instruct max4D_reg_avx(vecY dst, vecY src1, vecY src2) %{
15514   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
15515   match(Set dst (MaxV src1 src2));
15516   format %{ "vmaxpd  $dst,$src1,$src2\t! " %}
15517   ins_encode %{
15518     int vector_len = 1;
15519     __ vmaxpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
15520   %}
15521   ins_pipe( pipe_slow );
15522 %}
15523 
15524 instruct max4D_reg_evex(vecY dst, vecY src1, vecY src2) %{
15525   predicate(UseAVX > 2 && VM_Version::supports_avx512vl() && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
15526   match(Set dst (MaxV src1 src2));
15527   format %{ "vmaxpd  $dst,$src1,$src2\t! " %}
15528   ins_encode %{
15529     int vector_len = 1;
15530     __ vmaxpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
15531   %}
15532   ins_pipe( pipe_slow );
15533 %}
15534 
15535 instruct max8D_reg_evex(vecZ dst, vecZ src1, vecZ src2) %{
15536   predicate(UseAVX > 2 && VM_Version::supports_avx512vl() && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
15537   match(Set dst (MaxV src1 src2));
15538   format %{ "vmaxpd  $dst,$src1,$src2\t! " %}
15539   ins_encode %{
15540     int vector_len = 2;
15541     __ vmaxpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
15542   %}
15543   ins_pipe( pipe_slow );
15544 %}
15545 
15546 // ------------------------------ Shift ---------------------------------------
15547 
15548 // Left and right shift count vectors are the same on x86
15549 // (only lowest bits of xmm reg are used for count).
15550 instruct vshiftcnt(vecS dst, rRegI cnt) %{
15551   match(Set dst (LShiftCntV cnt));
15552   match(Set dst (RShiftCntV cnt));
15553   format %{ "movd    $dst,$cnt\t! load shift count" %}
15554   ins_encode %{
15555     __ movdl($dst$$XMMRegister, $cnt$$Register);
15556   %}
15557   ins_pipe( pipe_slow );
15558 %}
15559 
15560 // --------------------------------- Sqrt --------------------------------------
15561 
15562 // Floating point vector sqrt
15563 instruct vsqrt2D_reg(vecX dst, vecX src) %{
15564   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
15565   match(Set dst (SqrtVD src));
15566   format %{ "vsqrtpd  $dst,$src\t! sqrt packed2D" %}
15567   ins_encode %{
15568     int vector_len = 0;
15569     __ vsqrtpd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
15570   %}
15571   ins_pipe( pipe_slow );
15572 %}
15573 
15574 instruct vsqrt2D_mem(vecX dst, memory mem) %{
15575   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
15576   match(Set dst (SqrtVD (LoadVector mem)));
15577   format %{ "vsqrtpd  $dst,$mem\t! sqrt packed2D" %}
15578   ins_encode %{
15579     int vector_len = 0;
15580     __ vsqrtpd($dst$$XMMRegister, $mem$$Address, vector_len);
15581   %}
15582   ins_pipe( pipe_slow );
15583 %}
15584 
15585 instruct vsqrt4D_reg(vecY dst, vecY src) %{
15586   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
15587   match(Set dst (SqrtVD src));
15588   format %{ "vsqrtpd  $dst,$src\t! sqrt packed4D" %}
15589   ins_encode %{
15590     int vector_len = 1;
15591     __ vsqrtpd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
15592   %}
15593   ins_pipe( pipe_slow );
15594 %}
15595 
15596 instruct vsqrt4D_mem(vecY dst, memory mem) %{
15597   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
15598   match(Set dst (SqrtVD (LoadVector mem)));
15599   format %{ "vsqrtpd  $dst,$mem\t! sqrt packed4D" %}
15600   ins_encode %{
15601     int vector_len = 1;
15602     __ vsqrtpd($dst$$XMMRegister, $mem$$Address, vector_len);
15603   %}
15604   ins_pipe( pipe_slow );
15605 %}
15606 
15607 instruct vsqrt8D_reg(vecZ dst, vecZ src) %{
15608   predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
15609   match(Set dst (SqrtVD src));
15610   format %{ "vsqrtpd  $dst,$src\t! sqrt packed8D" %}
15611   ins_encode %{
15612     int vector_len = 2;
15613     __ vsqrtpd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
15614   %}
15615   ins_pipe( pipe_slow );
15616 %}
15617 
15618 instruct vsqrt8D_mem(vecZ dst, memory mem) %{
15619   predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
15620   match(Set dst (SqrtVD (LoadVector mem)));
15621   format %{ "vsqrtpd  $dst,$mem\t! sqrt packed8D" %}
15622   ins_encode %{
15623     int vector_len = 2;
15624     __ vsqrtpd($dst$$XMMRegister, $mem$$Address, vector_len);
15625   %}
15626   ins_pipe( pipe_slow );
15627 %}
15628 
15629 instruct vsqrt2F_reg(vecD dst, vecD src) %{
15630   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
15631   match(Set dst (SqrtVF src));
15632   format %{ "vsqrtps  $dst,$src\t! sqrt packed2F" %}
15633   ins_encode %{
15634     int vector_len = 0;
15635     __ vsqrtps($dst$$XMMRegister, $src$$XMMRegister, vector_len);
15636   %}
15637   ins_pipe( pipe_slow );
15638 %}
15639 
15640 instruct vsqrt2F_mem(vecD dst, memory mem) %{
15641   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
15642   match(Set dst (SqrtVF (LoadVector mem)));
15643   format %{ "vsqrtps  $dst,$mem\t! sqrt packed2F" %}
15644   ins_encode %{
15645     int vector_len = 0;
15646     __ vsqrtps($dst$$XMMRegister, $mem$$Address, vector_len);
15647   %}
15648   ins_pipe( pipe_slow );
15649 %}
15650 
15651 instruct vsqrt4F_reg(vecX dst, vecX src) %{
15652   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
15653   match(Set dst (SqrtVF src));
15654   format %{ "vsqrtps  $dst,$src\t! sqrt packed4F" %}
15655   ins_encode %{
15656     int vector_len = 0;
15657     __ vsqrtps($dst$$XMMRegister, $src$$XMMRegister, vector_len);
15658   %}
15659   ins_pipe( pipe_slow );
15660 %}
15661 
15662 instruct vsqrt4F_mem(vecX dst, memory mem) %{
15663   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
15664   match(Set dst (SqrtVF (LoadVector mem)));
15665   format %{ "vsqrtps  $dst,$mem\t! sqrt packed4F" %}
15666   ins_encode %{
15667     int vector_len = 0;
15668     __ vsqrtps($dst$$XMMRegister, $mem$$Address, vector_len);
15669   %}
15670   ins_pipe( pipe_slow );
15671 %}
15672 
15673 instruct vsqrt8F_reg(vecY dst, vecY src) %{
15674   predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
15675   match(Set dst (SqrtVF src));
15676   format %{ "vsqrtps  $dst,$src\t! sqrt packed8F" %}
15677   ins_encode %{
15678     int vector_len = 1;
15679     __ vsqrtps($dst$$XMMRegister, $src$$XMMRegister, vector_len);
15680   %}
15681   ins_pipe( pipe_slow );
15682 %}
15683 
15684 instruct vsqrt8F_mem(vecY dst, memory mem) %{
15685   predicate(UseAVX > 0 && n->as_Vector()->length() == 8);
15686   match(Set dst (SqrtVF (LoadVector mem)));
15687   format %{ "vsqrtps  $dst,$mem\t! sqrt packed8F" %}
15688   ins_encode %{
15689     int vector_len = 1;
15690     __ vsqrtps($dst$$XMMRegister, $mem$$Address, vector_len);
15691   %}
15692   ins_pipe( pipe_slow );
15693 %}
15694 
15695 instruct vsqrt16F_reg(vecZ dst, vecZ src) %{
15696   predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
15697   match(Set dst (SqrtVF src));
15698   format %{ "vsqrtps  $dst,$src\t! sqrt packed16F" %}
15699   ins_encode %{
15700     int vector_len = 2;
15701     __ vsqrtps($dst$$XMMRegister, $src$$XMMRegister, vector_len);
15702   %}
15703   ins_pipe( pipe_slow );
15704 %}
15705 
15706 instruct vsqrt16F_mem(vecZ dst, memory mem) %{
15707   predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
15708   match(Set dst (SqrtVF (LoadVector mem)));
15709   format %{ "vsqrtps  $dst,$mem\t! sqrt packed16F" %}
15710   ins_encode %{
15711     int vector_len = 2;
15712     __ vsqrtps($dst$$XMMRegister, $mem$$Address, vector_len);
15713   %}
15714   ins_pipe( pipe_slow );
15715 %}
15716 
15717 // ------------------------------ LeftShift -----------------------------------
15718 
15719 // Shorts/Chars vector left shift
15720 instruct vsll2S(vecS dst, vecS shift) %{
15721   predicate(UseAVX == 0 && n->as_Vector()->length() == 2);
15722   match(Set dst (LShiftVS dst shift));
15723   format %{ "psllw   $dst,$shift\t! left shift packed2S" %}
15724   ins_encode %{
15725     __ psllw($dst$$XMMRegister, $shift$$XMMRegister);
15726   %}
15727   ins_pipe( pipe_slow );
15728 %}
15729 
15730 instruct vsll2S_imm(vecS dst, immI8 shift) %{
15731   predicate(UseAVX == 0 && n->as_Vector()->length() == 2);
15732   match(Set dst (LShiftVS dst shift));
15733   format %{ "psllw   $dst,$shift\t! left shift packed2S" %}
15734   ins_encode %{
15735     __ psllw($dst$$XMMRegister, (int)$shift$$constant);
15736   %}
15737   ins_pipe( pipe_slow );
15738 %}
15739 
15740 instruct vsll2S_reg_avx(vecS dst, vecS src, vecS shift) %{
15741   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 2);
15742   match(Set dst (LShiftVS src shift));
15743   format %{ "vpsllw  $dst,$src,$shift\t! left shift packed2S" %}
15744   ins_encode %{
15745     int vector_len = 0;
15746     __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
15747   %}
15748   ins_pipe( pipe_slow );
15749 %}
15750 
15751 instruct vsll2S_reg_evex(vecS dst, vecS src, vecS shift) %{
15752   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 2);
15753   match(Set dst (LShiftVS src shift));
15754   format %{ "vpsllw  $dst,$src,$shift\t! left shift packed2S" %}
15755   ins_encode %{
15756     int vector_len = 0;
15757     __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
15758   %}
15759   ins_pipe( pipe_slow );
15760 %}
15761 
15762 instruct vsll2S_reg_evex_special(vecS dst, vecS src, vecS shift) %{
15763   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 2);
15764   match(Set dst (LShiftVS dst shift));
15765   effect(TEMP src);
15766   format %{ "vpsllw  $dst,$src,$shift\t! left shift packed2S" %}
15767   ins_encode %{
15768     int vector_len = 0;
15769     __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
15770   %}
15771   ins_pipe( pipe_slow );
15772 %}
15773 
15774 instruct vsll2S_reg_imm_avx(vecS dst, vecS src, immI8 shift) %{
15775   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 2);
15776   match(Set dst (LShiftVS src shift));
15777   format %{ "vpsllw  $dst,$src,$shift\t! left shift packed2S" %}
15778   ins_encode %{
15779     int vector_len = 0;
15780     __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
15781   %}
15782   ins_pipe( pipe_slow );
15783 %}
15784 
15785 instruct vsll2S_reg_imm_evex(vecS dst, vecS src, immI8 shift) %{
15786   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 2);
15787   match(Set dst (LShiftVS src shift));
15788   format %{ "vpsllw  $dst,$src,$shift\t! left shift packed2S" %}
15789   ins_encode %{
15790     int vector_len = 0;
15791     __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
15792   %}
15793   ins_pipe( pipe_slow );
15794 %}
15795 
15796 instruct vsll2S_reg_imm_evex_special(vecS dst, vecS src, immI8 shift) %{
15797   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 2);
15798   match(Set dst (LShiftVS dst shift));
15799   effect(TEMP src);
15800   format %{ "vpsllw  $dst,$src,$shift\t! left shift packed2S" %}
15801   ins_encode %{
15802     int vector_len = 0;
15803     __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
15804   %}
15805   ins_pipe( pipe_slow );
15806 %}
15807 
15808 instruct vsll4S(vecD dst, vecS shift) %{
15809   predicate(UseAVX == 0 && n->as_Vector()->length() == 4);
15810   match(Set dst (LShiftVS dst shift));
15811   format %{ "psllw   $dst,$shift\t! left shift packed4S" %}
15812   ins_encode %{
15813     __ psllw($dst$$XMMRegister, $shift$$XMMRegister);
15814   %}
15815   ins_pipe( pipe_slow );
15816 %}
15817 
15818 instruct vsll4S_imm(vecD dst, immI8 shift) %{
15819   predicate(UseAVX == 0 && n->as_Vector()->length() == 4);
15820   match(Set dst (LShiftVS dst shift));
15821   format %{ "psllw   $dst,$shift\t! left shift packed4S" %}
15822   ins_encode %{
15823     __ psllw($dst$$XMMRegister, (int)$shift$$constant);
15824   %}
15825   ins_pipe( pipe_slow );
15826 %}
15827 
15828 instruct vsll4S_reg_avx(vecD dst, vecD src, vecS shift) %{
15829   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 4);
15830   match(Set dst (LShiftVS src shift));
15831   format %{ "vpsllw  $dst,$src,$shift\t! left shift packed4S" %}
15832   ins_encode %{
15833     int vector_len = 0;
15834     __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
15835   %}
15836   ins_pipe( pipe_slow );
15837 %}
15838 
15839 instruct vsll4S_reg_evex(vecD dst, vecD src, vecS shift) %{
15840   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 4);
15841   match(Set dst (LShiftVS src shift));
15842   format %{ "vpsllw  $dst,$src,$shift\t! left shift packed4S" %}
15843   ins_encode %{
15844     int vector_len = 0;
15845     __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
15846   %}
15847   ins_pipe( pipe_slow );
15848 %}
15849 
15850 instruct vsll4S_reg_evex_special(vecD dst, vecD src, vecS shift) %{
15851   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 4);
15852   match(Set dst (LShiftVS dst shift));
15853   effect(TEMP src);
15854   format %{ "vpsllw  $dst,$src,$shift\t! left shift packed4S" %}
15855   ins_encode %{
15856     int vector_len = 0;
15857     __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
15858   %}
15859   ins_pipe( pipe_slow );
15860 %}
15861 
15862 instruct vsll4S_reg_imm_avx(vecD dst, vecD src, immI8 shift) %{
15863   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 4);
15864   match(Set dst (LShiftVS src shift));
15865   format %{ "vpsllw  $dst,$src,$shift\t! left shift packed4S" %}
15866   ins_encode %{
15867     int vector_len = 0;
15868     __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
15869   %}
15870   ins_pipe( pipe_slow );
15871 %}
15872 
15873 instruct vsll4S_reg_imm_evex(vecD dst, vecD src, immI8 shift) %{
15874   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 4);
15875   match(Set dst (LShiftVS src shift));
15876   format %{ "vpsllw  $dst,$src,$shift\t! left shift packed4S" %}
15877   ins_encode %{
15878     int vector_len = 0;
15879     __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
15880   %}
15881   ins_pipe( pipe_slow );
15882 %}
15883 
15884 instruct vsll4S_reg_imm_evex_special(vecD dst, vecD src, immI8 shift) %{
15885   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 4);
15886   match(Set dst (LShiftVS dst shift));
15887   effect(TEMP src);
15888   format %{ "vpsllw  $dst,$src,$shift\t! left shift packed4S" %}
15889   ins_encode %{
15890     int vector_len = 0;
15891     __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
15892   %}
15893   ins_pipe( pipe_slow );
15894 %}
15895 
15896 instruct vsll8S(vecX dst, vecS shift) %{
15897   predicate(UseAVX == 0 && n->as_Vector()->length() == 8);
15898   match(Set dst (LShiftVS dst shift));
15899   format %{ "psllw   $dst,$shift\t! left shift packed8S" %}
15900   ins_encode %{
15901     __ psllw($dst$$XMMRegister, $shift$$XMMRegister);
15902   %}
15903   ins_pipe( pipe_slow );
15904 %}
15905 
15906 instruct vsll8S_imm(vecX dst, immI8 shift) %{
15907   predicate(UseAVX == 0 && n->as_Vector()->length() == 8);
15908   match(Set dst (LShiftVS dst shift));
15909   format %{ "psllw   $dst,$shift\t! left shift packed8S" %}
15910   ins_encode %{
15911     __ psllw($dst$$XMMRegister, (int)$shift$$constant);
15912   %}
15913   ins_pipe( pipe_slow );
15914 %}
15915 
15916 instruct vsll8S_reg_avx(vecX dst, vecX src, vecS shift) %{
15917   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 8);
15918   match(Set dst (LShiftVS src shift));
15919   format %{ "vpsllw  $dst,$src,$shift\t! left shift packed8S" %}
15920   ins_encode %{
15921     int vector_len = 0;
15922     __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
15923   %}
15924   ins_pipe( pipe_slow );
15925 %}
15926 
15927 instruct vsll8S_reg_evex(vecX dst, vecX src, vecS shift) %{
15928   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 8);
15929   match(Set dst (LShiftVS src shift));
15930   format %{ "vpsllw  $dst,$src,$shift\t! left shift packed8S" %}
15931   ins_encode %{
15932     int vector_len = 0;
15933     __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
15934   %}
15935   ins_pipe( pipe_slow );
15936 %}
15937 
15938 instruct vsll8S_reg_evex_special(vecX dst, vecX src, vecS shift) %{
15939   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 8);
15940   match(Set dst (LShiftVS dst shift));
15941   effect(TEMP src);
15942   format %{ "vpsllw  $dst,$src,$shift\t! left shift packed8S" %}
15943   ins_encode %{
15944     int vector_len = 0;
15945     __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
15946   %}
15947   ins_pipe( pipe_slow );
15948 %}
15949 
15950 instruct vsll8S_reg_imm_avx(vecX dst, vecX src, immI8 shift) %{
15951   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 8);
15952   match(Set dst (LShiftVS src shift));
15953   format %{ "vpsllw  $dst,$src,$shift\t! left shift packed8S" %}
15954   ins_encode %{
15955     int vector_len = 0;
15956     __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
15957   %}
15958   ins_pipe( pipe_slow );
15959 %}
15960 
15961 instruct vsll8S_reg_imm_evex(vecX dst, vecX src, immI8 shift) %{
15962   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 8);
15963   match(Set dst (LShiftVS src shift));
15964   format %{ "vpsllw  $dst,$src,$shift\t! left shift packed8S" %}
15965   ins_encode %{
15966     int vector_len = 0;
15967     __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
15968   %}
15969   ins_pipe( pipe_slow );
15970 %}
15971 
15972 instruct vsll8S_reg_imm_evex_special(vecX dst, vecX src, immI8 shift) %{
15973   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 8);
15974   match(Set dst (LShiftVS dst shift));
15975   effect(TEMP src);
15976   format %{ "vpsllw  $dst,$src,$shift\t! left shift packed8S" %}
15977   ins_encode %{
15978     int vector_len = 0;
15979     __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
15980   %}
15981   ins_pipe( pipe_slow );
15982 %}
15983 
15984 instruct vsll16S_reg_avx(vecY dst, vecY src, vecS shift) %{
15985   predicate(VM_Version::supports_avx256only() && n->as_Vector()->length() == 16);
15986   match(Set dst (LShiftVS src shift));
15987   format %{ "vpsllw  $dst,$src,$shift\t! left shift packed16S" %}
15988   ins_encode %{
15989     int vector_len = 1;
15990     __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
15991   %}
15992   ins_pipe( pipe_slow );
15993 %}
15994 
15995 instruct vsll16S_reg_evex(vecY dst, vecY src, vecS shift) %{
15996   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 16);
15997   match(Set dst (LShiftVS src shift));
15998   format %{ "vpsllw  $dst,$src,$shift\t! left shift packed16S" %}
15999   ins_encode %{
16000     int vector_len = 1;
16001     __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16002   %}
16003   ins_pipe( pipe_slow );
16004 %}
16005 
16006 instruct vsll16S_reg_evex_special(vecY dst, vecY src, vecS shift) %{
16007   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 16);
16008   match(Set dst (LShiftVS dst shift));
16009   effect(TEMP src);
16010   format %{ "vpsllw  $dst,$src,$shift\t! left shift packed16S" %}
16011   ins_encode %{
16012     int vector_len = 1;
16013     __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16014   %}
16015   ins_pipe( pipe_slow );
16016 %}
16017 
16018 instruct vsll16S_reg_imm_avx(vecY dst, vecY src, immI8 shift) %{
16019   predicate(VM_Version::supports_avx256only() && n->as_Vector()->length() == 16);
16020   match(Set dst (LShiftVS src shift));
16021   format %{ "vpsllw  $dst,$src,$shift\t! left shift packed16S" %}
16022   ins_encode %{
16023     int vector_len = 1;
16024     __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16025   %}
16026   ins_pipe( pipe_slow );
16027 %}
16028 
16029 instruct vsll16S_reg_imm_evex(vecY dst, vecY src, immI8 shift) %{
16030   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 16);
16031   match(Set dst (LShiftVS src shift));
16032   format %{ "vpsllw  $dst,$src,$shift\t! left shift packed16S" %}
16033   ins_encode %{
16034     int vector_len = 1;
16035     __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16036   %}
16037   ins_pipe( pipe_slow );
16038 %}
16039 
16040 instruct vsll16S_reg_imm_evex_special(vecY dst, vecY src, immI8 shift) %{
16041   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 16);
16042   match(Set dst (LShiftVS dst shift));
16043   effect(TEMP src);
16044   format %{ "vpsllw  $dst,$src,$shift\t! left shift packed16S" %}
16045   ins_encode %{
16046     int vector_len = 1;
16047     __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16048   %}
16049   ins_pipe( pipe_slow );
16050 %}
16051 
16052 instruct vsll32S_reg(vecZ dst, vecZ src, vecS shift) %{
16053   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 32);
16054   match(Set dst (LShiftVS src shift));
16055   format %{ "vpsllw  $dst,$src,$shift\t! left shift packed32S" %}
16056   ins_encode %{
16057     int vector_len = 2;
16058     __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16059   %}
16060   ins_pipe( pipe_slow );
16061 %}
16062 
16063 instruct vsll32S_reg_imm(vecZ dst, vecZ src, immI8 shift) %{
16064   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 32);
16065   match(Set dst (LShiftVS src shift));
16066   format %{ "vpsllw  $dst,$src,$shift\t! left shift packed32S" %}
16067   ins_encode %{
16068     int vector_len = 2;
16069     __ vpsllw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16070   %}
16071   ins_pipe( pipe_slow );
16072 %}
16073 
16074 // Integers vector left shift
16075 instruct vsll2I(vecD dst, vecS shift) %{
16076   predicate(n->as_Vector()->length() == 2);
16077   match(Set dst (LShiftVI dst shift));
16078   format %{ "pslld   $dst,$shift\t! left shift packed2I" %}
16079   ins_encode %{
16080     __ pslld($dst$$XMMRegister, $shift$$XMMRegister);
16081   %}
16082   ins_pipe( pipe_slow );
16083 %}
16084 
16085 instruct vsll2I_imm(vecD dst, immI8 shift) %{
16086   predicate(n->as_Vector()->length() == 2);
16087   match(Set dst (LShiftVI dst shift));
16088   format %{ "pslld   $dst,$shift\t! left shift packed2I" %}
16089   ins_encode %{
16090     __ pslld($dst$$XMMRegister, (int)$shift$$constant);
16091   %}
16092   ins_pipe( pipe_slow );
16093 %}
16094 
16095 instruct vsll2I_reg(vecD dst, vecD src, vecS shift) %{
16096   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
16097   match(Set dst (LShiftVI src shift));
16098   format %{ "vpslld  $dst,$src,$shift\t! left shift packed2I" %}
16099   ins_encode %{
16100     int vector_len = 0;
16101     __ vpslld($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16102   %}
16103   ins_pipe( pipe_slow );
16104 %}
16105 
16106 instruct vsll2I_reg_imm(vecD dst, vecD src, immI8 shift) %{
16107   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
16108   match(Set dst (LShiftVI src shift));
16109   format %{ "vpslld  $dst,$src,$shift\t! left shift packed2I" %}
16110   ins_encode %{
16111     int vector_len = 0;
16112     __ vpslld($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16113   %}
16114   ins_pipe( pipe_slow );
16115 %}
16116 
16117 instruct vsll4I(vecX dst, vecS shift) %{
16118   predicate(n->as_Vector()->length() == 4);
16119   match(Set dst (LShiftVI dst shift));
16120   format %{ "pslld   $dst,$shift\t! left shift packed4I" %}
16121   ins_encode %{
16122     __ pslld($dst$$XMMRegister, $shift$$XMMRegister);
16123   %}
16124   ins_pipe( pipe_slow );
16125 %}
16126 
16127 instruct vsll4I_imm(vecX dst, immI8 shift) %{
16128   predicate(n->as_Vector()->length() == 4);
16129   match(Set dst (LShiftVI dst shift));
16130   format %{ "pslld   $dst,$shift\t! left shift packed4I" %}
16131   ins_encode %{
16132     __ pslld($dst$$XMMRegister, (int)$shift$$constant);
16133   %}
16134   ins_pipe( pipe_slow );
16135 %}
16136 
16137 instruct vsll4I_reg(vecX dst, vecX src, vecS shift) %{
16138   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
16139   match(Set dst (LShiftVI src shift));
16140   format %{ "vpslld  $dst,$src,$shift\t! left shift packed4I" %}
16141   ins_encode %{
16142     int vector_len = 0;
16143     __ vpslld($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16144   %}
16145   ins_pipe( pipe_slow );
16146 %}
16147 
16148 instruct vsll4I_reg_imm(vecX dst, vecX src, immI8 shift) %{
16149   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
16150   match(Set dst (LShiftVI src shift));
16151   format %{ "vpslld  $dst,$src,$shift\t! left shift packed4I" %}
16152   ins_encode %{
16153     int vector_len = 0;
16154     __ vpslld($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16155   %}
16156   ins_pipe( pipe_slow );
16157 %}
16158 
16159 instruct vsll8I_reg(vecY dst, vecY src, vecS shift) %{
16160   predicate(UseAVX > 1 && n->as_Vector()->length() == 8);
16161   match(Set dst (LShiftVI src shift));
16162   format %{ "vpslld  $dst,$src,$shift\t! left shift packed8I" %}
16163   ins_encode %{
16164     int vector_len = 1;
16165     __ vpslld($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16166   %}
16167   ins_pipe( pipe_slow );
16168 %}
16169 
16170 instruct vsll8I_reg_imm(vecY dst, vecY src, immI8 shift) %{
16171   predicate(UseAVX > 1 && n->as_Vector()->length() == 8);
16172   match(Set dst (LShiftVI src shift));
16173   format %{ "vpslld  $dst,$src,$shift\t! left shift packed8I" %}
16174   ins_encode %{
16175     int vector_len = 1;
16176     __ vpslld($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16177   %}
16178   ins_pipe( pipe_slow );
16179 %}
16180 
16181 instruct vsll16I_reg(vecZ dst, vecZ src, vecS shift) %{
16182   predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
16183   match(Set dst (LShiftVI src shift));
16184   format %{ "vpslld  $dst,$src,$shift\t! left shift packed16I" %}
16185   ins_encode %{
16186     int vector_len = 2;
16187     __ vpslld($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16188   %}
16189   ins_pipe( pipe_slow );
16190 %}
16191 
16192 instruct vsll16I_reg_imm(vecZ dst, vecZ src, immI8 shift) %{
16193   predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
16194   match(Set dst (LShiftVI src shift));
16195   format %{ "vpslld  $dst,$src,$shift\t! left shift packed16I" %}
16196   ins_encode %{
16197     int vector_len = 2;
16198     __ vpslld($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16199   %}
16200   ins_pipe( pipe_slow );
16201 %}
16202 
16203 // Longs vector left shift
16204 instruct vsll2L(vecX dst, vecS shift) %{
16205   predicate(n->as_Vector()->length() == 2);
16206   match(Set dst (LShiftVL dst shift));
16207   format %{ "psllq   $dst,$shift\t! left shift packed2L" %}
16208   ins_encode %{
16209     __ psllq($dst$$XMMRegister, $shift$$XMMRegister);
16210   %}
16211   ins_pipe( pipe_slow );
16212 %}
16213 
16214 instruct vsll2L_imm(vecX dst, immI8 shift) %{
16215   predicate(n->as_Vector()->length() == 2);
16216   match(Set dst (LShiftVL dst shift));
16217   format %{ "psllq   $dst,$shift\t! left shift packed2L" %}
16218   ins_encode %{
16219     __ psllq($dst$$XMMRegister, (int)$shift$$constant);
16220   %}
16221   ins_pipe( pipe_slow );
16222 %}
16223 
16224 instruct vsll2L_reg(vecX dst, vecX src, vecS shift) %{
16225   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
16226   match(Set dst (LShiftVL src shift));
16227   format %{ "vpsllq  $dst,$src,$shift\t! left shift packed2L" %}
16228   ins_encode %{
16229     int vector_len = 0;
16230     __ vpsllq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16231   %}
16232   ins_pipe( pipe_slow );
16233 %}
16234 
16235 instruct vsll2L_reg_imm(vecX dst, vecX src, immI8 shift) %{
16236   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
16237   match(Set dst (LShiftVL src shift));
16238   format %{ "vpsllq  $dst,$src,$shift\t! left shift packed2L" %}
16239   ins_encode %{
16240     int vector_len = 0;
16241     __ vpsllq($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16242   %}
16243   ins_pipe( pipe_slow );
16244 %}
16245 
16246 instruct vsll4L_reg(vecY dst, vecY src, vecS shift) %{
16247   predicate(UseAVX > 1 && n->as_Vector()->length() == 4);
16248   match(Set dst (LShiftVL src shift));
16249   format %{ "vpsllq  $dst,$src,$shift\t! left shift packed4L" %}
16250   ins_encode %{
16251     int vector_len = 1;
16252     __ vpsllq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16253   %}
16254   ins_pipe( pipe_slow );
16255 %}
16256 
16257 instruct vsll4L_reg_imm(vecY dst, vecY src, immI8 shift) %{
16258   predicate(UseAVX > 1 && n->as_Vector()->length() == 4);
16259   match(Set dst (LShiftVL src shift));
16260   format %{ "vpsllq  $dst,$src,$shift\t! left shift packed4L" %}
16261   ins_encode %{
16262     int vector_len = 1;
16263     __ vpsllq($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16264   %}
16265   ins_pipe( pipe_slow );
16266 %}
16267 
16268 instruct vsll8L_reg(vecZ dst, vecZ src, vecS shift) %{
16269   predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
16270   match(Set dst (LShiftVL src shift));
16271   format %{ "vpsllq  $dst,$src,$shift\t! left shift packed8L" %}
16272   ins_encode %{
16273     int vector_len = 2;
16274     __ vpsllq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16275   %}
16276   ins_pipe( pipe_slow );
16277 %}
16278 
16279 instruct vsll8L_reg_imm(vecZ dst, vecZ src, immI8 shift) %{
16280   predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
16281   match(Set dst (LShiftVL src shift));
16282   format %{ "vpsllq  $dst,$src,$shift\t! left shift packed8L" %}
16283   ins_encode %{
16284     int vector_len = 2;
16285     __ vpsllq($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16286   %}
16287   ins_pipe( pipe_slow );
16288 %}
16289 
16290 // ----------------------- LogicalRightShift -----------------------------------
16291 
16292 // Shorts vector logical right shift produces incorrect Java result
16293 // for negative data because java code convert short value into int with
16294 // sign extension before a shift. But char vectors are fine since chars are
16295 // unsigned values.
16296 
16297 instruct vsrl2S(vecS dst, vecS shift) %{
16298   predicate(UseAVX == 0 && n->as_Vector()->length() == 2);
16299   match(Set dst (URShiftVS dst shift));
16300   format %{ "psrlw   $dst,$shift\t! logical right shift packed2S" %}
16301   ins_encode %{
16302     __ psrlw($dst$$XMMRegister, $shift$$XMMRegister);
16303   %}
16304   ins_pipe( pipe_slow );
16305 %}
16306 
16307 instruct vsrl2S_imm(vecS dst, immI8 shift) %{
16308   predicate(UseAVX == 0 && n->as_Vector()->length() == 2);
16309   match(Set dst (URShiftVS dst shift));
16310   format %{ "psrlw   $dst,$shift\t! logical right shift packed2S" %}
16311   ins_encode %{
16312     __ psrlw($dst$$XMMRegister, (int)$shift$$constant);
16313   %}
16314   ins_pipe( pipe_slow );
16315 %}
16316 
16317 instruct vsrl2S_reg_avx(vecS dst, vecS src, vecS shift) %{
16318   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 2);
16319   match(Set dst (URShiftVS src shift));
16320   format %{ "vpsrlw  $dst,$src,$shift\t! logical right shift packed2S" %}
16321   ins_encode %{
16322     int vector_len = 0;
16323     __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16324   %}
16325   ins_pipe( pipe_slow );
16326 %}
16327 
16328 instruct vsrl2S_reg_evex(vecS dst, vecS src, vecS shift) %{
16329   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 2);
16330   match(Set dst (URShiftVS src shift));
16331   format %{ "vpsrlw  $dst,$src,$shift\t! logical right shift packed2S" %}
16332   ins_encode %{
16333     int vector_len = 0;
16334     __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16335   %}
16336   ins_pipe( pipe_slow );
16337 %}
16338 
16339 instruct vsrl2S_reg_evex_special(vecS dst, vecS src, vecS shift) %{
16340   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 2);
16341   match(Set dst (URShiftVS dst shift));
16342   effect(TEMP src);
16343   format %{ "vpsrlw  $dst,$src,$shift\t! logical right shift packed2S" %}
16344   ins_encode %{
16345     int vector_len = 0;
16346     __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16347   %}
16348   ins_pipe( pipe_slow );
16349 %}
16350 
16351 instruct vsrl2S_reg_imm_avx(vecS dst, vecS src, immI8 shift) %{
16352   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 2);
16353   match(Set dst (URShiftVS src shift));
16354   format %{ "vpsrlw  $dst,$src,$shift\t! logical right shift packed2S" %}
16355   ins_encode %{
16356     int vector_len = 0;
16357     __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16358   %}
16359   ins_pipe( pipe_slow );
16360 %}
16361 
16362 instruct vsrl2S_reg_imm_evex(vecS dst, vecS src, immI8 shift) %{
16363   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 2);
16364   match(Set dst (URShiftVS src shift));
16365   format %{ "vpsrlw  $dst,$src,$shift\t! logical right shift packed2S" %}
16366   ins_encode %{
16367     int vector_len = 0;
16368     __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16369   %}
16370   ins_pipe( pipe_slow );
16371 %}
16372 
16373 instruct vsrl2S_reg_imm_evex_special(vecS dst, vecS src, immI8 shift) %{
16374   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 2);
16375   match(Set dst (URShiftVS dst shift));
16376   effect(TEMP src);
16377   format %{ "vpsrlw  $dst,$src,$shift\t! logical right shift packed2S" %}
16378   ins_encode %{
16379     int vector_len = 0;
16380     __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16381   %}
16382   ins_pipe( pipe_slow );
16383 %}
16384 
16385 instruct vsrl4S(vecD dst, vecS shift) %{
16386   predicate(UseAVX == 0 && n->as_Vector()->length() == 4);
16387   match(Set dst (URShiftVS dst shift));
16388   format %{ "psrlw   $dst,$shift\t! logical right shift packed4S" %}
16389   ins_encode %{
16390     __ psrlw($dst$$XMMRegister, $shift$$XMMRegister);
16391   %}
16392   ins_pipe( pipe_slow );
16393 %}
16394 
16395 instruct vsrl4S_imm(vecD dst, immI8 shift) %{
16396   predicate(UseAVX == 0 && n->as_Vector()->length() == 4);
16397   match(Set dst (URShiftVS dst shift));
16398   format %{ "psrlw   $dst,$shift\t! logical right shift packed4S" %}
16399   ins_encode %{
16400     __ psrlw($dst$$XMMRegister, (int)$shift$$constant);
16401   %}
16402   ins_pipe( pipe_slow );
16403 %}
16404 
16405 instruct vsrl4S_reg_avx(vecD dst, vecD src, vecS shift) %{
16406   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 4);
16407   match(Set dst (URShiftVS src shift));
16408   format %{ "vpsrlw  $dst,$src,$shift\t! logical right shift packed4S" %}
16409   ins_encode %{
16410     int vector_len = 0;
16411     __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16412   %}
16413   ins_pipe( pipe_slow );
16414 %}
16415 
16416 instruct vsrl4S_reg_evex(vecD dst, vecD src, vecS shift) %{
16417   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 4);
16418   match(Set dst (URShiftVS src shift));
16419   format %{ "vpsrlw  $dst,$src,$shift\t! logical right shift packed4S" %}
16420   ins_encode %{
16421     int vector_len = 0;
16422     __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16423   %}
16424   ins_pipe( pipe_slow );
16425 %}
16426 
16427 instruct vsrl4S_reg_evex_special(vecD dst, vecD src, vecS shift) %{
16428   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 4);
16429   match(Set dst (URShiftVS dst shift));
16430   effect(TEMP src);
16431   format %{ "vpsrlw  $dst,$src,$shift\t! logical right shift packed4S" %}
16432   ins_encode %{
16433     int vector_len = 0;
16434     __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16435   %}
16436   ins_pipe( pipe_slow );
16437 %}
16438 
16439 instruct vsrl4S_reg_imm_avx(vecD dst, vecD src, immI8 shift) %{
16440   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 4);
16441   match(Set dst (URShiftVS src shift));
16442   format %{ "vpsrlw  $dst,$src,$shift\t! logical right shift packed4S" %}
16443   ins_encode %{
16444     int vector_len = 0;
16445     __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16446   %}
16447   ins_pipe( pipe_slow );
16448 %}
16449 
16450 instruct vsrl4S_reg_imm_evex(vecD dst, vecD src, immI8 shift) %{
16451   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 4);
16452   match(Set dst (URShiftVS src shift));
16453   format %{ "vpsrlw  $dst,$src,$shift\t! logical right shift packed4S" %}
16454   ins_encode %{
16455     int vector_len = 0;
16456     __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16457   %}
16458   ins_pipe( pipe_slow );
16459 %}
16460 
16461 instruct vsrl4S_reg_imm_evex_special(vecD dst, vecD src, immI8 shift) %{
16462   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 4);
16463   match(Set dst (URShiftVS dst shift));
16464   effect(TEMP src);
16465   format %{ "vpsrlw  $dst,$src,$shift\t! logical right shift packed4S" %}
16466   ins_encode %{
16467     int vector_len = 0;
16468     __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16469   %}
16470   ins_pipe( pipe_slow );
16471 %}
16472 
16473 instruct vsrl8S(vecX dst, vecS shift) %{
16474   predicate(UseAVX == 0 && n->as_Vector()->length() == 8);
16475   match(Set dst (URShiftVS dst shift));
16476   format %{ "psrlw   $dst,$shift\t! logical right shift packed8S" %}
16477   ins_encode %{
16478     __ psrlw($dst$$XMMRegister, $shift$$XMMRegister);
16479   %}
16480   ins_pipe( pipe_slow );
16481 %}
16482 
16483 instruct vsrl8S_imm(vecX dst, immI8 shift) %{
16484   predicate(UseAVX == 0 && n->as_Vector()->length() == 8);
16485   match(Set dst (URShiftVS dst shift));
16486   format %{ "psrlw   $dst,$shift\t! logical right shift packed8S" %}
16487   ins_encode %{
16488     __ psrlw($dst$$XMMRegister, (int)$shift$$constant);
16489   %}
16490   ins_pipe( pipe_slow );
16491 %}
16492 
16493 instruct vsrl8S_reg_avx(vecX dst, vecX src, vecS shift) %{
16494   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 8);
16495   match(Set dst (URShiftVS src shift));
16496   format %{ "vpsrlw  $dst,$src,$shift\t! logical right shift packed8S" %}
16497   ins_encode %{
16498     int vector_len = 0;
16499     __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16500   %}
16501   ins_pipe( pipe_slow );
16502 %}
16503 
16504 instruct vsrl8S_reg_evex(vecX dst, vecX src, vecS shift) %{
16505   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 8);
16506   match(Set dst (URShiftVS src shift));
16507   format %{ "vpsrlw  $dst,$src,$shift\t! logical right shift packed8S" %}
16508   ins_encode %{
16509     int vector_len = 0;
16510     __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16511   %}
16512   ins_pipe( pipe_slow );
16513 %}
16514 
16515 instruct vsrl8S_reg_evex_special(vecX dst, vecX src, vecS shift) %{
16516   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 8);
16517   match(Set dst (URShiftVS dst shift));
16518   effect(TEMP src);
16519   format %{ "vpsrlw  $dst,$src,$shift\t! logical right shift packed8S" %}
16520   ins_encode %{
16521     int vector_len = 0;
16522     __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16523   %}
16524   ins_pipe( pipe_slow );
16525 %}
16526 
16527 instruct vsrl8S_reg_imm_avx(vecX dst, vecX src, immI8 shift) %{
16528   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 8);
16529   match(Set dst (URShiftVS src shift));
16530   format %{ "vpsrlw  $dst,$src,$shift\t! logical right shift packed8S" %}
16531   ins_encode %{
16532     int vector_len = 0;
16533     __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16534   %}
16535   ins_pipe( pipe_slow );
16536 %}
16537 
16538 instruct vsrl8S_reg_imm_evex(vecX dst, vecX src, immI8 shift) %{
16539   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 8);
16540   match(Set dst (URShiftVS src shift));
16541   format %{ "vpsrlw  $dst,$src,$shift\t! logical right shift packed8S" %}
16542   ins_encode %{
16543     int vector_len = 0;
16544     __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16545   %}
16546   ins_pipe( pipe_slow );
16547 %}
16548 
16549 instruct vsrl8S_reg_imm_evex_special(vecX dst, vecX src, immI8 shift) %{
16550   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 8);
16551   match(Set dst (URShiftVS dst shift));
16552   effect(TEMP src);
16553   format %{ "vpsrlw  $dst,$src,$shift\t! logical right shift packed8S" %}
16554   ins_encode %{
16555     int vector_len = 0;
16556     __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16557   %}
16558   ins_pipe( pipe_slow );
16559 %}
16560 
16561 instruct vsrl16S_reg_avx(vecY dst, vecY src, vecS shift) %{
16562   predicate(VM_Version::supports_avx256only() && n->as_Vector()->length() == 16);
16563   match(Set dst (URShiftVS src shift));
16564   format %{ "vpsrlw  $dst,$src,$shift\t! logical right shift packed16S" %}
16565   ins_encode %{
16566     int vector_len = 1;
16567     __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16568   %}
16569   ins_pipe( pipe_slow );
16570 %}
16571 
16572 instruct vsrl16S_reg_evex(vecY dst, vecY src, vecS shift) %{
16573   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 16);
16574   match(Set dst (URShiftVS src shift));
16575   format %{ "vpsrlw  $dst,$src,$shift\t! logical right shift packed16S" %}
16576   ins_encode %{
16577     int vector_len = 1;
16578     __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16579   %}
16580   ins_pipe( pipe_slow );
16581 %}
16582 
16583 instruct vsrl16S_reg_evex_special(vecY dst, vecY src, vecS shift) %{
16584   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 16);
16585   match(Set dst (URShiftVS dst shift));
16586   effect(TEMP src);
16587   format %{ "vpsrlw  $dst,$src,$shift\t! logical right shift packed16S" %}
16588   ins_encode %{
16589     int vector_len = 1;
16590     __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16591   %}
16592   ins_pipe( pipe_slow );
16593 %}
16594 
16595 instruct vsrl16S_reg_imm_avx(vecY dst, vecY src, immI8 shift) %{
16596   predicate(VM_Version::supports_avx256only() && n->as_Vector()->length() == 16);
16597   match(Set dst (URShiftVS src shift));
16598   format %{ "vpsrlw  $dst,$src,$shift\t! logical right shift packed16S" %}
16599   ins_encode %{
16600     int vector_len = 1;
16601     __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16602   %}
16603   ins_pipe( pipe_slow );
16604 %}
16605 
16606 instruct vsrl16S_reg_imm_evex(vecY dst, vecY src, immI8 shift) %{
16607   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 16);
16608   match(Set dst (URShiftVS src shift));
16609   format %{ "vpsrlw  $dst,$src,$shift\t! logical right shift packed16S" %}
16610   ins_encode %{
16611     int vector_len = 1;
16612     __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16613   %}
16614   ins_pipe( pipe_slow );
16615 %}
16616 
16617 instruct vsrl16S_reg_imm_evex_special(vecY dst, vecY src, immI8 shift) %{
16618   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 16);
16619   match(Set dst (URShiftVS dst shift));
16620   effect(TEMP src);
16621   format %{ "vpsrlw  $dst,$src,$shift\t! logical right shift packed16S" %}
16622   ins_encode %{
16623     int vector_len = 1;
16624     __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16625   %}
16626   ins_pipe( pipe_slow );
16627 %}
16628 
16629 instruct vsrl32S_reg(vecZ dst, vecZ src, vecS shift) %{
16630   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 32);
16631   match(Set dst (URShiftVS src shift));
16632   format %{ "vpsrlw  $dst,$src,$shift\t! logical right shift packed32S" %}
16633   ins_encode %{
16634     int vector_len = 2;
16635     __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16636   %}
16637   ins_pipe( pipe_slow );
16638 %}
16639 
16640 instruct vsrl32S_reg_imm(vecZ dst, vecZ src, immI8 shift) %{
16641   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 32);
16642   match(Set dst (URShiftVS src shift));
16643   format %{ "vpsrlw  $dst,$src,$shift\t! logical right shift packed32S" %}
16644   ins_encode %{
16645     int vector_len = 2;
16646     __ vpsrlw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16647   %}
16648   ins_pipe( pipe_slow );
16649 %}
16650 
16651 // Integers vector logical right shift
16652 instruct vsrl2I(vecD dst, vecS shift) %{
16653   predicate(n->as_Vector()->length() == 2);
16654   match(Set dst (URShiftVI dst shift));
16655   format %{ "psrld   $dst,$shift\t! logical right shift packed2I" %}
16656   ins_encode %{
16657     __ psrld($dst$$XMMRegister, $shift$$XMMRegister);
16658   %}
16659   ins_pipe( pipe_slow );
16660 %}
16661 
16662 instruct vsrl2I_imm(vecD dst, immI8 shift) %{
16663   predicate(n->as_Vector()->length() == 2);
16664   match(Set dst (URShiftVI dst shift));
16665   format %{ "psrld   $dst,$shift\t! logical right shift packed2I" %}
16666   ins_encode %{
16667     __ psrld($dst$$XMMRegister, (int)$shift$$constant);
16668   %}
16669   ins_pipe( pipe_slow );
16670 %}
16671 
16672 instruct vsrl2I_reg(vecD dst, vecD src, vecS shift) %{
16673   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
16674   match(Set dst (URShiftVI src shift));
16675   format %{ "vpsrld  $dst,$src,$shift\t! logical right shift packed2I" %}
16676   ins_encode %{
16677     int vector_len = 0;
16678     __ vpsrld($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16679   %}
16680   ins_pipe( pipe_slow );
16681 %}
16682 
16683 instruct vsrl2I_reg_imm(vecD dst, vecD src, immI8 shift) %{
16684   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
16685   match(Set dst (URShiftVI src shift));
16686   format %{ "vpsrld  $dst,$src,$shift\t! logical right shift packed2I" %}
16687   ins_encode %{
16688     int vector_len = 0;
16689     __ vpsrld($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16690   %}
16691   ins_pipe( pipe_slow );
16692 %}
16693 
16694 instruct vsrl4I(vecX dst, vecS shift) %{
16695   predicate(n->as_Vector()->length() == 4);
16696   match(Set dst (URShiftVI dst shift));
16697   format %{ "psrld   $dst,$shift\t! logical right shift packed4I" %}
16698   ins_encode %{
16699     __ psrld($dst$$XMMRegister, $shift$$XMMRegister);
16700   %}
16701   ins_pipe( pipe_slow );
16702 %}
16703 
16704 instruct vsrl4I_imm(vecX dst, immI8 shift) %{
16705   predicate(n->as_Vector()->length() == 4);
16706   match(Set dst (URShiftVI dst shift));
16707   format %{ "psrld   $dst,$shift\t! logical right shift packed4I" %}
16708   ins_encode %{
16709     __ psrld($dst$$XMMRegister, (int)$shift$$constant);
16710   %}
16711   ins_pipe( pipe_slow );
16712 %}
16713 
16714 instruct vsrl4I_reg(vecX dst, vecX src, vecS shift) %{
16715   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
16716   match(Set dst (URShiftVI src shift));
16717   format %{ "vpsrld  $dst,$src,$shift\t! logical right shift packed4I" %}
16718   ins_encode %{
16719     int vector_len = 0;
16720     __ vpsrld($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16721   %}
16722   ins_pipe( pipe_slow );
16723 %}
16724 
16725 instruct vsrl4I_reg_imm(vecX dst, vecX src, immI8 shift) %{
16726   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
16727   match(Set dst (URShiftVI src shift));
16728   format %{ "vpsrld  $dst,$src,$shift\t! logical right shift packed4I" %}
16729   ins_encode %{
16730     int vector_len = 0;
16731     __ vpsrld($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16732   %}
16733   ins_pipe( pipe_slow );
16734 %}
16735 
16736 instruct vsrl8I_reg(vecY dst, vecY src, vecS shift) %{
16737   predicate(UseAVX > 1 && n->as_Vector()->length() == 8);
16738   match(Set dst (URShiftVI src shift));
16739   format %{ "vpsrld  $dst,$src,$shift\t! logical right shift packed8I" %}
16740   ins_encode %{
16741     int vector_len = 1;
16742     __ vpsrld($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16743   %}
16744   ins_pipe( pipe_slow );
16745 %}
16746 
16747 instruct vsrl8I_reg_imm(vecY dst, vecY src, immI8 shift) %{
16748   predicate(UseAVX > 1 && n->as_Vector()->length() == 8);
16749   match(Set dst (URShiftVI src shift));
16750   format %{ "vpsrld  $dst,$src,$shift\t! logical right shift packed8I" %}
16751   ins_encode %{
16752     int vector_len = 1;
16753     __ vpsrld($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16754   %}
16755   ins_pipe( pipe_slow );
16756 %}
16757 
16758 instruct vsrl16I_reg(vecZ dst, vecZ src, vecS shift) %{
16759   predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
16760   match(Set dst (URShiftVI src shift));
16761   format %{ "vpsrld  $dst,$src,$shift\t! logical right shift packed16I" %}
16762   ins_encode %{
16763     int vector_len = 2;
16764     __ vpsrld($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16765   %}
16766   ins_pipe( pipe_slow );
16767 %}
16768 
16769 instruct vsrl16I_reg_imm(vecZ dst, vecZ src, immI8 shift) %{
16770   predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
16771   match(Set dst (URShiftVI src shift));
16772   format %{ "vpsrld  $dst,$src,$shift\t! logical right shift packed16I" %}
16773   ins_encode %{
16774     int vector_len = 2;
16775     __ vpsrld($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16776   %}
16777   ins_pipe( pipe_slow );
16778 %}
16779 
16780 // Longs vector logical right shift
16781 instruct vsrl2L(vecX dst, vecS shift) %{
16782   predicate(n->as_Vector()->length() == 2);
16783   match(Set dst (URShiftVL dst shift));
16784   format %{ "psrlq   $dst,$shift\t! logical right shift packed2L" %}
16785   ins_encode %{
16786     __ psrlq($dst$$XMMRegister, $shift$$XMMRegister);
16787   %}
16788   ins_pipe( pipe_slow );
16789 %}
16790 
16791 instruct vsrl2L_imm(vecX dst, immI8 shift) %{
16792   predicate(n->as_Vector()->length() == 2);
16793   match(Set dst (URShiftVL dst shift));
16794   format %{ "psrlq   $dst,$shift\t! logical right shift packed2L" %}
16795   ins_encode %{
16796     __ psrlq($dst$$XMMRegister, (int)$shift$$constant);
16797   %}
16798   ins_pipe( pipe_slow );
16799 %}
16800 
16801 instruct vsrl2L_reg(vecX dst, vecX src, vecS shift) %{
16802   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
16803   match(Set dst (URShiftVL src shift));
16804   format %{ "vpsrlq  $dst,$src,$shift\t! logical right shift packed2L" %}
16805   ins_encode %{
16806     int vector_len = 0;
16807     __ vpsrlq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16808   %}
16809   ins_pipe( pipe_slow );
16810 %}
16811 
16812 instruct vsrl2L_reg_imm(vecX dst, vecX src, immI8 shift) %{
16813   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
16814   match(Set dst (URShiftVL src shift));
16815   format %{ "vpsrlq  $dst,$src,$shift\t! logical right shift packed2L" %}
16816   ins_encode %{
16817     int vector_len = 0;
16818     __ vpsrlq($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16819   %}
16820   ins_pipe( pipe_slow );
16821 %}
16822 
16823 instruct vsrl4L_reg(vecY dst, vecY src, vecS shift) %{
16824   predicate(UseAVX > 1 && n->as_Vector()->length() == 4);
16825   match(Set dst (URShiftVL src shift));
16826   format %{ "vpsrlq  $dst,$src,$shift\t! logical right shift packed4L" %}
16827   ins_encode %{
16828     int vector_len = 1;
16829     __ vpsrlq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16830   %}
16831   ins_pipe( pipe_slow );
16832 %}
16833 
16834 instruct vsrl4L_reg_imm(vecY dst, vecY src, immI8 shift) %{
16835   predicate(UseAVX > 1 && n->as_Vector()->length() == 4);
16836   match(Set dst (URShiftVL src shift));
16837   format %{ "vpsrlq  $dst,$src,$shift\t! logical right shift packed4L" %}
16838   ins_encode %{
16839     int vector_len = 1;
16840     __ vpsrlq($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16841   %}
16842   ins_pipe( pipe_slow );
16843 %}
16844 
16845 instruct vsrl8L_reg(vecZ dst, vecZ src, vecS shift) %{
16846   predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
16847   match(Set dst (URShiftVL src shift));
16848   format %{ "vpsrlq  $dst,$src,$shift\t! logical right shift packed8L" %}
16849   ins_encode %{
16850     int vector_len = 2;
16851     __ vpsrlq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16852   %}
16853   ins_pipe( pipe_slow );
16854 %}
16855 
16856 instruct vsrl8L_reg_imm(vecZ dst, vecZ src, immI8 shift) %{
16857   predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
16858   match(Set dst (URShiftVL src shift));
16859   format %{ "vpsrlq  $dst,$src,$shift\t! logical right shift packed8L" %}
16860   ins_encode %{
16861     int vector_len = 2;
16862     __ vpsrlq($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16863   %}
16864   ins_pipe( pipe_slow );
16865 %}
16866 
16867 // ------------------- ArithmeticRightShift -----------------------------------
16868 
16869 // Shorts/Chars vector arithmetic right shift
16870 instruct vsra2S(vecS dst, vecS shift) %{
16871   predicate(UseAVX == 0 && n->as_Vector()->length() == 2);
16872   match(Set dst (RShiftVS dst shift));
16873   format %{ "psraw   $dst,$shift\t! arithmetic right shift packed2S" %}
16874   ins_encode %{
16875     __ psraw($dst$$XMMRegister, $shift$$XMMRegister);
16876   %}
16877   ins_pipe( pipe_slow );
16878 %}
16879 
16880 instruct vsra2S_imm(vecS dst, immI8 shift) %{
16881   predicate(n->as_Vector()->length() == 2);
16882   match(Set dst (RShiftVS dst shift));
16883   format %{ "psraw   $dst,$shift\t! arithmetic right shift packed2S" %}
16884   ins_encode %{
16885     __ psraw($dst$$XMMRegister, (int)$shift$$constant);
16886   %}
16887   ins_pipe( pipe_slow );
16888 %}
16889 
16890 instruct vsra2S_reg_avx(vecS dst, vecS src, vecS shift) %{
16891   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 2);
16892   match(Set dst (RShiftVS src shift));
16893   format %{ "vpsraw  $dst,$src,$shift\t! arithmetic right shift packed2S" %}
16894   ins_encode %{
16895     int vector_len = 0;
16896     __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16897   %}
16898   ins_pipe( pipe_slow );
16899 %}
16900 
16901 instruct vsra2S_reg_evex(vecS dst, vecS src, vecS shift) %{
16902   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 2);
16903   match(Set dst (RShiftVS src shift));
16904   format %{ "vpsraw  $dst,$src,$shift\t! arithmetic right shift packed2S" %}
16905   ins_encode %{
16906     int vector_len = 0;
16907     __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16908   %}
16909   ins_pipe( pipe_slow );
16910 %}
16911 
16912 instruct vsra2S_reg_evex_special(vecS dst, vecS src, vecS shift) %{
16913   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 2);
16914   match(Set dst (RShiftVS dst shift));
16915   effect(TEMP src);
16916   format %{ "vpsraw  $dst,$src,$shift\t! arithmetic right shift packed2S" %}
16917   ins_encode %{
16918     int vector_len = 0;
16919     __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16920   %}
16921   ins_pipe( pipe_slow );
16922 %}
16923 
16924 instruct vsra2S_reg_imm_avx(vecS dst, vecS src, immI8 shift) %{
16925   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 2);
16926   match(Set dst (RShiftVS src shift));
16927   format %{ "vpsraw  $dst,$src,$shift\t! arithmetic right shift packed2S" %}
16928   ins_encode %{
16929     int vector_len = 0;
16930     __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16931   %}
16932   ins_pipe( pipe_slow );
16933 %}
16934 
16935 instruct vsra2S_reg_imm_evex(vecS dst, vecS src, immI8 shift) %{
16936   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 2);
16937   match(Set dst (RShiftVS src shift));
16938   format %{ "vpsraw  $dst,$src,$shift\t! arithmetic right shift packed2S" %}
16939   ins_encode %{
16940     int vector_len = 0;
16941     __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16942   %}
16943   ins_pipe( pipe_slow );
16944 %}
16945 
16946 instruct vsra2S_reg_imm_evex_special(vecS dst, vecS src, immI8 shift) %{
16947   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 2);
16948   match(Set dst (RShiftVS dst shift));
16949   effect(TEMP src);
16950   format %{ "vpsraw  $dst,$src,$shift\t! arithmetic right shift packed2S" %}
16951   ins_encode %{
16952     int vector_len = 0;
16953     __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
16954   %}
16955   ins_pipe( pipe_slow );
16956 %}
16957 
16958 instruct vsra4S(vecD dst, vecS shift) %{
16959   predicate(UseAVX == 0 && n->as_Vector()->length() == 4);
16960   match(Set dst (RShiftVS dst shift));
16961   format %{ "psraw   $dst,$shift\t! arithmetic right shift packed4S" %}
16962   ins_encode %{
16963     __ psraw($dst$$XMMRegister, $shift$$XMMRegister);
16964   %}
16965   ins_pipe( pipe_slow );
16966 %}
16967 
16968 instruct vsra4S_imm(vecD dst, immI8 shift) %{
16969   predicate(UseAVX == 0 && n->as_Vector()->length() == 4);
16970   match(Set dst (RShiftVS dst shift));
16971   format %{ "psraw   $dst,$shift\t! arithmetic right shift packed4S" %}
16972   ins_encode %{
16973     __ psraw($dst$$XMMRegister, (int)$shift$$constant);
16974   %}
16975   ins_pipe( pipe_slow );
16976 %}
16977 
16978 instruct vsra4S_reg_avx(vecD dst, vecD src, vecS shift) %{
16979   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 4);
16980   match(Set dst (RShiftVS src shift));
16981   format %{ "vpsraw  $dst,$src,$shift\t! arithmetic right shift packed4S" %}
16982   ins_encode %{
16983     int vector_len = 0;
16984     __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16985   %}
16986   ins_pipe( pipe_slow );
16987 %}
16988 
16989 instruct vsra4S_reg_evex(vecD dst, vecD src, vecS shift) %{
16990   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 4);
16991   match(Set dst (RShiftVS src shift));
16992   format %{ "vpsraw  $dst,$src,$shift\t! arithmetic right shift packed4S" %}
16993   ins_encode %{
16994     int vector_len = 0;
16995     __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
16996   %}
16997   ins_pipe( pipe_slow );
16998 %}
16999 
17000 instruct vsra4S_reg_evex_special(vecD dst, vecD src, vecS shift) %{
17001   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 4);
17002   match(Set dst (RShiftVS dst shift));
17003   effect(TEMP src);
17004   format %{ "vpsraw  $dst,$src,$shift\t! arithmetic right shift packed4S" %}
17005   ins_encode %{
17006     int vector_len = 0;
17007     __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17008   %}
17009   ins_pipe( pipe_slow );
17010 %}
17011 
17012 instruct vsra4S_reg_imm_avx(vecD dst, vecD src, immI8 shift) %{
17013   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 4);
17014   match(Set dst (RShiftVS src shift));
17015   format %{ "vpsraw  $dst,$src,$shift\t! arithmetic right shift packed4S" %}
17016   ins_encode %{
17017     int vector_len = 0;
17018     __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
17019   %}
17020   ins_pipe( pipe_slow );
17021 %}
17022 
17023 instruct vsra4S_reg_imm_evex(vecD dst, vecD src, immI8 shift) %{
17024   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 4);
17025   match(Set dst (RShiftVS src shift));
17026   format %{ "vpsraw  $dst,$src,$shift\t! arithmetic right shift packed4S" %}
17027   ins_encode %{
17028     int vector_len = 0;
17029     __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
17030   %}
17031   ins_pipe( pipe_slow );
17032 %}
17033 
17034 instruct vsra4S_reg_imm_evex_special(vecD dst, vecD src, immI8 shift) %{
17035   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 4);
17036   match(Set dst (RShiftVS dst shift));
17037   effect(TEMP src);
17038   format %{ "vpsraw  $dst,$src,$shift\t! arithmetic right shift packed4S" %}
17039   ins_encode %{
17040     int vector_len = 0;
17041     __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
17042   %}
17043   ins_pipe( pipe_slow );
17044 %}
17045 
17046 instruct vsra8S(vecX dst, vecS shift) %{
17047   predicate(UseAVX == 0 && n->as_Vector()->length() == 8);
17048   match(Set dst (RShiftVS dst shift));
17049   format %{ "psraw   $dst,$shift\t! arithmetic right shift packed8S" %}
17050   ins_encode %{
17051     __ psraw($dst$$XMMRegister, $shift$$XMMRegister);
17052   %}
17053   ins_pipe( pipe_slow );
17054 %}
17055 
17056 instruct vsra8S_imm(vecX dst, immI8 shift) %{
17057   predicate(UseAVX == 0 && n->as_Vector()->length() == 8);
17058   match(Set dst (RShiftVS dst shift));
17059   format %{ "psraw   $dst,$shift\t! arithmetic right shift packed8S" %}
17060   ins_encode %{
17061     __ psraw($dst$$XMMRegister, (int)$shift$$constant);
17062   %}
17063   ins_pipe( pipe_slow );
17064 %}
17065 
17066 instruct vsra8S_reg_avx(vecX dst, vecX src, vecS shift) %{
17067   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 8);
17068   match(Set dst (RShiftVS src shift));
17069   format %{ "vpsraw  $dst,$src,$shift\t! arithmetic right shift packed8S" %}
17070   ins_encode %{
17071     int vector_len = 0;
17072     __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17073   %}
17074   ins_pipe( pipe_slow );
17075 %}
17076 
17077 instruct vsra8S_reg_evex(vecX dst, vecX src, vecS shift) %{
17078   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 8);
17079   match(Set dst (RShiftVS src shift));
17080   format %{ "vpsraw  $dst,$src,$shift\t! arithmetic right shift packed8S" %}
17081   ins_encode %{
17082     int vector_len = 0;
17083     __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17084   %}
17085   ins_pipe( pipe_slow );
17086 %}
17087 
17088 instruct vsra8S_reg_evex_special(vecX dst, vecX src, vecS shift) %{
17089   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 8);
17090   match(Set dst (RShiftVS dst shift));
17091   effect(TEMP src);
17092   format %{ "vpsraw  $dst,$src,$shift\t! arithmetic right shift packed8S" %}
17093   ins_encode %{
17094     int vector_len = 0;
17095     __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17096   %}
17097   ins_pipe( pipe_slow );
17098 %}
17099 
17100 instruct vsra8S_reg_imm_avx(vecX dst, vecX src, immI8 shift) %{
17101   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 8);
17102   match(Set dst (RShiftVS src shift));
17103   format %{ "vpsraw  $dst,$src,$shift\t! arithmetic right shift packed8S" %}
17104   ins_encode %{
17105     int vector_len = 0;
17106     __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
17107   %}
17108   ins_pipe( pipe_slow );
17109 %}
17110 
17111 instruct vsra8S_reg_imm_evex(vecX dst, vecX src, immI8 shift) %{
17112   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 8);
17113   match(Set dst (RShiftVS src shift));
17114   format %{ "vpsraw  $dst,$src,$shift\t! arithmetic right shift packed8S" %}
17115   ins_encode %{
17116     int vector_len = 0;
17117     __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
17118   %}
17119   ins_pipe( pipe_slow );
17120 %}
17121 
17122 instruct vsra8S_reg_imm_evex_special(vecX dst, vecX src, immI8 shift) %{
17123   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 8);
17124   match(Set dst (RShiftVS dst shift));
17125   effect(TEMP src);
17126   format %{ "vpsraw  $dst,$src,$shift\t! arithmetic right shift packed8S" %}
17127   ins_encode %{
17128     int vector_len = 0;
17129     __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
17130   %}
17131   ins_pipe( pipe_slow );
17132 %}
17133 
17134 instruct vsra16S_reg_avx(vecY dst, vecY src, vecS shift) %{
17135   predicate(VM_Version::supports_avx256only() && n->as_Vector()->length() == 16);
17136   match(Set dst (RShiftVS src shift));
17137   format %{ "vpsraw  $dst,$src,$shift\t! arithmetic right shift packed16S" %}
17138   ins_encode %{
17139     int vector_len = 1;
17140     __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17141   %}
17142   ins_pipe( pipe_slow );
17143 %}
17144 
17145 instruct vsra16S_reg_evex(vecY dst, vecY src, vecS shift) %{
17146   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 16);
17147   match(Set dst (RShiftVS src shift));
17148   format %{ "vpsraw  $dst,$src,$shift\t! arithmetic right shift packed16S" %}
17149   ins_encode %{
17150     int vector_len = 1;
17151     __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17152   %}
17153   ins_pipe( pipe_slow );
17154 %}
17155 
17156 instruct vsra16S_reg_evex_special(vecY dst, vecY src, vecS shift) %{
17157   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 16);
17158   match(Set dst (RShiftVS dst shift));
17159   effect(TEMP src);
17160   format %{ "vpsraw  $dst,$src,$shift\t! arithmetic right shift packed16S" %}
17161   ins_encode %{
17162     int vector_len = 1;
17163     __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17164   %}
17165   ins_pipe( pipe_slow );
17166 %}
17167 
17168 instruct vsra16S_reg_imm_avx(vecY dst, vecY src, immI8 shift) %{
17169   predicate(VM_Version::supports_avxonly() && n->as_Vector()->length() == 16);
17170   match(Set dst (RShiftVS src shift));
17171   format %{ "vpsraw  $dst,$src,$shift\t! arithmetic right shift packed16S" %}
17172   ins_encode %{
17173     int vector_len = 1;
17174     __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
17175   %}
17176   ins_pipe( pipe_slow );
17177 %}
17178 
17179 instruct vsra16S_reg_imm_evex(vecY dst, vecY src, immI8 shift) %{
17180   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 16);
17181   match(Set dst (RShiftVS src shift));
17182   format %{ "vpsraw  $dst,$src,$shift\t! arithmetic right shift packed16S" %}
17183   ins_encode %{
17184     int vector_len = 1;
17185     __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
17186   %}
17187   ins_pipe( pipe_slow );
17188 %}
17189 
17190 instruct vsra16S_reg_imm_evex_special(vecY dst, vecY src, immI8 shift) %{
17191   predicate(VM_Version::supports_avx512nobw() && n->as_Vector()->length() == 16);
17192   match(Set dst (RShiftVS dst shift));
17193   effect(TEMP src);
17194   format %{ "vpsraw  $dst,$src,$shift\t! arithmetic right shift packed16S" %}
17195   ins_encode %{
17196     int vector_len = 1;
17197     __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
17198   %}
17199   ins_pipe( pipe_slow );
17200 %}
17201 
17202 instruct vsra32S_reg(vecZ dst, vecZ src, vecS shift) %{
17203   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 32);
17204   match(Set dst (RShiftVS src shift));
17205   format %{ "vpsraw  $dst,$src,$shift\t! arithmetic right shift packed32S" %}
17206   ins_encode %{
17207     int vector_len = 2;
17208     __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17209   %}
17210   ins_pipe( pipe_slow );
17211 %}
17212 
17213 instruct vsra32S_reg_imm(vecZ dst, vecZ src, immI8 shift) %{
17214   predicate(VM_Version::supports_avx512bw() && n->as_Vector()->length() == 32);
17215   match(Set dst (RShiftVS src shift));
17216   format %{ "vpsraw  $dst,$src,$shift\t! arithmetic right shift packed32S" %}
17217   ins_encode %{
17218     int vector_len = 2;
17219     __ vpsraw($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
17220   %}
17221   ins_pipe( pipe_slow );
17222 %}
17223 
17224 // Integers vector arithmetic right shift
17225 instruct vsra2I(vecD dst, vecS shift) %{
17226   predicate(n->as_Vector()->length() == 2);
17227   match(Set dst (RShiftVI dst shift));
17228   format %{ "psrad   $dst,$shift\t! arithmetic right shift packed2I" %}
17229   ins_encode %{
17230     __ psrad($dst$$XMMRegister, $shift$$XMMRegister);
17231   %}
17232   ins_pipe( pipe_slow );
17233 %}
17234 
17235 instruct vsra2I_imm(vecD dst, immI8 shift) %{
17236   predicate(n->as_Vector()->length() == 2);
17237   match(Set dst (RShiftVI dst shift));
17238   format %{ "psrad   $dst,$shift\t! arithmetic right shift packed2I" %}
17239   ins_encode %{
17240     __ psrad($dst$$XMMRegister, (int)$shift$$constant);
17241   %}
17242   ins_pipe( pipe_slow );
17243 %}
17244 
17245 instruct vsra2I_reg(vecD dst, vecD src, vecS shift) %{
17246   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
17247   match(Set dst (RShiftVI src shift));
17248   format %{ "vpsrad  $dst,$src,$shift\t! arithmetic right shift packed2I" %}
17249   ins_encode %{
17250     int vector_len = 0;
17251     __ vpsrad($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17252   %}
17253   ins_pipe( pipe_slow );
17254 %}
17255 
17256 instruct vsra2I_reg_imm(vecD dst, vecD src, immI8 shift) %{
17257   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
17258   match(Set dst (RShiftVI src shift));
17259   format %{ "vpsrad  $dst,$src,$shift\t! arithmetic right shift packed2I" %}
17260   ins_encode %{
17261     int vector_len = 0;
17262     __ vpsrad($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
17263   %}
17264   ins_pipe( pipe_slow );
17265 %}
17266 
17267 instruct vsra4I(vecX dst, vecS shift) %{
17268   predicate(n->as_Vector()->length() == 4);
17269   match(Set dst (RShiftVI dst shift));
17270   format %{ "psrad   $dst,$shift\t! arithmetic right shift packed4I" %}
17271   ins_encode %{
17272     __ psrad($dst$$XMMRegister, $shift$$XMMRegister);
17273   %}
17274   ins_pipe( pipe_slow );
17275 %}
17276 
17277 instruct vsra4I_imm(vecX dst, immI8 shift) %{
17278   predicate(n->as_Vector()->length() == 4);
17279   match(Set dst (RShiftVI dst shift));
17280   format %{ "psrad   $dst,$shift\t! arithmetic right shift packed4I" %}
17281   ins_encode %{
17282     __ psrad($dst$$XMMRegister, (int)$shift$$constant);
17283   %}
17284   ins_pipe( pipe_slow );
17285 %}
17286 
17287 instruct vsra4I_reg(vecX dst, vecX src, vecS shift) %{
17288   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
17289   match(Set dst (RShiftVI src shift));
17290   format %{ "vpsrad  $dst,$src,$shift\t! arithmetic right shift packed4I" %}
17291   ins_encode %{
17292     int vector_len = 0;
17293     __ vpsrad($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17294   %}
17295   ins_pipe( pipe_slow );
17296 %}
17297 
17298 instruct vsra4I_reg_imm(vecX dst, vecX src, immI8 shift) %{
17299   predicate(UseAVX > 0 && n->as_Vector()->length() == 4);
17300   match(Set dst (RShiftVI src shift));
17301   format %{ "vpsrad  $dst,$src,$shift\t! arithmetic right shift packed4I" %}
17302   ins_encode %{
17303     int vector_len = 0;
17304     __ vpsrad($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
17305   %}
17306   ins_pipe( pipe_slow );
17307 %}
17308 
17309 instruct vsra8I_reg(vecY dst, vecY src, vecS shift) %{
17310   predicate(UseAVX > 1 && n->as_Vector()->length() == 8);
17311   match(Set dst (RShiftVI src shift));
17312   format %{ "vpsrad  $dst,$src,$shift\t! arithmetic right shift packed8I" %}
17313   ins_encode %{
17314     int vector_len = 1;
17315     __ vpsrad($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17316   %}
17317   ins_pipe( pipe_slow );
17318 %}
17319 
17320 instruct vsra8I_reg_imm(vecY dst, vecY src, immI8 shift) %{
17321   predicate(UseAVX > 1 && n->as_Vector()->length() == 8);
17322   match(Set dst (RShiftVI src shift));
17323   format %{ "vpsrad  $dst,$src,$shift\t! arithmetic right shift packed8I" %}
17324   ins_encode %{
17325     int vector_len = 1;
17326     __ vpsrad($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
17327   %}
17328   ins_pipe( pipe_slow );
17329 %}
17330 
17331 instruct vsra16I_reg(vecZ dst, vecZ src, vecS shift) %{
17332   predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
17333   match(Set dst (RShiftVI src shift));
17334   format %{ "vpsrad  $dst,$src,$shift\t! arithmetic right shift packed16I" %}
17335   ins_encode %{
17336     int vector_len = 2;
17337     __ vpsrad($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17338   %}
17339   ins_pipe( pipe_slow );
17340 %}
17341 
17342 instruct vsra16I_reg_imm(vecZ dst, vecZ src, immI8 shift) %{
17343   predicate(UseAVX > 2 && n->as_Vector()->length() == 16);
17344   match(Set dst (RShiftVI src shift));
17345   format %{ "vpsrad  $dst,$src,$shift\t! arithmetic right shift packed16I" %}
17346   ins_encode %{
17347     int vector_len = 2;
17348     __ vpsrad($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
17349   %}
17350   ins_pipe( pipe_slow );
17351 %}
17352 
17353 // Long vector arithmetic right shift
17354 instruct vsra1L(vecD dst, vecD src, vecS shift, vecD tmp) %{
17355   predicate(n->as_Vector()->length() == 1);
17356   match(Set dst (RShiftVL src shift));
17357   effect(TEMP dst, TEMP tmp);
17358   format %{ "movdqu  $dst,$src\n\t"
17359             "psrlq   $dst,$shift\n\t"
17360             "movdqu  $tmp,[0x8000000000000000]\n\t"
17361             "psrlq   $tmp,$shift\n\t"
17362             "pxor    $dst,$tmp\n\t"
17363             "psubq   $dst,$tmp\t! arithmetic right shift packed1L" %}
17364   ins_encode %{
17365     __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
17366     __ psrlq($dst$$XMMRegister, $shift$$XMMRegister);
17367     __ movdqu($tmp$$XMMRegister, ExternalAddress(vector_all_ones_mask()));
17368     __ psrlq($tmp$$XMMRegister, $shift$$XMMRegister);
17369     __ pxor($dst$$XMMRegister, $tmp$$XMMRegister);
17370     __ psubq($dst$$XMMRegister, $tmp$$XMMRegister);
17371   %}
17372   ins_pipe( pipe_slow );
17373 %}
17374 
17375 instruct vsra1L_imm(vecD dst, vecD src, immI8 shift, vecD tmp) %{
17376   predicate(n->as_Vector()->length() == 1);
17377   match(Set dst (RShiftVL src shift));
17378   effect(TEMP dst, TEMP tmp);
17379   format %{ "movdqu  $dst,$src\n\t"
17380             "psrlq   $dst,$shift\n\t"
17381             "movdqu  $tmp,[0x8000000000000000]\n\t"
17382             "psrlq   $tmp,$shift\n\t"
17383             "pxor    $dst,$tmp\n\t"
17384             "psubq   $dst,$tmp\t! arithmetic right shift packed1L" %}
17385   ins_encode %{
17386     __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
17387     __ psrlq($dst$$XMMRegister, (int)$shift$$constant);
17388     __ movdqu($tmp$$XMMRegister, ExternalAddress(vector_all_ones_mask()));
17389     __ psrlq($tmp$$XMMRegister, (int)$shift$$constant);
17390     __ pxor($dst$$XMMRegister, $tmp$$XMMRegister);
17391     __ psubq($dst$$XMMRegister, $tmp$$XMMRegister);
17392   %}
17393   ins_pipe( pipe_slow );
17394 %}
17395 
17396 instruct vsra1L_reg(vecD dst, vecD src, vecS shift, vecD tmp) %{
17397   predicate(UseAVX > 0 && n->as_Vector()->length() == 1);
17398   match(Set dst (RShiftVL src shift));
17399   effect(TEMP dst, TEMP tmp);
17400   format %{ "vpsrlq   $dst,$src,$shift\n\t"
17401             "vmovdqu  $tmp,[0x8000000000000000]\n\t"
17402             "vpsrlq   $tmp,$tmp,$shift\n\t"
17403             "vpxor    $dst,$dst,$tmp\n\t"
17404             "vpsubq   $dst,$dst,$tmp\t! arithmetic right shift packed1L" %}
17405   ins_encode %{
17406     int vector_len = 0;
17407     __ vpsrlq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17408     __ vmovdqu($tmp$$XMMRegister, ExternalAddress(vector_all_ones_mask()));
17409     __ vpsrlq($tmp$$XMMRegister, $tmp$$XMMRegister, $shift$$XMMRegister, vector_len);
17410     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
17411     __ vpsubq($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
17412   %}
17413   ins_pipe( pipe_slow );
17414 %}
17415 
17416 instruct vsra1L_reg_imm(vecD dst, vecD src, immI8 shift, vecD tmp) %{
17417   predicate(UseAVX > 0 && n->as_Vector()->length() == 1);
17418   match(Set dst (RShiftVL src shift));
17419   effect(TEMP dst, TEMP tmp);
17420   format %{ "vpsrlq   $dst,$src,$shift\n\t"
17421             "vmovdqu  $tmp,[0x8000000000000000]\n\t"
17422             "vpsrlq   $tmp,$tmp,$shift\n\t"
17423             "vpxor    $dst,$dst,$tmp\n\t"
17424             "vpsubq   $dst,$dst,$tmp\t! arithmetic right shift packed1L" %}
17425   ins_encode %{
17426     int vector_len = 0;
17427     __ vpsrlq($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
17428     __ vmovdqu($tmp$$XMMRegister, ExternalAddress(vector_all_ones_mask()));
17429     __ vpsrlq($tmp$$XMMRegister, $tmp$$XMMRegister, (int)$shift$$constant, vector_len);
17430     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
17431     __ vpsubq($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
17432   %}
17433   ins_pipe( pipe_slow );
17434 %}
17435 
17436 instruct vsra1L_reg_evex(vecD dst, vecD src, vecS shift) %{
17437   predicate(UseAVX > 2 && n->as_Vector()->length() == 1);
17438   match(Set dst (RShiftVL src shift));
17439   format %{ "evpsraq  $dst,$src,$shift\t! arithmetic right shift packed1L" %}
17440   ins_encode %{
17441     int vector_len = 0;
17442     __ evpsraq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17443   %}
17444   ins_pipe( pipe_slow );
17445 %}
17446 
17447 instruct vsra2L_reg_imm(vecX dst, vecX src, immI8 shift, vecX tmp) %{
17448   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
17449   match(Set dst (RShiftVL src shift));
17450   effect(TEMP dst, TEMP tmp);
17451   format %{ "vpsrlq   $dst,$src,$shift\n\t"
17452             "vmovdqu  $tmp,[0x8000000000000000]\n\t"
17453             "vpsrlq   $tmp,$tmp,$shift\n\t"
17454             "vpxor    $dst,$dst,$tmp\n\t"
17455             "vpsubq   $dst,$dst,$tmp\t! arithmetic right shift packed2L" %}
17456   ins_encode %{
17457     int vector_len = 0;
17458     __ vpsrlq($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
17459     __ vmovdqu($tmp$$XMMRegister, ExternalAddress(vector_all_ones_mask()));
17460     __ vpsrlq($tmp$$XMMRegister, $tmp$$XMMRegister, (int)$shift$$constant, vector_len);
17461     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
17462     __ vpsubq($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
17463   %}
17464   ins_pipe( pipe_slow );
17465 %}
17466 
17467 instruct vsra2L_reg(vecX dst, vecX src, vecS shift, vecX tmp) %{
17468   predicate(UseAVX > 0 && n->as_Vector()->length() == 2);
17469   match(Set dst (RShiftVL src shift));
17470   effect(TEMP dst, TEMP tmp);
17471   format %{ "vpsrlq   $dst,$src,$shift\n\t"
17472             "vmovdqu  $tmp,[0x8000000000000000]\n\t"
17473             "vpsrlq   $tmp,$tmp,$shift\n\t"
17474             "vpxor    $dst,$dst,$tmp\n\t"
17475             "vpsubq   $dst,$dst,$tmp\t! arithmetic right shift packed2L" %}
17476   ins_encode %{
17477     int vector_len = 0;
17478     __ vpsrlq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17479     __ vmovdqu($tmp$$XMMRegister, ExternalAddress(vector_all_ones_mask()));
17480     __ vpsrlq($tmp$$XMMRegister, $tmp$$XMMRegister, $shift$$XMMRegister, vector_len);
17481     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
17482     __ vpsubq($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
17483   %}
17484   ins_pipe( pipe_slow );
17485 %}
17486 
17487 instruct vsra2L_reg_evex_imm(vecX dst, vecX src, immI8 shift) %{
17488   predicate(UseAVX > 2 && n->as_Vector()->length() == 2);
17489   match(Set dst (RShiftVL src shift));
17490   format %{ "evpsraq  $dst,$src,$shift\t! arithmetic right shift packed2L" %}
17491   ins_encode %{
17492     int vector_len = 0;
17493     __ evpsraq($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
17494   %}
17495   ins_pipe( pipe_slow );
17496 %}
17497 
17498 instruct vsra2L_reg_evex(vecX dst, vecX src, vecS shift) %{
17499   predicate(UseAVX > 2 && n->as_Vector()->length() == 2);
17500   match(Set dst (RShiftVL src shift));
17501   format %{ "evpsraq  $dst,$src,$shift\t! arithmetic right shift packed2L" %}
17502   ins_encode %{
17503     int vector_len = 0;
17504     __ evpsraq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17505   %}
17506   ins_pipe( pipe_slow );
17507 %}
17508 
17509 instruct vsra4L_reg_imm(vecY dst, vecY src, immI8 shift, vecY tmp) %{
17510   predicate(UseAVX > 1 && n->as_Vector()->length() == 4);
17511   match(Set dst (RShiftVL src shift));
17512   effect(TEMP dst, TEMP tmp);
17513   format %{ "vpsrlq   $dst,$src,$shift\n\t"
17514             "vmovdqu  $tmp,[0x8000000000000000]\n\t"
17515             "vpsrlq   $tmp,$tmp,$shift\n\t"
17516             "vpxor    $dst,$dst,$tmp\n\t"
17517             "vpsubq   $dst,$dst,$tmp\t! arithmetic right shift packed4L" %}
17518   ins_encode %{
17519     int vector_len = 1;
17520     __ vpsrlq($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
17521     __ vmovdqu($tmp$$XMMRegister, ExternalAddress(vector_all_ones_mask()));
17522     __ vpsrlq($tmp$$XMMRegister, $tmp$$XMMRegister, (int)$shift$$constant, vector_len);
17523     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
17524     __ vpsubq($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
17525   %}
17526   ins_pipe( pipe_slow );
17527 %}
17528 
17529 instruct vsra4L_reg(vecY dst, vecY src, vecS shift, vecY tmp) %{
17530   predicate(UseAVX > 1 && n->as_Vector()->length() == 4);
17531   match(Set dst (RShiftVL src shift));
17532   effect(TEMP dst, TEMP tmp);
17533   format %{ "vpsrlq   $dst,$src,$shift\n\t"
17534             "vmovdqu  $tmp,[0x8000000000000000]\n\t"
17535             "vpsrlq   $tmp,$tmp,$shift\n\t"
17536             "vpxor    $dst,$dst,$tmp\n\t"
17537             "vpsubq   $dst,$dst,$tmp\t! arithmetic right shift packed4L" %}
17538   ins_encode %{
17539     int vector_len = 1;
17540     __ vpsrlq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17541     __ vmovdqu($tmp$$XMMRegister, ExternalAddress(vector_all_ones_mask()));
17542     __ vpsrlq($tmp$$XMMRegister, $tmp$$XMMRegister, $shift$$XMMRegister, vector_len);
17543     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
17544     __ vpsubq($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
17545   %}
17546   ins_pipe( pipe_slow );
17547 %}
17548 
17549 instruct vsra4L_reg_evex_imm(vecY dst, vecY src, immI8 shift) %{
17550   predicate(UseAVX > 2 && n->as_Vector()->length() == 4);
17551   match(Set dst (RShiftVL src shift));
17552   format %{ "evpsraq  $dst,$src,$shift\t! arithmetic right shift packed2L" %}
17553   ins_encode %{
17554     int vector_len = 1;
17555     __ evpsraq($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
17556   %}
17557   ins_pipe( pipe_slow );
17558 %}
17559 
17560 instruct vsra4L_reg_evex(vecY dst, vecY src, vecS shift) %{
17561   predicate(UseAVX > 2 && n->as_Vector()->length() == 4);
17562   match(Set dst (RShiftVL src shift));
17563   format %{ "evpsraq  $dst,$src,$shift\t! arithmetic right shift packed4L" %}
17564   ins_encode %{
17565     int vector_len = 1;
17566     __ evpsraq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17567   %}
17568   ins_pipe( pipe_slow );
17569 %}
17570 
17571 instruct vsra8L_reg_evex_imm(vecZ dst, vecZ src, immI8 shift) %{
17572   predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
17573   match(Set dst (RShiftVL src shift));
17574   format %{ "evpsraq  $dst,$src,$shift\t! arithmetic right shift packed2L" %}
17575   ins_encode %{
17576     int vector_len = 2;
17577     __ evpsraq($dst$$XMMRegister, $src$$XMMRegister, (int)$shift$$constant, vector_len);
17578   %}
17579   ins_pipe( pipe_slow );
17580 %}
17581 
17582 instruct vsra8L_reg_evex(vecZ dst, vecZ src, vecS shift) %{
17583   predicate(UseAVX > 2 && n->as_Vector()->length() == 8);
17584   match(Set dst (RShiftVL src shift));
17585   format %{ "evpsraq  $dst,$src,$shift\t! arithmetic right shift packed8L" %}
17586   ins_encode %{
17587     int vector_len = 2;
17588     __ evpsraq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17589   %}
17590   ins_pipe( pipe_slow );
17591 %}
17592 
17593 // ------------------- Variable Bit Shift Left Logical -----------------------------
17594 //Integer Variable left shift
17595 instruct vsllv2I(vecD dst, vecD src, vecD shift) %{
17596   predicate(UseAVX > 1 && n->as_Vector()->length() == 2 && n->in(2)->Opcode() != Op_LShiftCntV);
17597   match(Set dst (LShiftVI src shift));
17598   format %{ "vpsllvd  $dst,$src,$shift\t! variable bit shift left shift packed2I" %}
17599   ins_encode %{
17600     int vector_len = 0;
17601     __ vpsllvd($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17602   %}
17603   ins_pipe( pipe_slow );
17604 %}
17605 
17606 instruct vsllv4I_reg(vecX dst, vecX src, vecX shift) %{
17607   predicate(UseAVX > 1 && n->as_Vector()->length() == 4 && n->in(2)->Opcode() != Op_LShiftCntV);
17608   match(Set dst (LShiftVI src shift));
17609   format %{ "vpsllvd  $dst,$src,$shift\t! variable bit shift left shift packed4I" %}
17610   ins_encode %{
17611     int vector_len = 0;
17612     __ vpsllvd($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17613   %}
17614   ins_pipe( pipe_slow );
17615 %}
17616 
17617 instruct vsllv4I_reg_evex(vecX dst, vecX src, vecX shift) %{
17618   predicate(UseAVX > 2 && n->as_Vector()->length() == 4 && n->in(2)->Opcode() != Op_LShiftCntV);
17619   match(Set dst (LShiftVI src shift));
17620   format %{ "vpsllvd  $dst,$src,$shift\t! variable bit shift left shift packed4I" %}
17621   ins_encode %{
17622     int vector_len = 0;
17623     __ vpsllvd($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17624   %}
17625   ins_pipe( pipe_slow );
17626 %}
17627 
17628 instruct vsllv8I_reg(vecY dst, vecY src, vecY shift) %{
17629   predicate(UseAVX > 1 && n->as_Vector()->length() == 8 && n->in(2)->Opcode() != Op_LShiftCntV);
17630   match(Set dst (LShiftVI src shift));
17631   format %{ "vpsllvd  $dst,$src,$shift\t! variable bit shift left shift packed8I" %}
17632   ins_encode %{
17633     int vector_len = 1;
17634     __ vpsllvd($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17635   %}
17636   ins_pipe( pipe_slow );
17637 %}
17638 
17639 instruct vsllv8I_reg_evex(vecY dst, vecY src, vecY shift) %{
17640   predicate(UseAVX > 2 && n->as_Vector()->length() == 8 && n->in(2)->Opcode() != Op_LShiftCntV);
17641   match(Set dst (LShiftVI src shift));
17642   format %{ "vpsllvd  $dst,$src,$shift\t! variable bit shift left shift packed8I" %}
17643   ins_encode %{
17644     int vector_len = 1;
17645     __ vpsllvd($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17646   %}
17647   ins_pipe( pipe_slow );
17648 %}
17649 
17650 instruct vsllv16I_reg_evex(vecZ dst, vecZ src, vecZ shift) %{
17651   predicate(UseAVX > 2 && n->as_Vector()->length() == 16 && n->in(2)->Opcode() != Op_LShiftCntV);
17652   match(Set dst (LShiftVI src shift));
17653   format %{ "vpsllvd  $dst,$src,$shift\t! variable bit shift left shift packed16I" %}
17654   ins_encode %{
17655     int vector_len = 2;
17656     __ vpsllvd($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17657   %}
17658   ins_pipe( pipe_slow );
17659 %}
17660 
17661 //Long Variable left shift
17662 instruct vsllv1L_reg(vecD dst, vecD src, vecD shift) %{
17663   predicate(UseAVX > 1 && n->as_Vector()->length() == 1 && n->in(2)->Opcode() != Op_LShiftCntV);
17664   match(Set dst (LShiftVL src shift));
17665   format %{ "vpsllvq  $dst,$src,$shift\t! variable bit shift left shift packed1L" %}
17666   ins_encode %{
17667     int vector_len = 0;
17668     __ vpsllvq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17669   %}
17670   ins_pipe( pipe_slow );
17671 %}
17672 
17673 instruct vsllv2L_reg(vecX dst, vecX src, vecX shift) %{
17674   predicate(UseAVX > 1 && n->as_Vector()->length() == 2 && n->in(2)->Opcode() != Op_LShiftCntV);
17675   match(Set dst (LShiftVL src shift));
17676   format %{ "vpsllvq  $dst,$src,$shift\t! variable bit shift left shift packed2L" %}
17677   ins_encode %{
17678     int vector_len = 0;
17679     __ vpsllvq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17680   %}
17681   ins_pipe( pipe_slow );
17682 %}
17683 
17684 instruct vsllv2L_reg_evex(vecX dst, vecX src, vecX shift) %{
17685   predicate(UseAVX > 2 && n->as_Vector()->length() == 2 && n->in(2)->Opcode() != Op_LShiftCntV);
17686   match(Set dst (LShiftVL src shift));
17687   format %{ "vpsllvq  $dst,$src,$shift\t! variable bit shift left shift packed2L" %}
17688   ins_encode %{
17689     int vector_len = 0;
17690     __ vpsllvq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17691   %}
17692   ins_pipe( pipe_slow );
17693 %}
17694 
17695 instruct vsllv4L_reg(vecY dst, vecY src, vecY shift) %{
17696   predicate(UseAVX > 1 && n->as_Vector()->length() == 4 && n->in(2)->Opcode() != Op_LShiftCntV);
17697   match(Set dst (LShiftVL src shift));
17698   format %{ "vpsllvq  $dst,$src,$shift\t! variable bit shift left shift packed4L" %}
17699   ins_encode %{
17700     int vector_len = 1;
17701     __ vpsllvq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17702   %}
17703   ins_pipe( pipe_slow );
17704 %}
17705 
17706 instruct vsllv4L_reg_evex(vecY dst, vecY src, vecY shift) %{
17707   predicate(UseAVX > 2 && n->as_Vector()->length() == 8 && n->in(2)->Opcode() != Op_LShiftCntV);
17708   match(Set dst (LShiftVL src shift));
17709   format %{ "vpsllvq  $dst,$src,$shift\t! variable bit shift left shift packed4L" %}
17710   ins_encode %{
17711     int vector_len = 1;
17712     __ vpsllvq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17713   %}
17714   ins_pipe( pipe_slow );
17715 %}
17716 
17717 instruct vsllv8L_reg_evex(vecZ dst, vecZ src, vecZ shift) %{
17718   predicate(UseAVX > 2 && n->as_Vector()->length() == 8 && n->in(2)->Opcode() != Op_LShiftCntV);
17719   match(Set dst (LShiftVL src shift));
17720   format %{ "vpsllvq  $dst,$src,$shift\t! variable bit shift left shift packed16I" %}
17721   ins_encode %{
17722     int vector_len = 2;
17723     __ vpsllvq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17724   %}
17725   ins_pipe( pipe_slow );
17726 %}
17727 
17728 // ------------------- Variable Bit Shift Right Logical -----------------------------
17729 //Integer Variable right shift
17730 instruct vsrlv2I_reg(vecD dst, vecD src, vecD shift) %{
17731   predicate(UseAVX > 1 && n->as_Vector()->length() == 2 && n->in(2)->Opcode() != Op_RShiftCntV);
17732   match(Set dst (URShiftVI src shift));
17733   format %{ "vpsrlvd  $dst,$src,$shift\t! variable bit shift right shift packed2I" %}
17734   ins_encode %{
17735     int vector_len = 0;
17736     __ vpsrlvd($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17737   %}
17738   ins_pipe( pipe_slow );
17739 %}
17740 
17741 instruct vsrlv4I_reg(vecX dst, vecX src, vecX shift) %{
17742   predicate(UseAVX > 1 && n->as_Vector()->length() == 4 && n->in(2)->Opcode() != Op_RShiftCntV);
17743   match(Set dst (URShiftVI src shift));
17744   format %{ "vpsrlvd  $dst,$src,$shift\t! variable bit shift right shift packed4I" %}
17745   ins_encode %{
17746     int vector_len = 0;
17747     __ vpsrlvd($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17748   %}
17749   ins_pipe( pipe_slow );
17750 %}
17751 
17752 instruct vsrlv4I_reg_evex(vecX dst, vecX src, vecX shift) %{
17753   predicate(UseAVX > 2 && n->as_Vector()->length() == 4 && n->in(2)->Opcode() != Op_RShiftCntV);
17754   match(Set dst (URShiftVI src shift));
17755   format %{ "vpsrlvd  $dst,$src,$shift\t! variable bit shift right shift packed4I" %}
17756   ins_encode %{
17757     int vector_len = 0;
17758     __ vpsrlvd($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17759   %}
17760   ins_pipe( pipe_slow );
17761 %}
17762 
17763 instruct vsrlv8I_reg(vecY dst, vecY src, vecY shift) %{
17764   predicate(UseAVX > 1 && n->as_Vector()->length() == 8 && n->in(2)->Opcode() != Op_RShiftCntV);
17765   match(Set dst (URShiftVI src shift));
17766   format %{ "vpsrlvd  $dst,$src,$shift\t! variable bit shift right shift packed8I" %}
17767   ins_encode %{
17768     int vector_len = 1;
17769     __ vpsrlvd($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17770   %}
17771   ins_pipe( pipe_slow );
17772 %}
17773 
17774 instruct vsrlv8I_reg_evex(vecY dst, vecY src, vecY shift) %{
17775   predicate(UseAVX > 2 && n->as_Vector()->length() == 8 && n->in(2)->Opcode() != Op_RShiftCntV);
17776   match(Set dst (URShiftVI src shift));
17777   format %{ "vpsrlvd  $dst,$src,$shift\t! variable bit shift right shift packed8I" %}
17778   ins_encode %{
17779     int vector_len = 1;
17780     __ vpsrlvd($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17781   %}
17782   ins_pipe( pipe_slow );
17783 %}
17784 
17785 instruct vsrlv16I_reg_evex(vecZ dst, vecZ src, vecZ shift) %{
17786   predicate(UseAVX > 2 && n->as_Vector()->length() == 16 && n->in(2)->Opcode() != Op_RShiftCntV);
17787   match(Set dst (URShiftVI src shift));
17788   format %{ "vpsrlvd  $dst,$src,$shift\t! variable bit shift right shift packed16I" %}
17789   ins_encode %{
17790     int vector_len = 2;
17791     __ vpsrlvd($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17792   %}
17793   ins_pipe( pipe_slow );
17794 %}
17795 
17796 //Long Variable right shift
17797 instruct vsrlv1L_reg(vecD dst, vecD src, vecD shift) %{
17798   predicate(UseAVX > 1 && n->as_Vector()->length() == 1 && n->in(2)->Opcode() != Op_RShiftCntV);
17799   match(Set dst (URShiftVL src shift));
17800   format %{ "vpsrlvq  $dst,$src,$shift\t! variable bit shift right shift packed1L" %}
17801   ins_encode %{
17802     int vector_len = 0;
17803     __ vpsrlvq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17804   %}
17805   ins_pipe( pipe_slow );
17806 %}
17807 
17808 instruct vsrlv2L_reg(vecX dst, vecX src, vecX shift) %{
17809   predicate(UseAVX > 1 && n->as_Vector()->length() == 2 && n->in(2)->Opcode() != Op_RShiftCntV);
17810   match(Set dst (URShiftVL src shift));
17811   format %{ "vpsrlvq  $dst,$src,$shift\t! variable bit shift right shift packed2L" %}
17812   ins_encode %{
17813     int vector_len = 0;
17814     __ vpsrlvq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17815   %}
17816   ins_pipe( pipe_slow );
17817 %}
17818 
17819 instruct vsrlv2L_reg_evex(vecX dst, vecX src, vecX shift) %{
17820   predicate(UseAVX > 2 && n->as_Vector()->length() == 2 && n->in(2)->Opcode() != Op_RShiftCntV);
17821   match(Set dst (URShiftVL src shift));
17822   format %{ "vpsrlvq  $dst,$src,$shift\t! variable bit shift right shift packed2L" %}
17823   ins_encode %{
17824     int vector_len = 0;
17825     __ vpsrlvq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17826   %}
17827   ins_pipe( pipe_slow );
17828 %}
17829 
17830 instruct vsrlv4L_reg(vecY dst, vecY src, vecY shift) %{
17831   predicate(UseAVX > 1 && n->as_Vector()->length() == 4 && n->in(2)->Opcode() != Op_RShiftCntV);
17832   match(Set dst (URShiftVL src shift));
17833   format %{ "vpsrlvq  $dst,$src,$shift\t! variable bit shift right shift packed4L" %}
17834   ins_encode %{
17835     int vector_len = 1;
17836     __ vpsrlvq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17837   %}
17838   ins_pipe( pipe_slow );
17839 %}
17840 
17841 instruct vsrlv4L_reg_evex(vecY dst, vecY src, vecY shift) %{
17842   predicate(UseAVX > 2 && n->as_Vector()->length() == 8 && n->in(2)->Opcode() != Op_RShiftCntV);
17843   match(Set dst (URShiftVL src shift));
17844   format %{ "vpsrlvq  $dst,$src,$shift\t! variable bit shift right shift packed4L" %}
17845   ins_encode %{
17846     int vector_len = 1;
17847     __ vpsrlvq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17848   %}
17849   ins_pipe( pipe_slow );
17850 %}
17851 
17852 instruct vsrlv8L_reg(vecZ dst, vecZ src, vecZ shift) %{
17853   predicate(UseAVX > 1 && n->as_Vector()->length() == 8 && n->in(2)->Opcode() != Op_RShiftCntV);
17854   match(Set dst (URShiftVL src shift));
17855   format %{ "vpsrlvq  $dst,$src,$shift\t! variable bit shift right shift packed8L" %}
17856   ins_encode %{
17857     int vector_len = 2;
17858     __ vpsrlvq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17859   %}
17860   ins_pipe( pipe_slow );
17861 %}
17862 
17863 // ------------------- Variable Bit Shift Right Arithmetic -----------------------------
17864 //Integer Variable right shift
17865 instruct vsrav2I_reg(vecD dst, vecD src, vecD shift) %{
17866   predicate(UseAVX > 1 && n->as_Vector()->length() == 2 && n->in(2)->Opcode() != Op_RShiftCntV);
17867   match(Set dst (RShiftVI src shift));
17868   format %{ "vpsravd  $dst,$src,$shift\t! variable bit shift right shift packed2I" %}
17869   ins_encode %{
17870     int vector_len = 0;
17871     __ vpsravd($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17872   %}
17873   ins_pipe( pipe_slow );
17874 %}
17875 
17876 instruct vsrav4I_reg(vecX dst, vecX src, vecX shift) %{
17877   predicate(UseAVX > 1 && n->as_Vector()->length() == 4 && n->in(2)->Opcode() != Op_RShiftCntV);
17878   match(Set dst (RShiftVI src shift));
17879   format %{ "vpsravd  $dst,$src,$shift\t! variable bit shift right shift packed4I" %}
17880   ins_encode %{
17881     int vector_len = 0;
17882     __ vpsravd($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17883   %}
17884   ins_pipe( pipe_slow );
17885 %}
17886 
17887 instruct vsrav4I_reg_evex(vecX dst, vecX src, vecX shift) %{
17888   predicate(UseAVX > 2 && n->as_Vector()->length() == 4 && n->in(2)->Opcode() != Op_RShiftCntV);
17889   match(Set dst (RShiftVI src shift));
17890   format %{ "vpsravd  $dst,$src,$shift\t! variable bit shift right shift packed4I" %}
17891   ins_encode %{
17892     int vector_len = 0;
17893     __ vpsravd($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17894   %}
17895   ins_pipe( pipe_slow );
17896 %}
17897 
17898 instruct vsrav8I_reg(vecY dst, vecY src, vecY shift) %{
17899   predicate(UseAVX > 1 && n->as_Vector()->length() == 8 && n->in(2)->Opcode() != Op_RShiftCntV);
17900   match(Set dst (RShiftVI src shift));
17901   format %{ "vpsravd  $dst,$src,$shift\t! variable bit shift right shift packed8I" %}
17902   ins_encode %{
17903     int vector_len = 1;
17904     __ vpsravd($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17905   %}
17906   ins_pipe( pipe_slow );
17907 %}
17908 
17909 instruct vsrav8I_reg_evex(vecY dst, vecY src, vecY shift) %{
17910   predicate(UseAVX > 2 && n->as_Vector()->length() == 8 && n->in(2)->Opcode() != Op_RShiftCntV);
17911   match(Set dst (RShiftVI src shift));
17912   format %{ "vpsravd  $dst,$src,$shift\t! variable bit shift right shift packed8I" %}
17913   ins_encode %{
17914     int vector_len = 1;
17915     __ vpsravd($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17916   %}
17917   ins_pipe( pipe_slow );
17918 %}
17919 
17920 instruct vsrav16I_reg_evex(vecZ dst, vecZ src, vecZ shift) %{
17921   predicate(UseAVX > 2 && n->as_Vector()->length() == 16 && n->in(2)->Opcode() != Op_RShiftCntV);
17922   match(Set dst (RShiftVI src shift));
17923   format %{ "vpsravd  $dst,$src,$shift\t! variable bit shift right shift packed16I" %}
17924   ins_encode %{
17925     int vector_len = 2;
17926     __ vpsravd($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17927   %}
17928   ins_pipe( pipe_slow );
17929 %}
17930 
17931 //Long Variable right shift arithmetic
17932 instruct vsrav1L_reg(vecD dst, vecD src, vecD shift, vecD tmp) %{
17933   predicate(UseAVX > 0 && n->as_Vector()->length() == 1 && n->in(2)->Opcode() != Op_RShiftCntV);
17934   match(Set dst (RShiftVL src shift));
17935   effect(TEMP dst, TEMP tmp);
17936   format %{ "vpsrlvq   $dst,$src,$shift\n\t"
17937             "vmovdqu   $tmp,[0x8000000000000000]\n\t"
17938             "vpsrlvq   $tmp,$tmp,$shift\n\t"
17939             "vpxor     $dst,$dst,$tmp\n\t"
17940             "vpsubq    $dst,$dst,$tmp\t! variable arithmetic right shift packed1L" %}
17941    ins_encode %{
17942      int vector_len = 0;
17943      __ vpsrlvq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17944      __ vmovdqu($tmp$$XMMRegister, ExternalAddress(vector_all_ones_mask()));
17945      __ vpsrlvq($tmp$$XMMRegister, $tmp$$XMMRegister, $shift$$XMMRegister, vector_len);
17946      __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
17947      __ vpsubq($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
17948    %}
17949    ins_pipe( pipe_slow );
17950  %}
17951 
17952 instruct vsrav1L_reg_evex(vecD dst, vecD src, vecD shift) %{
17953   predicate(UseAVX > 2 && n->as_Vector()->length() == 1 && n->in(2)->Opcode() != Op_RShiftCntV);
17954   match(Set dst (RShiftVL src shift));
17955   format %{ "evpsravq  $dst,$src,$shift\t! variable arithmetic right shift packed1L" %}
17956   ins_encode %{
17957     int vector_len = 0;
17958     __ evpsravq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17959   %}
17960   ins_pipe( pipe_slow );
17961 %}
17962 
17963 instruct vsrav2L_reg(vecX dst, vecX src, vecX shift, vecX tmp) %{
17964   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 && n->in(2)->Opcode() != Op_RShiftCntV);
17965   match(Set dst (RShiftVL src shift));
17966   effect(TEMP dst, TEMP tmp);
17967   format %{ "vpsrlvq   $dst,$src,$shift\n\t"
17968             "vmovdqu   $tmp,[0x8000000000000000]\n\t"
17969             "vpsrlvq   $tmp,$tmp,$shift\n\t"
17970             "vpxor     $dst,$dst,$tmp\n\t"
17971             "vpsubq    $dst,$dst,$tmp\t! variable arithmetic right shift packed2L" %}
17972   ins_encode %{
17973     int vector_len = 0;
17974     __ vpsrlvq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17975     __ vmovdqu($tmp$$XMMRegister, ExternalAddress(vector_all_ones_mask()));
17976     __ vpsrlvq($tmp$$XMMRegister, $tmp$$XMMRegister, $shift$$XMMRegister, vector_len);
17977     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
17978     __ vpsubq($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
17979   %}
17980   ins_pipe( pipe_slow );
17981 %}
17982 
17983 instruct vsrav2L_reg_evex(vecX dst, vecX src, vecX shift) %{
17984   predicate(UseAVX > 2 && n->as_Vector()->length() == 2 && n->in(2)->Opcode() != Op_RShiftCntV);
17985   match(Set dst (RShiftVL src shift));
17986   format %{ "evpsravq  $dst,$src,$shift\t! variable arithmetic right shift packed2L" %}
17987   ins_encode %{
17988     int vector_len = 0;
17989     __ evpsravq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
17990   %}
17991   ins_pipe( pipe_slow );
17992 %}
17993 
17994 instruct vsrav4L_reg(vecY dst, vecY src, vecY shift, vecY tmp) %{
17995   predicate(UseAVX > 1 && n->as_Vector()->length() == 4 && n->in(2)->Opcode() != Op_RShiftCntV);
17996   match(Set dst (RShiftVL src shift));
17997   effect(TEMP dst, TEMP tmp);
17998   format %{ "vpsrlvq   $dst,$src,$shift\n\t"
17999             "vmovdqu   $tmp,[0x8000000000000000]\n\t"
18000             "vpsrlvq   $tmp,$tmp,$shift\n\t"
18001             "vpxor     $dst,$dst,$tmp\n\t"
18002             "vpsubq    $dst,$dst,$tmp\t! variable arithmetic right shift packed4L" %}
18003   ins_encode %{
18004     int vector_len = 1;
18005     __ vpsrlvq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
18006     __ vmovdqu($tmp$$XMMRegister, ExternalAddress(vector_all_ones_mask()));
18007     __ vpsrlvq($tmp$$XMMRegister, $tmp$$XMMRegister, $shift$$XMMRegister, vector_len);
18008     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
18009     __ vpsubq($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
18010   %}
18011   ins_pipe( pipe_slow );
18012 %}
18013 
18014 instruct vsrav4L_reg_evex(vecY dst, vecY src, vecY shift) %{
18015   predicate(UseAVX > 2 && n->as_Vector()->length() == 4 && n->in(2)->Opcode() != Op_RShiftCntV);
18016   match(Set dst (RShiftVL src shift));
18017   format %{ "evpsravq  $dst,$src,$shift\t! variable bit shift right shift packed4L" %}
18018   ins_encode %{
18019     int vector_len = 1;
18020     __ evpsravq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
18021   %}
18022   ins_pipe( pipe_slow );
18023 %}
18024 
18025 instruct vsrav8L_reg_evex(vecZ dst, vecZ src, vecZ shift) %{
18026   predicate(UseAVX > 2 && n->as_Vector()->length() == 8 && n->in(2)->Opcode() != Op_RShiftCntV);
18027   match(Set dst (RShiftVL src shift));
18028   format %{ "evpsravq  $dst,$src,$shift\t! variable bit shift right shift packed8L" %}
18029   ins_encode %{
18030     int vector_len = 2;
18031     __ evpsravq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
18032   %}
18033   ins_pipe( pipe_slow );
18034 %}
18035 
18036 // --------------------------------- AND --------------------------------------
18037 
18038 instruct vand4B(vecS dst, vecS src) %{
18039   predicate(n->as_Vector()->length_in_bytes() == 4);
18040   match(Set dst (AndV dst src));
18041   format %{ "pand    $dst,$src\t! and vectors (4 bytes)" %}
18042   ins_encode %{
18043     __ pand($dst$$XMMRegister, $src$$XMMRegister);
18044   %}
18045   ins_pipe( pipe_slow );
18046 %}
18047 
18048 instruct vand4B_reg(vecS dst, vecS src1, vecS src2) %{
18049   predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 4);
18050   match(Set dst (AndV src1 src2));
18051   format %{ "vpand   $dst,$src1,$src2\t! and vectors (4 bytes)" %}
18052   ins_encode %{
18053     int vector_len = 0;
18054     __ vpand($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
18055   %}
18056   ins_pipe( pipe_slow );
18057 %}
18058 
18059 instruct vand4B_mem(vecS dst, vecS src, memory mem) %{
18060   predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 4);
18061   match(Set dst (AndV src (LoadVector mem)));
18062   format %{ "vpand   $dst,$src,$mem\t! and vectors (4 bytes)" %}
18063   ins_encode %{
18064     int vector_len = 0;
18065     __ vpand($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
18066   %}
18067   ins_pipe( pipe_slow );
18068 %}
18069 
18070 instruct vand8B(vecD dst, vecD src) %{
18071   predicate(n->as_Vector()->length_in_bytes() == 8);
18072   match(Set dst (AndV dst src));
18073   format %{ "pand    $dst,$src\t! and vectors (8 bytes)" %}
18074   ins_encode %{
18075     __ pand($dst$$XMMRegister, $src$$XMMRegister);
18076   %}
18077   ins_pipe( pipe_slow );
18078 %}
18079 
18080 instruct vand8B_reg(vecD dst, vecD src1, vecD src2) %{
18081   predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 8);
18082   match(Set dst (AndV src1 src2));
18083   format %{ "vpand   $dst,$src1,$src2\t! and vectors (8 bytes)" %}
18084   ins_encode %{
18085     int vector_len = 0;
18086     __ vpand($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
18087   %}
18088   ins_pipe( pipe_slow );
18089 %}
18090 
18091 instruct vand8B_mem(vecD dst, vecD src, memory mem) %{
18092   predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 8);
18093   match(Set dst (AndV src (LoadVector mem)));
18094   format %{ "vpand   $dst,$src,$mem\t! and vectors (8 bytes)" %}
18095   ins_encode %{
18096     int vector_len = 0;
18097     __ vpand($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
18098   %}
18099   ins_pipe( pipe_slow );
18100 %}
18101 
18102 instruct vand16B(vecX dst, vecX src) %{
18103   predicate(n->as_Vector()->length_in_bytes() == 16);
18104   match(Set dst (AndV dst src));
18105   format %{ "pand    $dst,$src\t! and vectors (16 bytes)" %}
18106   ins_encode %{
18107     __ pand($dst$$XMMRegister, $src$$XMMRegister);
18108   %}
18109   ins_pipe( pipe_slow );
18110 %}
18111 
18112 instruct vand16B_reg(vecX dst, vecX src1, vecX src2) %{
18113   predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 16);
18114   match(Set dst (AndV src1 src2));
18115   format %{ "vpand   $dst,$src1,$src2\t! and vectors (16 bytes)" %}
18116   ins_encode %{
18117     int vector_len = 0;
18118     __ vpand($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
18119   %}
18120   ins_pipe( pipe_slow );
18121 %}
18122 
18123 instruct vand16B_mem(vecX dst, vecX src, memory mem) %{
18124   predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 16);
18125   match(Set dst (AndV src (LoadVector mem)));
18126   format %{ "vpand   $dst,$src,$mem\t! and vectors (16 bytes)" %}
18127   ins_encode %{
18128     int vector_len = 0;
18129     __ vpand($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
18130   %}
18131   ins_pipe( pipe_slow );
18132 %}
18133 
18134 instruct vand32B_reg(vecY dst, vecY src1, vecY src2) %{
18135   predicate(UseAVX > 1 && n->as_Vector()->length_in_bytes() == 32);
18136   match(Set dst (AndV src1 src2));
18137   format %{ "vpand   $dst,$src1,$src2\t! and vectors (32 bytes)" %}
18138   ins_encode %{
18139     int vector_len = 1;
18140     __ vpand($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
18141   %}
18142   ins_pipe( pipe_slow );
18143 %}
18144 
18145 instruct vand32B_mem(vecY dst, vecY src, memory mem) %{
18146   predicate(UseAVX > 1 && n->as_Vector()->length_in_bytes() == 32);
18147   match(Set dst (AndV src (LoadVector mem)));
18148   format %{ "vpand   $dst,$src,$mem\t! and vectors (32 bytes)" %}
18149   ins_encode %{
18150     int vector_len = 1;
18151     __ vpand($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
18152   %}
18153   ins_pipe( pipe_slow );
18154 %}
18155 
18156 instruct vand64B_reg(vecZ dst, vecZ src1, vecZ src2) %{
18157   predicate(UseAVX > 2 && n->as_Vector()->length_in_bytes() == 64);
18158   match(Set dst (AndV src1 src2));
18159   format %{ "vpand   $dst,$src1,$src2\t! and vectors (64 bytes)" %}
18160   ins_encode %{
18161     int vector_len = 2;
18162     __ vpand($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
18163   %}
18164   ins_pipe( pipe_slow );
18165 %}
18166 
18167 instruct vand64B_mem(vecZ dst, vecZ src, memory mem) %{
18168   predicate(UseAVX > 2 && n->as_Vector()->length_in_bytes() == 64);
18169   match(Set dst (AndV src (LoadVector mem)));
18170   format %{ "vpand   $dst,$src,$mem\t! and vectors (64 bytes)" %}
18171   ins_encode %{
18172     int vector_len = 2;
18173     __ vpand($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
18174   %}
18175   ins_pipe( pipe_slow );
18176 %}
18177 
18178 // --------------------------------- OR ---------------------------------------
18179 
18180 instruct vor4B(vecS dst, vecS src) %{
18181   predicate(n->as_Vector()->length_in_bytes() == 4);
18182   match(Set dst (OrV dst src));
18183   format %{ "por     $dst,$src\t! or vectors (4 bytes)" %}
18184   ins_encode %{
18185     __ por($dst$$XMMRegister, $src$$XMMRegister);
18186   %}
18187   ins_pipe( pipe_slow );
18188 %}
18189 
18190 instruct vor4B_reg(vecS dst, vecS src1, vecS src2) %{
18191   predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 4);
18192   match(Set dst (OrV src1 src2));
18193   format %{ "vpor    $dst,$src1,$src2\t! or vectors (4 bytes)" %}
18194   ins_encode %{
18195     int vector_len = 0;
18196     __ vpor($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
18197   %}
18198   ins_pipe( pipe_slow );
18199 %}
18200 
18201 instruct vor4B_mem(vecS dst, vecS src, memory mem) %{
18202   predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 4);
18203   match(Set dst (OrV src (LoadVector mem)));
18204   format %{ "vpor    $dst,$src,$mem\t! or vectors (4 bytes)" %}
18205   ins_encode %{
18206     int vector_len = 0;
18207     __ vpor($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
18208   %}
18209   ins_pipe( pipe_slow );
18210 %}
18211 
18212 instruct vor8B(vecD dst, vecD src) %{
18213   predicate(n->as_Vector()->length_in_bytes() == 8);
18214   match(Set dst (OrV dst src));
18215   format %{ "por     $dst,$src\t! or vectors (8 bytes)" %}
18216   ins_encode %{
18217     __ por($dst$$XMMRegister, $src$$XMMRegister);
18218   %}
18219   ins_pipe( pipe_slow );
18220 %}
18221 
18222 instruct vor8B_reg(vecD dst, vecD src1, vecD src2) %{
18223   predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 8);
18224   match(Set dst (OrV src1 src2));
18225   format %{ "vpor    $dst,$src1,$src2\t! or vectors (8 bytes)" %}
18226   ins_encode %{
18227     int vector_len = 0;
18228     __ vpor($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
18229   %}
18230   ins_pipe( pipe_slow );
18231 %}
18232 
18233 instruct vor8B_mem(vecD dst, vecD src, memory mem) %{
18234   predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 4);
18235   match(Set dst (OrV src (LoadVector mem)));
18236   format %{ "vpor    $dst,$src,$mem\t! or vectors (8 bytes)" %}
18237   ins_encode %{
18238     int vector_len = 0;
18239     __ vpor($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
18240   %}
18241   ins_pipe( pipe_slow );
18242 %}
18243 
18244 instruct vor16B(vecX dst, vecX src) %{
18245   predicate(n->as_Vector()->length_in_bytes() == 16);
18246   match(Set dst (OrV dst src));
18247   format %{ "por     $dst,$src\t! or vectors (16 bytes)" %}
18248   ins_encode %{
18249     __ por($dst$$XMMRegister, $src$$XMMRegister);
18250   %}
18251   ins_pipe( pipe_slow );
18252 %}
18253 
18254 instruct vor16B_reg(vecX dst, vecX src1, vecX src2) %{
18255   predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 16);
18256   match(Set dst (OrV src1 src2));
18257   format %{ "vpor    $dst,$src1,$src2\t! or vectors (16 bytes)" %}
18258   ins_encode %{
18259     int vector_len = 0;
18260     __ vpor($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
18261   %}
18262   ins_pipe( pipe_slow );
18263 %}
18264 
18265 instruct vor16B_mem(vecX dst, vecX src, memory mem) %{
18266   predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 16);
18267   match(Set dst (OrV src (LoadVector mem)));
18268   format %{ "vpor    $dst,$src,$mem\t! or vectors (16 bytes)" %}
18269   ins_encode %{
18270     int vector_len = 0;
18271     __ vpor($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
18272   %}
18273   ins_pipe( pipe_slow );
18274 %}
18275 
18276 instruct vor32B_reg(vecY dst, vecY src1, vecY src2) %{
18277   predicate(UseAVX > 1 && n->as_Vector()->length_in_bytes() == 32);
18278   match(Set dst (OrV src1 src2));
18279   format %{ "vpor    $dst,$src1,$src2\t! or vectors (32 bytes)" %}
18280   ins_encode %{
18281     int vector_len = 1;
18282     __ vpor($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
18283   %}
18284   ins_pipe( pipe_slow );
18285 %}
18286 
18287 instruct vor32B_mem(vecY dst, vecY src, memory mem) %{
18288   predicate(UseAVX > 1 && n->as_Vector()->length_in_bytes() == 32);
18289   match(Set dst (OrV src (LoadVector mem)));
18290   format %{ "vpor    $dst,$src,$mem\t! or vectors (32 bytes)" %}
18291   ins_encode %{
18292     int vector_len = 1;
18293     __ vpor($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
18294   %}
18295   ins_pipe( pipe_slow );
18296 %}
18297 
18298 instruct vor64B_reg(vecZ dst, vecZ src1, vecZ src2) %{
18299   predicate(UseAVX > 2 && n->as_Vector()->length_in_bytes() == 64);
18300   match(Set dst (OrV src1 src2));
18301   format %{ "vpor    $dst,$src1,$src2\t! or vectors (64 bytes)" %}
18302   ins_encode %{
18303     int vector_len = 2;
18304     __ vpor($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
18305   %}
18306   ins_pipe( pipe_slow );
18307 %}
18308 
18309 instruct vor64B_mem(vecZ dst, vecZ src, memory mem) %{
18310   predicate(UseAVX > 2 && n->as_Vector()->length_in_bytes() == 64);
18311   match(Set dst (OrV src (LoadVector mem)));
18312   format %{ "vpor    $dst,$src,$mem\t! or vectors (64 bytes)" %}
18313   ins_encode %{
18314     int vector_len = 2;
18315     __ vpor($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
18316   %}
18317   ins_pipe( pipe_slow );
18318 %}
18319 
18320 // --------------------------------- XOR --------------------------------------
18321 
18322 instruct vxor4B(vecS dst, vecS src) %{
18323   predicate(n->as_Vector()->length_in_bytes() == 4);
18324   match(Set dst (XorV dst src));
18325   format %{ "pxor    $dst,$src\t! xor vectors (4 bytes)" %}
18326   ins_encode %{
18327     __ pxor($dst$$XMMRegister, $src$$XMMRegister);
18328   %}
18329   ins_pipe( pipe_slow );
18330 %}
18331 
18332 instruct vxor4B_reg(vecS dst, vecS src1, vecS src2) %{
18333   predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 4);
18334   match(Set dst (XorV src1 src2));
18335   format %{ "vpxor   $dst,$src1,$src2\t! xor vectors (4 bytes)" %}
18336   ins_encode %{
18337     int vector_len = 0;
18338     __ vpxor($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
18339   %}
18340   ins_pipe( pipe_slow );
18341 %}
18342 
18343 instruct vxor4B_mem(vecS dst, vecS src, memory mem) %{
18344   predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 4);
18345   match(Set dst (XorV src (LoadVector mem)));
18346   format %{ "vpxor   $dst,$src,$mem\t! xor vectors (4 bytes)" %}
18347   ins_encode %{
18348     int vector_len = 0;
18349     __ vpxor($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
18350   %}
18351   ins_pipe( pipe_slow );
18352 %}
18353 
18354 instruct vxor8B(vecD dst, vecD src) %{
18355   predicate(n->as_Vector()->length_in_bytes() == 8);
18356   match(Set dst (XorV dst src));
18357   format %{ "pxor    $dst,$src\t! xor vectors (8 bytes)" %}
18358   ins_encode %{
18359     __ pxor($dst$$XMMRegister, $src$$XMMRegister);
18360   %}
18361   ins_pipe( pipe_slow );
18362 %}
18363 
18364 instruct vxor8B_reg(vecD dst, vecD src1, vecD src2) %{
18365   predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 8);
18366   match(Set dst (XorV src1 src2));
18367   format %{ "vpxor   $dst,$src1,$src2\t! xor vectors (8 bytes)" %}
18368   ins_encode %{
18369     int vector_len = 0;
18370     __ vpxor($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
18371   %}
18372   ins_pipe( pipe_slow );
18373 %}
18374 
18375 instruct vxor8B_mem(vecD dst, vecD src, memory mem) %{
18376   predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 8);
18377   match(Set dst (XorV src (LoadVector mem)));
18378   format %{ "vpxor   $dst,$src,$mem\t! xor vectors (8 bytes)" %}
18379   ins_encode %{
18380     int vector_len = 0;
18381     __ vpxor($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
18382   %}
18383   ins_pipe( pipe_slow );
18384 %}
18385 
18386 instruct vxor16B(vecX dst, vecX src) %{
18387   predicate(n->as_Vector()->length_in_bytes() == 16);
18388   match(Set dst (XorV dst src));
18389   format %{ "pxor    $dst,$src\t! xor vectors (16 bytes)" %}
18390   ins_encode %{
18391     __ pxor($dst$$XMMRegister, $src$$XMMRegister);
18392   %}
18393   ins_pipe( pipe_slow );
18394 %}
18395 
18396 instruct vxor16B_reg(vecX dst, vecX src1, vecX src2) %{
18397   predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 16);
18398   match(Set dst (XorV src1 src2));
18399   format %{ "vpxor   $dst,$src1,$src2\t! xor vectors (16 bytes)" %}
18400   ins_encode %{
18401     int vector_len = 0;
18402     __ vpxor($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
18403   %}
18404   ins_pipe( pipe_slow );
18405 %}
18406 
18407 instruct vxor16B_mem(vecX dst, vecX src, memory mem) %{
18408   predicate(UseAVX > 0 && n->as_Vector()->length_in_bytes() == 16);
18409   match(Set dst (XorV src (LoadVector mem)));
18410   format %{ "vpxor   $dst,$src,$mem\t! xor vectors (16 bytes)" %}
18411   ins_encode %{
18412     int vector_len = 0;
18413     __ vpxor($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
18414   %}
18415   ins_pipe( pipe_slow );
18416 %}
18417 
18418 instruct vxor32B_reg(vecY dst, vecY src1, vecY src2) %{
18419   predicate(UseAVX > 1 && n->as_Vector()->length_in_bytes() == 32);
18420   match(Set dst (XorV src1 src2));
18421   format %{ "vpxor   $dst,$src1,$src2\t! xor vectors (32 bytes)" %}
18422   ins_encode %{
18423     int vector_len = 1;
18424     __ vpxor($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
18425   %}
18426   ins_pipe( pipe_slow );
18427 %}
18428 
18429 instruct vxor32B_mem(vecY dst, vecY src, memory mem) %{
18430   predicate(UseAVX > 1 && n->as_Vector()->length_in_bytes() == 32);
18431   match(Set dst (XorV src (LoadVector mem)));
18432   format %{ "vpxor   $dst,$src,$mem\t! xor vectors (32 bytes)" %}
18433   ins_encode %{
18434     int vector_len = 1;
18435     __ vpxor($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
18436   %}
18437   ins_pipe( pipe_slow );
18438 %}
18439 
18440 instruct vxor64B_reg(vecZ dst, vecZ src1, vecZ src2) %{
18441   predicate(UseAVX > 2 && n->as_Vector()->length_in_bytes() == 64);
18442   match(Set dst (XorV src1 src2));
18443   format %{ "vpxor   $dst,$src1,$src2\t! xor vectors (64 bytes)" %}
18444   ins_encode %{
18445     int vector_len = 2;
18446     __ vpxor($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
18447   %}
18448   ins_pipe( pipe_slow );
18449 %}
18450 
18451 instruct vxor64B_mem(vecZ dst, vecZ src, memory mem) %{
18452   predicate(UseAVX > 2 && n->as_Vector()->length_in_bytes() == 64);
18453   match(Set dst (XorV src (LoadVector mem)));
18454   format %{ "vpxor   $dst,$src,$mem\t! xor vectors (64 bytes)" %}
18455   ins_encode %{
18456     int vector_len = 2;
18457     __ vpxor($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vector_len);
18458   %}
18459   ins_pipe( pipe_slow );
18460 %}
18461 
18462 instruct vcvt4Bto4S_reg(vecD dst, vecS src) %{
18463   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
18464   match(Set dst (VectorCastB2X src));
18465   format %{ "vpmovsxbw   $dst,$src\t! convert 4B to 4S vector" %}
18466   ins_encode %{
18467     int vector_len = 0;
18468     __ vpmovsxbw($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18469   %}
18470   ins_pipe( pipe_slow );
18471 %}
18472 
18473 instruct vcvt8Bto8S_reg(vecX dst, vecD src) %{
18474   predicate(UseAVX > 0 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
18475   match(Set dst (VectorCastB2X src));
18476   format %{ "vpmovsxbw   $dst,$src\t! convert 8B to 8S vector" %}
18477   ins_encode %{
18478     int vector_len = 0;
18479     __ vpmovsxbw($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18480   %}
18481   ins_pipe( pipe_slow );
18482 %}
18483 
18484 instruct vcvt16Bto16S_reg(vecY dst, vecX src) %{
18485   predicate(UseAVX >= 2 && n->as_Vector()->length() == 16 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
18486   match(Set dst (VectorCastB2X src));
18487   format %{ "vpmovsxbw   $dst,$src\t! convert 16B to 16S vector" %}
18488   ins_encode %{
18489     int vector_len = 1;
18490     __ vpmovsxbw($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18491   %}
18492   ins_pipe( pipe_slow );
18493 %}
18494 
18495 instruct vcvt32Bto32S_reg(vecZ dst, vecY src) %{
18496   predicate(UseAVX > 2 && n->as_Vector()->length() == 32 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
18497   match(Set dst (VectorCastB2X src));
18498   format %{ "vpmovsxbw   $dst,$src\t! convert 32B to 32S vector" %}
18499   ins_encode %{
18500     int vector_len = 2;
18501     __ vpmovsxbw($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18502   %}
18503   ins_pipe( pipe_slow );
18504 %}
18505 
18506 instruct vcvt4Bto4I_reg(vecX dst, vecS src) %{
18507   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
18508   match(Set dst (VectorCastB2X src));
18509   format %{ "vpmovsxbd   $dst,$src\t! convert 4B to 4I vector" %}
18510   ins_encode %{
18511     int vector_len = 0;
18512     __ vpmovsxbd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18513   %}
18514   ins_pipe( pipe_slow );
18515 %}
18516 
18517 instruct vcvt8Bto8I_reg(vecY dst, vecD src) %{
18518   predicate(UseAVX >= 2 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
18519   match(Set dst (VectorCastB2X src));
18520   format %{ "vpmovsxbd   $dst,$src\t! convert 8B to 8I vector" %}
18521   ins_encode %{
18522     int vector_len = 1;
18523     __ vpmovsxbd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18524   %}
18525   ins_pipe( pipe_slow );
18526 %}
18527 
18528 instruct vcvt16Bto16I_reg(vecZ dst, vecX src) %{
18529   predicate(UseAVX > 2 && n->as_Vector()->length() == 16 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
18530   match(Set dst (VectorCastB2X src));
18531   format %{ "vpmovsxbd   $dst,$src\t! convert 16B to 16I vector" %}
18532   ins_encode %{
18533     int vector_len = 2;
18534     __ vpmovsxbd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18535   %}
18536   ins_pipe( pipe_slow );
18537 %}
18538 
18539 instruct vcvt4Bto4L_reg(vecY dst, vecS src) %{
18540   predicate(UseAVX >= 2 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
18541   match(Set dst (VectorCastB2X src));
18542   format %{ "vpmovsxbq   $dst,$src\t! convert 4B to 4L vector" %}
18543   ins_encode %{
18544     int vector_len = 1;
18545     __ vpmovsxbq($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18546   %}
18547   ins_pipe( pipe_slow );
18548 %}
18549 
18550 instruct vcvt8Bto8L_reg(vecZ dst, vecD src) %{
18551   predicate(UseAVX > 2 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
18552   match(Set dst (VectorCastB2X src));
18553   format %{ "vpmovsxbq   $dst,$src\t! convert 8B to 8L vector" %}
18554   ins_encode %{
18555     int vector_len = 2;
18556     __ vpmovsxbq($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18557   %}
18558   ins_pipe( pipe_slow );
18559 %}
18560 
18561 instruct vcvt4Bto4F_reg(vecX dst, vecS src) %{
18562   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
18563   match(Set dst (VectorCastB2X src));
18564   format %{ "vpmovsxbd   $dst,$src\n\t"
18565             "vcvtdq2ps   $dst,$dst\t! convert 4B to 4F vector" %}
18566   ins_encode %{
18567     int vector_len = 0;
18568     __ vpmovsxbd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18569     __ vcvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
18570   %}
18571   ins_pipe( pipe_slow );
18572 %}
18573 
18574 instruct vcvt8Bto8F_reg(vecY dst, vecD src) %{
18575   predicate(UseAVX >= 2 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
18576   match(Set dst (VectorCastB2X src));
18577   format %{ "vpmovsxbd   $dst,$src\n\t"
18578             "vcvtdq2ps   $dst,$dst\t! convert 8B to 8F vector" %}
18579   ins_encode %{
18580     int vector_len = 1;
18581     __ vpmovsxbd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18582     __ vcvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
18583   %}
18584   ins_pipe( pipe_slow );
18585 %}
18586 
18587 instruct vcvt16Bto16F_reg(vecZ dst, vecX src) %{
18588   predicate(UseAVX > 2 && n->as_Vector()->length() == 16 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
18589   match(Set dst (VectorCastB2X src));
18590   format %{ "vpmovsxbd   $dst,$src\n\t"
18591             "vcvtdq2ps   $dst,$dst\t! convert 16B to 16F vector" %}
18592   ins_encode %{
18593     int vector_len = 2;
18594     __ vpmovsxbd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18595     __ vcvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
18596   %}
18597   ins_pipe( pipe_slow );
18598 %}
18599 
18600 instruct vcvt4Bto4D_reg(vecY dst, vecS src) %{
18601   predicate(UseAVX >= 2 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
18602   match(Set dst (VectorCastB2X src));
18603   format %{ "vpmovsxbd   $dst,$src\n\t"
18604             "vcvtdq2pd   $dst,$dst\t! convert 4B to 4D vector" %}
18605   ins_encode %{
18606     __ vpmovsxbd($dst$$XMMRegister, $src$$XMMRegister, 0);
18607     __ vcvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister, 1);
18608   %}
18609   ins_pipe( pipe_slow );
18610 %}
18611 
18612 instruct vcvt8Bto8D_reg(vecZ dst, vecD src) %{
18613   predicate(UseAVX > 2 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
18614   match(Set dst (VectorCastB2X src));
18615   format %{ "vpmovsxbd   $dst,$src\n\t"
18616             "vcvtdq2pd   $dst,$dst\t! convert 8B to 8D vector" %}
18617   ins_encode %{
18618     __ vpmovsxbd($dst$$XMMRegister, $src$$XMMRegister, 1);
18619     __ vcvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister, 2);
18620   %}
18621   ins_pipe( pipe_slow );
18622 %}
18623 
18624 instruct vcvt4Sto4B_reg(vecS dst, vecD src, rRegL scratch) %{
18625   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
18626   effect(TEMP scratch);
18627   match(Set dst (VectorCastS2X src));
18628   format %{ "vpand      $dst,$src,[0x00FF00FF00FF00FF]\n\t"
18629             "vpackuswb  $dst,$dst\t! convert 4S to 4B vector" %}
18630   ins_encode %{
18631     int vector_len = 0;
18632     __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), vector_len, $scratch$$Register);
18633     __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vector_len);
18634   %}
18635   ins_pipe( pipe_slow );
18636 %}
18637 
18638 instruct vcvt8Sto8B_reg(vecD dst, vecX src, rRegL scratch) %{
18639   predicate(UseAVX > 0 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
18640   effect(TEMP scratch);
18641   match(Set dst (VectorCastS2X src));
18642   format %{ "vpand      $dst,$src,[0x00FF00FF00FF00FF]\n\t"
18643             "vpackuswb  $dst,$dst\t! convert 8S to 8B vector" %}
18644   ins_encode %{
18645     int vector_len = 0;
18646     __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), vector_len, $scratch$$Register);
18647     __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vector_len);
18648   %}
18649   ins_pipe( pipe_slow );
18650 %}
18651 
18652 instruct vcvt16Sto16B_reg(vecX dst, vecY src, vecY tmp, rRegL scratch) %{
18653   predicate(UseAVX > 0 && n->as_Vector()->length() == 16 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
18654   effect(TEMP scratch, TEMP tmp);
18655   match(Set dst (VectorCastS2X src));
18656     format %{ "vpand      $dst,$src,[0x00FF00FF00FF00FF]\n\t"
18657               "vextracti128 $tmp,$dst,0x1\n\t"
18658               "vpackuswb  $dst,$dst,$tmp\t! convert 16S to 16B vector" %}
18659   ins_encode %{
18660     int vector_len = 1;
18661     __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), vector_len, $scratch$$Register);
18662     __ vextracti128($tmp$$XMMRegister, $dst$$XMMRegister, 0x1);
18663     __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
18664   %}
18665   ins_pipe( pipe_slow );
18666 %}
18667 
18668 instruct vcvt32Sto32B_reg(vecY dst, vecZ src) %{
18669   predicate(UseAVX > 2 && n->as_Vector()->length() == 32 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
18670   match(Set dst (VectorCastS2X src));
18671     format %{ "evpmovwb   $dst,$src\t! convert 32S to 32B vector" %}
18672   ins_encode %{
18673     int vector_len = 2;
18674     __ evpmovwb($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18675   %}
18676   ins_pipe( pipe_slow );
18677 %}
18678 
18679 instruct vcvt2Sto2I_reg(vecD dst, vecS src) %{
18680   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
18681   match(Set dst (VectorCastS2X src));
18682   format %{ "vpmovsxwd   $dst,$src\t! convert 2S to 2I vector" %}
18683   ins_encode %{
18684     int vector_len = 0;
18685     __ vpmovsxwd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18686   %}
18687   ins_pipe( pipe_slow );
18688 %}
18689 
18690 instruct vcvt4Sto4I_reg(vecX dst, vecD src) %{
18691   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
18692   match(Set dst (VectorCastS2X src));
18693   format %{ "vpmovsxwd   $dst,$src\t! convert 4S to 4I vector" %}
18694   ins_encode %{
18695     int vector_len = 0;
18696     __ vpmovsxwd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18697   %}
18698   ins_pipe( pipe_slow );
18699 %}
18700 
18701 instruct vcvt8Sto8I_reg(vecY dst, vecX src) %{
18702   predicate(UseAVX >= 2 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
18703   match(Set dst (VectorCastS2X src));
18704   format %{ "vpmovsxwd   $dst,$src\t! convert 8S to 8I vector" %}
18705   ins_encode %{
18706     int vector_len = 1;
18707     __ vpmovsxwd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18708   %}
18709   ins_pipe( pipe_slow );
18710 %}
18711 
18712 instruct vcvt16Sto16I_reg(vecZ dst, vecY src) %{
18713   predicate(UseAVX > 2 && n->as_Vector()->length() == 16 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
18714   match(Set dst (VectorCastS2X src));
18715   format %{ "vpmovsxwd   $dst,$src\t! convert 16S to 16I vector" %}
18716   ins_encode %{
18717     int vector_len = 2;
18718     __ vpmovsxwd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18719   %}
18720   ins_pipe( pipe_slow );
18721 %}
18722 
18723 instruct vcvt2Sto2L_reg(vecX dst, vecS src) %{
18724   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
18725   match(Set dst (VectorCastS2X src));
18726   format %{ "vpmovsxwq   $dst,$src\t! convert 2S to 2L vector" %}
18727   ins_encode %{
18728     int vector_len = 0;
18729     __ vpmovsxwq($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18730   %}
18731   ins_pipe( pipe_slow );
18732 %}
18733 
18734 instruct vcvt4Sto4L_reg(vecY dst, vecD src) %{
18735   predicate(UseAVX >= 2 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
18736   match(Set dst (VectorCastS2X src));
18737   format %{ "vpmovsxwq   $dst,$src\t! convert 4S to 4L vector" %}
18738   ins_encode %{
18739     int vector_len = 1;
18740     __ vpmovsxwq($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18741   %}
18742   ins_pipe( pipe_slow );
18743 %}
18744 
18745 instruct vcvt8Sto8L_reg(vecZ dst, vecX src) %{
18746   predicate(UseAVX > 2 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
18747   match(Set dst (VectorCastS2X src));
18748   format %{ "vpmovsxwq   $dst,$src\t! convert 8S to 8L vector" %}
18749   ins_encode %{
18750     int vector_len = 2;
18751     __ vpmovsxwq($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18752   %}
18753   ins_pipe( pipe_slow );
18754 %}
18755 
18756 instruct vcvt2Sto2F_reg(vecD dst, vecS src) %{
18757   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
18758   match(Set dst (VectorCastS2X src));
18759   format %{ "vpmovsxwd   $dst,$src\n\t"
18760             "vcvtdq2ps   $dst,$dst\t! convert 2S to 2F vector" %}
18761   ins_encode %{
18762     int vector_len = 0;
18763     __ vpmovsxwd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18764     __ vcvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
18765   %}
18766   ins_pipe( pipe_slow );
18767 %}
18768 
18769 instruct vcvt4Sto4F_reg(vecX dst, vecD src) %{
18770   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
18771   match(Set dst (VectorCastS2X src));
18772   format %{ "vpmovsxwd   $dst,$src\n\t"
18773             "vcvtdq2ps   $dst,$dst\t! convert 4S to 4F vector" %}
18774   ins_encode %{
18775     int vector_len = 0;
18776     __ vpmovsxwd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18777     __ vcvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
18778   %}
18779   ins_pipe( pipe_slow );
18780 %}
18781 
18782 instruct vcvt8Sto8F_reg(vecY dst, vecX src) %{
18783   predicate(UseAVX >= 2 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
18784   match(Set dst (VectorCastS2X src));
18785   format %{ "vpmovsxwd   $dst,$src\n\t"
18786             "vcvtdq2ps   $dst,$dst\t! convert 8S to 8F vector" %}
18787   ins_encode %{
18788     int vector_len = 1;
18789     __ vpmovsxwd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18790     __ vcvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
18791   %}
18792   ins_pipe( pipe_slow );
18793 %}
18794 
18795 instruct vcvt16Sto16F_reg(vecZ dst, vecY src) %{
18796   predicate(UseAVX > 2 && n->as_Vector()->length() == 16 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
18797   match(Set dst (VectorCastS2X src));
18798   format %{ "vpmovsxwd   $dst,$src\n\t"
18799             "vcvtdq2ps   $dst,$dst\t! convert 16S to 16F vector" %}
18800   ins_encode %{
18801     int vector_len = 2;
18802     __ vpmovsxwd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18803     __ vcvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
18804   %}
18805   ins_pipe( pipe_slow );
18806 %}
18807 
18808 instruct vcvt2Sto2D_reg(vecX dst, vecS src) %{
18809   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
18810   match(Set dst (VectorCastS2X src));
18811   format %{ "vpmovsxwd   $dst,$src\n\t"
18812             "vcvtdq2pd   $dst,$dst\t! convert 2S to 2D vector" %}
18813   ins_encode %{
18814     int vector_len = 0;
18815     __ vpmovsxwd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18816     __ vcvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
18817   %}
18818   ins_pipe( pipe_slow );
18819 %}
18820 
18821 instruct vcvt4Sto4D_reg(vecY dst, vecD src) %{
18822   predicate(UseAVX >= 2 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
18823   match(Set dst (VectorCastS2X src));
18824   format %{ "vpmovsxwd   $dst,$src\n\t"
18825             "vcvtdq2pd   $dst,$dst\t! convert 4S to 4D vector" %}
18826   ins_encode %{
18827     int vector_len = 1;
18828     __ vpmovsxwd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18829     __ vcvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
18830   %}
18831   ins_pipe( pipe_slow );
18832 %}
18833 
18834 instruct vcvt8Sto8D_reg(vecZ dst, vecX src) %{
18835   predicate(UseAVX > 2 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
18836   match(Set dst (VectorCastS2X src));
18837   format %{ "vpmovsxwd   $dst,$src\n\t"
18838             "vcvtdq2pd   $dst,$dst\t! convert 8S to 8D vector" %}
18839   ins_encode %{
18840     int vector_len = 2;
18841     __ vpmovsxwd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18842     __ vcvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister, vector_len);
18843   %}
18844   ins_pipe( pipe_slow );
18845 %}
18846 
18847 instruct vcvt4Ito4B_reg(vecS dst, vecX src, rRegL scratch) %{
18848   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
18849   effect(TEMP scratch);
18850   match(Set dst (VectorCastI2X src));
18851   format %{ "vpand      $dst,$src,[0x000000FF000000FF]\n\t"
18852             "vpackusdw  $dst,$dst\n\t"
18853             "vpackuswb  $dst,$dst\t! convert 4I to 4B vector" %}
18854   ins_encode %{
18855     int vector_len = 0;
18856     __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_int_to_byte_mask()), vector_len, $scratch$$Register);
18857     __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vector_len);
18858     __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vector_len);
18859   %}
18860   ins_pipe( pipe_slow );
18861 %}
18862 
18863 instruct vcvt8Ito8B_reg(vecD dst, vecY src, vecY tmp, rRegL scratch) %{
18864   predicate(UseAVX >= 2 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
18865   effect(TEMP scratch, TEMP tmp);
18866   match(Set dst (VectorCastI2X src));
18867   format %{ "vpand      $dst,$src,[0x000000FF000000FF]\n\t"
18868             "vextracti128 $tmp,$dst,0x1\n\t"
18869             "vpackusdw  $dst,$dst,$tmp\n\t"
18870             "vpackuswb  $dst,$dst\t! convert 8I to 8B vector" %}
18871   ins_encode %{
18872     int vector_len = 1;
18873     __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_int_to_byte_mask()), vector_len, $scratch$$Register);
18874     __ vextracti128($tmp$$XMMRegister, $dst$$XMMRegister, 0x1);
18875     __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
18876     __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, 0);
18877   %}
18878   ins_pipe( pipe_slow );
18879 %}
18880 
18881 instruct vcvt16Ito16B_reg(vecX dst, vecZ src) %{
18882   predicate(UseAVX > 2 && n->as_Vector()->length() == 16 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
18883   match(Set dst (VectorCastI2X src));
18884     format %{ "evpmovdb   $dst,$src\t! convert 16I to 16B vector" %}
18885   ins_encode %{
18886     int vector_len = 2;
18887     __ evpmovdb($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18888   %}
18889   ins_pipe( pipe_slow );
18890 %}
18891 
18892 instruct vcvt2Ito2S_reg(vecS dst, vecD src, rRegL scratch) %{
18893   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
18894   effect(TEMP scratch);
18895   match(Set dst (VectorCastI2X src));
18896   format %{ "vpand      $dst,$src,[0x0000FFFF0000FFFF]\n\t"
18897             "vpackusdw  $dst,$dst\t! convert 2I to 2S vector" %}
18898   ins_encode %{
18899     int vector_len = 0;
18900     __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_int_to_short_mask()), vector_len, $scratch$$Register);
18901     __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vector_len);
18902   %}
18903   ins_pipe( pipe_slow );
18904 %}
18905 
18906 instruct vcvt4Ito4S_reg(vecD dst, vecX src, rRegL scratch) %{
18907   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
18908   effect(TEMP scratch);
18909   match(Set dst (VectorCastI2X src));
18910   format %{ "vpand      $dst,$src,[0x0000FFFF0000FFFF]\n\t"
18911             "vpackusdw  $dst,$dst\t! convert 4I to 4S vector" %}
18912   ins_encode %{
18913     int vector_len = 0;
18914     __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_int_to_short_mask()), vector_len, $scratch$$Register);
18915     __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vector_len);
18916   %}
18917   ins_pipe( pipe_slow );
18918 %}
18919 
18920 instruct vcvt8Ito8S_reg(vecX dst, vecY src, vecY tmp, rRegL scratch) %{
18921   predicate(UseAVX >= 2 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
18922   effect(TEMP scratch, TEMP tmp);
18923   match(Set dst (VectorCastI2X src));
18924   format %{ "vpand      $dst,$src,[0x0000FFFF0000FFFF]\n\t"
18925             "vextracti128 $tmp,$dst,0x1\n\t"
18926             "vpackusdw  $dst,$dst,$tmp\t! convert 8I to 8S vector" %}
18927   ins_encode %{
18928     int vector_len = 1;
18929     __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_int_to_short_mask()), vector_len, $scratch$$Register);
18930     __ vextracti128($tmp$$XMMRegister, $dst$$XMMRegister, 0x1);
18931     __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vector_len);
18932   %}
18933   ins_pipe( pipe_slow );
18934 %}
18935 
18936 instruct vcvt16Ito16S_reg(vecY dst, vecZ src) %{
18937   predicate(UseAVX > 2 && n->as_Vector()->length() == 16 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
18938   match(Set dst (VectorCastI2X src));
18939     format %{ "evpmovdw   $dst,$src\t! convert 16I to 16S vector" %}
18940   ins_encode %{
18941     int vector_len = 2;
18942     __ evpmovdw($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18943   %}
18944   ins_pipe( pipe_slow );
18945 %}
18946 
18947 instruct vcvt2Ito2L_reg(vecX dst, vecD src) %{
18948   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
18949   match(Set dst (VectorCastI2X src));
18950   format %{ "vpmovsxdq   $dst,$src\t! convert 2I to 2L vector" %}
18951   ins_encode %{
18952     int vector_len = 0;
18953     __ vpmovsxdq($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18954   %}
18955   ins_pipe( pipe_slow );
18956 %}
18957 
18958 instruct vcvt4Ito4L_reg(vecY dst, vecX src) %{
18959   predicate(UseAVX >= 2 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
18960   match(Set dst (VectorCastI2X src));
18961   format %{ "vpmovsxdq   $dst,$src\t! convert 4I to 4L vector" %}
18962   ins_encode %{
18963     int vector_len = 1;
18964     __ vpmovsxdq($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18965   %}
18966   ins_pipe( pipe_slow );
18967 %}
18968 
18969 instruct vcvt8Ito8L_reg(vecZ dst, vecY src) %{
18970   predicate(UseAVX >= 2 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
18971   match(Set dst (VectorCastI2X src));
18972   format %{ "vpmovsxdq   $dst,$src\t! convert 8I to 8L vector" %}
18973   ins_encode %{
18974     int vector_len = 2;
18975     __ vpmovsxdq($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18976   %}
18977   ins_pipe( pipe_slow );
18978 %}
18979 
18980 instruct vcvt2Ito2F_reg(vecD dst, vecD src) %{
18981   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
18982   match(Set dst (VectorCastI2X src));
18983   format %{ "vcvtdq2ps   $dst,$src\t! convert 2I to 2F vector" %}
18984   ins_encode %{
18985     int vector_len = 0;
18986     __ vcvtdq2ps($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18987   %}
18988   ins_pipe( pipe_slow );
18989 %}
18990 
18991 instruct vcvt4Ito4F_reg(vecX dst, vecX src) %{
18992   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
18993   match(Set dst (VectorCastI2X src));
18994   format %{ "vcvtdq2ps   $dst,$src\t! convert 4I to 4F vector" %}
18995   ins_encode %{
18996     int vector_len = 0;
18997     __ vcvtdq2ps($dst$$XMMRegister, $src$$XMMRegister, vector_len);
18998   %}
18999   ins_pipe( pipe_slow );
19000 %}
19001 
19002 instruct vcvt8Ito8F_reg(vecY dst, vecY src) %{
19003   predicate(UseAVX > 0 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19004   match(Set dst (VectorCastI2X src));
19005   format %{ "vcvtdq2ps   $dst,$src\t! convert 8I to 8F vector" %}
19006   ins_encode %{
19007     int vector_len = 1;
19008     __ vcvtdq2ps($dst$$XMMRegister, $src$$XMMRegister, vector_len);
19009   %}
19010   ins_pipe( pipe_slow );
19011 %}
19012 
19013 instruct vcvt16Ito16F_reg(vecY dst, vecY src) %{
19014   predicate(UseAVX > 2 && n->as_Vector()->length() == 16 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19015   match(Set dst (VectorCastI2X src));
19016   format %{ "vcvtdq2ps   $dst,$src\t! convert 16I to 16F vector" %}
19017   ins_encode %{
19018     int vector_len = 2;
19019     __ vcvtdq2ps($dst$$XMMRegister, $src$$XMMRegister, vector_len);
19020   %}
19021   ins_pipe( pipe_slow );
19022 %}
19023 
19024 instruct vcvt2Ito2D_reg(vecX dst, vecD src) %{
19025   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
19026   match(Set dst (VectorCastI2X src));
19027   format %{ "vcvtdq2pd   $dst,$src\t! convert 2I to 2D vector" %}
19028   ins_encode %{
19029     int vector_len = 0;
19030     __ vcvtdq2pd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
19031   %}
19032   ins_pipe( pipe_slow );
19033 %}
19034 
19035 instruct vcvt4Ito4D_reg(vecY dst, vecX src) %{
19036   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
19037   match(Set dst (VectorCastI2X src));
19038   format %{ "vcvtdq2pd   $dst,$src\t! convert 4I to 4D vector" %}
19039   ins_encode %{
19040     int vector_len = 1;
19041     __ vcvtdq2pd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
19042   %}
19043   ins_pipe( pipe_slow );
19044 %}
19045 
19046 instruct vcvt8Ito8D_reg(vecZ dst, vecY src) %{
19047   predicate(UseAVX >= 2 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
19048   match(Set dst (VectorCastI2X src));
19049   format %{ "vcvtdq2pd   $dst,$src\t! convert 8I to 8D vector" %}
19050   ins_encode %{
19051     int vector_len = 2;
19052     __ vcvtdq2pd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
19053   %}
19054   ins_pipe( pipe_slow );
19055 %}
19056 
19057 instruct vcvt4Lto4B_reg(vecS dst, vecY src, rRegL scratch) %{
19058   predicate(UseAVX >= 2 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
19059   match(Set dst (VectorCastL2X src));
19060   effect(TEMP scratch);
19061   format %{ "vpermilps  $dst,$src,8\n\t"
19062             "vpermpd    $dst,$dst,8\n\t"
19063             "vpand      $dst,$dst,[0x000000FF000000FF]\n\t"
19064             "vpackusdw  $dst,$dst\n\t"
19065             "vpackuswb  $dst,$dst\t! convert 4L to 4B vector" %}
19066   ins_encode %{
19067     int vector_len = 1;
19068     __ vpermilps($dst$$XMMRegister, $src$$XMMRegister, 8, vector_len);
19069     __ vpermpd($dst$$XMMRegister, $dst$$XMMRegister, 8, vector_len);
19070     // Since cast to int has been done, do rest of operations in 128.
19071     vector_len = 0;
19072     __ vpand($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_int_to_byte_mask()), vector_len, $scratch$$Register);
19073     __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vector_len);
19074     __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vector_len);
19075   %}
19076   ins_pipe( pipe_slow );
19077 %}
19078 
19079 instruct vcvt8Lto8B_reg(vecD dst, vecZ src) %{
19080   predicate(UseAVX > 2 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
19081   match(Set dst (VectorCastL2X src));
19082     format %{ "evpmovqb   $dst,$src\t! convert 8L to 8B vector" %}
19083   ins_encode %{
19084     int vector_len = 2;
19085     __ evpmovqb($dst$$XMMRegister, $src$$XMMRegister, vector_len);
19086   %}
19087   ins_pipe( pipe_slow );
19088 %}
19089 
19090 instruct vcvt2Lto2S_reg(vecS dst, vecX src, rRegL scratch) %{
19091   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
19092   match(Set dst (VectorCastL2X src));
19093   effect(TEMP scratch);
19094   format %{ "vpshufd    $dst,$src,8\n\t"
19095             "vpand      $dst,$dst,[0x0000FFFF0000FFFF]\n\t"
19096             "vpackusdw  $dst,$dst\t! convert 2L to 2S vector" %}
19097   ins_encode %{
19098     int vector_len = 0;
19099     __ vpshufd($dst$$XMMRegister, $src$$XMMRegister, 8, vector_len);
19100     __ vpand($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_int_to_short_mask()), vector_len, $scratch$$Register);
19101     __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vector_len);
19102   %}
19103   ins_pipe( pipe_slow );
19104 %}
19105 
19106 instruct vcvt4Lto4S_reg(vecD dst, vecY src, rRegL scratch) %{
19107   predicate(UseAVX >= 2 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
19108   match(Set dst (VectorCastL2X src));
19109   effect(TEMP scratch);
19110   format %{ "vpermilps  $dst,$src,8\n\t"
19111             "vpermpd    $dst,$dst,8\n\t"
19112             "vpand      $dst,$dst,[0x0000FFFF0000FFFF]\n\t"
19113             "vpackusdw  $dst,$dst\t! convert 4L to 4S vector" %}
19114   ins_encode %{
19115     int vector_len = 1;
19116     __ vpermilps($dst$$XMMRegister, $src$$XMMRegister, 8, vector_len);
19117     __ vpermpd($dst$$XMMRegister, $dst$$XMMRegister, 8, vector_len);
19118     // Since cast to int has been done, do rest of operations in 128.
19119     vector_len = 0;
19120     __ vpand($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_int_to_short_mask()), vector_len, $scratch$$Register);
19121     __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vector_len);
19122   %}
19123   ins_pipe( pipe_slow );
19124 %}
19125 
19126 instruct vcvt8Lto8S_reg(vecX dst, vecZ src) %{
19127   predicate(UseAVX > 2 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
19128   match(Set dst (VectorCastL2X src));
19129     format %{ "evpmovqw   $dst,$src\t! convert 8L to 8S vector" %}
19130   ins_encode %{
19131     int vector_len = 2;
19132     __ evpmovqw($dst$$XMMRegister, $src$$XMMRegister, vector_len);
19133   %}
19134   ins_pipe( pipe_slow );
19135 %}
19136 
19137 instruct vcvt1Lto1I_reg(vecS dst, vecD src) %{
19138   predicate(n->as_Vector()->length() == 1 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
19139   match(Set dst (VectorCastL2X src));
19140   format %{ "movdqu   $dst,$src\t! convert 1L to 1I vector" %}
19141   ins_encode %{
19142     // If register is the same, then move is not needed.
19143     if ($dst$$XMMRegister != $src$$XMMRegister) {
19144       __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
19145     }
19146   %}
19147   ins_pipe( pipe_slow );
19148 %}
19149 
19150 instruct vcvt2Lto2I_reg(vecD dst, vecX src) %{
19151   predicate(UseAVX == 0 && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
19152   match(Set dst (VectorCastL2X src));
19153   format %{ "pshufd   $dst,$src,8\t! convert 2L to 2I vector" %}
19154   ins_encode %{
19155     __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 8);
19156   %}
19157   ins_pipe( pipe_slow );
19158 %}
19159 
19160 instruct vcvt2Lto2I_reg_avx(vecD dst, vecX src) %{
19161   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
19162   match(Set dst (VectorCastL2X src));
19163   format %{ "vpshufd   $dst,$src,8\t! convert 2L to 2I vector" %}
19164   ins_encode %{
19165     int vector_len = 0;
19166     __ vpshufd($dst$$XMMRegister, $src$$XMMRegister, 8, vector_len);
19167   %}
19168   ins_pipe( pipe_slow );
19169 %}
19170 
19171 instruct vcvt4Lto4I_reg(vecX dst, vecY src) %{
19172   predicate(UseAVX >= 2 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
19173   match(Set dst (VectorCastL2X src));
19174   format %{ "vpermilps  $dst,$src,8\n\t"
19175           "vpermpd  $dst,$dst,8\t! convert 4L to 4I vector" %}
19176   ins_encode %{
19177     int vector_len = 1;
19178     __ vpermilps($dst$$XMMRegister, $src$$XMMRegister, 8, vector_len);
19179     __ vpermpd($dst$$XMMRegister, $dst$$XMMRegister, 8, vector_len);
19180   %}
19181   ins_pipe( pipe_slow );
19182 %}
19183 
19184 instruct vcvt8Lto8I_reg(vecY dst, vecZ src) %{
19185   predicate(UseAVX > 2 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_INT);
19186   match(Set dst (VectorCastL2X src));
19187     format %{ "evpmovqd   $dst,$src\t! convert 8L to 8I vector" %}
19188   ins_encode %{
19189     int vector_len = 2;
19190     __ evpmovqd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
19191   %}
19192   ins_pipe( pipe_slow );
19193 %}
19194 
19195 instruct vcvt2Lto2F_reg(vecD dst, vecX src) %{
19196   predicate(UseAVX > 2 && VM_Version::supports_avx512dq() && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19197   match(Set dst (VectorCastL2X src));
19198   format %{ "vcvtqq2ps   $dst,$src\t! convert 2L to 2F vector" %}
19199   ins_encode %{
19200     int vector_len = 0;
19201     __ evcvtqq2ps($dst$$XMMRegister, $src$$XMMRegister, vector_len);
19202   %}
19203   ins_pipe( pipe_slow );
19204 %}
19205 
19206 instruct vcvt4Lto4F_reg(vecX dst, vecY src) %{
19207   predicate(UseAVX > 2 && VM_Version::supports_avx512dq() && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19208   match(Set dst (VectorCastL2X src));
19209   format %{ "vcvtqq2ps   $dst,$src\t! convert 4L to 4F vector" %}
19210   ins_encode %{
19211     int vector_len = 1;
19212     __ evcvtqq2ps($dst$$XMMRegister, $src$$XMMRegister, vector_len);
19213   %}
19214   ins_pipe( pipe_slow );
19215 %}
19216 
19217 instruct vcvt8Lto8F_reg(vecY dst, vecZ src) %{
19218   predicate(UseAVX > 2 && VM_Version::supports_avx512dq() && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19219   match(Set dst (VectorCastL2X src));
19220   format %{ "vcvtqq2ps   $dst,$src\t! convert 8L to 8F vector" %}
19221   ins_encode %{
19222     int vector_len = 2;
19223     __ evcvtqq2ps($dst$$XMMRegister, $src$$XMMRegister, vector_len);
19224   %}
19225   ins_pipe( pipe_slow );
19226 %}
19227 
19228 instruct vcvt1Lto1D_reg(vecD dst, vecD src) %{
19229   predicate(UseAVX > 2 && VM_Version::supports_avx512dq() && n->as_Vector()->length() == 1 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
19230   match(Set dst (VectorCastL2X src));
19231   format %{ "vcvtqq2pd   $dst,$src\t! convert 1L to 1D vector" %}
19232   ins_encode %{
19233     int vector_len = 0;
19234     __ evcvtqq2pd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
19235   %}
19236   ins_pipe( pipe_slow );
19237 %}
19238 
19239 instruct vcvt2Lto2D_reg(vecX dst, vecX src) %{
19240   predicate(UseAVX > 2 && VM_Version::supports_avx512dq() && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
19241   match(Set dst (VectorCastL2X src));
19242   format %{ "vcvtqq2pd   $dst,$src\t! convert 2L to 2D vector" %}
19243   ins_encode %{
19244     int vector_len = 0;
19245     __ evcvtqq2pd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
19246   %}
19247   ins_pipe( pipe_slow );
19248 %}
19249 
19250 instruct vcvt4Lto4D_reg(vecY dst, vecY src) %{
19251   predicate(UseAVX > 2 && VM_Version::supports_avx512dq() && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
19252   match(Set dst (VectorCastL2X src));
19253   format %{ "vcvtqq2pd   $dst,$src\t! convert 4L to 4D vector" %}
19254   ins_encode %{
19255     int vector_len = 1;
19256     __ evcvtqq2pd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
19257   %}
19258   ins_pipe( pipe_slow );
19259 %}
19260 
19261 instruct vcvt8Lto8D_reg(vecZ dst, vecZ src) %{
19262   predicate(UseAVX > 2 && VM_Version::supports_avx512dq() && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
19263   match(Set dst (VectorCastL2X src));
19264   format %{ "vcvtqq2pd   $dst,$src\t! convert 8L to 8D vector" %}
19265   ins_encode %{
19266     int vector_len = 2;
19267     __ evcvtqq2pd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
19268   %}
19269   ins_pipe( pipe_slow );
19270 %}
19271 
19272 instruct vcvt2Fto2D_reg(vecX dst, vecD src) %{
19273   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
19274   match(Set dst (VectorCastF2X src));
19275   format %{ "vcvtps2pd   $dst,$src\t! convert 2F to 2D vector" %}
19276   ins_encode %{
19277     int vector_len = 0;
19278     __ vcvtps2pd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
19279   %}
19280   ins_pipe( pipe_slow );
19281 %}
19282 
19283 instruct vcvt4Fto4D_reg(vecY dst, vecX src) %{
19284   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
19285   match(Set dst (VectorCastF2X src));
19286   format %{ "vcvtps2pd   $dst,$src\t! convert 4F to 4D vector" %}
19287   ins_encode %{
19288     int vector_len = 1;
19289     __ vcvtps2pd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
19290   %}
19291   ins_pipe( pipe_slow );
19292 %}
19293 
19294 instruct vcvt8Fto8D_reg(vecZ dst, vecY src) %{
19295   predicate(UseAVX > 2 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
19296   match(Set dst (VectorCastF2X src));
19297   format %{ "evcvtps2pd   $dst,$src\t! convert 8F to 8D vector" %}
19298   ins_encode %{
19299     int vector_len = 2;
19300     __ evcvtps2pd($dst$$XMMRegister, $src$$XMMRegister, vector_len);
19301   %}
19302   ins_pipe( pipe_slow );
19303 %}
19304 
19305 instruct vcvt2Dto2F_reg(vecD dst, vecX src) %{
19306   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19307   match(Set dst (VectorCastD2X src));
19308   format %{ "vcvtpd2ps   $dst,$src\t! convert 2D to 2F vector" %}
19309   ins_encode %{
19310     int vector_len = 0;
19311     __ vcvtpd2ps($dst$$XMMRegister, $src$$XMMRegister, vector_len);
19312   %}
19313   ins_pipe( pipe_slow );
19314 %}
19315 
19316 instruct vcvt4Dto4F_reg(vecX dst, vecY src) %{
19317   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19318   match(Set dst (VectorCastD2X src));
19319   format %{ "vcvtpd2ps   $dst,$src\t! convert 4D to 4F vector" %}
19320   ins_encode %{
19321     int vector_len = 1;
19322     __ vcvtpd2ps($dst$$XMMRegister, $src$$XMMRegister, vector_len);
19323   %}
19324   ins_pipe( pipe_slow );
19325 %}
19326 
19327 instruct vcvt8Dto8F_reg(vecY dst, vecZ src) %{
19328   predicate(UseAVX > 2 && n->as_Vector()->length() == 8 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19329   match(Set dst (VectorCastD2X src));
19330   format %{ "evcvtpd2ps   $dst,$src\t! convert 8D to 8F vector" %}
19331   ins_encode %{
19332     int vector_len = 2;
19333     __ evcvtpd2ps($dst$$XMMRegister, $src$$XMMRegister, vector_len);
19334   %}
19335   ins_pipe( pipe_slow );
19336 %}
19337 
19338 instruct vcmpeq2F(vecD dst, vecD src1, vecD src2) %{
19339   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 &&
19340             n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
19341             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19342   match(Set dst (VectorMaskCmp src1 src2));
19343   format %{ "vcmpeqps  $dst,$src1,$src2\t! cmpeq packed2F" %}
19344   ins_encode %{
19345     int vector_len = 0;
19346     Assembler::ComparisonPredicateFP cmp = Assembler::EQ_OQ;  // ordered non-signaling
19347     __ vcmpps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19348   %}
19349   ins_pipe( pipe_slow );
19350 %}
19351 
19352 instruct vcmpeq4F(vecX dst, vecX src1, vecX src2) %{
19353   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 &&
19354             n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
19355             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19356   match(Set dst (VectorMaskCmp src1 src2));
19357   format %{ "vcmpeqps  $dst,$src1,$src2\t! cmpeq packed4F" %}
19358   ins_encode %{
19359     int vector_len = 0;
19360     Assembler::ComparisonPredicateFP cmp = Assembler::EQ_OQ;  // ordered non-signaling
19361     __ vcmpps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19362   %}
19363   ins_pipe( pipe_slow );
19364 %}
19365 
19366 instruct vcmpeq8F(vecY dst, vecY src1, vecY src2) %{
19367   predicate(UseAVX > 0 && n->as_Vector()->length() == 8 &&
19368             n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
19369             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19370   match(Set dst (VectorMaskCmp src1 src2));
19371   format %{ "vcmpeqps  $dst,$src1,$src2\t! cmpeq packed8F" %}
19372   ins_encode %{
19373     int vector_len = 1;
19374     Assembler::ComparisonPredicateFP cmp = Assembler::EQ_OQ;  // ordered non-signaling
19375     __ vcmpps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19376   %}
19377   ins_pipe( pipe_slow );
19378 %}
19379 
19380 instruct vcmpeq16F(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
19381   predicate(UseAVX > 2 && n->as_Vector()->length() == 16 &&
19382             n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
19383             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19384   match(Set dst (VectorMaskCmp src1 src2));
19385   effect(TEMP dst, TEMP scratch);
19386   format %{ "vcmpeqps  k2,$src1,$src2\n\t"
19387             "vmovdqu8  $dst, k2{z}, 0xFFFFFFFFFF \t! cmpeq packed16F" %}
19388   ins_encode %{
19389     int vector_len = 2;
19390     Assembler::ComparisonPredicateFP cmp = Assembler::EQ_OQ;  // ordered non-signaling
19391     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
19392     KRegister mask = k0; // The comparison itself is not being masked.
19393     __ evcmpps(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19394     __ evmovdqul($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
19395   %}
19396   ins_pipe( pipe_slow );
19397 %}
19398 
19399 instruct vcmplt2F(vecD dst, vecD src1, vecD src2) %{
19400   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 &&
19401             n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
19402             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19403   match(Set dst (VectorMaskCmp src1 src2));
19404   format %{ "vcmpltps  $dst,$src1,$src2\t! cmplt packed2F" %}
19405   ins_encode %{
19406     int vector_len = 0;
19407     Assembler::ComparisonPredicateFP cmp = Assembler::LT_OQ; //ordered non-signaling
19408     __ vcmpps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19409   %}
19410   ins_pipe( pipe_slow );
19411 %}
19412 
19413 instruct vcmplt4F(vecX dst, vecX src1, vecX src2) %{
19414   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 &&
19415             n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
19416             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19417   match(Set dst (VectorMaskCmp src1 src2));
19418   format %{ "vcmpltps  $dst,$src1,$src2\t! cmplt packed4F" %}
19419   ins_encode %{
19420     int vector_len = 0;
19421     Assembler::ComparisonPredicateFP cmp = Assembler::LT_OQ; //ordered non-signaling
19422     __ vcmpps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19423   %}
19424   ins_pipe( pipe_slow );
19425 %}
19426 
19427 instruct vcmplt8F(vecY dst, vecY src1, vecY src2) %{
19428   predicate(UseAVX > 0 && n->as_Vector()->length() == 8 &&
19429             n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
19430             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19431   match(Set dst (VectorMaskCmp src1 src2));
19432   format %{ "vcmpltps  $dst,$src1,$src2\t! cmplt packed8F" %}
19433   ins_encode %{
19434     int vector_len = 1;
19435     Assembler::ComparisonPredicateFP cmp = Assembler::LT_OQ; //ordered non-signaling
19436     __ vcmpps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19437   %}
19438   ins_pipe( pipe_slow );
19439 %}
19440 
19441 instruct vcmplt16F(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
19442   predicate(UseAVX > 2 && n->as_Vector()->length() == 16 &&
19443             n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
19444             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19445   match(Set dst (VectorMaskCmp src1 src2));
19446   effect(TEMP dst, TEMP scratch);
19447   format %{ "vcmpltps  k2,$src1,$src2\n\t"
19448             "vmovdqu8  $dst, k2{z}, 0xFFFFFFFFFF \t! cmplt packed16F" %}
19449   ins_encode %{
19450     int vector_len = 2;
19451     Assembler::ComparisonPredicateFP cmp = Assembler::LT_OQ; //ordered non-signaling
19452     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
19453     KRegister mask = k0; // The comparison itself is not being masked.
19454     __ evcmpps(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19455     __ evmovdqul($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
19456   %}
19457   ins_pipe( pipe_slow );
19458 %}
19459 
19460 instruct vcmpgt2F(vecD dst, vecD src1, vecD src2) %{
19461   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 &&
19462             n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
19463             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19464   match(Set dst (VectorMaskCmp src1 src2));
19465   format %{ "vcmpgtps  $dst,$src1,$src2\t! cmpgt packed2F" %}
19466   ins_encode %{
19467     int vector_len = 0;
19468     Assembler::ComparisonPredicateFP cmp = Assembler::GT_OQ; //ordered non-signaling
19469     __ vcmpps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19470   %}
19471   ins_pipe( pipe_slow );
19472 %}
19473 
19474 instruct vcmpgt4F(vecX dst, vecX src1, vecX src2) %{
19475   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 &&
19476             n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
19477             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19478   match(Set dst (VectorMaskCmp src1 src2));
19479   format %{ "vcmpgtps  $dst,$src1,$src2\t! cmpgt packed4F" %}
19480   ins_encode %{
19481     int vector_len = 0;
19482     Assembler::ComparisonPredicateFP cmp = Assembler::GT_OQ; //ordered non-signaling
19483     __ vcmpps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19484   %}
19485   ins_pipe( pipe_slow );
19486 %}
19487 
19488 instruct vcmpgt8F(vecY dst, vecY src1, vecY src2) %{
19489   predicate(UseAVX > 0 && n->as_Vector()->length() == 8 &&
19490             n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
19491             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19492   match(Set dst (VectorMaskCmp src1 src2));
19493   format %{ "vcmpgtps  $dst,$src1,$src2\t! cmpgt packed8F" %}
19494   ins_encode %{
19495     int vector_len = 1;
19496     Assembler::ComparisonPredicateFP cmp = Assembler::GT_OQ; //ordered non-signaling
19497     __ vcmpps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19498   %}
19499   ins_pipe( pipe_slow );
19500 %}
19501 
19502 instruct vcmpgt16F(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
19503   predicate(UseAVX > 2 && n->as_Vector()->length() == 16 &&
19504             n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
19505             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19506   match(Set dst (VectorMaskCmp src1 src2));
19507   effect(TEMP dst, TEMP scratch);
19508   format %{ "vcmpgtps  k2,$src1,$src2\n\t"
19509             "vmovdqu8  $dst, k2{z}, 0xFFFFFFFFFF \t! cmpgt packed16F" %}
19510   ins_encode %{
19511     int vector_len = 2;
19512     Assembler::ComparisonPredicateFP cmp = Assembler::GT_OQ; //ordered non-signaling
19513     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
19514     KRegister mask = k0; // The comparison itself is not being masked.
19515     __ evcmpps(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19516     __ evmovdqul($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
19517   %}
19518   ins_pipe( pipe_slow );
19519 %}
19520 
19521 instruct vcmpge2F(vecD dst, vecD src1, vecD src2) %{
19522   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 &&
19523             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
19524             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19525   match(Set dst (VectorMaskCmp src1 src2));
19526   format %{ "vcmpgeps  $dst,$src1,$src2\t! cmpge packed2F" %}
19527   ins_encode %{
19528     int vector_len = 0;
19529     Assembler::ComparisonPredicateFP cmp = Assembler::GE_OQ; //ordered non-signaling
19530     __ vcmpps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19531   %}
19532   ins_pipe( pipe_slow );
19533 %}
19534 
19535 instruct vcmpge4F(vecX dst, vecX src1, vecX src2) %{
19536   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 &&
19537             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
19538             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19539   match(Set dst (VectorMaskCmp src1 src2));
19540   format %{ "vcmpgeps  $dst,$src1,$src2\t! cmpge packed4F" %}
19541   ins_encode %{
19542     int vector_len = 0;
19543     Assembler::ComparisonPredicateFP cmp = Assembler::GE_OQ; //ordered non-signaling
19544     __ vcmpps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19545   %}
19546   ins_pipe( pipe_slow );
19547 %}
19548 
19549 instruct vcmpge8F(vecY dst, vecY src1, vecY src2) %{
19550   predicate(UseAVX > 0 && n->as_Vector()->length() == 8 &&
19551             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
19552             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19553   match(Set dst (VectorMaskCmp src1 src2));
19554   format %{ "vcmpgeps  $dst,$src1,$src2\t! cmpge packed8F" %}
19555   ins_encode %{
19556     int vector_len = 1;
19557     Assembler::ComparisonPredicateFP cmp = Assembler::GE_OQ; //ordered non-signaling
19558     __ vcmpps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19559   %}
19560   ins_pipe( pipe_slow );
19561 %}
19562 
19563 instruct vcmpge16F(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
19564   predicate(UseAVX > 2 && n->as_Vector()->length() == 16 &&
19565             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
19566             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19567   match(Set dst (VectorMaskCmp src1 src2));
19568   effect(TEMP dst, TEMP scratch);
19569   format %{ "vcmpgeps  k2,$src1,$src2\n\t"
19570             "vmovdqu8  $dst, k2{z}, 0xFFFFFFFFFF \t! cmpge packed16F" %}
19571   ins_encode %{
19572     int vector_len = 2;
19573     Assembler::ComparisonPredicateFP cmp = Assembler::GE_OQ; //ordered non-signaling
19574     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
19575     KRegister mask = k0; // The comparison itself is not being masked.
19576     __ evcmpps(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19577     __ evmovdqul($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
19578   %}
19579   ins_pipe( pipe_slow );
19580 %}
19581 
19582 instruct vcmple2F(vecD dst, vecD src1, vecD src2) %{
19583   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 &&
19584             n->as_VectorMaskCmp()->get_predicate() == BoolTest::le &&
19585             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19586   match(Set dst (VectorMaskCmp src1 src2));
19587   format %{ "vcmpleps  $dst,$src1,$src2\t! cmple packed2F" %}
19588   ins_encode %{
19589     int vector_len = 0;
19590     Assembler::ComparisonPredicateFP cmp = Assembler::LE_OQ; //ordered non-signaling
19591     __ vcmpps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19592   %}
19593   ins_pipe( pipe_slow );
19594 %}
19595 
19596 instruct vcmple4F(vecX dst, vecX src1, vecX src2) %{
19597   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 &&
19598             n->as_VectorMaskCmp()->get_predicate() == BoolTest::le &&
19599             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19600   match(Set dst (VectorMaskCmp src1 src2));
19601   format %{ "vcmpleps  $dst,$src1,$src2\t! cmple packed4F" %}
19602   ins_encode %{
19603     int vector_len = 0;
19604     Assembler::ComparisonPredicateFP cmp = Assembler::LE_OQ; //ordered non-signaling
19605     __ vcmpps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19606   %}
19607   ins_pipe( pipe_slow );
19608 %}
19609 
19610 instruct vcmple8F(vecY dst, vecY src1, vecY src2) %{
19611   predicate(UseAVX > 0 && n->as_Vector()->length() == 8 &&
19612             n->as_VectorMaskCmp()->get_predicate() == BoolTest::le &&
19613             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19614   match(Set dst (VectorMaskCmp src1 src2));
19615   format %{ "vcmpleps  $dst,$src1,$src2\t! cmple packed8F" %}
19616   ins_encode %{
19617     int vector_len = 1;
19618     Assembler::ComparisonPredicateFP cmp = Assembler::LE_OQ; //ordered non-signaling
19619     __ vcmpps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19620   %}
19621   ins_pipe( pipe_slow );
19622 %}
19623 
19624 instruct vcmple16F(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
19625   predicate(UseAVX > 2 && n->as_Vector()->length() == 16 &&
19626             n->as_VectorMaskCmp()->get_predicate() == BoolTest::le &&
19627             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19628   match(Set dst (VectorMaskCmp src1 src2));
19629   effect(TEMP dst, TEMP scratch);
19630   format %{ "vcmpleps  k2,$src1,$src2\n\t"
19631             "vmovdqu8  $dst, k2{z}, 0xFFFFFFFFFF \t! cmple packed16F" %}
19632   ins_encode %{
19633     int vector_len = 2;
19634     Assembler::ComparisonPredicateFP cmp = Assembler::LE_OQ; //ordered non-signaling
19635     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
19636     KRegister mask = k0; // The comparison itself is not being masked.
19637     __ evcmpps(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19638     __ evmovdqul($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
19639   %}
19640   ins_pipe( pipe_slow );
19641 %}
19642 
19643 instruct vcmpne2F(vecD dst, vecD src1, vecD src2) %{
19644   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 &&
19645             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ne &&
19646             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19647   match(Set dst (VectorMaskCmp src1 src2));
19648   format %{ "vcmpneps  $dst,$src1,$src2\t! cmpne packed2F" %}
19649   ins_encode %{
19650     int vector_len = 0;
19651     // As per JLS 15.21.1, != of NaNs is true. Thus use unordered compare.
19652     Assembler::ComparisonPredicateFP cmp = Assembler::NEQ_UQ; //unordered non-signaling
19653     __ vcmpps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19654   %}
19655   ins_pipe( pipe_slow );
19656 %}
19657 
19658 instruct vcmpne4F(vecX dst, vecX src1, vecX src2) %{
19659   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 &&
19660             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ne &&
19661             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19662   match(Set dst (VectorMaskCmp src1 src2));
19663   format %{ "vcmpneps  $dst,$src1,$src2\t! cmpne packed4F" %}
19664   ins_encode %{
19665     int vector_len = 0;
19666     // As per JLS 15.21.1, != of NaNs is true. Thus use unordered compare.
19667     Assembler::ComparisonPredicateFP cmp = Assembler::NEQ_UQ; //unordered non-signaling
19668     __ vcmpps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19669   %}
19670   ins_pipe( pipe_slow );
19671 %}
19672 
19673 instruct vcmpne8F(vecY dst, vecY src1, vecY src2) %{
19674   predicate(UseAVX > 0 && n->as_Vector()->length() == 8 &&
19675             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ne &&
19676             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19677   match(Set dst (VectorMaskCmp src1 src2));
19678   format %{ "vcmpneps  $dst,$src1,$src2\t! cmpne packed8F" %}
19679   ins_encode %{
19680     int vector_len = 1;
19681     // As per JLS 15.21.1, != of NaNs is true. Thus use unordered compare.
19682     Assembler::ComparisonPredicateFP cmp = Assembler::NEQ_UQ; //unordered non-signaling
19683     __ vcmpps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19684   %}
19685   ins_pipe( pipe_slow );
19686 %}
19687 
19688 instruct vcmpne16F(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
19689   predicate(UseAVX > 2 && n->as_Vector()->length() == 16 &&
19690             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ne &&
19691             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
19692   match(Set dst (VectorMaskCmp src1 src2));
19693   effect(TEMP dst, TEMP scratch);
19694   format %{ "vcmpneps  k2,$src1,$src2\n\t"
19695             "vmovdqu8  $dst, k2{z}, 0xFFFFFFFFFF \t! cmpne packed16F" %}
19696   ins_encode %{
19697     int vector_len = 2;
19698     // As per JLS 15.21.1, != of NaNs is true. Thus use unordered compare.
19699     Assembler::ComparisonPredicateFP cmp = Assembler::NEQ_UQ; //unordered non-signaling
19700     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
19701     KRegister mask = k0; // The comparison itself is not being masked.
19702     __ evcmpps(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19703     __ evmovdqul($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
19704   %}
19705   ins_pipe( pipe_slow );
19706 %}
19707 
19708 instruct vcmpeq1D(vecD dst, vecD src1, vecD src2) %{
19709   predicate(UseAVX > 0 && n->as_Vector()->length() == 1 &&
19710             n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
19711             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
19712   match(Set dst (VectorMaskCmp src1 src2));
19713   format %{ "vcmpeqpd  $dst,$src1,$src2\t! cmpeq packed1D" %}
19714   ins_encode %{
19715     int vector_len = 0;
19716     Assembler::ComparisonPredicateFP cmp = Assembler::EQ_OQ;  // ordered non-signaling
19717     __ vcmppd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19718   %}
19719   ins_pipe( pipe_slow );
19720 %}
19721 
19722 instruct vcmpeq2D(vecX dst, vecX src1, vecX src2) %{
19723   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 &&
19724             n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
19725             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
19726   match(Set dst (VectorMaskCmp src1 src2));
19727   format %{ "vcmpeqpd  $dst,$src1,$src2\t! cmpeq packed2D" %}
19728   ins_encode %{
19729     int vector_len = 0;
19730     Assembler::ComparisonPredicateFP cmp = Assembler::EQ_OQ;  // ordered non-signaling
19731     __ vcmppd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19732   %}
19733   ins_pipe( pipe_slow );
19734 %}
19735 
19736 instruct vcmpeq4D(vecY dst, vecY src1, vecY src2) %{
19737   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 &&
19738             n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
19739             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
19740   match(Set dst (VectorMaskCmp src1 src2));
19741   format %{ "vcmpeqpd  $dst,$src1,$src2\t! cmpeq packed4D" %}
19742   ins_encode %{
19743     int vector_len = 1;
19744     Assembler::ComparisonPredicateFP cmp = Assembler::EQ_OQ;  // ordered non-signaling
19745     __ vcmppd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19746   %}
19747   ins_pipe( pipe_slow );
19748 %}
19749 
19750 instruct vcmpeq8D(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
19751   predicate(UseAVX > 2 && n->as_Vector()->length() == 8 &&
19752             n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
19753             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
19754   match(Set dst (VectorMaskCmp src1 src2));
19755   effect(TEMP dst, TEMP scratch);
19756   format %{ "vcmpeqpd  k2,$src1,$src2\n\t"
19757             "vmovdqu8  $dst, k2{z}, 0xFFFFFFFFFF \t! cmpeq packed8D" %}
19758   ins_encode %{
19759     int vector_len = 2;
19760     Assembler::ComparisonPredicateFP cmp = Assembler::EQ_OQ;  // ordered non-signaling
19761     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
19762     KRegister mask = k0; // The comparison itself is not being masked.
19763     __ evcmppd(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19764     __ evmovdquq($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
19765   %}
19766   ins_pipe( pipe_slow );
19767 %}
19768 
19769 instruct vcmplt1D(vecD dst, vecD src1, vecD src2) %{
19770   predicate(UseAVX > 0 && n->as_Vector()->length() == 1 &&
19771             n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
19772             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
19773   match(Set dst (VectorMaskCmp src1 src2));
19774   format %{ "vcmpltpd  $dst,$src1,$src2\t! cmplt packed1D" %}
19775   ins_encode %{
19776     int vector_len = 0;
19777     Assembler::ComparisonPredicateFP cmp = Assembler::LT_OQ; //ordered non-signaling
19778     __ vcmppd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19779   %}
19780   ins_pipe( pipe_slow );
19781 %}
19782 
19783 instruct vcmplt2D(vecX dst, vecX src1, vecX src2) %{
19784   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 &&
19785             n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
19786             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
19787   match(Set dst (VectorMaskCmp src1 src2));
19788   format %{ "vcmpltpd  $dst,$src1,$src2\t! cmplt packed2D" %}
19789   ins_encode %{
19790     int vector_len = 0;
19791     Assembler::ComparisonPredicateFP cmp = Assembler::LT_OQ; //ordered non-signaling
19792     __ vcmppd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19793   %}
19794   ins_pipe( pipe_slow );
19795 %}
19796 
19797 instruct vcmplt4D(vecY dst, vecY src1, vecY src2) %{
19798   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 &&
19799             n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
19800             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
19801   match(Set dst (VectorMaskCmp src1 src2));
19802   format %{ "vcmpltpd  $dst,$src1,$src2\t! cmplt packed4D" %}
19803   ins_encode %{
19804     int vector_len = 1;
19805     Assembler::ComparisonPredicateFP cmp = Assembler::LT_OQ; //ordered non-signaling
19806     __ vcmppd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19807   %}
19808   ins_pipe( pipe_slow );
19809 %}
19810 
19811 instruct vcmplt8D(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
19812   predicate(UseAVX > 2 && n->as_Vector()->length() == 8 &&
19813             n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
19814             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
19815   match(Set dst (VectorMaskCmp src1 src2));
19816   effect(TEMP dst, TEMP scratch);
19817   format %{ "vcmpltpd  k2,$src1,$src2\n\t"
19818             "vmovdqu8  $dst, k2{z}, 0xFFFFFFFFFF \t! cmplt packed8D" %}
19819   ins_encode %{
19820     int vector_len = 2;
19821     Assembler::ComparisonPredicateFP cmp = Assembler::LT_OQ; //ordered non-signaling
19822     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
19823     KRegister mask = k0; // The comparison itself is not being masked.
19824     __ evcmppd(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19825     __ evmovdquq($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
19826   %}
19827   ins_pipe( pipe_slow );
19828 %}
19829 
19830 instruct vcmpgt1D(vecD dst, vecD src1, vecD src2) %{
19831   predicate(UseAVX > 0 && n->as_Vector()->length() == 1 &&
19832             n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
19833             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
19834   match(Set dst (VectorMaskCmp src1 src2));
19835   format %{ "vcmpgtpd  $dst,$src1,$src2\t! cmpgt packed1D" %}
19836   ins_encode %{
19837     int vector_len = 0;
19838     Assembler::ComparisonPredicateFP cmp = Assembler::GT_OQ; //ordered non-signaling
19839     __ vcmppd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19840   %}
19841   ins_pipe( pipe_slow );
19842 %}
19843 
19844 instruct vcmpgt2D(vecX dst, vecX src1, vecX src2) %{
19845   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 &&
19846             n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
19847             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
19848   match(Set dst (VectorMaskCmp src1 src2));
19849   format %{ "vcmpgtpd  $dst,$src1,$src2\t! cmpgt packed2D" %}
19850   ins_encode %{
19851     int vector_len = 0;
19852     Assembler::ComparisonPredicateFP cmp = Assembler::GT_OQ; //ordered non-signaling
19853     __ vcmppd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19854   %}
19855   ins_pipe( pipe_slow );
19856 %}
19857 
19858 instruct vcmpgt4D(vecY dst, vecY src1, vecY src2) %{
19859   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 &&
19860             n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
19861             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
19862   match(Set dst (VectorMaskCmp src1 src2));
19863   format %{ "vcmpgtpd  $dst,$src1,$src2\t! cmpgt packed4D" %}
19864   ins_encode %{
19865     int vector_len = 1;
19866     Assembler::ComparisonPredicateFP cmp = Assembler::GT_OQ; //ordered non-signaling
19867     __ vcmppd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19868   %}
19869   ins_pipe( pipe_slow );
19870 %}
19871 
19872 instruct vcmpgt8D(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
19873   predicate(UseAVX > 2 && n->as_Vector()->length() == 8 &&
19874             n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
19875             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
19876   match(Set dst (VectorMaskCmp src1 src2));
19877   effect(TEMP dst, TEMP scratch);
19878   format %{ "vcmpgtpd  k2,$src1,$src2\n\t"
19879             "vmovdqu8  $dst, k2{z}, 0xFFFFFFFFFF \t! cmpgt packed8D" %}
19880   ins_encode %{
19881     int vector_len = 2;
19882     Assembler::ComparisonPredicateFP cmp = Assembler::GT_OQ; //ordered non-signaling
19883     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
19884     KRegister mask = k0; // The comparison itself is not being masked.
19885     __ evcmppd(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19886     __ evmovdquq($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
19887   %}
19888   ins_pipe( pipe_slow );
19889 %}
19890 
19891 instruct vcmpge1D(vecD dst, vecD src1, vecD src2) %{
19892   predicate(UseAVX > 0 && n->as_Vector()->length() == 1 &&
19893             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
19894             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
19895   match(Set dst (VectorMaskCmp src1 src2));
19896   format %{ "vcmpgepd  $dst,$src1,$src2\t! cmpge packed1D" %}
19897   ins_encode %{
19898     int vector_len = 0;
19899     Assembler::ComparisonPredicateFP cmp = Assembler::GE_OQ; //ordered non-signaling
19900     __ vcmppd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19901   %}
19902   ins_pipe( pipe_slow );
19903 %}
19904 
19905 instruct vcmpge2D(vecX dst, vecX src1, vecX src2) %{
19906   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 &&
19907             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
19908             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
19909   match(Set dst (VectorMaskCmp src1 src2));
19910   format %{ "vcmpgepd  $dst,$src1,$src2\t! cmpge packed2D" %}
19911   ins_encode %{
19912     int vector_len = 0;
19913     Assembler::ComparisonPredicateFP cmp = Assembler::GE_OQ; //ordered non-signaling
19914     __ vcmppd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19915   %}
19916   ins_pipe( pipe_slow );
19917 %}
19918 
19919 instruct vcmpge4D(vecY dst, vecY src1, vecY src2) %{
19920   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 &&
19921             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
19922             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
19923   match(Set dst (VectorMaskCmp src1 src2));
19924   format %{ "vcmpgepd  $dst,$src1,$src2\t! cmpge packed4D" %}
19925   ins_encode %{
19926     int vector_len = 1;
19927     Assembler::ComparisonPredicateFP cmp = Assembler::GE_OQ; //ordered non-signaling
19928     __ vcmppd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19929   %}
19930   ins_pipe( pipe_slow );
19931 %}
19932 
19933 instruct vcmpge8D(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
19934   predicate(UseAVX > 2 && n->as_Vector()->length() == 8 &&
19935             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
19936             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
19937   match(Set dst (VectorMaskCmp src1 src2));
19938   effect(TEMP dst, TEMP scratch);
19939   format %{ "vcmpgepd  k2,$src1,$src2\n\t"
19940             "vmovdqu8  $dst, k2{z}, 0xFFFFFFFFFF \t! cmpge packed8D" %}
19941   ins_encode %{
19942     int vector_len = 2;
19943     Assembler::ComparisonPredicateFP cmp = Assembler::GE_OQ; //ordered non-signaling
19944     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
19945     KRegister mask = k0; // The comparison itself is not being masked.
19946     __ evcmppd(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19947     __ evmovdquq($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
19948   %}
19949   ins_pipe( pipe_slow );
19950 %}
19951 
19952 instruct vcmple1D(vecD dst, vecD src1, vecD src2) %{
19953   predicate(UseAVX > 0 && n->as_Vector()->length() == 1 &&
19954             n->as_VectorMaskCmp()->get_predicate() == BoolTest::le &&
19955             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
19956   match(Set dst (VectorMaskCmp src1 src2));
19957   format %{ "vcmplepd  $dst,$src1,$src2\t! cmple packed1D" %}
19958   ins_encode %{
19959     int vector_len = 0;
19960     Assembler::ComparisonPredicateFP cmp = Assembler::LE_OQ; //ordered non-signaling
19961     __ vcmppd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19962   %}
19963   ins_pipe( pipe_slow );
19964 %}
19965 
19966 instruct vcmple2D(vecX dst, vecX src1, vecX src2) %{
19967   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 &&
19968             n->as_VectorMaskCmp()->get_predicate() == BoolTest::le &&
19969             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
19970   match(Set dst (VectorMaskCmp src1 src2));
19971   format %{ "vcmplepd  $dst,$src1,$src2\t! cmple packed2D" %}
19972   ins_encode %{
19973     int vector_len = 0;
19974     Assembler::ComparisonPredicateFP cmp = Assembler::LE_OQ; //ordered non-signaling
19975     __ vcmppd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19976   %}
19977   ins_pipe( pipe_slow );
19978 %}
19979 
19980 instruct vcmple4D(vecY dst, vecY src1, vecY src2) %{
19981   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 &&
19982             n->as_VectorMaskCmp()->get_predicate() == BoolTest::le &&
19983             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
19984   match(Set dst (VectorMaskCmp src1 src2));
19985   format %{ "vcmplepd  $dst,$src1,$src2\t! cmple packed4D" %}
19986   ins_encode %{
19987     int vector_len = 1;
19988     Assembler::ComparisonPredicateFP cmp = Assembler::LE_OQ; //ordered non-signaling
19989     __ vcmppd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
19990   %}
19991   ins_pipe( pipe_slow );
19992 %}
19993 
19994 instruct vcmple8D(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
19995   predicate(UseAVX > 2 && n->as_Vector()->length() == 8 &&
19996             n->as_VectorMaskCmp()->get_predicate() == BoolTest::le &&
19997             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
19998   match(Set dst (VectorMaskCmp src1 src2));
19999   effect(TEMP dst, TEMP scratch);
20000   format %{ "vcmplepd  k2,$src1,$src2\n\t"
20001             "vmovdqu8  $dst, k2{z}, 0xFFFFFFFFFF \t! cmple packed8D" %}
20002   ins_encode %{
20003     int vector_len = 2;
20004     Assembler::ComparisonPredicateFP cmp = Assembler::LE_OQ; //ordered non-signaling
20005     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
20006     KRegister mask = k0; // The comparison itself is not being masked.
20007     __ evcmppd(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
20008     __ evmovdquq($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
20009   %}
20010   ins_pipe( pipe_slow );
20011 %}
20012 
20013 instruct vcmpne1D(vecD dst, vecD src1, vecD src2) %{
20014   predicate(UseAVX > 0 && n->as_Vector()->length() == 1 &&
20015             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ne &&
20016             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
20017   match(Set dst (VectorMaskCmp src1 src2));
20018   format %{ "vcmpnepd  $dst,$src1,$src2\t! cmpne packed1D" %}
20019   ins_encode %{
20020     int vector_len = 0;
20021     // As per JLS 15.21.1, != of NaNs is true. Thus use unordered compare.
20022     Assembler::ComparisonPredicateFP cmp = Assembler::NEQ_UQ; //unordered non-signaling
20023     __ vcmppd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
20024   %}
20025   ins_pipe( pipe_slow );
20026 %}
20027 
20028 instruct vcmpne2D(vecX dst, vecX src1, vecX src2) %{
20029   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 &&
20030             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ne &&
20031             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
20032   match(Set dst (VectorMaskCmp src1 src2));
20033   format %{ "vcmpnepd  $dst,$src1,$src2\t! cmpne packed2D" %}
20034   ins_encode %{
20035     int vector_len = 0;
20036     // As per JLS 15.21.1, != of NaNs is true. Thus use unordered compare.
20037     Assembler::ComparisonPredicateFP cmp = Assembler::NEQ_UQ; //unordered non-signaling
20038     __ vcmppd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
20039   %}
20040   ins_pipe( pipe_slow );
20041 %}
20042 
20043 instruct vcmpne4D(vecY dst, vecY src1, vecY src2) %{
20044   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 &&
20045             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ne &&
20046             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
20047   match(Set dst (VectorMaskCmp src1 src2));
20048   format %{ "vcmpnepd  $dst,$src1,$src2\t! cmpne packed4D" %}
20049   ins_encode %{
20050     int vector_len = 1;
20051     // As per JLS 15.21.1, != of NaNs is true. Thus use unordered compare.
20052     Assembler::ComparisonPredicateFP cmp = Assembler::NEQ_UQ; //unordered non-signaling
20053     __ vcmppd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
20054   %}
20055   ins_pipe( pipe_slow );
20056 %}
20057 
20058 instruct vcmpne8D(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
20059   predicate(UseAVX > 2 && n->as_Vector()->length() == 8 &&
20060             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ne &&
20061             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
20062   match(Set dst (VectorMaskCmp src1 src2));
20063   effect(TEMP dst, TEMP scratch);
20064   format %{ "vcmpnepd  k2,$src1,$src2\n\t"
20065             "vmovdqu8  $dst, k2{z}, 0xFFFFFFFFFF \t! cmpne packed8D" %}
20066   ins_encode %{
20067     int vector_len = 2;
20068     // As per JLS 15.21.1, != of NaNs is true. Thus use unordered compare.
20069     Assembler::ComparisonPredicateFP cmp = Assembler::NEQ_UQ; //unordered non-signaling
20070     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
20071     KRegister mask = k0; // The comparison itself is not being masked.
20072     __ evcmppd(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
20073     __ evmovdquq($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
20074   %}
20075   ins_pipe( pipe_slow );
20076 %}
20077 
20078 instruct vcmpeq2I(vecD dst, vecD src1, vecD src2) %{
20079   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 &&
20080             n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
20081             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
20082   match(Set dst (VectorMaskCmp src1 src2));
20083   format %{ "vpcmpeqd  $dst,$src1,$src2\n\t! cmpeq packed2I" %}
20084   ins_encode %{
20085     int vector_len = 0;
20086     __ vpcmpeqd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
20087   %}
20088   ins_pipe( pipe_slow );
20089 %}
20090 
20091 instruct vcmpeq4I(vecX dst, vecX src1, vecX src2) %{
20092   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 &&
20093             n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
20094             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
20095   match(Set dst (VectorMaskCmp src1 src2));
20096   format %{ "vpcmpeqd  $dst,$src1,$src2\n\t! cmpeq packed4I" %}
20097   ins_encode %{
20098     int vector_len = 0;
20099     __ vpcmpeqd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
20100   %}
20101   ins_pipe( pipe_slow );
20102 %}
20103 
20104 instruct vcmpeq8I(vecY dst, vecY src1, vecY src2) %{
20105   predicate(UseAVX > 1 && n->as_Vector()->length() == 8 &&
20106             n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
20107             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
20108   match(Set dst (VectorMaskCmp src1 src2));
20109   format %{ "vpcmpeqd  $dst,$src1,$src2\n\t! cmpeq packed8I" %}
20110   ins_encode %{
20111     int vector_len = 1;
20112     __ vpcmpeqd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
20113   %}
20114   ins_pipe( pipe_slow );
20115 %}
20116 
20117 instruct vcmpeq16I(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
20118   predicate(UseAVX > 2 && n->as_Vector()->length() == 16 &&
20119             n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
20120             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
20121   match(Set dst (VectorMaskCmp src1 src2));
20122   effect(TEMP dst, TEMP scratch);
20123   format %{ "vpcmpeqd  k2,$src1,$src2\n\t"
20124             "vmovdqu32 $dst, k2{z}, 0xFFFFFFFFFF \t! cmpeq packed16I" %}
20125   ins_encode %{
20126     int vector_len = 2;
20127     Assembler::ComparisonPredicate cmp = Assembler::eq;
20128     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
20129     KRegister mask = k0; // The comparison itself is not being masked.
20130     __ evpcmpd(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
20131     __ evmovdqul($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
20132   %}
20133   ins_pipe( pipe_slow );
20134 %}
20135 
20136 instruct vcmplt2I(vecD dst, vecD src1, vecD src2) %{
20137   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 &&
20138             n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
20139             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
20140   match(Set dst (VectorMaskCmp src1 src2));
20141   format %{ "vpcmpgtd  $dst,$src2,$src1\t! cmplt packed2I" %}
20142   ins_encode %{
20143     int vector_len = 0;
20144     __ vpcmpgtd($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, vector_len);
20145   %}
20146   ins_pipe( pipe_slow );
20147 %}
20148 
20149 instruct vcmplt4I(vecX dst, vecX src1, vecX src2) %{
20150   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 &&
20151             n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
20152             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
20153   match(Set dst (VectorMaskCmp src1 src2));
20154   format %{ "vpcmpgtd  $dst,$src2,$src1\t! cmplt packed4I" %}
20155   ins_encode %{
20156     int vector_len = 0;
20157     __ vpcmpgtd($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, vector_len);
20158   %}
20159   ins_pipe( pipe_slow );
20160 %}
20161 
20162 instruct vcmplt8I(vecY dst, vecY src1, vecY src2) %{
20163   predicate(UseAVX > 1 && n->as_Vector()->length() == 8 &&
20164             n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
20165             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
20166   match(Set dst (VectorMaskCmp src1 src2));
20167   format %{ "vpcmpgtd  $dst,$src2,$src1\t! cmplt packed8I" %}
20168   ins_encode %{
20169     int vector_len = 1;
20170     __ vpcmpgtd($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, vector_len);
20171   %}
20172   ins_pipe( pipe_slow );
20173 %}
20174 
20175 instruct vcmplt16I(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
20176   predicate(UseAVX > 2 && n->as_Vector()->length() == 16 &&
20177             n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
20178             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
20179   match(Set dst (VectorMaskCmp src1 src2));
20180   effect(TEMP dst, TEMP scratch);
20181   format %{ "vpcmpnled  k2,$src1,$src2\n\t"
20182             "vmovdqu32   $dst, k2{z}, 0xFFFFFFFFFF \t! cmpgt packed16I" %}
20183   ins_encode %{
20184     int vector_len = 2;
20185     Assembler::ComparisonPredicate cmp = Assembler::lt;
20186     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
20187     KRegister mask = k0; // The comparison itself is not being masked.
20188     __ evpcmpd(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
20189     __ evmovdqul($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
20190   %}
20191   ins_pipe( pipe_slow );
20192 %}
20193 
20194 instruct vcmpgt2I(vecD dst, vecD src1, vecD src2) %{
20195   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 &&
20196             n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
20197             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
20198   match(Set dst (VectorMaskCmp src1 src2));
20199   format %{ "vpcmpgtd  $dst,$src1,$src2\t! cmpgt packed2I" %}
20200   ins_encode %{
20201     int vector_len = 0;
20202     __ vpcmpgtd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
20203   %}
20204   ins_pipe( pipe_slow );
20205 %}
20206 
20207 instruct vcmpgt4I(vecX dst, vecX src1, vecX src2) %{
20208   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 &&
20209             n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
20210             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
20211   match(Set dst (VectorMaskCmp src1 src2));
20212   format %{ "vpcmpgtd  $dst,$src1,$src2\t! cmpgt packed4I" %}
20213   ins_encode %{
20214     int vector_len = 0;
20215     __ vpcmpgtd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
20216   %}
20217   ins_pipe( pipe_slow );
20218 %}
20219 
20220 instruct vcmpgt8I(vecY dst, vecY src1, vecY src2) %{
20221   predicate(UseAVX > 1 && n->as_Vector()->length() == 8 &&
20222             n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
20223             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
20224   match(Set dst (VectorMaskCmp src1 src2));
20225   format %{ "vpcmpgtd  $dst,$src1,$src2\t! cmpgt packed8I" %}
20226   ins_encode %{
20227     int vector_len = 1;
20228     __ vpcmpgtd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
20229   %}
20230   ins_pipe( pipe_slow );
20231 %}
20232 
20233 instruct vcmpgt16I(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
20234   predicate(UseAVX > 2 && n->as_Vector()->length() == 16 &&
20235             n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
20236             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
20237   match(Set dst (VectorMaskCmp src1 src2));
20238   effect(TEMP dst, TEMP scratch);
20239   format %{ "vpcmpnled  k2,$src1,$src2\n\t"
20240             "vmovdqu32   $dst, k2{z}, 0xFFFFFFFFFF \t! cmpgt packed16I" %}
20241   ins_encode %{
20242     int vector_len = 2;
20243     Assembler::ComparisonPredicate cmp = Assembler::nle;
20244     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
20245     KRegister mask = k0; // The comparison itself is not being masked.
20246     __ evpcmpd(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
20247     __ evmovdqul($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
20248   %}
20249   ins_pipe( pipe_slow );
20250 %}
20251 
20252 instruct vcmpge2I(vecD dst, vecD src1, vecD src2, rRegL scratch) %{
20253   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 &&
20254             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
20255             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
20256   match(Set dst (VectorMaskCmp src1 src2));
20257   effect(TEMP scratch);
20258   format %{ "vpcmpgtd  $dst,$src2,$src1\n\t"
20259             "vpxor $dst,$dst,0xFFFFFFFF\t! cmpge packed2I" %}
20260   ins_encode %{
20261     int vector_len = 0;
20262     __ vpcmpgtd($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, vector_len);
20263     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_all_bits_set()), vector_len, $scratch$$Register);
20264   %}
20265   ins_pipe( pipe_slow );
20266 %}
20267 
20268 instruct vcmpge4I(vecX dst, vecX src1, vecX src2, rRegL scratch) %{
20269   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 &&
20270             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
20271             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
20272   match(Set dst (VectorMaskCmp src1 src2));
20273   effect(TEMP scratch);
20274   format %{ "vpcmpgtd  $dst,$src2,$src1\n\t"
20275             "vpxor $dst,$dst,0xFFFFFFFF\t! cmpge packed4I" %}
20276   ins_encode %{
20277     int vector_len = 0;
20278     __ vpcmpgtd($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, vector_len);
20279     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_all_bits_set()), vector_len, $scratch$$Register);
20280   %}
20281   ins_pipe( pipe_slow );
20282 %}
20283 
20284 instruct vcmpge8I(vecY dst, vecY src1, vecY src2, rRegL scratch) %{
20285   predicate(UseAVX > 1 && n->as_Vector()->length() == 8 &&
20286             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
20287             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
20288   match(Set dst (VectorMaskCmp src1 src2));
20289   effect(TEMP scratch);
20290   format %{ "vpcmpgtd  $dst,$src2,$src1\n\t"
20291             "vpxor $dst,$dst,0xFFFFFFFF\t! cmpge packed8I" %}
20292   ins_encode %{
20293     int vector_len = 1;
20294     __ vpcmpgtd($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, vector_len);
20295     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_all_bits_set()), vector_len, $scratch$$Register);
20296   %}
20297   ins_pipe( pipe_slow );
20298 %}
20299 
20300 instruct vcmpge16I(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
20301   predicate(UseAVX > 2 && n->as_Vector()->length() == 16 &&
20302             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
20303             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
20304   match(Set dst (VectorMaskCmp src1 src2));
20305   effect(TEMP dst, TEMP scratch);
20306   format %{ "vpcmpnltd  k2,$src1,$src2\n\t"
20307             "vmovdqu32   $dst, k2{z}, 0xFFFFFFFFFF \t! cmpge packed16I" %}
20308   ins_encode %{
20309     int vector_len = 2;
20310     Assembler::ComparisonPredicate cmp = Assembler::nlt;
20311     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
20312     KRegister mask = k0; // The comparison itself is not being masked.
20313     __ evpcmpd(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
20314     __ evmovdqul($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
20315   %}
20316   ins_pipe( pipe_slow );
20317 %}
20318 
20319 instruct vcmple2I(vecD dst, vecD src1, vecD src2, rRegL scratch) %{
20320   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 &&
20321             n->as_VectorMaskCmp()->get_predicate() == BoolTest::le &&
20322             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
20323   match(Set dst (VectorMaskCmp src1 src2));
20324   effect(TEMP scratch);
20325   format %{ "vpcmpgtd  $dst,$src1,$src2\n\t"
20326             "vpxor $dst,$dst,0xFFFFFFFF\t! cmple packed2I" %}
20327   ins_encode %{
20328     int vector_len = 0;
20329     __ vpcmpgtd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
20330     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_all_bits_set()), vector_len, $scratch$$Register);
20331   %}
20332   ins_pipe( pipe_slow );
20333 %}
20334 
20335 instruct vcmple4I(vecX dst, vecX src1, vecX src2, rRegL scratch) %{
20336   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 &&
20337             n->as_VectorMaskCmp()->get_predicate() == BoolTest::le &&
20338             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
20339   match(Set dst (VectorMaskCmp src1 src2));
20340   effect(TEMP scratch);
20341   format %{ "vpcmpgtd  $dst,$src1,$src2\n\t"
20342             "vpxor $dst,$dst,0xFFFFFFFF\t! cmple packed4I" %}
20343   ins_encode %{
20344     int vector_len = 0;
20345     __ vpcmpgtd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
20346     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_all_bits_set()), vector_len, $scratch$$Register);
20347   %}
20348   ins_pipe( pipe_slow );
20349 %}
20350 
20351 instruct vcmple8I(vecY dst, vecY src1, vecY src2, rRegL scratch) %{
20352   predicate(UseAVX > 1 && n->as_Vector()->length() == 8 &&
20353             n->as_VectorMaskCmp()->get_predicate() == BoolTest::le &&
20354             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
20355   match(Set dst (VectorMaskCmp src1 src2));
20356   effect(TEMP scratch);
20357   format %{ "vpcmpgtd  $dst,$src1,$src2\n\t"
20358             "vpxor $dst,$dst,0xFFFFFFFF\t! cmple packed8I" %}
20359   ins_encode %{
20360     int vector_len = 1;
20361     __ vpcmpgtd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
20362     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_all_bits_set()), vector_len, $scratch$$Register);
20363   %}
20364   ins_pipe( pipe_slow );
20365 %}
20366 
20367 instruct vcmple16I(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
20368   predicate(UseAVX > 2 && n->as_Vector()->length() == 16 &&
20369             n->as_VectorMaskCmp()->get_predicate() == BoolTest::le &&
20370             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
20371   match(Set dst (VectorMaskCmp src1 src2));
20372   effect(TEMP dst, TEMP scratch);
20373   format %{ "vpcmpled  k2,$src1,$src2\n\t"
20374             "vmovdqu32   $dst, k2{z}, 0xFFFFFFFFFF \t! cmple packed16I" %}
20375   ins_encode %{
20376     int vector_len = 2;
20377     Assembler::ComparisonPredicate cmp = Assembler::le;
20378     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
20379     KRegister mask = k0; // The comparison itself is not being masked.
20380     __ evpcmpd(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
20381     __ evmovdqul($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
20382   %}
20383   ins_pipe( pipe_slow );
20384 %}
20385 
20386 instruct vcmpne2I(vecD dst, vecD src1, vecD src2, rRegL scratch) %{
20387   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 &&
20388             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ne &&
20389             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
20390   match(Set dst (VectorMaskCmp src1 src2));
20391   effect(TEMP scratch);
20392   format %{ "vpcmpeqd  $dst,$src1,$src2\n\t"
20393             "vpxor $dst,$dst,0xFFFFFFFF\t! cmpneq packed2I" %}
20394   ins_encode %{
20395     int vector_len = 0;
20396     __ vpcmpeqd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
20397     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_all_bits_set()), vector_len, $scratch$$Register);
20398   %}
20399   ins_pipe( pipe_slow );
20400 %}
20401 
20402 instruct vcmpne4I(vecX dst, vecX src1, vecX src2, rRegL scratch) %{
20403   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 &&
20404             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ne &&
20405             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
20406   match(Set dst (VectorMaskCmp src1 src2));
20407   effect(TEMP scratch);
20408   format %{ "vpcmpeqd  $dst,$src1,$src2\n\t"
20409             "vpxor $dst,$dst,0xFFFFFFFF\t! cmpneq packed4I" %}
20410   ins_encode %{
20411     int vector_len = 0;
20412     __ vpcmpeqd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
20413     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_all_bits_set()), vector_len, $scratch$$Register);
20414   %}
20415   ins_pipe( pipe_slow );
20416 %}
20417 
20418 instruct vcmpne8I(vecY dst, vecY src1, vecY src2, rRegL scratch) %{
20419   predicate(UseAVX > 1 && n->as_Vector()->length() == 8 &&
20420             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ne &&
20421             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
20422   match(Set dst (VectorMaskCmp src1 src2));
20423   effect(TEMP scratch);
20424   format %{ "vpcmpeqd  $dst,$src1,$src2\n\t"
20425             "vpxor $dst,$dst,0xFFFFFFFF\t! cmpneq packed8I" %}
20426   ins_encode %{
20427     int vector_len = 1;
20428     __ vpcmpeqd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
20429     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_all_bits_set()), vector_len, $scratch$$Register);
20430   %}
20431   ins_pipe( pipe_slow );
20432 %}
20433 
20434 instruct vcmpne16I(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
20435   predicate(UseAVX > 2 && n->as_Vector()->length() == 16 &&
20436             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ne &&
20437             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT);
20438   match(Set dst (VectorMaskCmp src1 src2));
20439   effect(TEMP dst, TEMP scratch);
20440   format %{ "vpcmpneqd  k2,$src1,$src2\n\t"
20441             "vmovdqu32   $dst, k2{z}, 0xFFFFFFFFFF \t! cmpneq packed16I" %}
20442   ins_encode %{
20443     int vector_len = 2;
20444     Assembler::ComparisonPredicate cmp = Assembler::neq;
20445     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
20446     KRegister mask = k0; // The comparison itself is not being masked.
20447     __ evpcmpd(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
20448     __ evmovdqul($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
20449   %}
20450   ins_pipe( pipe_slow );
20451 %}
20452 
20453 instruct vcmpeq8B(vecD dst, vecD src1, vecD src2) %{
20454   predicate(UseAVX > 0 && n->as_Vector()->length() == 8 &&
20455             n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
20456             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
20457   match(Set dst (VectorMaskCmp src1 src2));
20458   format %{ "vpcmpeqb  $dst,$src1,$src2\n\t! cmpeq packed8B" %}
20459   ins_encode %{
20460     int vector_len = 0;
20461     __ vpcmpeqb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
20462   %}
20463   ins_pipe( pipe_slow );
20464 %}
20465 
20466 instruct vcmpeq16B(vecX dst, vecX src1, vecX src2) %{
20467   predicate(UseAVX > 0 && n->as_Vector()->length() == 16 &&
20468             n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
20469             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
20470   match(Set dst (VectorMaskCmp src1 src2));
20471   format %{ "vpcmpeqb  $dst,$src1,$src2\n\t! cmpeq packed16B" %}
20472   ins_encode %{
20473     int vector_len = 0;
20474     __ vpcmpeqb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
20475   %}
20476   ins_pipe( pipe_slow );
20477 %}
20478 
20479 instruct vcmpeq32B(vecY dst, vecY src1, vecY src2) %{
20480   predicate(UseAVX > 1 && n->as_Vector()->length() == 32 &&
20481             n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
20482             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
20483   match(Set dst (VectorMaskCmp src1 src2));
20484   format %{ "vpcmpeqb  $dst,$src1,$src2\n\t! cmpeq packed32B" %}
20485   ins_encode %{
20486     int vector_len = 1;
20487     __ vpcmpeqb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
20488   %}
20489   ins_pipe( pipe_slow );
20490 %}
20491 
20492 instruct vcmpeq64B(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
20493   predicate(UseAVX > 2 && VM_Version::supports_avx512bw() && n->as_Vector()->length() == 64 &&
20494             n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
20495             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
20496   match(Set dst (VectorMaskCmp src1 src2));
20497   effect(TEMP dst, TEMP scratch);
20498   format %{ "vpcmpeqb  k2,$src1,$src2\n\t"
20499             "vmovdqu8 $dst, k2{z}, 0xFFFFFFFFFF \t! cmpeq packed64B" %}
20500   ins_encode %{
20501     int vector_len = 2;
20502     Assembler::ComparisonPredicate cmp = Assembler::eq;
20503     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
20504     KRegister mask = k0; // The comparison itself is not being masked.
20505     __ evpcmpb(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
20506     __ evmovdqub($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
20507   %}
20508   ins_pipe( pipe_slow );
20509 %}
20510 
20511 instruct vcmplt8B(vecD dst, vecD src1, vecD src2) %{
20512   predicate(UseAVX > 0 && n->as_Vector()->length() == 8 &&
20513             n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
20514             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
20515   match(Set dst (VectorMaskCmp src1 src2));
20516   format %{ "vpcmpgtb  $dst,$src2,$src1\t! cmplt packed8B" %}
20517   ins_encode %{
20518     int vector_len = 0;
20519     __ vpcmpgtb($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, vector_len);
20520   %}
20521   ins_pipe( pipe_slow );
20522 %}
20523 
20524 instruct vcmplt16B(vecX dst, vecX src1, vecX src2) %{
20525   predicate(UseAVX > 0 && n->as_Vector()->length() == 16 &&
20526             n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
20527             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
20528   match(Set dst (VectorMaskCmp src1 src2));
20529   format %{ "vpcmpgtb  $dst,$src2,$src1\t! cmplt packed16B" %}
20530   ins_encode %{
20531     int vector_len = 0;
20532     __ vpcmpgtb($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, vector_len);
20533   %}
20534   ins_pipe( pipe_slow );
20535 %}
20536 
20537 instruct vcmplt32B(vecY dst, vecY src1, vecY src2) %{
20538   predicate(UseAVX > 1 && n->as_Vector()->length() == 32 &&
20539             n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
20540             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
20541   match(Set dst (VectorMaskCmp src1 src2));
20542   format %{ "vpcmpgtb  $dst,$src2,$src1\t! cmplt packed32B" %}
20543   ins_encode %{
20544     int vector_len = 1;
20545     __ vpcmpgtb($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, vector_len);
20546   %}
20547   ins_pipe( pipe_slow );
20548 %}
20549 
20550 instruct vcmplt64B(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
20551   predicate(UseAVX > 2 && VM_Version::supports_avx512bw() && n->as_Vector()->length() == 64 &&
20552             n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
20553             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
20554   match(Set dst (VectorMaskCmp src1 src2));
20555   effect(TEMP dst, TEMP scratch);
20556   format %{ "vpcmpnleb  k2,$src1,$src2\n\t"
20557             "vmovdqu8   $dst, k2{z}, 0xFFFFFFFFFF \t! cmpgt packed64B" %}
20558   ins_encode %{
20559     int vector_len = 2;
20560     Assembler::ComparisonPredicate cmp = Assembler::lt;
20561     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
20562     KRegister mask = k0; // The comparison itself is not being masked.
20563     __ evpcmpb(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
20564     __ evmovdqub($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
20565   %}
20566   ins_pipe( pipe_slow );
20567 %}
20568 
20569 instruct vcmpgt8B(vecD dst, vecD src1, vecD src2) %{
20570   predicate(UseAVX > 0 && n->as_Vector()->length() == 8 &&
20571             n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
20572             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
20573   match(Set dst (VectorMaskCmp src1 src2));
20574   format %{ "vpcmpgtb  $dst,$src1,$src2\t! cmpgt packed8B" %}
20575   ins_encode %{
20576     int vector_len = 0;
20577     __ vpcmpgtb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
20578   %}
20579   ins_pipe( pipe_slow );
20580 %}
20581 
20582 instruct vcmpgt16B(vecX dst, vecX src1, vecX src2) %{
20583   predicate(UseAVX > 0 && n->as_Vector()->length() == 16 &&
20584             n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
20585             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
20586   match(Set dst (VectorMaskCmp src1 src2));
20587   format %{ "vpcmpgtb  $dst,$src1,$src2\t! cmpgt packed16B" %}
20588   ins_encode %{
20589     int vector_len = 0;
20590     __ vpcmpgtb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
20591   %}
20592   ins_pipe( pipe_slow );
20593 %}
20594 
20595 instruct vcmpgt32B(vecY dst, vecY src1, vecY src2) %{
20596   predicate(UseAVX > 1 && n->as_Vector()->length() == 32 &&
20597             n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
20598             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
20599   match(Set dst (VectorMaskCmp src1 src2));
20600   format %{ "vpcmpgtb  $dst,$src1,$src2\t! cmpgt packed32B" %}
20601   ins_encode %{
20602     int vector_len = 1;
20603     __ vpcmpgtb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
20604   %}
20605   ins_pipe( pipe_slow );
20606 %}
20607 
20608 instruct vcmpgt64B(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
20609   predicate(UseAVX > 2 && VM_Version::supports_avx512bw() && n->as_Vector()->length() == 64 &&
20610             n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
20611             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
20612   match(Set dst (VectorMaskCmp src1 src2));
20613   effect(TEMP dst, TEMP scratch);
20614   format %{ "vpcmpnleb  k2,$src1,$src2\n\t"
20615             "vmovdqu8   $dst, k2{z}, 0xFFFFFFFFFF \t! cmpgt packed64B" %}
20616   ins_encode %{
20617     int vector_len = 2;
20618     Assembler::ComparisonPredicate cmp = Assembler::nle;
20619     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
20620     KRegister mask = k0; // The comparison itself is not being masked.
20621     __ evpcmpb(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
20622     __ evmovdqub($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
20623   %}
20624   ins_pipe( pipe_slow );
20625 %}
20626 
20627 instruct vcmpge8B(vecD dst, vecD src1, vecD src2, rRegL scratch) %{
20628   predicate(UseAVX > 0 && n->as_Vector()->length() == 8 &&
20629             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
20630             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
20631   match(Set dst (VectorMaskCmp src1 src2));
20632   effect(TEMP scratch);
20633   format %{ "vpcmpgtb  $dst,$src2,$src1\n\t"
20634             "vpxor $dst,$dst,0xFFFFFFFF\t! cmpge packed8B" %}
20635   ins_encode %{
20636     int vector_len = 0;
20637     __ vpcmpgtb($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, vector_len);
20638     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_all_bits_set()), vector_len, $scratch$$Register);
20639   %}
20640   ins_pipe( pipe_slow );
20641 %}
20642 
20643 instruct vcmpge16B(vecX dst, vecX src1, vecX src2, rRegL scratch) %{
20644   predicate(UseAVX > 0 && n->as_Vector()->length() == 16 &&
20645             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
20646             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
20647   match(Set dst (VectorMaskCmp src1 src2));
20648   effect(TEMP scratch);
20649   format %{ "vpcmpgtb  $dst,$src2,$src1\n\t"
20650             "vpxor $dst,$dst,0xFFFFFFFF\t! cmpge packed16B" %}
20651   ins_encode %{
20652     int vector_len = 0;
20653     __ vpcmpgtb($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, vector_len);
20654     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_all_bits_set()), vector_len, $scratch$$Register);
20655   %}
20656   ins_pipe( pipe_slow );
20657 %}
20658 
20659 instruct extract8d(regD dst, vecZ src, vecZ tmp, immI idx) %{
20660   predicate(UseAVX > 2 && n->in(1)->bottom_type()->is_vect()->length() == 8);
20661   match(Set dst (ExtractD src idx));
20662   effect(TEMP tmp);
20663   ins_encode %{
20664     int vector_len = 2;
20665     int midx = 0x7 & $idx$$constant;
20666     if (midx == 0 && $dst$$XMMRegister != $src$$XMMRegister) {
20667       __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister);
20668     } else if (midx == 1) {
20669       __ vpshufpd($dst$$XMMRegister, $src$$XMMRegister, $src$$XMMRegister, midx, vector_len);  
20670     } else if (midx > 1 && midx <= 7) {
20671       int extr_idx1 = midx / 2;
20672       int extr_idx2 = midx % 2;
20673       __ vextractf32x4($tmp$$XMMRegister, $src$$XMMRegister, extr_idx1);
20674       __ vpshufpd($dst$$XMMRegister, $tmp$$XMMRegister, $tmp$$XMMRegister, extr_idx2, vector_len);
20675   }
20676   %}
20677   ins_pipe( pipe_slow );
20678 %}
20679 
20680 instruct extract4d(regD dst, vecY src, vecY tmp, immI idx) %{
20681   predicate(UseAVX > 0 && n->in(1)->bottom_type()->is_vect()->length() == 4);
20682   match(Set dst (ExtractD src idx));
20683   effect(TEMP tmp);
20684   ins_encode %{
20685     int vector_len = 1;
20686     int midx = 0x3 & $idx$$constant;
20687     if (midx == 0 && $dst$$XMMRegister != $src$$XMMRegister) {
20688       __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister);
20689     } else if (midx == 1) {
20690       __ vpshufpd($dst$$XMMRegister, $src$$XMMRegister, $src$$XMMRegister, midx, vector_len);  
20691     } else if (midx > 1 && midx <= 3) {
20692       __ vextractf128($tmp$$XMMRegister, $src$$XMMRegister, 0x1);
20693       __ vpshufpd($dst$$XMMRegister, $tmp$$XMMRegister, $tmp$$XMMRegister, midx - 2, vector_len);
20694     }
20695        
20696   %}
20697   ins_pipe( pipe_slow );
20698 %}
20699 
20700 instruct extract2d(regD dst, vecX src, immI idx) %{
20701   predicate(UseAVX > 0 && n->in(1)->bottom_type()->is_vect()->length() == 2);
20702   match(Set dst (ExtractD src idx));
20703   ins_encode %{
20704     int vector_len = 0;
20705     int midx = 0x1 & $idx$$constant;
20706     if (midx == 0 && $dst$$XMMRegister != $src$$XMMRegister) {
20707       __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister);
20708     } else if (midx >=1) {
20709       __ vpshufpd($dst$$XMMRegister, $src$$XMMRegister, $src$$XMMRegister, midx, vector_len);  
20710     } 
20711   %}
20712   ins_pipe( pipe_slow );
20713 %}
20714 
20715 instruct extract1d(regD dst, vecD src, immI idx) %{
20716   predicate(UseAVX > 0 && n->in(1)->bottom_type()->is_vect()->length() == 1);
20717   match(Set dst (ExtractD src idx));
20718   ins_encode %{
20719     int vector_len = 0;
20720     int midx = 0x1 & $idx$$constant;
20721     if (midx == 0 && $dst$$XMMRegister != $src$$XMMRegister) {
20722       __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister);
20723     } 
20724   %}
20725   ins_pipe( pipe_slow );
20726 %}
20727 
20728 instruct extract16f(regF dst, vecZ src, vecZ tmp, immI idx) %{
20729   predicate(UseAVX > 2 && n->in(1)->bottom_type()->is_vect()->length() == 16);
20730   match(Set dst (ExtractF src idx));
20731   effect(TEMP tmp);
20732   ins_encode %{
20733     int vector_len=2; 
20734     int midx = 0xF & $idx$$constant;
20735     if (midx == 0 && $dst$$XMMRegister != $src$$XMMRegister) {
20736       __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister);
20737     } else if (midx >= 1 && midx <= 3) {
20738       __ vpshufps($dst$$XMMRegister, $src$$XMMRegister, $src$$XMMRegister, midx, vector_len);  
20739     } else {
20740       int extr_idx1 = midx / 4;
20741       int extr_idx2 = midx % 4;
20742       __ vextractf32x4($tmp$$XMMRegister, $src$$XMMRegister, extr_idx1);
20743       __ vpshufps($dst$$XMMRegister, $tmp$$XMMRegister, $tmp$$XMMRegister, extr_idx2, vector_len);
20744   }
20745   %}
20746   ins_pipe( pipe_slow );
20747 %}
20748 
20749 instruct extract8f(regF dst, vecY src, vecY tmp, immI idx) %{
20750   predicate(UseAVX > 0 && n->in(1)->bottom_type()->is_vect()->length() == 8);
20751   match(Set dst (ExtractF src idx));
20752   effect(TEMP tmp);
20753   ins_encode %{
20754     int vector_len=1; 
20755     int midx = 0x7 & $idx$$constant;
20756     if (midx == 0 && $dst$$XMMRegister != $src$$XMMRegister) {
20757       __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister);
20758     } else if (midx >= 1 && midx <= 3) {
20759       __ vpshufps($dst$$XMMRegister, $src$$XMMRegister, $src$$XMMRegister, midx, vector_len);  
20760     } else if (midx >= 4) {
20761       __ vextractf128($tmp$$XMMRegister, $src$$XMMRegister, 0x1);
20762       __ vpshufps($dst$$XMMRegister, $tmp$$XMMRegister, $tmp$$XMMRegister, midx - 4, vector_len);
20763     }
20764   %}
20765   ins_pipe( pipe_slow );
20766 %}
20767 
20768 instruct extract4f(regF dst, vecX src, immI idx) %{
20769   predicate(UseAVX > 0 && n->in(1)->bottom_type()->is_vect()->length() == 4);
20770   match(Set dst (ExtractF src idx));
20771   ins_encode %{
20772     int vector_len=0;
20773     int midx = 0x3 & $idx$$constant;
20774     if (midx == 0 && $dst$$XMMRegister != $src$$XMMRegister) {
20775       __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister);
20776     } else if (midx >= 1 && midx <= 3) {
20777       __ vpshufps($dst$$XMMRegister, $src$$XMMRegister, $src$$XMMRegister, midx, vector_len);  
20778     } 
20779   %}
20780   ins_pipe( pipe_slow );
20781 %}
20782 
20783 instruct extract2f(regF dst, vecD src, immI idx) %{
20784   predicate(UseAVX > 0 && n->in(1)->bottom_type()->is_vect()->length() == 2);
20785   match(Set dst (ExtractF src idx));
20786   ins_encode %{
20787     int vector_len=0;
20788     int midx = 0x1 & $idx$$constant;
20789     if (midx == 0 && $dst$$XMMRegister != $src$$XMMRegister) {
20790       __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister);
20791     } else {
20792       __ vpshufps($dst$$XMMRegister, $src$$XMMRegister, $src$$XMMRegister, midx, vector_len);  
20793     } 
20794   %}
20795   ins_pipe( pipe_slow );
20796 %}
20797  
20798 instruct extract8l(rRegL dst, vecZ src, vecZ tmp, immI idx) %{
20799   predicate(UseAVX > 2 && n->in(1)->bottom_type()->is_vect()->length() == 8);
20800   match(Set dst (ExtractL src idx));
20801   effect(TEMP tmp);
20802   ins_encode %{
20803     int midx = 0x7 & $idx$$constant;
20804     if (midx == 0) {
20805       __ movdl($dst$$Register, $src$$XMMRegister);
20806     } else if (midx >= 1 && midx <= 3) {
20807       __ pextrq($dst$$Register, $src$$XMMRegister, midx);
20808     } 
20809    else if (midx >= 4 && midx <= 7) {
20810     __ vextracti128($tmp$$XMMRegister, $src$$XMMRegister, 0x1); 
20811       __ pextrq($dst$$Register, $tmp$$XMMRegister, midx-4);
20812     } 
20813   %}
20814   ins_pipe( pipe_slow );
20815 %}
20816 
20817 instruct extract4l(rRegL dst, vecY src, immI idx) %{
20818   predicate(UseAVX > 0 && n->in(1)->bottom_type()->is_vect()->length() == 4);
20819   match(Set dst (ExtractL src idx));
20820   ins_encode %{
20821     int midx = 0x3 & $idx$$constant;
20822     if (midx == 0) {
20823       __ movdl($dst$$Register, $src$$XMMRegister);
20824     } else if (midx >= 1 && midx <= 3) {
20825       __ pextrq($dst$$Register, $src$$XMMRegister, midx);
20826     } 
20827   %}
20828   ins_pipe( pipe_slow );
20829 %}
20830 
20831 instruct extract2l(rRegL dst, vecX src, immI idx) %{
20832   predicate(UseAVX > 0 && n->in(1)->bottom_type()->is_vect()->length() == 2);
20833   match(Set dst (ExtractL src idx));
20834   ins_encode %{
20835     int midx = 0x1 & $idx$$constant;
20836     if (midx == 0) {
20837       __ movdl($dst$$Register, $src$$XMMRegister);
20838     } else if (midx >= 1) {
20839       __ pextrq($dst$$Register, $src$$XMMRegister, midx);
20840     } 
20841   %}
20842   ins_pipe( pipe_slow );
20843 %}
20844 
20845 instruct extract1l(rRegL dst, vecD src, immI idx) %{
20846   predicate(UseAVX > 0 && n->in(1)->bottom_type()->is_vect()->length() == 1);
20847   match(Set dst (ExtractL src idx));
20848   ins_encode %{
20849     int midx = 0x1 & $idx$$constant;
20850     if (midx == 0) {
20851       __ movdl($dst$$Register, $src$$XMMRegister);
20852     } 
20853   %}
20854   ins_pipe( pipe_slow );
20855 %}
20856 
20857 instruct extract16i(rRegI dst, vecZ src, vecZ tmp, immI idx) %{
20858   predicate(UseAVX > 2 && n->in(1)->bottom_type()->is_vect()->length() == 16);
20859   match(Set dst (ExtractI src idx));
20860   effect(TEMP tmp);
20861     ins_encode %{
20862     int midx = 0xF & $idx$$constant;
20863     if (midx == 0) {
20864       __ movdl($dst$$Register, $src$$XMMRegister);
20865     }
20866     else if (midx >= 1 && midx <= 3) {
20867       __ pextrd($dst$$Register, $src$$XMMRegister, midx);
20868     } 
20869     else {
20870       // Using 4 because there are 4 ints in 128-bit
20871       int extr_idx1 = midx / 4;
20872       int extr_idx2 = midx % 4;
20873       __ vextracti32x4($tmp$$XMMRegister, $src$$XMMRegister, extr_idx1);
20874       __ pextrd($dst$$Register, $tmp$$XMMRegister, extr_idx2);
20875     }
20876   %}
20877   ins_pipe( pipe_slow );
20878 %}
20879  
20880 instruct extract8i(rRegI dst, vecY src, vecY tmp, immI idx) %{
20881   predicate(UseAVX > 0 && n->in(1)->bottom_type()->is_vect()->length() == 8);
20882   match(Set dst (ExtractI src idx));
20883   effect(TEMP tmp);
20884   ins_encode %{
20885     int midx = 0x7 & $idx$$constant;
20886     if (midx == 0) {
20887       __ movdl($dst$$Register, $src$$XMMRegister);
20888     } else if (midx >= 1 && midx <= 3) {
20889       __ pextrd($dst$$Register, $src$$XMMRegister, midx);
20890     } else if (midx >= 4) {
20891       __ vextracti128($tmp$$XMMRegister, $src$$XMMRegister, 0x1);
20892       __ pextrd($dst$$Register, $tmp$$XMMRegister, midx - 4);
20893     }
20894   %}
20895   ins_pipe( pipe_slow );
20896 %}
20897 
20898 instruct extract4i(rRegI dst, vecX src, immI idx) %{
20899   predicate(UseAVX > 0 && n->in(1)->bottom_type()->is_vect()->length() == 4);
20900   match(Set dst (ExtractI src idx));
20901   ins_encode %{
20902     int midx = 0x3 & $idx$$constant;
20903     if (midx == 0) {
20904       __ movdl($dst$$Register, $src$$XMMRegister);
20905     } else if (midx >= 1 && midx <= 3) {
20906       __ pextrd($dst$$Register, $src$$XMMRegister, midx);
20907     } 
20908   %}
20909   ins_pipe( pipe_slow );
20910 %}
20911 
20912 instruct extract2i(rRegI dst, vecD src, immI idx) %{
20913   predicate(UseAVX > 0 && n->in(1)->bottom_type()->is_vect()->length() == 2);
20914   match(Set dst (ExtractI src idx));
20915   ins_encode %{
20916     int midx = 0x1 & $idx$$constant;
20917     if (midx == 0) {
20918       __ movdl($dst$$Register, $src$$XMMRegister);
20919     } else if (midx >= 1) {
20920       __ pextrd($dst$$Register, $src$$XMMRegister, midx);
20921     } 
20922   %}
20923   ins_pipe( pipe_slow );
20924 %}
20925  
20926 instruct extract32s(rRegI dst, vecZ src, vecZ tmp, immI idx) %{
20927   predicate(UseAVX > 2 && n->in(1)->bottom_type()->is_vect()->length() == 32);
20928   match(Set dst (ExtractS src idx));
20929   effect(TEMP tmp);
20930     ins_encode %{
20931     int midx = 0x1F & $idx$$constant;
20932     if (midx == 0) {
20933       __ movdl($dst$$Register, $src$$XMMRegister);
20934     __ movswl($dst$$Register, $dst$$Register);
20935     }
20936     else if (midx >= 1 && midx <= 7) {
20937       __ pextrw($dst$$Register, $src$$XMMRegister, midx);
20938     __ movswl($dst$$Register, $dst$$Register);
20939     } 
20940     else {
20941       int extr_idx1 = midx / 8;
20942       int extr_idx2 = midx % 8;
20943       __ vextracti32x4($tmp$$XMMRegister, $src$$XMMRegister, extr_idx1);
20944       __ pextrw($dst$$Register, $tmp$$XMMRegister, extr_idx2);
20945     __ movswl($dst$$Register, $dst$$Register);
20946     }
20947   %}
20948   ins_pipe( pipe_slow );
20949 %}
20950 
20951 instruct extract16s(rRegI dst, vecY src, vecY tmp, immI idx) %{
20952   predicate(UseAVX > 0 && n->in(1)->bottom_type()->is_vect()->length() == 16);
20953   match(Set dst (ExtractS src idx));
20954   effect(TEMP tmp);
20955   ins_encode %{
20956     int midx = 0xF & $idx$$constant;
20957     if (midx == 0) {
20958       __ movdl($dst$$Register, $src$$XMMRegister);
20959       __ movswl($dst$$Register, $dst$$Register);
20960     } else if (midx >= 1 && midx <= 7) {
20961       __ pextrw($dst$$Register, $src$$XMMRegister, midx);
20962       __ movswl($dst$$Register, $dst$$Register);
20963     }
20964   else if (midx >= 8 && midx <= 15) {
20965     __ vextracti128($tmp$$XMMRegister, $src$$XMMRegister, 0x1);
20966     __ pextrw($dst$$Register, $tmp$$XMMRegister, midx-8);
20967       __ movswl($dst$$Register, $dst$$Register);
20968   }
20969   %}
20970   ins_pipe( pipe_slow );
20971 %}
20972  
20973 instruct extract8s(rRegI dst, vecX src, immI idx) %{
20974   predicate(UseAVX > 0 && n->in(1)->bottom_type()->is_vect()->length() == 8);
20975   match(Set dst (ExtractS src idx));
20976   ins_encode %{
20977     int midx = 0x7 & $idx$$constant;
20978     if (midx == 0) {
20979       __ movdl($dst$$Register, $src$$XMMRegister);
20980       __ movswl($dst$$Register, $dst$$Register);
20981     } else if (midx >= 1) {
20982       __ pextrw($dst$$Register, $src$$XMMRegister, midx);
20983       __ movswl($dst$$Register, $dst$$Register);
20984     }
20985   %}
20986   ins_pipe( pipe_slow );
20987 %}
20988 
20989 instruct extract4s(rRegI dst, vecD src, immI idx) %{
20990   predicate(UseAVX > 0 && n->in(1)->bottom_type()->is_vect()->length() == 4);
20991   match(Set dst (ExtractS src idx));
20992   ins_encode %{
20993     int midx = 0x3 & $idx$$constant;
20994     if (midx == 0) {
20995       __ movdl($dst$$Register, $src$$XMMRegister);
20996       __ movswl($dst$$Register, $dst$$Register);
20997     } else if (midx >= 1) {
20998       __ pextrw($dst$$Register, $src$$XMMRegister, midx);
20999       __ movswl($dst$$Register, $dst$$Register);
21000     }
21001   %}
21002   ins_pipe( pipe_slow );
21003 %}
21004  
21005 instruct extract64b(rRegI dst, vecZ src, vecZ tmp, immI idx) %{
21006   predicate(UseAVX > 2 && n->in(1)->bottom_type()->is_vect()->length() == 64);
21007   match(Set dst (ExtractB src idx));
21008   effect(TEMP tmp);
21009     ins_encode %{
21010     int midx = 0x3F & $idx$$constant;
21011     if (midx == 0) {
21012       __ movdl($dst$$Register, $src$$XMMRegister);
21013     __ movsbl($dst$$Register, $dst$$Register);
21014     }
21015     else if (midx >= 1 && midx <= 15) {
21016       __ pextrb($dst$$Register, $src$$XMMRegister, midx);
21017     __ movsbl($dst$$Register, $dst$$Register);
21018     } 
21019     else {
21020       int extr_idx1 = midx / 16;
21021       int extr_idx2 = midx % 16;
21022       __ vextracti32x4($tmp$$XMMRegister, $src$$XMMRegister, extr_idx1);
21023       __ pextrb($dst$$Register, $tmp$$XMMRegister, extr_idx2);
21024     __ movsbl($dst$$Register, $dst$$Register);
21025     }
21026   %}
21027   ins_pipe( pipe_slow );
21028 %}
21029 
21030 instruct extract32b(rRegI dst, vecY src, vecY tmp, immI idx) %{
21031   predicate(UseAVX > 0 && n->in(1)->bottom_type()->is_vect()->length() == 32);
21032   match(Set dst (ExtractB src idx));
21033   effect(TEMP tmp);
21034     ins_encode %{
21035     int midx = 0x1F & $idx$$constant;
21036     if (midx == 0) {
21037       __ movdl($dst$$Register, $src$$XMMRegister);
21038     __ movsbl($dst$$Register, $dst$$Register);
21039     }
21040     else if (midx >= 1 && midx <= 15) {
21041       __ pextrb($dst$$Register, $src$$XMMRegister, midx);
21042     __ movsbl($dst$$Register, $dst$$Register);
21043     } 
21044     else {
21045       int extr_idx1 = midx / 16;
21046       int extr_idx2 = midx % 16;
21047       __ vextracti32x4($tmp$$XMMRegister, $src$$XMMRegister, extr_idx1);
21048       __ pextrb($dst$$Register, $tmp$$XMMRegister, extr_idx2);
21049     __ movsbl($dst$$Register, $dst$$Register);
21050     }
21051   %}
21052   ins_pipe( pipe_slow );
21053 %}
21054 
21055 instruct extract16b(rRegI dst, vecX src, immI idx) %{
21056   predicate(UseAVX > 0 && n->in(1)->bottom_type()->is_vect()->length() == 16);
21057   match(Set dst (ExtractB src idx));
21058   ins_encode %{
21059     int midx = 0xF & $idx$$constant;
21060     if (midx == 0) {
21061       __ movdl($dst$$Register, $src$$XMMRegister);
21062       __ movsbl($dst$$Register, $dst$$Register);
21063     } else if (midx >= 1) {
21064       __ pextrb($dst$$Register, $src$$XMMRegister, midx);
21065       __ movsbl($dst$$Register, $dst$$Register);
21066     }
21067   %}
21068   ins_pipe( pipe_slow );
21069 %}
21070 
21071 instruct extract8b(rRegI dst, vecD src, immI idx) %{
21072   predicate(UseAVX > 0 && n->in(1)->bottom_type()->is_vect()->length() == 8);
21073   match(Set dst (ExtractB src idx));
21074   ins_encode %{
21075     int midx = 0x7 & $idx$$constant;
21076     if (midx == 0) {
21077       __ movdl($dst$$Register, $src$$XMMRegister);
21078       __ movsbl($dst$$Register, $dst$$Register);
21079     } else if (midx >= 1) {
21080       __ pextrb($dst$$Register, $src$$XMMRegister, midx);
21081       __ movsbl($dst$$Register, $dst$$Register);
21082     }
21083   %}
21084   ins_pipe( pipe_slow );
21085 %}
21086 
21087 instruct vcmpge32B(vecY dst, vecY src1, vecY src2, rRegL scratch) %{
21088   predicate(UseAVX > 1 && n->as_Vector()->length() == 32 &&
21089             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
21090             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
21091   match(Set dst (VectorMaskCmp src1 src2));
21092   effect(TEMP scratch);
21093   format %{ "vpcmpgtb  $dst,$src2,$src1\n  "
21094             "vpxor $dst,$dst,0xFFFFFFFF\t! cmpge packed32B" %}
21095   ins_encode %{
21096     int vector_len = 1;
21097     __ vpcmpgtb($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, vector_len);
21098     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_all_bits_set()), vector_len, $scratch$$Register);
21099   %}
21100   ins_pipe( pipe_slow );
21101 %}
21102 
21103 instruct vcmpge64B(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
21104   predicate(UseAVX > 2 && VM_Version::supports_avx512bw() && n->as_Vector()->length() == 64 &&
21105             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
21106             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
21107   match(Set dst (VectorMaskCmp src1 src2));
21108   effect(TEMP dst, TEMP scratch);
21109   format %{ "vpcmpnltb  k2,$src1,$src2\n\t"
21110             "vmovdqu8   $dst, k2{z}, 0xFFFFFFFFFF \t! cmpge packed64B" %}
21111   ins_encode %{
21112     int vector_len = 2;
21113     Assembler::ComparisonPredicate cmp = Assembler::nlt;
21114     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
21115     KRegister mask = k0; // The comparison itself is not being masked.
21116     __ evpcmpb(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
21117     __ evmovdqub($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
21118   %}
21119   ins_pipe( pipe_slow );
21120 %}
21121 
21122 instruct vcmple8B(vecD dst, vecD src1, vecD src2, rRegL scratch) %{
21123   predicate(UseAVX > 0 && n->as_Vector()->length() == 8 &&
21124             n->as_VectorMaskCmp()->get_predicate() == BoolTest::le &&
21125             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
21126   match(Set dst (VectorMaskCmp src1 src2));
21127   effect(TEMP scratch);
21128   format %{ "vpcmpgtb  $dst,$src1,$src2\n\t"
21129             "vpxor $dst,$dst,0xFFFFFFFF\t! cmple packed8B" %}
21130   ins_encode %{
21131     int vector_len = 0;
21132     __ vpcmpgtb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
21133     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_all_bits_set()), vector_len, $scratch$$Register);
21134   %}
21135   ins_pipe( pipe_slow );
21136 %}
21137 
21138 instruct vcmple16B(vecX dst, vecX src1, vecX src2, rRegL scratch) %{
21139   predicate(UseAVX > 0 && n->as_Vector()->length() == 16 &&
21140             n->as_VectorMaskCmp()->get_predicate() == BoolTest::le &&
21141             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
21142   match(Set dst (VectorMaskCmp src1 src2));
21143   effect(TEMP scratch);
21144   format %{ "vpcmpgtb  $dst,$src1,$src2\n\t"
21145             "vpxor $dst,$dst,0xFFFFFFFF\t! cmple packed16B" %}
21146   ins_encode %{
21147     int vector_len = 0;
21148     __ vpcmpgtb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
21149     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_all_bits_set()), vector_len, $scratch$$Register);
21150   %}
21151   ins_pipe( pipe_slow );
21152 %}
21153 
21154 instruct vcmple32B(vecY dst, vecY src1, vecY src2, rRegL scratch) %{
21155   predicate(UseAVX > 1 && n->as_Vector()->length() == 32 &&
21156             n->as_VectorMaskCmp()->get_predicate() == BoolTest::le &&
21157             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
21158   match(Set dst (VectorMaskCmp src1 src2));
21159   effect(TEMP scratch);
21160   format %{ "vpcmpgtb  $dst,$src1,$src2\n\t"
21161             "vpxor $dst,$dst,0xFFFFFFFF\t! cmple packed32B" %}
21162   ins_encode %{
21163     int vector_len = 1;
21164     __ vpcmpgtb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
21165     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_all_bits_set()), vector_len, $scratch$$Register);
21166   %}
21167   ins_pipe( pipe_slow );
21168 %}
21169 
21170 instruct vcmple64B(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
21171   predicate(UseAVX > 2 && VM_Version::supports_avx512bw() && n->as_Vector()->length() == 64 &&
21172             n->as_VectorMaskCmp()->get_predicate() == BoolTest::le &&
21173             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
21174   match(Set dst (VectorMaskCmp src1 src2));
21175   effect(TEMP dst, TEMP scratch);
21176   format %{ "vpcmpleb  k2,$src1,$src2\n\t"
21177             "vmovdqu8   $dst, k2{z}, 0xFFFFFFFFFF \t! cmple packed64B" %}
21178   ins_encode %{
21179     int vector_len = 2;
21180     Assembler::ComparisonPredicate cmp = Assembler::le;
21181     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
21182     KRegister mask = k0; // The comparison itself is not being masked.
21183     __ evpcmpb(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
21184     __ evmovdqub($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
21185   %}
21186   ins_pipe( pipe_slow );
21187 %}
21188 
21189 instruct vcmpne8B(vecD dst, vecD src1, vecD src2, rRegL scratch) %{
21190   predicate(UseAVX > 0 && n->as_Vector()->length() == 8 &&
21191             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ne &&
21192             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
21193   match(Set dst (VectorMaskCmp src1 src2));
21194   effect(TEMP scratch);
21195   format %{ "vpcmpeqb  $dst,$src1,$src2\n\t"
21196             "vpxor $dst,$dst,0xFFFFFFFF\t! cmpneq packed8B" %}
21197   ins_encode %{
21198     int vector_len = 0;
21199     __ vpcmpeqb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
21200     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_all_bits_set()), vector_len, $scratch$$Register);
21201   %}
21202   ins_pipe( pipe_slow );
21203 %}
21204 
21205 instruct vcmpne16B(vecX dst, vecX src1, vecX src2, rRegL scratch) %{
21206   predicate(UseAVX > 0 && n->as_Vector()->length() == 16 &&
21207             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ne &&
21208             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
21209   match(Set dst (VectorMaskCmp src1 src2));
21210   effect(TEMP scratch);
21211   format %{ "vpcmpeqb  $dst,$src1,$src2\n\t"
21212             "vpxor $dst,$dst,0xFFFFFFFF\t! cmpneq packed16B" %}
21213   ins_encode %{
21214     int vector_len = 0;
21215     __ vpcmpeqb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
21216     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_all_bits_set()), vector_len, $scratch$$Register);
21217   %}
21218   ins_pipe( pipe_slow );
21219 %}
21220 
21221 instruct vcmpne32B(vecY dst, vecY src1, vecY src2, rRegL scratch) %{
21222   predicate(UseAVX > 1 && n->as_Vector()->length() == 32 &&
21223             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ne &&
21224             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
21225   match(Set dst (VectorMaskCmp src1 src2));
21226   effect(TEMP scratch);
21227   format %{ "vpcmpeqb  $dst,$src1,$src2\n\t"
21228             "vpxor $dst,$dst,0xFFFFFFFF\t! cmpneq packed32B" %}
21229   ins_encode %{
21230     int vector_len = 1;
21231     __ vpcmpeqb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
21232     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_all_bits_set()), vector_len, $scratch$$Register);
21233   %}
21234   ins_pipe( pipe_slow );
21235 %}
21236 
21237 instruct vcmpne64B(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
21238   predicate(UseAVX > 2 && VM_Version::supports_avx512bw() && n->as_Vector()->length() == 64 &&
21239             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ne &&
21240             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
21241   match(Set dst (VectorMaskCmp src1 src2));
21242   effect(TEMP dst, TEMP scratch);
21243   format %{ "vpcmpneqb  k2,$src1,$src2\n\t"
21244             "vmovdqu8   $dst, k2{z}, 0xFFFFFFFFFF \t! cmpneq packed64B" %}
21245   ins_encode %{
21246     int vector_len = 2;
21247     Assembler::ComparisonPredicate cmp = Assembler::neq;
21248     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
21249     KRegister mask = k0; // The comparison itself is not being masked.
21250     __ evpcmpb(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
21251     __ evmovdqub($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
21252   %}
21253   ins_pipe( pipe_slow );
21254 %}
21255 
21256 instruct vcmpeq4S(vecD dst, vecD src1, vecD src2) %{
21257   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 &&
21258             n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
21259             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
21260   match(Set dst (VectorMaskCmp src1 src2));
21261   format %{ "vpcmpeqw  $dst,$src1,$src2\n\t! cmpeq packed4S" %}
21262   ins_encode %{
21263     int vector_len = 0;
21264     __ vpcmpeqw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
21265   %}
21266   ins_pipe( pipe_slow );
21267 %}
21268 
21269 instruct vcmpeq8S(vecX dst, vecX src1, vecX src2) %{
21270   predicate(UseAVX > 0 && n->as_Vector()->length() == 8 &&
21271             n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
21272             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
21273   match(Set dst (VectorMaskCmp src1 src2));
21274   format %{ "vpcmpeqw  $dst,$src1,$src2\n\t! cmpeq packed8S" %}
21275   ins_encode %{
21276     int vector_len = 0;
21277     __ vpcmpeqw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
21278   %}
21279   ins_pipe( pipe_slow );
21280 %}
21281 
21282 instruct vcmpeq16S(vecY dst, vecY src1, vecY src2) %{
21283   predicate(UseAVX > 1 && n->as_Vector()->length() == 16 &&
21284             n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
21285             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
21286   match(Set dst (VectorMaskCmp src1 src2));
21287   format %{ "vpcmpeqw  $dst,$src1,$src2\n\t! cmpeq packed16S" %}
21288   ins_encode %{
21289     int vector_len = 1;
21290     __ vpcmpeqw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
21291   %}
21292   ins_pipe( pipe_slow );
21293 %}
21294 
21295 instruct vcmpeq32S(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
21296   predicate(UseAVX > 2 && n->as_Vector()->length() == 32 &&
21297             n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
21298             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
21299   match(Set dst (VectorMaskCmp src1 src2));
21300   effect(TEMP dst, TEMP scratch);
21301   format %{ "vpcmpeqw  k2,$src1,$src2\n\t"
21302             "vmovdqu16 $dst, k2{z}, 0xFFFFFFFFFF \t! cmpeq packed32S" %}
21303   ins_encode %{
21304     int vector_len = 2;
21305     Assembler::ComparisonPredicate cmp = Assembler::eq;
21306     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
21307     KRegister mask = k0; // The comparison itself is not being masked.
21308     __ evpcmpw(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
21309     __ evmovdquw($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
21310   %}
21311   ins_pipe( pipe_slow );
21312 %}
21313 
21314 instruct vcmplt4S(vecD dst, vecD src1, vecD src2) %{
21315   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 &&
21316             n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
21317             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
21318   match(Set dst (VectorMaskCmp src1 src2));
21319   format %{ "vpcmpgtw  $dst,$src2,$src1\t! cmplt packed4S" %}
21320   ins_encode %{
21321     int vector_len = 0;
21322     __ vpcmpgtw($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, vector_len);
21323   %}
21324   ins_pipe( pipe_slow );
21325 %}
21326 
21327 instruct vcmplt8S(vecX dst, vecX src1, vecX src2) %{
21328   predicate(UseAVX > 0 && n->as_Vector()->length() == 8 &&
21329             n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
21330             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
21331   match(Set dst (VectorMaskCmp src1 src2));
21332   format %{ "vpcmpgtw  $dst,$src2,$src1\t! cmplt packed8S" %}
21333   ins_encode %{
21334     int vector_len = 0;
21335     __ vpcmpgtw($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, vector_len);
21336   %}
21337   ins_pipe( pipe_slow );
21338 %}
21339 
21340 instruct vcmplt16S(vecY dst, vecY src1, vecY src2) %{
21341   predicate(UseAVX > 1 && n->as_Vector()->length() == 16 &&
21342             n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
21343             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
21344   match(Set dst (VectorMaskCmp src1 src2));
21345   format %{ "vpcmpgtw  $dst,$src2,$src1\t! cmplt packed16S" %}
21346   ins_encode %{
21347     int vector_len = 1;
21348     __ vpcmpgtw($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, vector_len);
21349   %}
21350   ins_pipe( pipe_slow );
21351 %}
21352 
21353 instruct vcmplt32S(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
21354   predicate(UseAVX > 2 && n->as_Vector()->length() == 32 &&
21355             n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
21356             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
21357   match(Set dst (VectorMaskCmp src1 src2));
21358   effect(TEMP dst, TEMP scratch);
21359   format %{ "vpcmpnlew  k2,$src1,$src2\n\t"
21360             "vmovdqu16   $dst, k2{z}, 0xFFFFFFFFFF \t! cmpgt packed32S" %}
21361   ins_encode %{
21362     int vector_len = 2;
21363     Assembler::ComparisonPredicate cmp = Assembler::lt;
21364     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
21365     KRegister mask = k0; // The comparison itself is not being masked.
21366     __ evpcmpw(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
21367     __ evmovdquw($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
21368   %}
21369   ins_pipe( pipe_slow );
21370 %}
21371 
21372 instruct vcmpgt4S(vecD dst, vecD src1, vecD src2) %{
21373   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 &&
21374             n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
21375             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
21376   match(Set dst (VectorMaskCmp src1 src2));
21377   format %{ "vpcmpgtw  $dst,$src1,$src2\t! cmpgt packed4S" %}
21378   ins_encode %{
21379     int vector_len = 0;
21380     __ vpcmpgtw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
21381   %}
21382   ins_pipe( pipe_slow );
21383 %}
21384 
21385 instruct vcmpgt8S(vecX dst, vecX src1, vecX src2) %{
21386   predicate(UseAVX > 0 && n->as_Vector()->length() == 8 &&
21387             n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
21388             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
21389   match(Set dst (VectorMaskCmp src1 src2));
21390   format %{ "vpcmpgtw  $dst,$src1,$src2\t! cmpgt packed8S" %}
21391   ins_encode %{
21392     int vector_len = 0;
21393     __ vpcmpgtw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
21394   %}
21395   ins_pipe( pipe_slow );
21396 %}
21397 
21398 instruct vcmpgt16S(vecY dst, vecY src1, vecY src2) %{
21399   predicate(UseAVX > 1 && n->as_Vector()->length() == 16 &&
21400             n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
21401             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
21402   match(Set dst (VectorMaskCmp src1 src2));
21403   format %{ "vpcmpgtw  $dst,$src1,$src2\t! cmpgt packed16S" %}
21404   ins_encode %{
21405     int vector_len = 1;
21406     __ vpcmpgtw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
21407   %}
21408   ins_pipe( pipe_slow );
21409 %}
21410 
21411 instruct vcmpgt32S(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
21412   predicate(UseAVX > 2 && n->as_Vector()->length() == 32 &&
21413             n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
21414             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
21415   match(Set dst (VectorMaskCmp src1 src2));
21416   effect(TEMP dst, TEMP scratch);
21417   format %{ "vpcmpnlew  k2,$src1,$src2\n\t"
21418             "vmovdqu16   $dst, k2{z}, 0xFFFFFFFFFF \t! cmpgt packed32S" %}
21419   ins_encode %{
21420     int vector_len = 2;
21421     Assembler::ComparisonPredicate cmp = Assembler::nle;
21422     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
21423     KRegister mask = k0; // The comparison itself is not being masked.
21424     __ evpcmpw(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
21425     __ evmovdquw($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
21426   %}
21427   ins_pipe( pipe_slow );
21428 %}
21429 
21430 instruct vcmpge4S(vecD dst, vecD src1, vecD src2, rRegL scratch) %{
21431   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 &&
21432             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
21433             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
21434   match(Set dst (VectorMaskCmp src1 src2));
21435   effect(TEMP scratch);
21436   format %{ "vpcmpgtw  $dst,$src2,$src1\n\t"
21437             "vpxor $dst,$dst,0xFFFFFFFF\t! cmpge packed4S" %}
21438   ins_encode %{
21439     int vector_len = 0;
21440     __ vpcmpgtw($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, vector_len);
21441     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_all_bits_set()), vector_len, $scratch$$Register);
21442   %}
21443   ins_pipe( pipe_slow );
21444 %}
21445 
21446 instruct vcmpge8S(vecX dst, vecX src1, vecX src2, rRegL scratch) %{
21447   predicate(UseAVX > 0 && n->as_Vector()->length() == 8 &&
21448             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
21449             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
21450   match(Set dst (VectorMaskCmp src1 src2));
21451   effect(TEMP scratch);
21452   format %{ "vpcmpgtw  $dst,$src2,$src1\n\t"
21453             "vpxor $dst,$dst,0xFFFFFFFF\t! cmpge packed8S" %}
21454   ins_encode %{
21455     int vector_len = 0;
21456     __ vpcmpgtw($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, vector_len);
21457     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_all_bits_set()), vector_len, $scratch$$Register);
21458   %}
21459   ins_pipe( pipe_slow );
21460 %}
21461 
21462 instruct vcmpge16S(vecY dst, vecY src1, vecY src2, rRegL scratch) %{
21463   predicate(UseAVX > 1 && n->as_Vector()->length() == 16 &&
21464             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
21465             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
21466   match(Set dst (VectorMaskCmp src1 src2));
21467   effect(TEMP scratch);
21468   format %{ "vpcmpgtw  $dst,$src2,$src1\n\t"
21469             "vpxor $dst,$dst,0xFFFFFFFF\t! cmpge packed16S" %}
21470   ins_encode %{
21471     int vector_len = 1;
21472     __ vpcmpgtw($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, vector_len);
21473     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_all_bits_set()), vector_len, $scratch$$Register);
21474   %}
21475   ins_pipe( pipe_slow );
21476 %}
21477 
21478 instruct vcmpge32S(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
21479   predicate(UseAVX > 2 && n->as_Vector()->length() == 32 &&
21480             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
21481             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
21482   match(Set dst (VectorMaskCmp src1 src2));
21483   effect(TEMP dst, TEMP scratch);
21484   format %{ "vpcmpnltw  k2,$src1,$src2\n\t"
21485             "vmovdqu16   $dst, k2{z}, 0xFFFFFFFFFF \t! cmpge packed32S" %}
21486   ins_encode %{
21487     int vector_len = 2;
21488     Assembler::ComparisonPredicate cmp = Assembler::nlt;
21489     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
21490     KRegister mask = k0; // The comparison itself is not being masked.
21491     __ evpcmpw(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
21492     __ evmovdquw($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
21493   %}
21494   ins_pipe( pipe_slow );
21495 %}
21496 
21497 instruct vcmple4S(vecD dst, vecD src1, vecD src2, rRegL scratch) %{
21498   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 &&
21499             n->as_VectorMaskCmp()->get_predicate() == BoolTest::le &&
21500             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
21501   match(Set dst (VectorMaskCmp src1 src2));
21502   effect(TEMP scratch);
21503   format %{ "vpcmpgtw  $dst,$src1,$src2\n\t"
21504             "vpxor $dst,$dst,0xFFFFFFFF\t! cmple packed4S" %}
21505   ins_encode %{
21506     int vector_len = 0;
21507     __ vpcmpgtw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
21508     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_all_bits_set()), vector_len, $scratch$$Register);
21509   %}
21510   ins_pipe( pipe_slow );
21511 %}
21512 
21513 instruct vcmple8S(vecX dst, vecX src1, vecX src2, rRegL scratch) %{
21514   predicate(UseAVX > 0 && n->as_Vector()->length() == 8 &&
21515             n->as_VectorMaskCmp()->get_predicate() == BoolTest::le &&
21516             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
21517   match(Set dst (VectorMaskCmp src1 src2));
21518   effect(TEMP scratch);
21519   format %{ "vpcmpgtw  $dst,$src1,$src2\n\t"
21520             "vpxor $dst,$dst,0xFFFFFFFF\t! cmple packed8S" %}
21521   ins_encode %{
21522     int vector_len = 0;
21523     __ vpcmpgtw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
21524     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_all_bits_set()), vector_len, $scratch$$Register);
21525   %}
21526   ins_pipe( pipe_slow );
21527 %}
21528 
21529 instruct vcmple16S(vecY dst, vecY src1, vecY src2, rRegL scratch) %{
21530   predicate(UseAVX > 1 && n->as_Vector()->length() == 16 &&
21531             n->as_VectorMaskCmp()->get_predicate() == BoolTest::le &&
21532             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
21533   match(Set dst (VectorMaskCmp src1 src2));
21534   effect(TEMP scratch);
21535   format %{ "vpcmpgtw  $dst,$src1,$src2\n\t"
21536             "vpxor $dst,$dst,0xFFFFFFFF\t! cmple packed16S" %}
21537   ins_encode %{
21538     int vector_len = 1;
21539     __ vpcmpgtw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
21540     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_all_bits_set()), vector_len, $scratch$$Register);
21541   %}
21542   ins_pipe( pipe_slow );
21543 %}
21544 
21545 instruct vcmple32S(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
21546   predicate(UseAVX > 2 && n->as_Vector()->length() == 32 &&
21547             n->as_VectorMaskCmp()->get_predicate() == BoolTest::le &&
21548             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
21549   match(Set dst (VectorMaskCmp src1 src2));
21550   effect(TEMP dst, TEMP scratch);
21551   format %{ "vpcmplew  k2,$src1,$src2\n\t"
21552             "vmovdqu16   $dst, k2{z}, 0xFFFFFFFFFF \t! cmple packed32S" %}
21553   ins_encode %{
21554     int vector_len = 2;
21555     Assembler::ComparisonPredicate cmp = Assembler::le;
21556     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
21557     KRegister mask = k0; // The comparison itself is not being masked.
21558     __ evpcmpw(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
21559     __ evmovdquw($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
21560   %}
21561   ins_pipe( pipe_slow );
21562 %}
21563 
21564 instruct vcmpne4S(vecD dst, vecD src1, vecD src2, rRegL scratch) %{
21565   predicate(UseAVX > 0 && n->as_Vector()->length() == 4 &&
21566             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ne &&
21567             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
21568   match(Set dst (VectorMaskCmp src1 src2));
21569   effect(TEMP scratch);
21570   format %{ "vpcmpeqw  $dst,$src1,$src2\n\t"
21571             "vpxor $dst,$dst,0xFFFFFFFF\t! cmpneq packed4S" %}
21572   ins_encode %{
21573     int vector_len = 0;
21574     __ vpcmpeqw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
21575     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_all_bits_set()), vector_len, $scratch$$Register);
21576   %}
21577   ins_pipe( pipe_slow );
21578 %}
21579 
21580 instruct vcmpne8S(vecX dst, vecX src1, vecX src2, rRegL scratch) %{
21581   predicate(UseAVX > 0 && n->as_Vector()->length() == 8 &&
21582             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ne &&
21583             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
21584   match(Set dst (VectorMaskCmp src1 src2));
21585   effect(TEMP scratch);
21586   format %{ "vpcmpeqw  $dst,$src1,$src2\n\t"
21587             "vpxor $dst,$dst,0xFFFFFFFF\t! cmpneq packed8S" %}
21588   ins_encode %{
21589     int vector_len = 0;
21590     __ vpcmpeqw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
21591     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_all_bits_set()), vector_len, $scratch$$Register);
21592   %}
21593   ins_pipe( pipe_slow );
21594 %}
21595 
21596 instruct vcmpne16S(vecY dst, vecY src1, vecY src2, rRegL scratch) %{
21597   predicate(UseAVX > 1 && n->as_Vector()->length() == 16 &&
21598             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ne &&
21599             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
21600   match(Set dst (VectorMaskCmp src1 src2));
21601   effect(TEMP scratch);
21602   format %{ "vpcmpeqw  $dst,$src1,$src2\n\t"
21603             "vpxor $dst,$dst,0xFFFFFFFF\t! cmpneq packed16S" %}
21604   ins_encode %{
21605     int vector_len = 1;
21606     __ vpcmpeqw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
21607     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_all_bits_set()), vector_len, $scratch$$Register);
21608   %}
21609   ins_pipe( pipe_slow );
21610 %}
21611 
21612 instruct vcmpne32S(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
21613   predicate(UseAVX > 2 && n->as_Vector()->length() == 32 &&
21614             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ne &&
21615             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
21616   match(Set dst (VectorMaskCmp src1 src2));
21617   effect(TEMP dst, TEMP scratch);
21618   format %{ "vpcmpneqw  k2,$src1,$src2\n\t"
21619             "vmovdqu16   $dst, k2{z}, 0xFFFFFFFFFF \t! cmpneq packed32S" %}
21620   ins_encode %{
21621     int vector_len = 2;
21622     Assembler::ComparisonPredicate cmp = Assembler::neq;
21623     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
21624     KRegister mask = k0; // The comparison itself is not being masked.
21625     __ evpcmpw(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
21626     __ evmovdquw($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
21627   %}
21628   ins_pipe( pipe_slow );
21629 %}
21630 
21631 instruct vcmpeq1L(vecD dst, vecD src1, vecD src2) %{
21632   predicate(UseAVX > 0 && n->as_Vector()->length() == 1 &&
21633             n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
21634             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
21635   match(Set dst (VectorMaskCmp src1 src2));
21636   format %{ "vpcmpeqq  $dst,$src1,$src2\n\t! cmpeq packed1L" %}
21637   ins_encode %{
21638     int vector_len = 0;
21639     __ vpcmpeqq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
21640   %}
21641   ins_pipe( pipe_slow );
21642 %}
21643 
21644 instruct vcmpeq2L(vecX dst, vecX src1, vecX src2) %{
21645   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 &&
21646             n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
21647             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
21648   match(Set dst (VectorMaskCmp src1 src2));
21649   format %{ "vpcmpeqq  $dst,$src1,$src2\n\t! cmpeq packed2L" %}
21650   ins_encode %{
21651     int vector_len = 0;
21652     __ vpcmpeqq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
21653   %}
21654   ins_pipe( pipe_slow );
21655 %}
21656 
21657 instruct vcmpeq4L(vecY dst, vecY src1, vecY src2) %{
21658   predicate(UseAVX > 1 && n->as_Vector()->length() == 4 &&
21659             n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
21660             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
21661   match(Set dst (VectorMaskCmp src1 src2));
21662   format %{ "vpcmpeqq  $dst,$src1,$src2\n\t! cmpeq packed4L" %}
21663   ins_encode %{
21664     int vector_len = 1;
21665     __ vpcmpeqq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
21666   %}
21667   ins_pipe( pipe_slow );
21668 %}
21669 
21670 instruct vcmpeq8L(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
21671   predicate(UseAVX > 2 && n->as_Vector()->length() == 8 &&
21672             n->as_VectorMaskCmp()->get_predicate() == BoolTest::eq &&
21673             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
21674   match(Set dst (VectorMaskCmp src1 src2));
21675   effect(TEMP dst, TEMP scratch);
21676   format %{ "vpcmpeqq  k2,$src1,$src2\n\t"
21677             "vmovdqu64 $dst, k2{z}, 0xFFFFFFFFFF \t! cmpeq packed8L" %}
21678   ins_encode %{
21679     int vector_len = 2;
21680     Assembler::ComparisonPredicate cmp = Assembler::eq;
21681     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
21682     KRegister mask = k0; // The comparison itself is not being masked.
21683     __ evpcmpq(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
21684     __ evmovdquq($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
21685   %}
21686   ins_pipe( pipe_slow );
21687 %}
21688 
21689 instruct vcmplt1L(vecD dst, vecD src1, vecD src2) %{
21690   predicate(UseAVX > 0 && n->as_Vector()->length() == 1 &&
21691             n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
21692             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
21693   match(Set dst (VectorMaskCmp src1 src2));
21694   format %{ "vpcmpgtq  $dst,$src2,$src1\t! cmplt packed1L" %}
21695   ins_encode %{
21696     int vector_len = 0;
21697     __ vpcmpgtq($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, vector_len);
21698   %}
21699   ins_pipe( pipe_slow );
21700 %}
21701 
21702 instruct vcmplt2L(vecX dst, vecX src1, vecX src2) %{
21703   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 &&
21704             n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
21705             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
21706   match(Set dst (VectorMaskCmp src1 src2));
21707   format %{ "vpcmpgtq  $dst,$src2,$src1\t! cmplt packed2L" %}
21708   ins_encode %{
21709     int vector_len = 0;
21710     __ vpcmpgtq($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, vector_len);
21711   %}
21712   ins_pipe( pipe_slow );
21713 %}
21714 
21715 instruct vcmplt4L(vecY dst, vecY src1, vecY src2) %{
21716   predicate(UseAVX > 1 && n->as_Vector()->length() == 4 &&
21717             n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
21718             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
21719   match(Set dst (VectorMaskCmp src1 src2));
21720   format %{ "vpcmpgtq  $dst,$src2,$src1\t! cmplt packed4L" %}
21721   ins_encode %{
21722     int vector_len = 1;
21723     __ vpcmpgtq($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, vector_len);
21724   %}
21725   ins_pipe( pipe_slow );
21726 %}
21727 
21728 instruct vcmplt8L(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
21729   predicate(UseAVX > 2 && n->as_Vector()->length() == 8 &&
21730             n->as_VectorMaskCmp()->get_predicate() == BoolTest::lt &&
21731             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
21732   match(Set dst (VectorMaskCmp src1 src2));
21733   effect(TEMP dst, TEMP scratch);
21734   format %{ "vpcmpnleq  k2,$src1,$src2\n\t"
21735             "vmovdqu64   $dst, k2{z}, 0xFFFFFFFFFF \t! cmpgt packed8L" %}
21736   ins_encode %{
21737     int vector_len = 2;
21738     Assembler::ComparisonPredicate cmp = Assembler::lt;
21739     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
21740     KRegister mask = k0; // The comparison itself is not being masked.
21741     __ evpcmpq(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
21742     __ evmovdquq($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
21743   %}
21744   ins_pipe( pipe_slow );
21745 %}
21746 
21747 instruct vcmpgt1L(vecD dst, vecD src1, vecD src2) %{
21748   predicate(UseAVX > 0 && n->as_Vector()->length() == 1 &&
21749             n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
21750             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
21751   match(Set dst (VectorMaskCmp src1 src2));
21752   format %{ "vpcmpgtq  $dst,$src1,$src2\t! cmpgt packed1L" %}
21753   ins_encode %{
21754     int vector_len = 0;
21755     __ vpcmpgtq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
21756   %}
21757   ins_pipe( pipe_slow );
21758 %}
21759 
21760 instruct vcmpgt2L(vecX dst, vecX src1, vecX src2) %{
21761   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 &&
21762             n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
21763             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
21764   match(Set dst (VectorMaskCmp src1 src2));
21765   format %{ "vpcmpgtq  $dst,$src1,$src2\t! cmpgt packed2L" %}
21766   ins_encode %{
21767     int vector_len = 0;
21768     __ vpcmpgtq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
21769   %}
21770   ins_pipe( pipe_slow );
21771 %}
21772 
21773 instruct vcmpgt4L(vecY dst, vecY src1, vecY src2) %{
21774   predicate(UseAVX > 1 && n->as_Vector()->length() == 4 &&
21775             n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
21776             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
21777   match(Set dst (VectorMaskCmp src1 src2));
21778   format %{ "vpcmpgtq  $dst,$src1,$src2\t! cmpgt packed4L" %}
21779   ins_encode %{
21780     int vector_len = 1;
21781     __ vpcmpgtq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vector_len);
21782   %}
21783   ins_pipe( pipe_slow );
21784 %}
21785 
21786 instruct vcmpgt8L(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
21787   predicate(UseAVX > 2 && n->as_Vector()->length() == 8 &&
21788             n->as_VectorMaskCmp()->get_predicate() == BoolTest::gt &&
21789             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
21790   match(Set dst (VectorMaskCmp src1 src2));
21791   effect(TEMP dst, TEMP scratch);
21792   format %{ "vpcmpnleq  k2,$src1,$src2\n\t"
21793             "vmovdqu64   $dst, k2{z}, 0xFFFFFFFFFF \t! cmpgt packed8L" %}
21794   ins_encode %{
21795     int vector_len = 2;
21796     Assembler::ComparisonPredicate cmp = Assembler::nle;
21797     KRegister ktmp = k2; // Use a hardcoded temp due to no k register allocation.
21798     KRegister mask = k0; // The comparison itself is not being masked.
21799     __ evpcmpq(ktmp, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vector_len);
21800     __ evmovdquq($dst$$XMMRegister, ktmp, ExternalAddress(vector_all_bits_set()), false, vector_len, $scratch$$Register);
21801   %}
21802   ins_pipe( pipe_slow );
21803 %}
21804 
21805 instruct vcmpge1L(vecD dst, vecD src1, vecD src2, rRegL scratch) %{
21806   predicate(UseAVX > 0 && n->as_Vector()->length() == 1 &&
21807             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
21808             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
21809   match(Set dst (VectorMaskCmp src1 src2));
21810   effect(TEMP scratch);
21811   format %{ "vpcmpgtq  $dst,$src2,$src1\n\t"
21812             "vpxor $dst,$dst,0xFFFFFFFF\t! cmpge packed1L" %}
21813   ins_encode %{
21814     int vector_len = 0;
21815     __ vpcmpgtq($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, vector_len);
21816     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_all_bits_set()), vector_len, $scratch$$Register);
21817   %}
21818   ins_pipe( pipe_slow );
21819 %}
21820 
21821 instruct vcmpge2L(vecX dst, vecX src1, vecX src2, rRegL scratch) %{
21822   predicate(UseAVX > 0 && n->as_Vector()->length() == 2 &&
21823             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
21824             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
21825   match(Set dst (VectorMaskCmp src1 src2));
21826   effect(TEMP scratch);
21827   format %{ "vpcmpgtq  $dst,$src2,$src1\n\t"
21828             "vpxor $dst,$dst,0xFFFFFFFF\t! cmpge packed2L" %}
21829   ins_encode %{
21830     int vector_len = 0;
21831     __ vpcmpgtq($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, vector_len);
21832     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_all_bits_set()), vector_len, $scratch$$Register);
21833   %}
21834   ins_pipe( pipe_slow );
21835 %}
21836 
21837 instruct vcmpge4L(vecY dst, vecY src1, vecY src2, rRegL scratch) %{
21838   predicate(UseAVX > 1 && n->as_Vector()->length() == 4 &&
21839             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
21840             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
21841   match(Set dst (VectorMaskCmp src1 src2));
21842   effect(TEMP scratch);
21843   format %{ "vpcmpgtq  $dst,$src2,$src1\n\t"
21844             "vpxor $dst,$dst,0xFFFFFFFF\t! cmpge packed4L" %}
21845   ins_encode %{
21846     int vector_len = 1;
21847     __ vpcmpgtq($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, vector_len);
21848     __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_all_bits_set()), vector_len, $scratch$$Register);
21849   %}
21850   ins_pipe( pipe_slow );
21851 %}
21852 
21853 instruct vcmpge8L(vecZ dst, vecZ src1, vecZ src2, rRegL scratch) %{
21854   predicate(UseAVX > 2 && n->as_Vector()->length() == 8 &&
21855             n->as_VectorMaskCmp()->get_predicate() == BoolTest::ge &&
21856             n->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
21857   match(Set dst (Ve