204 struct {
205 uint32_t fsgsbase : 1,
206 : 2,
207 bmi1 : 1,
208 : 1,
209 avx2 : 1,
210 : 2,
211 bmi2 : 1,
212 erms : 1,
213 : 1,
214 rtm : 1,
215 : 4,
216 avx512f : 1,
217 avx512dq : 1,
218 : 1,
219 adx : 1,
220 : 6,
221 avx512pf : 1,
222 avx512er : 1,
223 avx512cd : 1,
224 : 1,
225 avx512bw : 1,
226 avx512vl : 1;
227 } bits;
228 };
229
230 union XemXcr0Eax {
231 uint32_t value;
232 struct {
233 uint32_t x87 : 1,
234 sse : 1,
235 ymm : 1,
236 bndregs : 1,
237 bndcsr : 1,
238 opmask : 1,
239 zmm512 : 1,
240 zmm32 : 1,
241 : 24;
242 } bits;
243 };
244
265 CPU_SSE4A = (1 << 10),
266 CPU_SSE4_1 = (1 << 11),
267 CPU_SSE4_2 = (1 << 12),
268 CPU_POPCNT = (1 << 13),
269 CPU_LZCNT = (1 << 14),
270 CPU_TSC = (1 << 15),
271 CPU_TSCINV = (1 << 16),
272 CPU_AVX = (1 << 17),
273 CPU_AVX2 = (1 << 18),
274 CPU_AES = (1 << 19),
275 CPU_ERMS = (1 << 20), // enhanced 'rep movsb/stosb' instructions
276 CPU_CLMUL = (1 << 21), // carryless multiply for CRC
277 CPU_BMI1 = (1 << 22),
278 CPU_BMI2 = (1 << 23),
279 CPU_RTM = (1 << 24), // Restricted Transactional Memory instructions
280 CPU_ADX = (1 << 25),
281 CPU_AVX512F = (1 << 26), // AVX 512bit foundation instructions
282 CPU_AVX512DQ = (1 << 27),
283 CPU_AVX512PF = (1 << 28),
284 CPU_AVX512ER = (1 << 29),
285 CPU_AVX512CD = (1 << 30),
286 CPU_AVX512BW = (1 << 31)
287 };
288
289 #define CPU_AVX512VL UCONST64(0x100000000) // EVEX instructions with smaller vector length : enums are limited to 32bit
290
291 enum Extended_Family {
292 // AMD
293 CPU_FAMILY_AMD_11H = 0x11,
294 // Intel
295 CPU_FAMILY_INTEL_CORE = 6,
296 CPU_MODEL_NEHALEM = 0x1e,
297 CPU_MODEL_NEHALEM_EP = 0x1a,
298 CPU_MODEL_NEHALEM_EX = 0x2e,
299 CPU_MODEL_WESTMERE = 0x25,
300 CPU_MODEL_WESTMERE_EP = 0x2c,
301 CPU_MODEL_WESTMERE_EX = 0x2f,
302 CPU_MODEL_SANDYBRIDGE = 0x2a,
303 CPU_MODEL_SANDYBRIDGE_EP = 0x2d,
304 CPU_MODEL_IVYBRIDGE_EP = 0x3a,
305 CPU_MODEL_HASWELL_E3 = 0x3c,
306 CPU_MODEL_HASWELL_E7 = 0x3f,
307 CPU_MODEL_BROADWELL = 0x3d,
308 CPU_MODEL_SKYLAKE = CPU_MODEL_HASWELL_E3
309 };
499 result |= CPU_CLMUL;
500 if (_cpuid_info.sef_cpuid7_ebx.bits.rtm != 0)
501 result |= CPU_RTM;
502
503 // AMD features.
504 if (is_amd()) {
505 if ((_cpuid_info.ext_cpuid1_edx.bits.tdnow != 0) ||
506 (_cpuid_info.ext_cpuid1_ecx.bits.prefetchw != 0))
507 result |= CPU_3DNOW_PREFETCH;
508 if (_cpuid_info.ext_cpuid1_ecx.bits.lzcnt != 0)
509 result |= CPU_LZCNT;
510 if (_cpuid_info.ext_cpuid1_ecx.bits.sse4a != 0)
511 result |= CPU_SSE4A;
512 }
513 // Intel features.
514 if(is_intel()) {
515 if(_cpuid_info.sef_cpuid7_ebx.bits.adx != 0)
516 result |= CPU_ADX;
517 if(_cpuid_info.sef_cpuid7_ebx.bits.bmi2 != 0)
518 result |= CPU_BMI2;
519 if(_cpuid_info.ext_cpuid1_ecx.bits.lzcnt_intel != 0)
520 result |= CPU_LZCNT;
521 // for Intel, ecx.bits.misalignsse bit (bit 8) indicates support for prefetchw
522 if (_cpuid_info.ext_cpuid1_ecx.bits.misalignsse != 0) {
523 result |= CPU_3DNOW_PREFETCH;
524 }
525 }
526
527 return result;
528 }
529
530 static bool os_supports_avx_vectors() {
531 bool retVal = false;
532 if (supports_evex()) {
533 // Verify that OS save/restore all bits of EVEX registers
534 // during signal processing.
535 int nreg = 2 LP64_ONLY(+2);
536 retVal = true;
537 for (int i = 0; i < 16 * nreg; i++) { // 64 bytes per zmm register
538 if (_cpuid_info.zmm_save[i] != ymm_test_value()) {
704 static bool supports_tsc() { return (_features & CPU_TSC) != 0; }
705 static bool supports_aes() { return (_features & CPU_AES) != 0; }
706 static bool supports_erms() { return (_features & CPU_ERMS) != 0; }
707 static bool supports_clmul() { return (_features & CPU_CLMUL) != 0; }
708 static bool supports_rtm() { return (_features & CPU_RTM) != 0; }
709 static bool supports_bmi1() { return (_features & CPU_BMI1) != 0; }
710 static bool supports_bmi2() { return (_features & CPU_BMI2) != 0; }
711 static bool supports_adx() { return (_features & CPU_ADX) != 0; }
712 static bool supports_evex() { return (_features & CPU_AVX512F) != 0; }
713 static bool supports_avx512dq() { return (_features & CPU_AVX512DQ) != 0; }
714 static bool supports_avx512pf() { return (_features & CPU_AVX512PF) != 0; }
715 static bool supports_avx512er() { return (_features & CPU_AVX512ER) != 0; }
716 static bool supports_avx512cd() { return (_features & CPU_AVX512CD) != 0; }
717 static bool supports_avx512bw() { return (_features & CPU_AVX512BW) != 0; }
718 static bool supports_avx512vl() { return (_features & CPU_AVX512VL) != 0; }
719 static bool supports_avx512vlbw() { return (supports_avx512bw() && supports_avx512vl()); }
720 static bool supports_avx512novl() { return (supports_evex() && !supports_avx512vl()); }
721 static bool supports_avx512nobw() { return (supports_evex() && !supports_avx512bw()); }
722 static bool supports_avx256only() { return (supports_avx2() && !supports_evex()); }
723 static bool supports_avxonly() { return ((supports_avx2() || supports_avx()) && !supports_evex()); }
724 // Intel features
725 static bool is_intel_family_core() { return is_intel() &&
726 extended_cpu_family() == CPU_FAMILY_INTEL_CORE; }
727
728 static bool is_intel_tsc_synched_at_init() {
729 if (is_intel_family_core()) {
730 uint32_t ext_model = extended_cpu_model();
731 if (ext_model == CPU_MODEL_NEHALEM_EP ||
732 ext_model == CPU_MODEL_WESTMERE_EP ||
733 ext_model == CPU_MODEL_SANDYBRIDGE_EP ||
734 ext_model == CPU_MODEL_IVYBRIDGE_EP) {
735 // <= 2-socket invariant tsc support. EX versions are usually used
736 // in > 2-socket systems and likely don't synchronize tscs at
737 // initialization.
738 // Code that uses tsc values must be prepared for them to arbitrarily
739 // jump forward or backward.
740 return true;
741 }
742 }
743 return false;
|
204 struct {
205 uint32_t fsgsbase : 1,
206 : 2,
207 bmi1 : 1,
208 : 1,
209 avx2 : 1,
210 : 2,
211 bmi2 : 1,
212 erms : 1,
213 : 1,
214 rtm : 1,
215 : 4,
216 avx512f : 1,
217 avx512dq : 1,
218 : 1,
219 adx : 1,
220 : 6,
221 avx512pf : 1,
222 avx512er : 1,
223 avx512cd : 1,
224 sha : 1,
225 avx512bw : 1,
226 avx512vl : 1;
227 } bits;
228 };
229
230 union XemXcr0Eax {
231 uint32_t value;
232 struct {
233 uint32_t x87 : 1,
234 sse : 1,
235 ymm : 1,
236 bndregs : 1,
237 bndcsr : 1,
238 opmask : 1,
239 zmm512 : 1,
240 zmm32 : 1,
241 : 24;
242 } bits;
243 };
244
265 CPU_SSE4A = (1 << 10),
266 CPU_SSE4_1 = (1 << 11),
267 CPU_SSE4_2 = (1 << 12),
268 CPU_POPCNT = (1 << 13),
269 CPU_LZCNT = (1 << 14),
270 CPU_TSC = (1 << 15),
271 CPU_TSCINV = (1 << 16),
272 CPU_AVX = (1 << 17),
273 CPU_AVX2 = (1 << 18),
274 CPU_AES = (1 << 19),
275 CPU_ERMS = (1 << 20), // enhanced 'rep movsb/stosb' instructions
276 CPU_CLMUL = (1 << 21), // carryless multiply for CRC
277 CPU_BMI1 = (1 << 22),
278 CPU_BMI2 = (1 << 23),
279 CPU_RTM = (1 << 24), // Restricted Transactional Memory instructions
280 CPU_ADX = (1 << 25),
281 CPU_AVX512F = (1 << 26), // AVX 512bit foundation instructions
282 CPU_AVX512DQ = (1 << 27),
283 CPU_AVX512PF = (1 << 28),
284 CPU_AVX512ER = (1 << 29),
285 CPU_AVX512CD = (1 << 30)
286 // Keeping sign bit 31 unassigned.
287 };
288
289 #define CPU_AVX512BW ((uint64_t)UCONST64(0x100000000)) // enums are limited to 31 bit
290 #define CPU_AVX512VL ((uint64_t)UCONST64(0x200000000)) // EVEX instructions with smaller vector length
291 #define CPU_SHA ((uint64_t)UCONST64(0x400000000)) // SHA instructions
292
293 enum Extended_Family {
294 // AMD
295 CPU_FAMILY_AMD_11H = 0x11,
296 // Intel
297 CPU_FAMILY_INTEL_CORE = 6,
298 CPU_MODEL_NEHALEM = 0x1e,
299 CPU_MODEL_NEHALEM_EP = 0x1a,
300 CPU_MODEL_NEHALEM_EX = 0x2e,
301 CPU_MODEL_WESTMERE = 0x25,
302 CPU_MODEL_WESTMERE_EP = 0x2c,
303 CPU_MODEL_WESTMERE_EX = 0x2f,
304 CPU_MODEL_SANDYBRIDGE = 0x2a,
305 CPU_MODEL_SANDYBRIDGE_EP = 0x2d,
306 CPU_MODEL_IVYBRIDGE_EP = 0x3a,
307 CPU_MODEL_HASWELL_E3 = 0x3c,
308 CPU_MODEL_HASWELL_E7 = 0x3f,
309 CPU_MODEL_BROADWELL = 0x3d,
310 CPU_MODEL_SKYLAKE = CPU_MODEL_HASWELL_E3
311 };
501 result |= CPU_CLMUL;
502 if (_cpuid_info.sef_cpuid7_ebx.bits.rtm != 0)
503 result |= CPU_RTM;
504
505 // AMD features.
506 if (is_amd()) {
507 if ((_cpuid_info.ext_cpuid1_edx.bits.tdnow != 0) ||
508 (_cpuid_info.ext_cpuid1_ecx.bits.prefetchw != 0))
509 result |= CPU_3DNOW_PREFETCH;
510 if (_cpuid_info.ext_cpuid1_ecx.bits.lzcnt != 0)
511 result |= CPU_LZCNT;
512 if (_cpuid_info.ext_cpuid1_ecx.bits.sse4a != 0)
513 result |= CPU_SSE4A;
514 }
515 // Intel features.
516 if(is_intel()) {
517 if(_cpuid_info.sef_cpuid7_ebx.bits.adx != 0)
518 result |= CPU_ADX;
519 if(_cpuid_info.sef_cpuid7_ebx.bits.bmi2 != 0)
520 result |= CPU_BMI2;
521 if (_cpuid_info.sef_cpuid7_ebx.bits.sha != 0)
522 result |= CPU_SHA;
523 if(_cpuid_info.ext_cpuid1_ecx.bits.lzcnt_intel != 0)
524 result |= CPU_LZCNT;
525 // for Intel, ecx.bits.misalignsse bit (bit 8) indicates support for prefetchw
526 if (_cpuid_info.ext_cpuid1_ecx.bits.misalignsse != 0) {
527 result |= CPU_3DNOW_PREFETCH;
528 }
529 }
530
531 return result;
532 }
533
534 static bool os_supports_avx_vectors() {
535 bool retVal = false;
536 if (supports_evex()) {
537 // Verify that OS save/restore all bits of EVEX registers
538 // during signal processing.
539 int nreg = 2 LP64_ONLY(+2);
540 retVal = true;
541 for (int i = 0; i < 16 * nreg; i++) { // 64 bytes per zmm register
542 if (_cpuid_info.zmm_save[i] != ymm_test_value()) {
708 static bool supports_tsc() { return (_features & CPU_TSC) != 0; }
709 static bool supports_aes() { return (_features & CPU_AES) != 0; }
710 static bool supports_erms() { return (_features & CPU_ERMS) != 0; }
711 static bool supports_clmul() { return (_features & CPU_CLMUL) != 0; }
712 static bool supports_rtm() { return (_features & CPU_RTM) != 0; }
713 static bool supports_bmi1() { return (_features & CPU_BMI1) != 0; }
714 static bool supports_bmi2() { return (_features & CPU_BMI2) != 0; }
715 static bool supports_adx() { return (_features & CPU_ADX) != 0; }
716 static bool supports_evex() { return (_features & CPU_AVX512F) != 0; }
717 static bool supports_avx512dq() { return (_features & CPU_AVX512DQ) != 0; }
718 static bool supports_avx512pf() { return (_features & CPU_AVX512PF) != 0; }
719 static bool supports_avx512er() { return (_features & CPU_AVX512ER) != 0; }
720 static bool supports_avx512cd() { return (_features & CPU_AVX512CD) != 0; }
721 static bool supports_avx512bw() { return (_features & CPU_AVX512BW) != 0; }
722 static bool supports_avx512vl() { return (_features & CPU_AVX512VL) != 0; }
723 static bool supports_avx512vlbw() { return (supports_avx512bw() && supports_avx512vl()); }
724 static bool supports_avx512novl() { return (supports_evex() && !supports_avx512vl()); }
725 static bool supports_avx512nobw() { return (supports_evex() && !supports_avx512bw()); }
726 static bool supports_avx256only() { return (supports_avx2() && !supports_evex()); }
727 static bool supports_avxonly() { return ((supports_avx2() || supports_avx()) && !supports_evex()); }
728 static bool supports_sha() { return (_features & CPU_SHA) != 0; }
729 // Intel features
730 static bool is_intel_family_core() { return is_intel() &&
731 extended_cpu_family() == CPU_FAMILY_INTEL_CORE; }
732
733 static bool is_intel_tsc_synched_at_init() {
734 if (is_intel_family_core()) {
735 uint32_t ext_model = extended_cpu_model();
736 if (ext_model == CPU_MODEL_NEHALEM_EP ||
737 ext_model == CPU_MODEL_WESTMERE_EP ||
738 ext_model == CPU_MODEL_SANDYBRIDGE_EP ||
739 ext_model == CPU_MODEL_IVYBRIDGE_EP) {
740 // <= 2-socket invariant tsc support. EX versions are usually used
741 // in > 2-socket systems and likely don't synchronize tscs at
742 // initialization.
743 // Code that uses tsc values must be prepared for them to arbitrarily
744 // jump forward or backward.
745 return true;
746 }
747 }
748 return false;
|