328 CPU_AVX512DQ = (1 << 27),
329 CPU_AVX512PF = (1 << 28),
330 CPU_AVX512ER = (1 << 29),
331 CPU_AVX512CD = (1 << 30)
332 // Keeping sign bit 31 unassigned.
333 };
334
335 #define CPU_AVX512BW ((uint64_t)UCONST64(0x100000000)) // enums are limited to 31 bit
336 #define CPU_AVX512VL ((uint64_t)UCONST64(0x200000000)) // EVEX instructions with smaller vector length
337 #define CPU_SHA ((uint64_t)UCONST64(0x400000000)) // SHA instructions
338 #define CPU_FMA ((uint64_t)UCONST64(0x800000000)) // FMA instructions
339 #define CPU_VZEROUPPER ((uint64_t)UCONST64(0x1000000000)) // Vzeroupper instruction
340 #define CPU_AVX512_VPOPCNTDQ ((uint64_t)UCONST64(0x2000000000)) // Vector popcount
341 #define CPU_AVX512_VPCLMULQDQ ((uint64_t)UCONST64(0x4000000000)) //Vector carryless multiplication
342 #define CPU_VAES ((uint64_t)UCONST64(0x8000000000)) // Vector AES instructions
343 #define CPU_VNNI ((uint64_t)UCONST64(0x10000000000)) // Vector Neural Network Instructions
344
345 #define CPU_FLUSH ((uint64_t)UCONST64(0x20000000000)) // flush instruction
346 #define CPU_FLUSHOPT ((uint64_t)UCONST64(0x40000000000)) // flushopt instruction
347 #define CPU_CLWB ((uint64_t)UCONST64(0x80000000000)) // clwb instruction
348
349 enum Extended_Family {
350 // AMD
351 CPU_FAMILY_AMD_11H = 0x11,
352 // ZX
353 CPU_FAMILY_ZX_CORE_F6 = 6,
354 CPU_FAMILY_ZX_CORE_F7 = 7,
355 // Intel
356 CPU_FAMILY_INTEL_CORE = 6,
357 CPU_MODEL_NEHALEM = 0x1e,
358 CPU_MODEL_NEHALEM_EP = 0x1a,
359 CPU_MODEL_NEHALEM_EX = 0x2e,
360 CPU_MODEL_WESTMERE = 0x25,
361 CPU_MODEL_WESTMERE_EP = 0x2c,
362 CPU_MODEL_WESTMERE_EX = 0x2f,
363 CPU_MODEL_SANDYBRIDGE = 0x2a,
364 CPU_MODEL_SANDYBRIDGE_EP = 0x2d,
365 CPU_MODEL_IVYBRIDGE_EP = 0x3a,
366 CPU_MODEL_HASWELL_E3 = 0x3c,
367 CPU_MODEL_HASWELL_E7 = 0x3f,
550 if (_cpuid_info.sef_cpuid7_ebx.bits.avx512cd != 0)
551 result |= CPU_AVX512CD;
552 if (_cpuid_info.sef_cpuid7_ebx.bits.avx512dq != 0)
553 result |= CPU_AVX512DQ;
554 if (_cpuid_info.sef_cpuid7_ebx.bits.avx512pf != 0)
555 result |= CPU_AVX512PF;
556 if (_cpuid_info.sef_cpuid7_ebx.bits.avx512er != 0)
557 result |= CPU_AVX512ER;
558 if (_cpuid_info.sef_cpuid7_ebx.bits.avx512bw != 0)
559 result |= CPU_AVX512BW;
560 if (_cpuid_info.sef_cpuid7_ebx.bits.avx512vl != 0)
561 result |= CPU_AVX512VL;
562 if (_cpuid_info.sef_cpuid7_ecx.bits.avx512_vpopcntdq != 0)
563 result |= CPU_AVX512_VPOPCNTDQ;
564 if (_cpuid_info.sef_cpuid7_ecx.bits.avx512_vpclmulqdq != 0)
565 result |= CPU_AVX512_VPCLMULQDQ;
566 if (_cpuid_info.sef_cpuid7_ecx.bits.vaes != 0)
567 result |= CPU_VAES;
568 if (_cpuid_info.sef_cpuid7_ecx.bits.avx512_vnni != 0)
569 result |= CPU_VNNI;
570 }
571 }
572 if (_cpuid_info.sef_cpuid7_ebx.bits.bmi1 != 0)
573 result |= CPU_BMI1;
574 if (_cpuid_info.std_cpuid1_edx.bits.tsc != 0)
575 result |= CPU_TSC;
576 if (_cpuid_info.ext_cpuid7_edx.bits.tsc_invariance != 0)
577 result |= CPU_TSCINV;
578 if (_cpuid_info.std_cpuid1_ecx.bits.aes != 0)
579 result |= CPU_AES;
580 if (_cpuid_info.sef_cpuid7_ebx.bits.erms != 0)
581 result |= CPU_ERMS;
582 if (_cpuid_info.std_cpuid1_ecx.bits.clmul != 0)
583 result |= CPU_CLMUL;
584 if (_cpuid_info.sef_cpuid7_ebx.bits.rtm != 0)
585 result |= CPU_RTM;
586 if (_cpuid_info.sef_cpuid7_ebx.bits.adx != 0)
587 result |= CPU_ADX;
588 if (_cpuid_info.sef_cpuid7_ebx.bits.bmi2 != 0)
589 result |= CPU_BMI2;
841 static bool supports_avx512pf() { return (_features & CPU_AVX512PF) != 0; }
842 static bool supports_avx512er() { return (_features & CPU_AVX512ER) != 0; }
843 static bool supports_avx512cd() { return (_features & CPU_AVX512CD) != 0; }
844 static bool supports_avx512bw() { return (_features & CPU_AVX512BW) != 0; }
845 static bool supports_avx512vl() { return (_features & CPU_AVX512VL) != 0; }
846 static bool supports_avx512vlbw() { return (supports_evex() && supports_avx512bw() && supports_avx512vl()); }
847 static bool supports_avx512vldq() { return (supports_evex() && supports_avx512dq() && supports_avx512vl()); }
848 static bool supports_avx512vlbwdq() { return (supports_evex() && supports_avx512vl() &&
849 supports_avx512bw() && supports_avx512dq()); }
850 static bool supports_avx512novl() { return (supports_evex() && !supports_avx512vl()); }
851 static bool supports_avx512nobw() { return (supports_evex() && !supports_avx512bw()); }
852 static bool supports_avx256only() { return (supports_avx2() && !supports_evex()); }
853 static bool supports_avxonly() { return ((supports_avx2() || supports_avx()) && !supports_evex()); }
854 static bool supports_sha() { return (_features & CPU_SHA) != 0; }
855 static bool supports_fma() { return (_features & CPU_FMA) != 0 && supports_avx(); }
856 static bool supports_vzeroupper() { return (_features & CPU_VZEROUPPER) != 0; }
857 static bool supports_vpopcntdq() { return (_features & CPU_AVX512_VPOPCNTDQ) != 0; }
858 static bool supports_avx512_vpclmulqdq() { return (_features & CPU_AVX512_VPCLMULQDQ) != 0; }
859 static bool supports_vaes() { return (_features & CPU_VAES) != 0; }
860 static bool supports_vnni() { return (_features & CPU_VNNI) != 0; }
861
862 // Intel features
863 static bool is_intel_family_core() { return is_intel() &&
864 extended_cpu_family() == CPU_FAMILY_INTEL_CORE; }
865
866 static bool is_intel_tsc_synched_at_init() {
867 if (is_intel_family_core()) {
868 uint32_t ext_model = extended_cpu_model();
869 if (ext_model == CPU_MODEL_NEHALEM_EP ||
870 ext_model == CPU_MODEL_WESTMERE_EP ||
871 ext_model == CPU_MODEL_SANDYBRIDGE_EP ||
872 ext_model == CPU_MODEL_IVYBRIDGE_EP) {
873 // <= 2-socket invariant tsc support. EX versions are usually used
874 // in > 2-socket systems and likely don't synchronize tscs at
875 // initialization.
876 // Code that uses tsc values must be prepared for them to arbitrarily
877 // jump forward or backward.
878 return true;
879 }
880 }
|
328 CPU_AVX512DQ = (1 << 27),
329 CPU_AVX512PF = (1 << 28),
330 CPU_AVX512ER = (1 << 29),
331 CPU_AVX512CD = (1 << 30)
332 // Keeping sign bit 31 unassigned.
333 };
334
335 #define CPU_AVX512BW ((uint64_t)UCONST64(0x100000000)) // enums are limited to 31 bit
336 #define CPU_AVX512VL ((uint64_t)UCONST64(0x200000000)) // EVEX instructions with smaller vector length
337 #define CPU_SHA ((uint64_t)UCONST64(0x400000000)) // SHA instructions
338 #define CPU_FMA ((uint64_t)UCONST64(0x800000000)) // FMA instructions
339 #define CPU_VZEROUPPER ((uint64_t)UCONST64(0x1000000000)) // Vzeroupper instruction
340 #define CPU_AVX512_VPOPCNTDQ ((uint64_t)UCONST64(0x2000000000)) // Vector popcount
341 #define CPU_AVX512_VPCLMULQDQ ((uint64_t)UCONST64(0x4000000000)) //Vector carryless multiplication
342 #define CPU_VAES ((uint64_t)UCONST64(0x8000000000)) // Vector AES instructions
343 #define CPU_VNNI ((uint64_t)UCONST64(0x10000000000)) // Vector Neural Network Instructions
344
345 #define CPU_FLUSH ((uint64_t)UCONST64(0x20000000000)) // flush instruction
346 #define CPU_FLUSHOPT ((uint64_t)UCONST64(0x40000000000)) // flushopt instruction
347 #define CPU_CLWB ((uint64_t)UCONST64(0x80000000000)) // clwb instruction
348 #define CPU_VBMI2 ((uint64_t)UCONST64(0x100000000000)) // VBMI2 shift left double instructions
349
350
351 enum Extended_Family {
352 // AMD
353 CPU_FAMILY_AMD_11H = 0x11,
354 // ZX
355 CPU_FAMILY_ZX_CORE_F6 = 6,
356 CPU_FAMILY_ZX_CORE_F7 = 7,
357 // Intel
358 CPU_FAMILY_INTEL_CORE = 6,
359 CPU_MODEL_NEHALEM = 0x1e,
360 CPU_MODEL_NEHALEM_EP = 0x1a,
361 CPU_MODEL_NEHALEM_EX = 0x2e,
362 CPU_MODEL_WESTMERE = 0x25,
363 CPU_MODEL_WESTMERE_EP = 0x2c,
364 CPU_MODEL_WESTMERE_EX = 0x2f,
365 CPU_MODEL_SANDYBRIDGE = 0x2a,
366 CPU_MODEL_SANDYBRIDGE_EP = 0x2d,
367 CPU_MODEL_IVYBRIDGE_EP = 0x3a,
368 CPU_MODEL_HASWELL_E3 = 0x3c,
369 CPU_MODEL_HASWELL_E7 = 0x3f,
552 if (_cpuid_info.sef_cpuid7_ebx.bits.avx512cd != 0)
553 result |= CPU_AVX512CD;
554 if (_cpuid_info.sef_cpuid7_ebx.bits.avx512dq != 0)
555 result |= CPU_AVX512DQ;
556 if (_cpuid_info.sef_cpuid7_ebx.bits.avx512pf != 0)
557 result |= CPU_AVX512PF;
558 if (_cpuid_info.sef_cpuid7_ebx.bits.avx512er != 0)
559 result |= CPU_AVX512ER;
560 if (_cpuid_info.sef_cpuid7_ebx.bits.avx512bw != 0)
561 result |= CPU_AVX512BW;
562 if (_cpuid_info.sef_cpuid7_ebx.bits.avx512vl != 0)
563 result |= CPU_AVX512VL;
564 if (_cpuid_info.sef_cpuid7_ecx.bits.avx512_vpopcntdq != 0)
565 result |= CPU_AVX512_VPOPCNTDQ;
566 if (_cpuid_info.sef_cpuid7_ecx.bits.avx512_vpclmulqdq != 0)
567 result |= CPU_AVX512_VPCLMULQDQ;
568 if (_cpuid_info.sef_cpuid7_ecx.bits.vaes != 0)
569 result |= CPU_VAES;
570 if (_cpuid_info.sef_cpuid7_ecx.bits.avx512_vnni != 0)
571 result |= CPU_VNNI;
572 if (_cpuid_info.sef_cpuid7_ecx.bits.avx512_vbmi2 != 0)
573 result |= CPU_VBMI2;
574 }
575 }
576 if (_cpuid_info.sef_cpuid7_ebx.bits.bmi1 != 0)
577 result |= CPU_BMI1;
578 if (_cpuid_info.std_cpuid1_edx.bits.tsc != 0)
579 result |= CPU_TSC;
580 if (_cpuid_info.ext_cpuid7_edx.bits.tsc_invariance != 0)
581 result |= CPU_TSCINV;
582 if (_cpuid_info.std_cpuid1_ecx.bits.aes != 0)
583 result |= CPU_AES;
584 if (_cpuid_info.sef_cpuid7_ebx.bits.erms != 0)
585 result |= CPU_ERMS;
586 if (_cpuid_info.std_cpuid1_ecx.bits.clmul != 0)
587 result |= CPU_CLMUL;
588 if (_cpuid_info.sef_cpuid7_ebx.bits.rtm != 0)
589 result |= CPU_RTM;
590 if (_cpuid_info.sef_cpuid7_ebx.bits.adx != 0)
591 result |= CPU_ADX;
592 if (_cpuid_info.sef_cpuid7_ebx.bits.bmi2 != 0)
593 result |= CPU_BMI2;
845 static bool supports_avx512pf() { return (_features & CPU_AVX512PF) != 0; }
846 static bool supports_avx512er() { return (_features & CPU_AVX512ER) != 0; }
847 static bool supports_avx512cd() { return (_features & CPU_AVX512CD) != 0; }
848 static bool supports_avx512bw() { return (_features & CPU_AVX512BW) != 0; }
849 static bool supports_avx512vl() { return (_features & CPU_AVX512VL) != 0; }
850 static bool supports_avx512vlbw() { return (supports_evex() && supports_avx512bw() && supports_avx512vl()); }
851 static bool supports_avx512vldq() { return (supports_evex() && supports_avx512dq() && supports_avx512vl()); }
852 static bool supports_avx512vlbwdq() { return (supports_evex() && supports_avx512vl() &&
853 supports_avx512bw() && supports_avx512dq()); }
854 static bool supports_avx512novl() { return (supports_evex() && !supports_avx512vl()); }
855 static bool supports_avx512nobw() { return (supports_evex() && !supports_avx512bw()); }
856 static bool supports_avx256only() { return (supports_avx2() && !supports_evex()); }
857 static bool supports_avxonly() { return ((supports_avx2() || supports_avx()) && !supports_evex()); }
858 static bool supports_sha() { return (_features & CPU_SHA) != 0; }
859 static bool supports_fma() { return (_features & CPU_FMA) != 0 && supports_avx(); }
860 static bool supports_vzeroupper() { return (_features & CPU_VZEROUPPER) != 0; }
861 static bool supports_vpopcntdq() { return (_features & CPU_AVX512_VPOPCNTDQ) != 0; }
862 static bool supports_avx512_vpclmulqdq() { return (_features & CPU_AVX512_VPCLMULQDQ) != 0; }
863 static bool supports_vaes() { return (_features & CPU_VAES) != 0; }
864 static bool supports_vnni() { return (_features & CPU_VNNI) != 0; }
865 static bool supports_vbmi2() { return (_features & CPU_VBMI2) != 0; }
866
867 // Intel features
868 static bool is_intel_family_core() { return is_intel() &&
869 extended_cpu_family() == CPU_FAMILY_INTEL_CORE; }
870
871 static bool is_intel_tsc_synched_at_init() {
872 if (is_intel_family_core()) {
873 uint32_t ext_model = extended_cpu_model();
874 if (ext_model == CPU_MODEL_NEHALEM_EP ||
875 ext_model == CPU_MODEL_WESTMERE_EP ||
876 ext_model == CPU_MODEL_SANDYBRIDGE_EP ||
877 ext_model == CPU_MODEL_IVYBRIDGE_EP) {
878 // <= 2-socket invariant tsc support. EX versions are usually used
879 // in > 2-socket systems and likely don't synchronize tscs at
880 // initialization.
881 // Code that uses tsc values must be prepared for them to arbitrarily
882 // jump forward or backward.
883 return true;
884 }
885 }
|