6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2011, 2016 by Delphix. All rights reserved.
24 * Copyright 2013 Nexenta Systems, Inc. All rights reserved.
25 * Copyright 2014 Josef "Jeff" Sipek <jeffpc@josefsipek.net>
26 */
27 /*
28 * Copyright (c) 2010, Intel Corporation.
29 * All rights reserved.
30 */
31 /*
32 * Portions Copyright 2009 Advanced Micro Devices, Inc.
33 */
34 /*
35 * Copyright 2020 Joyent, Inc.
36 */
37
38 /*
39 * CPU Identification logic
40 *
41 * The purpose of this file and its companion, cpuid_subr.c, is to help deal
42 * with the identification of CPUs, their features, and their topologies. More
43 * specifically, this file helps drive the following:
44 *
45 * 1. Enumeration of features of the processor which are used by the kernel to
46 * determine what features to enable or disable. These may be instruction set
47 * enhancements or features that we use.
48 *
49 * 2. Enumeration of instruction set architecture (ISA) additions that userland
50 * will be told about through the auxiliary vector.
51 *
52 * 3. Understanding the physical topology of the CPU such as the number of
53 * caches, how many cores it has, whether or not it supports symmetric
54 * multi-processing (SMT), etc.
55 *
56 * ------------------------
1210 * enumerate support for disabling TSX. In general, we'd rather use this method
1211 * when available as it doesn't require disabling hyperthreading to be
1212 * effective. Currently we basically are relying on microcode for processors
1213 * that enumerate MDS_NO.
1214 *
1215 * The microcode features are enumerated as part of the IA32_ARCH_CAPABILITIES.
1216 * When bit 7 (IA32_ARCH_CAP_TSX_CTRL) is present, then we are given two
1217 * different powers. The first allows us to cause all transactions to
1218 * immediately abort. The second gives us a means of disabling TSX completely,
1219 * which includes removing it from cpuid. If we have support for this in
1220 * microcode during the first cpuid pass, then we'll disable TSX completely such
1221 * that user land never has a chance to observe the bit. However, if we are late
1222 * loading the microcode, then we must use the functionality to cause
1223 * transactions to automatically abort. This is necessary for user land's sake.
1224 * Once a program sees a cpuid bit, it must not be taken away.
1225 *
1226 * We track whether or not we should do this based on what cpuid pass we're in.
1227 * Whenever we hit cpuid_scan_security() on the boot CPU and we're still on pass
1228 * 1 of the cpuid logic, then we can completely turn off TSX. Notably this
1229 * should happen twice. Once in the normal cpuid_pass1() code and then a second
1230 * time after we do the initial microcode update.
1231 *
1232 * If TAA has been fixed, then it will be enumerated in IA32_ARCH_CAPABILITIES
1233 * as TAA_NO. In such a case, we will still disable TSX: it's proven to be an
1234 * unfortunate feature in a number of ways, and taking the opportunity to
1235 * finally be able to turn it off is likely to be of benefit in the future.
1236 *
1237 * SUMMARY
1238 *
1239 * The following table attempts to summarize the mitigations for various issues
1240 * and what's done in various places:
1241 *
1242 * - Spectre v1: Not currently mitigated
1243 * - swapgs: lfences after swapgs paths
1244 * - Spectre v2: Retpolines/RSB Stuffing or EIBRS if HW support
1245 * - Meltdown: Kernel Page Table Isolation
1246 * - Spectre v3a: Updated CPU microcode
1247 * - Spectre v4: Not currently mitigated
1248 * - SpectreRSB: SMEP and RSB Stuffing
1249 * - L1TF: spec_uarch_flush, SMT exclusion, requires microcode
1250 * - MDS: x86_md_clear, requires microcode, disabling SMT
2822 } else {
2823 x86_taa_mitigation = X86_TAA_NOTHING;
2824 }
2825 return;
2826 }
2827
2828 /*
2829 * We have TSX_CTRL, but we can only fully disable TSX if we're early
2830 * enough in boot.
2831 *
2832 * Otherwise, we'll fall back to causing transactions to abort as our
2833 * mitigation. TSX-using code will always take the fallback path.
2834 */
2835 if (cpi->cpi_pass < 4) {
2836 x86_taa_mitigation = X86_TAA_TSX_DISABLE;
2837 } else {
2838 x86_taa_mitigation = X86_TAA_TSX_FORCE_ABORT;
2839 }
2840 }
2841
2842 static void
2843 cpuid_apply_tsx(x86_taa_mitigation_t taa)
2844 {
2845 uint64_t val;
2846
2847 switch (taa) {
2848 case X86_TAA_TSX_DISABLE:
2849 val = rdmsr(MSR_IA32_TSX_CTRL);
2850 val |= IA32_TSX_CTRL_CPUID_CLEAR | IA32_TSX_CTRL_RTM_DISABLE;
2851 wrmsr(MSR_IA32_TSX_CTRL, val);
2852 break;
2853 case X86_TAA_TSX_FORCE_ABORT:
2854 val = rdmsr(MSR_IA32_TSX_CTRL);
2855 val |= IA32_TSX_CTRL_RTM_DISABLE;
2856 wrmsr(MSR_IA32_TSX_CTRL, val);
2857 break;
2858 case X86_TAA_HW_MITIGATED:
2859 case X86_TAA_MD_CLEAR:
2860 case X86_TAA_DISABLED:
2861 case X86_TAA_NOTHING:
2862 break;
2863 }
2864 }
2865
2866 static void
2867 cpuid_scan_security(cpu_t *cpu, uchar_t *featureset)
2868 {
2869 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
2870 x86_spectrev2_mitigation_t v2mit;
2871
2872 if (cpi->cpi_vendor == X86_VENDOR_AMD &&
2873 cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_8) {
2959 if (reg & IA32_ARCH_CAP_TAA_NO) {
2960 add_x86_feature(featureset,
2961 X86FSET_TAA_NO);
2962 }
2963 }
2964 no_trap();
2965 }
2966 #endif /* !__xpv */
2967
2968 if (ecp->cp_edx & CPUID_INTC_EDX_7_0_SSBD)
2969 add_x86_feature(featureset, X86FSET_SSBD);
2970
2971 if (ecp->cp_edx & CPUID_INTC_EDX_7_0_FLUSH_CMD)
2972 add_x86_feature(featureset, X86FSET_FLUSH_CMD);
2973 }
2974
2975 /*
2976 * Take care of certain mitigations on the non-boot CPU. The boot CPU
2977 * will have already run this function and determined what we need to
2978 * do. This gives us a hook for per-HW thread mitigations such as
2979 * enhanced IBRS, or disabling TSX. For TSX disabling, we need to be
2980 * careful that we've had a chance to load ucode that enables the new
2981 * MSRs.
2982 */
2983 if (cpu->cpu_id != 0) {
2984 if (x86_spectrev2_mitigation == X86_SPECTREV2_ENHANCED_IBRS) {
2985 cpuid_enable_enhanced_ibrs();
2986 }
2987
2988 if (cpi->cpi_pass >= 1)
2989 cpuid_apply_tsx(x86_taa_mitigation);
2990 return;
2991 }
2992
2993 /*
2994 * Go through and initialize various security mechanisms that we should
2995 * only do on a single CPU. This includes Spectre V2, L1TF, MDS, and
2996 * TAA.
2997 */
2998
2999 /*
3000 * By default we've come in with retpolines enabled. Check whether we
3001 * should disable them or enable enhanced IBRS. RSB stuffing is enabled
3002 * by default, but disabled if we are using enhanced IBRS.
3003 */
3004 if (x86_disable_spectrev2 != 0) {
3005 v2mit = X86_SPECTREV2_DISABLED;
3006 } else if (is_x86_feature(featureset, X86FSET_IBRS_ALL)) {
3007 cpuid_enable_enhanced_ibrs();
3008 v2mit = X86_SPECTREV2_ENHANCED_IBRS;
3009 #ifndef __xpv
3032 * md_clear does.
3033 */
3034
3035 /*
3036 * Update whether or not we need to be taking explicit action against
3037 * MDS.
3038 */
3039 cpuid_update_md_clear(cpu, featureset);
3040
3041 /*
3042 * Determine whether SMT exclusion is required and whether or not we
3043 * need to perform an l1d flush.
3044 */
3045 cpuid_update_l1d_flush(cpu, featureset);
3046
3047 /*
3048 * Determine what our mitigation strategy should be for TAA and then
3049 * also apply TAA mitigations.
3050 */
3051 cpuid_update_tsx(cpu, featureset);
3052 cpuid_apply_tsx(x86_taa_mitigation);
3053 }
3054
3055 /*
3056 * Setup XFeature_Enabled_Mask register. Required by xsave feature.
3057 */
3058 void
3059 setup_xfem(void)
3060 {
3061 uint64_t flags = XFEATURE_LEGACY_FP;
3062
3063 ASSERT(is_x86_feature(x86_featureset, X86FSET_XSAVE));
3064
3065 if (is_x86_feature(x86_featureset, X86FSET_SSE))
3066 flags |= XFEATURE_SSE;
3067
3068 if (is_x86_feature(x86_featureset, X86FSET_AVX))
3069 flags |= XFEATURE_AVX;
3070
3071 if (is_x86_feature(x86_featureset, X86FSET_AVX512F))
3072 flags |= XFEATURE_AVX512;
|
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2011, 2016 by Delphix. All rights reserved.
24 * Copyright 2013 Nexenta Systems, Inc. All rights reserved.
25 * Copyright 2014 Josef "Jeff" Sipek <jeffpc@josefsipek.net>
26 * Copyright 2020 Joyent, Inc.
27 */
28 /*
29 * Copyright (c) 2010, Intel Corporation.
30 * All rights reserved.
31 */
32 /*
33 * Portions Copyright 2009 Advanced Micro Devices, Inc.
34 */
35
36 /*
37 * CPU Identification logic
38 *
39 * The purpose of this file and its companion, cpuid_subr.c, is to help deal
40 * with the identification of CPUs, their features, and their topologies. More
41 * specifically, this file helps drive the following:
42 *
43 * 1. Enumeration of features of the processor which are used by the kernel to
44 * determine what features to enable or disable. These may be instruction set
45 * enhancements or features that we use.
46 *
47 * 2. Enumeration of instruction set architecture (ISA) additions that userland
48 * will be told about through the auxiliary vector.
49 *
50 * 3. Understanding the physical topology of the CPU such as the number of
51 * caches, how many cores it has, whether or not it supports symmetric
52 * multi-processing (SMT), etc.
53 *
54 * ------------------------
1208 * enumerate support for disabling TSX. In general, we'd rather use this method
1209 * when available as it doesn't require disabling hyperthreading to be
1210 * effective. Currently we basically are relying on microcode for processors
1211 * that enumerate MDS_NO.
1212 *
1213 * The microcode features are enumerated as part of the IA32_ARCH_CAPABILITIES.
1214 * When bit 7 (IA32_ARCH_CAP_TSX_CTRL) is present, then we are given two
1215 * different powers. The first allows us to cause all transactions to
1216 * immediately abort. The second gives us a means of disabling TSX completely,
1217 * which includes removing it from cpuid. If we have support for this in
1218 * microcode during the first cpuid pass, then we'll disable TSX completely such
1219 * that user land never has a chance to observe the bit. However, if we are late
1220 * loading the microcode, then we must use the functionality to cause
1221 * transactions to automatically abort. This is necessary for user land's sake.
1222 * Once a program sees a cpuid bit, it must not be taken away.
1223 *
1224 * We track whether or not we should do this based on what cpuid pass we're in.
1225 * Whenever we hit cpuid_scan_security() on the boot CPU and we're still on pass
1226 * 1 of the cpuid logic, then we can completely turn off TSX. Notably this
1227 * should happen twice. Once in the normal cpuid_pass1() code and then a second
1228 * time after we do the initial microcode update. As a result we need to be
1229 * careful in cpuid_apply_tsx() to only use the MSR if we've loaded a suitable
1230 * microcode on the current CPU (which happens prior to cpuid_pass_ucode()).
1231 *
1232 * If TAA has been fixed, then it will be enumerated in IA32_ARCH_CAPABILITIES
1233 * as TAA_NO. In such a case, we will still disable TSX: it's proven to be an
1234 * unfortunate feature in a number of ways, and taking the opportunity to
1235 * finally be able to turn it off is likely to be of benefit in the future.
1236 *
1237 * SUMMARY
1238 *
1239 * The following table attempts to summarize the mitigations for various issues
1240 * and what's done in various places:
1241 *
1242 * - Spectre v1: Not currently mitigated
1243 * - swapgs: lfences after swapgs paths
1244 * - Spectre v2: Retpolines/RSB Stuffing or EIBRS if HW support
1245 * - Meltdown: Kernel Page Table Isolation
1246 * - Spectre v3a: Updated CPU microcode
1247 * - Spectre v4: Not currently mitigated
1248 * - SpectreRSB: SMEP and RSB Stuffing
1249 * - L1TF: spec_uarch_flush, SMT exclusion, requires microcode
1250 * - MDS: x86_md_clear, requires microcode, disabling SMT
2822 } else {
2823 x86_taa_mitigation = X86_TAA_NOTHING;
2824 }
2825 return;
2826 }
2827
2828 /*
2829 * We have TSX_CTRL, but we can only fully disable TSX if we're early
2830 * enough in boot.
2831 *
2832 * Otherwise, we'll fall back to causing transactions to abort as our
2833 * mitigation. TSX-using code will always take the fallback path.
2834 */
2835 if (cpi->cpi_pass < 4) {
2836 x86_taa_mitigation = X86_TAA_TSX_DISABLE;
2837 } else {
2838 x86_taa_mitigation = X86_TAA_TSX_FORCE_ABORT;
2839 }
2840 }
2841
2842 /*
2843 * As mentioned, we should only touch the MSR when we've got a suitable
2844 * microcode loaded on this CPU.
2845 */
2846 static void
2847 cpuid_apply_tsx(x86_taa_mitigation_t taa, uchar_t *featureset)
2848 {
2849 uint64_t val;
2850
2851 switch (taa) {
2852 case X86_TAA_TSX_DISABLE:
2853 if (!is_x86_feature(featureset, X86FSET_TSX_CTRL))
2854 return;
2855 val = rdmsr(MSR_IA32_TSX_CTRL);
2856 val |= IA32_TSX_CTRL_CPUID_CLEAR | IA32_TSX_CTRL_RTM_DISABLE;
2857 wrmsr(MSR_IA32_TSX_CTRL, val);
2858 break;
2859 case X86_TAA_TSX_FORCE_ABORT:
2860 if (!is_x86_feature(featureset, X86FSET_TSX_CTRL))
2861 return;
2862 val = rdmsr(MSR_IA32_TSX_CTRL);
2863 val |= IA32_TSX_CTRL_RTM_DISABLE;
2864 wrmsr(MSR_IA32_TSX_CTRL, val);
2865 break;
2866 case X86_TAA_HW_MITIGATED:
2867 case X86_TAA_MD_CLEAR:
2868 case X86_TAA_DISABLED:
2869 case X86_TAA_NOTHING:
2870 break;
2871 }
2872 }
2873
2874 static void
2875 cpuid_scan_security(cpu_t *cpu, uchar_t *featureset)
2876 {
2877 struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
2878 x86_spectrev2_mitigation_t v2mit;
2879
2880 if (cpi->cpi_vendor == X86_VENDOR_AMD &&
2881 cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_8) {
2967 if (reg & IA32_ARCH_CAP_TAA_NO) {
2968 add_x86_feature(featureset,
2969 X86FSET_TAA_NO);
2970 }
2971 }
2972 no_trap();
2973 }
2974 #endif /* !__xpv */
2975
2976 if (ecp->cp_edx & CPUID_INTC_EDX_7_0_SSBD)
2977 add_x86_feature(featureset, X86FSET_SSBD);
2978
2979 if (ecp->cp_edx & CPUID_INTC_EDX_7_0_FLUSH_CMD)
2980 add_x86_feature(featureset, X86FSET_FLUSH_CMD);
2981 }
2982
2983 /*
2984 * Take care of certain mitigations on the non-boot CPU. The boot CPU
2985 * will have already run this function and determined what we need to
2986 * do. This gives us a hook for per-HW thread mitigations such as
2987 * enhanced IBRS, or disabling TSX.
2988 */
2989 if (cpu->cpu_id != 0) {
2990 if (x86_spectrev2_mitigation == X86_SPECTREV2_ENHANCED_IBRS) {
2991 cpuid_enable_enhanced_ibrs();
2992 }
2993
2994 cpuid_apply_tsx(x86_taa_mitigation, featureset);
2995 return;
2996 }
2997
2998 /*
2999 * Go through and initialize various security mechanisms that we should
3000 * only do on a single CPU. This includes Spectre V2, L1TF, MDS, and
3001 * TAA.
3002 */
3003
3004 /*
3005 * By default we've come in with retpolines enabled. Check whether we
3006 * should disable them or enable enhanced IBRS. RSB stuffing is enabled
3007 * by default, but disabled if we are using enhanced IBRS.
3008 */
3009 if (x86_disable_spectrev2 != 0) {
3010 v2mit = X86_SPECTREV2_DISABLED;
3011 } else if (is_x86_feature(featureset, X86FSET_IBRS_ALL)) {
3012 cpuid_enable_enhanced_ibrs();
3013 v2mit = X86_SPECTREV2_ENHANCED_IBRS;
3014 #ifndef __xpv
3037 * md_clear does.
3038 */
3039
3040 /*
3041 * Update whether or not we need to be taking explicit action against
3042 * MDS.
3043 */
3044 cpuid_update_md_clear(cpu, featureset);
3045
3046 /*
3047 * Determine whether SMT exclusion is required and whether or not we
3048 * need to perform an l1d flush.
3049 */
3050 cpuid_update_l1d_flush(cpu, featureset);
3051
3052 /*
3053 * Determine what our mitigation strategy should be for TAA and then
3054 * also apply TAA mitigations.
3055 */
3056 cpuid_update_tsx(cpu, featureset);
3057 cpuid_apply_tsx(x86_taa_mitigation, featureset);
3058 }
3059
3060 /*
3061 * Setup XFeature_Enabled_Mask register. Required by xsave feature.
3062 */
3063 void
3064 setup_xfem(void)
3065 {
3066 uint64_t flags = XFEATURE_LEGACY_FP;
3067
3068 ASSERT(is_x86_feature(x86_featureset, X86FSET_XSAVE));
3069
3070 if (is_x86_feature(x86_featureset, X86FSET_SSE))
3071 flags |= XFEATURE_SSE;
3072
3073 if (is_x86_feature(x86_featureset, X86FSET_AVX))
3074 flags |= XFEATURE_AVX;
3075
3076 if (is_x86_feature(x86_featureset, X86FSET_AVX512F))
3077 flags |= XFEATURE_AVX512;
|