Print this page
update


   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  23  * Copyright (c) 2011, 2016 by Delphix. All rights reserved.
  24  * Copyright 2013 Nexenta Systems, Inc. All rights reserved.
  25  * Copyright 2014 Josef "Jeff" Sipek <jeffpc@josefsipek.net>

  26  */
  27 /*
  28  * Copyright (c) 2010, Intel Corporation.
  29  * All rights reserved.
  30  */
  31 /*
  32  * Portions Copyright 2009 Advanced Micro Devices, Inc.
  33  */
  34 /*
  35  * Copyright 2020 Joyent, Inc.
  36  */
  37 
  38 /*
  39  * CPU Identification logic
  40  *
  41  * The purpose of this file and its companion, cpuid_subr.c, is to help deal
  42  * with the identification of CPUs, their features, and their topologies. More
  43  * specifically, this file helps drive the following:
  44  *
  45  * 1. Enumeration of features of the processor which are used by the kernel to
  46  *    determine what features to enable or disable. These may be instruction set
  47  *    enhancements or features that we use.
  48  *
  49  * 2. Enumeration of instruction set architecture (ISA) additions that userland
  50  *    will be told about through the auxiliary vector.
  51  *
  52  * 3. Understanding the physical topology of the CPU such as the number of
  53  *    caches, how many cores it has, whether or not it supports symmetric
  54  *    multi-processing (SMT), etc.
  55  *
  56  * ------------------------


1210  * enumerate support for disabling TSX. In general, we'd rather use this method
1211  * when available as it doesn't require disabling hyperthreading to be
1212  * effective. Currently we basically are relying on microcode for processors
1213  * that enumerate MDS_NO.
1214  *
1215  * The microcode features are enumerated as part of the IA32_ARCH_CAPABILITIES.
1216  * When bit 7 (IA32_ARCH_CAP_TSX_CTRL) is present, then we are given two
1217  * different powers. The first allows us to cause all transactions to
1218  * immediately abort. The second gives us a means of disabling TSX completely,
1219  * which includes removing it from cpuid. If we have support for this in
1220  * microcode during the first cpuid pass, then we'll disable TSX completely such
1221  * that user land never has a chance to observe the bit. However, if we are late
1222  * loading the microcode, then we must use the functionality to cause
1223  * transactions to automatically abort. This is necessary for user land's sake.
1224  * Once a program sees a cpuid bit, it must not be taken away.
1225  *
1226  * We track whether or not we should do this based on what cpuid pass we're in.
1227  * Whenever we hit cpuid_scan_security() on the boot CPU and we're still on pass
1228  * 1 of the cpuid logic, then we can completely turn off TSX. Notably this
1229  * should happen twice. Once in the normal cpuid_pass1() code and then a second
1230  * time after we do the initial microcode update.


1231  *
1232  * If TAA has been fixed, then it will be enumerated in IA32_ARCH_CAPABILITIES
1233  * as TAA_NO. In such a case, we will still disable TSX: it's proven to be an
1234  * unfortunate feature in a number of ways, and taking the opportunity to
1235  * finally be able to turn it off is likely to be of benefit in the future.
1236  *
1237  * SUMMARY
1238  *
1239  * The following table attempts to summarize the mitigations for various issues
1240  * and what's done in various places:
1241  *
1242  *  - Spectre v1: Not currently mitigated
1243  *  - swapgs: lfences after swapgs paths
1244  *  - Spectre v2: Retpolines/RSB Stuffing or EIBRS if HW support
1245  *  - Meltdown: Kernel Page Table Isolation
1246  *  - Spectre v3a: Updated CPU microcode
1247  *  - Spectre v4: Not currently mitigated
1248  *  - SpectreRSB: SMEP and RSB Stuffing
1249  *  - L1TF: spec_uarch_flush, SMT exclusion, requires microcode
1250  *  - MDS: x86_md_clear, requires microcode, disabling SMT


2822                 } else {
2823                         x86_taa_mitigation = X86_TAA_NOTHING;
2824                 }
2825                 return;
2826         }
2827 
2828         /*
2829          * We have TSX_CTRL, but we can only fully disable TSX if we're early
2830          * enough in boot.
2831          *
2832          * Otherwise, we'll fall back to causing transactions to abort as our
2833          * mitigation. TSX-using code will always take the fallback path.
2834          */
2835         if (cpi->cpi_pass < 4) {
2836                 x86_taa_mitigation = X86_TAA_TSX_DISABLE;
2837         } else {
2838                 x86_taa_mitigation = X86_TAA_TSX_FORCE_ABORT;
2839         }
2840 }
2841 




2842 static void
2843 cpuid_apply_tsx(x86_taa_mitigation_t taa)
2844 {
2845         uint64_t val;
2846 
2847         switch (taa) {
2848         case X86_TAA_TSX_DISABLE:


2849                 val = rdmsr(MSR_IA32_TSX_CTRL);
2850                 val |= IA32_TSX_CTRL_CPUID_CLEAR | IA32_TSX_CTRL_RTM_DISABLE;
2851                 wrmsr(MSR_IA32_TSX_CTRL, val);
2852                 break;
2853         case X86_TAA_TSX_FORCE_ABORT:


2854                 val = rdmsr(MSR_IA32_TSX_CTRL);
2855                 val |= IA32_TSX_CTRL_RTM_DISABLE;
2856                 wrmsr(MSR_IA32_TSX_CTRL, val);
2857                 break;
2858         case X86_TAA_HW_MITIGATED:
2859         case X86_TAA_MD_CLEAR:
2860         case X86_TAA_DISABLED:
2861         case X86_TAA_NOTHING:
2862                 break;
2863         }
2864 }
2865 
2866 static void
2867 cpuid_scan_security(cpu_t *cpu, uchar_t *featureset)
2868 {
2869         struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
2870         x86_spectrev2_mitigation_t v2mit;
2871 
2872         if (cpi->cpi_vendor == X86_VENDOR_AMD &&
2873             cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_8) {


2959                                 if (reg & IA32_ARCH_CAP_TAA_NO) {
2960                                         add_x86_feature(featureset,
2961                                             X86FSET_TAA_NO);
2962                                 }
2963                         }
2964                         no_trap();
2965                 }
2966 #endif  /* !__xpv */
2967 
2968                 if (ecp->cp_edx & CPUID_INTC_EDX_7_0_SSBD)
2969                         add_x86_feature(featureset, X86FSET_SSBD);
2970 
2971                 if (ecp->cp_edx & CPUID_INTC_EDX_7_0_FLUSH_CMD)
2972                         add_x86_feature(featureset, X86FSET_FLUSH_CMD);
2973         }
2974 
2975         /*
2976          * Take care of certain mitigations on the non-boot CPU. The boot CPU
2977          * will have already run this function and determined what we need to
2978          * do. This gives us a hook for per-HW thread mitigations such as
2979          * enhanced IBRS, or disabling TSX.  For TSX disabling, we need to be
2980          * careful that we've had a chance to load ucode that enables the new
2981          * MSRs.
2982          */
2983         if (cpu->cpu_id != 0) {
2984                 if (x86_spectrev2_mitigation == X86_SPECTREV2_ENHANCED_IBRS) {
2985                         cpuid_enable_enhanced_ibrs();
2986                 }
2987 
2988                 if (cpi->cpi_pass >= 1)
2989                         cpuid_apply_tsx(x86_taa_mitigation);
2990                 return;
2991         }
2992 
2993         /*
2994          * Go through and initialize various security mechanisms that we should
2995          * only do on a single CPU. This includes Spectre V2, L1TF, MDS, and
2996          * TAA.
2997          */
2998 
2999         /*
3000          * By default we've come in with retpolines enabled. Check whether we
3001          * should disable them or enable enhanced IBRS. RSB stuffing is enabled
3002          * by default, but disabled if we are using enhanced IBRS.
3003          */
3004         if (x86_disable_spectrev2 != 0) {
3005                 v2mit = X86_SPECTREV2_DISABLED;
3006         } else if (is_x86_feature(featureset, X86FSET_IBRS_ALL)) {
3007                 cpuid_enable_enhanced_ibrs();
3008                 v2mit = X86_SPECTREV2_ENHANCED_IBRS;
3009 #ifndef __xpv


3032          * md_clear does.
3033          */
3034 
3035         /*
3036          * Update whether or not we need to be taking explicit action against
3037          * MDS.
3038          */
3039         cpuid_update_md_clear(cpu, featureset);
3040 
3041         /*
3042          * Determine whether SMT exclusion is required and whether or not we
3043          * need to perform an l1d flush.
3044          */
3045         cpuid_update_l1d_flush(cpu, featureset);
3046 
3047         /*
3048          * Determine what our mitigation strategy should be for TAA and then
3049          * also apply TAA mitigations.
3050          */
3051         cpuid_update_tsx(cpu, featureset);
3052         cpuid_apply_tsx(x86_taa_mitigation);
3053 }
3054 
3055 /*
3056  * Setup XFeature_Enabled_Mask register. Required by xsave feature.
3057  */
3058 void
3059 setup_xfem(void)
3060 {
3061         uint64_t flags = XFEATURE_LEGACY_FP;
3062 
3063         ASSERT(is_x86_feature(x86_featureset, X86FSET_XSAVE));
3064 
3065         if (is_x86_feature(x86_featureset, X86FSET_SSE))
3066                 flags |= XFEATURE_SSE;
3067 
3068         if (is_x86_feature(x86_featureset, X86FSET_AVX))
3069                 flags |= XFEATURE_AVX;
3070 
3071         if (is_x86_feature(x86_featureset, X86FSET_AVX512F))
3072                 flags |= XFEATURE_AVX512;




   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  23  * Copyright (c) 2011, 2016 by Delphix. All rights reserved.
  24  * Copyright 2013 Nexenta Systems, Inc. All rights reserved.
  25  * Copyright 2014 Josef "Jeff" Sipek <jeffpc@josefsipek.net>
  26  * Copyright 2020 Joyent, Inc.
  27  */
  28 /*
  29  * Copyright (c) 2010, Intel Corporation.
  30  * All rights reserved.
  31  */
  32 /*
  33  * Portions Copyright 2009 Advanced Micro Devices, Inc.
  34  */



  35 
  36 /*
  37  * CPU Identification logic
  38  *
  39  * The purpose of this file and its companion, cpuid_subr.c, is to help deal
  40  * with the identification of CPUs, their features, and their topologies. More
  41  * specifically, this file helps drive the following:
  42  *
  43  * 1. Enumeration of features of the processor which are used by the kernel to
  44  *    determine what features to enable or disable. These may be instruction set
  45  *    enhancements or features that we use.
  46  *
  47  * 2. Enumeration of instruction set architecture (ISA) additions that userland
  48  *    will be told about through the auxiliary vector.
  49  *
  50  * 3. Understanding the physical topology of the CPU such as the number of
  51  *    caches, how many cores it has, whether or not it supports symmetric
  52  *    multi-processing (SMT), etc.
  53  *
  54  * ------------------------


1208  * enumerate support for disabling TSX. In general, we'd rather use this method
1209  * when available as it doesn't require disabling hyperthreading to be
1210  * effective. Currently we basically are relying on microcode for processors
1211  * that enumerate MDS_NO.
1212  *
1213  * The microcode features are enumerated as part of the IA32_ARCH_CAPABILITIES.
1214  * When bit 7 (IA32_ARCH_CAP_TSX_CTRL) is present, then we are given two
1215  * different powers. The first allows us to cause all transactions to
1216  * immediately abort. The second gives us a means of disabling TSX completely,
1217  * which includes removing it from cpuid. If we have support for this in
1218  * microcode during the first cpuid pass, then we'll disable TSX completely such
1219  * that user land never has a chance to observe the bit. However, if we are late
1220  * loading the microcode, then we must use the functionality to cause
1221  * transactions to automatically abort. This is necessary for user land's sake.
1222  * Once a program sees a cpuid bit, it must not be taken away.
1223  *
1224  * We track whether or not we should do this based on what cpuid pass we're in.
1225  * Whenever we hit cpuid_scan_security() on the boot CPU and we're still on pass
1226  * 1 of the cpuid logic, then we can completely turn off TSX. Notably this
1227  * should happen twice. Once in the normal cpuid_pass1() code and then a second
1228  * time after we do the initial microcode update.  As a result we need to be
1229  * careful in cpuid_apply_tsx() to only use the MSR if we've loaded a suitable
1230  * microcode on the current CPU (which happens prior to cpuid_pass_ucode()).
1231  *
1232  * If TAA has been fixed, then it will be enumerated in IA32_ARCH_CAPABILITIES
1233  * as TAA_NO. In such a case, we will still disable TSX: it's proven to be an
1234  * unfortunate feature in a number of ways, and taking the opportunity to
1235  * finally be able to turn it off is likely to be of benefit in the future.
1236  *
1237  * SUMMARY
1238  *
1239  * The following table attempts to summarize the mitigations for various issues
1240  * and what's done in various places:
1241  *
1242  *  - Spectre v1: Not currently mitigated
1243  *  - swapgs: lfences after swapgs paths
1244  *  - Spectre v2: Retpolines/RSB Stuffing or EIBRS if HW support
1245  *  - Meltdown: Kernel Page Table Isolation
1246  *  - Spectre v3a: Updated CPU microcode
1247  *  - Spectre v4: Not currently mitigated
1248  *  - SpectreRSB: SMEP and RSB Stuffing
1249  *  - L1TF: spec_uarch_flush, SMT exclusion, requires microcode
1250  *  - MDS: x86_md_clear, requires microcode, disabling SMT


2822                 } else {
2823                         x86_taa_mitigation = X86_TAA_NOTHING;
2824                 }
2825                 return;
2826         }
2827 
2828         /*
2829          * We have TSX_CTRL, but we can only fully disable TSX if we're early
2830          * enough in boot.
2831          *
2832          * Otherwise, we'll fall back to causing transactions to abort as our
2833          * mitigation. TSX-using code will always take the fallback path.
2834          */
2835         if (cpi->cpi_pass < 4) {
2836                 x86_taa_mitigation = X86_TAA_TSX_DISABLE;
2837         } else {
2838                 x86_taa_mitigation = X86_TAA_TSX_FORCE_ABORT;
2839         }
2840 }
2841 
2842 /*
2843  * As mentioned, we should only touch the MSR when we've got a suitable
2844  * microcode loaded on this CPU.
2845  */
2846 static void
2847 cpuid_apply_tsx(x86_taa_mitigation_t taa, uchar_t *featureset)
2848 {
2849         uint64_t val;
2850 
2851         switch (taa) {
2852         case X86_TAA_TSX_DISABLE:
2853                 if (!is_x86_feature(featureset, X86FSET_TSX_CTRL))
2854                         return;
2855                 val = rdmsr(MSR_IA32_TSX_CTRL);
2856                 val |= IA32_TSX_CTRL_CPUID_CLEAR | IA32_TSX_CTRL_RTM_DISABLE;
2857                 wrmsr(MSR_IA32_TSX_CTRL, val);
2858                 break;
2859         case X86_TAA_TSX_FORCE_ABORT:
2860                 if (!is_x86_feature(featureset, X86FSET_TSX_CTRL))
2861                         return;
2862                 val = rdmsr(MSR_IA32_TSX_CTRL);
2863                 val |= IA32_TSX_CTRL_RTM_DISABLE;
2864                 wrmsr(MSR_IA32_TSX_CTRL, val);
2865                 break;
2866         case X86_TAA_HW_MITIGATED:
2867         case X86_TAA_MD_CLEAR:
2868         case X86_TAA_DISABLED:
2869         case X86_TAA_NOTHING:
2870                 break;
2871         }
2872 }
2873 
2874 static void
2875 cpuid_scan_security(cpu_t *cpu, uchar_t *featureset)
2876 {
2877         struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
2878         x86_spectrev2_mitigation_t v2mit;
2879 
2880         if (cpi->cpi_vendor == X86_VENDOR_AMD &&
2881             cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_8) {


2967                                 if (reg & IA32_ARCH_CAP_TAA_NO) {
2968                                         add_x86_feature(featureset,
2969                                             X86FSET_TAA_NO);
2970                                 }
2971                         }
2972                         no_trap();
2973                 }
2974 #endif  /* !__xpv */
2975 
2976                 if (ecp->cp_edx & CPUID_INTC_EDX_7_0_SSBD)
2977                         add_x86_feature(featureset, X86FSET_SSBD);
2978 
2979                 if (ecp->cp_edx & CPUID_INTC_EDX_7_0_FLUSH_CMD)
2980                         add_x86_feature(featureset, X86FSET_FLUSH_CMD);
2981         }
2982 
2983         /*
2984          * Take care of certain mitigations on the non-boot CPU. The boot CPU
2985          * will have already run this function and determined what we need to
2986          * do. This gives us a hook for per-HW thread mitigations such as
2987          * enhanced IBRS, or disabling TSX.


2988          */
2989         if (cpu->cpu_id != 0) {
2990                 if (x86_spectrev2_mitigation == X86_SPECTREV2_ENHANCED_IBRS) {
2991                         cpuid_enable_enhanced_ibrs();
2992                 }
2993 
2994                 cpuid_apply_tsx(x86_taa_mitigation, featureset);

2995                 return;
2996         }
2997 
2998         /*
2999          * Go through and initialize various security mechanisms that we should
3000          * only do on a single CPU. This includes Spectre V2, L1TF, MDS, and
3001          * TAA.
3002          */
3003 
3004         /*
3005          * By default we've come in with retpolines enabled. Check whether we
3006          * should disable them or enable enhanced IBRS. RSB stuffing is enabled
3007          * by default, but disabled if we are using enhanced IBRS.
3008          */
3009         if (x86_disable_spectrev2 != 0) {
3010                 v2mit = X86_SPECTREV2_DISABLED;
3011         } else if (is_x86_feature(featureset, X86FSET_IBRS_ALL)) {
3012                 cpuid_enable_enhanced_ibrs();
3013                 v2mit = X86_SPECTREV2_ENHANCED_IBRS;
3014 #ifndef __xpv


3037          * md_clear does.
3038          */
3039 
3040         /*
3041          * Update whether or not we need to be taking explicit action against
3042          * MDS.
3043          */
3044         cpuid_update_md_clear(cpu, featureset);
3045 
3046         /*
3047          * Determine whether SMT exclusion is required and whether or not we
3048          * need to perform an l1d flush.
3049          */
3050         cpuid_update_l1d_flush(cpu, featureset);
3051 
3052         /*
3053          * Determine what our mitigation strategy should be for TAA and then
3054          * also apply TAA mitigations.
3055          */
3056         cpuid_update_tsx(cpu, featureset);
3057         cpuid_apply_tsx(x86_taa_mitigation, featureset);
3058 }
3059 
3060 /*
3061  * Setup XFeature_Enabled_Mask register. Required by xsave feature.
3062  */
3063 void
3064 setup_xfem(void)
3065 {
3066         uint64_t flags = XFEATURE_LEGACY_FP;
3067 
3068         ASSERT(is_x86_feature(x86_featureset, X86FSET_XSAVE));
3069 
3070         if (is_x86_feature(x86_featureset, X86FSET_SSE))
3071                 flags |= XFEATURE_SSE;
3072 
3073         if (is_x86_feature(x86_featureset, X86FSET_AVX))
3074                 flags |= XFEATURE_AVX;
3075 
3076         if (is_x86_feature(x86_featureset, X86FSET_AVX512F))
3077                 flags |= XFEATURE_AVX512;