1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
  24  */
  25 /*
  26  * Copyright 2018 Joyent, Inc.
  27  * Copyright (c) 2016, 2017 by Delphix. All rights reserved.
  28  */
  29 
  30 /*
  31  * PSMI 1.1 extensions are supported only in 2.6 and later versions.
  32  * PSMI 1.2 extensions are supported only in 2.7 and later versions.
  33  * PSMI 1.3 and 1.4 extensions are supported in Solaris 10.
  34  * PSMI 1.5 extensions are supported in Solaris Nevada.
  35  * PSMI 1.6 extensions are supported in Solaris Nevada.
  36  * PSMI 1.7 extensions are supported in Solaris Nevada.
  37  */
  38 #define PSMI_1_7
  39 
  40 #include <sys/processor.h>
  41 #include <sys/time.h>
  42 #include <sys/psm.h>
  43 #include <sys/smp_impldefs.h>
  44 #include <sys/cram.h>
  45 #include <sys/acpi/acpi.h>
  46 #include <sys/acpica.h>
  47 #include <sys/psm_common.h>
  48 #include <sys/apic.h>
  49 #include <sys/pit.h>
  50 #include <sys/ddi.h>
  51 #include <sys/sunddi.h>
  52 #include <sys/ddi_impldefs.h>
  53 #include <sys/pci.h>
  54 #include <sys/promif.h>
  55 #include <sys/x86_archext.h>
  56 #include <sys/cpc_impl.h>
  57 #include <sys/uadmin.h>
  58 #include <sys/panic.h>
  59 #include <sys/debug.h>
  60 #include <sys/archsystm.h>
  61 #include <sys/trap.h>
  62 #include <sys/machsystm.h>
  63 #include <sys/sysmacros.h>
  64 #include <sys/cpuvar.h>
  65 #include <sys/rm_platter.h>
  66 #include <sys/privregs.h>
  67 #include <sys/note.h>
  68 #include <sys/pci_intr_lib.h>
  69 #include <sys/spl.h>
  70 #include <sys/clock.h>
  71 #include <sys/dditypes.h>
  72 #include <sys/sunddi.h>
  73 #include <sys/x_call.h>
  74 #include <sys/reboot.h>
  75 #include <sys/hpet.h>
  76 #include <sys/apic_common.h>
  77 #include <sys/apic_timer.h>
  78 
  79 static void     apic_record_ioapic_rdt(void *intrmap_private,
  80                     ioapic_rdt_t *irdt);
  81 static void     apic_record_msi(void *intrmap_private, msi_regs_t *mregs);
  82 
  83 /*
  84  * Common routines between pcplusmp & apix (taken from apic.c).
  85  */
  86 
  87 int     apic_clkinit(int);
  88 hrtime_t apic_gethrtime(void);
  89 void    apic_send_ipi(int, int);
  90 void    apic_set_idlecpu(processorid_t);
  91 void    apic_unset_idlecpu(processorid_t);
  92 void    apic_shutdown(int, int);
  93 void    apic_preshutdown(int, int);
  94 processorid_t   apic_get_next_processorid(processorid_t);
  95 
  96 hrtime_t apic_gettime();
  97 
  98 enum apic_ioapic_method_type apix_mul_ioapic_method = APIC_MUL_IOAPIC_PCPLUSMP;
  99 
 100 /* Now the ones for Dynamic Interrupt distribution */
 101 int     apic_enable_dynamic_migration = 0;
 102 
 103 /* maximum loop count when sending Start IPIs. */
 104 int apic_sipi_max_loop_count = 0x1000;
 105 
 106 /*
 107  * These variables are frequently accessed in apic_intr_enter(),
 108  * apic_intr_exit and apic_setspl, so group them together
 109  */
 110 volatile uint32_t *apicadr =  NULL;     /* virtual addr of local APIC   */
 111 int apic_setspl_delay = 1;              /* apic_setspl - delay enable   */
 112 int apic_clkvect;
 113 
 114 /* vector at which error interrupts come in */
 115 int apic_errvect;
 116 int apic_enable_error_intr = 1;
 117 int apic_error_display_delay = 100;
 118 
 119 /* vector at which performance counter overflow interrupts come in */
 120 int apic_cpcovf_vect;
 121 int apic_enable_cpcovf_intr = 1;
 122 
 123 /* vector at which CMCI interrupts come in */
 124 int apic_cmci_vect;
 125 extern void cmi_cmci_trap(void);
 126 
 127 lock_t apic_mode_switch_lock;
 128 
 129 int apic_pir_vect;
 130 
 131 /*
 132  * Patchable global variables.
 133  */
 134 int     apic_forceload = 0;
 135 
 136 int     apic_coarse_hrtime = 1;         /* 0 - use accurate slow gethrtime() */
 137 
 138 int     apic_flat_model = 0;            /* 0 - clustered. 1 - flat */
 139 int     apic_panic_on_nmi = 0;
 140 int     apic_panic_on_apic_error = 0;
 141 
 142 int     apic_verbose = 0;       /* 0x1ff */
 143 
 144 #ifdef DEBUG
 145 int     apic_debug = 0;
 146 int     apic_restrict_vector = 0;
 147 
 148 int     apic_debug_msgbuf[APIC_DEBUG_MSGBUFSIZE];
 149 int     apic_debug_msgbufindex = 0;
 150 
 151 #endif /* DEBUG */
 152 
 153 uint_t apic_nticks = 0;
 154 uint_t apic_skipped_redistribute = 0;
 155 
 156 uint_t last_count_read = 0;
 157 lock_t  apic_gethrtime_lock;
 158 volatile int    apic_hrtime_stamp = 0;
 159 volatile hrtime_t apic_nsec_since_boot = 0;
 160 
 161 static  hrtime_t        apic_last_hrtime = 0;
 162 int             apic_hrtime_error = 0;
 163 int             apic_remote_hrterr = 0;
 164 int             apic_num_nmis = 0;
 165 int             apic_apic_error = 0;
 166 int             apic_num_apic_errors = 0;
 167 int             apic_num_cksum_errors = 0;
 168 
 169 int     apic_error = 0;
 170 
 171 static  int     apic_cmos_ssb_set = 0;
 172 
 173 /* use to make sure only one cpu handles the nmi */
 174 lock_t  apic_nmi_lock;
 175 /* use to make sure only one cpu handles the error interrupt */
 176 lock_t  apic_error_lock;
 177 
 178 static  struct {
 179         uchar_t cntl;
 180         uchar_t data;
 181 } aspen_bmc[] = {
 182         { CC_SMS_WR_START,      0x18 },         /* NetFn/LUN */
 183         { CC_SMS_WR_NEXT,       0x24 },         /* Cmd SET_WATCHDOG_TIMER */
 184         { CC_SMS_WR_NEXT,       0x84 },         /* DataByte 1: SMS/OS no log */
 185         { CC_SMS_WR_NEXT,       0x2 },          /* DataByte 2: Power Down */
 186         { CC_SMS_WR_NEXT,       0x0 },          /* DataByte 3: no pre-timeout */
 187         { CC_SMS_WR_NEXT,       0x0 },          /* DataByte 4: timer expir. */
 188         { CC_SMS_WR_NEXT,       0xa },          /* DataByte 5: init countdown */
 189         { CC_SMS_WR_END,        0x0 },          /* DataByte 6: init countdown */
 190 
 191         { CC_SMS_WR_START,      0x18 },         /* NetFn/LUN */
 192         { CC_SMS_WR_END,        0x22 }          /* Cmd RESET_WATCHDOG_TIMER */
 193 };
 194 
 195 static  struct {
 196         int     port;
 197         uchar_t data;
 198 } sitka_bmc[] = {
 199         { SMS_COMMAND_REGISTER, SMS_WRITE_START },
 200         { SMS_DATA_REGISTER,    0x18 },         /* NetFn/LUN */
 201         { SMS_DATA_REGISTER,    0x24 },         /* Cmd SET_WATCHDOG_TIMER */
 202         { SMS_DATA_REGISTER,    0x84 },         /* DataByte 1: SMS/OS no log */
 203         { SMS_DATA_REGISTER,    0x2 },          /* DataByte 2: Power Down */
 204         { SMS_DATA_REGISTER,    0x0 },          /* DataByte 3: no pre-timeout */
 205         { SMS_DATA_REGISTER,    0x0 },          /* DataByte 4: timer expir. */
 206         { SMS_DATA_REGISTER,    0xa },          /* DataByte 5: init countdown */
 207         { SMS_COMMAND_REGISTER, SMS_WRITE_END },
 208         { SMS_DATA_REGISTER,    0x0 },          /* DataByte 6: init countdown */
 209 
 210         { SMS_COMMAND_REGISTER, SMS_WRITE_START },
 211         { SMS_DATA_REGISTER,    0x18 },         /* NetFn/LUN */
 212         { SMS_COMMAND_REGISTER, SMS_WRITE_END },
 213         { SMS_DATA_REGISTER,    0x22 }          /* Cmd RESET_WATCHDOG_TIMER */
 214 };
 215 
 216 /* Patchable global variables. */
 217 int             apic_kmdb_on_nmi = 0;           /* 0 - no, 1 - yes enter kmdb */
 218 uint32_t        apic_divide_reg_init = 0;       /* 0 - divide by 2 */
 219 
 220 /* default apic ops without interrupt remapping */
 221 static apic_intrmap_ops_t apic_nointrmap_ops = {
 222         (int (*)(int))return_instr,
 223         (void (*)(int))return_instr,
 224         (void (*)(void **, dev_info_t *, uint16_t, int, uchar_t))return_instr,
 225         (void (*)(void *, void *, uint16_t, int))return_instr,
 226         (void (*)(void **))return_instr,
 227         apic_record_ioapic_rdt,
 228         apic_record_msi,
 229 };
 230 
 231 apic_intrmap_ops_t *apic_vt_ops = &apic_nointrmap_ops;
 232 apic_cpus_info_t        *apic_cpus = NULL;
 233 cpuset_t        apic_cpumask;
 234 uint_t          apic_picinit_called;
 235 
 236 /* Flag to indicate that we need to shut down all processors */
 237 static uint_t   apic_shutdown_processors;
 238 
 239 /*
 240  * Probe the ioapic method for apix module. Called in apic_probe_common()
 241  */
 242 int
 243 apic_ioapic_method_probe()
 244 {
 245         if (apix_enable == 0)
 246                 return (PSM_SUCCESS);
 247 
 248         /*
 249          * Set IOAPIC EOI handling method. The priority from low to high is:
 250          *      1. IOxAPIC: with EOI register
 251          *      2. IOMMU interrupt mapping
 252          *      3. Mask-Before-EOI method for systems without boot
 253          *      interrupt routing, such as systems with only one IOAPIC;
 254          *      NVIDIA CK8-04/MCP55 systems; systems with bridge solution
 255          *      which disables the boot interrupt routing already.
 256          *      4. Directed EOI
 257          */
 258         if (apic_io_ver[0] >= 0x20)
 259                 apix_mul_ioapic_method = APIC_MUL_IOAPIC_IOXAPIC;
 260         if ((apic_io_max == 1) || (apic_nvidia_io_max == apic_io_max))
 261                 apix_mul_ioapic_method = APIC_MUL_IOAPIC_MASK;
 262         if (apic_directed_EOI_supported())
 263                 apix_mul_ioapic_method = APIC_MUL_IOAPIC_DEOI;
 264 
 265         /* fall back to pcplusmp */
 266         if (apix_mul_ioapic_method == APIC_MUL_IOAPIC_PCPLUSMP) {
 267                 /* make sure apix is after pcplusmp in /etc/mach */
 268                 apix_enable = 0; /* go ahead with pcplusmp install next */
 269                 return (PSM_FAILURE);
 270         }
 271 
 272         return (PSM_SUCCESS);
 273 }
 274 
 275 /*
 276  * handler for APIC Error interrupt. Just print a warning and continue
 277  */
 278 int
 279 apic_error_intr()
 280 {
 281         uint_t  error0, error1, error;
 282         uint_t  i;
 283 
 284         /*
 285          * We need to write before read as per 7.4.17 of system prog manual.
 286          * We do both and or the results to be safe
 287          */
 288         error0 = apic_reg_ops->apic_read(APIC_ERROR_STATUS);
 289         apic_reg_ops->apic_write(APIC_ERROR_STATUS, 0);
 290         error1 = apic_reg_ops->apic_read(APIC_ERROR_STATUS);
 291         error = error0 | error1;
 292 
 293         /*
 294          * Clear the APIC error status (do this on all cpus that enter here)
 295          * (two writes are required due to the semantics of accessing the
 296          * error status register.)
 297          */
 298         apic_reg_ops->apic_write(APIC_ERROR_STATUS, 0);
 299         apic_reg_ops->apic_write(APIC_ERROR_STATUS, 0);
 300 
 301         /*
 302          * Prevent more than 1 CPU from handling error interrupt causing
 303          * double printing (interleave of characters from multiple
 304          * CPU's when using prom_printf)
 305          */
 306         if (lock_try(&apic_error_lock) == 0)
 307                 return (error ? DDI_INTR_CLAIMED : DDI_INTR_UNCLAIMED);
 308         if (error) {
 309 #if     DEBUG
 310                 if (apic_debug)
 311                         debug_enter("pcplusmp: APIC Error interrupt received");
 312 #endif /* DEBUG */
 313                 if (apic_panic_on_apic_error)
 314                         cmn_err(CE_PANIC,
 315                             "APIC Error interrupt on CPU %d. Status = %x",
 316                             psm_get_cpu_id(), error);
 317                 else {
 318                         if ((error & ~APIC_CS_ERRORS) == 0) {
 319                                 /* cksum error only */
 320                                 apic_error |= APIC_ERR_APIC_ERROR;
 321                                 apic_apic_error |= error;
 322                                 apic_num_apic_errors++;
 323                                 apic_num_cksum_errors++;
 324                         } else {
 325                                 /*
 326                                  * prom_printf is the best shot we have of
 327                                  * something which is problem free from
 328                                  * high level/NMI type of interrupts
 329                                  */
 330                                 prom_printf("APIC Error interrupt on CPU %d. "
 331                                     "Status 0 = %x, Status 1 = %x\n",
 332                                     psm_get_cpu_id(), error0, error1);
 333                                 apic_error |= APIC_ERR_APIC_ERROR;
 334                                 apic_apic_error |= error;
 335                                 apic_num_apic_errors++;
 336                                 for (i = 0; i < apic_error_display_delay; i++) {
 337                                         tenmicrosec();
 338                                 }
 339                                 /*
 340                                  * provide more delay next time limited to
 341                                  * roughly 1 clock tick time
 342                                  */
 343                                 if (apic_error_display_delay < 500)
 344                                         apic_error_display_delay *= 2;
 345                         }
 346                 }
 347                 lock_clear(&apic_error_lock);
 348                 return (DDI_INTR_CLAIMED);
 349         } else {
 350                 lock_clear(&apic_error_lock);
 351                 return (DDI_INTR_UNCLAIMED);
 352         }
 353 }
 354 
 355 /*
 356  * Turn off the mask bit in the performance counter Local Vector Table entry.
 357  */
 358 void
 359 apic_cpcovf_mask_clear(void)
 360 {
 361         apic_reg_ops->apic_write(APIC_PCINT_VECT,
 362             (apic_reg_ops->apic_read(APIC_PCINT_VECT) & ~APIC_LVT_MASK));
 363 }
 364 
 365 /*ARGSUSED*/
 366 static int
 367 apic_cmci_enable(xc_arg_t arg1, xc_arg_t arg2, xc_arg_t arg3)
 368 {
 369         apic_reg_ops->apic_write(APIC_CMCI_VECT, apic_cmci_vect);
 370         return (0);
 371 }
 372 
 373 /*ARGSUSED*/
 374 static int
 375 apic_cmci_disable(xc_arg_t arg1, xc_arg_t arg2, xc_arg_t arg3)
 376 {
 377         apic_reg_ops->apic_write(APIC_CMCI_VECT, apic_cmci_vect | AV_MASK);
 378         return (0);
 379 }
 380 
 381 void
 382 apic_cmci_setup(processorid_t cpuid, boolean_t enable)
 383 {
 384         cpuset_t        cpu_set;
 385 
 386         CPUSET_ONLY(cpu_set, cpuid);
 387 
 388         if (enable) {
 389                 xc_call(NULL, NULL, NULL, CPUSET2BV(cpu_set),
 390                     (xc_func_t)apic_cmci_enable);
 391         } else {
 392                 xc_call(NULL, NULL, NULL, CPUSET2BV(cpu_set),
 393                     (xc_func_t)apic_cmci_disable);
 394         }
 395 }
 396 
 397 static void
 398 apic_disable_local_apic(void)
 399 {
 400         apic_reg_ops->apic_write_task_reg(APIC_MASK_ALL);
 401         apic_reg_ops->apic_write(APIC_LOCAL_TIMER, AV_MASK);
 402 
 403         /* local intr reg 0 */
 404         apic_reg_ops->apic_write(APIC_INT_VECT0, AV_MASK);
 405 
 406         /* disable NMI */
 407         apic_reg_ops->apic_write(APIC_INT_VECT1, AV_MASK);
 408 
 409         /* and error interrupt */
 410         apic_reg_ops->apic_write(APIC_ERR_VECT, AV_MASK);
 411 
 412         /* and perf counter intr */
 413         apic_reg_ops->apic_write(APIC_PCINT_VECT, AV_MASK);
 414 
 415         apic_reg_ops->apic_write(APIC_SPUR_INT_REG, APIC_SPUR_INTR);
 416 }
 417 
 418 static void
 419 apic_cpu_send_SIPI(processorid_t cpun, boolean_t start)
 420 {
 421         int             loop_count;
 422         uint32_t        vector;
 423         uint_t          apicid;
 424         ulong_t         iflag;
 425 
 426         apicid =  apic_cpus[cpun].aci_local_id;
 427 
 428         /*
 429          * Interrupts on current CPU will be disabled during the
 430          * steps in order to avoid unwanted side effects from
 431          * executing interrupt handlers on a problematic BIOS.
 432          */
 433         iflag = intr_clear();
 434 
 435         if (start) {
 436                 outb(CMOS_ADDR, SSB);
 437                 outb(CMOS_DATA, BIOS_SHUTDOWN);
 438         }
 439 
 440         /*
 441          * According to X2APIC specification in section '2.3.5.1' of
 442          * Interrupt Command Register Semantics, the semantics of
 443          * programming the Interrupt Command Register to dispatch an interrupt
 444          * is simplified. A single MSR write to the 64-bit ICR is required
 445          * for dispatching an interrupt. Specifically, with the 64-bit MSR
 446          * interface to ICR, system software is not required to check the
 447          * status of the delivery status bit prior to writing to the ICR
 448          * to send an IPI. With the removal of the Delivery Status bit,
 449          * system software no longer has a reason to read the ICR. It remains
 450          * readable only to aid in debugging.
 451          */
 452 #ifdef  DEBUG
 453         APIC_AV_PENDING_SET();
 454 #else
 455         if (apic_mode == LOCAL_APIC) {
 456                 APIC_AV_PENDING_SET();
 457         }
 458 #endif /* DEBUG */
 459 
 460         /* for integrated - make sure there is one INIT IPI in buffer */
 461         /* for external - it will wake up the cpu */
 462         apic_reg_ops->apic_write_int_cmd(apicid, AV_ASSERT | AV_RESET);
 463 
 464         /* If only 1 CPU is installed, PENDING bit will not go low */
 465         for (loop_count = apic_sipi_max_loop_count; loop_count; loop_count--) {
 466                 if (apic_mode == LOCAL_APIC &&
 467                     apic_reg_ops->apic_read(APIC_INT_CMD1) & AV_PENDING)
 468                         apic_ret();
 469                 else
 470                         break;
 471         }
 472 
 473         apic_reg_ops->apic_write_int_cmd(apicid, AV_DEASSERT | AV_RESET);
 474         drv_usecwait(20000);            /* 20 milli sec */
 475 
 476         if (apic_cpus[cpun].aci_local_ver >= APIC_INTEGRATED_VERS) {
 477                 /* integrated apic */
 478 
 479                 vector = (rm_platter_pa >> MMU_PAGESHIFT) &
 480                     (APIC_VECTOR_MASK | APIC_IPL_MASK);
 481 
 482                 /* to offset the INIT IPI queue up in the buffer */
 483                 apic_reg_ops->apic_write_int_cmd(apicid, vector | AV_STARTUP);
 484                 drv_usecwait(200);              /* 20 micro sec */
 485 
 486                 /*
 487                  * send the second SIPI (Startup IPI) as recommended by Intel
 488                  * software development manual.
 489                  */
 490                 apic_reg_ops->apic_write_int_cmd(apicid, vector | AV_STARTUP);
 491                 drv_usecwait(200);      /* 20 micro sec */
 492         }
 493 
 494         intr_restore(iflag);
 495 }
 496 
 497 /*ARGSUSED1*/
 498 int
 499 apic_cpu_start(processorid_t cpun, caddr_t arg)
 500 {
 501         ASSERT(MUTEX_HELD(&cpu_lock));
 502 
 503         if (!apic_cpu_in_range(cpun)) {
 504                 return (EINVAL);
 505         }
 506 
 507         /*
 508          * Switch to apic_common_send_ipi for safety during starting other CPUs.
 509          */
 510         if (apic_mode == LOCAL_X2APIC) {
 511                 apic_switch_ipi_callback(B_TRUE);
 512         }
 513 
 514         apic_cmos_ssb_set = 1;
 515         apic_cpu_send_SIPI(cpun, B_TRUE);
 516 
 517         return (0);
 518 }
 519 
 520 /*
 521  * Put CPU into halted state with interrupts disabled.
 522  */
 523 /*ARGSUSED1*/
 524 int
 525 apic_cpu_stop(processorid_t cpun, caddr_t arg)
 526 {
 527         int             rc;
 528         cpu_t           *cp;
 529         extern cpuset_t cpu_ready_set;
 530         extern void cpu_idle_intercept_cpu(cpu_t *cp);
 531 
 532         ASSERT(MUTEX_HELD(&cpu_lock));
 533 
 534         if (!apic_cpu_in_range(cpun)) {
 535                 return (EINVAL);
 536         }
 537         if (apic_cpus[cpun].aci_local_ver < APIC_INTEGRATED_VERS) {
 538                 return (ENOTSUP);
 539         }
 540 
 541         cp = cpu_get(cpun);
 542         ASSERT(cp != NULL);
 543         ASSERT((cp->cpu_flags & CPU_OFFLINE) != 0);
 544         ASSERT((cp->cpu_flags & CPU_QUIESCED) != 0);
 545         ASSERT((cp->cpu_flags & CPU_ENABLE) == 0);
 546 
 547         /* Clear CPU_READY flag to disable cross calls. */
 548         cp->cpu_flags &= ~CPU_READY;
 549         CPUSET_ATOMIC_DEL(cpu_ready_set, cpun);
 550         rc = xc_flush_cpu(cp);
 551         if (rc != 0) {
 552                 CPUSET_ATOMIC_ADD(cpu_ready_set, cpun);
 553                 cp->cpu_flags |= CPU_READY;
 554                 return (rc);
 555         }
 556 
 557         /* Intercept target CPU at a safe point before powering it off. */
 558         cpu_idle_intercept_cpu(cp);
 559 
 560         apic_cpu_send_SIPI(cpun, B_FALSE);
 561         cp->cpu_flags &= ~CPU_RUNNING;
 562 
 563         return (0);
 564 }
 565 
 566 int
 567 apic_cpu_ops(psm_cpu_request_t *reqp)
 568 {
 569         if (reqp == NULL) {
 570                 return (EINVAL);
 571         }
 572 
 573         switch (reqp->pcr_cmd) {
 574         case PSM_CPU_ADD:
 575                 return (apic_cpu_add(reqp));
 576 
 577         case PSM_CPU_REMOVE:
 578                 return (apic_cpu_remove(reqp));
 579 
 580         case PSM_CPU_STOP:
 581                 return (apic_cpu_stop(reqp->req.cpu_stop.cpuid,
 582                     reqp->req.cpu_stop.ctx));
 583 
 584         default:
 585                 return (ENOTSUP);
 586         }
 587 }
 588 
 589 #ifdef  DEBUG
 590 int     apic_break_on_cpu = 9;
 591 int     apic_stretch_interrupts = 0;
 592 int     apic_stretch_ISR = 1 << 3;        /* IPL of 3 matches nothing now */
 593 #endif /* DEBUG */
 594 
 595 /*
 596  * generates an interprocessor interrupt to another CPU. Any changes made to
 597  * this routine must be accompanied by similar changes to
 598  * apic_common_send_ipi().
 599  */
 600 void
 601 apic_send_ipi(int cpun, int ipl)
 602 {
 603         int vector;
 604         ulong_t flag;
 605 
 606         vector = apic_resv_vector[ipl];
 607 
 608         ASSERT((vector >= APIC_BASE_VECT) && (vector <= APIC_SPUR_INTR));
 609 
 610         flag = intr_clear();
 611 
 612         APIC_AV_PENDING_SET();
 613 
 614         apic_reg_ops->apic_write_int_cmd(apic_cpus[cpun].aci_local_id,
 615             vector);
 616 
 617         intr_restore(flag);
 618 }
 619 
 620 void
 621 apic_send_pir_ipi(processorid_t cpun)
 622 {
 623         const int vector = apic_pir_vect;
 624         ulong_t flag;
 625 
 626         ASSERT((vector >= APIC_BASE_VECT) && (vector <= APIC_SPUR_INTR));
 627 
 628         flag = intr_clear();
 629 
 630         /* Self-IPI for inducing PIR makes no sense. */
 631         if ((cpun != psm_get_cpu_id())) {
 632                 APIC_AV_PENDING_SET();
 633                 apic_reg_ops->apic_write_int_cmd(apic_cpus[cpun].aci_local_id,
 634                     vector);
 635         }
 636 
 637         intr_restore(flag);
 638 }
 639 
 640 int
 641 apic_get_pir_ipivect(void)
 642 {
 643         return (apic_pir_vect);
 644 }
 645 
 646 /*ARGSUSED*/
 647 void
 648 apic_set_idlecpu(processorid_t cpun)
 649 {
 650 }
 651 
 652 /*ARGSUSED*/
 653 void
 654 apic_unset_idlecpu(processorid_t cpun)
 655 {
 656 }
 657 
 658 
 659 void
 660 apic_ret()
 661 {
 662 }
 663 
 664 /*
 665  * If apic_coarse_time == 1, then apic_gettime() is used instead of
 666  * apic_gethrtime().  This is used for performance instead of accuracy.
 667  */
 668 
 669 hrtime_t
 670 apic_gettime()
 671 {
 672         int old_hrtime_stamp;
 673         hrtime_t temp;
 674 
 675         /*
 676          * In one-shot mode, we do not keep time, so if anyone
 677          * calls psm_gettime() directly, we vector over to
 678          * gethrtime().
 679          * one-shot mode MUST NOT be enabled if this psm is the source of
 680          * hrtime.
 681          */
 682 
 683         if (apic_oneshot)
 684                 return (gethrtime());
 685 
 686 
 687 gettime_again:
 688         while ((old_hrtime_stamp = apic_hrtime_stamp) & 1)
 689                 apic_ret();
 690 
 691         temp = apic_nsec_since_boot;
 692 
 693         if (apic_hrtime_stamp != old_hrtime_stamp) {    /* got an interrupt */
 694                 goto gettime_again;
 695         }
 696         return (temp);
 697 }
 698 
 699 /*
 700  * Here we return the number of nanoseconds since booting.  Note every
 701  * clock interrupt increments apic_nsec_since_boot by the appropriate
 702  * amount.
 703  */
 704 hrtime_t
 705 apic_gethrtime(void)
 706 {
 707         int curr_timeval, countval, elapsed_ticks;
 708         int old_hrtime_stamp, status;
 709         hrtime_t temp;
 710         uint32_t cpun;
 711         ulong_t oflags;
 712 
 713         /*
 714          * In one-shot mode, we do not keep time, so if anyone
 715          * calls psm_gethrtime() directly, we vector over to
 716          * gethrtime().
 717          * one-shot mode MUST NOT be enabled if this psm is the source of
 718          * hrtime.
 719          */
 720 
 721         if (apic_oneshot)
 722                 return (gethrtime());
 723 
 724         oflags = intr_clear();  /* prevent migration */
 725 
 726         cpun = apic_reg_ops->apic_read(APIC_LID_REG);
 727         if (apic_mode == LOCAL_APIC)
 728                 cpun >>= APIC_ID_BIT_OFFSET;
 729 
 730         lock_set(&apic_gethrtime_lock);
 731 
 732 gethrtime_again:
 733         while ((old_hrtime_stamp = apic_hrtime_stamp) & 1)
 734                 apic_ret();
 735 
 736         /*
 737          * Check to see which CPU we are on.  Note the time is kept on
 738          * the local APIC of CPU 0.  If on CPU 0, simply read the current
 739          * counter.  If on another CPU, issue a remote read command to CPU 0.
 740          */
 741         if (cpun == apic_cpus[0].aci_local_id) {
 742                 countval = apic_reg_ops->apic_read(APIC_CURR_COUNT);
 743         } else {
 744 #ifdef  DEBUG
 745                 APIC_AV_PENDING_SET();
 746 #else
 747                 if (apic_mode == LOCAL_APIC)
 748                         APIC_AV_PENDING_SET();
 749 #endif /* DEBUG */
 750 
 751                 apic_reg_ops->apic_write_int_cmd(
 752                     apic_cpus[0].aci_local_id, APIC_CURR_ADD | AV_REMOTE);
 753 
 754                 while ((status = apic_reg_ops->apic_read(APIC_INT_CMD1))
 755                     & AV_READ_PENDING) {
 756                         apic_ret();
 757                 }
 758 
 759                 if (status & AV_REMOTE_STATUS)      /* 1 = valid */
 760                         countval = apic_reg_ops->apic_read(APIC_REMOTE_READ);
 761                 else {  /* 0 = invalid */
 762                         apic_remote_hrterr++;
 763                         /*
 764                          * return last hrtime right now, will need more
 765                          * testing if change to retry
 766                          */
 767                         temp = apic_last_hrtime;
 768 
 769                         lock_clear(&apic_gethrtime_lock);
 770 
 771                         intr_restore(oflags);
 772 
 773                         return (temp);
 774                 }
 775         }
 776         if (countval > last_count_read)
 777                 countval = 0;
 778         else
 779                 last_count_read = countval;
 780 
 781         elapsed_ticks = apic_hertz_count - countval;
 782 
 783         curr_timeval = APIC_TICKS_TO_NSECS(elapsed_ticks);
 784         temp = apic_nsec_since_boot + curr_timeval;
 785 
 786         if (apic_hrtime_stamp != old_hrtime_stamp) {    /* got an interrupt */
 787                 /* we might have clobbered last_count_read. Restore it */
 788                 last_count_read = apic_hertz_count;
 789                 goto gethrtime_again;
 790         }
 791 
 792         if (temp < apic_last_hrtime) {
 793                 /* return last hrtime if error occurs */
 794                 apic_hrtime_error++;
 795                 temp = apic_last_hrtime;
 796         }
 797         else
 798                 apic_last_hrtime = temp;
 799 
 800         lock_clear(&apic_gethrtime_lock);
 801         intr_restore(oflags);
 802 
 803         return (temp);
 804 }
 805 
 806 /* apic NMI handler */
 807 /*ARGSUSED*/
 808 void
 809 apic_nmi_intr(caddr_t arg, struct regs *rp)
 810 {
 811         if (apic_shutdown_processors) {
 812                 apic_disable_local_apic();
 813                 return;
 814         }
 815 
 816         apic_error |= APIC_ERR_NMI;
 817 
 818         if (!lock_try(&apic_nmi_lock))
 819                 return;
 820         apic_num_nmis++;
 821 
 822         if (apic_kmdb_on_nmi && psm_debugger()) {
 823                 debug_enter("NMI received: entering kmdb\n");
 824         } else if (apic_panic_on_nmi) {
 825                 /* Keep panic from entering kmdb. */
 826                 nopanicdebug = 1;
 827                 panic("NMI received\n");
 828         } else {
 829                 /*
 830                  * prom_printf is the best shot we have of something which is
 831                  * problem free from high level/NMI type of interrupts
 832                  */
 833                 prom_printf("NMI received\n");
 834         }
 835 
 836         lock_clear(&apic_nmi_lock);
 837 }
 838 
 839 processorid_t
 840 apic_get_next_processorid(processorid_t cpu_id)
 841 {
 842 
 843         int i;
 844 
 845         if (cpu_id == -1)
 846                 return ((processorid_t)0);
 847 
 848         for (i = cpu_id + 1; i < NCPU; i++) {
 849                 if (apic_cpu_in_range(i))
 850                         return (i);
 851         }
 852 
 853         return ((processorid_t)-1);
 854 }
 855 
 856 int
 857 apic_cpu_add(psm_cpu_request_t *reqp)
 858 {
 859         int i, rv = 0;
 860         ulong_t iflag;
 861         boolean_t first = B_TRUE;
 862         uchar_t localver = 0;
 863         uint32_t localid, procid;
 864         processorid_t cpuid = (processorid_t)-1;
 865         mach_cpu_add_arg_t *ap;
 866 
 867         ASSERT(reqp != NULL);
 868         reqp->req.cpu_add.cpuid = (processorid_t)-1;
 869 
 870         /* Check whether CPU hotplug is supported. */
 871         if (!plat_dr_support_cpu() || apic_max_nproc == -1) {
 872                 return (ENOTSUP);
 873         }
 874 
 875         ap = (mach_cpu_add_arg_t *)reqp->req.cpu_add.argp;
 876         switch (ap->type) {
 877         case MACH_CPU_ARG_LOCAL_APIC:
 878                 localid = ap->arg.apic.apic_id;
 879                 procid = ap->arg.apic.proc_id;
 880                 if (localid >= 255 || procid > 255) {
 881                         cmn_err(CE_WARN,
 882                             "!apic: apicid(%u) or procid(%u) is invalid.",
 883                             localid, procid);
 884                         return (EINVAL);
 885                 }
 886                 break;
 887 
 888         case MACH_CPU_ARG_LOCAL_X2APIC:
 889                 localid = ap->arg.apic.apic_id;
 890                 procid = ap->arg.apic.proc_id;
 891                 if (localid >= UINT32_MAX) {
 892                         cmn_err(CE_WARN,
 893                             "!apic: x2apicid(%u) is invalid.", localid);
 894                         return (EINVAL);
 895                 } else if (localid >= 255 && apic_mode == LOCAL_APIC) {
 896                         cmn_err(CE_WARN, "!apic: system is in APIC mode, "
 897                             "can't support x2APIC processor.");
 898                         return (ENOTSUP);
 899                 }
 900                 break;
 901 
 902         default:
 903                 cmn_err(CE_WARN,
 904                     "!apic: unknown argument type %d to apic_cpu_add().",
 905                     ap->type);
 906                 return (EINVAL);
 907         }
 908 
 909         /* Use apic_ioapic_lock to sync with apic_get_next_bind_cpu. */
 910         iflag = intr_clear();
 911         lock_set(&apic_ioapic_lock);
 912 
 913         /* Check whether local APIC id already exists. */
 914         for (i = 0; i < apic_nproc; i++) {
 915                 if (!CPU_IN_SET(apic_cpumask, i))
 916                         continue;
 917                 if (apic_cpus[i].aci_local_id == localid) {
 918                         lock_clear(&apic_ioapic_lock);
 919                         intr_restore(iflag);
 920                         cmn_err(CE_WARN,
 921                             "!apic: local apic id %u already exists.",
 922                             localid);
 923                         return (EEXIST);
 924                 } else if (apic_cpus[i].aci_processor_id == procid) {
 925                         lock_clear(&apic_ioapic_lock);
 926                         intr_restore(iflag);
 927                         cmn_err(CE_WARN,
 928                             "!apic: processor id %u already exists.",
 929                             (int)procid);
 930                         return (EEXIST);
 931                 }
 932 
 933                 /*
 934                  * There's no local APIC version number available in MADT table,
 935                  * so assume that all CPUs are homogeneous and use local APIC
 936                  * version number of the first existing CPU.
 937                  */
 938                 if (first) {
 939                         first = B_FALSE;
 940                         localver = apic_cpus[i].aci_local_ver;
 941                 }
 942         }
 943         ASSERT(first == B_FALSE);
 944 
 945         /*
 946          * Try to assign the same cpuid if APIC id exists in the dirty cache.
 947          */
 948         for (i = 0; i < apic_max_nproc; i++) {
 949                 if (CPU_IN_SET(apic_cpumask, i)) {
 950                         ASSERT((apic_cpus[i].aci_status & APIC_CPU_FREE) == 0);
 951                         continue;
 952                 }
 953                 ASSERT(apic_cpus[i].aci_status & APIC_CPU_FREE);
 954                 if ((apic_cpus[i].aci_status & APIC_CPU_DIRTY) &&
 955                     apic_cpus[i].aci_local_id == localid &&
 956                     apic_cpus[i].aci_processor_id == procid) {
 957                         cpuid = i;
 958                         break;
 959                 }
 960         }
 961 
 962         /* Avoid the dirty cache and allocate fresh slot if possible. */
 963         if (cpuid == (processorid_t)-1) {
 964                 for (i = 0; i < apic_max_nproc; i++) {
 965                         if ((apic_cpus[i].aci_status & APIC_CPU_FREE) &&
 966                             (apic_cpus[i].aci_status & APIC_CPU_DIRTY) == 0) {
 967                                 cpuid = i;
 968                                 break;
 969                         }
 970                 }
 971         }
 972 
 973         /* Try to find any free slot as last resort. */
 974         if (cpuid == (processorid_t)-1) {
 975                 for (i = 0; i < apic_max_nproc; i++) {
 976                         if (apic_cpus[i].aci_status & APIC_CPU_FREE) {
 977                                 cpuid = i;
 978                                 break;
 979                         }
 980                 }
 981         }
 982 
 983         if (cpuid == (processorid_t)-1) {
 984                 lock_clear(&apic_ioapic_lock);
 985                 intr_restore(iflag);
 986                 cmn_err(CE_NOTE,
 987                     "!apic: failed to allocate cpu id for processor %u.",
 988                     procid);
 989                 rv = EAGAIN;
 990         } else if (ACPI_FAILURE(acpica_map_cpu(cpuid, procid))) {
 991                 lock_clear(&apic_ioapic_lock);
 992                 intr_restore(iflag);
 993                 cmn_err(CE_NOTE,
 994                     "!apic: failed to build mapping for processor %u.",
 995                     procid);
 996                 rv = EBUSY;
 997         } else {
 998                 ASSERT(cpuid >= 0 && cpuid < NCPU);
 999                 ASSERT(cpuid < apic_max_nproc && cpuid < max_ncpus);
1000                 bzero(&apic_cpus[cpuid], sizeof (apic_cpus[0]));
1001                 apic_cpus[cpuid].aci_processor_id = procid;
1002                 apic_cpus[cpuid].aci_local_id = localid;
1003                 apic_cpus[cpuid].aci_local_ver = localver;
1004                 CPUSET_ATOMIC_ADD(apic_cpumask, cpuid);
1005                 if (cpuid >= apic_nproc) {
1006                         apic_nproc = cpuid + 1;
1007                 }
1008                 lock_clear(&apic_ioapic_lock);
1009                 intr_restore(iflag);
1010                 reqp->req.cpu_add.cpuid = cpuid;
1011         }
1012 
1013         return (rv);
1014 }
1015 
1016 int
1017 apic_cpu_remove(psm_cpu_request_t *reqp)
1018 {
1019         int i;
1020         ulong_t iflag;
1021         processorid_t cpuid;
1022 
1023         /* Check whether CPU hotplug is supported. */
1024         if (!plat_dr_support_cpu() || apic_max_nproc == -1) {
1025                 return (ENOTSUP);
1026         }
1027 
1028         cpuid = reqp->req.cpu_remove.cpuid;
1029 
1030         /* Use apic_ioapic_lock to sync with apic_get_next_bind_cpu. */
1031         iflag = intr_clear();
1032         lock_set(&apic_ioapic_lock);
1033 
1034         if (!apic_cpu_in_range(cpuid)) {
1035                 lock_clear(&apic_ioapic_lock);
1036                 intr_restore(iflag);
1037                 cmn_err(CE_WARN,
1038                     "!apic: cpuid %d doesn't exist in apic_cpus array.",
1039                     cpuid);
1040                 return (ENODEV);
1041         }
1042         ASSERT((apic_cpus[cpuid].aci_status & APIC_CPU_FREE) == 0);
1043 
1044         if (ACPI_FAILURE(acpica_unmap_cpu(cpuid))) {
1045                 lock_clear(&apic_ioapic_lock);
1046                 intr_restore(iflag);
1047                 return (ENOENT);
1048         }
1049 
1050         if (cpuid == apic_nproc - 1) {
1051                 /*
1052                  * We are removing the highest numbered cpuid so we need to
1053                  * find the next highest cpuid as the new value for apic_nproc.
1054                  */
1055                 for (i = apic_nproc; i > 0; i--) {
1056                         if (CPU_IN_SET(apic_cpumask, i - 1)) {
1057                                 apic_nproc = i;
1058                                 break;
1059                         }
1060                 }
1061                 /* at least one CPU left */
1062                 ASSERT(i > 0);
1063         }
1064         CPUSET_ATOMIC_DEL(apic_cpumask, cpuid);
1065         /* mark slot as free and keep it in the dirty cache */
1066         apic_cpus[cpuid].aci_status = APIC_CPU_FREE | APIC_CPU_DIRTY;
1067 
1068         lock_clear(&apic_ioapic_lock);
1069         intr_restore(iflag);
1070 
1071         return (0);
1072 }
1073 
1074 /*
1075  * Return the number of ticks the APIC decrements in SF nanoseconds.
1076  * The fixed-frequency PIT (aka 8254) is used for the measurement.
1077  */
1078 static uint64_t
1079 apic_calibrate_impl()
1080 {
1081         uint8_t         pit_tick_lo;
1082         uint16_t        pit_tick, target_pit_tick, pit_ticks_adj;
1083         uint32_t        pit_ticks;
1084         uint32_t        start_apic_tick, end_apic_tick, apic_ticks;
1085         ulong_t         iflag;
1086 
1087         apic_reg_ops->apic_write(APIC_DIVIDE_REG, apic_divide_reg_init);
1088         apic_reg_ops->apic_write(APIC_INIT_COUNT, APIC_MAXVAL);
1089 
1090         iflag = intr_clear();
1091 
1092         do {
1093                 pit_tick_lo = inb(PITCTR0_PORT);
1094                 pit_tick = (inb(PITCTR0_PORT) << 8) | pit_tick_lo;
1095         } while (pit_tick < APIC_TIME_MIN ||
1096             pit_tick_lo <= APIC_LB_MIN || pit_tick_lo >= APIC_LB_MAX);
1097 
1098         /*
1099          * Wait for the PIT to decrement by 5 ticks to ensure
1100          * we didn't start in the middle of a tick.
1101          * Compare with 0x10 for the wrap around case.
1102          */
1103         target_pit_tick = pit_tick - 5;
1104         do {
1105                 pit_tick_lo = inb(PITCTR0_PORT);
1106                 pit_tick = (inb(PITCTR0_PORT) << 8) | pit_tick_lo;
1107         } while (pit_tick > target_pit_tick || pit_tick_lo < 0x10);
1108 
1109         start_apic_tick = apic_reg_ops->apic_read(APIC_CURR_COUNT);
1110 
1111         /*
1112          * Wait for the PIT to decrement by APIC_TIME_COUNT ticks
1113          */
1114         target_pit_tick = pit_tick - APIC_TIME_COUNT;
1115         do {
1116                 pit_tick_lo = inb(PITCTR0_PORT);
1117                 pit_tick = (inb(PITCTR0_PORT) << 8) | pit_tick_lo;
1118         } while (pit_tick > target_pit_tick || pit_tick_lo < 0x10);
1119 
1120         end_apic_tick = apic_reg_ops->apic_read(APIC_CURR_COUNT);
1121 
1122         intr_restore(iflag);
1123 
1124         apic_ticks = start_apic_tick - end_apic_tick;
1125 
1126         /* The PIT might have decremented by more ticks than planned */
1127         pit_ticks_adj = target_pit_tick - pit_tick;
1128         /* total number of PIT ticks corresponding to apic_ticks */
1129         pit_ticks = APIC_TIME_COUNT + pit_ticks_adj;
1130 
1131         /*
1132          * Determine the number of nanoseconds per APIC clock tick
1133          * and then determine how many APIC ticks to interrupt at the
1134          * desired frequency
1135          * apic_ticks / (pitticks / PIT_HZ) = apic_ticks_per_s
1136          * (apic_ticks * PIT_HZ) / pitticks = apic_ticks_per_s
1137          * apic_ticks_per_ns = (apic_ticks * PIT_HZ) / (pitticks * 10^9)
1138          * apic_ticks_per_SFns =
1139          * (SF * apic_ticks * PIT_HZ) / (pitticks * 10^9)
1140          */
1141         return ((SF * apic_ticks * PIT_HZ) / ((uint64_t)pit_ticks * NANOSEC));
1142 }
1143 
1144 /*
1145  * It was found empirically that 5 measurements seem sufficient to give a good
1146  * accuracy. Most spurious measurements are higher than the target value thus
1147  * we eliminate up to 2/5 spurious measurements.
1148  */
1149 #define APIC_CALIBRATE_MEASUREMENTS             5
1150 
1151 #define APIC_CALIBRATE_PERCENT_OFF_WARNING      10
1152 
1153 /*
1154  * Return the number of ticks the APIC decrements in SF nanoseconds.
1155  * Several measurements are taken to filter out outliers.
1156  */
1157 uint64_t
1158 apic_calibrate()
1159 {
1160         uint64_t        measurements[APIC_CALIBRATE_MEASUREMENTS];
1161         int             median_idx;
1162         uint64_t        median;
1163 
1164         /*
1165          * When running under a virtual machine, the emulated PIT and APIC
1166          * counters do not always return the right values and can roll over.
1167          * Those spurious measurements are relatively rare but could
1168          * significantly affect the calibration.
1169          * Therefore we take several measurements and then keep the median.
1170          * The median is preferred to the average here as we only want to
1171          * discard outliers.
1172          */
1173         for (int i = 0; i < APIC_CALIBRATE_MEASUREMENTS; i++)
1174                 measurements[i] = apic_calibrate_impl();
1175 
1176         /*
1177          * sort results and retrieve median.
1178          */
1179         for (int i = 0; i < APIC_CALIBRATE_MEASUREMENTS; i++) {
1180                 for (int j = i + 1; j < APIC_CALIBRATE_MEASUREMENTS; j++) {
1181                         if (measurements[j] < measurements[i]) {
1182                                 uint64_t tmp = measurements[i];
1183                                 measurements[i] = measurements[j];
1184                                 measurements[j] = tmp;
1185                         }
1186                 }
1187         }
1188         median_idx = APIC_CALIBRATE_MEASUREMENTS / 2;
1189         median = measurements[median_idx];
1190 
1191 #if (APIC_CALIBRATE_MEASUREMENTS >= 3)
1192         /*
1193          * Check that measurements are consistent. Post a warning
1194          * if the three middle values are not close to each other.
1195          */
1196         uint64_t delta_warn = median *
1197             APIC_CALIBRATE_PERCENT_OFF_WARNING / 100;
1198         if ((median - measurements[median_idx - 1]) > delta_warn ||
1199             (measurements[median_idx + 1] - median) > delta_warn) {
1200                 cmn_err(CE_WARN, "apic_calibrate measurements lack "
1201                     "precision: %llu, %llu, %llu.",
1202                     (u_longlong_t)measurements[median_idx - 1],
1203                     (u_longlong_t)median,
1204                     (u_longlong_t)measurements[median_idx + 1]);
1205         }
1206 #endif
1207 
1208         return (median);
1209 }
1210 
1211 /*
1212  * Initialise the APIC timer on the local APIC of CPU 0 to the desired
1213  * frequency.  Note at this stage in the boot sequence, the boot processor
1214  * is the only active processor.
1215  * hertz value of 0 indicates a one-shot mode request.  In this case
1216  * the function returns the resolution (in nanoseconds) for the hardware
1217  * timer interrupt.  If one-shot mode capability is not available,
1218  * the return value will be 0. apic_enable_oneshot is a global switch
1219  * for disabling the functionality.
1220  * A non-zero positive value for hertz indicates a periodic mode request.
1221  * In this case the hardware will be programmed to generate clock interrupts
1222  * at hertz frequency and returns the resolution of interrupts in
1223  * nanosecond.
1224  */
1225 
1226 int
1227 apic_clkinit(int hertz)
1228 {
1229         int             ret;
1230 
1231         apic_int_busy_mark = (apic_int_busy_mark *
1232             apic_sample_factor_redistribution) / 100;
1233         apic_int_free_mark = (apic_int_free_mark *
1234             apic_sample_factor_redistribution) / 100;
1235         apic_diff_for_redistribution = (apic_diff_for_redistribution *
1236             apic_sample_factor_redistribution) / 100;
1237 
1238         ret = apic_timer_init(hertz);
1239         return (ret);
1240 
1241 }
1242 
1243 /*
1244  * apic_preshutdown:
1245  * Called early in shutdown whilst we can still access filesystems to do
1246  * things like loading modules which will be required to complete shutdown
1247  * after filesystems are all unmounted.
1248  */
1249 void
1250 apic_preshutdown(int cmd, int fcn)
1251 {
1252         APIC_VERBOSE_POWEROFF(("apic_preshutdown(%d,%d); m=%d a=%d\n",
1253             cmd, fcn, apic_poweroff_method, apic_enable_acpi));
1254 }
1255 
1256 void
1257 apic_shutdown(int cmd, int fcn)
1258 {
1259         int restarts, attempts;
1260         int i;
1261         uchar_t byte;
1262         ulong_t iflag;
1263 
1264         hpet_acpi_fini();
1265 
1266         /* Send NMI to all CPUs except self to do per processor shutdown */
1267         iflag = intr_clear();
1268 #ifdef  DEBUG
1269         APIC_AV_PENDING_SET();
1270 #else
1271         if (apic_mode == LOCAL_APIC)
1272                 APIC_AV_PENDING_SET();
1273 #endif /* DEBUG */
1274         apic_shutdown_processors = 1;
1275         apic_reg_ops->apic_write(APIC_INT_CMD1,
1276             AV_NMI | AV_LEVEL | AV_SH_ALL_EXCSELF);
1277 
1278         /* restore cmos shutdown byte before reboot */
1279         if (apic_cmos_ssb_set) {
1280                 outb(CMOS_ADDR, SSB);
1281                 outb(CMOS_DATA, 0);
1282         }
1283 
1284         ioapic_disable_redirection();
1285 
1286         /*      disable apic mode if imcr present       */
1287         if (apic_imcrp) {
1288                 outb(APIC_IMCR_P1, (uchar_t)APIC_IMCR_SELECT);
1289                 outb(APIC_IMCR_P2, (uchar_t)APIC_IMCR_PIC);
1290         }
1291 
1292         apic_disable_local_apic();
1293 
1294         intr_restore(iflag);
1295 
1296         /* remainder of function is for shutdown cases only */
1297         if (cmd != A_SHUTDOWN)
1298                 return;
1299 
1300         /*
1301          * Switch system back into Legacy-Mode if using ACPI and
1302          * not powering-off.  Some BIOSes need to remain in ACPI-mode
1303          * for power-off to succeed (Dell Dimension 4600)
1304          * Do not disable ACPI while doing fastreboot
1305          */
1306         if (apic_enable_acpi && fcn != AD_POWEROFF && fcn != AD_FASTREBOOT)
1307                 (void) AcpiDisable();
1308 
1309         if (fcn == AD_FASTREBOOT) {
1310                 apic_reg_ops->apic_write(APIC_INT_CMD1,
1311                     AV_ASSERT | AV_RESET | AV_SH_ALL_EXCSELF);
1312         }
1313 
1314         /* remainder of function is for shutdown+poweroff case only */
1315         if (fcn != AD_POWEROFF)
1316                 return;
1317 
1318         switch (apic_poweroff_method) {
1319                 case APIC_POWEROFF_VIA_RTC:
1320 
1321                         /* select the extended NVRAM bank in the RTC */
1322                         outb(CMOS_ADDR, RTC_REGA);
1323                         byte = inb(CMOS_DATA);
1324                         outb(CMOS_DATA, (byte | EXT_BANK));
1325 
1326                         outb(CMOS_ADDR, PFR_REG);
1327 
1328                         /* for Predator must toggle the PAB bit */
1329                         byte = inb(CMOS_DATA);
1330 
1331                         /*
1332                          * clear power active bar, wakeup alarm and
1333                          * kickstart
1334                          */
1335                         byte &= ~(PAB_CBIT | WF_FLAG | KS_FLAG);
1336                         outb(CMOS_DATA, byte);
1337 
1338                         /* delay before next write */
1339                         drv_usecwait(1000);
1340 
1341                         /* for S40 the following would suffice */
1342                         byte = inb(CMOS_DATA);
1343 
1344                         /* power active bar control bit */
1345                         byte |= PAB_CBIT;
1346                         outb(CMOS_DATA, byte);
1347 
1348                         break;
1349 
1350                 case APIC_POWEROFF_VIA_ASPEN_BMC:
1351                         restarts = 0;
1352 restart_aspen_bmc:
1353                         if (++restarts == 3)
1354                                 break;
1355                         attempts = 0;
1356                         do {
1357                                 byte = inb(MISMIC_FLAG_REGISTER);
1358                                 byte &= MISMIC_BUSY_MASK;
1359                                 if (byte != 0) {
1360                                         drv_usecwait(1000);
1361                                         if (attempts >= 3)
1362                                                 goto restart_aspen_bmc;
1363                                         ++attempts;
1364                                 }
1365                         } while (byte != 0);
1366                         outb(MISMIC_CNTL_REGISTER, CC_SMS_GET_STATUS);
1367                         byte = inb(MISMIC_FLAG_REGISTER);
1368                         byte |= 0x1;
1369                         outb(MISMIC_FLAG_REGISTER, byte);
1370                         i = 0;
1371                         for (; i < (sizeof (aspen_bmc)/sizeof (aspen_bmc[0]));
1372                             i++) {
1373                                 attempts = 0;
1374                                 do {
1375                                         byte = inb(MISMIC_FLAG_REGISTER);
1376                                         byte &= MISMIC_BUSY_MASK;
1377                                         if (byte != 0) {
1378                                                 drv_usecwait(1000);
1379                                                 if (attempts >= 3)
1380                                                         goto restart_aspen_bmc;
1381                                                 ++attempts;
1382                                         }
1383                                 } while (byte != 0);
1384                                 outb(MISMIC_CNTL_REGISTER, aspen_bmc[i].cntl);
1385                                 outb(MISMIC_DATA_REGISTER, aspen_bmc[i].data);
1386                                 byte = inb(MISMIC_FLAG_REGISTER);
1387                                 byte |= 0x1;
1388                                 outb(MISMIC_FLAG_REGISTER, byte);
1389                         }
1390                         break;
1391 
1392                 case APIC_POWEROFF_VIA_SITKA_BMC:
1393                         restarts = 0;
1394 restart_sitka_bmc:
1395                         if (++restarts == 3)
1396                                 break;
1397                         attempts = 0;
1398                         do {
1399                                 byte = inb(SMS_STATUS_REGISTER);
1400                                 byte &= SMS_STATE_MASK;
1401                                 if ((byte == SMS_READ_STATE) ||
1402                                     (byte == SMS_WRITE_STATE)) {
1403                                         drv_usecwait(1000);
1404                                         if (attempts >= 3)
1405                                                 goto restart_sitka_bmc;
1406                                         ++attempts;
1407                                 }
1408                         } while ((byte == SMS_READ_STATE) ||
1409                             (byte == SMS_WRITE_STATE));
1410                         outb(SMS_COMMAND_REGISTER, SMS_GET_STATUS);
1411                         i = 0;
1412                         for (; i < (sizeof (sitka_bmc)/sizeof (sitka_bmc[0]));
1413                             i++) {
1414                                 attempts = 0;
1415                                 do {
1416                                         byte = inb(SMS_STATUS_REGISTER);
1417                                         byte &= SMS_IBF_MASK;
1418                                         if (byte != 0) {
1419                                                 drv_usecwait(1000);
1420                                                 if (attempts >= 3)
1421                                                         goto restart_sitka_bmc;
1422                                                 ++attempts;
1423                                         }
1424                                 } while (byte != 0);
1425                                 outb(sitka_bmc[i].port, sitka_bmc[i].data);
1426                         }
1427                         break;
1428 
1429                 case APIC_POWEROFF_NONE:
1430 
1431                         /* If no APIC direct method, we will try using ACPI */
1432                         if (apic_enable_acpi) {
1433                                 if (acpi_poweroff() == 1)
1434                                         return;
1435                         } else
1436                                 return;
1437 
1438                         break;
1439         }
1440         /*
1441          * Wait a limited time here for power to go off.
1442          * If the power does not go off, then there was a
1443          * problem and we should continue to the halt which
1444          * prints a message for the user to press a key to
1445          * reboot.
1446          */
1447         drv_usecwait(7000000); /* wait seven seconds */
1448 
1449 }
1450 
1451 cyclic_id_t apic_cyclic_id;
1452 
1453 /*
1454  * The following functions are in the platform specific file so that they
1455  * can be different functions depending on whether we are running on
1456  * bare metal or a hypervisor.
1457  */
1458 
1459 /*
1460  * map an apic for memory-mapped access
1461  */
1462 uint32_t *
1463 mapin_apic(uint32_t addr, size_t len, int flags)
1464 {
1465         return ((void *)psm_map_phys(addr, len, flags));
1466 }
1467 
1468 uint32_t *
1469 mapin_ioapic(uint32_t addr, size_t len, int flags)
1470 {
1471         return (mapin_apic(addr, len, flags));
1472 }
1473 
1474 /*
1475  * unmap an apic
1476  */
1477 void
1478 mapout_apic(caddr_t addr, size_t len)
1479 {
1480         psm_unmap_phys(addr, len);
1481 }
1482 
1483 void
1484 mapout_ioapic(caddr_t addr, size_t len)
1485 {
1486         mapout_apic(addr, len);
1487 }
1488 
1489 uint32_t
1490 ioapic_read(int ioapic_ix, uint32_t reg)
1491 {
1492         volatile uint32_t *ioapic;
1493 
1494         ioapic = apicioadr[ioapic_ix];
1495         ioapic[APIC_IO_REG] = reg;
1496         return (ioapic[APIC_IO_DATA]);
1497 }
1498 
1499 void
1500 ioapic_write(int ioapic_ix, uint32_t reg, uint32_t value)
1501 {
1502         volatile uint32_t *ioapic;
1503 
1504         ioapic = apicioadr[ioapic_ix];
1505         ioapic[APIC_IO_REG] = reg;
1506         ioapic[APIC_IO_DATA] = value;
1507 }
1508 
1509 void
1510 ioapic_write_eoi(int ioapic_ix, uint32_t value)
1511 {
1512         volatile uint32_t *ioapic;
1513 
1514         ioapic = apicioadr[ioapic_ix];
1515         ioapic[APIC_IO_EOI] = value;
1516 }
1517 
1518 /*
1519  * Round-robin algorithm to find the next CPU with interrupts enabled.
1520  * It can't share the same static variable apic_next_bind_cpu with
1521  * apic_get_next_bind_cpu(), since that will cause all interrupts to be
1522  * bound to CPU1 at boot time.  During boot, only CPU0 is online with
1523  * interrupts enabled when apic_get_next_bind_cpu() and apic_find_cpu()
1524  * are called.  However, the pcplusmp driver assumes that there will be
1525  * boot_ncpus CPUs configured eventually so it tries to distribute all
1526  * interrupts among CPU0 - CPU[boot_ncpus - 1].  Thus to prevent all
1527  * interrupts being targetted at CPU1, we need to use a dedicated static
1528  * variable for find_next_cpu() instead of sharing apic_next_bind_cpu.
1529  */
1530 
1531 processorid_t
1532 apic_find_cpu(int flag)
1533 {
1534         int i;
1535         static processorid_t acid = 0;
1536 
1537         /* Find the first CPU with the passed-in flag set */
1538         for (i = 0; i < apic_nproc; i++) {
1539                 if (++acid >= apic_nproc) {
1540                         acid = 0;
1541                 }
1542                 if (apic_cpu_in_range(acid) &&
1543                     (apic_cpus[acid].aci_status & flag)) {
1544                         break;
1545                 }
1546         }
1547 
1548         ASSERT((apic_cpus[acid].aci_status & flag) != 0);
1549         return (acid);
1550 }
1551 
1552 void
1553 apic_intrmap_init(int apic_mode)
1554 {
1555         int suppress_brdcst_eoi = 0;
1556 
1557         /*
1558          * Intel Software Developer's Manual 3A, 10.12.7:
1559          *
1560          * Routing of device interrupts to local APIC units operating in
1561          * x2APIC mode requires use of the interrupt-remapping architecture
1562          * specified in the Intel Virtualization Technology for Directed
1563          * I/O, Revision 1.3.  Because of this, BIOS must enumerate support
1564          * for and software must enable this interrupt remapping with
1565          * Extended Interrupt Mode Enabled before it enabling x2APIC mode in
1566          * the local APIC units.
1567          *
1568          *
1569          * In other words, to use the APIC in x2APIC mode, we need interrupt
1570          * remapping.  Since we don't start up the IOMMU by default, we
1571          * won't be able to do any interrupt remapping and therefore have to
1572          * use the APIC in traditional 'local APIC' mode with memory mapped
1573          * I/O.
1574          */
1575 
1576         if (psm_vt_ops != NULL) {
1577                 if (((apic_intrmap_ops_t *)psm_vt_ops)->
1578                     apic_intrmap_init(apic_mode) == DDI_SUCCESS) {
1579 
1580                         apic_vt_ops = psm_vt_ops;
1581 
1582                         /*
1583                          * We leverage the interrupt remapping engine to
1584                          * suppress broadcast EOI; thus we must send the
1585                          * directed EOI with the directed-EOI handler.
1586                          */
1587                         if (apic_directed_EOI_supported() == 0) {
1588                                 suppress_brdcst_eoi = 1;
1589                         }
1590 
1591                         apic_vt_ops->apic_intrmap_enable(suppress_brdcst_eoi);
1592 
1593                         if (apic_detect_x2apic()) {
1594                                 apic_enable_x2apic();
1595                         }
1596 
1597                         if (apic_directed_EOI_supported() == 0) {
1598                                 apic_set_directed_EOI_handler();
1599                         }
1600                 }
1601         }
1602 }
1603 
1604 /*ARGSUSED*/
1605 static void
1606 apic_record_ioapic_rdt(void *intrmap_private, ioapic_rdt_t *irdt)
1607 {
1608         irdt->ir_hi <<= APIC_ID_BIT_OFFSET;
1609 }
1610 
1611 /*ARGSUSED*/
1612 static void
1613 apic_record_msi(void *intrmap_private, msi_regs_t *mregs)
1614 {
1615         mregs->mr_addr = MSI_ADDR_HDR |
1616             (MSI_ADDR_RH_FIXED << MSI_ADDR_RH_SHIFT) |
1617             (MSI_ADDR_DM_PHYSICAL << MSI_ADDR_DM_SHIFT) |
1618             (mregs->mr_addr << MSI_ADDR_DEST_SHIFT);
1619         mregs->mr_data = (MSI_DATA_TM_EDGE << MSI_DATA_TM_SHIFT) |
1620             mregs->mr_data;
1621 }
1622 
1623 /*
1624  * Functions from apic_introp.c
1625  *
1626  * Those functions are used by apic_intr_ops().
1627  */
1628 
1629 /*
1630  * MSI support flag:
1631  * reflects whether MSI is supported at APIC level
1632  * it can also be patched through /etc/system
1633  *
1634  *  0 = default value - don't know and need to call apic_check_msi_support()
1635  *      to find out then set it accordingly
1636  *  1 = supported
1637  * -1 = not supported
1638  */
1639 int     apic_support_msi = 0;
1640 
1641 /* Multiple vector support for MSI-X */
1642 int     apic_msix_enable = 1;
1643 
1644 /* Multiple vector support for MSI */
1645 int     apic_multi_msi_enable = 1;
1646 
1647 /*
1648  * Check whether the system supports MSI.
1649  *
1650  * MSI is required for PCI-E and for PCI versions later than 2.2, so if we find
1651  * a PCI-E bus or we find a PCI bus whose version we know is >= 2.2, then we
1652  * return PSM_SUCCESS to indicate this system supports MSI.
1653  *
1654  * (Currently the only way we check whether a given PCI bus supports >= 2.2 is
1655  * by detecting if we are running inside the KVM hypervisor, which guarantees
1656  * this version number.)
1657  */
1658 int
1659 apic_check_msi_support()
1660 {
1661         dev_info_t *cdip;
1662         char dev_type[16];
1663         int dev_len;
1664         int hwenv = get_hwenv();
1665 
1666         DDI_INTR_IMPLDBG((CE_CONT, "apic_check_msi_support:\n"));
1667 
1668         /*
1669          * check whether the first level children of root_node have
1670          * PCI-E or PCI capability.
1671          */
1672         for (cdip = ddi_get_child(ddi_root_node()); cdip != NULL;
1673             cdip = ddi_get_next_sibling(cdip)) {
1674 
1675                 DDI_INTR_IMPLDBG((CE_CONT, "apic_check_msi_support: cdip: 0x%p,"
1676                     " driver: %s, binding: %s, nodename: %s\n", (void *)cdip,
1677                     ddi_driver_name(cdip), ddi_binding_name(cdip),
1678                     ddi_node_name(cdip)));
1679                 dev_len = sizeof (dev_type);
1680                 if (ddi_getlongprop_buf(DDI_DEV_T_ANY, cdip, DDI_PROP_DONTPASS,
1681                     "device_type", (caddr_t)dev_type, &dev_len)
1682                     != DDI_PROP_SUCCESS)
1683                         continue;
1684                 if (strcmp(dev_type, "pciex") == 0)
1685                         return (PSM_SUCCESS);
1686                 if (strcmp(dev_type, "pci") == 0 &&
1687                     (hwenv == HW_KVM || hwenv == HW_BHYVE))
1688                         return (PSM_SUCCESS);
1689         }
1690 
1691         /* MSI is not supported on this system */
1692         DDI_INTR_IMPLDBG((CE_CONT, "apic_check_msi_support: no 'pciex' "
1693             "device_type found\n"));
1694         return (PSM_FAILURE);
1695 }
1696 
1697 /*
1698  * apic_pci_msi_unconfigure:
1699  *
1700  * This and next two interfaces are copied from pci_intr_lib.c
1701  * Do ensure that these two files stay in sync.
1702  * These needed to be copied over here to avoid a deadlock situation on
1703  * certain mp systems that use MSI interrupts.
1704  *
1705  * IMPORTANT regards next three interfaces:
1706  * i) are called only for MSI/X interrupts.
1707  * ii) called with interrupts disabled, and must not block
1708  */
1709 void
1710 apic_pci_msi_unconfigure(dev_info_t *rdip, int type, int inum)
1711 {
1712         ushort_t                msi_ctrl;
1713         int                     cap_ptr = i_ddi_get_msi_msix_cap_ptr(rdip);
1714         ddi_acc_handle_t        handle = i_ddi_get_pci_config_handle(rdip);
1715 
1716         ASSERT((handle != NULL) && (cap_ptr != 0));
1717 
1718         if (type == DDI_INTR_TYPE_MSI) {
1719                 msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSI_CTRL);
1720                 msi_ctrl &= (~PCI_MSI_MME_MASK);
1721                 pci_config_put16(handle, cap_ptr + PCI_MSI_CTRL, msi_ctrl);
1722                 pci_config_put32(handle, cap_ptr + PCI_MSI_ADDR_OFFSET, 0);
1723 
1724                 if (msi_ctrl &  PCI_MSI_64BIT_MASK) {
1725                         pci_config_put16(handle,
1726                             cap_ptr + PCI_MSI_64BIT_DATA, 0);
1727                         pci_config_put32(handle,
1728                             cap_ptr + PCI_MSI_ADDR_OFFSET + 4, 0);
1729                 } else {
1730                         pci_config_put16(handle,
1731                             cap_ptr + PCI_MSI_32BIT_DATA, 0);
1732                 }
1733 
1734         } else if (type == DDI_INTR_TYPE_MSIX) {
1735                 uintptr_t       off;
1736                 uint32_t        mask;
1737                 ddi_intr_msix_t *msix_p = i_ddi_get_msix(rdip);
1738 
1739                 ASSERT(msix_p != NULL);
1740 
1741                 /* Offset into "inum"th entry in the MSI-X table & mask it */
1742                 off = (uintptr_t)msix_p->msix_tbl_addr + (inum *
1743                     PCI_MSIX_VECTOR_SIZE) + PCI_MSIX_VECTOR_CTRL_OFFSET;
1744 
1745                 mask = ddi_get32(msix_p->msix_tbl_hdl, (uint32_t *)off);
1746 
1747                 ddi_put32(msix_p->msix_tbl_hdl, (uint32_t *)off, (mask | 1));
1748 
1749                 /* Offset into the "inum"th entry in the MSI-X table */
1750                 off = (uintptr_t)msix_p->msix_tbl_addr +
1751                     (inum * PCI_MSIX_VECTOR_SIZE);
1752 
1753                 /* Reset the "data" and "addr" bits */
1754                 ddi_put32(msix_p->msix_tbl_hdl,
1755                     (uint32_t *)(off + PCI_MSIX_DATA_OFFSET), 0);
1756                 ddi_put64(msix_p->msix_tbl_hdl, (uint64_t *)off, 0);
1757         }
1758 }
1759 
1760 /*
1761  * apic_pci_msi_disable_mode:
1762  */
1763 void
1764 apic_pci_msi_disable_mode(dev_info_t *rdip, int type)
1765 {
1766         ushort_t                msi_ctrl;
1767         int                     cap_ptr = i_ddi_get_msi_msix_cap_ptr(rdip);
1768         ddi_acc_handle_t        handle = i_ddi_get_pci_config_handle(rdip);
1769 
1770         ASSERT((handle != NULL) && (cap_ptr != 0));
1771 
1772         if (type == DDI_INTR_TYPE_MSI) {
1773                 msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSI_CTRL);
1774                 if (!(msi_ctrl & PCI_MSI_ENABLE_BIT))
1775                         return;
1776 
1777                 msi_ctrl &= ~PCI_MSI_ENABLE_BIT;    /* MSI disable */
1778                 pci_config_put16(handle, cap_ptr + PCI_MSI_CTRL, msi_ctrl);
1779 
1780         } else if (type == DDI_INTR_TYPE_MSIX) {
1781                 msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSIX_CTRL);
1782                 if (msi_ctrl & PCI_MSIX_ENABLE_BIT) {
1783                         msi_ctrl &= ~PCI_MSIX_ENABLE_BIT;
1784                         pci_config_put16(handle, cap_ptr + PCI_MSIX_CTRL,
1785                             msi_ctrl);
1786                 }
1787         }
1788 }
1789 
1790 uint32_t
1791 apic_get_localapicid(uint32_t cpuid)
1792 {
1793         ASSERT(cpuid < apic_nproc && apic_cpus != NULL);
1794 
1795         return (apic_cpus[cpuid].aci_local_id);
1796 }
1797 
1798 uchar_t
1799 apic_get_ioapicid(uchar_t ioapicindex)
1800 {
1801         ASSERT(ioapicindex < MAX_IO_APIC);
1802 
1803         return (apic_io_id[ioapicindex]);
1804 }