1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25 /*
26 * Copyright 2018 Joyent, Inc.
27 * Copyright (c) 2016, 2017 by Delphix. All rights reserved.
28 */
29
30 /*
31 * PSMI 1.1 extensions are supported only in 2.6 and later versions.
32 * PSMI 1.2 extensions are supported only in 2.7 and later versions.
33 * PSMI 1.3 and 1.4 extensions are supported in Solaris 10.
34 * PSMI 1.5 extensions are supported in Solaris Nevada.
35 * PSMI 1.6 extensions are supported in Solaris Nevada.
36 * PSMI 1.7 extensions are supported in Solaris Nevada.
37 */
38 #define PSMI_1_7
39
40 #include <sys/processor.h>
41 #include <sys/time.h>
42 #include <sys/psm.h>
43 #include <sys/smp_impldefs.h>
44 #include <sys/cram.h>
45 #include <sys/acpi/acpi.h>
46 #include <sys/acpica.h>
47 #include <sys/psm_common.h>
48 #include <sys/apic.h>
49 #include <sys/pit.h>
50 #include <sys/ddi.h>
51 #include <sys/sunddi.h>
52 #include <sys/ddi_impldefs.h>
53 #include <sys/pci.h>
54 #include <sys/promif.h>
55 #include <sys/x86_archext.h>
56 #include <sys/cpc_impl.h>
57 #include <sys/uadmin.h>
58 #include <sys/panic.h>
59 #include <sys/debug.h>
60 #include <sys/archsystm.h>
61 #include <sys/trap.h>
62 #include <sys/machsystm.h>
63 #include <sys/sysmacros.h>
64 #include <sys/cpuvar.h>
65 #include <sys/rm_platter.h>
66 #include <sys/privregs.h>
67 #include <sys/note.h>
68 #include <sys/pci_intr_lib.h>
69 #include <sys/spl.h>
70 #include <sys/clock.h>
71 #include <sys/dditypes.h>
72 #include <sys/sunddi.h>
73 #include <sys/x_call.h>
74 #include <sys/reboot.h>
75 #include <sys/hpet.h>
76 #include <sys/apic_common.h>
77 #include <sys/apic_timer.h>
78
79 static void apic_record_ioapic_rdt(void *intrmap_private,
80 ioapic_rdt_t *irdt);
81 static void apic_record_msi(void *intrmap_private, msi_regs_t *mregs);
82
83 /*
84 * Common routines between pcplusmp & apix (taken from apic.c).
85 */
86
87 int apic_clkinit(int);
88 hrtime_t apic_gethrtime(void);
89 void apic_send_ipi(int, int);
90 void apic_set_idlecpu(processorid_t);
91 void apic_unset_idlecpu(processorid_t);
92 void apic_shutdown(int, int);
93 void apic_preshutdown(int, int);
94 processorid_t apic_get_next_processorid(processorid_t);
95
96 hrtime_t apic_gettime();
97
98 enum apic_ioapic_method_type apix_mul_ioapic_method = APIC_MUL_IOAPIC_PCPLUSMP;
99
100 /* Now the ones for Dynamic Interrupt distribution */
101 int apic_enable_dynamic_migration = 0;
102
103 /* maximum loop count when sending Start IPIs. */
104 int apic_sipi_max_loop_count = 0x1000;
105
106 /*
107 * These variables are frequently accessed in apic_intr_enter(),
108 * apic_intr_exit and apic_setspl, so group them together
109 */
110 volatile uint32_t *apicadr = NULL; /* virtual addr of local APIC */
111 int apic_setspl_delay = 1; /* apic_setspl - delay enable */
112 int apic_clkvect;
113
114 /* vector at which error interrupts come in */
115 int apic_errvect;
116 int apic_enable_error_intr = 1;
117 int apic_error_display_delay = 100;
118
119 /* vector at which performance counter overflow interrupts come in */
120 int apic_cpcovf_vect;
121 int apic_enable_cpcovf_intr = 1;
122
123 /* vector at which CMCI interrupts come in */
124 int apic_cmci_vect;
125 extern void cmi_cmci_trap(void);
126
127 lock_t apic_mode_switch_lock;
128
129 int apic_pir_vect;
130
131 /*
132 * Patchable global variables.
133 */
134 int apic_forceload = 0;
135
136 int apic_coarse_hrtime = 1; /* 0 - use accurate slow gethrtime() */
137
138 int apic_flat_model = 0; /* 0 - clustered. 1 - flat */
139 int apic_panic_on_nmi = 0;
140 int apic_panic_on_apic_error = 0;
141
142 int apic_verbose = 0; /* 0x1ff */
143
144 #ifdef DEBUG
145 int apic_debug = 0;
146 int apic_restrict_vector = 0;
147
148 int apic_debug_msgbuf[APIC_DEBUG_MSGBUFSIZE];
149 int apic_debug_msgbufindex = 0;
150
151 #endif /* DEBUG */
152
153 uint_t apic_nticks = 0;
154 uint_t apic_skipped_redistribute = 0;
155
156 uint_t last_count_read = 0;
157 lock_t apic_gethrtime_lock;
158 volatile int apic_hrtime_stamp = 0;
159 volatile hrtime_t apic_nsec_since_boot = 0;
160
161 static hrtime_t apic_last_hrtime = 0;
162 int apic_hrtime_error = 0;
163 int apic_remote_hrterr = 0;
164 int apic_num_nmis = 0;
165 int apic_apic_error = 0;
166 int apic_num_apic_errors = 0;
167 int apic_num_cksum_errors = 0;
168
169 int apic_error = 0;
170
171 static int apic_cmos_ssb_set = 0;
172
173 /* use to make sure only one cpu handles the nmi */
174 lock_t apic_nmi_lock;
175 /* use to make sure only one cpu handles the error interrupt */
176 lock_t apic_error_lock;
177
178 static struct {
179 uchar_t cntl;
180 uchar_t data;
181 } aspen_bmc[] = {
182 { CC_SMS_WR_START, 0x18 }, /* NetFn/LUN */
183 { CC_SMS_WR_NEXT, 0x24 }, /* Cmd SET_WATCHDOG_TIMER */
184 { CC_SMS_WR_NEXT, 0x84 }, /* DataByte 1: SMS/OS no log */
185 { CC_SMS_WR_NEXT, 0x2 }, /* DataByte 2: Power Down */
186 { CC_SMS_WR_NEXT, 0x0 }, /* DataByte 3: no pre-timeout */
187 { CC_SMS_WR_NEXT, 0x0 }, /* DataByte 4: timer expir. */
188 { CC_SMS_WR_NEXT, 0xa }, /* DataByte 5: init countdown */
189 { CC_SMS_WR_END, 0x0 }, /* DataByte 6: init countdown */
190
191 { CC_SMS_WR_START, 0x18 }, /* NetFn/LUN */
192 { CC_SMS_WR_END, 0x22 } /* Cmd RESET_WATCHDOG_TIMER */
193 };
194
195 static struct {
196 int port;
197 uchar_t data;
198 } sitka_bmc[] = {
199 { SMS_COMMAND_REGISTER, SMS_WRITE_START },
200 { SMS_DATA_REGISTER, 0x18 }, /* NetFn/LUN */
201 { SMS_DATA_REGISTER, 0x24 }, /* Cmd SET_WATCHDOG_TIMER */
202 { SMS_DATA_REGISTER, 0x84 }, /* DataByte 1: SMS/OS no log */
203 { SMS_DATA_REGISTER, 0x2 }, /* DataByte 2: Power Down */
204 { SMS_DATA_REGISTER, 0x0 }, /* DataByte 3: no pre-timeout */
205 { SMS_DATA_REGISTER, 0x0 }, /* DataByte 4: timer expir. */
206 { SMS_DATA_REGISTER, 0xa }, /* DataByte 5: init countdown */
207 { SMS_COMMAND_REGISTER, SMS_WRITE_END },
208 { SMS_DATA_REGISTER, 0x0 }, /* DataByte 6: init countdown */
209
210 { SMS_COMMAND_REGISTER, SMS_WRITE_START },
211 { SMS_DATA_REGISTER, 0x18 }, /* NetFn/LUN */
212 { SMS_COMMAND_REGISTER, SMS_WRITE_END },
213 { SMS_DATA_REGISTER, 0x22 } /* Cmd RESET_WATCHDOG_TIMER */
214 };
215
216 /* Patchable global variables. */
217 int apic_kmdb_on_nmi = 0; /* 0 - no, 1 - yes enter kmdb */
218 uint32_t apic_divide_reg_init = 0; /* 0 - divide by 2 */
219
220 /* default apic ops without interrupt remapping */
221 static apic_intrmap_ops_t apic_nointrmap_ops = {
222 (int (*)(int))return_instr,
223 (void (*)(int))return_instr,
224 (void (*)(void **, dev_info_t *, uint16_t, int, uchar_t))return_instr,
225 (void (*)(void *, void *, uint16_t, int))return_instr,
226 (void (*)(void **))return_instr,
227 apic_record_ioapic_rdt,
228 apic_record_msi,
229 };
230
231 apic_intrmap_ops_t *apic_vt_ops = &apic_nointrmap_ops;
232 apic_cpus_info_t *apic_cpus = NULL;
233 cpuset_t apic_cpumask;
234 uint_t apic_picinit_called;
235
236 /* Flag to indicate that we need to shut down all processors */
237 static uint_t apic_shutdown_processors;
238
239 /*
240 * Probe the ioapic method for apix module. Called in apic_probe_common()
241 */
242 int
243 apic_ioapic_method_probe()
244 {
245 if (apix_enable == 0)
246 return (PSM_SUCCESS);
247
248 /*
249 * Set IOAPIC EOI handling method. The priority from low to high is:
250 * 1. IOxAPIC: with EOI register
251 * 2. IOMMU interrupt mapping
252 * 3. Mask-Before-EOI method for systems without boot
253 * interrupt routing, such as systems with only one IOAPIC;
254 * NVIDIA CK8-04/MCP55 systems; systems with bridge solution
255 * which disables the boot interrupt routing already.
256 * 4. Directed EOI
257 */
258 if (apic_io_ver[0] >= 0x20)
259 apix_mul_ioapic_method = APIC_MUL_IOAPIC_IOXAPIC;
260 if ((apic_io_max == 1) || (apic_nvidia_io_max == apic_io_max))
261 apix_mul_ioapic_method = APIC_MUL_IOAPIC_MASK;
262 if (apic_directed_EOI_supported())
263 apix_mul_ioapic_method = APIC_MUL_IOAPIC_DEOI;
264
265 /* fall back to pcplusmp */
266 if (apix_mul_ioapic_method == APIC_MUL_IOAPIC_PCPLUSMP) {
267 /* make sure apix is after pcplusmp in /etc/mach */
268 apix_enable = 0; /* go ahead with pcplusmp install next */
269 return (PSM_FAILURE);
270 }
271
272 return (PSM_SUCCESS);
273 }
274
275 /*
276 * handler for APIC Error interrupt. Just print a warning and continue
277 */
278 int
279 apic_error_intr()
280 {
281 uint_t error0, error1, error;
282 uint_t i;
283
284 /*
285 * We need to write before read as per 7.4.17 of system prog manual.
286 * We do both and or the results to be safe
287 */
288 error0 = apic_reg_ops->apic_read(APIC_ERROR_STATUS);
289 apic_reg_ops->apic_write(APIC_ERROR_STATUS, 0);
290 error1 = apic_reg_ops->apic_read(APIC_ERROR_STATUS);
291 error = error0 | error1;
292
293 /*
294 * Clear the APIC error status (do this on all cpus that enter here)
295 * (two writes are required due to the semantics of accessing the
296 * error status register.)
297 */
298 apic_reg_ops->apic_write(APIC_ERROR_STATUS, 0);
299 apic_reg_ops->apic_write(APIC_ERROR_STATUS, 0);
300
301 /*
302 * Prevent more than 1 CPU from handling error interrupt causing
303 * double printing (interleave of characters from multiple
304 * CPU's when using prom_printf)
305 */
306 if (lock_try(&apic_error_lock) == 0)
307 return (error ? DDI_INTR_CLAIMED : DDI_INTR_UNCLAIMED);
308 if (error) {
309 #if DEBUG
310 if (apic_debug)
311 debug_enter("pcplusmp: APIC Error interrupt received");
312 #endif /* DEBUG */
313 if (apic_panic_on_apic_error)
314 cmn_err(CE_PANIC,
315 "APIC Error interrupt on CPU %d. Status = %x",
316 psm_get_cpu_id(), error);
317 else {
318 if ((error & ~APIC_CS_ERRORS) == 0) {
319 /* cksum error only */
320 apic_error |= APIC_ERR_APIC_ERROR;
321 apic_apic_error |= error;
322 apic_num_apic_errors++;
323 apic_num_cksum_errors++;
324 } else {
325 /*
326 * prom_printf is the best shot we have of
327 * something which is problem free from
328 * high level/NMI type of interrupts
329 */
330 prom_printf("APIC Error interrupt on CPU %d. "
331 "Status 0 = %x, Status 1 = %x\n",
332 psm_get_cpu_id(), error0, error1);
333 apic_error |= APIC_ERR_APIC_ERROR;
334 apic_apic_error |= error;
335 apic_num_apic_errors++;
336 for (i = 0; i < apic_error_display_delay; i++) {
337 tenmicrosec();
338 }
339 /*
340 * provide more delay next time limited to
341 * roughly 1 clock tick time
342 */
343 if (apic_error_display_delay < 500)
344 apic_error_display_delay *= 2;
345 }
346 }
347 lock_clear(&apic_error_lock);
348 return (DDI_INTR_CLAIMED);
349 } else {
350 lock_clear(&apic_error_lock);
351 return (DDI_INTR_UNCLAIMED);
352 }
353 }
354
355 /*
356 * Turn off the mask bit in the performance counter Local Vector Table entry.
357 */
358 void
359 apic_cpcovf_mask_clear(void)
360 {
361 apic_reg_ops->apic_write(APIC_PCINT_VECT,
362 (apic_reg_ops->apic_read(APIC_PCINT_VECT) & ~APIC_LVT_MASK));
363 }
364
365 /*ARGSUSED*/
366 static int
367 apic_cmci_enable(xc_arg_t arg1, xc_arg_t arg2, xc_arg_t arg3)
368 {
369 apic_reg_ops->apic_write(APIC_CMCI_VECT, apic_cmci_vect);
370 return (0);
371 }
372
373 /*ARGSUSED*/
374 static int
375 apic_cmci_disable(xc_arg_t arg1, xc_arg_t arg2, xc_arg_t arg3)
376 {
377 apic_reg_ops->apic_write(APIC_CMCI_VECT, apic_cmci_vect | AV_MASK);
378 return (0);
379 }
380
381 void
382 apic_cmci_setup(processorid_t cpuid, boolean_t enable)
383 {
384 cpuset_t cpu_set;
385
386 CPUSET_ONLY(cpu_set, cpuid);
387
388 if (enable) {
389 xc_call(NULL, NULL, NULL, CPUSET2BV(cpu_set),
390 (xc_func_t)apic_cmci_enable);
391 } else {
392 xc_call(NULL, NULL, NULL, CPUSET2BV(cpu_set),
393 (xc_func_t)apic_cmci_disable);
394 }
395 }
396
397 static void
398 apic_disable_local_apic(void)
399 {
400 apic_reg_ops->apic_write_task_reg(APIC_MASK_ALL);
401 apic_reg_ops->apic_write(APIC_LOCAL_TIMER, AV_MASK);
402
403 /* local intr reg 0 */
404 apic_reg_ops->apic_write(APIC_INT_VECT0, AV_MASK);
405
406 /* disable NMI */
407 apic_reg_ops->apic_write(APIC_INT_VECT1, AV_MASK);
408
409 /* and error interrupt */
410 apic_reg_ops->apic_write(APIC_ERR_VECT, AV_MASK);
411
412 /* and perf counter intr */
413 apic_reg_ops->apic_write(APIC_PCINT_VECT, AV_MASK);
414
415 apic_reg_ops->apic_write(APIC_SPUR_INT_REG, APIC_SPUR_INTR);
416 }
417
418 static void
419 apic_cpu_send_SIPI(processorid_t cpun, boolean_t start)
420 {
421 int loop_count;
422 uint32_t vector;
423 uint_t apicid;
424 ulong_t iflag;
425
426 apicid = apic_cpus[cpun].aci_local_id;
427
428 /*
429 * Interrupts on current CPU will be disabled during the
430 * steps in order to avoid unwanted side effects from
431 * executing interrupt handlers on a problematic BIOS.
432 */
433 iflag = intr_clear();
434
435 if (start) {
436 outb(CMOS_ADDR, SSB);
437 outb(CMOS_DATA, BIOS_SHUTDOWN);
438 }
439
440 /*
441 * According to X2APIC specification in section '2.3.5.1' of
442 * Interrupt Command Register Semantics, the semantics of
443 * programming the Interrupt Command Register to dispatch an interrupt
444 * is simplified. A single MSR write to the 64-bit ICR is required
445 * for dispatching an interrupt. Specifically, with the 64-bit MSR
446 * interface to ICR, system software is not required to check the
447 * status of the delivery status bit prior to writing to the ICR
448 * to send an IPI. With the removal of the Delivery Status bit,
449 * system software no longer has a reason to read the ICR. It remains
450 * readable only to aid in debugging.
451 */
452 #ifdef DEBUG
453 APIC_AV_PENDING_SET();
454 #else
455 if (apic_mode == LOCAL_APIC) {
456 APIC_AV_PENDING_SET();
457 }
458 #endif /* DEBUG */
459
460 /* for integrated - make sure there is one INIT IPI in buffer */
461 /* for external - it will wake up the cpu */
462 apic_reg_ops->apic_write_int_cmd(apicid, AV_ASSERT | AV_RESET);
463
464 /* If only 1 CPU is installed, PENDING bit will not go low */
465 for (loop_count = apic_sipi_max_loop_count; loop_count; loop_count--) {
466 if (apic_mode == LOCAL_APIC &&
467 apic_reg_ops->apic_read(APIC_INT_CMD1) & AV_PENDING)
468 apic_ret();
469 else
470 break;
471 }
472
473 apic_reg_ops->apic_write_int_cmd(apicid, AV_DEASSERT | AV_RESET);
474 drv_usecwait(20000); /* 20 milli sec */
475
476 if (apic_cpus[cpun].aci_local_ver >= APIC_INTEGRATED_VERS) {
477 /* integrated apic */
478
479 vector = (rm_platter_pa >> MMU_PAGESHIFT) &
480 (APIC_VECTOR_MASK | APIC_IPL_MASK);
481
482 /* to offset the INIT IPI queue up in the buffer */
483 apic_reg_ops->apic_write_int_cmd(apicid, vector | AV_STARTUP);
484 drv_usecwait(200); /* 20 micro sec */
485
486 /*
487 * send the second SIPI (Startup IPI) as recommended by Intel
488 * software development manual.
489 */
490 apic_reg_ops->apic_write_int_cmd(apicid, vector | AV_STARTUP);
491 drv_usecwait(200); /* 20 micro sec */
492 }
493
494 intr_restore(iflag);
495 }
496
497 /*ARGSUSED1*/
498 int
499 apic_cpu_start(processorid_t cpun, caddr_t arg)
500 {
501 ASSERT(MUTEX_HELD(&cpu_lock));
502
503 if (!apic_cpu_in_range(cpun)) {
504 return (EINVAL);
505 }
506
507 /*
508 * Switch to apic_common_send_ipi for safety during starting other CPUs.
509 */
510 if (apic_mode == LOCAL_X2APIC) {
511 apic_switch_ipi_callback(B_TRUE);
512 }
513
514 apic_cmos_ssb_set = 1;
515 apic_cpu_send_SIPI(cpun, B_TRUE);
516
517 return (0);
518 }
519
520 /*
521 * Put CPU into halted state with interrupts disabled.
522 */
523 /*ARGSUSED1*/
524 int
525 apic_cpu_stop(processorid_t cpun, caddr_t arg)
526 {
527 int rc;
528 cpu_t *cp;
529 extern cpuset_t cpu_ready_set;
530 extern void cpu_idle_intercept_cpu(cpu_t *cp);
531
532 ASSERT(MUTEX_HELD(&cpu_lock));
533
534 if (!apic_cpu_in_range(cpun)) {
535 return (EINVAL);
536 }
537 if (apic_cpus[cpun].aci_local_ver < APIC_INTEGRATED_VERS) {
538 return (ENOTSUP);
539 }
540
541 cp = cpu_get(cpun);
542 ASSERT(cp != NULL);
543 ASSERT((cp->cpu_flags & CPU_OFFLINE) != 0);
544 ASSERT((cp->cpu_flags & CPU_QUIESCED) != 0);
545 ASSERT((cp->cpu_flags & CPU_ENABLE) == 0);
546
547 /* Clear CPU_READY flag to disable cross calls. */
548 cp->cpu_flags &= ~CPU_READY;
549 CPUSET_ATOMIC_DEL(cpu_ready_set, cpun);
550 rc = xc_flush_cpu(cp);
551 if (rc != 0) {
552 CPUSET_ATOMIC_ADD(cpu_ready_set, cpun);
553 cp->cpu_flags |= CPU_READY;
554 return (rc);
555 }
556
557 /* Intercept target CPU at a safe point before powering it off. */
558 cpu_idle_intercept_cpu(cp);
559
560 apic_cpu_send_SIPI(cpun, B_FALSE);
561 cp->cpu_flags &= ~CPU_RUNNING;
562
563 return (0);
564 }
565
566 int
567 apic_cpu_ops(psm_cpu_request_t *reqp)
568 {
569 if (reqp == NULL) {
570 return (EINVAL);
571 }
572
573 switch (reqp->pcr_cmd) {
574 case PSM_CPU_ADD:
575 return (apic_cpu_add(reqp));
576
577 case PSM_CPU_REMOVE:
578 return (apic_cpu_remove(reqp));
579
580 case PSM_CPU_STOP:
581 return (apic_cpu_stop(reqp->req.cpu_stop.cpuid,
582 reqp->req.cpu_stop.ctx));
583
584 default:
585 return (ENOTSUP);
586 }
587 }
588
589 #ifdef DEBUG
590 int apic_break_on_cpu = 9;
591 int apic_stretch_interrupts = 0;
592 int apic_stretch_ISR = 1 << 3; /* IPL of 3 matches nothing now */
593 #endif /* DEBUG */
594
595 /*
596 * generates an interprocessor interrupt to another CPU. Any changes made to
597 * this routine must be accompanied by similar changes to
598 * apic_common_send_ipi().
599 */
600 void
601 apic_send_ipi(int cpun, int ipl)
602 {
603 int vector;
604 ulong_t flag;
605
606 vector = apic_resv_vector[ipl];
607
608 ASSERT((vector >= APIC_BASE_VECT) && (vector <= APIC_SPUR_INTR));
609
610 flag = intr_clear();
611
612 APIC_AV_PENDING_SET();
613
614 apic_reg_ops->apic_write_int_cmd(apic_cpus[cpun].aci_local_id,
615 vector);
616
617 intr_restore(flag);
618 }
619
620 void
621 apic_send_pir_ipi(processorid_t cpun)
622 {
623 const int vector = apic_pir_vect;
624 ulong_t flag;
625
626 ASSERT((vector >= APIC_BASE_VECT) && (vector <= APIC_SPUR_INTR));
627
628 flag = intr_clear();
629
630 /* Self-IPI for inducing PIR makes no sense. */
631 if ((cpun != psm_get_cpu_id())) {
632 APIC_AV_PENDING_SET();
633 apic_reg_ops->apic_write_int_cmd(apic_cpus[cpun].aci_local_id,
634 vector);
635 }
636
637 intr_restore(flag);
638 }
639
640 int
641 apic_get_pir_ipivect(void)
642 {
643 return (apic_pir_vect);
644 }
645
646 /*ARGSUSED*/
647 void
648 apic_set_idlecpu(processorid_t cpun)
649 {
650 }
651
652 /*ARGSUSED*/
653 void
654 apic_unset_idlecpu(processorid_t cpun)
655 {
656 }
657
658
659 void
660 apic_ret()
661 {
662 }
663
664 /*
665 * If apic_coarse_time == 1, then apic_gettime() is used instead of
666 * apic_gethrtime(). This is used for performance instead of accuracy.
667 */
668
669 hrtime_t
670 apic_gettime()
671 {
672 int old_hrtime_stamp;
673 hrtime_t temp;
674
675 /*
676 * In one-shot mode, we do not keep time, so if anyone
677 * calls psm_gettime() directly, we vector over to
678 * gethrtime().
679 * one-shot mode MUST NOT be enabled if this psm is the source of
680 * hrtime.
681 */
682
683 if (apic_oneshot)
684 return (gethrtime());
685
686
687 gettime_again:
688 while ((old_hrtime_stamp = apic_hrtime_stamp) & 1)
689 apic_ret();
690
691 temp = apic_nsec_since_boot;
692
693 if (apic_hrtime_stamp != old_hrtime_stamp) { /* got an interrupt */
694 goto gettime_again;
695 }
696 return (temp);
697 }
698
699 /*
700 * Here we return the number of nanoseconds since booting. Note every
701 * clock interrupt increments apic_nsec_since_boot by the appropriate
702 * amount.
703 */
704 hrtime_t
705 apic_gethrtime(void)
706 {
707 int curr_timeval, countval, elapsed_ticks;
708 int old_hrtime_stamp, status;
709 hrtime_t temp;
710 uint32_t cpun;
711 ulong_t oflags;
712
713 /*
714 * In one-shot mode, we do not keep time, so if anyone
715 * calls psm_gethrtime() directly, we vector over to
716 * gethrtime().
717 * one-shot mode MUST NOT be enabled if this psm is the source of
718 * hrtime.
719 */
720
721 if (apic_oneshot)
722 return (gethrtime());
723
724 oflags = intr_clear(); /* prevent migration */
725
726 cpun = apic_reg_ops->apic_read(APIC_LID_REG);
727 if (apic_mode == LOCAL_APIC)
728 cpun >>= APIC_ID_BIT_OFFSET;
729
730 lock_set(&apic_gethrtime_lock);
731
732 gethrtime_again:
733 while ((old_hrtime_stamp = apic_hrtime_stamp) & 1)
734 apic_ret();
735
736 /*
737 * Check to see which CPU we are on. Note the time is kept on
738 * the local APIC of CPU 0. If on CPU 0, simply read the current
739 * counter. If on another CPU, issue a remote read command to CPU 0.
740 */
741 if (cpun == apic_cpus[0].aci_local_id) {
742 countval = apic_reg_ops->apic_read(APIC_CURR_COUNT);
743 } else {
744 #ifdef DEBUG
745 APIC_AV_PENDING_SET();
746 #else
747 if (apic_mode == LOCAL_APIC)
748 APIC_AV_PENDING_SET();
749 #endif /* DEBUG */
750
751 apic_reg_ops->apic_write_int_cmd(
752 apic_cpus[0].aci_local_id, APIC_CURR_ADD | AV_REMOTE);
753
754 while ((status = apic_reg_ops->apic_read(APIC_INT_CMD1))
755 & AV_READ_PENDING) {
756 apic_ret();
757 }
758
759 if (status & AV_REMOTE_STATUS) /* 1 = valid */
760 countval = apic_reg_ops->apic_read(APIC_REMOTE_READ);
761 else { /* 0 = invalid */
762 apic_remote_hrterr++;
763 /*
764 * return last hrtime right now, will need more
765 * testing if change to retry
766 */
767 temp = apic_last_hrtime;
768
769 lock_clear(&apic_gethrtime_lock);
770
771 intr_restore(oflags);
772
773 return (temp);
774 }
775 }
776 if (countval > last_count_read)
777 countval = 0;
778 else
779 last_count_read = countval;
780
781 elapsed_ticks = apic_hertz_count - countval;
782
783 curr_timeval = APIC_TICKS_TO_NSECS(elapsed_ticks);
784 temp = apic_nsec_since_boot + curr_timeval;
785
786 if (apic_hrtime_stamp != old_hrtime_stamp) { /* got an interrupt */
787 /* we might have clobbered last_count_read. Restore it */
788 last_count_read = apic_hertz_count;
789 goto gethrtime_again;
790 }
791
792 if (temp < apic_last_hrtime) {
793 /* return last hrtime if error occurs */
794 apic_hrtime_error++;
795 temp = apic_last_hrtime;
796 }
797 else
798 apic_last_hrtime = temp;
799
800 lock_clear(&apic_gethrtime_lock);
801 intr_restore(oflags);
802
803 return (temp);
804 }
805
806 /* apic NMI handler */
807 /*ARGSUSED*/
808 void
809 apic_nmi_intr(caddr_t arg, struct regs *rp)
810 {
811 if (apic_shutdown_processors) {
812 apic_disable_local_apic();
813 return;
814 }
815
816 apic_error |= APIC_ERR_NMI;
817
818 if (!lock_try(&apic_nmi_lock))
819 return;
820 apic_num_nmis++;
821
822 if (apic_kmdb_on_nmi && psm_debugger()) {
823 debug_enter("NMI received: entering kmdb\n");
824 } else if (apic_panic_on_nmi) {
825 /* Keep panic from entering kmdb. */
826 nopanicdebug = 1;
827 panic("NMI received\n");
828 } else {
829 /*
830 * prom_printf is the best shot we have of something which is
831 * problem free from high level/NMI type of interrupts
832 */
833 prom_printf("NMI received\n");
834 }
835
836 lock_clear(&apic_nmi_lock);
837 }
838
839 processorid_t
840 apic_get_next_processorid(processorid_t cpu_id)
841 {
842
843 int i;
844
845 if (cpu_id == -1)
846 return ((processorid_t)0);
847
848 for (i = cpu_id + 1; i < NCPU; i++) {
849 if (apic_cpu_in_range(i))
850 return (i);
851 }
852
853 return ((processorid_t)-1);
854 }
855
856 int
857 apic_cpu_add(psm_cpu_request_t *reqp)
858 {
859 int i, rv = 0;
860 ulong_t iflag;
861 boolean_t first = B_TRUE;
862 uchar_t localver = 0;
863 uint32_t localid, procid;
864 processorid_t cpuid = (processorid_t)-1;
865 mach_cpu_add_arg_t *ap;
866
867 ASSERT(reqp != NULL);
868 reqp->req.cpu_add.cpuid = (processorid_t)-1;
869
870 /* Check whether CPU hotplug is supported. */
871 if (!plat_dr_support_cpu() || apic_max_nproc == -1) {
872 return (ENOTSUP);
873 }
874
875 ap = (mach_cpu_add_arg_t *)reqp->req.cpu_add.argp;
876 switch (ap->type) {
877 case MACH_CPU_ARG_LOCAL_APIC:
878 localid = ap->arg.apic.apic_id;
879 procid = ap->arg.apic.proc_id;
880 if (localid >= 255 || procid > 255) {
881 cmn_err(CE_WARN,
882 "!apic: apicid(%u) or procid(%u) is invalid.",
883 localid, procid);
884 return (EINVAL);
885 }
886 break;
887
888 case MACH_CPU_ARG_LOCAL_X2APIC:
889 localid = ap->arg.apic.apic_id;
890 procid = ap->arg.apic.proc_id;
891 if (localid >= UINT32_MAX) {
892 cmn_err(CE_WARN,
893 "!apic: x2apicid(%u) is invalid.", localid);
894 return (EINVAL);
895 } else if (localid >= 255 && apic_mode == LOCAL_APIC) {
896 cmn_err(CE_WARN, "!apic: system is in APIC mode, "
897 "can't support x2APIC processor.");
898 return (ENOTSUP);
899 }
900 break;
901
902 default:
903 cmn_err(CE_WARN,
904 "!apic: unknown argument type %d to apic_cpu_add().",
905 ap->type);
906 return (EINVAL);
907 }
908
909 /* Use apic_ioapic_lock to sync with apic_get_next_bind_cpu. */
910 iflag = intr_clear();
911 lock_set(&apic_ioapic_lock);
912
913 /* Check whether local APIC id already exists. */
914 for (i = 0; i < apic_nproc; i++) {
915 if (!CPU_IN_SET(apic_cpumask, i))
916 continue;
917 if (apic_cpus[i].aci_local_id == localid) {
918 lock_clear(&apic_ioapic_lock);
919 intr_restore(iflag);
920 cmn_err(CE_WARN,
921 "!apic: local apic id %u already exists.",
922 localid);
923 return (EEXIST);
924 } else if (apic_cpus[i].aci_processor_id == procid) {
925 lock_clear(&apic_ioapic_lock);
926 intr_restore(iflag);
927 cmn_err(CE_WARN,
928 "!apic: processor id %u already exists.",
929 (int)procid);
930 return (EEXIST);
931 }
932
933 /*
934 * There's no local APIC version number available in MADT table,
935 * so assume that all CPUs are homogeneous and use local APIC
936 * version number of the first existing CPU.
937 */
938 if (first) {
939 first = B_FALSE;
940 localver = apic_cpus[i].aci_local_ver;
941 }
942 }
943 ASSERT(first == B_FALSE);
944
945 /*
946 * Try to assign the same cpuid if APIC id exists in the dirty cache.
947 */
948 for (i = 0; i < apic_max_nproc; i++) {
949 if (CPU_IN_SET(apic_cpumask, i)) {
950 ASSERT((apic_cpus[i].aci_status & APIC_CPU_FREE) == 0);
951 continue;
952 }
953 ASSERT(apic_cpus[i].aci_status & APIC_CPU_FREE);
954 if ((apic_cpus[i].aci_status & APIC_CPU_DIRTY) &&
955 apic_cpus[i].aci_local_id == localid &&
956 apic_cpus[i].aci_processor_id == procid) {
957 cpuid = i;
958 break;
959 }
960 }
961
962 /* Avoid the dirty cache and allocate fresh slot if possible. */
963 if (cpuid == (processorid_t)-1) {
964 for (i = 0; i < apic_max_nproc; i++) {
965 if ((apic_cpus[i].aci_status & APIC_CPU_FREE) &&
966 (apic_cpus[i].aci_status & APIC_CPU_DIRTY) == 0) {
967 cpuid = i;
968 break;
969 }
970 }
971 }
972
973 /* Try to find any free slot as last resort. */
974 if (cpuid == (processorid_t)-1) {
975 for (i = 0; i < apic_max_nproc; i++) {
976 if (apic_cpus[i].aci_status & APIC_CPU_FREE) {
977 cpuid = i;
978 break;
979 }
980 }
981 }
982
983 if (cpuid == (processorid_t)-1) {
984 lock_clear(&apic_ioapic_lock);
985 intr_restore(iflag);
986 cmn_err(CE_NOTE,
987 "!apic: failed to allocate cpu id for processor %u.",
988 procid);
989 rv = EAGAIN;
990 } else if (ACPI_FAILURE(acpica_map_cpu(cpuid, procid))) {
991 lock_clear(&apic_ioapic_lock);
992 intr_restore(iflag);
993 cmn_err(CE_NOTE,
994 "!apic: failed to build mapping for processor %u.",
995 procid);
996 rv = EBUSY;
997 } else {
998 ASSERT(cpuid >= 0 && cpuid < NCPU);
999 ASSERT(cpuid < apic_max_nproc && cpuid < max_ncpus);
1000 bzero(&apic_cpus[cpuid], sizeof (apic_cpus[0]));
1001 apic_cpus[cpuid].aci_processor_id = procid;
1002 apic_cpus[cpuid].aci_local_id = localid;
1003 apic_cpus[cpuid].aci_local_ver = localver;
1004 CPUSET_ATOMIC_ADD(apic_cpumask, cpuid);
1005 if (cpuid >= apic_nproc) {
1006 apic_nproc = cpuid + 1;
1007 }
1008 lock_clear(&apic_ioapic_lock);
1009 intr_restore(iflag);
1010 reqp->req.cpu_add.cpuid = cpuid;
1011 }
1012
1013 return (rv);
1014 }
1015
1016 int
1017 apic_cpu_remove(psm_cpu_request_t *reqp)
1018 {
1019 int i;
1020 ulong_t iflag;
1021 processorid_t cpuid;
1022
1023 /* Check whether CPU hotplug is supported. */
1024 if (!plat_dr_support_cpu() || apic_max_nproc == -1) {
1025 return (ENOTSUP);
1026 }
1027
1028 cpuid = reqp->req.cpu_remove.cpuid;
1029
1030 /* Use apic_ioapic_lock to sync with apic_get_next_bind_cpu. */
1031 iflag = intr_clear();
1032 lock_set(&apic_ioapic_lock);
1033
1034 if (!apic_cpu_in_range(cpuid)) {
1035 lock_clear(&apic_ioapic_lock);
1036 intr_restore(iflag);
1037 cmn_err(CE_WARN,
1038 "!apic: cpuid %d doesn't exist in apic_cpus array.",
1039 cpuid);
1040 return (ENODEV);
1041 }
1042 ASSERT((apic_cpus[cpuid].aci_status & APIC_CPU_FREE) == 0);
1043
1044 if (ACPI_FAILURE(acpica_unmap_cpu(cpuid))) {
1045 lock_clear(&apic_ioapic_lock);
1046 intr_restore(iflag);
1047 return (ENOENT);
1048 }
1049
1050 if (cpuid == apic_nproc - 1) {
1051 /*
1052 * We are removing the highest numbered cpuid so we need to
1053 * find the next highest cpuid as the new value for apic_nproc.
1054 */
1055 for (i = apic_nproc; i > 0; i--) {
1056 if (CPU_IN_SET(apic_cpumask, i - 1)) {
1057 apic_nproc = i;
1058 break;
1059 }
1060 }
1061 /* at least one CPU left */
1062 ASSERT(i > 0);
1063 }
1064 CPUSET_ATOMIC_DEL(apic_cpumask, cpuid);
1065 /* mark slot as free and keep it in the dirty cache */
1066 apic_cpus[cpuid].aci_status = APIC_CPU_FREE | APIC_CPU_DIRTY;
1067
1068 lock_clear(&apic_ioapic_lock);
1069 intr_restore(iflag);
1070
1071 return (0);
1072 }
1073
1074 /*
1075 * Return the number of ticks the APIC decrements in SF nanoseconds.
1076 * The fixed-frequency PIT (aka 8254) is used for the measurement.
1077 */
1078 static uint64_t
1079 apic_calibrate_impl()
1080 {
1081 uint8_t pit_tick_lo;
1082 uint16_t pit_tick, target_pit_tick, pit_ticks_adj;
1083 uint32_t pit_ticks;
1084 uint32_t start_apic_tick, end_apic_tick, apic_ticks;
1085 ulong_t iflag;
1086
1087 apic_reg_ops->apic_write(APIC_DIVIDE_REG, apic_divide_reg_init);
1088 apic_reg_ops->apic_write(APIC_INIT_COUNT, APIC_MAXVAL);
1089
1090 iflag = intr_clear();
1091
1092 do {
1093 pit_tick_lo = inb(PITCTR0_PORT);
1094 pit_tick = (inb(PITCTR0_PORT) << 8) | pit_tick_lo;
1095 } while (pit_tick < APIC_TIME_MIN ||
1096 pit_tick_lo <= APIC_LB_MIN || pit_tick_lo >= APIC_LB_MAX);
1097
1098 /*
1099 * Wait for the PIT to decrement by 5 ticks to ensure
1100 * we didn't start in the middle of a tick.
1101 * Compare with 0x10 for the wrap around case.
1102 */
1103 target_pit_tick = pit_tick - 5;
1104 do {
1105 pit_tick_lo = inb(PITCTR0_PORT);
1106 pit_tick = (inb(PITCTR0_PORT) << 8) | pit_tick_lo;
1107 } while (pit_tick > target_pit_tick || pit_tick_lo < 0x10);
1108
1109 start_apic_tick = apic_reg_ops->apic_read(APIC_CURR_COUNT);
1110
1111 /*
1112 * Wait for the PIT to decrement by APIC_TIME_COUNT ticks
1113 */
1114 target_pit_tick = pit_tick - APIC_TIME_COUNT;
1115 do {
1116 pit_tick_lo = inb(PITCTR0_PORT);
1117 pit_tick = (inb(PITCTR0_PORT) << 8) | pit_tick_lo;
1118 } while (pit_tick > target_pit_tick || pit_tick_lo < 0x10);
1119
1120 end_apic_tick = apic_reg_ops->apic_read(APIC_CURR_COUNT);
1121
1122 intr_restore(iflag);
1123
1124 apic_ticks = start_apic_tick - end_apic_tick;
1125
1126 /* The PIT might have decremented by more ticks than planned */
1127 pit_ticks_adj = target_pit_tick - pit_tick;
1128 /* total number of PIT ticks corresponding to apic_ticks */
1129 pit_ticks = APIC_TIME_COUNT + pit_ticks_adj;
1130
1131 /*
1132 * Determine the number of nanoseconds per APIC clock tick
1133 * and then determine how many APIC ticks to interrupt at the
1134 * desired frequency
1135 * apic_ticks / (pitticks / PIT_HZ) = apic_ticks_per_s
1136 * (apic_ticks * PIT_HZ) / pitticks = apic_ticks_per_s
1137 * apic_ticks_per_ns = (apic_ticks * PIT_HZ) / (pitticks * 10^9)
1138 * apic_ticks_per_SFns =
1139 * (SF * apic_ticks * PIT_HZ) / (pitticks * 10^9)
1140 */
1141 return ((SF * apic_ticks * PIT_HZ) / ((uint64_t)pit_ticks * NANOSEC));
1142 }
1143
1144 /*
1145 * It was found empirically that 5 measurements seem sufficient to give a good
1146 * accuracy. Most spurious measurements are higher than the target value thus
1147 * we eliminate up to 2/5 spurious measurements.
1148 */
1149 #define APIC_CALIBRATE_MEASUREMENTS 5
1150
1151 #define APIC_CALIBRATE_PERCENT_OFF_WARNING 10
1152
1153 /*
1154 * Return the number of ticks the APIC decrements in SF nanoseconds.
1155 * Several measurements are taken to filter out outliers.
1156 */
1157 uint64_t
1158 apic_calibrate()
1159 {
1160 uint64_t measurements[APIC_CALIBRATE_MEASUREMENTS];
1161 int median_idx;
1162 uint64_t median;
1163
1164 /*
1165 * When running under a virtual machine, the emulated PIT and APIC
1166 * counters do not always return the right values and can roll over.
1167 * Those spurious measurements are relatively rare but could
1168 * significantly affect the calibration.
1169 * Therefore we take several measurements and then keep the median.
1170 * The median is preferred to the average here as we only want to
1171 * discard outliers.
1172 */
1173 for (int i = 0; i < APIC_CALIBRATE_MEASUREMENTS; i++)
1174 measurements[i] = apic_calibrate_impl();
1175
1176 /*
1177 * sort results and retrieve median.
1178 */
1179 for (int i = 0; i < APIC_CALIBRATE_MEASUREMENTS; i++) {
1180 for (int j = i + 1; j < APIC_CALIBRATE_MEASUREMENTS; j++) {
1181 if (measurements[j] < measurements[i]) {
1182 uint64_t tmp = measurements[i];
1183 measurements[i] = measurements[j];
1184 measurements[j] = tmp;
1185 }
1186 }
1187 }
1188 median_idx = APIC_CALIBRATE_MEASUREMENTS / 2;
1189 median = measurements[median_idx];
1190
1191 #if (APIC_CALIBRATE_MEASUREMENTS >= 3)
1192 /*
1193 * Check that measurements are consistent. Post a warning
1194 * if the three middle values are not close to each other.
1195 */
1196 uint64_t delta_warn = median *
1197 APIC_CALIBRATE_PERCENT_OFF_WARNING / 100;
1198 if ((median - measurements[median_idx - 1]) > delta_warn ||
1199 (measurements[median_idx + 1] - median) > delta_warn) {
1200 cmn_err(CE_WARN, "apic_calibrate measurements lack "
1201 "precision: %llu, %llu, %llu.",
1202 (u_longlong_t)measurements[median_idx - 1],
1203 (u_longlong_t)median,
1204 (u_longlong_t)measurements[median_idx + 1]);
1205 }
1206 #endif
1207
1208 return (median);
1209 }
1210
1211 /*
1212 * Initialise the APIC timer on the local APIC of CPU 0 to the desired
1213 * frequency. Note at this stage in the boot sequence, the boot processor
1214 * is the only active processor.
1215 * hertz value of 0 indicates a one-shot mode request. In this case
1216 * the function returns the resolution (in nanoseconds) for the hardware
1217 * timer interrupt. If one-shot mode capability is not available,
1218 * the return value will be 0. apic_enable_oneshot is a global switch
1219 * for disabling the functionality.
1220 * A non-zero positive value for hertz indicates a periodic mode request.
1221 * In this case the hardware will be programmed to generate clock interrupts
1222 * at hertz frequency and returns the resolution of interrupts in
1223 * nanosecond.
1224 */
1225
1226 int
1227 apic_clkinit(int hertz)
1228 {
1229 int ret;
1230
1231 apic_int_busy_mark = (apic_int_busy_mark *
1232 apic_sample_factor_redistribution) / 100;
1233 apic_int_free_mark = (apic_int_free_mark *
1234 apic_sample_factor_redistribution) / 100;
1235 apic_diff_for_redistribution = (apic_diff_for_redistribution *
1236 apic_sample_factor_redistribution) / 100;
1237
1238 ret = apic_timer_init(hertz);
1239 return (ret);
1240
1241 }
1242
1243 /*
1244 * apic_preshutdown:
1245 * Called early in shutdown whilst we can still access filesystems to do
1246 * things like loading modules which will be required to complete shutdown
1247 * after filesystems are all unmounted.
1248 */
1249 void
1250 apic_preshutdown(int cmd, int fcn)
1251 {
1252 APIC_VERBOSE_POWEROFF(("apic_preshutdown(%d,%d); m=%d a=%d\n",
1253 cmd, fcn, apic_poweroff_method, apic_enable_acpi));
1254 }
1255
1256 void
1257 apic_shutdown(int cmd, int fcn)
1258 {
1259 int restarts, attempts;
1260 int i;
1261 uchar_t byte;
1262 ulong_t iflag;
1263
1264 hpet_acpi_fini();
1265
1266 /* Send NMI to all CPUs except self to do per processor shutdown */
1267 iflag = intr_clear();
1268 #ifdef DEBUG
1269 APIC_AV_PENDING_SET();
1270 #else
1271 if (apic_mode == LOCAL_APIC)
1272 APIC_AV_PENDING_SET();
1273 #endif /* DEBUG */
1274 apic_shutdown_processors = 1;
1275 apic_reg_ops->apic_write(APIC_INT_CMD1,
1276 AV_NMI | AV_LEVEL | AV_SH_ALL_EXCSELF);
1277
1278 /* restore cmos shutdown byte before reboot */
1279 if (apic_cmos_ssb_set) {
1280 outb(CMOS_ADDR, SSB);
1281 outb(CMOS_DATA, 0);
1282 }
1283
1284 ioapic_disable_redirection();
1285
1286 /* disable apic mode if imcr present */
1287 if (apic_imcrp) {
1288 outb(APIC_IMCR_P1, (uchar_t)APIC_IMCR_SELECT);
1289 outb(APIC_IMCR_P2, (uchar_t)APIC_IMCR_PIC);
1290 }
1291
1292 apic_disable_local_apic();
1293
1294 intr_restore(iflag);
1295
1296 /* remainder of function is for shutdown cases only */
1297 if (cmd != A_SHUTDOWN)
1298 return;
1299
1300 /*
1301 * Switch system back into Legacy-Mode if using ACPI and
1302 * not powering-off. Some BIOSes need to remain in ACPI-mode
1303 * for power-off to succeed (Dell Dimension 4600)
1304 * Do not disable ACPI while doing fastreboot
1305 */
1306 if (apic_enable_acpi && fcn != AD_POWEROFF && fcn != AD_FASTREBOOT)
1307 (void) AcpiDisable();
1308
1309 if (fcn == AD_FASTREBOOT) {
1310 apic_reg_ops->apic_write(APIC_INT_CMD1,
1311 AV_ASSERT | AV_RESET | AV_SH_ALL_EXCSELF);
1312 }
1313
1314 /* remainder of function is for shutdown+poweroff case only */
1315 if (fcn != AD_POWEROFF)
1316 return;
1317
1318 switch (apic_poweroff_method) {
1319 case APIC_POWEROFF_VIA_RTC:
1320
1321 /* select the extended NVRAM bank in the RTC */
1322 outb(CMOS_ADDR, RTC_REGA);
1323 byte = inb(CMOS_DATA);
1324 outb(CMOS_DATA, (byte | EXT_BANK));
1325
1326 outb(CMOS_ADDR, PFR_REG);
1327
1328 /* for Predator must toggle the PAB bit */
1329 byte = inb(CMOS_DATA);
1330
1331 /*
1332 * clear power active bar, wakeup alarm and
1333 * kickstart
1334 */
1335 byte &= ~(PAB_CBIT | WF_FLAG | KS_FLAG);
1336 outb(CMOS_DATA, byte);
1337
1338 /* delay before next write */
1339 drv_usecwait(1000);
1340
1341 /* for S40 the following would suffice */
1342 byte = inb(CMOS_DATA);
1343
1344 /* power active bar control bit */
1345 byte |= PAB_CBIT;
1346 outb(CMOS_DATA, byte);
1347
1348 break;
1349
1350 case APIC_POWEROFF_VIA_ASPEN_BMC:
1351 restarts = 0;
1352 restart_aspen_bmc:
1353 if (++restarts == 3)
1354 break;
1355 attempts = 0;
1356 do {
1357 byte = inb(MISMIC_FLAG_REGISTER);
1358 byte &= MISMIC_BUSY_MASK;
1359 if (byte != 0) {
1360 drv_usecwait(1000);
1361 if (attempts >= 3)
1362 goto restart_aspen_bmc;
1363 ++attempts;
1364 }
1365 } while (byte != 0);
1366 outb(MISMIC_CNTL_REGISTER, CC_SMS_GET_STATUS);
1367 byte = inb(MISMIC_FLAG_REGISTER);
1368 byte |= 0x1;
1369 outb(MISMIC_FLAG_REGISTER, byte);
1370 i = 0;
1371 for (; i < (sizeof (aspen_bmc)/sizeof (aspen_bmc[0]));
1372 i++) {
1373 attempts = 0;
1374 do {
1375 byte = inb(MISMIC_FLAG_REGISTER);
1376 byte &= MISMIC_BUSY_MASK;
1377 if (byte != 0) {
1378 drv_usecwait(1000);
1379 if (attempts >= 3)
1380 goto restart_aspen_bmc;
1381 ++attempts;
1382 }
1383 } while (byte != 0);
1384 outb(MISMIC_CNTL_REGISTER, aspen_bmc[i].cntl);
1385 outb(MISMIC_DATA_REGISTER, aspen_bmc[i].data);
1386 byte = inb(MISMIC_FLAG_REGISTER);
1387 byte |= 0x1;
1388 outb(MISMIC_FLAG_REGISTER, byte);
1389 }
1390 break;
1391
1392 case APIC_POWEROFF_VIA_SITKA_BMC:
1393 restarts = 0;
1394 restart_sitka_bmc:
1395 if (++restarts == 3)
1396 break;
1397 attempts = 0;
1398 do {
1399 byte = inb(SMS_STATUS_REGISTER);
1400 byte &= SMS_STATE_MASK;
1401 if ((byte == SMS_READ_STATE) ||
1402 (byte == SMS_WRITE_STATE)) {
1403 drv_usecwait(1000);
1404 if (attempts >= 3)
1405 goto restart_sitka_bmc;
1406 ++attempts;
1407 }
1408 } while ((byte == SMS_READ_STATE) ||
1409 (byte == SMS_WRITE_STATE));
1410 outb(SMS_COMMAND_REGISTER, SMS_GET_STATUS);
1411 i = 0;
1412 for (; i < (sizeof (sitka_bmc)/sizeof (sitka_bmc[0]));
1413 i++) {
1414 attempts = 0;
1415 do {
1416 byte = inb(SMS_STATUS_REGISTER);
1417 byte &= SMS_IBF_MASK;
1418 if (byte != 0) {
1419 drv_usecwait(1000);
1420 if (attempts >= 3)
1421 goto restart_sitka_bmc;
1422 ++attempts;
1423 }
1424 } while (byte != 0);
1425 outb(sitka_bmc[i].port, sitka_bmc[i].data);
1426 }
1427 break;
1428
1429 case APIC_POWEROFF_NONE:
1430
1431 /* If no APIC direct method, we will try using ACPI */
1432 if (apic_enable_acpi) {
1433 if (acpi_poweroff() == 1)
1434 return;
1435 } else
1436 return;
1437
1438 break;
1439 }
1440 /*
1441 * Wait a limited time here for power to go off.
1442 * If the power does not go off, then there was a
1443 * problem and we should continue to the halt which
1444 * prints a message for the user to press a key to
1445 * reboot.
1446 */
1447 drv_usecwait(7000000); /* wait seven seconds */
1448
1449 }
1450
1451 cyclic_id_t apic_cyclic_id;
1452
1453 /*
1454 * The following functions are in the platform specific file so that they
1455 * can be different functions depending on whether we are running on
1456 * bare metal or a hypervisor.
1457 */
1458
1459 /*
1460 * map an apic for memory-mapped access
1461 */
1462 uint32_t *
1463 mapin_apic(uint32_t addr, size_t len, int flags)
1464 {
1465 return ((void *)psm_map_phys(addr, len, flags));
1466 }
1467
1468 uint32_t *
1469 mapin_ioapic(uint32_t addr, size_t len, int flags)
1470 {
1471 return (mapin_apic(addr, len, flags));
1472 }
1473
1474 /*
1475 * unmap an apic
1476 */
1477 void
1478 mapout_apic(caddr_t addr, size_t len)
1479 {
1480 psm_unmap_phys(addr, len);
1481 }
1482
1483 void
1484 mapout_ioapic(caddr_t addr, size_t len)
1485 {
1486 mapout_apic(addr, len);
1487 }
1488
1489 uint32_t
1490 ioapic_read(int ioapic_ix, uint32_t reg)
1491 {
1492 volatile uint32_t *ioapic;
1493
1494 ioapic = apicioadr[ioapic_ix];
1495 ioapic[APIC_IO_REG] = reg;
1496 return (ioapic[APIC_IO_DATA]);
1497 }
1498
1499 void
1500 ioapic_write(int ioapic_ix, uint32_t reg, uint32_t value)
1501 {
1502 volatile uint32_t *ioapic;
1503
1504 ioapic = apicioadr[ioapic_ix];
1505 ioapic[APIC_IO_REG] = reg;
1506 ioapic[APIC_IO_DATA] = value;
1507 }
1508
1509 void
1510 ioapic_write_eoi(int ioapic_ix, uint32_t value)
1511 {
1512 volatile uint32_t *ioapic;
1513
1514 ioapic = apicioadr[ioapic_ix];
1515 ioapic[APIC_IO_EOI] = value;
1516 }
1517
1518 /*
1519 * Round-robin algorithm to find the next CPU with interrupts enabled.
1520 * It can't share the same static variable apic_next_bind_cpu with
1521 * apic_get_next_bind_cpu(), since that will cause all interrupts to be
1522 * bound to CPU1 at boot time. During boot, only CPU0 is online with
1523 * interrupts enabled when apic_get_next_bind_cpu() and apic_find_cpu()
1524 * are called. However, the pcplusmp driver assumes that there will be
1525 * boot_ncpus CPUs configured eventually so it tries to distribute all
1526 * interrupts among CPU0 - CPU[boot_ncpus - 1]. Thus to prevent all
1527 * interrupts being targetted at CPU1, we need to use a dedicated static
1528 * variable for find_next_cpu() instead of sharing apic_next_bind_cpu.
1529 */
1530
1531 processorid_t
1532 apic_find_cpu(int flag)
1533 {
1534 int i;
1535 static processorid_t acid = 0;
1536
1537 /* Find the first CPU with the passed-in flag set */
1538 for (i = 0; i < apic_nproc; i++) {
1539 if (++acid >= apic_nproc) {
1540 acid = 0;
1541 }
1542 if (apic_cpu_in_range(acid) &&
1543 (apic_cpus[acid].aci_status & flag)) {
1544 break;
1545 }
1546 }
1547
1548 ASSERT((apic_cpus[acid].aci_status & flag) != 0);
1549 return (acid);
1550 }
1551
1552 void
1553 apic_intrmap_init(int apic_mode)
1554 {
1555 int suppress_brdcst_eoi = 0;
1556
1557 /*
1558 * Intel Software Developer's Manual 3A, 10.12.7:
1559 *
1560 * Routing of device interrupts to local APIC units operating in
1561 * x2APIC mode requires use of the interrupt-remapping architecture
1562 * specified in the Intel Virtualization Technology for Directed
1563 * I/O, Revision 1.3. Because of this, BIOS must enumerate support
1564 * for and software must enable this interrupt remapping with
1565 * Extended Interrupt Mode Enabled before it enabling x2APIC mode in
1566 * the local APIC units.
1567 *
1568 *
1569 * In other words, to use the APIC in x2APIC mode, we need interrupt
1570 * remapping. Since we don't start up the IOMMU by default, we
1571 * won't be able to do any interrupt remapping and therefore have to
1572 * use the APIC in traditional 'local APIC' mode with memory mapped
1573 * I/O.
1574 */
1575
1576 if (psm_vt_ops != NULL) {
1577 if (((apic_intrmap_ops_t *)psm_vt_ops)->
1578 apic_intrmap_init(apic_mode) == DDI_SUCCESS) {
1579
1580 apic_vt_ops = psm_vt_ops;
1581
1582 /*
1583 * We leverage the interrupt remapping engine to
1584 * suppress broadcast EOI; thus we must send the
1585 * directed EOI with the directed-EOI handler.
1586 */
1587 if (apic_directed_EOI_supported() == 0) {
1588 suppress_brdcst_eoi = 1;
1589 }
1590
1591 apic_vt_ops->apic_intrmap_enable(suppress_brdcst_eoi);
1592
1593 if (apic_detect_x2apic()) {
1594 apic_enable_x2apic();
1595 }
1596
1597 if (apic_directed_EOI_supported() == 0) {
1598 apic_set_directed_EOI_handler();
1599 }
1600 }
1601 }
1602 }
1603
1604 /*ARGSUSED*/
1605 static void
1606 apic_record_ioapic_rdt(void *intrmap_private, ioapic_rdt_t *irdt)
1607 {
1608 irdt->ir_hi <<= APIC_ID_BIT_OFFSET;
1609 }
1610
1611 /*ARGSUSED*/
1612 static void
1613 apic_record_msi(void *intrmap_private, msi_regs_t *mregs)
1614 {
1615 mregs->mr_addr = MSI_ADDR_HDR |
1616 (MSI_ADDR_RH_FIXED << MSI_ADDR_RH_SHIFT) |
1617 (MSI_ADDR_DM_PHYSICAL << MSI_ADDR_DM_SHIFT) |
1618 (mregs->mr_addr << MSI_ADDR_DEST_SHIFT);
1619 mregs->mr_data = (MSI_DATA_TM_EDGE << MSI_DATA_TM_SHIFT) |
1620 mregs->mr_data;
1621 }
1622
1623 /*
1624 * Functions from apic_introp.c
1625 *
1626 * Those functions are used by apic_intr_ops().
1627 */
1628
1629 /*
1630 * MSI support flag:
1631 * reflects whether MSI is supported at APIC level
1632 * it can also be patched through /etc/system
1633 *
1634 * 0 = default value - don't know and need to call apic_check_msi_support()
1635 * to find out then set it accordingly
1636 * 1 = supported
1637 * -1 = not supported
1638 */
1639 int apic_support_msi = 0;
1640
1641 /* Multiple vector support for MSI-X */
1642 int apic_msix_enable = 1;
1643
1644 /* Multiple vector support for MSI */
1645 int apic_multi_msi_enable = 1;
1646
1647 /*
1648 * Check whether the system supports MSI.
1649 *
1650 * MSI is required for PCI-E and for PCI versions later than 2.2, so if we find
1651 * a PCI-E bus or we find a PCI bus whose version we know is >= 2.2, then we
1652 * return PSM_SUCCESS to indicate this system supports MSI.
1653 *
1654 * (Currently the only way we check whether a given PCI bus supports >= 2.2 is
1655 * by detecting if we are running inside the KVM hypervisor, which guarantees
1656 * this version number.)
1657 */
1658 int
1659 apic_check_msi_support()
1660 {
1661 dev_info_t *cdip;
1662 char dev_type[16];
1663 int dev_len;
1664 int hwenv = get_hwenv();
1665
1666 DDI_INTR_IMPLDBG((CE_CONT, "apic_check_msi_support:\n"));
1667
1668 /*
1669 * check whether the first level children of root_node have
1670 * PCI-E or PCI capability.
1671 */
1672 for (cdip = ddi_get_child(ddi_root_node()); cdip != NULL;
1673 cdip = ddi_get_next_sibling(cdip)) {
1674
1675 DDI_INTR_IMPLDBG((CE_CONT, "apic_check_msi_support: cdip: 0x%p,"
1676 " driver: %s, binding: %s, nodename: %s\n", (void *)cdip,
1677 ddi_driver_name(cdip), ddi_binding_name(cdip),
1678 ddi_node_name(cdip)));
1679 dev_len = sizeof (dev_type);
1680 if (ddi_getlongprop_buf(DDI_DEV_T_ANY, cdip, DDI_PROP_DONTPASS,
1681 "device_type", (caddr_t)dev_type, &dev_len)
1682 != DDI_PROP_SUCCESS)
1683 continue;
1684 if (strcmp(dev_type, "pciex") == 0)
1685 return (PSM_SUCCESS);
1686 if (strcmp(dev_type, "pci") == 0 &&
1687 (hwenv == HW_KVM || hwenv == HW_BHYVE))
1688 return (PSM_SUCCESS);
1689 }
1690
1691 /* MSI is not supported on this system */
1692 DDI_INTR_IMPLDBG((CE_CONT, "apic_check_msi_support: no 'pciex' "
1693 "device_type found\n"));
1694 return (PSM_FAILURE);
1695 }
1696
1697 /*
1698 * apic_pci_msi_unconfigure:
1699 *
1700 * This and next two interfaces are copied from pci_intr_lib.c
1701 * Do ensure that these two files stay in sync.
1702 * These needed to be copied over here to avoid a deadlock situation on
1703 * certain mp systems that use MSI interrupts.
1704 *
1705 * IMPORTANT regards next three interfaces:
1706 * i) are called only for MSI/X interrupts.
1707 * ii) called with interrupts disabled, and must not block
1708 */
1709 void
1710 apic_pci_msi_unconfigure(dev_info_t *rdip, int type, int inum)
1711 {
1712 ushort_t msi_ctrl;
1713 int cap_ptr = i_ddi_get_msi_msix_cap_ptr(rdip);
1714 ddi_acc_handle_t handle = i_ddi_get_pci_config_handle(rdip);
1715
1716 ASSERT((handle != NULL) && (cap_ptr != 0));
1717
1718 if (type == DDI_INTR_TYPE_MSI) {
1719 msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSI_CTRL);
1720 msi_ctrl &= (~PCI_MSI_MME_MASK);
1721 pci_config_put16(handle, cap_ptr + PCI_MSI_CTRL, msi_ctrl);
1722 pci_config_put32(handle, cap_ptr + PCI_MSI_ADDR_OFFSET, 0);
1723
1724 if (msi_ctrl & PCI_MSI_64BIT_MASK) {
1725 pci_config_put16(handle,
1726 cap_ptr + PCI_MSI_64BIT_DATA, 0);
1727 pci_config_put32(handle,
1728 cap_ptr + PCI_MSI_ADDR_OFFSET + 4, 0);
1729 } else {
1730 pci_config_put16(handle,
1731 cap_ptr + PCI_MSI_32BIT_DATA, 0);
1732 }
1733
1734 } else if (type == DDI_INTR_TYPE_MSIX) {
1735 uintptr_t off;
1736 uint32_t mask;
1737 ddi_intr_msix_t *msix_p = i_ddi_get_msix(rdip);
1738
1739 ASSERT(msix_p != NULL);
1740
1741 /* Offset into "inum"th entry in the MSI-X table & mask it */
1742 off = (uintptr_t)msix_p->msix_tbl_addr + (inum *
1743 PCI_MSIX_VECTOR_SIZE) + PCI_MSIX_VECTOR_CTRL_OFFSET;
1744
1745 mask = ddi_get32(msix_p->msix_tbl_hdl, (uint32_t *)off);
1746
1747 ddi_put32(msix_p->msix_tbl_hdl, (uint32_t *)off, (mask | 1));
1748
1749 /* Offset into the "inum"th entry in the MSI-X table */
1750 off = (uintptr_t)msix_p->msix_tbl_addr +
1751 (inum * PCI_MSIX_VECTOR_SIZE);
1752
1753 /* Reset the "data" and "addr" bits */
1754 ddi_put32(msix_p->msix_tbl_hdl,
1755 (uint32_t *)(off + PCI_MSIX_DATA_OFFSET), 0);
1756 ddi_put64(msix_p->msix_tbl_hdl, (uint64_t *)off, 0);
1757 }
1758 }
1759
1760 /*
1761 * apic_pci_msi_disable_mode:
1762 */
1763 void
1764 apic_pci_msi_disable_mode(dev_info_t *rdip, int type)
1765 {
1766 ushort_t msi_ctrl;
1767 int cap_ptr = i_ddi_get_msi_msix_cap_ptr(rdip);
1768 ddi_acc_handle_t handle = i_ddi_get_pci_config_handle(rdip);
1769
1770 ASSERT((handle != NULL) && (cap_ptr != 0));
1771
1772 if (type == DDI_INTR_TYPE_MSI) {
1773 msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSI_CTRL);
1774 if (!(msi_ctrl & PCI_MSI_ENABLE_BIT))
1775 return;
1776
1777 msi_ctrl &= ~PCI_MSI_ENABLE_BIT; /* MSI disable */
1778 pci_config_put16(handle, cap_ptr + PCI_MSI_CTRL, msi_ctrl);
1779
1780 } else if (type == DDI_INTR_TYPE_MSIX) {
1781 msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSIX_CTRL);
1782 if (msi_ctrl & PCI_MSIX_ENABLE_BIT) {
1783 msi_ctrl &= ~PCI_MSIX_ENABLE_BIT;
1784 pci_config_put16(handle, cap_ptr + PCI_MSIX_CTRL,
1785 msi_ctrl);
1786 }
1787 }
1788 }
1789
1790 uint32_t
1791 apic_get_localapicid(uint32_t cpuid)
1792 {
1793 ASSERT(cpuid < apic_nproc && apic_cpus != NULL);
1794
1795 return (apic_cpus[cpuid].aci_local_id);
1796 }
1797
1798 uchar_t
1799 apic_get_ioapicid(uchar_t ioapicindex)
1800 {
1801 ASSERT(ioapicindex < MAX_IO_APIC);
1802
1803 return (apic_io_id[ioapicindex]);
1804 }