OS-2366 ddi_periodic_add(9F) is entirely rubbish

@@ -24,10 +24,13 @@
  */
 /*
  * Copyright (c) 2010, Intel Corporation.
  * All rights reserved.
  */
+/*
+ * Copyright (c) 2013, Joyent, Inc.  All rights reserved.
+ */
 
 /*
  * To understand how the pcplusmp module interacts with the interrupt subsystem
  * read the theory statement in uts/i86pc/os/intr.c.
  */

@@ -71,10 +74,11 @@
 #include <sys/privregs.h>
 #include <sys/note.h>
 #include <sys/pci_intr_lib.h>
 #include <sys/spl.h>
 #include <sys/clock.h>
+#include <sys/cyclic.h>
 #include <sys/dditypes.h>
 #include <sys/sunddi.h>
 #include <sys/x_call.h>
 #include <sys/reboot.h>
 #include <sys/hpet.h>

@@ -992,27 +996,39 @@
  */
 static void
 apic_post_cyclic_setup(void *arg)
 {
 _NOTE(ARGUNUSED(arg))
+
+        cyc_handler_t cyh;
+        cyc_time_t cyt;
+
         /* cpu_lock is held */
         /* set up a periodic handler for intr redistribution */
 
         /*
          * In peridoc mode intr redistribution processing is done in
          * apic_intr_enter during clk intr processing
          */
         if (!apic_oneshot)
                 return;
+
         /*
          * Register a periodical handler for the redistribution processing.
-         * On X86, CY_LOW_LEVEL is mapped to the level 2 interrupt, so
-         * DDI_IPL_2 should be passed to ddi_periodic_add() here.
+         * Though we would generally prefer to use the DDI interface for
+         * periodic handler invocation, ddi_periodic_add(9F), we are
+         * unfortunately already holding cpu_lock, which ddi_periodic_add will
+         * attempt to take for us.  Thus, we add our own cyclic directly:
          */
-        apic_periodic_id = ddi_periodic_add(
-            (void (*)(void *))apic_redistribute_compute, NULL,
-            apic_redistribute_sample_interval, DDI_IPL_2);
+        cyh.cyh_func = (void (*)(void *))apic_redistribute_compute;
+        cyh.cyh_arg = NULL;
+        cyh.cyh_level = CY_LOW_LEVEL;
+
+        cyt.cyt_when = 0;
+        cyt.cyt_interval = apic_redistribute_sample_interval;
+
+        apic_cyclic_id = cyclic_add(&cyh, &cyt);
 }
 
 static void
 apic_redistribute_compute(void)
 {