1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26 
  27 /*
  28  * Copyright (c) 2014 by Delphix. All rights reserved.
  29  */
  30 
  31 #include <sys/types.h>
  32 #include <sys/xpv_support.h>
  33 #include <sys/hypervisor.h>
  34 #include <sys/machsystm.h>
  35 #include <sys/mutex.h>
  36 #include <sys/cmn_err.h>
  37 #include <sys/dditypes.h>
  38 #include <sys/atomic.h>
  39 #include <sys/sysmacros.h>
  40 #include <sys/cpu.h>
  41 #include <sys/psw.h>
  42 #include <sys/psm.h>
  43 #include <sys/sdt.h>
  44 
  45 extern dev_info_t *xpv_dip;
  46 static ddi_intr_handle_t *evtchn_ihp = NULL;
  47 static ddi_softint_handle_t evtchn_to_handle[NR_EVENT_CHANNELS];
  48 kmutex_t ec_lock;
  49 
  50 static int evtchn_callback_irq = -1;
  51 
  52 static volatile ulong_t *pending_events;
  53 static volatile ulong_t *masked_events;
  54 
  55 /* log2(NBBY * sizeof (ulong)) */
  56 #ifdef __amd64
  57 #define EVTCHN_SHIFT    6
  58 #else /* __i386 */
  59 #define EVTCHN_SHIFT    5
  60 #endif
  61 
  62 /* Atomically get and clear a ulong from memory. */
  63 #define GET_AND_CLEAR(src, targ) {                              \
  64         membar_enter();                                         \
  65         do {                                                    \
  66                 targ = *src;                                    \
  67         } while (atomic_cas_ulong(src, targ, 0) != targ);       \
  68 }
  69 
  70 /* Get the first and last bits set in a bitmap */
  71 #define GET_BOUNDS(bitmap, low, high)    {      \
  72         int _i;                                         \
  73         low = high = -1;                                \
  74         for (_i = 0; _i <= sizeof (ulong_t); _i++)                   \
  75                 if (bitmap & (1UL << _i)) {   \
  76                         if (low == -1)                  \
  77                                 low = _i;               \
  78                         high = _i;                      \
  79                 }                                       \
  80 }
  81 
  82 void
  83 ec_bind_evtchn_to_handler(int evtchn, pri_t pri, ec_handler_fcn_t handler,
  84     void *arg1)
  85 {
  86         ddi_softint_handle_t hdl;
  87 
  88         if (evtchn < 0 || evtchn >= NR_EVENT_CHANNELS) {
  89                 cmn_err(CE_WARN, "Binding invalid event channel: %d", evtchn);
  90                 return;
  91         }
  92 
  93         (void) ddi_intr_add_softint(xpv_dip, &hdl, pri, handler, (caddr_t)arg1);
  94         mutex_enter(&ec_lock);
  95         ASSERT(evtchn_to_handle[evtchn] == NULL);
  96         evtchn_to_handle[evtchn] = hdl;
  97         mutex_exit(&ec_lock);
  98 
  99         /* Let the hypervisor know we're prepared to handle this event */
 100         hypervisor_unmask_event(evtchn);
 101 }
 102 
 103 void
 104 ec_unbind_evtchn(int evtchn)
 105 {
 106         evtchn_close_t close;
 107         ddi_softint_handle_t hdl;
 108 
 109         if (evtchn < 0 || evtchn >= NR_EVENT_CHANNELS) {
 110                 cmn_err(CE_WARN, "Unbinding invalid event channel: %d", evtchn);
 111                 return;
 112         }
 113 
 114         /*
 115          * Let the hypervisor know we're no longer prepared to handle this
 116          * event
 117          */
 118         hypervisor_mask_event(evtchn);
 119 
 120         /* Cleanup the event handler metadata */
 121         mutex_enter(&ec_lock);
 122         hdl = evtchn_to_handle[evtchn];
 123         evtchn_to_handle[evtchn] = NULL;
 124         mutex_exit(&ec_lock);
 125 
 126         close.port = evtchn;
 127         (void) HYPERVISOR_event_channel_op(EVTCHNOP_close, &close);
 128         (void) ddi_intr_remove_softint(hdl);
 129 }
 130 
 131 void
 132 ec_notify_via_evtchn(unsigned int port)
 133 {
 134         evtchn_send_t send;
 135 
 136         if ((int)port == -1)
 137                 return;
 138         send.port = port;
 139         (void) HYPERVISOR_event_channel_op(EVTCHNOP_send, &send);
 140 }
 141 
 142 void
 143 hypervisor_unmask_event(unsigned int ev)
 144 {
 145         int index = ev >> EVTCHN_SHIFT;
 146         ulong_t bit = 1UL << (ev & ((1UL << EVTCHN_SHIFT) - 1));
 147         volatile ulong_t *maskp;
 148         evtchn_unmask_t unmask;
 149 
 150         /*
 151          * index,bit contain the event number as an index into the
 152          * masked-events bitmask. Set it to 0.
 153          */
 154         maskp = &masked_events[index];
 155         atomic_and_ulong(maskp, ~bit);
 156 
 157         /* Let the hypervisor know the event has been unmasked */
 158         unmask.port = ev;
 159         if (HYPERVISOR_event_channel_op(EVTCHNOP_unmask, &unmask) != 0)
 160                 panic("xen_evtchn_unmask() failed");
 161 }
 162 
 163 /* Set a bit in an evtchan mask word */
 164 void
 165 hypervisor_mask_event(uint_t ev)
 166 {
 167         int index = ev >> EVTCHN_SHIFT;
 168         ulong_t bit = 1UL << (ev & ((1UL << EVTCHN_SHIFT) - 1));
 169         volatile ulong_t *maskp;
 170 
 171         maskp = &masked_events[index];
 172         atomic_or_ulong(maskp, bit);
 173 }
 174 
 175 void
 176 hypervisor_clear_event(uint_t ev)
 177 {
 178         int index = ev >> EVTCHN_SHIFT;
 179         ulong_t bit = 1UL << (ev & ((1UL << EVTCHN_SHIFT) - 1));
 180         volatile ulong_t *maskp;
 181 
 182         maskp = &pending_events[index];
 183         atomic_and_ulong(maskp, ~bit);
 184 }
 185 
 186 int
 187 xen_alloc_unbound_evtchn(int domid, int *evtchnp)
 188 {
 189         evtchn_alloc_unbound_t alloc;
 190         int err;
 191 
 192         alloc.dom = DOMID_SELF;
 193         alloc.remote_dom = (domid_t)domid;
 194 
 195         if ((err = HYPERVISOR_event_channel_op(EVTCHNOP_alloc_unbound,
 196             &alloc)) == 0) {
 197                 *evtchnp = alloc.port;
 198                 /* ensure evtchn is masked till we're ready to use it */
 199                 (void) hypervisor_mask_event(*evtchnp);
 200         } else {
 201                 err = xen_xlate_errcode(err);
 202         }
 203 
 204         return (err);
 205 }
 206 
 207 int
 208 xen_bind_interdomain(int domid, int remote_port, int *port)
 209 {
 210         evtchn_bind_interdomain_t bind;
 211         int err;
 212 
 213         bind.remote_dom = (domid_t)domid;
 214         bind.remote_port = remote_port;
 215         if ((err = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain,
 216             &bind)) == 0)
 217                 *port = bind.local_port;
 218         else
 219                 err = xen_xlate_errcode(err);
 220         return (err);
 221 }
 222 
 223 /*ARGSUSED*/
 224 uint_t
 225 evtchn_callback_fcn(caddr_t arg0, caddr_t arg1)
 226 {
 227         ulong_t pending_word;
 228         int i, j, port;
 229         volatile struct vcpu_info *vci;
 230         uint_t rv = DDI_INTR_UNCLAIMED;
 231         ddi_softint_handle_t hdl;
 232         int low, high;
 233         ulong_t sels;
 234 
 235         /*
 236          * Xen hard-codes all notifications to VCPU0, so we bind
 237          * ourselves via xpv.conf.  Note that this also assumes that all
 238          * evtchns are bound to VCPU0, which is true by default.
 239          */
 240         ASSERT(CPU->cpu_id == 0);
 241 
 242         vci = &HYPERVISOR_shared_info->vcpu_info[0];
 243 
 244 again:
 245         DTRACE_PROBE2(evtchn__scan__start, int, vci->evtchn_upcall_pending,
 246             ulong_t, vci->evtchn_pending_sel);
 247 
 248         atomic_and_8(&vci->evtchn_upcall_pending, 0);
 249 
 250         /*
 251          * Find the upper and lower bounds in which we need to search for
 252          * pending events.
 253          */
 254         GET_AND_CLEAR(&vci->evtchn_pending_sel, sels);
 255 
 256         /* sels == 1 is by far the most common case.  Make it fast */
 257         if (sels == 1)
 258                 low = high = 0;
 259         else if (sels == 0)
 260                 return (rv);
 261         else
 262                 GET_BOUNDS(sels, low, high);
 263 
 264         /* Scan the port list, looking for words with bits set */
 265         for (i = low; i <= high; i++) {
 266                 ulong_t tmp;
 267 
 268                 GET_AND_CLEAR(&pending_events[i], tmp);
 269                 pending_word = tmp & ~(masked_events[i]);
 270 
 271                 /* Scan the bits in the word, looking for pending events */
 272                 while (pending_word != 0) {
 273                         j = lowbit(pending_word) - 1;
 274                         port = (i << EVTCHN_SHIFT) + j;
 275                         pending_word = pending_word & ~(1UL << j);
 276 
 277                         /*
 278                          * If there is a handler registered for this event,
 279                          * schedule a softint of the appropriate priority
 280                          * to execute it.
 281                          */
 282                         if ((hdl = evtchn_to_handle[port]) != NULL) {
 283                                 (void) ddi_intr_trigger_softint(hdl, NULL);
 284                                 rv = DDI_INTR_CLAIMED;
 285                         }
 286                 }
 287         }
 288         DTRACE_PROBE2(evtchn__scan__end, int, vci->evtchn_upcall_pending,
 289             ulong_t, vci->evtchn_pending_sel);
 290 
 291         if ((volatile uint8_t)vci->evtchn_upcall_pending ||
 292             ((volatile ulong_t)vci->evtchn_pending_sel))
 293                 goto again;
 294 
 295         return (rv);
 296 }
 297 
 298 static int
 299 set_hvm_callback(int irq)
 300 {
 301         struct xen_hvm_param xhp;
 302 
 303         xhp.domid = DOMID_SELF;
 304         xhp.index = HVM_PARAM_CALLBACK_IRQ;
 305         xhp.value = irq;
 306         return (HYPERVISOR_hvm_op(HVMOP_set_param, &xhp));
 307 }
 308 
 309 void
 310 ec_fini()
 311 {
 312         int i;
 313 
 314         for (i = 0; i < NR_EVENT_CHANNELS; i++)
 315                 ec_unbind_evtchn(i);
 316 
 317         evtchn_callback_irq = -1;
 318         if (evtchn_ihp != NULL) {
 319                 (void) ddi_intr_disable(*evtchn_ihp);
 320                 (void) ddi_intr_remove_handler(*evtchn_ihp);
 321                 (void) ddi_intr_free(*evtchn_ihp);
 322                 kmem_free(evtchn_ihp, sizeof (ddi_intr_handle_t));
 323                 evtchn_ihp = NULL;
 324         }
 325 }
 326 
 327 int
 328 ec_init(void)
 329 {
 330         int i;
 331         int rv, actual;
 332         ddi_intr_handle_t *ihp;
 333 
 334         /*
 335          * Translate the variable-sized pending and masked event bitmasks
 336          * into constant-sized arrays of uint32_t's.
 337          */
 338         pending_events = &HYPERVISOR_shared_info->evtchn_pending[0];
 339         masked_events = &HYPERVISOR_shared_info->evtchn_mask[0];
 340 
 341         /*
 342          * Clear our event handler structures and prevent the hypervisor
 343          * from triggering any events.
 344          */
 345         mutex_init(&ec_lock, NULL, MUTEX_SPIN, (void *)ipltospl(SPL7));
 346         for (i = 0; i < NR_EVENT_CHANNELS; i++) {
 347                 evtchn_to_handle[i] = NULL;
 348                 (void) hypervisor_mask_event(i);
 349         }
 350 
 351         /*
 352          * Allocate and initialize an interrupt handler to process the
 353          * hypervisor's "hey you have events pending!" interrupt.
 354          */
 355         ihp = kmem_zalloc(sizeof (ddi_intr_handle_t), KM_SLEEP);
 356         rv = ddi_intr_alloc(xpv_dip, ihp, DDI_INTR_TYPE_FIXED, 0, 1, &actual,
 357             DDI_INTR_ALLOC_NORMAL);
 358         if (rv < 0 || actual != 1) {
 359                 cmn_err(CE_WARN, "Could not allocate evtchn interrupt: %d",
 360                     rv);
 361                 return (-1);
 362         }
 363 
 364         rv = ddi_intr_add_handler(*ihp, evtchn_callback_fcn, NULL, NULL);
 365         if (rv < 0) {
 366                 (void) ddi_intr_free(*ihp);
 367                 cmn_err(CE_WARN, "Could not attach evtchn handler");
 368                 return (-1);
 369         }
 370         evtchn_ihp = ihp;
 371 
 372         if (ddi_intr_enable(*ihp) != DDI_SUCCESS) {
 373                 cmn_err(CE_WARN, "Could not enable evtchn interrupts\n");
 374                 return (-1);
 375         }
 376 
 377         /* Tell the hypervisor which interrupt we're waiting on. */
 378         evtchn_callback_irq = ((ddi_intr_handle_impl_t *)*ihp)->ih_vector;
 379 
 380         if (set_hvm_callback(evtchn_callback_irq) != 0) {
 381                 cmn_err(CE_WARN, "Couldn't register evtchn callback");
 382                 return (-1);
 383         }
 384         return (0);
 385 }
 386 
 387 void
 388 ec_resume(void)
 389 {
 390         int i;
 391 
 392         /* New event-channel space is not 'live' yet. */
 393         for (i = 0; i < NR_EVENT_CHANNELS; i++)
 394                 (void) hypervisor_mask_event(i);
 395         if (set_hvm_callback(evtchn_callback_irq) != 0)
 396                 cmn_err(CE_WARN, "Couldn't register evtchn callback");
 397 
 398 }