1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /*
28 * Copyright (c) 2014 by Delphix. All rights reserved.
29 */
30
31 #include <sys/types.h>
32 #include <sys/xpv_support.h>
33 #include <sys/hypervisor.h>
34 #include <sys/machsystm.h>
35 #include <sys/mutex.h>
36 #include <sys/cmn_err.h>
37 #include <sys/dditypes.h>
38 #include <sys/atomic.h>
39 #include <sys/sysmacros.h>
40 #include <sys/cpu.h>
41 #include <sys/psw.h>
42 #include <sys/psm.h>
43 #include <sys/sdt.h>
44
45 extern dev_info_t *xpv_dip;
46 static ddi_intr_handle_t *evtchn_ihp = NULL;
47 static ddi_softint_handle_t evtchn_to_handle[NR_EVENT_CHANNELS];
48 kmutex_t ec_lock;
49
50 static int evtchn_callback_irq = -1;
51
52 static volatile ulong_t *pending_events;
53 static volatile ulong_t *masked_events;
54
55 /* log2(NBBY * sizeof (ulong)) */
56 #ifdef __amd64
57 #define EVTCHN_SHIFT 6
58 #else /* __i386 */
59 #define EVTCHN_SHIFT 5
60 #endif
61
62 /* Atomically get and clear a ulong from memory. */
63 #define GET_AND_CLEAR(src, targ) { \
64 membar_enter(); \
65 do { \
66 targ = *src; \
67 } while (atomic_cas_ulong(src, targ, 0) != targ); \
68 }
69
70 /* Get the first and last bits set in a bitmap */
71 #define GET_BOUNDS(bitmap, low, high) { \
72 int _i; \
73 low = high = -1; \
74 for (_i = 0; _i <= sizeof (ulong_t); _i++) \
75 if (bitmap & (1UL << _i)) { \
76 if (low == -1) \
77 low = _i; \
78 high = _i; \
79 } \
80 }
81
82 void
83 ec_bind_evtchn_to_handler(int evtchn, pri_t pri, ec_handler_fcn_t handler,
84 void *arg1)
85 {
86 ddi_softint_handle_t hdl;
87
88 if (evtchn < 0 || evtchn >= NR_EVENT_CHANNELS) {
89 cmn_err(CE_WARN, "Binding invalid event channel: %d", evtchn);
90 return;
91 }
92
93 (void) ddi_intr_add_softint(xpv_dip, &hdl, pri, handler, (caddr_t)arg1);
94 mutex_enter(&ec_lock);
95 ASSERT(evtchn_to_handle[evtchn] == NULL);
96 evtchn_to_handle[evtchn] = hdl;
97 mutex_exit(&ec_lock);
98
99 /* Let the hypervisor know we're prepared to handle this event */
100 hypervisor_unmask_event(evtchn);
101 }
102
103 void
104 ec_unbind_evtchn(int evtchn)
105 {
106 evtchn_close_t close;
107 ddi_softint_handle_t hdl;
108
109 if (evtchn < 0 || evtchn >= NR_EVENT_CHANNELS) {
110 cmn_err(CE_WARN, "Unbinding invalid event channel: %d", evtchn);
111 return;
112 }
113
114 /*
115 * Let the hypervisor know we're no longer prepared to handle this
116 * event
117 */
118 hypervisor_mask_event(evtchn);
119
120 /* Cleanup the event handler metadata */
121 mutex_enter(&ec_lock);
122 hdl = evtchn_to_handle[evtchn];
123 evtchn_to_handle[evtchn] = NULL;
124 mutex_exit(&ec_lock);
125
126 close.port = evtchn;
127 (void) HYPERVISOR_event_channel_op(EVTCHNOP_close, &close);
128 (void) ddi_intr_remove_softint(hdl);
129 }
130
131 void
132 ec_notify_via_evtchn(unsigned int port)
133 {
134 evtchn_send_t send;
135
136 if ((int)port == -1)
137 return;
138 send.port = port;
139 (void) HYPERVISOR_event_channel_op(EVTCHNOP_send, &send);
140 }
141
142 void
143 hypervisor_unmask_event(unsigned int ev)
144 {
145 int index = ev >> EVTCHN_SHIFT;
146 ulong_t bit = 1UL << (ev & ((1UL << EVTCHN_SHIFT) - 1));
147 volatile ulong_t *maskp;
148 evtchn_unmask_t unmask;
149
150 /*
151 * index,bit contain the event number as an index into the
152 * masked-events bitmask. Set it to 0.
153 */
154 maskp = &masked_events[index];
155 atomic_and_ulong(maskp, ~bit);
156
157 /* Let the hypervisor know the event has been unmasked */
158 unmask.port = ev;
159 if (HYPERVISOR_event_channel_op(EVTCHNOP_unmask, &unmask) != 0)
160 panic("xen_evtchn_unmask() failed");
161 }
162
163 /* Set a bit in an evtchan mask word */
164 void
165 hypervisor_mask_event(uint_t ev)
166 {
167 int index = ev >> EVTCHN_SHIFT;
168 ulong_t bit = 1UL << (ev & ((1UL << EVTCHN_SHIFT) - 1));
169 volatile ulong_t *maskp;
170
171 maskp = &masked_events[index];
172 atomic_or_ulong(maskp, bit);
173 }
174
175 void
176 hypervisor_clear_event(uint_t ev)
177 {
178 int index = ev >> EVTCHN_SHIFT;
179 ulong_t bit = 1UL << (ev & ((1UL << EVTCHN_SHIFT) - 1));
180 volatile ulong_t *maskp;
181
182 maskp = &pending_events[index];
183 atomic_and_ulong(maskp, ~bit);
184 }
185
186 int
187 xen_alloc_unbound_evtchn(int domid, int *evtchnp)
188 {
189 evtchn_alloc_unbound_t alloc;
190 int err;
191
192 alloc.dom = DOMID_SELF;
193 alloc.remote_dom = (domid_t)domid;
194
195 if ((err = HYPERVISOR_event_channel_op(EVTCHNOP_alloc_unbound,
196 &alloc)) == 0) {
197 *evtchnp = alloc.port;
198 /* ensure evtchn is masked till we're ready to use it */
199 (void) hypervisor_mask_event(*evtchnp);
200 } else {
201 err = xen_xlate_errcode(err);
202 }
203
204 return (err);
205 }
206
207 int
208 xen_bind_interdomain(int domid, int remote_port, int *port)
209 {
210 evtchn_bind_interdomain_t bind;
211 int err;
212
213 bind.remote_dom = (domid_t)domid;
214 bind.remote_port = remote_port;
215 if ((err = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain,
216 &bind)) == 0)
217 *port = bind.local_port;
218 else
219 err = xen_xlate_errcode(err);
220 return (err);
221 }
222
223 /*ARGSUSED*/
224 uint_t
225 evtchn_callback_fcn(caddr_t arg0, caddr_t arg1)
226 {
227 ulong_t pending_word;
228 int i, j, port;
229 volatile struct vcpu_info *vci;
230 uint_t rv = DDI_INTR_UNCLAIMED;
231 ddi_softint_handle_t hdl;
232 int low, high;
233 ulong_t sels;
234
235 /*
236 * Xen hard-codes all notifications to VCPU0, so we bind
237 * ourselves via xpv.conf. Note that this also assumes that all
238 * evtchns are bound to VCPU0, which is true by default.
239 */
240 ASSERT(CPU->cpu_id == 0);
241
242 vci = &HYPERVISOR_shared_info->vcpu_info[0];
243
244 again:
245 DTRACE_PROBE2(evtchn__scan__start, int, vci->evtchn_upcall_pending,
246 ulong_t, vci->evtchn_pending_sel);
247
248 atomic_and_8(&vci->evtchn_upcall_pending, 0);
249
250 /*
251 * Find the upper and lower bounds in which we need to search for
252 * pending events.
253 */
254 GET_AND_CLEAR(&vci->evtchn_pending_sel, sels);
255
256 /* sels == 1 is by far the most common case. Make it fast */
257 if (sels == 1)
258 low = high = 0;
259 else if (sels == 0)
260 return (rv);
261 else
262 GET_BOUNDS(sels, low, high);
263
264 /* Scan the port list, looking for words with bits set */
265 for (i = low; i <= high; i++) {
266 ulong_t tmp;
267
268 GET_AND_CLEAR(&pending_events[i], tmp);
269 pending_word = tmp & ~(masked_events[i]);
270
271 /* Scan the bits in the word, looking for pending events */
272 while (pending_word != 0) {
273 j = lowbit(pending_word) - 1;
274 port = (i << EVTCHN_SHIFT) + j;
275 pending_word = pending_word & ~(1UL << j);
276
277 /*
278 * If there is a handler registered for this event,
279 * schedule a softint of the appropriate priority
280 * to execute it.
281 */
282 if ((hdl = evtchn_to_handle[port]) != NULL) {
283 (void) ddi_intr_trigger_softint(hdl, NULL);
284 rv = DDI_INTR_CLAIMED;
285 }
286 }
287 }
288 DTRACE_PROBE2(evtchn__scan__end, int, vci->evtchn_upcall_pending,
289 ulong_t, vci->evtchn_pending_sel);
290
291 if ((volatile uint8_t)vci->evtchn_upcall_pending ||
292 ((volatile ulong_t)vci->evtchn_pending_sel))
293 goto again;
294
295 return (rv);
296 }
297
298 static int
299 set_hvm_callback(int irq)
300 {
301 struct xen_hvm_param xhp;
302
303 xhp.domid = DOMID_SELF;
304 xhp.index = HVM_PARAM_CALLBACK_IRQ;
305 xhp.value = irq;
306 return (HYPERVISOR_hvm_op(HVMOP_set_param, &xhp));
307 }
308
309 void
310 ec_fini()
311 {
312 int i;
313
314 for (i = 0; i < NR_EVENT_CHANNELS; i++)
315 ec_unbind_evtchn(i);
316
317 evtchn_callback_irq = -1;
318 if (evtchn_ihp != NULL) {
319 (void) ddi_intr_disable(*evtchn_ihp);
320 (void) ddi_intr_remove_handler(*evtchn_ihp);
321 (void) ddi_intr_free(*evtchn_ihp);
322 kmem_free(evtchn_ihp, sizeof (ddi_intr_handle_t));
323 evtchn_ihp = NULL;
324 }
325 }
326
327 int
328 ec_init(void)
329 {
330 int i;
331 int rv, actual;
332 ddi_intr_handle_t *ihp;
333
334 /*
335 * Translate the variable-sized pending and masked event bitmasks
336 * into constant-sized arrays of uint32_t's.
337 */
338 pending_events = &HYPERVISOR_shared_info->evtchn_pending[0];
339 masked_events = &HYPERVISOR_shared_info->evtchn_mask[0];
340
341 /*
342 * Clear our event handler structures and prevent the hypervisor
343 * from triggering any events.
344 */
345 mutex_init(&ec_lock, NULL, MUTEX_SPIN, (void *)ipltospl(SPL7));
346 for (i = 0; i < NR_EVENT_CHANNELS; i++) {
347 evtchn_to_handle[i] = NULL;
348 (void) hypervisor_mask_event(i);
349 }
350
351 /*
352 * Allocate and initialize an interrupt handler to process the
353 * hypervisor's "hey you have events pending!" interrupt.
354 */
355 ihp = kmem_zalloc(sizeof (ddi_intr_handle_t), KM_SLEEP);
356 rv = ddi_intr_alloc(xpv_dip, ihp, DDI_INTR_TYPE_FIXED, 0, 1, &actual,
357 DDI_INTR_ALLOC_NORMAL);
358 if (rv < 0 || actual != 1) {
359 cmn_err(CE_WARN, "Could not allocate evtchn interrupt: %d",
360 rv);
361 return (-1);
362 }
363
364 rv = ddi_intr_add_handler(*ihp, evtchn_callback_fcn, NULL, NULL);
365 if (rv < 0) {
366 (void) ddi_intr_free(*ihp);
367 cmn_err(CE_WARN, "Could not attach evtchn handler");
368 return (-1);
369 }
370 evtchn_ihp = ihp;
371
372 if (ddi_intr_enable(*ihp) != DDI_SUCCESS) {
373 cmn_err(CE_WARN, "Could not enable evtchn interrupts\n");
374 return (-1);
375 }
376
377 /* Tell the hypervisor which interrupt we're waiting on. */
378 evtchn_callback_irq = ((ddi_intr_handle_impl_t *)*ihp)->ih_irq;
379
380 if (set_hvm_callback(evtchn_callback_irq) != 0) {
381 cmn_err(CE_WARN, "Couldn't register evtchn callback");
382 return (-1);
383 }
384 return (0);
385 }
386
387 void
388 ec_resume(void)
389 {
390 int i;
391
392 /* New event-channel space is not 'live' yet. */
393 for (i = 0; i < NR_EVENT_CHANNELS; i++)
394 (void) hypervisor_mask_event(i);
395 if (set_hvm_callback(evtchn_callback_irq) != 0)
396 cmn_err(CE_WARN, "Couldn't register evtchn callback");
397
398 }