1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /*
28 * tavor_event.c
29 * Tavor Interrupt and Event Processing Routines
30 *
31 * Implements all the routines necessary for allocating, freeing, and
32 * handling all of the various event types that the Tavor hardware can
33 * generate.
34 * These routines include the main Tavor interrupt service routine
35 * (tavor_isr()) as well as all the code necessary to setup and handle
36 * events from each of the many event queues used by the Tavor device.
37 */
38
39 #include <sys/types.h>
40 #include <sys/conf.h>
41 #include <sys/ddi.h>
42 #include <sys/sunddi.h>
43 #include <sys/modctl.h>
44
45 #include <sys/ib/adapters/tavor/tavor.h>
46
47 static void tavor_eq_poll(tavor_state_t *state, tavor_eqhdl_t eq);
48 static void tavor_eq_catastrophic(tavor_state_t *state);
49 static int tavor_eq_alloc(tavor_state_t *state, uint32_t log_eq_size,
50 uint_t intr, tavor_eqhdl_t *eqhdl);
51 static int tavor_eq_free(tavor_state_t *state, tavor_eqhdl_t *eqhdl);
52 static int tavor_eq_handler_init(tavor_state_t *state, tavor_eqhdl_t eq,
53 uint_t evt_type_mask, int (*eqfunc)(tavor_state_t *state,
54 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe));
55 static int tavor_eq_handler_fini(tavor_state_t *state, tavor_eqhdl_t eq);
56 static void tavor_eqe_sync(tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe, uint_t flag,
57 uint_t force_sync);
58 static int tavor_port_state_change_handler(tavor_state_t *state,
59 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
60 static int tavor_comm_estbl_handler(tavor_state_t *state,
61 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
62 static int tavor_local_wq_cat_err_handler(tavor_state_t *state,
63 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
64 static int tavor_invreq_local_wq_err_handler(tavor_state_t *state,
65 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
66 static int tavor_local_acc_vio_wq_err_handler(tavor_state_t *state,
67 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
68 static int tavor_sendq_drained_handler(tavor_state_t *state,
69 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
70 static int tavor_path_mig_handler(tavor_state_t *state,
71 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
72 static int tavor_path_mig_err_handler(tavor_state_t *state,
73 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
74 static int tavor_srq_catastrophic_handler(tavor_state_t *state,
75 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
76 static int tavor_srq_last_wqe_reached_handler(tavor_state_t *state,
77 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
78 static int tavor_ecc_detection_handler(tavor_state_t *state,
79 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
80 static int tavor_no_eqhandler(tavor_state_t *state, tavor_eqhdl_t eq,
81 tavor_hw_eqe_t *eqe);
82
83
84 /*
85 * tavor_eq_init_all
86 * Context: Only called from attach() path context
87 */
88 int
89 tavor_eq_init_all(tavor_state_t *state)
90 {
91 uint_t log_eq_size, intr_num;
92 uint_t num_eq, num_eq_init, num_eq_unmap;
93 int status, i;
94 char *errormsg;
95
96 TAVOR_TNF_ENTER(tavor_eq_init_all);
97
98 /*
99 * For now, all Event Queues default to the same size (pulled from
100 * the current configuration profile) and are all assigned to the
101 * same interrupt or MSI. In the future we may support assigning
102 * EQs to specific interrupts or MSIs XXX
103 */
104 log_eq_size = state->ts_cfg_profile->cp_log_default_eq_sz;
105
106 /*
107 * If MSI is to be used, then set intr_num to the MSI number
108 * (currently zero because we're using only one) or'd with the
109 * MSI enable flag. Otherwise, for regular (i.e. 'legacy') interrupt,
110 * use the 'inta_pin' value returned by QUERY_ADAPTER.
111 */
112 if (state->ts_intr_type_chosen == DDI_INTR_TYPE_MSI) {
113 intr_num = TAVOR_EQ_MSI_ENABLE_FLAG | 0;
114 } else {
115 intr_num = state->ts_adapter.inta_pin;
116 }
117
118 /*
119 * Total number of supported EQs is hardcoded. Tavor hardware
120 * supports up to 64 EQs. We are currently using only 45 of them
121 * We will set aside the first 32 for use with Completion Queues (CQ)
122 * and reserve a few of the other 32 for each specific class of event
123 * (see below for more details).
124 */
125 num_eq = TAVOR_NUM_EQ_USED;
126
127 /*
128 * The "num_eq_unmap" variable is used in any possible failure
129 * cleanup (below) to indicate which events queues might require
130 * possible event class unmapping.
131 */
132 num_eq_unmap = 0;
133
134 /*
135 * Allocate and initialize all the Event Queues. If any of these
136 * EQ allocations fail then jump to the end, cleanup what had been
137 * successfully initialized, and return an error.
138 */
139 for (i = 0; i < num_eq; i++) {
140 status = tavor_eq_alloc(state, log_eq_size, intr_num,
141 &state->ts_eqhdl[i]);
142 if (status != DDI_SUCCESS) {
143 /* Set "status" and "errormsg" and goto failure */
144 TAVOR_TNF_FAIL(status, "failed EQ alloc");
145 num_eq_init = i;
146 goto all_eq_init_fail;
147 }
148 }
149 num_eq_init = num_eq;
150
151 /*
152 * Setup EQ0-EQ31 for use with Completion Queues. Note: We can
153 * cast the return value to void here because, when we use the
154 * TAVOR_EVT_NO_MASK flag, it is not possible for
155 * tavor_eq_handler_init() to return an error.
156 */
157 for (i = 0; i < 32; i++) {
158 (void) tavor_eq_handler_init(state, state->ts_eqhdl[i],
159 TAVOR_EVT_NO_MASK, tavor_cq_handler);
160 }
161 num_eq_unmap = 32;
162
163 /*
164 * Setup EQ32 for handling Completion Queue Error Events.
165 *
166 * These events include things like CQ overflow or CQ access
167 * violation errors. If this setup fails for any reason (which, in
168 * general, it really never should), then jump to the end, cleanup
169 * everything that has been successfully initialized, and return an
170 * error.
171 */
172 status = tavor_eq_handler_init(state, state->ts_eqhdl[32],
173 TAVOR_EVT_MSK_CQ_ERRORS, tavor_cq_err_handler);
174 if (status != DDI_SUCCESS) {
175 /* Set "status" and "errormsg" and goto failure */
176 TAVOR_TNF_FAIL(status, "completion queue error event");
177 goto all_eq_init_fail;
178 }
179 num_eq_unmap = 33;
180
181 /*
182 * Setup EQ33 for handling Port State Change Events
183 *
184 * These events include things like Port Up and Port Down events.
185 * If this setup fails for any reason (which, in general, it really
186 * never should), then undo all previous EQ mapping, jump to the end,
187 * cleanup everything that has been successfully initialized, and
188 * return an error.
189 */
190 status = tavor_eq_handler_init(state, state->ts_eqhdl[33],
191 TAVOR_EVT_MSK_PORT_STATE_CHANGE, tavor_port_state_change_handler);
192 if (status != DDI_SUCCESS) {
193 /* Set "status" and "errormsg" and goto failure */
194 TAVOR_TNF_FAIL(status, "port state change event");
195 goto all_eq_init_fail;
196 }
197 num_eq_unmap = 34;
198
199 /*
200 * Setup EQ34 for handling Communication Established Events
201 *
202 * These events correspond to the IB affiliated asynchronous events
203 * that are used for connection management. If this setup fails for
204 * any reason (which, in general, it really never should), then undo
205 * all previous EQ mapping, jump to the end, cleanup everything that
206 * has been successfully initialized, and return an error.
207 */
208 status = tavor_eq_handler_init(state, state->ts_eqhdl[34],
209 TAVOR_EVT_MSK_COMM_ESTABLISHED, tavor_comm_estbl_handler);
210 if (status != DDI_SUCCESS) {
211 /* Set "status" and "errormsg" and goto failure */
212 TAVOR_TNF_FAIL(status, "communication established event");
213 goto all_eq_init_fail;
214 }
215 num_eq_unmap = 35;
216
217 /*
218 * Setup EQ35 for handling Command Completion Events
219 *
220 * These events correspond to the Tavor generated events that are used
221 * to indicate Tavor firmware command completion. These events are
222 * only generated when Tavor firmware commands are posted using the
223 * asynchronous completion mechanism. If this setup fails for any
224 * reason (which, in general, it really never should), then undo all
225 * previous EQ mapping, jump to the end, cleanup everything that has
226 * been successfully initialized, and return an error.
227 */
228 status = tavor_eq_handler_init(state, state->ts_eqhdl[35],
229 TAVOR_EVT_MSK_COMMAND_INTF_COMP, tavor_cmd_complete_handler);
230 if (status != DDI_SUCCESS) {
231 /* Set "status" and "errormsg" and goto failure */
232 TAVOR_TNF_FAIL(status, "command completion event");
233 goto all_eq_init_fail;
234 }
235 num_eq_unmap = 36;
236
237 /*
238 * Setup EQ36 for handling Local WQ Catastrophic Error Events
239 *
240 * These events correspond to the similarly-named IB affiliated
241 * asynchronous error type. If this setup fails for any reason
242 * (which, in general, it really never should), then undo all previous
243 * EQ mapping, jump to the end, cleanup everything that has been
244 * successfully initialized, and return an error.
245 */
246 status = tavor_eq_handler_init(state, state->ts_eqhdl[36],
247 TAVOR_EVT_MSK_LOCAL_WQ_CAT_ERROR, tavor_local_wq_cat_err_handler);
248 if (status != DDI_SUCCESS) {
249 /* Set "status" and "errormsg" and goto failure */
250 TAVOR_TNF_FAIL(status, "local WQ catastrophic error event");
251 goto all_eq_init_fail;
252 }
253 num_eq_unmap = 37;
254
255 /*
256 * Setup EQ37 for handling Invalid Req Local WQ Error Events
257 *
258 * These events also correspond to the similarly-named IB affiliated
259 * asynchronous error type. If this setup fails for any reason
260 * (which, in general, it really never should), then undo all previous
261 * EQ mapping, jump to the end, cleanup everything that has been
262 * successfully initialized, and return an error.
263 */
264 status = tavor_eq_handler_init(state, state->ts_eqhdl[37],
265 TAVOR_EVT_MSK_INV_REQ_LOCAL_WQ_ERROR,
266 tavor_invreq_local_wq_err_handler);
267 if (status != DDI_SUCCESS) {
268 /* Set "status" and "errormsg" and goto failure */
269 TAVOR_TNF_FAIL(status, "invalid req local WQ error event");
270 goto all_eq_init_fail;
271 }
272 num_eq_unmap = 38;
273
274 /*
275 * Setup EQ38 for handling Local Access Violation WQ Error Events
276 *
277 * These events also correspond to the similarly-named IB affiliated
278 * asynchronous error type. If this setup fails for any reason
279 * (which, in general, it really never should), then undo all previous
280 * EQ mapping, jump to the end, cleanup everything that has been
281 * successfully initialized, and return an error.
282 */
283 status = tavor_eq_handler_init(state, state->ts_eqhdl[38],
284 TAVOR_EVT_MSK_LOCAL_ACC_VIO_WQ_ERROR,
285 tavor_local_acc_vio_wq_err_handler);
286 if (status != DDI_SUCCESS) {
287 /* Set "status" and "errormsg" and goto failure */
288 TAVOR_TNF_FAIL(status, "local access violation WQ error event");
289 goto all_eq_init_fail;
290 }
291 num_eq_unmap = 39;
292
293 /*
294 * Setup EQ39 for handling Send Queue Drained Events
295 *
296 * These events correspond to the IB affiliated asynchronous events
297 * that are used to indicate completion of a Send Queue Drained QP
298 * state transition. If this setup fails for any reason (which, in
299 * general, it really never should), then undo all previous EQ
300 * mapping, jump to the end, cleanup everything that has been
301 * successfully initialized, and return an error.
302 */
303 status = tavor_eq_handler_init(state, state->ts_eqhdl[39],
304 TAVOR_EVT_MSK_SEND_QUEUE_DRAINED, tavor_sendq_drained_handler);
305 if (status != DDI_SUCCESS) {
306 /* Set "status" and "errormsg" and goto failure */
307 TAVOR_TNF_FAIL(status, "send queue drained event");
308 goto all_eq_init_fail;
309 }
310 num_eq_unmap = 40;
311
312 /*
313 * Setup EQ40 for handling Path Migration Succeeded Events
314 *
315 * These events correspond to the IB affiliated asynchronous events
316 * that are used to indicate successful completion of a path
317 * migration. If this setup fails for any reason (which, in general,
318 * it really never should), then undo all previous EQ mapping, jump
319 * to the end, cleanup everything that has been successfully
320 * initialized, and return an error.
321 */
322 status = tavor_eq_handler_init(state, state->ts_eqhdl[40],
323 TAVOR_EVT_MSK_PATH_MIGRATED, tavor_path_mig_handler);
324 if (status != DDI_SUCCESS) {
325 /* Set "status" and "errormsg" and goto failure */
326 TAVOR_TNF_FAIL(status, "path migration succeeded event");
327 goto all_eq_init_fail;
328 }
329 num_eq_unmap = 41;
330
331 /*
332 * Setup EQ41 for handling Path Migration Failed Events
333 *
334 * These events correspond to the IB affiliated asynchronous events
335 * that are used to indicate that path migration was not successful.
336 * If this setup fails for any reason (which, in general, it really
337 * never should), then undo all previous EQ mapping, jump to the end,
338 * cleanup everything that has been successfully initialized, and
339 * return an error.
340 */
341 status = tavor_eq_handler_init(state, state->ts_eqhdl[41],
342 TAVOR_EVT_MSK_PATH_MIGRATE_FAILED, tavor_path_mig_err_handler);
343 if (status != DDI_SUCCESS) {
344 /* Set "status" and "errormsg" and goto failure */
345 TAVOR_TNF_FAIL(status, "path migration failed event");
346 goto all_eq_init_fail;
347 }
348 num_eq_unmap = 42;
349
350 /*
351 * Setup EQ42 for handling Local Catastrophic Error Events
352 *
353 * These events correspond to the similarly-named IB unaffiliated
354 * asynchronous error type. If this setup fails for any reason
355 * (which, in general, it really never should), then undo all previous
356 * EQ mapping, jump to the end, cleanup everything that has been
357 * successfully initialized, and return an error.
358 *
359 * This error is unique, in that an EQE is not generated if this event
360 * occurs. Instead, an interrupt is called and we must poll the
361 * Catastrophic Error buffer in CR-Space. This mapping is setup simply
362 * to enable this error reporting. We pass in a NULL handler since it
363 * will never be called.
364 */
365 status = tavor_eq_handler_init(state, state->ts_eqhdl[42],
366 TAVOR_EVT_MSK_LOCAL_CAT_ERROR, NULL);
367 if (status != DDI_SUCCESS) {
368 /* Set "status" and "errormsg" and goto failure */
369 TAVOR_TNF_FAIL(status, "local catastrophic error event");
370 goto all_eq_init_fail;
371 }
372 num_eq_unmap = 43;
373
374 /*
375 * Setup EQ43 for handling SRQ Catastrophic Error Events
376 *
377 * These events correspond to the similarly-named IB affiliated
378 * asynchronous error type. If this setup fails for any reason
379 * (which, in general, it really never should), then undo all previous
380 * EQ mapping, jump to the end, cleanup everything that has been
381 * successfully initialized, and return an error.
382 */
383 status = tavor_eq_handler_init(state, state->ts_eqhdl[43],
384 TAVOR_EVT_MSK_SRQ_CATASTROPHIC_ERROR,
385 tavor_srq_catastrophic_handler);
386 if (status != DDI_SUCCESS) {
387 /* Set "status" and "errormsg" and goto failure */
388 TAVOR_TNF_FAIL(status, "srq catastrophic error event");
389 goto all_eq_init_fail;
390 }
391 num_eq_unmap = 44;
392
393 /*
394 * Setup EQ44 for handling SRQ Last WQE Reached Events
395 *
396 * These events correspond to the similarly-named IB affiliated
397 * asynchronous event type. If this setup fails for any reason
398 * (which, in general, it really never should), then undo all previous
399 * EQ mapping, jump to the end, cleanup everything that has been
400 * successfully initialized, and return an error.
401 */
402 status = tavor_eq_handler_init(state, state->ts_eqhdl[44],
403 TAVOR_EVT_MSK_SRQ_LAST_WQE_REACHED,
404 tavor_srq_last_wqe_reached_handler);
405 if (status != DDI_SUCCESS) {
406 /* Set "status" and "errormsg" and goto failure */
407 TAVOR_TNF_FAIL(status, "srq last wqe reached event");
408 goto all_eq_init_fail;
409 }
410 num_eq_unmap = 45;
411
412 /*
413 * Setup EQ45 for handling ECC error detection events
414 *
415 * These events correspond to the similarly-named IB affiliated
416 * asynchronous event type. If this setup fails for any reason
417 * (which, in general, it really never should), then undo all previous
418 * EQ mapping, jump to the end, cleanup everything that has been
419 * successfully initialized, and return an error.
420 */
421 status = tavor_eq_handler_init(state, state->ts_eqhdl[45],
422 TAVOR_EVT_MSK_ECC_DETECTION,
423 tavor_ecc_detection_handler);
424 if (status != DDI_SUCCESS) {
425 /* Set "status" and "errormsg" and goto failure */
426 TAVOR_TNF_FAIL(status, "ecc detection event");
427 goto all_eq_init_fail;
428 }
429 num_eq_unmap = 46;
430
431 /*
432 * Setup EQ46 to catch all other types of events. Specifically, we
433 * do not catch the "Local EEC Catastrophic Error Event" because we
434 * should have no EEC (the Tavor driver does not support RD). We also
435 * choose not to handle any of the address translation page fault
436 * event types. Since we are not doing any page fault handling (and
437 * since the Tavor firmware does not currently support any such
438 * handling), we allow these events to go to the catch-all handler.
439 */
440 status = tavor_eq_handler_init(state, state->ts_eqhdl[46],
441 TAVOR_EVT_CATCHALL_MASK, tavor_no_eqhandler);
442 if (status != DDI_SUCCESS) {
443 /* Set "status" and "errormsg" and goto failure */
444 TAVOR_TNF_FAIL(status, "all other events");
445 TNF_PROBE_0(tavor_eq_init_all_allothershdlr_fail,
446 TAVOR_TNF_ERROR, "");
447 goto all_eq_init_fail;
448 }
449
450 TAVOR_TNF_EXIT(tavor_eq_init_all);
451 return (DDI_SUCCESS);
452
453 all_eq_init_fail:
454 /* Unmap any of the partially mapped EQs from above */
455 for (i = 0; i < num_eq_unmap; i++) {
456 (void) tavor_eq_handler_fini(state, state->ts_eqhdl[i]);
457 }
458
459 /* Free up any of the partially allocated EQs from above */
460 for (i = 0; i < num_eq_init; i++) {
461 (void) tavor_eq_free(state, &state->ts_eqhdl[i]);
462 }
463 TNF_PROBE_1(tavor_eq_init_all_fail, TAVOR_TNF_ERROR, "",
464 tnf_string, msg, errormsg);
465 TAVOR_TNF_EXIT(tavor_eq_init_all);
466 return (status);
467 }
468
469
470 /*
471 * tavor_eq_fini_all
472 * Context: Only called from attach() and/or detach() path contexts
473 */
474 int
475 tavor_eq_fini_all(tavor_state_t *state)
476 {
477 uint_t num_eq;
478 int status, i;
479
480 TAVOR_TNF_ENTER(tavor_eq_fini_all);
481
482 /*
483 * Grab the total number of supported EQs again. This is the same
484 * hardcoded value that was used above (during the event queue
485 * initialization.)
486 */
487 num_eq = TAVOR_NUM_EQ_USED;
488
489 /*
490 * For each of the event queues that we initialized and mapped
491 * earlier, attempt to unmap the events from the EQ.
492 */
493 for (i = 0; i < num_eq; i++) {
494 status = tavor_eq_handler_fini(state, state->ts_eqhdl[i]);
495 if (status != DDI_SUCCESS) {
496 TNF_PROBE_0(tavor_eq_fini_all_eqhdlfini_fail,
497 TAVOR_TNF_ERROR, "");
498 TAVOR_TNF_EXIT(tavor_eq_fini_all);
499 return (DDI_FAILURE);
500 }
501 }
502
503 /*
504 * Teardown and free up all the Event Queues that were allocated
505 * earlier.
506 */
507 for (i = 0; i < num_eq; i++) {
508 status = tavor_eq_free(state, &state->ts_eqhdl[i]);
509 if (status != DDI_SUCCESS) {
510 TNF_PROBE_0(tavor_eq_fini_all_eqfree_fail,
511 TAVOR_TNF_ERROR, "");
512 TAVOR_TNF_EXIT(tavor_eq_fini_all);
513 return (DDI_FAILURE);
514 }
515 }
516
517 TAVOR_TNF_EXIT(tavor_eq_fini_all);
518 return (DDI_SUCCESS);
519 }
520
521
522 /*
523 * tavor_eq_arm_all
524 * Context: Only called from attach() and/or detach() path contexts
525 */
526 void
527 tavor_eq_arm_all(tavor_state_t *state)
528 {
529 uint_t num_eq;
530 int i;
531
532 TAVOR_TNF_ENTER(tavor_eq_arm_all);
533
534 /*
535 * Grab the total number of supported EQs again. This is the same
536 * hardcoded value that was used above (during the event queue
537 * initialization.)
538 */
539 num_eq = TAVOR_NUM_EQ_USED;
540
541 /*
542 * For each of the event queues that we initialized and mapped
543 * earlier, attempt to arm it for event generation.
544 */
545 for (i = 0; i < num_eq; i++) {
546 tavor_eq_doorbell(state, TAVOR_EQDB_REARM_EQ, i, 0);
547 }
548
549 TAVOR_TNF_EXIT(tavor_eq_arm_all);
550 }
551
552
553 /*
554 * tavor_isr()
555 * Context: Only called from interrupt context (and during panic)
556 */
557 /* ARGSUSED */
558 uint_t
559 tavor_isr(caddr_t arg1, caddr_t arg2)
560 {
561 tavor_state_t *state;
562 uint64_t *ecr, *clr_int;
563 uint64_t ecrreg, int_mask;
564 uint_t status;
565 int i;
566
567 TAVOR_TNF_ENTER(tavor_isr);
568
569 /*
570 * Grab the Tavor softstate pointer from the input parameter
571 */
572 state = (tavor_state_t *)arg1;
573
574 /*
575 * Find the pointers to the ECR and clr_INT registers
576 */
577 ecr = state->ts_cmd_regs.ecr;
578 clr_int = state->ts_cmd_regs.clr_int;
579
580 /*
581 * Read the ECR register. Each of the 64 bits in the ECR register
582 * corresponds to an event queue. If a bit is set, then the
583 * corresponding event queue has fired.
584 */
585 ecrreg = ddi_get64(state->ts_reg_cmdhdl, ecr);
586
587 /*
588 * As long as there are bits set (i.e. as long as there are still
589 * EQs in the "fired" state), call tavor_eq_poll() to process each
590 * fired EQ. If no ECR bits are set, do not claim the interrupt.
591 */
592 status = DDI_INTR_UNCLAIMED;
593 do {
594 i = 0;
595 while (ecrreg != 0x0) {
596 if (ecrreg & 0x1) {
597 tavor_eq_poll(state, state->ts_eqhdl[i]);
598 status = DDI_INTR_CLAIMED;
599 }
600 ecrreg = ecrreg >> 1;
601 i++;
602 }
603
604 /*
605 * Clear the interrupt. Note: Depending on the type of
606 * event (interrupt or MSI), we need to use a different
607 * mask to clear the event. In the case of MSI, the bit
608 * to clear corresponds to the MSI number, and for legacy
609 * interrupts the bit corresponds to the value in 'inta_pin'.
610 */
611 if (state->ts_intr_type_chosen == DDI_INTR_TYPE_MSI) {
612 int_mask = ((uint64_t)1 << 0);
613 } else {
614 int_mask = ((uint64_t)1 << state->ts_adapter.inta_pin);
615 }
616 ddi_put64(state->ts_reg_cmdhdl, clr_int, int_mask);
617
618 /* Reread the ECR register */
619 ecrreg = ddi_get64(state->ts_reg_cmdhdl, ecr);
620
621 } while (ecrreg != 0x0);
622
623 TAVOR_TNF_EXIT(tavor_isr);
624 return (status);
625 }
626
627
628 /*
629 * tavor_eq_doorbell
630 * Context: Only called from interrupt context
631 */
632 void
633 tavor_eq_doorbell(tavor_state_t *state, uint32_t eq_cmd, uint32_t eqn,
634 uint32_t eq_param)
635 {
636 uint64_t doorbell = 0;
637
638 /* Build the doorbell from the parameters */
639 doorbell = ((uint64_t)eq_cmd << TAVOR_EQDB_CMD_SHIFT) |
640 ((uint64_t)eqn << TAVOR_EQDB_EQN_SHIFT) | eq_param;
641
642 TNF_PROBE_1_DEBUG(tavor_eq_doorbell, TAVOR_TNF_TRACE, "",
643 tnf_ulong, doorbell, doorbell);
644
645 /* Write the doorbell to UAR */
646 TAVOR_UAR_DOORBELL(state, (uint64_t *)&state->ts_uar->eq,
647 doorbell);
648 }
649
650 /*
651 * tavor_eq_poll
652 * Context: Only called from interrupt context (and during panic)
653 */
654 static void
655 tavor_eq_poll(tavor_state_t *state, tavor_eqhdl_t eq)
656 {
657 uint64_t *clr_ecr;
658 tavor_hw_eqe_t *eqe;
659 uint64_t ecr_mask;
660 uint32_t cons_indx, wrap_around_mask;
661 int (*eqfunction)(tavor_state_t *state, tavor_eqhdl_t eq,
662 tavor_hw_eqe_t *eqe);
663
664 TAVOR_TNF_ENTER(tavor_eq_poll);
665
666 /* Find the pointer to the clr_ECR register */
667 clr_ecr = state->ts_cmd_regs.clr_ecr;
668
669 /*
670 * Check for Local Catastrophic Error If we have this kind of error,
671 * then we don't need to do anything else here, as this kind of
672 * catastrophic error is handled separately. So we call the
673 * catastrophic handler, clear the ECR and then return.
674 */
675 if (eq->eq_evttypemask == TAVOR_EVT_MSK_LOCAL_CAT_ERROR) {
676 /*
677 * Call Catastrophic Error handler
678 */
679 tavor_eq_catastrophic(state);
680
681 /*
682 * Clear the ECR. Specifically, clear the bit corresponding
683 * to the event queue just processed.
684 */
685 ecr_mask = ((uint64_t)1 << eq->eq_eqnum);
686 ddi_put64(state->ts_reg_cmdhdl, clr_ecr, ecr_mask);
687
688 TAVOR_TNF_EXIT(tavor_eq_poll);
689 return;
690 }
691
692 /* Get the consumer pointer index */
693 cons_indx = eq->eq_consindx;
694
695 /*
696 * Calculate the wrap around mask. Note: This operation only works
697 * because all Tavor event queues have power-of-2 sizes
698 */
699 wrap_around_mask = (eq->eq_bufsz - 1);
700
701 /* Calculate the pointer to the first EQ entry */
702 eqe = &eq->eq_buf[cons_indx];
703
704 /*
705 * Sync the current EQE to read
706 * We need to force a ddi_dma_sync() here (independent of how the
707 * EQ was mapped) because it is possible for us to receive the
708 * interrupt, do a read of the ECR, and have each of these
709 * operations complete successfully even though the hardware's DMA
710 * to the EQ has not yet completed.
711 */
712 tavor_eqe_sync(eq, eqe, DDI_DMA_SYNC_FORCPU, TAVOR_EQ_SYNC_FORCE);
713
714 /*
715 * Pull the handler function for this EQ from the Tavor Event Queue
716 * handle
717 */
718 eqfunction = eq->eq_func;
719
720 /*
721 * Keep pulling entries from the EQ until we find an entry owner by
722 * the hardware. As long as there the EQE's owned by SW, process
723 * each entry by calling its handler function and updating the EQ
724 * consumer index.
725 */
726 do {
727 while (TAVOR_EQE_OWNER_IS_SW(eq, eqe)) {
728 /*
729 * Call the EQ handler function. But only call if we
730 * are not in polled I/O mode (i.e. not processing
731 * because of a system panic). Note: We don't call
732 * the EQ handling functions from a system panic
733 * because we are primarily concerned only with
734 * ensuring that the event queues do not overflow (or,
735 * more specifically, the event queue associated with
736 * the CQ that is being used in the sync/dump process).
737 * Also, we don't want to make any upcalls (to the
738 * IBTF) because we can't guarantee when/if those
739 * calls would ever return. And, if we're in panic,
740 * then we reached here through a PollCQ() call (from
741 * tavor_cq_poll()), and we need to ensure that we
742 * successfully return any work completions to the
743 * caller.
744 */
745 if (ddi_in_panic() == 0) {
746 eqfunction(state, eq, eqe);
747 }
748
749 /* Reset entry to hardware ownership */
750 TAVOR_EQE_OWNER_SET_HW(eq, eqe);
751
752 /* Sync the current EQE for device */
753 tavor_eqe_sync(eq, eqe, DDI_DMA_SYNC_FORDEV,
754 TAVOR_EQ_SYNC_NORMAL);
755
756 /* Increment the consumer index */
757 cons_indx = (cons_indx + 1) & wrap_around_mask;
758
759 /* Update the pointer to the next EQ entry */
760 eqe = &eq->eq_buf[cons_indx];
761
762 /* Sync the next EQE to read */
763 tavor_eqe_sync(eq, eqe, DDI_DMA_SYNC_FORCPU,
764 TAVOR_EQ_SYNC_NORMAL);
765 }
766
767 /*
768 * Clear the ECR. Specifically, clear the bit corresponding
769 * to the event queue just processed.
770 */
771 ecr_mask = ((uint64_t)1 << eq->eq_eqnum);
772 ddi_put64(state->ts_reg_cmdhdl, clr_ecr, ecr_mask);
773
774 /* Write an EQ doorbell to update the consumer index */
775 eq->eq_consindx = cons_indx;
776 tavor_eq_doorbell(state, TAVOR_EQDB_SET_CONSINDX, eq->eq_eqnum,
777 cons_indx);
778
779 /* Write another EQ doorbell to rearm */
780 tavor_eq_doorbell(state, TAVOR_EQDB_REARM_EQ, eq->eq_eqnum, 0);
781
782 /*
783 * NOTE: Due to the nature of Mellanox hardware, we do not have
784 * to do an explicit PIO read to ensure that the doorbell write
785 * has been flushed to the hardware. There is state encoded in
786 * the doorbell information we write which makes this
787 * unnecessary. We can be assured that if an event needs to be
788 * generated, the hardware will make sure that it is, solving
789 * the possible race condition.
790 */
791
792 /* Sync the next EQE to read */
793 tavor_eqe_sync(eq, eqe, DDI_DMA_SYNC_FORCPU,
794 TAVOR_EQ_SYNC_NORMAL);
795
796 } while (TAVOR_EQE_OWNER_IS_SW(eq, eqe));
797
798 TAVOR_TNF_EXIT(tavor_eq_poll);
799 }
800
801
802 /*
803 * tavor_eq_catastrophic
804 * Context: Only called from interrupt context (and during panic)
805 */
806 static void
807 tavor_eq_catastrophic(tavor_state_t *state)
808 {
809 ibt_async_code_t type;
810 ibc_async_event_t event;
811 uint32_t *base_addr;
812 uint32_t buf_size;
813 uint32_t word;
814 uint8_t err_type;
815 uint32_t err_buf;
816 int i;
817
818 TAVOR_TNF_ENTER(tavor_eq_catastrophic);
819
820 bzero(&event, sizeof (ibc_async_event_t));
821
822 base_addr = (uint32_t *)(uintptr_t)(
823 (uintptr_t)state->ts_reg_cmd_baseaddr +
824 state->ts_fw.error_buf_addr);
825 buf_size = state->ts_fw.error_buf_sz;
826
827 word = ddi_get32(state->ts_reg_cmdhdl, base_addr);
828
829 err_type = (word & 0xFF000000) >> 24;
830 type = IBT_ERROR_LOCAL_CATASTROPHIC;
831
832 switch (err_type) {
833 case TAVOR_CATASTROPHIC_INTERNAL_ERROR:
834 cmn_err(CE_WARN, "Catastrophic Internal Error: 0x%02x",
835 err_type);
836
837 break;
838
839 case TAVOR_CATASTROPHIC_UPLINK_BUS_ERROR:
840 cmn_err(CE_WARN, "Catastrophic Uplink Bus Error: 0x%02x",
841 err_type);
842
843 break;
844
845 case TAVOR_CATASTROPHIC_DDR_DATA_ERROR:
846 cmn_err(CE_WARN, "Catastrophic DDR Data Error: 0x%02x",
847 err_type);
848
849 break;
850
851 case TAVOR_CATASTROPHIC_INTERNAL_PARITY_ERROR:
852 cmn_err(CE_WARN, "Catastrophic Internal Parity Error: 0x%02x",
853 err_type);
854
855 break;
856
857 default:
858 /* Unknown type of Catastrophic error */
859 cmn_err(CE_WARN, "Catastrophic Unknown Error: 0x%02x",
860 err_type);
861
862 break;
863 }
864
865 /*
866 * Read in the catastrophic error buffer from the hardware, printing
867 * only to the log file only
868 */
869 for (i = 0; i < buf_size; i += 4) {
870 base_addr = (uint32_t *)((uintptr_t)(state->ts_reg_cmd_baseaddr
871 + state->ts_fw.error_buf_addr + (i * 4)));
872 err_buf = ddi_get32(state->ts_reg_cmdhdl, base_addr);
873 cmn_err(CE_WARN, "catastrophic_error[%02x]: %08X", i, err_buf);
874 }
875
876 /*
877 * We also call the IBTF here to inform it of the catastrophic error.
878 * Note: Since no event information (i.e. QP handles, CQ handles,
879 * etc.) is necessary, we pass a NULL pointer instead of a pointer to
880 * an empty ibc_async_event_t struct.
881 *
882 * But we also check if "ts_ibtfpriv" is NULL. If it is then it
883 * means that we've have either received this event before we
884 * finished attaching to the IBTF or we've received it while we
885 * are in the process of detaching.
886 */
887 if (state->ts_ibtfpriv != NULL) {
888 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
889 }
890
891 TAVOR_TNF_EXIT(tavor_eq_catastrophic);
892 }
893
894
895 /*
896 * tavor_eq_alloc()
897 * Context: Only called from attach() path context
898 */
899 static int
900 tavor_eq_alloc(tavor_state_t *state, uint32_t log_eq_size, uint_t intr,
901 tavor_eqhdl_t *eqhdl)
902 {
903 tavor_rsrc_t *eqc, *rsrc;
904 tavor_hw_eqc_t eqc_entry;
905 tavor_eqhdl_t eq;
906 ibt_mr_attr_t mr_attr;
907 tavor_mr_options_t op;
908 tavor_pdhdl_t pd;
909 tavor_mrhdl_t mr;
910 tavor_hw_eqe_t *buf;
911 uint64_t addr;
912 uint32_t lkey;
913 uint_t dma_xfer_mode;
914 int status, i;
915 char *errormsg;
916
917 TAVOR_TNF_ENTER(tavor_eq_alloc);
918
919 /* Use the internal protection domain (PD) for setting up EQs */
920 pd = state->ts_pdhdl_internal;
921
922 /* Increment the reference count on the protection domain (PD) */
923 tavor_pd_refcnt_inc(pd);
924
925 /*
926 * Allocate an EQ context entry. This will be filled in with all
927 * the necessary parameters to define the Event Queue. And then
928 * ownership will be passed to the hardware in the final step
929 * below. If we fail here, we must undo the protection domain
930 * reference count.
931 */
932 status = tavor_rsrc_alloc(state, TAVOR_EQC, 1, TAVOR_SLEEP, &eqc);
933 if (status != DDI_SUCCESS) {
934 /* Set "status" and "errormsg" and goto failure */
935 TAVOR_TNF_FAIL(DDI_FAILURE, "failed EQ context");
936 goto eqalloc_fail1;
937 }
938
939 /*
940 * Allocate the software structure for tracking the event queue (i.e.
941 * the Tavor Event Queue handle). If we fail here, we must undo the
942 * protection domain reference count and the previous resource
943 * allocation.
944 */
945 status = tavor_rsrc_alloc(state, TAVOR_EQHDL, 1, TAVOR_SLEEP, &rsrc);
946 if (status != DDI_SUCCESS) {
947 /* Set "status" and "errormsg" and goto failure */
948 TAVOR_TNF_FAIL(DDI_FAILURE, "failed EQ handler");
949 goto eqalloc_fail2;
950 }
951 eq = (tavor_eqhdl_t)rsrc->tr_addr;
952
953 /*
954 * Allocate the memory for Event Queue. Note: Although we use the
955 * common queue allocation routine, we always specify
956 * TAVOR_QUEUE_LOCATION_NORMAL (i.e. EQ located in system memory)
957 * because it would be inefficient to have EQs located in DDR memory.
958 * This is primarily because EQs are read from (by software) more
959 * than they are written to. Also note that, unlike Tavor QP work
960 * queues, event queues do not have the same strict alignment
961 * requirements. It is sufficient for the EQ memory to be both
962 * aligned to and bound to addresses which are a multiple of EQE size.
963 */
964 eq->eq_eqinfo.qa_size = (1 << log_eq_size) * sizeof (tavor_hw_eqe_t);
965 eq->eq_eqinfo.qa_alloc_align = sizeof (tavor_hw_eqe_t);
966 eq->eq_eqinfo.qa_bind_align = sizeof (tavor_hw_eqe_t);
967 eq->eq_eqinfo.qa_location = TAVOR_QUEUE_LOCATION_NORMAL;
968 status = tavor_queue_alloc(state, &eq->eq_eqinfo, TAVOR_SLEEP);
969 if (status != DDI_SUCCESS) {
970 /* Set "status" and "errormsg" and goto failure */
971 TAVOR_TNF_FAIL(DDI_FAILURE, "failed event queue");
972 goto eqalloc_fail3;
973 }
974 buf = (tavor_hw_eqe_t *)eq->eq_eqinfo.qa_buf_aligned;
975
976 /*
977 * Initialize each of the Event Queue Entries (EQE) by setting their
978 * ownership to hardware ("owner" bit set to HW). This is in
979 * preparation for the final transfer of ownership (below) of the
980 * EQ context itself.
981 */
982 for (i = 0; i < (1 << log_eq_size); i++) {
983 TAVOR_EQE_OWNER_SET_HW(eq, &buf[i]);
984 }
985
986 /*
987 * Register the memory for the EQ. The memory for the EQ must
988 * be registered in the Tavor TPT tables. This gives us the LKey
989 * to specify in the EQ context below.
990 *
991 * Because we are in the attach path we use NOSLEEP here so that we
992 * SPIN in the HCR since the event queues are not setup yet, and we
993 * cannot NOSPIN at this point in time.
994 */
995 mr_attr.mr_vaddr = (uint64_t)(uintptr_t)buf;
996 mr_attr.mr_len = eq->eq_eqinfo.qa_size;
997 mr_attr.mr_as = NULL;
998 mr_attr.mr_flags = IBT_MR_NOSLEEP | IBT_MR_ENABLE_LOCAL_WRITE;
999 dma_xfer_mode = state->ts_cfg_profile->cp_streaming_consistent;
1000 if (dma_xfer_mode == DDI_DMA_STREAMING) {
1001 mr_attr.mr_flags |= IBT_MR_NONCOHERENT;
1002 }
1003 op.mro_bind_type = state->ts_cfg_profile->cp_iommu_bypass;
1004 op.mro_bind_dmahdl = eq->eq_eqinfo.qa_dmahdl;
1005 op.mro_bind_override_addr = 0;
1006 status = tavor_mr_register(state, pd, &mr_attr, &mr, &op);
1007 if (status != DDI_SUCCESS) {
1008 /* Set "status" and "errormsg" and goto failure */
1009 TAVOR_TNF_FAIL(DDI_FAILURE, "failed register mr");
1010 goto eqalloc_fail4;
1011 }
1012 addr = mr->mr_bindinfo.bi_addr;
1013 lkey = mr->mr_lkey;
1014
1015 /* Determine if later ddi_dma_sync will be necessary */
1016 eq->eq_sync = TAVOR_EQ_IS_SYNC_REQ(state, eq->eq_eqinfo);
1017
1018 /* Sync entire EQ for use by the hardware (if necessary) */
1019 if (eq->eq_sync) {
1020 (void) ddi_dma_sync(mr->mr_bindinfo.bi_dmahdl, 0,
1021 eq->eq_eqinfo.qa_size, DDI_DMA_SYNC_FORDEV);
1022 }
1023
1024 /*
1025 * Fill in the EQC entry. This is the final step before passing
1026 * ownership of the EQC entry to the Tavor hardware. We use all of
1027 * the information collected/calculated above to fill in the
1028 * requisite portions of the EQC. Note: We create all EQs in the
1029 * "fired" state. We will arm them later (after our interrupt
1030 * routine had been registered.)
1031 */
1032 bzero(&eqc_entry, sizeof (tavor_hw_eqc_t));
1033 eqc_entry.owner = TAVOR_HW_OWNER;
1034 eqc_entry.xlat = TAVOR_VA2PA_XLAT_ENABLED;
1035 eqc_entry.state = TAVOR_EQ_FIRED;
1036 eqc_entry.start_addr_h = (addr >> 32);
1037 eqc_entry.start_addr_l = (addr & 0xFFFFFFFF);
1038 eqc_entry.log_eq_sz = log_eq_size;
1039 eqc_entry.usr_page = 0;
1040 eqc_entry.pd = pd->pd_pdnum;
1041 eqc_entry.intr = intr;
1042 eqc_entry.lkey = lkey;
1043
1044 /*
1045 * Write the EQC entry to hardware. Lastly, we pass ownership of
1046 * the entry to the hardware (using the Tavor SW2HW_EQ firmware
1047 * command). Note: in general, this operation shouldn't fail. But
1048 * if it does, we have to undo everything we've done above before
1049 * returning error.
1050 */
1051 status = tavor_cmn_ownership_cmd_post(state, SW2HW_EQ, &eqc_entry,
1052 sizeof (tavor_hw_eqc_t), eqc->tr_indx, TAVOR_CMD_NOSLEEP_SPIN);
1053 if (status != TAVOR_CMD_SUCCESS) {
1054 cmn_err(CE_CONT, "Tavor: SW2HW_EQ command failed: %08x\n",
1055 status);
1056 TNF_PROBE_1(tavor_eq_alloc_sw2hw_eq_cmd_fail,
1057 TAVOR_TNF_ERROR, "", tnf_uint, status, status);
1058 /* Set "status" and "errormsg" and goto failure */
1059 TAVOR_TNF_FAIL(ibc_get_ci_failure(0), "tavor SW2HW_EQ command");
1060 goto eqalloc_fail5;
1061 }
1062
1063 /*
1064 * Fill in the rest of the Tavor Event Queue handle. Having
1065 * successfully transferred ownership of the EQC, we can update the
1066 * following fields for use in further operations on the EQ.
1067 */
1068 eq->eq_eqcrsrcp = eqc;
1069 eq->eq_rsrcp = rsrc;
1070 eq->eq_consindx = 0;
1071 eq->eq_eqnum = eqc->tr_indx;
1072 eq->eq_buf = buf;
1073 eq->eq_bufsz = (1 << log_eq_size);
1074 eq->eq_mrhdl = mr;
1075 *eqhdl = eq;
1076
1077 TAVOR_TNF_EXIT(tavor_eq_alloc);
1078 return (DDI_SUCCESS);
1079
1080 /*
1081 * The following is cleanup for all possible failure cases in this routine
1082 */
1083 eqalloc_fail5:
1084 if (tavor_mr_deregister(state, &mr, TAVOR_MR_DEREG_ALL,
1085 TAVOR_NOSLEEP) != DDI_SUCCESS) {
1086 TAVOR_WARNING(state, "failed to deregister EQ memory");
1087 }
1088 eqalloc_fail4:
1089 tavor_queue_free(state, &eq->eq_eqinfo);
1090 eqalloc_fail3:
1091 tavor_rsrc_free(state, &rsrc);
1092 eqalloc_fail2:
1093 tavor_rsrc_free(state, &eqc);
1094 eqalloc_fail1:
1095 tavor_pd_refcnt_dec(pd);
1096 eqalloc_fail:
1097 TNF_PROBE_1(tavor_eq_alloc_fail, TAVOR_TNF_ERROR, "",
1098 tnf_string, msg, errormsg);
1099 TAVOR_TNF_EXIT(tavor_eq_alloc);
1100 return (status);
1101 }
1102
1103
1104 /*
1105 * tavor_eq_free()
1106 * Context: Only called from attach() and/or detach() path contexts
1107 */
1108 static int
1109 tavor_eq_free(tavor_state_t *state, tavor_eqhdl_t *eqhdl)
1110 {
1111 tavor_rsrc_t *eqc, *rsrc;
1112 tavor_hw_eqc_t eqc_entry;
1113 tavor_pdhdl_t pd;
1114 tavor_mrhdl_t mr;
1115 tavor_eqhdl_t eq;
1116 uint32_t eqnum;
1117 int status;
1118
1119 TAVOR_TNF_ENTER(tavor_eq_free);
1120
1121 /*
1122 * Pull all the necessary information from the Tavor Event Queue
1123 * handle. This is necessary here because the resource for the
1124 * EQ handle is going to be freed up as part of this operation.
1125 */
1126 eq = *eqhdl;
1127 eqc = eq->eq_eqcrsrcp;
1128 rsrc = eq->eq_rsrcp;
1129 pd = state->ts_pdhdl_internal;
1130 mr = eq->eq_mrhdl;
1131 eqnum = eq->eq_eqnum;
1132
1133 /*
1134 * Reclaim EQC entry from hardware (using the Tavor HW2SW_EQ
1135 * firmware command). If the ownership transfer fails for any reason,
1136 * then it is an indication that something (either in HW or SW) has
1137 * gone seriously wrong.
1138 */
1139 status = tavor_cmn_ownership_cmd_post(state, HW2SW_EQ, &eqc_entry,
1140 sizeof (tavor_hw_eqc_t), eqnum, TAVOR_CMD_NOSLEEP_SPIN);
1141 if (status != TAVOR_CMD_SUCCESS) {
1142 TAVOR_WARNING(state, "failed to reclaim EQC ownership");
1143 cmn_err(CE_CONT, "Tavor: HW2SW_EQ command failed: %08x\n",
1144 status);
1145 TNF_PROBE_1(tavor_eq_free_hw2sw_eq_cmd_fail,
1146 TAVOR_TNF_ERROR, "", tnf_uint, status, status);
1147 TAVOR_TNF_EXIT(tavor_eq_free);
1148 return (DDI_FAILURE);
1149 }
1150
1151 /*
1152 * Deregister the memory for the Event Queue. If this fails
1153 * for any reason, then it is an indication that something (either
1154 * in HW or SW) has gone seriously wrong. So we print a warning
1155 * message and continue.
1156 */
1157 status = tavor_mr_deregister(state, &mr, TAVOR_MR_DEREG_ALL,
1158 TAVOR_NOSLEEP);
1159 if (status != DDI_SUCCESS) {
1160 TAVOR_WARNING(state, "failed to deregister EQ memory");
1161 TNF_PROBE_0(tavor_eq_free_dereg_mr_fail, TAVOR_TNF_ERROR, "");
1162 TAVOR_TNF_EXIT(tavor_eq_free);
1163 }
1164
1165 /* Free the memory for the EQ */
1166 tavor_queue_free(state, &eq->eq_eqinfo);
1167
1168 /* Free the Tavor Event Queue handle */
1169 tavor_rsrc_free(state, &rsrc);
1170
1171 /* Free up the EQC entry resource */
1172 tavor_rsrc_free(state, &eqc);
1173
1174 /* Decrement the reference count on the protection domain (PD) */
1175 tavor_pd_refcnt_dec(pd);
1176
1177 /* Set the eqhdl pointer to NULL and return success */
1178 *eqhdl = NULL;
1179
1180 TAVOR_TNF_EXIT(tavor_eq_free);
1181 return (DDI_SUCCESS);
1182 }
1183
1184
1185 /*
1186 * tavor_eq_handler_init
1187 * Context: Only called from attach() path context
1188 */
1189 static int
1190 tavor_eq_handler_init(tavor_state_t *state, tavor_eqhdl_t eq,
1191 uint_t evt_type_mask, int (*eq_func)(tavor_state_t *state,
1192 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe))
1193 {
1194 int status;
1195
1196 TAVOR_TNF_ENTER(tavor_eq_handler_init);
1197
1198 /*
1199 * Save away the EQ handler function and the event type mask. These
1200 * will be used later during interrupt and event queue processing.
1201 */
1202 eq->eq_func = eq_func;
1203 eq->eq_evttypemask = evt_type_mask;
1204
1205 /*
1206 * Map the EQ to a specific class of event (or events) depending
1207 * on the mask value passed in. The TAVOR_EVT_NO_MASK means not
1208 * to attempt associating the EQ with any specific class of event.
1209 * This is particularly useful when initializing the events queues
1210 * used for CQ events. The mapping is done using the Tavor MAP_EQ
1211 * firmware command. Note: This command should not, in general, fail.
1212 * If it does, then something (probably HW related) has gone seriously
1213 * wrong.
1214 */
1215 if (evt_type_mask != TAVOR_EVT_NO_MASK) {
1216 status = tavor_map_eq_cmd_post(state,
1217 TAVOR_CMD_MAP_EQ_EVT_MAP, eq->eq_eqnum, evt_type_mask,
1218 TAVOR_CMD_NOSLEEP_SPIN);
1219 if (status != TAVOR_CMD_SUCCESS) {
1220 cmn_err(CE_CONT, "Tavor: MAP_EQ command failed: "
1221 "%08x\n", status);
1222 TNF_PROBE_1(tavor_eq_handler_init_map_eq_cmd_fail,
1223 TAVOR_TNF_ERROR, "", tnf_uint, status, status);
1224 TAVOR_TNF_EXIT(tavor_eq_handler_init);
1225 return (DDI_FAILURE);
1226 }
1227 }
1228
1229 TAVOR_TNF_EXIT(tavor_eq_handler_init);
1230 return (DDI_SUCCESS);
1231 }
1232
1233
1234 /*
1235 * tavor_eq_handler_fini
1236 * Context: Only called from attach() and/or detach() path contexts
1237 */
1238 static int
1239 tavor_eq_handler_fini(tavor_state_t *state, tavor_eqhdl_t eq)
1240 {
1241 int status;
1242
1243 TAVOR_TNF_ENTER(tavor_eq_handler_fini);
1244
1245 /*
1246 * Unmap the EQ from the event class to which it had been previously
1247 * mapped. The unmapping is done using the Tavor MAP_EQ (in much
1248 * the same way that the initial mapping was done). The difference,
1249 * however, is in the TAVOR_EQ_EVT_UNMAP flag that is passed to the
1250 * MAP_EQ firmware command. The TAVOR_EVT_NO_MASK (which may have
1251 * been passed in at init time) still means that no association has
1252 * been made between the EQ and any specific class of event (and,
1253 * hence, no unmapping is necessary). Note: This command should not,
1254 * in general, fail. If it does, then something (probably HW related)
1255 * has gone seriously wrong.
1256 */
1257 if (eq->eq_evttypemask != TAVOR_EVT_NO_MASK) {
1258 status = tavor_map_eq_cmd_post(state,
1259 TAVOR_CMD_MAP_EQ_EVT_UNMAP, eq->eq_eqnum,
1260 eq->eq_evttypemask, TAVOR_CMD_NOSLEEP_SPIN);
1261 if (status != TAVOR_CMD_SUCCESS) {
1262 cmn_err(CE_CONT, "Tavor: MAP_EQ command failed: "
1263 "%08x\n", status);
1264 TNF_PROBE_1(tavor_eq_handler_fini_map_eq_cmd_fail,
1265 TAVOR_TNF_ERROR, "", tnf_uint, status, status);
1266 TAVOR_TNF_EXIT(tavor_eq_handler_fini);
1267 return (DDI_FAILURE);
1268 }
1269 }
1270
1271 TAVOR_TNF_EXIT(tavor_eq_handler_fini);
1272 return (DDI_SUCCESS);
1273 }
1274
1275
1276 /*
1277 * tavor_eqe_sync()
1278 * Context: Can be called from interrupt or base context.
1279 *
1280 * Typically, this routine does nothing unless the EQ memory is
1281 * mapped as DDI_DMA_STREAMING. However, there is a condition where
1282 * ddi_dma_sync() is necessary even if the memory was mapped in
1283 * consistent mode. The "force_sync" parameter is used here to force
1284 * the call to ddi_dma_sync() independent of how the EQ memory was
1285 * mapped.
1286 */
1287 static void
1288 tavor_eqe_sync(tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe, uint_t flag,
1289 uint_t force_sync)
1290 {
1291 ddi_dma_handle_t dmahdl;
1292 off_t offset;
1293 int status;
1294
1295 TAVOR_TNF_ENTER(tavor_eqe_sync);
1296
1297 /* Determine if EQ needs to be synced or not */
1298 if ((eq->eq_sync == 0) && (force_sync == TAVOR_EQ_SYNC_NORMAL)) {
1299 TAVOR_TNF_EXIT(tavor_eqe_sync);
1300 return;
1301 }
1302
1303 /* Get the DMA handle from EQ context */
1304 dmahdl = eq->eq_mrhdl->mr_bindinfo.bi_dmahdl;
1305
1306 /* Calculate offset of next EQE */
1307 offset = (off_t)((uintptr_t)eqe - (uintptr_t)&eq->eq_buf[0]);
1308 status = ddi_dma_sync(dmahdl, offset, sizeof (tavor_hw_eqe_t), flag);
1309 if (status != DDI_SUCCESS) {
1310 TNF_PROBE_0(tavor_eqe_sync_getnextentry_fail,
1311 TAVOR_TNF_ERROR, "");
1312 TAVOR_TNF_EXIT(tavor_eqe_sync);
1313 return;
1314 }
1315
1316 TAVOR_TNF_EXIT(tavor_eqe_sync);
1317 }
1318
1319
1320 /*
1321 * tavor_port_state_change_handler()
1322 * Context: Only called from interrupt context
1323 */
1324 static int
1325 tavor_port_state_change_handler(tavor_state_t *state, tavor_eqhdl_t eq,
1326 tavor_hw_eqe_t *eqe)
1327 {
1328 ibc_async_event_t event;
1329 ibt_async_code_t type;
1330 uint_t port, subtype;
1331 uint_t eqe_evttype;
1332 char link_msg[24];
1333
1334 TAVOR_TNF_ENTER(tavor_port_state_change_handler);
1335
1336 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
1337
1338 ASSERT(eqe_evttype == TAVOR_EVT_PORT_STATE_CHANGE ||
1339 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
1340
1341 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
1342 TNF_PROBE_0(tavor_port_state_change_eq_overflow_condition,
1343 TAVOR_TNF_ERROR, "");
1344 tavor_eq_overflow_handler(state, eq, eqe);
1345
1346 TAVOR_TNF_EXIT(tavor_port_state_change_handler);
1347 return (DDI_FAILURE);
1348 }
1349
1350 /*
1351 * Depending on the type of Port State Change event, pass the
1352 * appropriate asynch event to the IBTF.
1353 */
1354 port = TAVOR_EQE_PORTNUM_GET(eq, eqe);
1355
1356 /* Check for valid port number in event */
1357 if ((port == 0) || (port > state->ts_cfg_profile->cp_num_ports)) {
1358 TAVOR_WARNING(state, "Unexpected port number in port state "
1359 "change event");
1360 cmn_err(CE_CONT, " Port number: %02x\n", port);
1361 TAVOR_TNF_EXIT(tavor_port_state_change_handler);
1362 return (DDI_FAILURE);
1363 }
1364
1365 subtype = TAVOR_EQE_EVTSUBTYPE_GET(eq, eqe);
1366 if (subtype == TAVOR_PORT_LINK_ACTIVE) {
1367 event.ev_port = port;
1368 type = IBT_EVENT_PORT_UP;
1369
1370 (void) snprintf(link_msg, 23, "port %d up", port);
1371 ddi_dev_report_fault(state->ts_dip, DDI_SERVICE_RESTORED,
1372 DDI_EXTERNAL_FAULT, link_msg);
1373 } else if (subtype == TAVOR_PORT_LINK_DOWN) {
1374 event.ev_port = port;
1375 type = IBT_ERROR_PORT_DOWN;
1376
1377 (void) snprintf(link_msg, 23, "port %d down", port);
1378 ddi_dev_report_fault(state->ts_dip, DDI_SERVICE_LOST,
1379 DDI_EXTERNAL_FAULT, link_msg);
1380 } else {
1381 TAVOR_WARNING(state, "Unexpected subtype in port state change "
1382 "event");
1383 cmn_err(CE_CONT, " Event type: %02x, subtype: %02x\n",
1384 TAVOR_EQE_EVTTYPE_GET(eq, eqe), subtype);
1385 TAVOR_TNF_EXIT(tavor_port_state_change_handler);
1386 return (DDI_FAILURE);
1387 }
1388
1389 /*
1390 * Deliver the event to the IBTF. Note: If "ts_ibtfpriv" is NULL,
1391 * then we have either received this event before we finished
1392 * attaching to the IBTF or we've received it while we are in the
1393 * process of detaching.
1394 */
1395 if (state->ts_ibtfpriv != NULL) {
1396 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
1397 }
1398
1399 TAVOR_TNF_EXIT(tavor_port_state_change_handler);
1400 return (DDI_SUCCESS);
1401 }
1402
1403
1404 /*
1405 * tavor_comm_estbl_handler()
1406 * Context: Only called from interrupt context
1407 */
1408 static int
1409 tavor_comm_estbl_handler(tavor_state_t *state, tavor_eqhdl_t eq,
1410 tavor_hw_eqe_t *eqe)
1411 {
1412 tavor_qphdl_t qp;
1413 uint_t qpnum;
1414 ibc_async_event_t event;
1415 ibt_async_code_t type;
1416 uint_t eqe_evttype;
1417
1418 TAVOR_TNF_ENTER(tavor_comm_estbl_handler);
1419
1420 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
1421
1422 ASSERT(eqe_evttype == TAVOR_EVT_COMM_ESTABLISHED ||
1423 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
1424
1425 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
1426 TNF_PROBE_0(tavor_comm_estbl_eq_overflow_condition,
1427 TAVOR_TNF_ERROR, "");
1428 tavor_eq_overflow_handler(state, eq, eqe);
1429
1430 TAVOR_TNF_EXIT(tavor_comm_estbl_handler);
1431 return (DDI_FAILURE);
1432 }
1433
1434 /* Get the QP handle from QP number in event descriptor */
1435 qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
1436 qp = tavor_qphdl_from_qpnum(state, qpnum);
1437
1438 /*
1439 * If the QP handle is NULL, this is probably an indication
1440 * that the QP has been freed already. In which case, we
1441 * should not deliver this event.
1442 *
1443 * We also check that the QP number in the handle is the
1444 * same as the QP number in the event queue entry. This
1445 * extra check allows us to handle the case where a QP was
1446 * freed and then allocated again in the time it took to
1447 * handle the event queue processing. By constantly incrementing
1448 * the non-constrained portion of the QP number every time
1449 * a new QP is allocated, we mitigate (somewhat) the chance
1450 * that a stale event could be passed to the client's QP
1451 * handler.
1452 *
1453 * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it
1454 * means that we've have either received this event before we
1455 * finished attaching to the IBTF or we've received it while we
1456 * are in the process of detaching.
1457 */
1458 if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
1459 (state->ts_ibtfpriv != NULL)) {
1460 event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
1461 type = IBT_EVENT_COM_EST_QP;
1462
1463 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
1464 } else {
1465 TNF_PROBE_2(tavor_comm_estbl_handler_dropped_event,
1466 TAVOR_TNF_ERROR, "", tnf_uint, ev_qpnum, qpnum,
1467 tnf_uint, hdl_qpnum, qpnum);
1468 }
1469
1470 TAVOR_TNF_EXIT(tavor_comm_estbl_handler);
1471 return (DDI_SUCCESS);
1472 }
1473
1474
1475 /*
1476 * tavor_local_wq_cat_err_handler()
1477 * Context: Only called from interrupt context
1478 */
1479 static int
1480 tavor_local_wq_cat_err_handler(tavor_state_t *state, tavor_eqhdl_t eq,
1481 tavor_hw_eqe_t *eqe)
1482 {
1483 tavor_qphdl_t qp;
1484 uint_t qpnum;
1485 ibc_async_event_t event;
1486 ibt_async_code_t type;
1487 uint_t eqe_evttype;
1488
1489 TAVOR_TNF_ENTER(tavor_local_wq_cat_err_handler);
1490
1491 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
1492
1493 ASSERT(eqe_evttype == TAVOR_EVT_LOCAL_WQ_CAT_ERROR ||
1494 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
1495
1496 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
1497 TNF_PROBE_0(tavor_local_wq_cat_err_eq_overflow_condition,
1498 TAVOR_TNF_ERROR, "");
1499 tavor_eq_overflow_handler(state, eq, eqe);
1500
1501 TAVOR_TNF_EXIT(tavor_local_wq_cat_err_handler);
1502 return (DDI_FAILURE);
1503 }
1504
1505 /* Get the QP handle from QP number in event descriptor */
1506 qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
1507 qp = tavor_qphdl_from_qpnum(state, qpnum);
1508
1509 /*
1510 * If the QP handle is NULL, this is probably an indication
1511 * that the QP has been freed already. In which case, we
1512 * should not deliver this event.
1513 *
1514 * We also check that the QP number in the handle is the
1515 * same as the QP number in the event queue entry. This
1516 * extra check allows us to handle the case where a QP was
1517 * freed and then allocated again in the time it took to
1518 * handle the event queue processing. By constantly incrementing
1519 * the non-constrained portion of the QP number every time
1520 * a new QP is allocated, we mitigate (somewhat) the chance
1521 * that a stale event could be passed to the client's QP
1522 * handler.
1523 *
1524 * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it
1525 * means that we've have either received this event before we
1526 * finished attaching to the IBTF or we've received it while we
1527 * are in the process of detaching.
1528 */
1529 if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
1530 (state->ts_ibtfpriv != NULL)) {
1531 event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
1532 type = IBT_ERROR_CATASTROPHIC_QP;
1533
1534 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
1535 } else {
1536 TNF_PROBE_2(tavor_local_wq_cat_err_handler_dropped_event,
1537 TAVOR_TNF_ERROR, "", tnf_uint, ev_qpnum, qpnum,
1538 tnf_uint, hdl_qpnum, qpnum);
1539 }
1540
1541 TAVOR_TNF_EXIT(tavor_local_wq_cat_err_handler);
1542 return (DDI_SUCCESS);
1543 }
1544
1545
1546 /*
1547 * tavor_invreq_local_wq_err_handler()
1548 * Context: Only called from interrupt context
1549 */
1550 static int
1551 tavor_invreq_local_wq_err_handler(tavor_state_t *state, tavor_eqhdl_t eq,
1552 tavor_hw_eqe_t *eqe)
1553 {
1554 tavor_qphdl_t qp;
1555 uint_t qpnum;
1556 ibc_async_event_t event;
1557 ibt_async_code_t type;
1558 uint_t eqe_evttype;
1559
1560 TAVOR_TNF_ENTER(tavor_invreq_local_wq_err_handler);
1561
1562 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
1563
1564 ASSERT(eqe_evttype == TAVOR_EVT_INV_REQ_LOCAL_WQ_ERROR ||
1565 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
1566
1567 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
1568 TNF_PROBE_0(tavor_invreq_local_wq_err_eq_overflow_condition,
1569 TAVOR_TNF_ERROR, "");
1570 tavor_eq_overflow_handler(state, eq, eqe);
1571
1572 TAVOR_TNF_EXIT(tavor_port_state_change_handler);
1573 return (DDI_FAILURE);
1574 }
1575
1576 /* Get the QP handle from QP number in event descriptor */
1577 qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
1578 qp = tavor_qphdl_from_qpnum(state, qpnum);
1579
1580 /*
1581 * If the QP handle is NULL, this is probably an indication
1582 * that the QP has been freed already. In which case, we
1583 * should not deliver this event.
1584 *
1585 * We also check that the QP number in the handle is the
1586 * same as the QP number in the event queue entry. This
1587 * extra check allows us to handle the case where a QP was
1588 * freed and then allocated again in the time it took to
1589 * handle the event queue processing. By constantly incrementing
1590 * the non-constrained portion of the QP number every time
1591 * a new QP is allocated, we mitigate (somewhat) the chance
1592 * that a stale event could be passed to the client's QP
1593 * handler.
1594 *
1595 * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it
1596 * means that we've have either received this event before we
1597 * finished attaching to the IBTF or we've received it while we
1598 * are in the process of detaching.
1599 */
1600 if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
1601 (state->ts_ibtfpriv != NULL)) {
1602 event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
1603 type = IBT_ERROR_INVALID_REQUEST_QP;
1604
1605 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
1606 } else {
1607 TNF_PROBE_2(tavor_invreq_local_wq_err_handler_dropped_event,
1608 TAVOR_TNF_ERROR, "", tnf_uint, ev_qpnum, qpnum,
1609 tnf_uint, hdl_qpnum, qpnum);
1610 }
1611
1612 TAVOR_TNF_EXIT(tavor_invreq_local_wq_err_handler);
1613 return (DDI_SUCCESS);
1614 }
1615
1616
1617 /*
1618 * tavor_local_acc_vio_wq_err_handler()
1619 * Context: Only called from interrupt context
1620 */
1621 static int
1622 tavor_local_acc_vio_wq_err_handler(tavor_state_t *state, tavor_eqhdl_t eq,
1623 tavor_hw_eqe_t *eqe)
1624 {
1625 tavor_qphdl_t qp;
1626 uint_t qpnum;
1627 ibc_async_event_t event;
1628 ibt_async_code_t type;
1629 uint_t eqe_evttype;
1630
1631 TAVOR_TNF_ENTER(tavor_local_acc_vio_wq_err_handler);
1632
1633 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
1634
1635 ASSERT(eqe_evttype == TAVOR_EVT_LOCAL_ACC_VIO_WQ_ERROR ||
1636 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
1637
1638 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
1639 TNF_PROBE_0(tavor_local_acc_vio_wq_err_eq_overflow_condition,
1640 TAVOR_TNF_ERROR, "");
1641 tavor_eq_overflow_handler(state, eq, eqe);
1642
1643 TAVOR_TNF_EXIT(tavor_local_acc_vio_wq_err_handler);
1644 return (DDI_FAILURE);
1645 }
1646
1647 /* Get the QP handle from QP number in event descriptor */
1648 qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
1649 qp = tavor_qphdl_from_qpnum(state, qpnum);
1650
1651 /*
1652 * If the QP handle is NULL, this is probably an indication
1653 * that the QP has been freed already. In which case, we
1654 * should not deliver this event.
1655 *
1656 * We also check that the QP number in the handle is the
1657 * same as the QP number in the event queue entry. This
1658 * extra check allows us to handle the case where a QP was
1659 * freed and then allocated again in the time it took to
1660 * handle the event queue processing. By constantly incrementing
1661 * the non-constrained portion of the QP number every time
1662 * a new QP is allocated, we mitigate (somewhat) the chance
1663 * that a stale event could be passed to the client's QP
1664 * handler.
1665 *
1666 * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it
1667 * means that we've have either received this event before we
1668 * finished attaching to the IBTF or we've received it while we
1669 * are in the process of detaching.
1670 */
1671 if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
1672 (state->ts_ibtfpriv != NULL)) {
1673 event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
1674 type = IBT_ERROR_ACCESS_VIOLATION_QP;
1675
1676 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
1677 } else {
1678 TNF_PROBE_2(tavor_local_acc_vio_wq_err_handler_dropped_event,
1679 TAVOR_TNF_ERROR, "", tnf_uint, ev_qpnum, qpnum,
1680 tnf_uint, hdl_qpnum, qpnum);
1681 }
1682
1683 TAVOR_TNF_EXIT(tavor_local_acc_vio_wq_err_handler);
1684 return (DDI_SUCCESS);
1685 }
1686
1687
1688 /*
1689 * tavor_sendq_drained_handler()
1690 * Context: Only called from interrupt context
1691 */
1692 static int
1693 tavor_sendq_drained_handler(tavor_state_t *state, tavor_eqhdl_t eq,
1694 tavor_hw_eqe_t *eqe)
1695 {
1696 tavor_qphdl_t qp;
1697 uint_t qpnum;
1698 ibc_async_event_t event;
1699 uint_t forward_sqd_event;
1700 ibt_async_code_t type;
1701 uint_t eqe_evttype;
1702
1703 TAVOR_TNF_ENTER(tavor_sendq_drained_handler);
1704
1705 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
1706
1707 ASSERT(eqe_evttype == TAVOR_EVT_SEND_QUEUE_DRAINED ||
1708 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
1709
1710 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
1711 TNF_PROBE_0(tavor_sendq_drained_eq_overflow_condition,
1712 TAVOR_TNF_ERROR, "");
1713 tavor_eq_overflow_handler(state, eq, eqe);
1714
1715 TAVOR_TNF_EXIT(tavor_sendq_drained_handler);
1716 return (DDI_FAILURE);
1717 }
1718
1719 /* Get the QP handle from QP number in event descriptor */
1720 qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
1721 qp = tavor_qphdl_from_qpnum(state, qpnum);
1722
1723 /*
1724 * If the QP handle is NULL, this is probably an indication
1725 * that the QP has been freed already. In which case, we
1726 * should not deliver this event.
1727 *
1728 * We also check that the QP number in the handle is the
1729 * same as the QP number in the event queue entry. This
1730 * extra check allows us to handle the case where a QP was
1731 * freed and then allocated again in the time it took to
1732 * handle the event queue processing. By constantly incrementing
1733 * the non-constrained portion of the QP number every time
1734 * a new QP is allocated, we mitigate (somewhat) the chance
1735 * that a stale event could be passed to the client's QP
1736 * handler.
1737 *
1738 * And then we check if "ts_ibtfpriv" is NULL. If it is then it
1739 * means that we've have either received this event before we
1740 * finished attaching to the IBTF or we've received it while we
1741 * are in the process of detaching.
1742 */
1743 if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
1744 (state->ts_ibtfpriv != NULL)) {
1745 event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
1746 type = IBT_EVENT_SQD;
1747
1748 /*
1749 * Grab the QP lock and update the QP state to reflect that
1750 * the Send Queue Drained event has arrived. Also determine
1751 * whether the event is intended to be forwarded on to the
1752 * consumer or not. This information is used below in
1753 * determining whether or not to call the IBTF.
1754 */
1755 mutex_enter(&qp->qp_lock);
1756 forward_sqd_event = qp->qp_forward_sqd_event;
1757 qp->qp_forward_sqd_event = 0;
1758 qp->qp_sqd_still_draining = 0;
1759 mutex_exit(&qp->qp_lock);
1760
1761 if (forward_sqd_event != 0) {
1762 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
1763 }
1764 } else {
1765 TNF_PROBE_2(tavor_sendq_drained_handler_dropped_event,
1766 TAVOR_TNF_ERROR, "", tnf_uint, ev_qpnum, qpnum,
1767 tnf_uint, hdl_qpnum, qpnum);
1768 }
1769
1770 TAVOR_TNF_EXIT(tavor_sendq_drained_handler);
1771 return (DDI_SUCCESS);
1772 }
1773
1774
1775 /*
1776 * tavor_path_mig_handler()
1777 * Context: Only called from interrupt context
1778 */
1779 static int
1780 tavor_path_mig_handler(tavor_state_t *state, tavor_eqhdl_t eq,
1781 tavor_hw_eqe_t *eqe)
1782 {
1783 tavor_qphdl_t qp;
1784 uint_t qpnum;
1785 ibc_async_event_t event;
1786 ibt_async_code_t type;
1787 uint_t eqe_evttype;
1788
1789 TAVOR_TNF_ENTER(tavor_path_mig_handler);
1790
1791 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
1792
1793 ASSERT(eqe_evttype == TAVOR_EVT_PATH_MIGRATED ||
1794 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
1795
1796 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
1797 TNF_PROBE_0(tavor_path_mig_eq_overflow_condition,
1798 TAVOR_TNF_ERROR, "");
1799 tavor_eq_overflow_handler(state, eq, eqe);
1800
1801 TAVOR_TNF_EXIT(tavor_path_mig_handler);
1802 return (DDI_FAILURE);
1803 }
1804
1805 /* Get the QP handle from QP number in event descriptor */
1806 qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
1807 qp = tavor_qphdl_from_qpnum(state, qpnum);
1808
1809 /*
1810 * If the QP handle is NULL, this is probably an indication
1811 * that the QP has been freed already. In which case, we
1812 * should not deliver this event.
1813 *
1814 * We also check that the QP number in the handle is the
1815 * same as the QP number in the event queue entry. This
1816 * extra check allows us to handle the case where a QP was
1817 * freed and then allocated again in the time it took to
1818 * handle the event queue processing. By constantly incrementing
1819 * the non-constrained portion of the QP number every time
1820 * a new QP is allocated, we mitigate (somewhat) the chance
1821 * that a stale event could be passed to the client's QP
1822 * handler.
1823 *
1824 * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it
1825 * means that we've have either received this event before we
1826 * finished attaching to the IBTF or we've received it while we
1827 * are in the process of detaching.
1828 */
1829 if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
1830 (state->ts_ibtfpriv != NULL)) {
1831 event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
1832 type = IBT_EVENT_PATH_MIGRATED_QP;
1833
1834 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
1835 } else {
1836 TNF_PROBE_2(tavor_path_mig_handler_dropped_event,
1837 TAVOR_TNF_ERROR, "", tnf_uint, ev_qpnum, qpnum,
1838 tnf_uint, hdl_qpnum, qpnum);
1839 }
1840
1841 TAVOR_TNF_EXIT(tavor_path_mig_handler);
1842 return (DDI_SUCCESS);
1843 }
1844
1845
1846 /*
1847 * tavor_path_mig_err_handler()
1848 * Context: Only called from interrupt context
1849 */
1850 static int
1851 tavor_path_mig_err_handler(tavor_state_t *state, tavor_eqhdl_t eq,
1852 tavor_hw_eqe_t *eqe)
1853 {
1854 tavor_qphdl_t qp;
1855 uint_t qpnum;
1856 ibc_async_event_t event;
1857 ibt_async_code_t type;
1858 uint_t eqe_evttype;
1859
1860 TAVOR_TNF_ENTER(tavor_path_mig_err_handler);
1861
1862 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
1863
1864 ASSERT(eqe_evttype == TAVOR_EVT_PATH_MIGRATE_FAILED ||
1865 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
1866
1867 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
1868 TNF_PROBE_0(tavor_path_mig_err_eq_overflow_condition,
1869 TAVOR_TNF_ERROR, "");
1870 tavor_eq_overflow_handler(state, eq, eqe);
1871
1872 TAVOR_TNF_EXIT(tavor_path_mig_err_handler);
1873 return (DDI_FAILURE);
1874 }
1875
1876 /* Get the QP handle from QP number in event descriptor */
1877 qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
1878 qp = tavor_qphdl_from_qpnum(state, qpnum);
1879
1880 /*
1881 * If the QP handle is NULL, this is probably an indication
1882 * that the QP has been freed already. In which case, we
1883 * should not deliver this event.
1884 *
1885 * We also check that the QP number in the handle is the
1886 * same as the QP number in the event queue entry. This
1887 * extra check allows us to handle the case where a QP was
1888 * freed and then allocated again in the time it took to
1889 * handle the event queue processing. By constantly incrementing
1890 * the non-constrained portion of the QP number every time
1891 * a new QP is allocated, we mitigate (somewhat) the chance
1892 * that a stale event could be passed to the client's QP
1893 * handler.
1894 *
1895 * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it
1896 * means that we've have either received this event before we
1897 * finished attaching to the IBTF or we've received it while we
1898 * are in the process of detaching.
1899 */
1900 if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
1901 (state->ts_ibtfpriv != NULL)) {
1902 event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
1903 type = IBT_ERROR_PATH_MIGRATE_REQ_QP;
1904
1905 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
1906 } else {
1907 TNF_PROBE_2(tavor_path_mig_err_handler_dropped_event,
1908 TAVOR_TNF_ERROR, "", tnf_uint, ev_qpnum, qpnum,
1909 tnf_uint, hdl_qpnum, qpnum);
1910 }
1911
1912 TAVOR_TNF_EXIT(tavor_path_mig_err_handler);
1913 return (DDI_SUCCESS);
1914 }
1915
1916
1917 /*
1918 * tavor_srq_catastrophic_handler()
1919 * Context: Only called from interrupt context
1920 */
1921 static int
1922 tavor_srq_catastrophic_handler(tavor_state_t *state, tavor_eqhdl_t eq,
1923 tavor_hw_eqe_t *eqe)
1924 {
1925 tavor_qphdl_t qp;
1926 uint_t qpnum;
1927 ibc_async_event_t event;
1928 ibt_async_code_t type;
1929 uint_t eqe_evttype;
1930
1931 TAVOR_TNF_ENTER(tavor_srq_catastrophic_handler);
1932
1933 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
1934
1935 ASSERT(eqe_evttype == TAVOR_EVT_SRQ_CATASTROPHIC_ERROR ||
1936 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
1937
1938 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
1939 TNF_PROBE_0(tavor_srq_catastrophic_overflow_condition,
1940 TAVOR_TNF_ERROR, "");
1941 tavor_eq_overflow_handler(state, eq, eqe);
1942
1943 TAVOR_TNF_EXIT(tavor_srq_catastrophic_handler);
1944 return (DDI_FAILURE);
1945 }
1946
1947 /* Get the QP handle from QP number in event descriptor */
1948 qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
1949 qp = tavor_qphdl_from_qpnum(state, qpnum);
1950
1951 /*
1952 * If the QP handle is NULL, this is probably an indication
1953 * that the QP has been freed already. In which case, we
1954 * should not deliver this event.
1955 *
1956 * We also check that the QP number in the handle is the
1957 * same as the QP number in the event queue entry. This
1958 * extra check allows us to handle the case where a QP was
1959 * freed and then allocated again in the time it took to
1960 * handle the event queue processing. By constantly incrementing
1961 * the non-constrained portion of the QP number every time
1962 * a new QP is allocated, we mitigate (somewhat) the chance
1963 * that a stale event could be passed to the client's QP
1964 * handler.
1965 *
1966 * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it
1967 * means that we've have either received this event before we
1968 * finished attaching to the IBTF or we've received it while we
1969 * are in the process of detaching.
1970 */
1971 if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
1972 (state->ts_ibtfpriv != NULL)) {
1973 event.ev_srq_hdl = (ibt_srq_hdl_t)qp->qp_srqhdl->srq_hdlrarg;
1974 type = IBT_ERROR_CATASTROPHIC_SRQ;
1975
1976 mutex_enter(&qp->qp_srqhdl->srq_lock);
1977 qp->qp_srqhdl->srq_state = TAVOR_SRQ_STATE_ERROR;
1978 mutex_exit(&qp->qp_srqhdl->srq_lock);
1979
1980 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
1981 } else {
1982 TNF_PROBE_2(tavor_srq_catastrophic_handler_dropped_event,
1983 TAVOR_TNF_ERROR, "", tnf_uint, ev_qpnum, qpnum,
1984 tnf_uint, hdl_qpnum, qpnum);
1985 }
1986
1987 TAVOR_TNF_EXIT(tavor_srq_catastrophic_handler);
1988 return (DDI_SUCCESS);
1989 }
1990
1991
1992 /*
1993 * tavor_srq_last_wqe_reached_handler()
1994 * Context: Only called from interrupt context
1995 */
1996 static int
1997 tavor_srq_last_wqe_reached_handler(tavor_state_t *state, tavor_eqhdl_t eq,
1998 tavor_hw_eqe_t *eqe)
1999 {
2000 tavor_qphdl_t qp;
2001 uint_t qpnum;
2002 ibc_async_event_t event;
2003 ibt_async_code_t type;
2004 uint_t eqe_evttype;
2005
2006 TAVOR_TNF_ENTER(tavor_srq_last_wqe_reached_handler);
2007
2008 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
2009
2010 ASSERT(eqe_evttype == TAVOR_EVT_SRQ_LAST_WQE_REACHED ||
2011 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
2012
2013 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
2014 TNF_PROBE_0(tavor_srq_last_wqe_reached_over_condition,
2015 TAVOR_TNF_ERROR, "");
2016 tavor_eq_overflow_handler(state, eq, eqe);
2017
2018 TAVOR_TNF_EXIT(tavor_srq_last_wqe_reached_handler);
2019 return (DDI_FAILURE);
2020 }
2021
2022 /* Get the QP handle from QP number in event descriptor */
2023 qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
2024 qp = tavor_qphdl_from_qpnum(state, qpnum);
2025
2026 /*
2027 * If the QP handle is NULL, this is probably an indication
2028 * that the QP has been freed already. In which case, we
2029 * should not deliver this event.
2030 *
2031 * We also check that the QP number in the handle is the
2032 * same as the QP number in the event queue entry. This
2033 * extra check allows us to handle the case where a QP was
2034 * freed and then allocated again in the time it took to
2035 * handle the event queue processing. By constantly incrementing
2036 * the non-constrained portion of the QP number every time
2037 * a new QP is allocated, we mitigate (somewhat) the chance
2038 * that a stale event could be passed to the client's QP
2039 * handler.
2040 *
2041 * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it
2042 * means that we've have either received this event before we
2043 * finished attaching to the IBTF or we've received it while we
2044 * are in the process of detaching.
2045 */
2046 if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
2047 (state->ts_ibtfpriv != NULL)) {
2048 event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
2049 type = IBT_EVENT_EMPTY_CHAN;
2050
2051 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
2052 } else {
2053 TNF_PROBE_2(tavor_srq_last_wqe_reached_dropped_event,
2054 TAVOR_TNF_ERROR, "", tnf_uint, ev_qpnum, qpnum,
2055 tnf_uint, hdl_qpnum, qpnum);
2056 }
2057
2058 TAVOR_TNF_EXIT(tavor_srq_last_wqe_reached_handler);
2059 return (DDI_SUCCESS);
2060 }
2061
2062
2063 /*
2064 * tavor_ecc_detection_handler()
2065 * Context: Only called from interrupt context
2066 */
2067 static int
2068 tavor_ecc_detection_handler(tavor_state_t *state, tavor_eqhdl_t eq,
2069 tavor_hw_eqe_t *eqe)
2070 {
2071 uint_t eqe_evttype;
2072 uint_t data;
2073 int i;
2074
2075 TAVOR_TNF_ENTER(tavor_ecc_detection_handler);
2076
2077 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
2078
2079 ASSERT(eqe_evttype == TAVOR_EVT_ECC_DETECTION ||
2080 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
2081
2082 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
2083 TNF_PROBE_0(tavor_ecc_detection_eq_overflow_condition,
2084 TAVOR_TNF_ERROR, "");
2085 tavor_eq_overflow_handler(state, eq, eqe);
2086
2087 TAVOR_TNF_EXIT(tavor_ecc_detection_handler);
2088 return (DDI_FAILURE);
2089 }
2090
2091 /*
2092 * The "ECC Detection Event" indicates that a correctable single-bit
2093 * has occurred with the attached DDR. The EQE provides some
2094 * additional information about the errored EQ. So we print a warning
2095 * message here along with that additional information.
2096 */
2097 TAVOR_WARNING(state, "ECC Correctable Error Event Detected");
2098 for (i = 0; i < sizeof (tavor_hw_eqe_t) >> 2; i++) {
2099 data = ((uint_t *)eqe)[i];
2100 cmn_err(CE_CONT, "! EQE[%02x]: %08x\n", i, data);
2101 }
2102
2103 TAVOR_TNF_EXIT(tavor_ecc_detection_handler);
2104 return (DDI_SUCCESS);
2105 }
2106
2107
2108 /*
2109 * tavor_eq_overflow_handler()
2110 * Context: Only called from interrupt context
2111 */
2112 void
2113 tavor_eq_overflow_handler(tavor_state_t *state, tavor_eqhdl_t eq,
2114 tavor_hw_eqe_t *eqe)
2115 {
2116 uint_t error_type, data;
2117
2118 TAVOR_TNF_ENTER(tavor_eq_overflow_handler);
2119
2120 ASSERT(TAVOR_EQE_EVTTYPE_GET(eq, eqe) == TAVOR_EVT_EQ_OVERFLOW);
2121
2122 /*
2123 * The "Event Queue Overflow Event" indicates that something has
2124 * probably gone seriously wrong with some hardware (or, perhaps,
2125 * with the software... though it's unlikely in this case). The EQE
2126 * provides some additional information about the errored EQ. So we
2127 * print a warning message here along with that additional information.
2128 */
2129 error_type = TAVOR_EQE_OPERRTYPE_GET(eq, eqe);
2130 data = TAVOR_EQE_OPERRDATA_GET(eq, eqe);
2131
2132 TAVOR_WARNING(state, "Event Queue overflow");
2133 cmn_err(CE_CONT, " Error type: %02x, data: %08x\n", error_type, data);
2134
2135 TAVOR_TNF_EXIT(tavor_eq_overflow_handler);
2136 }
2137
2138
2139 /*
2140 * tavor_no_eqhandler
2141 * Context: Only called from interrupt context
2142 */
2143 /* ARGSUSED */
2144 static int
2145 tavor_no_eqhandler(tavor_state_t *state, tavor_eqhdl_t eq,
2146 tavor_hw_eqe_t *eqe)
2147 {
2148 uint_t data;
2149 int i;
2150
2151 TAVOR_TNF_ENTER(tavor_no_eqhandler);
2152
2153 /*
2154 * This "unexpected event" handler (or "catch-all" handler) will
2155 * receive all events for which no other handler has been registered.
2156 * If we end up here, then something has probably gone seriously wrong
2157 * with the Tavor hardware (or, perhaps, with the software... though
2158 * it's unlikely in this case). The EQE provides all the information
2159 * about the event. So we print a warning message here along with
2160 * the contents of the EQE.
2161 */
2162 TAVOR_WARNING(state, "Unexpected Event handler");
2163 cmn_err(CE_CONT, " Event type: %02x, subtype: %02x\n",
2164 TAVOR_EQE_EVTTYPE_GET(eq, eqe), TAVOR_EQE_EVTSUBTYPE_GET(eq, eqe));
2165 for (i = 0; i < sizeof (tavor_hw_eqe_t) >> 2; i++) {
2166 data = ((uint_t *)eqe)[i];
2167 cmn_err(CE_CONT, " EQE[%02x]: %08x\n", i, data);
2168 }
2169
2170 TAVOR_TNF_EXIT(tavor_no_eqhandler);
2171 return (DDI_SUCCESS);
2172 }