1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25
26 /*
27 * hermon_umap.c
28 * Hermon Userland Mapping Routines
29 *
30 * Implements all the routines necessary for enabling direct userland
31 * access to the Hermon hardware. This includes all routines necessary for
32 * maintaining the "userland resources database" and all the support routines
33 * for the devmap calls.
34 */
35
36 #include <sys/types.h>
37 #include <sys/conf.h>
38 #include <sys/ddi.h>
39 #include <sys/sunddi.h>
40 #include <sys/modctl.h>
41 #include <sys/file.h>
42 #include <sys/avl.h>
43 #include <sys/sysmacros.h>
44
45 #include <sys/ib/adapters/hermon/hermon.h>
46
47 /* Hermon HCA state pointer (extern) */
48 extern void *hermon_statep;
49
50 /* Hermon HCA Userland Resource Database (extern) */
51 extern hermon_umap_db_t hermon_userland_rsrc_db;
52
53 static int hermon_umap_uarpg(hermon_state_t *state, devmap_cookie_t dhp,
54 hermon_rsrc_t *rsrcp, uint64_t offset, size_t *maplen, int *err);
55 static int hermon_umap_cqmem(hermon_state_t *state, devmap_cookie_t dhp,
56 hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err);
57 static int hermon_umap_qpmem(hermon_state_t *state, devmap_cookie_t dhp,
58 hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err);
59 static int hermon_umap_srqmem(hermon_state_t *state, devmap_cookie_t dhp,
60 hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err);
61 static int hermon_umap_dbrecmem(hermon_state_t *state, devmap_cookie_t dhp,
62 hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err);
63 static int hermon_devmap_umem_map(devmap_cookie_t dhp, dev_t dev, uint_t flags,
64 offset_t off, size_t len, void **pvtp);
65 static int hermon_devmap_umem_dup(devmap_cookie_t dhp, void *pvtp,
66 devmap_cookie_t new_dhp, void **new_pvtp);
67 static void hermon_devmap_umem_unmap(devmap_cookie_t dhp, void *pvtp,
68 offset_t off, size_t len, devmap_cookie_t new_dhp1, void **pvtp1,
69 devmap_cookie_t new_dhp2, void **pvtp2);
70 static int hermon_devmap_dbrecmem_map(devmap_cookie_t dhp, dev_t dev,
71 uint_t flags, offset_t off, size_t len, void **pvtp);
72 static int hermon_devmap_dbrecmem_dup(devmap_cookie_t dhp, void *pvtp,
73 devmap_cookie_t new_dhp, void **new_pvtp);
74 static void hermon_devmap_dbrecmem_unmap(devmap_cookie_t dhp, void *pvtp,
75 offset_t off, size_t len, devmap_cookie_t new_dhp1, void **pvtp1,
76 devmap_cookie_t new_dhp2, void **pvtp2);
77 static int hermon_devmap_devmem_map(devmap_cookie_t dhp, dev_t dev,
78 uint_t flags, offset_t off, size_t len, void **pvtp);
79 static int hermon_devmap_devmem_dup(devmap_cookie_t dhp, void *pvtp,
80 devmap_cookie_t new_dhp, void **new_pvtp);
81 static void hermon_devmap_devmem_unmap(devmap_cookie_t dhp, void *pvtp,
82 offset_t off, size_t len, devmap_cookie_t new_dhp1, void **pvtp1,
83 devmap_cookie_t new_dhp2, void **pvtp2);
84 static ibt_status_t hermon_umap_mr_data_in(hermon_mrhdl_t mr,
85 ibt_mr_data_in_t *data, size_t data_sz);
86 static ibt_status_t hermon_umap_cq_data_out(hermon_cqhdl_t cq,
87 mlnx_umap_cq_data_out_t *data, size_t data_sz);
88 static ibt_status_t hermon_umap_qp_data_out(hermon_qphdl_t qp,
89 mlnx_umap_qp_data_out_t *data, size_t data_sz);
90 static ibt_status_t hermon_umap_srq_data_out(hermon_srqhdl_t srq,
91 mlnx_umap_srq_data_out_t *data, size_t data_sz);
92 static ibt_status_t hermon_umap_pd_data_out(hermon_pdhdl_t pd,
93 mlnx_umap_pd_data_out_t *data, size_t data_sz);
94 static int hermon_umap_db_compare(const void *query, const void *entry);
95
96
97 /*
98 * These callbacks are passed to devmap_umem_setup() and devmap_devmem_setup(),
99 * respectively. They are used to handle (among other things) partial
100 * unmappings and to provide a method for invalidating mappings inherited
101 * as a result of a fork(2) system call.
102 */
103 static struct devmap_callback_ctl hermon_devmap_umem_cbops = {
104 DEVMAP_OPS_REV,
105 hermon_devmap_umem_map,
106 NULL,
107 hermon_devmap_umem_dup,
108 hermon_devmap_umem_unmap
109 };
110 static struct devmap_callback_ctl hermon_devmap_devmem_cbops = {
111 DEVMAP_OPS_REV,
112 hermon_devmap_devmem_map,
113 NULL,
114 hermon_devmap_devmem_dup,
115 hermon_devmap_devmem_unmap
116 };
117 static struct devmap_callback_ctl hermon_devmap_dbrecmem_cbops = {
118 DEVMAP_OPS_REV,
119 hermon_devmap_dbrecmem_map,
120 NULL,
121 hermon_devmap_dbrecmem_dup,
122 hermon_devmap_dbrecmem_unmap
123 };
124
125 /*
126 * hermon_devmap()
127 * Context: Can be called from user context.
128 */
129 /* ARGSUSED */
130 int
131 hermon_devmap(dev_t dev, devmap_cookie_t dhp, offset_t off, size_t len,
132 size_t *maplen, uint_t model)
133 {
134 hermon_state_t *state;
135 hermon_rsrc_t *rsrcp;
136 minor_t instance;
137 uint64_t key, value;
138 uint64_t bf_offset = 0;
139 uint_t type;
140 int err, status;
141
142 /* Get Hermon softstate structure from instance */
143 instance = HERMON_DEV_INSTANCE(dev);
144 state = ddi_get_soft_state(hermon_statep, instance);
145 if (state == NULL) {
146 return (ENXIO);
147 }
148
149 /*
150 * Access to Hermon devmap interface is not allowed in
151 * "maintenance mode".
152 */
153 if (state->hs_operational_mode == HERMON_MAINTENANCE_MODE) {
154 return (EFAULT);
155 }
156
157 /*
158 * The bottom bits of "offset" are undefined (number depends on
159 * system PAGESIZE). Shifting these off leaves us with a "key".
160 * The "key" is actually a combination of both a real key value
161 * (for the purpose of database lookup) and a "type" value. We
162 * extract this information before doing the database lookup.
163 */
164 key = off >> PAGESHIFT;
165 type = key & MLNX_UMAP_RSRC_TYPE_MASK;
166 key = key >> MLNX_UMAP_RSRC_TYPE_SHIFT;
167 if (type == MLNX_UMAP_BLUEFLAMEPG_RSRC) {
168 if (state->hs_devlim.blu_flm == 0) {
169 return (EFAULT);
170 }
171 bf_offset = state->hs_bf_offset;
172 type = MLNX_UMAP_UARPG_RSRC;
173 }
174 status = hermon_umap_db_find(instance, key, type, &value, 0, NULL);
175 if (status == DDI_SUCCESS) {
176 rsrcp = (hermon_rsrc_t *)(uintptr_t)value;
177
178 switch (type) {
179 case MLNX_UMAP_UARPG_RSRC:
180 /*
181 * Double check that process who open()'d Hermon is
182 * same process attempting to mmap() UAR page.
183 */
184 if (key != ddi_get_pid()) {
185 return (EINVAL);
186 }
187
188 /* Map the UAR page out for userland access */
189 status = hermon_umap_uarpg(state, dhp, rsrcp, bf_offset,
190 maplen, &err);
191 if (status != DDI_SUCCESS) {
192 return (err);
193 }
194 break;
195
196 case MLNX_UMAP_CQMEM_RSRC:
197 /* Map the CQ memory out for userland access */
198 status = hermon_umap_cqmem(state, dhp, rsrcp, off,
199 maplen, &err);
200 if (status != DDI_SUCCESS) {
201 return (err);
202 }
203 break;
204
205 case MLNX_UMAP_QPMEM_RSRC:
206 /* Map the QP memory out for userland access */
207 status = hermon_umap_qpmem(state, dhp, rsrcp, off,
208 maplen, &err);
209 if (status != DDI_SUCCESS) {
210 return (err);
211 }
212 break;
213
214 case MLNX_UMAP_SRQMEM_RSRC:
215 /* Map the SRQ memory out for userland access */
216 status = hermon_umap_srqmem(state, dhp, rsrcp, off,
217 maplen, &err);
218 if (status != DDI_SUCCESS) {
219 return (err);
220 }
221 break;
222
223 case MLNX_UMAP_DBRMEM_RSRC:
224 /*
225 * Map the doorbell record memory out for
226 * userland access
227 */
228 status = hermon_umap_dbrecmem(state, dhp, rsrcp, off,
229 maplen, &err);
230 if (status != DDI_SUCCESS) {
231 return (err);
232 }
233 break;
234
235 default:
236 HERMON_WARNING(state, "unexpected rsrc type in devmap");
237 return (EINVAL);
238 }
239 } else {
240 return (EINVAL);
241 }
242
243 return (0);
244 }
245
246
247 /*
248 * hermon_umap_uarpg()
249 * Context: Can be called from user context.
250 */
251 static int
252 hermon_umap_uarpg(hermon_state_t *state, devmap_cookie_t dhp,
253 hermon_rsrc_t *rsrcp, uint64_t offset, size_t *maplen, int *err)
254 {
255 int status;
256 uint_t maxprot;
257 ddi_device_acc_attr_t *accattrp = &state->hs_reg_accattr;
258 ddi_device_acc_attr_t accattr;
259
260 if (offset != 0) { /* Hermon Blueflame */
261 /* Try to use write coalescing data ordering */
262 accattr = *accattrp;
263 accattr.devacc_attr_dataorder = DDI_STORECACHING_OK_ACC;
264 accattrp = &accattr;
265 }
266
267 /* Map out the UAR page (doorbell page) */
268 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
269 status = devmap_devmem_setup(dhp, state->hs_dip,
270 &hermon_devmap_devmem_cbops, HERMON_UAR_BAR, (rsrcp->hr_indx <<
271 PAGESHIFT) + offset, PAGESIZE, maxprot, DEVMAP_ALLOW_REMAP,
272 accattrp);
273 if (status < 0) {
274 *err = status;
275 return (DDI_FAILURE);
276 }
277
278 *maplen = PAGESIZE;
279 return (DDI_SUCCESS);
280 }
281
282
283 /*
284 * hermon_umap_cqmem()
285 * Context: Can be called from user context.
286 */
287 /* ARGSUSED */
288 static int
289 hermon_umap_cqmem(hermon_state_t *state, devmap_cookie_t dhp,
290 hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err)
291 {
292 hermon_cqhdl_t cq;
293 size_t size;
294 uint_t maxprot;
295 int status;
296
297 /* Extract the Hermon CQ handle pointer from the hermon_rsrc_t */
298 cq = (hermon_cqhdl_t)rsrcp->hr_addr;
299
300 /* Round-up the CQ size to system page size */
301 size = ptob(btopr(cq->cq_resize_hdl ?
302 cq->cq_resize_hdl->cq_cqinfo.qa_size : cq->cq_cqinfo.qa_size));
303
304 /* Map out the CQ memory - use resize_hdl if non-NULL */
305 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
306 status = devmap_umem_setup(dhp, state->hs_dip,
307 &hermon_devmap_umem_cbops, cq->cq_resize_hdl ?
308 cq->cq_resize_hdl->cq_cqinfo.qa_umemcookie :
309 cq->cq_cqinfo.qa_umemcookie, 0, size,
310 maxprot, (DEVMAP_ALLOW_REMAP | DEVMAP_DEFAULTS), NULL);
311 if (status < 0) {
312 *err = status;
313 return (DDI_FAILURE);
314 }
315 *maplen = size;
316
317 return (DDI_SUCCESS);
318 }
319
320
321 /*
322 * hermon_umap_qpmem()
323 * Context: Can be called from user context.
324 */
325 /* ARGSUSED */
326 static int
327 hermon_umap_qpmem(hermon_state_t *state, devmap_cookie_t dhp,
328 hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err)
329 {
330 hermon_qphdl_t qp;
331 offset_t offset;
332 size_t size;
333 uint_t maxprot;
334 int status;
335
336 /* Extract the Hermon QP handle pointer from the hermon_rsrc_t */
337 qp = (hermon_qphdl_t)rsrcp->hr_addr;
338
339 /*
340 * Calculate the offset of the first work queue (send or recv) into
341 * the memory (ddi_umem_alloc()) allocated previously for the QP.
342 */
343 offset = (offset_t)((uintptr_t)qp->qp_wqinfo.qa_buf_aligned -
344 (uintptr_t)qp->qp_wqinfo.qa_buf_real);
345
346 /* Round-up the QP work queue sizes to system page size */
347 size = ptob(btopr(qp->qp_wqinfo.qa_size));
348
349 /* Map out the QP memory */
350 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
351 status = devmap_umem_setup(dhp, state->hs_dip,
352 &hermon_devmap_umem_cbops, qp->qp_wqinfo.qa_umemcookie, offset,
353 size, maxprot, (DEVMAP_ALLOW_REMAP | DEVMAP_DEFAULTS), NULL);
354 if (status < 0) {
355 *err = status;
356 return (DDI_FAILURE);
357 }
358 *maplen = size;
359
360 return (DDI_SUCCESS);
361 }
362
363
364 /*
365 * hermon_umap_srqmem()
366 * Context: Can be called from user context.
367 */
368 /* ARGSUSED */
369 static int
370 hermon_umap_srqmem(hermon_state_t *state, devmap_cookie_t dhp,
371 hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err)
372 {
373 hermon_srqhdl_t srq;
374 offset_t offset;
375 size_t size;
376 uint_t maxprot;
377 int status;
378
379 /* Extract the Hermon SRQ handle pointer from the hermon_rsrc_t */
380 srq = (hermon_srqhdl_t)rsrcp->hr_addr;
381
382 /*
383 * Calculate the offset of the first shared recv queue into the memory
384 * (ddi_umem_alloc()) allocated previously for the SRQ.
385 */
386 offset = (offset_t)((uintptr_t)srq->srq_wqinfo.qa_buf_aligned -
387 (uintptr_t)srq->srq_wqinfo.qa_buf_real);
388
389 /* Round-up the SRQ work queue sizes to system page size */
390 size = ptob(btopr(srq->srq_wqinfo.qa_size));
391
392 /* Map out the SRQ memory */
393 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
394 status = devmap_umem_setup(dhp, state->hs_dip,
395 &hermon_devmap_umem_cbops, srq->srq_wqinfo.qa_umemcookie, offset,
396 size, maxprot, (DEVMAP_ALLOW_REMAP | DEVMAP_DEFAULTS), NULL);
397 if (status < 0) {
398 *err = status;
399 return (DDI_FAILURE);
400 }
401 *maplen = size;
402
403 return (DDI_SUCCESS);
404 }
405
406
407 /*
408 * hermon_devmap_dbrecmem()
409 * Context: Can be called from user context.
410 */
411 /* ARGSUSED */
412 static int
413 hermon_umap_dbrecmem(hermon_state_t *state, devmap_cookie_t dhp,
414 hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err)
415 {
416 hermon_udbr_page_t *pagep;
417 offset_t offset;
418 size_t size;
419 uint_t maxprot;
420 int status;
421
422 /* We stored the udbr_page pointer, and not a hermon_rsrc_t */
423 pagep = (hermon_udbr_page_t *)rsrcp;
424
425 /*
426 * Calculate the offset of the doorbell records into the memory
427 * (ddi_umem_alloc()) allocated previously for them.
428 */
429 offset = 0;
430
431 /* Round-up the doorbell page to system page size */
432 size = PAGESIZE;
433
434 /* Map out the Doorbell Record memory */
435 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
436 status = devmap_umem_setup(dhp, state->hs_dip,
437 &hermon_devmap_dbrecmem_cbops, pagep->upg_umemcookie, offset,
438 size, maxprot, (DEVMAP_ALLOW_REMAP | DEVMAP_DEFAULTS), NULL);
439 if (status < 0) {
440 *err = status;
441 return (DDI_FAILURE);
442 }
443 *maplen = size;
444
445 return (DDI_SUCCESS);
446 }
447
448
449 /*
450 * hermon_devmap_umem_map()
451 * Context: Can be called from kernel context.
452 */
453 /* ARGSUSED */
454 static int
455 hermon_devmap_umem_map(devmap_cookie_t dhp, dev_t dev, uint_t flags,
456 offset_t off, size_t len, void **pvtp)
457 {
458 hermon_state_t *state;
459 hermon_devmap_track_t *dvm_track;
460 hermon_cqhdl_t cq;
461 hermon_qphdl_t qp;
462 hermon_srqhdl_t srq;
463 minor_t instance;
464 uint64_t key;
465 uint_t type;
466
467 /* Get Hermon softstate structure from instance */
468 instance = HERMON_DEV_INSTANCE(dev);
469 state = ddi_get_soft_state(hermon_statep, instance);
470 if (state == NULL) {
471 return (ENXIO);
472 }
473
474 /*
475 * The bottom bits of "offset" are undefined (number depends on
476 * system PAGESIZE). Shifting these off leaves us with a "key".
477 * The "key" is actually a combination of both a real key value
478 * (for the purpose of database lookup) and a "type" value. Although
479 * we are not going to do any database lookup per se, we do want
480 * to extract the "key" and the "type" (to enable faster lookup of
481 * the appropriate CQ or QP handle).
482 */
483 key = off >> PAGESHIFT;
484 type = key & MLNX_UMAP_RSRC_TYPE_MASK;
485 key = key >> MLNX_UMAP_RSRC_TYPE_SHIFT;
486
487 /*
488 * Allocate an entry to track the mapping and unmapping (specifically,
489 * partial unmapping) of this resource.
490 */
491 dvm_track = (hermon_devmap_track_t *)kmem_zalloc(
492 sizeof (hermon_devmap_track_t), KM_SLEEP);
493 dvm_track->hdt_offset = off;
494 dvm_track->hdt_state = state;
495 dvm_track->hdt_refcnt = 1;
496 mutex_init(&dvm_track->hdt_lock, NULL, MUTEX_DRIVER,
497 DDI_INTR_PRI(state->hs_intrmsi_pri));
498
499 /*
500 * Depending of the type of resource that has been mapped out, we
501 * need to update the QP or CQ handle to reflect that it has, in
502 * fact, been mapped. This allows the driver code which frees a QP
503 * or a CQ to know whether it is appropriate to do a
504 * devmap_devmem_remap() to invalidate the userland mapping for the
505 * corresponding queue's memory.
506 */
507 if (type == MLNX_UMAP_CQMEM_RSRC) {
508
509 /* Use "key" (CQ number) to do fast lookup of CQ handle */
510 cq = hermon_cqhdl_from_cqnum(state, key);
511
512 /*
513 * Update the handle to the userland mapping. Note: If
514 * the CQ already has a valid userland mapping, then stop
515 * and return failure.
516 */
517 mutex_enter(&cq->cq_lock);
518 if (cq->cq_umap_dhp == NULL) {
519 cq->cq_umap_dhp = dhp;
520 dvm_track->hdt_size = cq->cq_cqinfo.qa_size;
521 mutex_exit(&cq->cq_lock);
522 } else if (cq->cq_resize_hdl &&
523 (cq->cq_resize_hdl->cq_umap_dhp == NULL)) {
524 cq->cq_resize_hdl->cq_umap_dhp = dhp;
525 dvm_track->hdt_size =
526 cq->cq_resize_hdl->cq_cqinfo.qa_size;
527 mutex_exit(&cq->cq_lock);
528 } else {
529 mutex_exit(&cq->cq_lock);
530 goto umem_map_fail;
531 }
532
533 } else if (type == MLNX_UMAP_QPMEM_RSRC) {
534
535 /* Use "key" (QP number) to do fast lookup of QP handle */
536 qp = hermon_qphdl_from_qpnum(state, key);
537
538 /*
539 * Update the handle to the userland mapping. Note: If
540 * the CQ already has a valid userland mapping, then stop
541 * and return failure.
542 */
543 mutex_enter(&qp->qp_lock);
544 if (qp->qp_umap_dhp == NULL) {
545 qp->qp_umap_dhp = dhp;
546 dvm_track->hdt_size = qp->qp_wqinfo.qa_size;
547 mutex_exit(&qp->qp_lock);
548 } else {
549 mutex_exit(&qp->qp_lock);
550 goto umem_map_fail;
551 }
552
553 } else if (type == MLNX_UMAP_SRQMEM_RSRC) {
554
555 /* Use "key" (SRQ number) to do fast lookup on SRQ handle */
556 srq = hermon_srqhdl_from_srqnum(state, key);
557
558 /*
559 * Update the handle to the userland mapping. Note: If the
560 * SRQ already has a valid userland mapping, then stop and
561 * return failure.
562 */
563 mutex_enter(&srq->srq_lock);
564 if (srq->srq_umap_dhp == NULL) {
565 srq->srq_umap_dhp = dhp;
566 dvm_track->hdt_size = srq->srq_wqinfo.qa_size;
567 mutex_exit(&srq->srq_lock);
568 } else {
569 mutex_exit(&srq->srq_lock);
570 goto umem_map_fail;
571 }
572 }
573
574 /*
575 * Pass the private "Hermon devmap tracking structure" back. This
576 * pointer will be returned in subsequent "unmap" callbacks.
577 */
578 *pvtp = dvm_track;
579
580 return (DDI_SUCCESS);
581
582 umem_map_fail:
583 mutex_destroy(&dvm_track->hdt_lock);
584 kmem_free(dvm_track, sizeof (hermon_devmap_track_t));
585 return (DDI_FAILURE);
586 }
587
588
589 /*
590 * hermon_devmap_umem_dup()
591 * Context: Can be called from kernel context.
592 */
593 /* ARGSUSED */
594 static int
595 hermon_devmap_umem_dup(devmap_cookie_t dhp, void *pvtp, devmap_cookie_t new_dhp,
596 void **new_pvtp)
597 {
598 hermon_state_t *state;
599 hermon_devmap_track_t *dvm_track, *new_dvm_track;
600 uint_t maxprot;
601 int status;
602
603 /*
604 * Extract the Hermon softstate pointer from "Hermon devmap tracking
605 * structure" (in "pvtp").
606 */
607 dvm_track = (hermon_devmap_track_t *)pvtp;
608 state = dvm_track->hdt_state;
609
610 /*
611 * Since this devmap_dup() entry point is generally called
612 * when a process does fork(2), it is incumbent upon the driver
613 * to insure that the child does not inherit a valid copy of
614 * the parent's QP or CQ resource. This is accomplished by using
615 * devmap_devmem_remap() to invalidate the child's mapping to the
616 * kernel memory.
617 */
618 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
619 status = devmap_devmem_remap(new_dhp, state->hs_dip, 0, 0,
620 dvm_track->hdt_size, maxprot, DEVMAP_MAPPING_INVALID, NULL);
621 if (status != DDI_SUCCESS) {
622 HERMON_WARNING(state, "failed in hermon_devmap_umem_dup()");
623 return (status);
624 }
625
626 /*
627 * Allocate a new entry to track the subsequent unmapping
628 * (specifically, all partial unmappings) of the child's newly
629 * invalidated resource. Note: Setting the "hdt_size" field to
630 * zero here is an indication to the devmap_unmap() entry point
631 * that this mapping is invalid, and that its subsequent unmapping
632 * should not affect any of the parent's CQ or QP resources.
633 */
634 new_dvm_track = (hermon_devmap_track_t *)kmem_zalloc(
635 sizeof (hermon_devmap_track_t), KM_SLEEP);
636 new_dvm_track->hdt_offset = 0;
637 new_dvm_track->hdt_state = state;
638 new_dvm_track->hdt_refcnt = 1;
639 new_dvm_track->hdt_size = 0;
640 mutex_init(&new_dvm_track->hdt_lock, NULL, MUTEX_DRIVER,
641 DDI_INTR_PRI(state->hs_intrmsi_pri));
642 *new_pvtp = new_dvm_track;
643
644 return (DDI_SUCCESS);
645 }
646
647
648 /*
649 * hermon_devmap_umem_unmap()
650 * Context: Can be called from kernel context.
651 */
652 /* ARGSUSED */
653 static void
654 hermon_devmap_umem_unmap(devmap_cookie_t dhp, void *pvtp, offset_t off,
655 size_t len, devmap_cookie_t new_dhp1, void **pvtp1,
656 devmap_cookie_t new_dhp2, void **pvtp2)
657 {
658 hermon_state_t *state;
659 hermon_rsrc_t *rsrcp;
660 hermon_devmap_track_t *dvm_track;
661 hermon_cqhdl_t cq;
662 hermon_qphdl_t qp;
663 hermon_srqhdl_t srq;
664 uint64_t key, value;
665 uint_t type;
666 uint_t size;
667 int status;
668
669 /*
670 * Extract the Hermon softstate pointer from "Hermon devmap tracking
671 * structure" (in "pvtp").
672 */
673 dvm_track = (hermon_devmap_track_t *)pvtp;
674 state = dvm_track->hdt_state;
675
676 /*
677 * Extract the "offset" from the "Hermon devmap tracking structure".
678 * Note: The input argument "off" is ignored here because the
679 * Hermon mapping interfaces define a very specific meaning to
680 * each "logical offset". Also extract the "key" and "type" encoded
681 * in the logical offset.
682 */
683 key = dvm_track->hdt_offset >> PAGESHIFT;
684 type = key & MLNX_UMAP_RSRC_TYPE_MASK;
685 key = key >> MLNX_UMAP_RSRC_TYPE_SHIFT;
686
687 /*
688 * Extract the "size" of the mapping. If this size is determined
689 * to be zero, then it is an indication of a previously invalidated
690 * mapping, and no CQ or QP resources should be affected.
691 */
692 size = dvm_track->hdt_size;
693
694 /*
695 * If only the "middle portion of a given mapping is being unmapped,
696 * then we are effectively creating one new piece of mapped memory.
697 * (Original region is divided into three pieces of which the middle
698 * piece is being removed. This leaves two pieces. Since we started
699 * with one piece and now have two pieces, we need to increment the
700 * counter in the "Hermon devmap tracking structure".
701 *
702 * If, however, the whole mapped region is being unmapped, then we
703 * have started with one region which we are completely removing.
704 * In this case, we need to decrement the counter in the "Hermon
705 * devmap tracking structure".
706 *
707 * In each of the remaining cases, we will have started with one
708 * mapped region and ended with one (different) region. So no counter
709 * modification is necessary.
710 */
711 mutex_enter(&dvm_track->hdt_lock);
712 if ((new_dhp1 == NULL) && (new_dhp2 == NULL)) {
713 dvm_track->hdt_refcnt--;
714 } else if ((new_dhp1 != NULL) && (new_dhp2 != NULL)) {
715 dvm_track->hdt_refcnt++;
716 }
717 mutex_exit(&dvm_track->hdt_lock);
718
719 /*
720 * For each of the cases where the region is being divided, then we
721 * need to pass back the "Hermon devmap tracking structure". This way
722 * we get it back when each of the remaining pieces is subsequently
723 * unmapped.
724 */
725 if (new_dhp1 != NULL) {
726 *pvtp1 = pvtp;
727 }
728 if (new_dhp2 != NULL) {
729 *pvtp2 = pvtp;
730 }
731
732 /*
733 * If the "Hermon devmap tracking structure" is no longer being
734 * referenced, then free it up. Otherwise, return.
735 */
736 if (dvm_track->hdt_refcnt == 0) {
737 mutex_destroy(&dvm_track->hdt_lock);
738 kmem_free(dvm_track, sizeof (hermon_devmap_track_t));
739
740 /*
741 * If the mapping was invalid (see explanation above), then
742 * no further processing is necessary.
743 */
744 if (size == 0) {
745 return;
746 }
747 } else {
748 return;
749 }
750
751 /*
752 * Now that we can guarantee that the user memory is fully unmapped,
753 * we can use the "key" and "type" values to try to find the entry
754 * in the "userland resources database". If it's found, then it
755 * indicates that the queue memory (CQ or QP) has not yet been freed.
756 * In this case, we update the corresponding CQ or QP handle to
757 * indicate that the "devmap_devmem_remap()" call will be unnecessary.
758 * If it's _not_ found, then it indicates that the CQ or QP memory
759 * was, in fact, freed before it was unmapped (thus requiring a
760 * previous invalidation by remapping - which will already have
761 * been done in the free routine).
762 */
763 status = hermon_umap_db_find(state->hs_instance, key, type, &value,
764 0, NULL);
765 if (status == DDI_SUCCESS) {
766 /*
767 * Depending on the type of the mapped resource (CQ or QP),
768 * update handle to indicate that no invalidation remapping
769 * will be necessary.
770 */
771 if (type == MLNX_UMAP_CQMEM_RSRC) {
772
773 /* Use "value" to convert to CQ handle */
774 rsrcp = (hermon_rsrc_t *)(uintptr_t)value;
775 cq = (hermon_cqhdl_t)rsrcp->hr_addr;
776
777 /*
778 * Invalidate the handle to the userland mapping.
779 * Note: We must ensure that the mapping being
780 * unmapped here is the current one for the CQ. It
781 * is possible that it might not be if this CQ has
782 * been resized and the previous CQ memory has not
783 * yet been unmapped. But in that case, because of
784 * the devmap_devmem_remap(), there is no longer any
785 * association between the mapping and the real CQ
786 * kernel memory.
787 */
788 mutex_enter(&cq->cq_lock);
789 if (cq->cq_umap_dhp == dhp) {
790 cq->cq_umap_dhp = NULL;
791 if (cq->cq_resize_hdl) {
792 /* resize is DONE, switch queues */
793 hermon_cq_resize_helper(state, cq);
794 }
795 } else {
796 if (cq->cq_resize_hdl &&
797 cq->cq_resize_hdl->cq_umap_dhp == dhp) {
798 /*
799 * Unexpected case. munmap of the
800 * cq_resize buf, and not the
801 * original buf.
802 */
803 cq->cq_resize_hdl->cq_umap_dhp = NULL;
804 }
805 }
806 mutex_exit(&cq->cq_lock);
807
808 } else if (type == MLNX_UMAP_QPMEM_RSRC) {
809
810 /* Use "value" to convert to QP handle */
811 rsrcp = (hermon_rsrc_t *)(uintptr_t)value;
812 qp = (hermon_qphdl_t)rsrcp->hr_addr;
813
814 /*
815 * Invalidate the handle to the userland mapping.
816 * Note: we ensure that the mapping being unmapped
817 * here is the current one for the QP. This is
818 * more of a sanity check here since, unlike CQs
819 * (above) we do not support resize of QPs.
820 */
821 mutex_enter(&qp->qp_lock);
822 if (qp->qp_umap_dhp == dhp) {
823 qp->qp_umap_dhp = NULL;
824 }
825 mutex_exit(&qp->qp_lock);
826
827 } else if (type == MLNX_UMAP_SRQMEM_RSRC) {
828
829 /* Use "value" to convert to SRQ handle */
830 rsrcp = (hermon_rsrc_t *)(uintptr_t)value;
831 srq = (hermon_srqhdl_t)rsrcp->hr_addr;
832
833 /*
834 * Invalidate the handle to the userland mapping.
835 * Note: we ensure that the mapping being unmapped
836 * here is the current one for the QP. This is
837 * more of a sanity check here since, unlike CQs
838 * (above) we do not support resize of QPs.
839 */
840 mutex_enter(&srq->srq_lock);
841 if (srq->srq_umap_dhp == dhp) {
842 srq->srq_umap_dhp = NULL;
843 }
844 mutex_exit(&srq->srq_lock);
845 }
846 }
847 }
848
849
850 /*
851 * hermon_devmap_devmem_map()
852 * Context: Can be called from kernel context.
853 */
854 /* ARGSUSED */
855 static int
856 hermon_devmap_dbrecmem_map(devmap_cookie_t dhp, dev_t dev, uint_t flags,
857 offset_t off, size_t len, void **pvtp)
858 {
859 hermon_state_t *state;
860 hermon_devmap_track_t *dvm_track;
861 hermon_cqhdl_t cq;
862 hermon_qphdl_t qp;
863 hermon_srqhdl_t srq;
864 minor_t instance;
865 uint64_t key;
866 uint_t type;
867
868 /* Get Hermon softstate structure from instance */
869 instance = HERMON_DEV_INSTANCE(dev);
870 state = ddi_get_soft_state(hermon_statep, instance);
871 if (state == NULL) {
872 return (ENXIO);
873 }
874
875 /*
876 * The bottom bits of "offset" are undefined (number depends on
877 * system PAGESIZE). Shifting these off leaves us with a "key".
878 * The "key" is actually a combination of both a real key value
879 * (for the purpose of database lookup) and a "type" value. Although
880 * we are not going to do any database lookup per se, we do want
881 * to extract the "key" and the "type" (to enable faster lookup of
882 * the appropriate CQ or QP handle).
883 */
884 key = off >> PAGESHIFT;
885 type = key & MLNX_UMAP_RSRC_TYPE_MASK;
886 key = key >> MLNX_UMAP_RSRC_TYPE_SHIFT;
887
888 /*
889 * Allocate an entry to track the mapping and unmapping (specifically,
890 * partial unmapping) of this resource.
891 */
892 dvm_track = (hermon_devmap_track_t *)kmem_zalloc(
893 sizeof (hermon_devmap_track_t), KM_SLEEP);
894 dvm_track->hdt_offset = off;
895 dvm_track->hdt_state = state;
896 dvm_track->hdt_refcnt = 1;
897 mutex_init(&dvm_track->hdt_lock, NULL, MUTEX_DRIVER,
898 DDI_INTR_PRI(state->hs_intrmsi_pri));
899
900 /*
901 * Depending of the type of resource that has been mapped out, we
902 * need to update the QP or CQ handle to reflect that it has, in
903 * fact, been mapped. This allows the driver code which frees a QP
904 * or a CQ to know whether it is appropriate to do a
905 * devmap_devmem_remap() to invalidate the userland mapping for the
906 * corresponding queue's memory.
907 */
908 if (type == MLNX_UMAP_CQMEM_RSRC) {
909
910 /* Use "key" (CQ number) to do fast lookup of CQ handle */
911 cq = hermon_cqhdl_from_cqnum(state, key);
912
913 /*
914 * Update the handle to the userland mapping. Note: If
915 * the CQ already has a valid userland mapping, then stop
916 * and return failure.
917 */
918 mutex_enter(&cq->cq_lock);
919 if (cq->cq_umap_dhp == NULL) {
920 cq->cq_umap_dhp = dhp;
921 dvm_track->hdt_size = cq->cq_cqinfo.qa_size;
922 mutex_exit(&cq->cq_lock);
923 } else {
924 mutex_exit(&cq->cq_lock);
925 goto umem_map_fail;
926 }
927
928 } else if (type == MLNX_UMAP_QPMEM_RSRC) {
929
930 /* Use "key" (QP number) to do fast lookup of QP handle */
931 qp = hermon_qphdl_from_qpnum(state, key);
932
933 /*
934 * Update the handle to the userland mapping. Note: If
935 * the CQ already has a valid userland mapping, then stop
936 * and return failure.
937 */
938 mutex_enter(&qp->qp_lock);
939 if (qp->qp_umap_dhp == NULL) {
940 qp->qp_umap_dhp = dhp;
941 dvm_track->hdt_size = qp->qp_wqinfo.qa_size;
942 mutex_exit(&qp->qp_lock);
943 } else {
944 mutex_exit(&qp->qp_lock);
945 goto umem_map_fail;
946 }
947
948 } else if (type == MLNX_UMAP_SRQMEM_RSRC) {
949
950 /* Use "key" (SRQ number) to do fast lookup on SRQ handle */
951 srq = hermon_srqhdl_from_srqnum(state, key);
952
953 /*
954 * Update the handle to the userland mapping. Note: If the
955 * SRQ already has a valid userland mapping, then stop and
956 * return failure.
957 */
958 mutex_enter(&srq->srq_lock);
959 if (srq->srq_umap_dhp == NULL) {
960 srq->srq_umap_dhp = dhp;
961 dvm_track->hdt_size = srq->srq_wqinfo.qa_size;
962 mutex_exit(&srq->srq_lock);
963 } else {
964 mutex_exit(&srq->srq_lock);
965 goto umem_map_fail;
966 }
967 }
968
969 /*
970 * Pass the private "Hermon devmap tracking structure" back. This
971 * pointer will be returned in subsequent "unmap" callbacks.
972 */
973 *pvtp = dvm_track;
974
975 return (DDI_SUCCESS);
976
977 umem_map_fail:
978 mutex_destroy(&dvm_track->hdt_lock);
979 kmem_free(dvm_track, sizeof (hermon_devmap_track_t));
980 return (DDI_FAILURE);
981 }
982
983
984 /*
985 * hermon_devmap_dbrecmem_dup()
986 * Context: Can be called from kernel context.
987 */
988 /* ARGSUSED */
989 static int
990 hermon_devmap_dbrecmem_dup(devmap_cookie_t dhp, void *pvtp,
991 devmap_cookie_t new_dhp, void **new_pvtp)
992 {
993 hermon_state_t *state;
994 hermon_devmap_track_t *dvm_track, *new_dvm_track;
995 uint_t maxprot;
996 int status;
997
998 /*
999 * Extract the Hermon softstate pointer from "Hermon devmap tracking
1000 * structure" (in "pvtp").
1001 */
1002 dvm_track = (hermon_devmap_track_t *)pvtp;
1003 state = dvm_track->hdt_state;
1004
1005 /*
1006 * Since this devmap_dup() entry point is generally called
1007 * when a process does fork(2), it is incumbent upon the driver
1008 * to insure that the child does not inherit a valid copy of
1009 * the parent's QP or CQ resource. This is accomplished by using
1010 * devmap_devmem_remap() to invalidate the child's mapping to the
1011 * kernel memory.
1012 */
1013 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
1014 status = devmap_devmem_remap(new_dhp, state->hs_dip, 0, 0,
1015 dvm_track->hdt_size, maxprot, DEVMAP_MAPPING_INVALID, NULL);
1016 if (status != DDI_SUCCESS) {
1017 HERMON_WARNING(state, "failed in hermon_devmap_dbrecmem_dup()");
1018 return (status);
1019 }
1020
1021 /*
1022 * Allocate a new entry to track the subsequent unmapping
1023 * (specifically, all partial unmappings) of the child's newly
1024 * invalidated resource. Note: Setting the "hdt_size" field to
1025 * zero here is an indication to the devmap_unmap() entry point
1026 * that this mapping is invalid, and that its subsequent unmapping
1027 * should not affect any of the parent's CQ or QP resources.
1028 */
1029 new_dvm_track = (hermon_devmap_track_t *)kmem_zalloc(
1030 sizeof (hermon_devmap_track_t), KM_SLEEP);
1031 new_dvm_track->hdt_offset = 0;
1032 new_dvm_track->hdt_state = state;
1033 new_dvm_track->hdt_refcnt = 1;
1034 new_dvm_track->hdt_size = 0;
1035 mutex_init(&new_dvm_track->hdt_lock, NULL, MUTEX_DRIVER,
1036 DDI_INTR_PRI(state->hs_intrmsi_pri));
1037 *new_pvtp = new_dvm_track;
1038
1039 return (DDI_SUCCESS);
1040 }
1041
1042
1043 /*
1044 * hermon_devmap_dbrecmem_unmap()
1045 * Context: Can be called from kernel context.
1046 */
1047 /* ARGSUSED */
1048 static void
1049 hermon_devmap_dbrecmem_unmap(devmap_cookie_t dhp, void *pvtp, offset_t off,
1050 size_t len, devmap_cookie_t new_dhp1, void **pvtp1,
1051 devmap_cookie_t new_dhp2, void **pvtp2)
1052 {
1053 hermon_state_t *state;
1054 hermon_rsrc_t *rsrcp;
1055 hermon_devmap_track_t *dvm_track;
1056 hermon_cqhdl_t cq;
1057 hermon_qphdl_t qp;
1058 hermon_srqhdl_t srq;
1059 uint64_t key, value;
1060 uint_t type;
1061 uint_t size;
1062 int status;
1063
1064 /*
1065 * Extract the Hermon softstate pointer from "Hermon devmap tracking
1066 * structure" (in "pvtp").
1067 */
1068 dvm_track = (hermon_devmap_track_t *)pvtp;
1069 state = dvm_track->hdt_state;
1070
1071 /*
1072 * Extract the "offset" from the "Hermon devmap tracking structure".
1073 * Note: The input argument "off" is ignored here because the
1074 * Hermon mapping interfaces define a very specific meaning to
1075 * each "logical offset". Also extract the "key" and "type" encoded
1076 * in the logical offset.
1077 */
1078 key = dvm_track->hdt_offset >> PAGESHIFT;
1079 type = key & MLNX_UMAP_RSRC_TYPE_MASK;
1080 key = key >> MLNX_UMAP_RSRC_TYPE_SHIFT;
1081
1082 /*
1083 * Extract the "size" of the mapping. If this size is determined
1084 * to be zero, then it is an indication of a previously invalidated
1085 * mapping, and no CQ or QP resources should be affected.
1086 */
1087 size = dvm_track->hdt_size;
1088
1089 /*
1090 * If only the "middle portion of a given mapping is being unmapped,
1091 * then we are effectively creating one new piece of mapped memory.
1092 * (Original region is divided into three pieces of which the middle
1093 * piece is being removed. This leaves two pieces. Since we started
1094 * with one piece and now have two pieces, we need to increment the
1095 * counter in the "Hermon devmap tracking structure".
1096 *
1097 * If, however, the whole mapped region is being unmapped, then we
1098 * have started with one region which we are completely removing.
1099 * In this case, we need to decrement the counter in the "Hermon
1100 * devmap tracking structure".
1101 *
1102 * In each of the remaining cases, we will have started with one
1103 * mapped region and ended with one (different) region. So no counter
1104 * modification is necessary.
1105 */
1106 mutex_enter(&dvm_track->hdt_lock);
1107 if ((new_dhp1 == NULL) && (new_dhp2 == NULL)) {
1108 dvm_track->hdt_refcnt--;
1109 } else if ((new_dhp1 != NULL) && (new_dhp2 != NULL)) {
1110 dvm_track->hdt_refcnt++;
1111 }
1112 mutex_exit(&dvm_track->hdt_lock);
1113
1114 /*
1115 * For each of the cases where the region is being divided, then we
1116 * need to pass back the "Hermon devmap tracking structure". This way
1117 * we get it back when each of the remaining pieces is subsequently
1118 * unmapped.
1119 */
1120 if (new_dhp1 != NULL) {
1121 *pvtp1 = pvtp;
1122 }
1123 if (new_dhp2 != NULL) {
1124 *pvtp2 = pvtp;
1125 }
1126
1127 /*
1128 * If the "Hermon devmap tracking structure" is no longer being
1129 * referenced, then free it up. Otherwise, return.
1130 */
1131 if (dvm_track->hdt_refcnt == 0) {
1132 mutex_destroy(&dvm_track->hdt_lock);
1133 kmem_free(dvm_track, sizeof (hermon_devmap_track_t));
1134
1135 /*
1136 * If the mapping was invalid (see explanation above), then
1137 * no further processing is necessary.
1138 */
1139 if (size == 0) {
1140 return;
1141 }
1142 } else {
1143 return;
1144 }
1145
1146 /*
1147 * Now that we can guarantee that the user memory is fully unmapped,
1148 * we can use the "key" and "type" values to try to find the entry
1149 * in the "userland resources database". If it's found, then it
1150 * indicates that the queue memory (CQ or QP) has not yet been freed.
1151 * In this case, we update the corresponding CQ or QP handle to
1152 * indicate that the "devmap_devmem_remap()" call will be unnecessary.
1153 * If it's _not_ found, then it indicates that the CQ or QP memory
1154 * was, in fact, freed before it was unmapped (thus requiring a
1155 * previous invalidation by remapping - which will already have
1156 * been done in the free routine).
1157 */
1158 status = hermon_umap_db_find(state->hs_instance, key, type, &value,
1159 0, NULL);
1160 if (status == DDI_SUCCESS) {
1161 /*
1162 * Depending on the type of the mapped resource (CQ or QP),
1163 * update handle to indicate that no invalidation remapping
1164 * will be necessary.
1165 */
1166 if (type == MLNX_UMAP_CQMEM_RSRC) {
1167
1168 /* Use "value" to convert to CQ handle */
1169 rsrcp = (hermon_rsrc_t *)(uintptr_t)value;
1170 cq = (hermon_cqhdl_t)rsrcp->hr_addr;
1171
1172 /*
1173 * Invalidate the handle to the userland mapping.
1174 * Note: We must ensure that the mapping being
1175 * unmapped here is the current one for the CQ. It
1176 * is possible that it might not be if this CQ has
1177 * been resized and the previous CQ memory has not
1178 * yet been unmapped. But in that case, because of
1179 * the devmap_devmem_remap(), there is no longer any
1180 * association between the mapping and the real CQ
1181 * kernel memory.
1182 */
1183 mutex_enter(&cq->cq_lock);
1184 if (cq->cq_umap_dhp == dhp) {
1185 cq->cq_umap_dhp = NULL;
1186 }
1187 mutex_exit(&cq->cq_lock);
1188
1189 } else if (type == MLNX_UMAP_QPMEM_RSRC) {
1190
1191 /* Use "value" to convert to QP handle */
1192 rsrcp = (hermon_rsrc_t *)(uintptr_t)value;
1193 qp = (hermon_qphdl_t)rsrcp->hr_addr;
1194
1195 /*
1196 * Invalidate the handle to the userland mapping.
1197 * Note: we ensure that the mapping being unmapped
1198 * here is the current one for the QP. This is
1199 * more of a sanity check here since, unlike CQs
1200 * (above) we do not support resize of QPs.
1201 */
1202 mutex_enter(&qp->qp_lock);
1203 if (qp->qp_umap_dhp == dhp) {
1204 qp->qp_umap_dhp = NULL;
1205 }
1206 mutex_exit(&qp->qp_lock);
1207
1208 } else if (type == MLNX_UMAP_SRQMEM_RSRC) {
1209
1210 /* Use "value" to convert to SRQ handle */
1211 rsrcp = (hermon_rsrc_t *)(uintptr_t)value;
1212 srq = (hermon_srqhdl_t)rsrcp->hr_addr;
1213
1214 /*
1215 * Invalidate the handle to the userland mapping.
1216 * Note: we ensure that the mapping being unmapped
1217 * here is the current one for the QP. This is
1218 * more of a sanity check here since, unlike CQs
1219 * (above) we do not support resize of QPs.
1220 */
1221 mutex_enter(&srq->srq_lock);
1222 if (srq->srq_umap_dhp == dhp) {
1223 srq->srq_umap_dhp = NULL;
1224 }
1225 mutex_exit(&srq->srq_lock);
1226 }
1227 }
1228 }
1229
1230
1231 /*
1232 * hermon_devmap_devmem_map()
1233 * Context: Can be called from kernel context.
1234 */
1235 /* ARGSUSED */
1236 static int
1237 hermon_devmap_devmem_map(devmap_cookie_t dhp, dev_t dev, uint_t flags,
1238 offset_t off, size_t len, void **pvtp)
1239 {
1240 hermon_state_t *state;
1241 hermon_devmap_track_t *dvm_track;
1242 minor_t instance;
1243
1244 /* Get Hermon softstate structure from instance */
1245 instance = HERMON_DEV_INSTANCE(dev);
1246 state = ddi_get_soft_state(hermon_statep, instance);
1247 if (state == NULL) {
1248 return (ENXIO);
1249 }
1250
1251 /*
1252 * Allocate an entry to track the mapping and unmapping of this
1253 * resource. Note: We don't need to initialize the "refcnt" or
1254 * "offset" fields here, nor do we need to initialize the mutex
1255 * used with the "refcnt". Since UAR pages are single pages, they
1256 * are not subject to "partial" unmappings. This makes these other
1257 * fields unnecessary.
1258 */
1259 dvm_track = (hermon_devmap_track_t *)kmem_zalloc(
1260 sizeof (hermon_devmap_track_t), KM_SLEEP);
1261 dvm_track->hdt_state = state;
1262 dvm_track->hdt_size = (uint_t)PAGESIZE;
1263
1264 /*
1265 * Pass the private "Hermon devmap tracking structure" back. This
1266 * pointer will be returned in a subsequent "unmap" callback.
1267 */
1268 *pvtp = dvm_track;
1269
1270 return (DDI_SUCCESS);
1271 }
1272
1273
1274 /*
1275 * hermon_devmap_devmem_dup()
1276 * Context: Can be called from kernel context.
1277 */
1278 /* ARGSUSED */
1279 static int
1280 hermon_devmap_devmem_dup(devmap_cookie_t dhp, void *pvtp,
1281 devmap_cookie_t new_dhp, void **new_pvtp)
1282 {
1283 hermon_state_t *state;
1284 hermon_devmap_track_t *dvm_track;
1285 uint_t maxprot;
1286 int status;
1287
1288 /*
1289 * Extract the Hermon softstate pointer from "Hermon devmap tracking
1290 * structure" (in "pvtp"). Note: If the tracking structure is NULL
1291 * here, it means that the mapping corresponds to an invalid mapping.
1292 * In this case, it can be safely ignored ("new_pvtp" set to NULL).
1293 */
1294 dvm_track = (hermon_devmap_track_t *)pvtp;
1295 if (dvm_track == NULL) {
1296 *new_pvtp = NULL;
1297 return (DDI_SUCCESS);
1298 }
1299
1300 state = dvm_track->hdt_state;
1301
1302 /*
1303 * Since this devmap_dup() entry point is generally called
1304 * when a process does fork(2), it is incumbent upon the driver
1305 * to insure that the child does not inherit a valid copy of
1306 * the parent's resource. This is accomplished by using
1307 * devmap_devmem_remap() to invalidate the child's mapping to the
1308 * kernel memory.
1309 */
1310 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
1311 status = devmap_devmem_remap(new_dhp, state->hs_dip, 0, 0,
1312 dvm_track->hdt_size, maxprot, DEVMAP_MAPPING_INVALID, NULL);
1313 if (status != DDI_SUCCESS) {
1314 HERMON_WARNING(state, "failed in hermon_devmap_devmem_dup()");
1315 return (status);
1316 }
1317
1318 /*
1319 * Since the region is invalid, there is no need for us to
1320 * allocate and continue to track an additional "Hermon devmap
1321 * tracking structure". Instead we return NULL here, which is an
1322 * indication to the devmap_unmap() entry point that this entry
1323 * can be safely ignored.
1324 */
1325 *new_pvtp = NULL;
1326
1327 return (DDI_SUCCESS);
1328 }
1329
1330
1331 /*
1332 * hermon_devmap_devmem_unmap()
1333 * Context: Can be called from kernel context.
1334 */
1335 /* ARGSUSED */
1336 static void
1337 hermon_devmap_devmem_unmap(devmap_cookie_t dhp, void *pvtp, offset_t off,
1338 size_t len, devmap_cookie_t new_dhp1, void **pvtp1,
1339 devmap_cookie_t new_dhp2, void **pvtp2)
1340 {
1341 hermon_devmap_track_t *dvm_track;
1342
1343 /*
1344 * Free up the "Hermon devmap tracking structure" (in "pvtp").
1345 * There cannot be "partial" unmappings here because all UAR pages
1346 * are single pages. Note: If the tracking structure is NULL here,
1347 * it means that the mapping corresponds to an invalid mapping. In
1348 * this case, it can be safely ignored.
1349 */
1350 dvm_track = (hermon_devmap_track_t *)pvtp;
1351 if (dvm_track == NULL) {
1352 return;
1353 }
1354
1355 kmem_free(dvm_track, sizeof (hermon_devmap_track_t));
1356 }
1357
1358
1359 /*
1360 * hermon_umap_ci_data_in()
1361 * Context: Can be called from user or kernel context.
1362 */
1363 /* ARGSUSED */
1364 ibt_status_t
1365 hermon_umap_ci_data_in(hermon_state_t *state, ibt_ci_data_flags_t flags,
1366 ibt_object_type_t object, void *hdl, void *data_p, size_t data_sz)
1367 {
1368 int status;
1369
1370 /*
1371 * Depending on the type of object about which additional information
1372 * is being provided (currently only MR is supported), we call the
1373 * appropriate resource-specific function.
1374 */
1375 switch (object) {
1376 case IBT_HDL_MR:
1377 status = hermon_umap_mr_data_in((hermon_mrhdl_t)hdl,
1378 (ibt_mr_data_in_t *)data_p, data_sz);
1379 if (status != DDI_SUCCESS) {
1380 return (status);
1381 }
1382 break;
1383
1384 /*
1385 * For other possible valid IBT types, we return IBT_NOT_SUPPORTED,
1386 * since the Hermon driver does not support these.
1387 */
1388 case IBT_HDL_HCA:
1389 case IBT_HDL_QP:
1390 case IBT_HDL_CQ:
1391 case IBT_HDL_PD:
1392 case IBT_HDL_MW:
1393 case IBT_HDL_AH:
1394 case IBT_HDL_SCHED:
1395 case IBT_HDL_EEC:
1396 case IBT_HDL_RDD:
1397 case IBT_HDL_SRQ:
1398 return (IBT_NOT_SUPPORTED);
1399
1400 /*
1401 * Any other types are invalid.
1402 */
1403 default:
1404 return (IBT_INVALID_PARAM);
1405 }
1406
1407 return (DDI_SUCCESS);
1408 }
1409
1410
1411 /*
1412 * hermon_umap_mr_data_in()
1413 * Context: Can be called from user or kernel context.
1414 */
1415 static ibt_status_t
1416 hermon_umap_mr_data_in(hermon_mrhdl_t mr, ibt_mr_data_in_t *data,
1417 size_t data_sz)
1418 {
1419 if (data->mr_rev != IBT_MR_DATA_IN_IF_VERSION) {
1420 return (IBT_NOT_SUPPORTED);
1421 }
1422
1423 /* Check for valid MR handle pointer */
1424 if (mr == NULL) {
1425 return (IBT_MR_HDL_INVALID);
1426 }
1427
1428 /* Check for valid MR input structure size */
1429 if (data_sz < sizeof (ibt_mr_data_in_t)) {
1430 return (IBT_INSUFF_RESOURCE);
1431 }
1432
1433 /*
1434 * Ensure that the MR corresponds to userland memory and that it is
1435 * a currently valid memory region as well.
1436 */
1437 mutex_enter(&mr->mr_lock);
1438 if ((mr->mr_is_umem == 0) || (mr->mr_umemcookie == NULL)) {
1439 mutex_exit(&mr->mr_lock);
1440 return (IBT_MR_HDL_INVALID);
1441 }
1442
1443 /*
1444 * If it has passed all the above checks, then extract the callback
1445 * function and argument from the input structure. Copy them into
1446 * the MR handle. This function will be called only if the memory
1447 * corresponding to the MR handle gets a umem_lockmemory() callback.
1448 */
1449 mr->mr_umem_cbfunc = data->mr_func;
1450 mr->mr_umem_cbarg1 = data->mr_arg1;
1451 mr->mr_umem_cbarg2 = data->mr_arg2;
1452 mutex_exit(&mr->mr_lock);
1453
1454 return (DDI_SUCCESS);
1455 }
1456
1457
1458 /*
1459 * hermon_umap_ci_data_out()
1460 * Context: Can be called from user or kernel context.
1461 */
1462 /* ARGSUSED */
1463 ibt_status_t
1464 hermon_umap_ci_data_out(hermon_state_t *state, ibt_ci_data_flags_t flags,
1465 ibt_object_type_t object, void *hdl, void *data_p, size_t data_sz)
1466 {
1467 int status;
1468
1469 /*
1470 * Depending on the type of object about which additional information
1471 * is being requested (CQ or QP), we call the appropriate resource-
1472 * specific mapping function.
1473 */
1474 switch (object) {
1475 case IBT_HDL_CQ:
1476 status = hermon_umap_cq_data_out((hermon_cqhdl_t)hdl,
1477 (mlnx_umap_cq_data_out_t *)data_p, data_sz);
1478 if (status != DDI_SUCCESS) {
1479 return (status);
1480 }
1481 break;
1482
1483 case IBT_HDL_QP:
1484 status = hermon_umap_qp_data_out((hermon_qphdl_t)hdl,
1485 (mlnx_umap_qp_data_out_t *)data_p, data_sz);
1486 if (status != DDI_SUCCESS) {
1487 return (status);
1488 }
1489 break;
1490
1491 case IBT_HDL_SRQ:
1492 status = hermon_umap_srq_data_out((hermon_srqhdl_t)hdl,
1493 (mlnx_umap_srq_data_out_t *)data_p, data_sz);
1494 if (status != DDI_SUCCESS) {
1495 return (status);
1496 }
1497 break;
1498
1499 case IBT_HDL_PD:
1500 status = hermon_umap_pd_data_out((hermon_pdhdl_t)hdl,
1501 (mlnx_umap_pd_data_out_t *)data_p, data_sz);
1502 if (status != DDI_SUCCESS) {
1503 return (status);
1504 }
1505 break;
1506
1507 /*
1508 * For other possible valid IBT types, we return IBT_NOT_SUPPORTED,
1509 * since the Hermon driver does not support these.
1510 */
1511 case IBT_HDL_HCA:
1512 case IBT_HDL_MR:
1513 case IBT_HDL_MW:
1514 case IBT_HDL_AH:
1515 case IBT_HDL_SCHED:
1516 case IBT_HDL_EEC:
1517 case IBT_HDL_RDD:
1518 return (IBT_NOT_SUPPORTED);
1519
1520 /*
1521 * Any other types are invalid.
1522 */
1523 default:
1524 return (IBT_INVALID_PARAM);
1525 }
1526
1527 return (DDI_SUCCESS);
1528 }
1529
1530
1531 /*
1532 * hermon_umap_cq_data_out()
1533 * Context: Can be called from user or kernel context.
1534 */
1535 static ibt_status_t
1536 hermon_umap_cq_data_out(hermon_cqhdl_t cq, mlnx_umap_cq_data_out_t *data,
1537 size_t data_sz)
1538 {
1539 /* Check for valid CQ handle pointer */
1540 if (cq == NULL) {
1541 return (IBT_CQ_HDL_INVALID);
1542 }
1543
1544 /* Check for valid CQ mapping structure size */
1545 if (data_sz < sizeof (mlnx_umap_cq_data_out_t)) {
1546 return (IBT_INSUFF_RESOURCE);
1547 }
1548
1549 /* deal with cq_alloc() verses cq_resize() */
1550 if (cq->cq_resize_hdl) {
1551 data->mcq_maplen = cq->cq_resize_hdl->cq_cqinfo.qa_size;
1552 data->mcq_numcqe = cq->cq_resize_hdl->cq_bufsz;
1553 } else {
1554 data->mcq_maplen = cq->cq_cqinfo.qa_size;
1555 data->mcq_numcqe = cq->cq_bufsz;
1556 }
1557
1558 /*
1559 * If it has passed all the above checks, then fill in all the useful
1560 * mapping information (including the mapping offset that will be
1561 * passed back to the devmap() interface during a subsequent mmap()
1562 * call.
1563 *
1564 * The "offset" for CQ mmap()'s looks like this:
1565 * +----------------------------------------+--------+--------------+
1566 * | CQ Number | 0x33 | Reserved (0) |
1567 * +----------------------------------------+--------+--------------+
1568 * (64 - 8 - PAGESHIFT) bits 8 bits PAGESHIFT bits
1569 *
1570 * This returns information about the mapping offset, the length of
1571 * the CQ memory, the CQ number (for use in later CQ doorbells), the
1572 * number of CQEs the CQ memory can hold, and the size of each CQE.
1573 */
1574 data->mcq_rev = MLNX_UMAP_IF_VERSION;
1575 data->mcq_mapoffset = ((((uint64_t)cq->cq_cqnum <<
1576 MLNX_UMAP_RSRC_TYPE_SHIFT) | MLNX_UMAP_CQMEM_RSRC) << PAGESHIFT);
1577 data->mcq_cqnum = cq->cq_cqnum;
1578 data->mcq_cqesz = sizeof (hermon_hw_cqe_t);
1579
1580 /* doorbell record fields */
1581 data->mcq_polldbr_mapoffset = cq->cq_dbr_mapoffset;
1582 data->mcq_polldbr_maplen = PAGESIZE;
1583 data->mcq_polldbr_offset = (uintptr_t)cq->cq_arm_ci_vdbr &
1584 PAGEOFFSET;
1585 data->mcq_armdbr_mapoffset = cq->cq_dbr_mapoffset;
1586 data->mcq_armdbr_maplen = PAGESIZE;
1587 data->mcq_armdbr_offset = data->mcq_polldbr_offset + 4;
1588
1589 return (DDI_SUCCESS);
1590 }
1591
1592
1593 /*
1594 * hermon_umap_qp_data_out()
1595 * Context: Can be called from user or kernel context.
1596 */
1597 static ibt_status_t
1598 hermon_umap_qp_data_out(hermon_qphdl_t qp, mlnx_umap_qp_data_out_t *data,
1599 size_t data_sz)
1600 {
1601 /* Check for valid QP handle pointer */
1602 if (qp == NULL) {
1603 return (IBT_QP_HDL_INVALID);
1604 }
1605
1606 /* Check for valid QP mapping structure size */
1607 if (data_sz < sizeof (mlnx_umap_qp_data_out_t)) {
1608 return (IBT_INSUFF_RESOURCE);
1609 }
1610
1611 /*
1612 * If it has passed all the checks, then fill in all the useful
1613 * mapping information (including the mapping offset that will be
1614 * passed back to the devmap() interface during a subsequent mmap()
1615 * call.
1616 *
1617 * The "offset" for QP mmap()'s looks like this:
1618 * +----------------------------------------+--------+--------------+
1619 * | QP Number | 0x44 | Reserved (0) |
1620 * +----------------------------------------+--------+--------------+
1621 * (64 - 8 - PAGESHIFT) bits 8 bits PAGESHIFT bits
1622 *
1623 * This returns information about the mapping offset, the length of
1624 * the QP memory, and the QP number (for use in later send and recv
1625 * doorbells). It also returns the following information for both
1626 * the receive work queue and the send work queue, respectively: the
1627 * offset (from the base mapped address) of the start of the given
1628 * work queue, the 64-bit IB virtual address that corresponds to
1629 * the base mapped address (needed for posting WQEs though the
1630 * QP doorbells), the number of WQEs the given work queue can hold,
1631 * and the size of each WQE for the given work queue.
1632 */
1633 data->mqp_rev = MLNX_UMAP_IF_VERSION;
1634 data->mqp_mapoffset = ((((uint64_t)qp->qp_qpnum <<
1635 MLNX_UMAP_RSRC_TYPE_SHIFT) | MLNX_UMAP_QPMEM_RSRC) << PAGESHIFT);
1636 data->mqp_maplen = qp->qp_wqinfo.qa_size;
1637 data->mqp_qpnum = qp->qp_qpnum;
1638
1639 /*
1640 * If this QP is associated with a shared receive queue (SRQ),
1641 * then return invalid RecvQ parameters. Otherwise, return
1642 * the proper parameter values.
1643 */
1644 if (qp->qp_alloc_flags & IBT_QP_USES_SRQ) {
1645 data->mqp_rq_off = (uint32_t)qp->qp_wqinfo.qa_size;
1646 data->mqp_rq_desc_addr = (uint32_t)qp->qp_wqinfo.qa_size;
1647 data->mqp_rq_numwqe = 0;
1648 data->mqp_rq_wqesz = 0;
1649 data->mqp_rdbr_mapoffset = 0;
1650 data->mqp_rdbr_maplen = 0;
1651 data->mqp_rdbr_offset = 0;
1652 } else {
1653 data->mqp_rq_off = (uintptr_t)qp->qp_rq_buf -
1654 (uintptr_t)qp->qp_wqinfo.qa_buf_aligned;
1655 data->mqp_rq_desc_addr = (uint32_t)((uintptr_t)qp->qp_rq_buf -
1656 qp->qp_desc_off);
1657 data->mqp_rq_numwqe = qp->qp_rq_bufsz;
1658 data->mqp_rq_wqesz = (1 << qp->qp_rq_log_wqesz);
1659
1660 /* doorbell record fields */
1661 data->mqp_rdbr_mapoffset = qp->qp_rdbr_mapoffset;
1662 data->mqp_rdbr_maplen = PAGESIZE;
1663 data->mqp_rdbr_offset = (uintptr_t)qp->qp_rq_vdbr &
1664 PAGEOFFSET;
1665 }
1666 data->mqp_sq_off = (uintptr_t)qp->qp_sq_buf -
1667 (uintptr_t)qp->qp_wqinfo.qa_buf_aligned;
1668 data->mqp_sq_desc_addr = (uint32_t)((uintptr_t)qp->qp_sq_buf -
1669 qp->qp_desc_off);
1670 data->mqp_sq_numwqe = qp->qp_sq_bufsz;
1671 data->mqp_sq_wqesz = (1 << qp->qp_sq_log_wqesz);
1672 data->mqp_sq_headroomwqes = qp->qp_sq_hdrmwqes;
1673
1674 /* doorbell record fields */
1675 data->mqp_sdbr_mapoffset = 0;
1676 data->mqp_sdbr_maplen = 0;
1677 data->mqp_sdbr_offset = 0;
1678
1679 return (DDI_SUCCESS);
1680 }
1681
1682
1683 /*
1684 * hermon_umap_srq_data_out()
1685 * Context: Can be called from user or kernel context.
1686 */
1687 static ibt_status_t
1688 hermon_umap_srq_data_out(hermon_srqhdl_t srq, mlnx_umap_srq_data_out_t *data,
1689 size_t data_sz)
1690 {
1691 /* Check for valid SRQ handle pointer */
1692 if (srq == NULL) {
1693 return (IBT_SRQ_HDL_INVALID);
1694 }
1695
1696 /* Check for valid SRQ mapping structure size */
1697 if (data_sz < sizeof (mlnx_umap_srq_data_out_t)) {
1698 return (IBT_INSUFF_RESOURCE);
1699 }
1700
1701 /*
1702 * If it has passed all the checks, then fill in all the useful
1703 * mapping information (including the mapping offset that will be
1704 * passed back to the devmap() interface during a subsequent mmap()
1705 * call.
1706 *
1707 * The "offset" for SRQ mmap()'s looks like this:
1708 * +----------------------------------------+--------+--------------+
1709 * | SRQ Number | 0x66 | Reserved (0) |
1710 * +----------------------------------------+--------+--------------+
1711 * (64 - 8 - PAGESHIFT) bits 8 bits PAGESHIFT bits
1712 *
1713 * This returns information about the mapping offset, the length of the
1714 * SRQ memory, and the SRQ number (for use in later send and recv
1715 * doorbells). It also returns the following information for the
1716 * shared receive queue: the offset (from the base mapped address) of
1717 * the start of the given work queue, the 64-bit IB virtual address
1718 * that corresponds to the base mapped address (needed for posting WQEs
1719 * though the QP doorbells), the number of WQEs the given work queue
1720 * can hold, and the size of each WQE for the given work queue.
1721 */
1722 data->msrq_rev = MLNX_UMAP_IF_VERSION;
1723 data->msrq_mapoffset = ((((uint64_t)srq->srq_srqnum <<
1724 MLNX_UMAP_RSRC_TYPE_SHIFT) | MLNX_UMAP_SRQMEM_RSRC) << PAGESHIFT);
1725 data->msrq_maplen = srq->srq_wqinfo.qa_size;
1726 data->msrq_srqnum = srq->srq_srqnum;
1727
1728 data->msrq_desc_addr = (uint32_t)((uintptr_t)srq->srq_wq_buf -
1729 srq->srq_desc_off);
1730 data->msrq_numwqe = srq->srq_wq_bufsz;
1731 data->msrq_wqesz = (1 << srq->srq_wq_log_wqesz);
1732
1733 /* doorbell record fields */
1734 data->msrq_rdbr_mapoffset = srq->srq_rdbr_mapoffset;
1735 data->msrq_rdbr_maplen = PAGESIZE;
1736 data->msrq_rdbr_offset = (uintptr_t)srq->srq_wq_vdbr &
1737 PAGEOFFSET;
1738
1739 return (DDI_SUCCESS);
1740 }
1741
1742
1743 /*
1744 * hermon_umap_pd_data_out()
1745 * Context: Can be called from user or kernel context.
1746 */
1747 static ibt_status_t
1748 hermon_umap_pd_data_out(hermon_pdhdl_t pd, mlnx_umap_pd_data_out_t *data,
1749 size_t data_sz)
1750 {
1751 /* Check for valid PD handle pointer */
1752 if (pd == NULL) {
1753 return (IBT_PD_HDL_INVALID);
1754 }
1755
1756 /* Check for valid PD mapping structure size */
1757 if (data_sz < sizeof (mlnx_umap_pd_data_out_t)) {
1758 return (IBT_INSUFF_RESOURCE);
1759 }
1760
1761 /*
1762 * If it has passed all the checks, then fill the PD table index
1763 * (the PD table allocated index for the PD pd_pdnum).
1764 */
1765 data->mpd_rev = MLNX_UMAP_IF_VERSION;
1766 data->mpd_pdnum = pd->pd_pdnum;
1767
1768 return (DDI_SUCCESS);
1769 }
1770
1771
1772 /*
1773 * hermon_umap_db_init()
1774 * Context: Only called from attach() path context
1775 */
1776 void
1777 hermon_umap_db_init(void)
1778 {
1779 /*
1780 * Initialize the lock used by the Hermon "userland resources database"
1781 * This is used to ensure atomic access to add, remove, and find
1782 * entries in the database.
1783 */
1784 mutex_init(&hermon_userland_rsrc_db.hdl_umapdb_lock, NULL,
1785 MUTEX_DRIVER, NULL);
1786
1787 /*
1788 * Initialize the AVL tree used for the "userland resources
1789 * database". Using an AVL tree here provides the ability to
1790 * scale the database size to large numbers of resources. The
1791 * entries in the tree are "hermon_umap_db_entry_t" (see
1792 * hermon_umap.h). The tree is searched with the help of the
1793 * hermon_umap_db_compare() routine.
1794 */
1795 avl_create(&hermon_userland_rsrc_db.hdl_umapdb_avl,
1796 hermon_umap_db_compare, sizeof (hermon_umap_db_entry_t),
1797 offsetof(hermon_umap_db_entry_t, hdbe_avlnode));
1798 }
1799
1800
1801 /*
1802 * hermon_umap_db_fini()
1803 * Context: Only called from attach() and/or detach() path contexts
1804 */
1805 void
1806 hermon_umap_db_fini(void)
1807 {
1808 /* Destroy the AVL tree for the "userland resources database" */
1809 avl_destroy(&hermon_userland_rsrc_db.hdl_umapdb_avl);
1810
1811 /* Destroy the lock for the "userland resources database" */
1812 mutex_destroy(&hermon_userland_rsrc_db.hdl_umapdb_lock);
1813 }
1814
1815
1816 /*
1817 * hermon_umap_db_alloc()
1818 * Context: Can be called from user or kernel context.
1819 */
1820 hermon_umap_db_entry_t *
1821 hermon_umap_db_alloc(uint_t instance, uint64_t key, uint_t type, uint64_t value)
1822 {
1823 hermon_umap_db_entry_t *umapdb;
1824
1825 /* Allocate an entry to add to the "userland resources database" */
1826 umapdb = kmem_zalloc(sizeof (hermon_umap_db_entry_t), KM_NOSLEEP);
1827 if (umapdb == NULL) {
1828 return (NULL);
1829 }
1830
1831 /* Fill in the fields in the database entry */
1832 umapdb->hdbe_common.hdb_instance = instance;
1833 umapdb->hdbe_common.hdb_type = type;
1834 umapdb->hdbe_common.hdb_key = key;
1835 umapdb->hdbe_common.hdb_value = value;
1836
1837 return (umapdb);
1838 }
1839
1840
1841 /*
1842 * hermon_umap_db_free()
1843 * Context: Can be called from user or kernel context.
1844 */
1845 void
1846 hermon_umap_db_free(hermon_umap_db_entry_t *umapdb)
1847 {
1848 /* Free the database entry */
1849 kmem_free(umapdb, sizeof (hermon_umap_db_entry_t));
1850 }
1851
1852
1853 /*
1854 * hermon_umap_db_add()
1855 * Context: Can be called from user or kernel context.
1856 */
1857 void
1858 hermon_umap_db_add(hermon_umap_db_entry_t *umapdb)
1859 {
1860 mutex_enter(&hermon_userland_rsrc_db.hdl_umapdb_lock);
1861 hermon_umap_db_add_nolock(umapdb);
1862 mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock);
1863 }
1864
1865
1866 /*
1867 * hermon_umap_db_add_nolock()
1868 * Context: Can be called from user or kernel context.
1869 */
1870 void
1871 hermon_umap_db_add_nolock(hermon_umap_db_entry_t *umapdb)
1872 {
1873 hermon_umap_db_query_t query;
1874 avl_index_t where;
1875
1876 ASSERT(MUTEX_HELD(&hermon_userland_rsrc_db.hdl_umapdb_lock));
1877
1878 /*
1879 * Copy the common portion of the "to-be-added" database entry
1880 * into the "hermon_umap_db_query_t" structure. We use this structure
1881 * (with no flags set) to find the appropriate location in the
1882 * "userland resources database" for the new entry to be added.
1883 *
1884 * Note: we expect that this entry should not be found in the
1885 * database (unless something bad has happened).
1886 */
1887 query.hqdb_common = umapdb->hdbe_common;
1888 query.hqdb_flags = 0;
1889 (void) avl_find(&hermon_userland_rsrc_db.hdl_umapdb_avl, &query,
1890 &where);
1891
1892 /*
1893 * Now, using the "where" field from the avl_find() operation
1894 * above, we will insert the new database entry ("umapdb").
1895 */
1896 avl_insert(&hermon_userland_rsrc_db.hdl_umapdb_avl, umapdb,
1897 where);
1898 }
1899
1900
1901 /*
1902 * hermon_umap_db_find()
1903 * Context: Can be called from user or kernel context.
1904 */
1905 int
1906 hermon_umap_db_find(uint_t instance, uint64_t key, uint_t type,
1907 uint64_t *value, uint_t flag, hermon_umap_db_entry_t **umapdb)
1908 {
1909 int status;
1910
1911 mutex_enter(&hermon_userland_rsrc_db.hdl_umapdb_lock);
1912 status = hermon_umap_db_find_nolock(instance, key, type, value, flag,
1913 umapdb);
1914 mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock);
1915
1916 return (status);
1917 }
1918
1919
1920 /*
1921 * hermon_umap_db_find_nolock()
1922 * Context: Can be called from user or kernel context.
1923 */
1924 int
1925 hermon_umap_db_find_nolock(uint_t instance, uint64_t key, uint_t type,
1926 uint64_t *value, uint_t flags, hermon_umap_db_entry_t **umapdb)
1927 {
1928 hermon_umap_db_query_t query;
1929 hermon_umap_db_entry_t *entry;
1930 avl_index_t where;
1931
1932 ASSERT(MUTEX_HELD(&hermon_userland_rsrc_db.hdl_umapdb_lock));
1933
1934 /*
1935 * Fill in key, type, instance, and flags values of the
1936 * hermon_umap_db_query_t in preparation for the database
1937 * lookup.
1938 */
1939 query.hqdb_flags = flags;
1940 query.hqdb_common.hdb_key = key;
1941 query.hqdb_common.hdb_type = type;
1942 query.hqdb_common.hdb_instance = instance;
1943
1944 /*
1945 * Perform the database query. If no entry is found, then
1946 * return failure, else continue.
1947 */
1948 entry = (hermon_umap_db_entry_t *)avl_find(
1949 &hermon_userland_rsrc_db.hdl_umapdb_avl, &query, &where);
1950 if (entry == NULL) {
1951 return (DDI_FAILURE);
1952 }
1953
1954 /*
1955 * If the flags argument specifies that the entry should
1956 * be removed if found, then call avl_remove() to remove
1957 * the entry from the database.
1958 */
1959 if (flags & HERMON_UMAP_DB_REMOVE) {
1960
1961 avl_remove(&hermon_userland_rsrc_db.hdl_umapdb_avl, entry);
1962
1963 /*
1964 * The database entry is returned with the expectation
1965 * that the caller will use hermon_umap_db_free() to
1966 * free the entry's memory. ASSERT that this is non-NULL.
1967 * NULL pointer should never be passed for the
1968 * HERMON_UMAP_DB_REMOVE case.
1969 */
1970 ASSERT(umapdb != NULL);
1971 }
1972
1973 /*
1974 * If the caller would like visibility to the database entry
1975 * (indicated through the use of a non-NULL "umapdb" argument),
1976 * then fill it in.
1977 */
1978 if (umapdb != NULL) {
1979 *umapdb = entry;
1980 }
1981
1982 /* Extract value field from database entry and return success */
1983 *value = entry->hdbe_common.hdb_value;
1984
1985 return (DDI_SUCCESS);
1986 }
1987
1988
1989 /*
1990 * hermon_umap_umemlock_cb()
1991 * Context: Can be called from callback context.
1992 */
1993 void
1994 hermon_umap_umemlock_cb(ddi_umem_cookie_t *umem_cookie)
1995 {
1996 hermon_umap_db_entry_t *umapdb;
1997 hermon_state_t *state;
1998 hermon_rsrc_t *rsrcp;
1999 hermon_mrhdl_t mr;
2000 uint64_t value;
2001 uint_t instance;
2002 int status;
2003 void (*mr_callback)(void *, void *);
2004 void *mr_cbarg1, *mr_cbarg2;
2005
2006 /*
2007 * If this was userland memory, then we need to remove its entry
2008 * from the "userland resources database". Note: We use the
2009 * HERMON_UMAP_DB_IGNORE_INSTANCE flag here because we don't know
2010 * which instance was used when the entry was added (but we want
2011 * to know after the entry is found using the other search criteria).
2012 */
2013 status = hermon_umap_db_find(0, (uint64_t)(uintptr_t)umem_cookie,
2014 MLNX_UMAP_MRMEM_RSRC, &value, (HERMON_UMAP_DB_REMOVE |
2015 HERMON_UMAP_DB_IGNORE_INSTANCE), &umapdb);
2016 if (status == DDI_SUCCESS) {
2017 instance = umapdb->hdbe_common.hdb_instance;
2018 state = ddi_get_soft_state(hermon_statep, instance);
2019 if (state == NULL) {
2020 cmn_err(CE_WARN, "Unable to match Hermon instance\n");
2021 return;
2022 }
2023
2024 /* Free the database entry */
2025 hermon_umap_db_free(umapdb);
2026
2027 /* Use "value" to convert to an MR handle */
2028 rsrcp = (hermon_rsrc_t *)(uintptr_t)value;
2029 mr = (hermon_mrhdl_t)rsrcp->hr_addr;
2030
2031 /*
2032 * If a callback has been provided, call it first. This
2033 * callback is expected to do any cleanup necessary to
2034 * guarantee that the subsequent MR deregister (below)
2035 * will succeed. Specifically, this means freeing up memory
2036 * windows which might have been associated with the MR.
2037 */
2038 mutex_enter(&mr->mr_lock);
2039 mr_callback = mr->mr_umem_cbfunc;
2040 mr_cbarg1 = mr->mr_umem_cbarg1;
2041 mr_cbarg2 = mr->mr_umem_cbarg2;
2042 mutex_exit(&mr->mr_lock);
2043 if (mr_callback != NULL) {
2044 mr_callback(mr_cbarg1, mr_cbarg2);
2045 }
2046
2047 /*
2048 * Then call hermon_mr_deregister() to release the resources
2049 * associated with the MR handle. Note: Because this routine
2050 * will also check for whether the ddi_umem_cookie_t is in the
2051 * database, it will take responsibility for disabling the
2052 * memory region and calling ddi_umem_unlock().
2053 */
2054 status = hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
2055 HERMON_SLEEP);
2056 if (status != DDI_SUCCESS) {
2057 HERMON_WARNING(state, "Unexpected failure in "
2058 "deregister from callback\n");
2059 }
2060 }
2061 }
2062
2063
2064 /*
2065 * hermon_umap_db_compare()
2066 * Context: Can be called from user or kernel context.
2067 */
2068 static int
2069 hermon_umap_db_compare(const void *q, const void *e)
2070 {
2071 hermon_umap_db_common_t *entry_common, *query_common;
2072 uint_t query_flags;
2073
2074 entry_common = &((hermon_umap_db_entry_t *)e)->hdbe_common;
2075 query_common = &((hermon_umap_db_query_t *)q)->hqdb_common;
2076 query_flags = ((hermon_umap_db_query_t *)q)->hqdb_flags;
2077
2078 /*
2079 * The first comparison is done on the "key" value in "query"
2080 * and "entry". If they are not equal, then the appropriate
2081 * search direction is returned. Else, we continue by
2082 * comparing "type".
2083 */
2084 if (query_common->hdb_key < entry_common->hdb_key) {
2085 return (-1);
2086 } else if (query_common->hdb_key > entry_common->hdb_key) {
2087 return (+1);
2088 }
2089
2090 /*
2091 * If the search reaches this point, then "query" and "entry"
2092 * have equal key values. So we continue be comparing their
2093 * "type" values. Again, if they are not equal, then the
2094 * appropriate search direction is returned. Else, we continue
2095 * by comparing "instance".
2096 */
2097 if (query_common->hdb_type < entry_common->hdb_type) {
2098 return (-1);
2099 } else if (query_common->hdb_type > entry_common->hdb_type) {
2100 return (+1);
2101 }
2102
2103 /*
2104 * If the search reaches this point, then "query" and "entry"
2105 * have exactly the same key and type values. Now we consult
2106 * the "flags" field in the query to determine whether the
2107 * "instance" is relevant to the search. If the
2108 * HERMON_UMAP_DB_IGNORE_INSTANCE flags is set, then return
2109 * success (0) here. Otherwise, continue the search by comparing
2110 * instance values and returning the appropriate search direction.
2111 */
2112 if (query_flags & HERMON_UMAP_DB_IGNORE_INSTANCE) {
2113 return (0);
2114 }
2115
2116 /*
2117 * If the search has reached this point, then "query" and "entry"
2118 * can only be differentiated by their instance values. If these
2119 * are not equal, then return the appropriate search direction.
2120 * Else, we return success (0).
2121 */
2122 if (query_common->hdb_instance < entry_common->hdb_instance) {
2123 return (-1);
2124 } else if (query_common->hdb_instance > entry_common->hdb_instance) {
2125 return (+1);
2126 }
2127
2128 /* Everything matches... so return success */
2129 return (0);
2130 }
2131
2132
2133 /*
2134 * hermon_umap_db_set_onclose_cb()
2135 * Context: Can be called from user or kernel context.
2136 */
2137 int
2138 hermon_umap_db_set_onclose_cb(dev_t dev, uint64_t flag,
2139 int (*callback)(void *), void *arg)
2140 {
2141 hermon_umap_db_priv_t *priv;
2142 hermon_umap_db_entry_t *umapdb;
2143 minor_t instance;
2144 uint64_t value;
2145 int status;
2146
2147 instance = HERMON_DEV_INSTANCE(dev);
2148 if (instance == (minor_t)-1) {
2149 return (DDI_FAILURE);
2150 }
2151
2152 if (flag != HERMON_ONCLOSE_FLASH_INPROGRESS) {
2153 return (DDI_FAILURE);
2154 }
2155
2156 /*
2157 * Grab the lock for the "userland resources database" and find
2158 * the entry corresponding to this minor number. Once it's found,
2159 * allocate (if necessary) and add an entry (in the "hdb_priv"
2160 * field) to indicate that further processing may be needed during
2161 * Hermon's close() handling.
2162 */
2163 mutex_enter(&hermon_userland_rsrc_db.hdl_umapdb_lock);
2164 status = hermon_umap_db_find_nolock(instance, dev,
2165 MLNX_UMAP_PID_RSRC, &value, 0, &umapdb);
2166 if (status != DDI_SUCCESS) {
2167 mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock);
2168 return (DDI_FAILURE);
2169 }
2170
2171 priv = (hermon_umap_db_priv_t *)umapdb->hdbe_common.hdb_priv;
2172 if (priv == NULL) {
2173 priv = (hermon_umap_db_priv_t *)kmem_zalloc(
2174 sizeof (hermon_umap_db_priv_t), KM_NOSLEEP);
2175 if (priv == NULL) {
2176 mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock);
2177 return (DDI_FAILURE);
2178 }
2179 }
2180
2181 /*
2182 * Save away the callback and argument to be used during Hermon's
2183 * close() processing.
2184 */
2185 priv->hdp_cb = callback;
2186 priv->hdp_arg = arg;
2187
2188 umapdb->hdbe_common.hdb_priv = (void *)priv;
2189 mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock);
2190
2191 return (DDI_SUCCESS);
2192 }
2193
2194
2195 /*
2196 * hermon_umap_db_clear_onclose_cb()
2197 * Context: Can be called from user or kernel context.
2198 */
2199 int
2200 hermon_umap_db_clear_onclose_cb(dev_t dev, uint64_t flag)
2201 {
2202 hermon_umap_db_priv_t *priv;
2203 hermon_umap_db_entry_t *umapdb;
2204 minor_t instance;
2205 uint64_t value;
2206 int status;
2207
2208 instance = HERMON_DEV_INSTANCE(dev);
2209 if (instance == (minor_t)-1) {
2210 return (DDI_FAILURE);
2211 }
2212
2213 if (flag != HERMON_ONCLOSE_FLASH_INPROGRESS) {
2214 return (DDI_FAILURE);
2215 }
2216
2217 /*
2218 * Grab the lock for the "userland resources database" and find
2219 * the entry corresponding to this minor number. Once it's found,
2220 * remove the entry (in the "hdb_priv" field) that indicated the
2221 * need for further processing during Hermon's close(). Free the
2222 * entry, if appropriate.
2223 */
2224 mutex_enter(&hermon_userland_rsrc_db.hdl_umapdb_lock);
2225 status = hermon_umap_db_find_nolock(instance, dev,
2226 MLNX_UMAP_PID_RSRC, &value, 0, &umapdb);
2227 if (status != DDI_SUCCESS) {
2228 mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock);
2229 return (DDI_FAILURE);
2230 }
2231
2232 priv = (hermon_umap_db_priv_t *)umapdb->hdbe_common.hdb_priv;
2233 if (priv != NULL) {
2234 kmem_free(priv, sizeof (hermon_umap_db_priv_t));
2235 priv = NULL;
2236 }
2237
2238 umapdb->hdbe_common.hdb_priv = (void *)priv;
2239 mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock);
2240 return (DDI_SUCCESS);
2241 }
2242
2243
2244 /*
2245 * hermon_umap_db_clear_onclose_cb()
2246 * Context: Can be called from user or kernel context.
2247 */
2248 int
2249 hermon_umap_db_handle_onclose_cb(hermon_umap_db_priv_t *priv)
2250 {
2251 int (*callback)(void *);
2252
2253 ASSERT(MUTEX_HELD(&hermon_userland_rsrc_db.hdl_umapdb_lock));
2254
2255 /*
2256 * Call the callback.
2257 * Note: Currently there is only one callback (in "hdp_cb"), but
2258 * in the future there may be more, depending on what other types
2259 * of interaction there are between userland processes and the
2260 * driver.
2261 */
2262 callback = priv->hdp_cb;
2263 return (callback(priv->hdp_arg));
2264 }