1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25
26 /*
27 * hermon_umap.c
28 * Hermon Userland Mapping Routines
29 *
30 * Implements all the routines necessary for enabling direct userland
31 * access to the Hermon hardware. This includes all routines necessary for
32 * maintaining the "userland resources database" and all the support routines
33 * for the devmap calls.
34 */
35
36 #include <sys/types.h>
37 #include <sys/conf.h>
38 #include <sys/ddi.h>
39 #include <sys/sunddi.h>
40 #include <sys/modctl.h>
41 #include <sys/file.h>
42 #include <sys/avl.h>
43 #include <sys/sysmacros.h>
44
45 #include <sys/ib/adapters/hermon/hermon.h>
46
47 /* Hermon HCA state pointer (extern) */
48 extern void *hermon_statep;
49
50 /* Hermon HCA Userland Resource Database (extern) */
51 extern hermon_umap_db_t hermon_userland_rsrc_db;
52
53 static int hermon_umap_uarpg(hermon_state_t *state, devmap_cookie_t dhp,
54 hermon_rsrc_t *rsrcp, uint64_t offset, size_t *maplen, int *err);
55 static int hermon_umap_cqmem(hermon_state_t *state, devmap_cookie_t dhp,
56 hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err);
57 static int hermon_umap_qpmem(hermon_state_t *state, devmap_cookie_t dhp,
58 hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err);
59 static int hermon_umap_srqmem(hermon_state_t *state, devmap_cookie_t dhp,
60 hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err);
61 static int hermon_umap_dbrecmem(hermon_state_t *state, devmap_cookie_t dhp,
62 hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err);
63 static int hermon_devmap_umem_map(devmap_cookie_t dhp, dev_t dev, uint_t flags,
64 offset_t off, size_t len, void **pvtp);
65 static int hermon_devmap_umem_dup(devmap_cookie_t dhp, void *pvtp,
66 devmap_cookie_t new_dhp, void **new_pvtp);
67 static void hermon_devmap_umem_unmap(devmap_cookie_t dhp, void *pvtp,
68 offset_t off, size_t len, devmap_cookie_t new_dhp1, void **pvtp1,
69 devmap_cookie_t new_dhp2, void **pvtp2);
70 static int hermon_devmap_dbrecmem_map(devmap_cookie_t dhp, dev_t dev,
71 uint_t flags, offset_t off, size_t len, void **pvtp);
72 static int hermon_devmap_dbrecmem_dup(devmap_cookie_t dhp, void *pvtp,
73 devmap_cookie_t new_dhp, void **new_pvtp);
74 static void hermon_devmap_dbrecmem_unmap(devmap_cookie_t dhp, void *pvtp,
75 offset_t off, size_t len, devmap_cookie_t new_dhp1, void **pvtp1,
76 devmap_cookie_t new_dhp2, void **pvtp2);
77 static int hermon_devmap_devmem_map(devmap_cookie_t dhp, dev_t dev,
78 uint_t flags, offset_t off, size_t len, void **pvtp);
79 static int hermon_devmap_devmem_dup(devmap_cookie_t dhp, void *pvtp,
80 devmap_cookie_t new_dhp, void **new_pvtp);
81 static void hermon_devmap_devmem_unmap(devmap_cookie_t dhp, void *pvtp,
82 offset_t off, size_t len, devmap_cookie_t new_dhp1, void **pvtp1,
83 devmap_cookie_t new_dhp2, void **pvtp2);
84 static ibt_status_t hermon_umap_mr_data_in(hermon_mrhdl_t mr,
85 ibt_mr_data_in_t *data, size_t data_sz);
86 static ibt_status_t hermon_umap_cq_data_out(hermon_cqhdl_t cq,
87 mlnx_umap_cq_data_out_t *data, size_t data_sz);
88 static ibt_status_t hermon_umap_qp_data_out(hermon_qphdl_t qp,
89 mlnx_umap_qp_data_out_t *data, size_t data_sz);
90 static ibt_status_t hermon_umap_srq_data_out(hermon_srqhdl_t srq,
91 mlnx_umap_srq_data_out_t *data, size_t data_sz);
92 static ibt_status_t hermon_umap_pd_data_out(hermon_pdhdl_t pd,
93 mlnx_umap_pd_data_out_t *data, size_t data_sz);
94 static int hermon_umap_db_compare(const void *query, const void *entry);
95
96
97 /*
98 * These callbacks are passed to devmap_umem_setup() and devmap_devmem_setup(),
99 * respectively. They are used to handle (among other things) partial
100 * unmappings and to provide a method for invalidating mappings inherited
101 * as a result of a fork(2) system call.
102 */
103 static struct devmap_callback_ctl hermon_devmap_umem_cbops = {
104 DEVMAP_OPS_REV,
105 hermon_devmap_umem_map,
106 NULL,
107 hermon_devmap_umem_dup,
108 hermon_devmap_umem_unmap
109 };
110 static struct devmap_callback_ctl hermon_devmap_devmem_cbops = {
111 DEVMAP_OPS_REV,
112 hermon_devmap_devmem_map,
113 NULL,
114 hermon_devmap_devmem_dup,
115 hermon_devmap_devmem_unmap
116 };
117 static struct devmap_callback_ctl hermon_devmap_dbrecmem_cbops = {
118 DEVMAP_OPS_REV,
119 hermon_devmap_dbrecmem_map,
120 NULL,
121 hermon_devmap_dbrecmem_dup,
122 hermon_devmap_dbrecmem_unmap
123 };
124
125 /*
126 * hermon_devmap()
127 * Context: Can be called from user context.
128 */
129 /* ARGSUSED */
130 int
131 hermon_devmap(dev_t dev, devmap_cookie_t dhp, offset_t off, size_t len,
132 size_t *maplen, uint_t model)
133 {
134 hermon_state_t *state;
135 hermon_rsrc_t *rsrcp;
136 minor_t instance;
137 uint64_t key, value;
138 uint64_t bf_offset = 0;
139 uint_t type;
140 int err, status;
141
142 /* Get Hermon softstate structure from instance */
143 instance = HERMON_DEV_INSTANCE(dev);
144 state = ddi_get_soft_state(hermon_statep, instance);
145 if (state == NULL) {
146 return (ENXIO);
147 }
148
149 /*
150 * Access to Hermon devmap interface is not allowed in
151 * "maintenance mode".
152 */
153 if (state->hs_operational_mode == HERMON_MAINTENANCE_MODE) {
154 return (EFAULT);
155 }
156
157 /*
158 * The bottom bits of "offset" are undefined (number depends on
159 * system PAGESIZE). Shifting these off leaves us with a "key".
160 * The "key" is actually a combination of both a real key value
161 * (for the purpose of database lookup) and a "type" value. We
162 * extract this information before doing the database lookup.
163 */
164 key = off >> PAGESHIFT;
165 type = key & MLNX_UMAP_RSRC_TYPE_MASK;
166 key = key >> MLNX_UMAP_RSRC_TYPE_SHIFT;
167 if (type == MLNX_UMAP_BLUEFLAMEPG_RSRC) {
168 if (state->hs_devlim.blu_flm == 0) {
169 return (EFAULT);
170 }
171 bf_offset = state->hs_bf_offset;
172 type = MLNX_UMAP_UARPG_RSRC;
173 }
174 status = hermon_umap_db_find(instance, key, type, &value, 0, NULL);
175 if (status == DDI_SUCCESS) {
176 rsrcp = (hermon_rsrc_t *)(uintptr_t)value;
177
178 switch (type) {
179 case MLNX_UMAP_UARPG_RSRC:
180 /*
181 * Double check that process who open()'d Hermon is
182 * same process attempting to mmap() UAR page.
183 */
184 if (key != ddi_get_pid()) {
185 return (EINVAL);
186 }
187
188 /* Map the UAR page out for userland access */
189 status = hermon_umap_uarpg(state, dhp, rsrcp, bf_offset,
190 maplen, &err);
191 if (status != DDI_SUCCESS) {
192 return (err);
193 }
194 break;
195
196 case MLNX_UMAP_CQMEM_RSRC:
197 /* Map the CQ memory out for userland access */
198 status = hermon_umap_cqmem(state, dhp, rsrcp, off,
199 maplen, &err);
200 if (status != DDI_SUCCESS) {
201 return (err);
202 }
203 break;
204
205 case MLNX_UMAP_QPMEM_RSRC:
206 /* Map the QP memory out for userland access */
207 status = hermon_umap_qpmem(state, dhp, rsrcp, off,
208 maplen, &err);
209 if (status != DDI_SUCCESS) {
210 return (err);
211 }
212 break;
213
214 case MLNX_UMAP_SRQMEM_RSRC:
215 /* Map the SRQ memory out for userland access */
216 status = hermon_umap_srqmem(state, dhp, rsrcp, off,
217 maplen, &err);
218 if (status != DDI_SUCCESS) {
219 return (err);
220 }
221 break;
222
223 case MLNX_UMAP_DBRMEM_RSRC:
224 /*
225 * Map the doorbell record memory out for
226 * userland access
227 */
228 status = hermon_umap_dbrecmem(state, dhp, rsrcp, off,
229 maplen, &err);
230 if (status != DDI_SUCCESS) {
231 return (err);
232 }
233 break;
234
235 default:
236 HERMON_WARNING(state, "unexpected rsrc type in devmap");
237 return (EINVAL);
238 }
239 } else {
240 return (EINVAL);
241 }
242
243 return (0);
244 }
245
246
247 /*
248 * hermon_umap_uarpg()
249 * Context: Can be called from user context.
250 */
251 static int
252 hermon_umap_uarpg(hermon_state_t *state, devmap_cookie_t dhp,
253 hermon_rsrc_t *rsrcp, uint64_t offset, size_t *maplen, int *err)
254 {
255 int status;
256 uint_t maxprot;
257 ddi_device_acc_attr_t *accattrp = &state->hs_reg_accattr;
258 ddi_device_acc_attr_t accattr;
259
260 if (offset != 0) { /* Hermon Blueflame */
261 /* Try to use write coalescing data ordering */
262 accattr = *accattrp;
263 accattr.devacc_attr_dataorder = DDI_STORECACHING_OK_ACC;
264 accattrp = &accattr;
265 }
266
267 /* Map out the UAR page (doorbell page) */
268 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
269 status = devmap_devmem_setup(dhp, state->hs_dip,
270 &hermon_devmap_devmem_cbops, HERMON_UAR_BAR, (rsrcp->hr_indx <<
271 PAGESHIFT) + offset, PAGESIZE, maxprot, DEVMAP_ALLOW_REMAP,
272 accattrp);
273 if (status < 0) {
274 *err = status;
275 return (DDI_FAILURE);
276 }
277
278 *maplen = PAGESIZE;
279 return (DDI_SUCCESS);
280 }
281
282
283 /*
284 * hermon_umap_cqmem()
285 * Context: Can be called from user context.
286 */
287 /* ARGSUSED */
288 static int
289 hermon_umap_cqmem(hermon_state_t *state, devmap_cookie_t dhp,
290 hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err)
291 {
292 hermon_cqhdl_t cq;
293 size_t size;
294 uint_t maxprot;
295 int status;
296
297 /* Extract the Hermon CQ handle pointer from the hermon_rsrc_t */
298 cq = (hermon_cqhdl_t)rsrcp->hr_addr;
299
300 /* Round-up the CQ size to system page size */
301 size = ptob(btopr(cq->cq_resize_hdl ?
302 cq->cq_resize_hdl->cq_cqinfo.qa_size : cq->cq_cqinfo.qa_size));
303
304 /* Map out the CQ memory - use resize_hdl if non-NULL */
305 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
306 status = devmap_umem_setup(dhp, state->hs_dip,
307 &hermon_devmap_umem_cbops, cq->cq_resize_hdl ?
308 cq->cq_resize_hdl->cq_cqinfo.qa_umemcookie :
309 cq->cq_cqinfo.qa_umemcookie, 0, size,
310 maxprot, (DEVMAP_ALLOW_REMAP | DEVMAP_DEFAULTS), NULL);
311 if (status < 0) {
312 *err = status;
313 return (DDI_FAILURE);
314 }
315 *maplen = size;
316
317 return (DDI_SUCCESS);
318 }
319
320
321 /*
322 * hermon_umap_qpmem()
323 * Context: Can be called from user context.
324 */
325 /* ARGSUSED */
326 static int
327 hermon_umap_qpmem(hermon_state_t *state, devmap_cookie_t dhp,
328 hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err)
329 {
330 hermon_qphdl_t qp;
331 offset_t offset;
332 size_t size;
333 uint_t maxprot;
334 int status;
335
336 /* Extract the Hermon QP handle pointer from the hermon_rsrc_t */
337 qp = (hermon_qphdl_t)rsrcp->hr_addr;
338
339 /*
340 * Calculate the offset of the first work queue (send or recv) into
341 * the memory (ddi_umem_alloc()) allocated previously for the QP.
342 */
343 offset = (offset_t)((uintptr_t)qp->qp_wqinfo.qa_buf_aligned -
344 (uintptr_t)qp->qp_wqinfo.qa_buf_real);
345
346 /* Round-up the QP work queue sizes to system page size */
347 size = ptob(btopr(qp->qp_wqinfo.qa_size));
348
349 /* Map out the QP memory */
350 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
351 status = devmap_umem_setup(dhp, state->hs_dip,
352 &hermon_devmap_umem_cbops, qp->qp_wqinfo.qa_umemcookie, offset,
353 size, maxprot, (DEVMAP_ALLOW_REMAP | DEVMAP_DEFAULTS), NULL);
354 if (status < 0) {
355 *err = status;
356 return (DDI_FAILURE);
357 }
358 *maplen = size;
359
360 return (DDI_SUCCESS);
361 }
362
363
364 /*
365 * hermon_umap_srqmem()
366 * Context: Can be called from user context.
367 */
368 /* ARGSUSED */
369 static int
370 hermon_umap_srqmem(hermon_state_t *state, devmap_cookie_t dhp,
371 hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err)
372 {
373 hermon_srqhdl_t srq;
374 offset_t offset;
375 size_t size;
376 uint_t maxprot;
377 int status;
378
379 /* Extract the Hermon SRQ handle pointer from the hermon_rsrc_t */
380 srq = (hermon_srqhdl_t)rsrcp->hr_addr;
381
382 /*
383 * Calculate the offset of the first shared recv queue into the memory
384 * (ddi_umem_alloc()) allocated previously for the SRQ.
385 */
386 offset = (offset_t)((uintptr_t)srq->srq_wqinfo.qa_buf_aligned -
387 (uintptr_t)srq->srq_wqinfo.qa_buf_real);
388
389 /* Round-up the SRQ work queue sizes to system page size */
390 size = ptob(btopr(srq->srq_wqinfo.qa_size));
391
392 /* Map out the SRQ memory */
393 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
394 status = devmap_umem_setup(dhp, state->hs_dip,
395 &hermon_devmap_umem_cbops, srq->srq_wqinfo.qa_umemcookie, offset,
396 size, maxprot, (DEVMAP_ALLOW_REMAP | DEVMAP_DEFAULTS), NULL);
397 if (status < 0) {
398 *err = status;
399 return (DDI_FAILURE);
400 }
401 *maplen = size;
402
403 return (DDI_SUCCESS);
404 }
405
406
407 /*
408 * hermon_devmap_dbrecmem()
409 * Context: Can be called from user context.
410 */
411 /* ARGSUSED */
412 static int
413 hermon_umap_dbrecmem(hermon_state_t *state, devmap_cookie_t dhp,
414 hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err)
415 {
416 hermon_udbr_page_t *pagep;
417 offset_t offset;
418 size_t size;
419 uint_t maxprot;
420 int status;
421
422 /* We stored the udbr_page pointer, and not a hermon_rsrc_t */
423 pagep = (hermon_udbr_page_t *)rsrcp;
424
425 /*
426 * Calculate the offset of the doorbell records into the memory
427 * (ddi_umem_alloc()) allocated previously for them.
428 */
429 offset = 0;
430
431 /* Round-up the doorbell page to system page size */
432 size = PAGESIZE;
433
434 /* Map out the Doorbell Record memory */
435 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
436 status = devmap_umem_setup(dhp, state->hs_dip,
437 &hermon_devmap_dbrecmem_cbops, pagep->upg_umemcookie, offset,
438 size, maxprot, (DEVMAP_ALLOW_REMAP | DEVMAP_DEFAULTS), NULL);
439 if (status < 0) {
440 *err = status;
441 return (DDI_FAILURE);
442 }
443 *maplen = size;
444
445 return (DDI_SUCCESS);
446 }
447
448
449 /*
450 * hermon_devmap_umem_map()
451 * Context: Can be called from kernel context.
452 */
453 /* ARGSUSED */
454 static int
455 hermon_devmap_umem_map(devmap_cookie_t dhp, dev_t dev, uint_t flags,
456 offset_t off, size_t len, void **pvtp)
457 {
458 hermon_state_t *state;
459 hermon_devmap_track_t *dvm_track;
460 hermon_cqhdl_t cq;
461 hermon_qphdl_t qp;
462 hermon_srqhdl_t srq;
463 minor_t instance;
464 uint64_t key;
465 uint_t type;
466
467 /* Get Hermon softstate structure from instance */
468 instance = HERMON_DEV_INSTANCE(dev);
469 state = ddi_get_soft_state(hermon_statep, instance);
470 if (state == NULL) {
471 return (ENXIO);
472 }
473
474 /*
475 * The bottom bits of "offset" are undefined (number depends on
476 * system PAGESIZE). Shifting these off leaves us with a "key".
477 * The "key" is actually a combination of both a real key value
478 * (for the purpose of database lookup) and a "type" value. Although
479 * we are not going to do any database lookup per se, we do want
480 * to extract the "key" and the "type" (to enable faster lookup of
481 * the appropriate CQ or QP handle).
482 */
483 key = off >> PAGESHIFT;
484 type = key & MLNX_UMAP_RSRC_TYPE_MASK;
485 key = key >> MLNX_UMAP_RSRC_TYPE_SHIFT;
486
487 /*
488 * Allocate an entry to track the mapping and unmapping (specifically,
489 * partial unmapping) of this resource.
490 */
491 dvm_track = (hermon_devmap_track_t *)kmem_zalloc(
492 sizeof (hermon_devmap_track_t), KM_SLEEP);
493 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
494 dvm_track->hdt_offset = off;
495 dvm_track->hdt_state = state;
496 dvm_track->hdt_refcnt = 1;
497 mutex_init(&dvm_track->hdt_lock, NULL, MUTEX_DRIVER,
498 DDI_INTR_PRI(state->hs_intrmsi_pri));
499
500 /*
501 * Depending of the type of resource that has been mapped out, we
502 * need to update the QP or CQ handle to reflect that it has, in
503 * fact, been mapped. This allows the driver code which frees a QP
504 * or a CQ to know whether it is appropriate to do a
505 * devmap_devmem_remap() to invalidate the userland mapping for the
506 * corresponding queue's memory.
507 */
508 if (type == MLNX_UMAP_CQMEM_RSRC) {
509
510 /* Use "key" (CQ number) to do fast lookup of CQ handle */
511 cq = hermon_cqhdl_from_cqnum(state, key);
512
513 /*
514 * Update the handle to the userland mapping. Note: If
515 * the CQ already has a valid userland mapping, then stop
516 * and return failure.
517 */
518 mutex_enter(&cq->cq_lock);
519 if (cq->cq_umap_dhp == NULL) {
520 cq->cq_umap_dhp = dhp;
521 dvm_track->hdt_size = cq->cq_cqinfo.qa_size;
522 mutex_exit(&cq->cq_lock);
523 } else if (cq->cq_resize_hdl &&
524 (cq->cq_resize_hdl->cq_umap_dhp == NULL)) {
525 cq->cq_resize_hdl->cq_umap_dhp = dhp;
526 dvm_track->hdt_size =
527 cq->cq_resize_hdl->cq_cqinfo.qa_size;
528 mutex_exit(&cq->cq_lock);
529 } else {
530 mutex_exit(&cq->cq_lock);
531 goto umem_map_fail;
532 }
533
534 } else if (type == MLNX_UMAP_QPMEM_RSRC) {
535
536 /* Use "key" (QP number) to do fast lookup of QP handle */
537 qp = hermon_qphdl_from_qpnum(state, key);
538
539 /*
540 * Update the handle to the userland mapping. Note: If
541 * the CQ already has a valid userland mapping, then stop
542 * and return failure.
543 */
544 mutex_enter(&qp->qp_lock);
545 if (qp->qp_umap_dhp == NULL) {
546 qp->qp_umap_dhp = dhp;
547 dvm_track->hdt_size = qp->qp_wqinfo.qa_size;
548 mutex_exit(&qp->qp_lock);
549 } else {
550 mutex_exit(&qp->qp_lock);
551 goto umem_map_fail;
552 }
553
554 } else if (type == MLNX_UMAP_SRQMEM_RSRC) {
555
556 /* Use "key" (SRQ number) to do fast lookup on SRQ handle */
557 srq = hermon_srqhdl_from_srqnum(state, key);
558
559 /*
560 * Update the handle to the userland mapping. Note: If the
561 * SRQ already has a valid userland mapping, then stop and
562 * return failure.
563 */
564 mutex_enter(&srq->srq_lock);
565 if (srq->srq_umap_dhp == NULL) {
566 srq->srq_umap_dhp = dhp;
567 dvm_track->hdt_size = srq->srq_wqinfo.qa_size;
568 mutex_exit(&srq->srq_lock);
569 } else {
570 mutex_exit(&srq->srq_lock);
571 goto umem_map_fail;
572 }
573 }
574
575 /*
576 * Pass the private "Hermon devmap tracking structure" back. This
577 * pointer will be returned in subsequent "unmap" callbacks.
578 */
579 *pvtp = dvm_track;
580
581 return (DDI_SUCCESS);
582
583 umem_map_fail:
584 mutex_destroy(&dvm_track->hdt_lock);
585 kmem_free(dvm_track, sizeof (hermon_devmap_track_t));
586 return (DDI_FAILURE);
587 }
588
589
590 /*
591 * hermon_devmap_umem_dup()
592 * Context: Can be called from kernel context.
593 */
594 /* ARGSUSED */
595 static int
596 hermon_devmap_umem_dup(devmap_cookie_t dhp, void *pvtp, devmap_cookie_t new_dhp,
597 void **new_pvtp)
598 {
599 hermon_state_t *state;
600 hermon_devmap_track_t *dvm_track, *new_dvm_track;
601 uint_t maxprot;
602 int status;
603
604 /*
605 * Extract the Hermon softstate pointer from "Hermon devmap tracking
606 * structure" (in "pvtp").
607 */
608 dvm_track = (hermon_devmap_track_t *)pvtp;
609 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
610 state = dvm_track->hdt_state;
611
612 /*
613 * Since this devmap_dup() entry point is generally called
614 * when a process does fork(2), it is incumbent upon the driver
615 * to insure that the child does not inherit a valid copy of
616 * the parent's QP or CQ resource. This is accomplished by using
617 * devmap_devmem_remap() to invalidate the child's mapping to the
618 * kernel memory.
619 */
620 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
621 status = devmap_devmem_remap(new_dhp, state->hs_dip, 0, 0,
622 dvm_track->hdt_size, maxprot, DEVMAP_MAPPING_INVALID, NULL);
623 if (status != DDI_SUCCESS) {
624 HERMON_WARNING(state, "failed in hermon_devmap_umem_dup()");
625 return (status);
626 }
627
628 /*
629 * Allocate a new entry to track the subsequent unmapping
630 * (specifically, all partial unmappings) of the child's newly
631 * invalidated resource. Note: Setting the "hdt_size" field to
632 * zero here is an indication to the devmap_unmap() entry point
633 * that this mapping is invalid, and that its subsequent unmapping
634 * should not affect any of the parent's CQ or QP resources.
635 */
636 new_dvm_track = (hermon_devmap_track_t *)kmem_zalloc(
637 sizeof (hermon_devmap_track_t), KM_SLEEP);
638 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*new_dvm_track))
639 new_dvm_track->hdt_offset = 0;
640 new_dvm_track->hdt_state = state;
641 new_dvm_track->hdt_refcnt = 1;
642 new_dvm_track->hdt_size = 0;
643 mutex_init(&new_dvm_track->hdt_lock, NULL, MUTEX_DRIVER,
644 DDI_INTR_PRI(state->hs_intrmsi_pri));
645 *new_pvtp = new_dvm_track;
646
647 return (DDI_SUCCESS);
648 }
649
650
651 /*
652 * hermon_devmap_umem_unmap()
653 * Context: Can be called from kernel context.
654 */
655 /* ARGSUSED */
656 static void
657 hermon_devmap_umem_unmap(devmap_cookie_t dhp, void *pvtp, offset_t off,
658 size_t len, devmap_cookie_t new_dhp1, void **pvtp1,
659 devmap_cookie_t new_dhp2, void **pvtp2)
660 {
661 hermon_state_t *state;
662 hermon_rsrc_t *rsrcp;
663 hermon_devmap_track_t *dvm_track;
664 hermon_cqhdl_t cq;
665 hermon_qphdl_t qp;
666 hermon_srqhdl_t srq;
667 uint64_t key, value;
668 uint_t type;
669 uint_t size;
670 int status;
671
672 /*
673 * Extract the Hermon softstate pointer from "Hermon devmap tracking
674 * structure" (in "pvtp").
675 */
676 dvm_track = (hermon_devmap_track_t *)pvtp;
677 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
678 state = dvm_track->hdt_state;
679
680 /*
681 * Extract the "offset" from the "Hermon devmap tracking structure".
682 * Note: The input argument "off" is ignored here because the
683 * Hermon mapping interfaces define a very specific meaning to
684 * each "logical offset". Also extract the "key" and "type" encoded
685 * in the logical offset.
686 */
687 key = dvm_track->hdt_offset >> PAGESHIFT;
688 type = key & MLNX_UMAP_RSRC_TYPE_MASK;
689 key = key >> MLNX_UMAP_RSRC_TYPE_SHIFT;
690
691 /*
692 * Extract the "size" of the mapping. If this size is determined
693 * to be zero, then it is an indication of a previously invalidated
694 * mapping, and no CQ or QP resources should be affected.
695 */
696 size = dvm_track->hdt_size;
697
698 /*
699 * If only the "middle portion of a given mapping is being unmapped,
700 * then we are effectively creating one new piece of mapped memory.
701 * (Original region is divided into three pieces of which the middle
702 * piece is being removed. This leaves two pieces. Since we started
703 * with one piece and now have two pieces, we need to increment the
704 * counter in the "Hermon devmap tracking structure".
705 *
706 * If, however, the whole mapped region is being unmapped, then we
707 * have started with one region which we are completely removing.
708 * In this case, we need to decrement the counter in the "Hermon
709 * devmap tracking structure".
710 *
711 * In each of the remaining cases, we will have started with one
712 * mapped region and ended with one (different) region. So no counter
713 * modification is necessary.
714 */
715 mutex_enter(&dvm_track->hdt_lock);
716 if ((new_dhp1 == NULL) && (new_dhp2 == NULL)) {
717 dvm_track->hdt_refcnt--;
718 } else if ((new_dhp1 != NULL) && (new_dhp2 != NULL)) {
719 dvm_track->hdt_refcnt++;
720 }
721 mutex_exit(&dvm_track->hdt_lock);
722
723 /*
724 * For each of the cases where the region is being divided, then we
725 * need to pass back the "Hermon devmap tracking structure". This way
726 * we get it back when each of the remaining pieces is subsequently
727 * unmapped.
728 */
729 if (new_dhp1 != NULL) {
730 *pvtp1 = pvtp;
731 }
732 if (new_dhp2 != NULL) {
733 *pvtp2 = pvtp;
734 }
735
736 /*
737 * If the "Hermon devmap tracking structure" is no longer being
738 * referenced, then free it up. Otherwise, return.
739 */
740 if (dvm_track->hdt_refcnt == 0) {
741 mutex_destroy(&dvm_track->hdt_lock);
742 kmem_free(dvm_track, sizeof (hermon_devmap_track_t));
743
744 /*
745 * If the mapping was invalid (see explanation above), then
746 * no further processing is necessary.
747 */
748 if (size == 0) {
749 return;
750 }
751 } else {
752 return;
753 }
754
755 /*
756 * Now that we can guarantee that the user memory is fully unmapped,
757 * we can use the "key" and "type" values to try to find the entry
758 * in the "userland resources database". If it's found, then it
759 * indicates that the queue memory (CQ or QP) has not yet been freed.
760 * In this case, we update the corresponding CQ or QP handle to
761 * indicate that the "devmap_devmem_remap()" call will be unnecessary.
762 * If it's _not_ found, then it indicates that the CQ or QP memory
763 * was, in fact, freed before it was unmapped (thus requiring a
764 * previous invalidation by remapping - which will already have
765 * been done in the free routine).
766 */
767 status = hermon_umap_db_find(state->hs_instance, key, type, &value,
768 0, NULL);
769 if (status == DDI_SUCCESS) {
770 /*
771 * Depending on the type of the mapped resource (CQ or QP),
772 * update handle to indicate that no invalidation remapping
773 * will be necessary.
774 */
775 if (type == MLNX_UMAP_CQMEM_RSRC) {
776
777 /* Use "value" to convert to CQ handle */
778 rsrcp = (hermon_rsrc_t *)(uintptr_t)value;
779 cq = (hermon_cqhdl_t)rsrcp->hr_addr;
780
781 /*
782 * Invalidate the handle to the userland mapping.
783 * Note: We must ensure that the mapping being
784 * unmapped here is the current one for the CQ. It
785 * is possible that it might not be if this CQ has
786 * been resized and the previous CQ memory has not
787 * yet been unmapped. But in that case, because of
788 * the devmap_devmem_remap(), there is no longer any
789 * association between the mapping and the real CQ
790 * kernel memory.
791 */
792 mutex_enter(&cq->cq_lock);
793 if (cq->cq_umap_dhp == dhp) {
794 cq->cq_umap_dhp = NULL;
795 if (cq->cq_resize_hdl) {
796 /* resize is DONE, switch queues */
797 hermon_cq_resize_helper(state, cq);
798 }
799 } else {
800 if (cq->cq_resize_hdl &&
801 cq->cq_resize_hdl->cq_umap_dhp == dhp) {
802 /*
803 * Unexpected case. munmap of the
804 * cq_resize buf, and not the
805 * original buf.
806 */
807 cq->cq_resize_hdl->cq_umap_dhp = NULL;
808 }
809 }
810 mutex_exit(&cq->cq_lock);
811
812 } else if (type == MLNX_UMAP_QPMEM_RSRC) {
813
814 /* Use "value" to convert to QP handle */
815 rsrcp = (hermon_rsrc_t *)(uintptr_t)value;
816 qp = (hermon_qphdl_t)rsrcp->hr_addr;
817
818 /*
819 * Invalidate the handle to the userland mapping.
820 * Note: we ensure that the mapping being unmapped
821 * here is the current one for the QP. This is
822 * more of a sanity check here since, unlike CQs
823 * (above) we do not support resize of QPs.
824 */
825 mutex_enter(&qp->qp_lock);
826 if (qp->qp_umap_dhp == dhp) {
827 qp->qp_umap_dhp = NULL;
828 }
829 mutex_exit(&qp->qp_lock);
830
831 } else if (type == MLNX_UMAP_SRQMEM_RSRC) {
832
833 /* Use "value" to convert to SRQ handle */
834 rsrcp = (hermon_rsrc_t *)(uintptr_t)value;
835 srq = (hermon_srqhdl_t)rsrcp->hr_addr;
836
837 /*
838 * Invalidate the handle to the userland mapping.
839 * Note: we ensure that the mapping being unmapped
840 * here is the current one for the QP. This is
841 * more of a sanity check here since, unlike CQs
842 * (above) we do not support resize of QPs.
843 */
844 mutex_enter(&srq->srq_lock);
845 if (srq->srq_umap_dhp == dhp) {
846 srq->srq_umap_dhp = NULL;
847 }
848 mutex_exit(&srq->srq_lock);
849 }
850 }
851 }
852
853
854 /*
855 * hermon_devmap_devmem_map()
856 * Context: Can be called from kernel context.
857 */
858 /* ARGSUSED */
859 static int
860 hermon_devmap_dbrecmem_map(devmap_cookie_t dhp, dev_t dev, uint_t flags,
861 offset_t off, size_t len, void **pvtp)
862 {
863 hermon_state_t *state;
864 hermon_devmap_track_t *dvm_track;
865 hermon_cqhdl_t cq;
866 hermon_qphdl_t qp;
867 hermon_srqhdl_t srq;
868 minor_t instance;
869 uint64_t key;
870 uint_t type;
871
872 /* Get Hermon softstate structure from instance */
873 instance = HERMON_DEV_INSTANCE(dev);
874 state = ddi_get_soft_state(hermon_statep, instance);
875 if (state == NULL) {
876 return (ENXIO);
877 }
878
879 /*
880 * The bottom bits of "offset" are undefined (number depends on
881 * system PAGESIZE). Shifting these off leaves us with a "key".
882 * The "key" is actually a combination of both a real key value
883 * (for the purpose of database lookup) and a "type" value. Although
884 * we are not going to do any database lookup per se, we do want
885 * to extract the "key" and the "type" (to enable faster lookup of
886 * the appropriate CQ or QP handle).
887 */
888 key = off >> PAGESHIFT;
889 type = key & MLNX_UMAP_RSRC_TYPE_MASK;
890 key = key >> MLNX_UMAP_RSRC_TYPE_SHIFT;
891
892 /*
893 * Allocate an entry to track the mapping and unmapping (specifically,
894 * partial unmapping) of this resource.
895 */
896 dvm_track = (hermon_devmap_track_t *)kmem_zalloc(
897 sizeof (hermon_devmap_track_t), KM_SLEEP);
898 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
899 dvm_track->hdt_offset = off;
900 dvm_track->hdt_state = state;
901 dvm_track->hdt_refcnt = 1;
902 mutex_init(&dvm_track->hdt_lock, NULL, MUTEX_DRIVER,
903 DDI_INTR_PRI(state->hs_intrmsi_pri));
904
905 /*
906 * Depending of the type of resource that has been mapped out, we
907 * need to update the QP or CQ handle to reflect that it has, in
908 * fact, been mapped. This allows the driver code which frees a QP
909 * or a CQ to know whether it is appropriate to do a
910 * devmap_devmem_remap() to invalidate the userland mapping for the
911 * corresponding queue's memory.
912 */
913 if (type == MLNX_UMAP_CQMEM_RSRC) {
914
915 /* Use "key" (CQ number) to do fast lookup of CQ handle */
916 cq = hermon_cqhdl_from_cqnum(state, key);
917
918 /*
919 * Update the handle to the userland mapping. Note: If
920 * the CQ already has a valid userland mapping, then stop
921 * and return failure.
922 */
923 mutex_enter(&cq->cq_lock);
924 if (cq->cq_umap_dhp == NULL) {
925 cq->cq_umap_dhp = dhp;
926 dvm_track->hdt_size = cq->cq_cqinfo.qa_size;
927 mutex_exit(&cq->cq_lock);
928 } else {
929 mutex_exit(&cq->cq_lock);
930 goto umem_map_fail;
931 }
932
933 } else if (type == MLNX_UMAP_QPMEM_RSRC) {
934
935 /* Use "key" (QP number) to do fast lookup of QP handle */
936 qp = hermon_qphdl_from_qpnum(state, key);
937
938 /*
939 * Update the handle to the userland mapping. Note: If
940 * the CQ already has a valid userland mapping, then stop
941 * and return failure.
942 */
943 mutex_enter(&qp->qp_lock);
944 if (qp->qp_umap_dhp == NULL) {
945 qp->qp_umap_dhp = dhp;
946 dvm_track->hdt_size = qp->qp_wqinfo.qa_size;
947 mutex_exit(&qp->qp_lock);
948 } else {
949 mutex_exit(&qp->qp_lock);
950 goto umem_map_fail;
951 }
952
953 } else if (type == MLNX_UMAP_SRQMEM_RSRC) {
954
955 /* Use "key" (SRQ number) to do fast lookup on SRQ handle */
956 srq = hermon_srqhdl_from_srqnum(state, key);
957
958 /*
959 * Update the handle to the userland mapping. Note: If the
960 * SRQ already has a valid userland mapping, then stop and
961 * return failure.
962 */
963 mutex_enter(&srq->srq_lock);
964 if (srq->srq_umap_dhp == NULL) {
965 srq->srq_umap_dhp = dhp;
966 dvm_track->hdt_size = srq->srq_wqinfo.qa_size;
967 mutex_exit(&srq->srq_lock);
968 } else {
969 mutex_exit(&srq->srq_lock);
970 goto umem_map_fail;
971 }
972 }
973
974 /*
975 * Pass the private "Hermon devmap tracking structure" back. This
976 * pointer will be returned in subsequent "unmap" callbacks.
977 */
978 *pvtp = dvm_track;
979
980 return (DDI_SUCCESS);
981
982 umem_map_fail:
983 mutex_destroy(&dvm_track->hdt_lock);
984 kmem_free(dvm_track, sizeof (hermon_devmap_track_t));
985 return (DDI_FAILURE);
986 }
987
988
989 /*
990 * hermon_devmap_dbrecmem_dup()
991 * Context: Can be called from kernel context.
992 */
993 /* ARGSUSED */
994 static int
995 hermon_devmap_dbrecmem_dup(devmap_cookie_t dhp, void *pvtp,
996 devmap_cookie_t new_dhp, void **new_pvtp)
997 {
998 hermon_state_t *state;
999 hermon_devmap_track_t *dvm_track, *new_dvm_track;
1000 uint_t maxprot;
1001 int status;
1002
1003 /*
1004 * Extract the Hermon softstate pointer from "Hermon devmap tracking
1005 * structure" (in "pvtp").
1006 */
1007 dvm_track = (hermon_devmap_track_t *)pvtp;
1008 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
1009 state = dvm_track->hdt_state;
1010
1011 /*
1012 * Since this devmap_dup() entry point is generally called
1013 * when a process does fork(2), it is incumbent upon the driver
1014 * to insure that the child does not inherit a valid copy of
1015 * the parent's QP or CQ resource. This is accomplished by using
1016 * devmap_devmem_remap() to invalidate the child's mapping to the
1017 * kernel memory.
1018 */
1019 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
1020 status = devmap_devmem_remap(new_dhp, state->hs_dip, 0, 0,
1021 dvm_track->hdt_size, maxprot, DEVMAP_MAPPING_INVALID, NULL);
1022 if (status != DDI_SUCCESS) {
1023 HERMON_WARNING(state, "failed in hermon_devmap_dbrecmem_dup()");
1024 return (status);
1025 }
1026
1027 /*
1028 * Allocate a new entry to track the subsequent unmapping
1029 * (specifically, all partial unmappings) of the child's newly
1030 * invalidated resource. Note: Setting the "hdt_size" field to
1031 * zero here is an indication to the devmap_unmap() entry point
1032 * that this mapping is invalid, and that its subsequent unmapping
1033 * should not affect any of the parent's CQ or QP resources.
1034 */
1035 new_dvm_track = (hermon_devmap_track_t *)kmem_zalloc(
1036 sizeof (hermon_devmap_track_t), KM_SLEEP);
1037 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*new_dvm_track))
1038 new_dvm_track->hdt_offset = 0;
1039 new_dvm_track->hdt_state = state;
1040 new_dvm_track->hdt_refcnt = 1;
1041 new_dvm_track->hdt_size = 0;
1042 mutex_init(&new_dvm_track->hdt_lock, NULL, MUTEX_DRIVER,
1043 DDI_INTR_PRI(state->hs_intrmsi_pri));
1044 *new_pvtp = new_dvm_track;
1045
1046 return (DDI_SUCCESS);
1047 }
1048
1049
1050 /*
1051 * hermon_devmap_dbrecmem_unmap()
1052 * Context: Can be called from kernel context.
1053 */
1054 /* ARGSUSED */
1055 static void
1056 hermon_devmap_dbrecmem_unmap(devmap_cookie_t dhp, void *pvtp, offset_t off,
1057 size_t len, devmap_cookie_t new_dhp1, void **pvtp1,
1058 devmap_cookie_t new_dhp2, void **pvtp2)
1059 {
1060 hermon_state_t *state;
1061 hermon_rsrc_t *rsrcp;
1062 hermon_devmap_track_t *dvm_track;
1063 hermon_cqhdl_t cq;
1064 hermon_qphdl_t qp;
1065 hermon_srqhdl_t srq;
1066 uint64_t key, value;
1067 uint_t type;
1068 uint_t size;
1069 int status;
1070
1071 /*
1072 * Extract the Hermon softstate pointer from "Hermon devmap tracking
1073 * structure" (in "pvtp").
1074 */
1075 dvm_track = (hermon_devmap_track_t *)pvtp;
1076 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
1077 state = dvm_track->hdt_state;
1078
1079 /*
1080 * Extract the "offset" from the "Hermon devmap tracking structure".
1081 * Note: The input argument "off" is ignored here because the
1082 * Hermon mapping interfaces define a very specific meaning to
1083 * each "logical offset". Also extract the "key" and "type" encoded
1084 * in the logical offset.
1085 */
1086 key = dvm_track->hdt_offset >> PAGESHIFT;
1087 type = key & MLNX_UMAP_RSRC_TYPE_MASK;
1088 key = key >> MLNX_UMAP_RSRC_TYPE_SHIFT;
1089
1090 /*
1091 * Extract the "size" of the mapping. If this size is determined
1092 * to be zero, then it is an indication of a previously invalidated
1093 * mapping, and no CQ or QP resources should be affected.
1094 */
1095 size = dvm_track->hdt_size;
1096
1097 /*
1098 * If only the "middle portion of a given mapping is being unmapped,
1099 * then we are effectively creating one new piece of mapped memory.
1100 * (Original region is divided into three pieces of which the middle
1101 * piece is being removed. This leaves two pieces. Since we started
1102 * with one piece and now have two pieces, we need to increment the
1103 * counter in the "Hermon devmap tracking structure".
1104 *
1105 * If, however, the whole mapped region is being unmapped, then we
1106 * have started with one region which we are completely removing.
1107 * In this case, we need to decrement the counter in the "Hermon
1108 * devmap tracking structure".
1109 *
1110 * In each of the remaining cases, we will have started with one
1111 * mapped region and ended with one (different) region. So no counter
1112 * modification is necessary.
1113 */
1114 mutex_enter(&dvm_track->hdt_lock);
1115 if ((new_dhp1 == NULL) && (new_dhp2 == NULL)) {
1116 dvm_track->hdt_refcnt--;
1117 } else if ((new_dhp1 != NULL) && (new_dhp2 != NULL)) {
1118 dvm_track->hdt_refcnt++;
1119 }
1120 mutex_exit(&dvm_track->hdt_lock);
1121
1122 /*
1123 * For each of the cases where the region is being divided, then we
1124 * need to pass back the "Hermon devmap tracking structure". This way
1125 * we get it back when each of the remaining pieces is subsequently
1126 * unmapped.
1127 */
1128 if (new_dhp1 != NULL) {
1129 *pvtp1 = pvtp;
1130 }
1131 if (new_dhp2 != NULL) {
1132 *pvtp2 = pvtp;
1133 }
1134
1135 /*
1136 * If the "Hermon devmap tracking structure" is no longer being
1137 * referenced, then free it up. Otherwise, return.
1138 */
1139 if (dvm_track->hdt_refcnt == 0) {
1140 mutex_destroy(&dvm_track->hdt_lock);
1141 kmem_free(dvm_track, sizeof (hermon_devmap_track_t));
1142
1143 /*
1144 * If the mapping was invalid (see explanation above), then
1145 * no further processing is necessary.
1146 */
1147 if (size == 0) {
1148 return;
1149 }
1150 } else {
1151 return;
1152 }
1153
1154 /*
1155 * Now that we can guarantee that the user memory is fully unmapped,
1156 * we can use the "key" and "type" values to try to find the entry
1157 * in the "userland resources database". If it's found, then it
1158 * indicates that the queue memory (CQ or QP) has not yet been freed.
1159 * In this case, we update the corresponding CQ or QP handle to
1160 * indicate that the "devmap_devmem_remap()" call will be unnecessary.
1161 * If it's _not_ found, then it indicates that the CQ or QP memory
1162 * was, in fact, freed before it was unmapped (thus requiring a
1163 * previous invalidation by remapping - which will already have
1164 * been done in the free routine).
1165 */
1166 status = hermon_umap_db_find(state->hs_instance, key, type, &value,
1167 0, NULL);
1168 if (status == DDI_SUCCESS) {
1169 /*
1170 * Depending on the type of the mapped resource (CQ or QP),
1171 * update handle to indicate that no invalidation remapping
1172 * will be necessary.
1173 */
1174 if (type == MLNX_UMAP_CQMEM_RSRC) {
1175
1176 /* Use "value" to convert to CQ handle */
1177 rsrcp = (hermon_rsrc_t *)(uintptr_t)value;
1178 cq = (hermon_cqhdl_t)rsrcp->hr_addr;
1179
1180 /*
1181 * Invalidate the handle to the userland mapping.
1182 * Note: We must ensure that the mapping being
1183 * unmapped here is the current one for the CQ. It
1184 * is possible that it might not be if this CQ has
1185 * been resized and the previous CQ memory has not
1186 * yet been unmapped. But in that case, because of
1187 * the devmap_devmem_remap(), there is no longer any
1188 * association between the mapping and the real CQ
1189 * kernel memory.
1190 */
1191 mutex_enter(&cq->cq_lock);
1192 if (cq->cq_umap_dhp == dhp) {
1193 cq->cq_umap_dhp = NULL;
1194 }
1195 mutex_exit(&cq->cq_lock);
1196
1197 } else if (type == MLNX_UMAP_QPMEM_RSRC) {
1198
1199 /* Use "value" to convert to QP handle */
1200 rsrcp = (hermon_rsrc_t *)(uintptr_t)value;
1201 qp = (hermon_qphdl_t)rsrcp->hr_addr;
1202
1203 /*
1204 * Invalidate the handle to the userland mapping.
1205 * Note: we ensure that the mapping being unmapped
1206 * here is the current one for the QP. This is
1207 * more of a sanity check here since, unlike CQs
1208 * (above) we do not support resize of QPs.
1209 */
1210 mutex_enter(&qp->qp_lock);
1211 if (qp->qp_umap_dhp == dhp) {
1212 qp->qp_umap_dhp = NULL;
1213 }
1214 mutex_exit(&qp->qp_lock);
1215
1216 } else if (type == MLNX_UMAP_SRQMEM_RSRC) {
1217
1218 /* Use "value" to convert to SRQ handle */
1219 rsrcp = (hermon_rsrc_t *)(uintptr_t)value;
1220 srq = (hermon_srqhdl_t)rsrcp->hr_addr;
1221
1222 /*
1223 * Invalidate the handle to the userland mapping.
1224 * Note: we ensure that the mapping being unmapped
1225 * here is the current one for the QP. This is
1226 * more of a sanity check here since, unlike CQs
1227 * (above) we do not support resize of QPs.
1228 */
1229 mutex_enter(&srq->srq_lock);
1230 if (srq->srq_umap_dhp == dhp) {
1231 srq->srq_umap_dhp = NULL;
1232 }
1233 mutex_exit(&srq->srq_lock);
1234 }
1235 }
1236 }
1237
1238
1239 /*
1240 * hermon_devmap_devmem_map()
1241 * Context: Can be called from kernel context.
1242 */
1243 /* ARGSUSED */
1244 static int
1245 hermon_devmap_devmem_map(devmap_cookie_t dhp, dev_t dev, uint_t flags,
1246 offset_t off, size_t len, void **pvtp)
1247 {
1248 hermon_state_t *state;
1249 hermon_devmap_track_t *dvm_track;
1250 minor_t instance;
1251
1252 /* Get Hermon softstate structure from instance */
1253 instance = HERMON_DEV_INSTANCE(dev);
1254 state = ddi_get_soft_state(hermon_statep, instance);
1255 if (state == NULL) {
1256 return (ENXIO);
1257 }
1258
1259 /*
1260 * Allocate an entry to track the mapping and unmapping of this
1261 * resource. Note: We don't need to initialize the "refcnt" or
1262 * "offset" fields here, nor do we need to initialize the mutex
1263 * used with the "refcnt". Since UAR pages are single pages, they
1264 * are not subject to "partial" unmappings. This makes these other
1265 * fields unnecessary.
1266 */
1267 dvm_track = (hermon_devmap_track_t *)kmem_zalloc(
1268 sizeof (hermon_devmap_track_t), KM_SLEEP);
1269 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
1270 dvm_track->hdt_state = state;
1271 dvm_track->hdt_size = (uint_t)PAGESIZE;
1272
1273 /*
1274 * Pass the private "Hermon devmap tracking structure" back. This
1275 * pointer will be returned in a subsequent "unmap" callback.
1276 */
1277 *pvtp = dvm_track;
1278
1279 return (DDI_SUCCESS);
1280 }
1281
1282
1283 /*
1284 * hermon_devmap_devmem_dup()
1285 * Context: Can be called from kernel context.
1286 */
1287 /* ARGSUSED */
1288 static int
1289 hermon_devmap_devmem_dup(devmap_cookie_t dhp, void *pvtp,
1290 devmap_cookie_t new_dhp, void **new_pvtp)
1291 {
1292 hermon_state_t *state;
1293 hermon_devmap_track_t *dvm_track;
1294 uint_t maxprot;
1295 int status;
1296
1297 /*
1298 * Extract the Hermon softstate pointer from "Hermon devmap tracking
1299 * structure" (in "pvtp"). Note: If the tracking structure is NULL
1300 * here, it means that the mapping corresponds to an invalid mapping.
1301 * In this case, it can be safely ignored ("new_pvtp" set to NULL).
1302 */
1303 dvm_track = (hermon_devmap_track_t *)pvtp;
1304 if (dvm_track == NULL) {
1305 *new_pvtp = NULL;
1306 return (DDI_SUCCESS);
1307 }
1308
1309 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
1310 state = dvm_track->hdt_state;
1311
1312 /*
1313 * Since this devmap_dup() entry point is generally called
1314 * when a process does fork(2), it is incumbent upon the driver
1315 * to insure that the child does not inherit a valid copy of
1316 * the parent's resource. This is accomplished by using
1317 * devmap_devmem_remap() to invalidate the child's mapping to the
1318 * kernel memory.
1319 */
1320 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
1321 status = devmap_devmem_remap(new_dhp, state->hs_dip, 0, 0,
1322 dvm_track->hdt_size, maxprot, DEVMAP_MAPPING_INVALID, NULL);
1323 if (status != DDI_SUCCESS) {
1324 HERMON_WARNING(state, "failed in hermon_devmap_devmem_dup()");
1325 return (status);
1326 }
1327
1328 /*
1329 * Since the region is invalid, there is no need for us to
1330 * allocate and continue to track an additional "Hermon devmap
1331 * tracking structure". Instead we return NULL here, which is an
1332 * indication to the devmap_unmap() entry point that this entry
1333 * can be safely ignored.
1334 */
1335 *new_pvtp = NULL;
1336
1337 return (DDI_SUCCESS);
1338 }
1339
1340
1341 /*
1342 * hermon_devmap_devmem_unmap()
1343 * Context: Can be called from kernel context.
1344 */
1345 /* ARGSUSED */
1346 static void
1347 hermon_devmap_devmem_unmap(devmap_cookie_t dhp, void *pvtp, offset_t off,
1348 size_t len, devmap_cookie_t new_dhp1, void **pvtp1,
1349 devmap_cookie_t new_dhp2, void **pvtp2)
1350 {
1351 hermon_devmap_track_t *dvm_track;
1352
1353 /*
1354 * Free up the "Hermon devmap tracking structure" (in "pvtp").
1355 * There cannot be "partial" unmappings here because all UAR pages
1356 * are single pages. Note: If the tracking structure is NULL here,
1357 * it means that the mapping corresponds to an invalid mapping. In
1358 * this case, it can be safely ignored.
1359 */
1360 dvm_track = (hermon_devmap_track_t *)pvtp;
1361 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
1362 if (dvm_track == NULL) {
1363 return;
1364 }
1365
1366 kmem_free(dvm_track, sizeof (hermon_devmap_track_t));
1367 }
1368
1369
1370 /*
1371 * hermon_umap_ci_data_in()
1372 * Context: Can be called from user or kernel context.
1373 */
1374 /* ARGSUSED */
1375 ibt_status_t
1376 hermon_umap_ci_data_in(hermon_state_t *state, ibt_ci_data_flags_t flags,
1377 ibt_object_type_t object, void *hdl, void *data_p, size_t data_sz)
1378 {
1379 int status;
1380
1381 /*
1382 * Depending on the type of object about which additional information
1383 * is being provided (currently only MR is supported), we call the
1384 * appropriate resource-specific function.
1385 */
1386 switch (object) {
1387 case IBT_HDL_MR:
1388 status = hermon_umap_mr_data_in((hermon_mrhdl_t)hdl,
1389 (ibt_mr_data_in_t *)data_p, data_sz);
1390 if (status != DDI_SUCCESS) {
1391 return (status);
1392 }
1393 break;
1394
1395 /*
1396 * For other possible valid IBT types, we return IBT_NOT_SUPPORTED,
1397 * since the Hermon driver does not support these.
1398 */
1399 case IBT_HDL_HCA:
1400 case IBT_HDL_QP:
1401 case IBT_HDL_CQ:
1402 case IBT_HDL_PD:
1403 case IBT_HDL_MW:
1404 case IBT_HDL_AH:
1405 case IBT_HDL_SCHED:
1406 case IBT_HDL_EEC:
1407 case IBT_HDL_RDD:
1408 case IBT_HDL_SRQ:
1409 return (IBT_NOT_SUPPORTED);
1410
1411 /*
1412 * Any other types are invalid.
1413 */
1414 default:
1415 return (IBT_INVALID_PARAM);
1416 }
1417
1418 return (DDI_SUCCESS);
1419 }
1420
1421
1422 /*
1423 * hermon_umap_mr_data_in()
1424 * Context: Can be called from user or kernel context.
1425 */
1426 static ibt_status_t
1427 hermon_umap_mr_data_in(hermon_mrhdl_t mr, ibt_mr_data_in_t *data,
1428 size_t data_sz)
1429 {
1430 if (data->mr_rev != IBT_MR_DATA_IN_IF_VERSION) {
1431 return (IBT_NOT_SUPPORTED);
1432 }
1433
1434 /* Check for valid MR handle pointer */
1435 if (mr == NULL) {
1436 return (IBT_MR_HDL_INVALID);
1437 }
1438
1439 /* Check for valid MR input structure size */
1440 if (data_sz < sizeof (ibt_mr_data_in_t)) {
1441 return (IBT_INSUFF_RESOURCE);
1442 }
1443 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*data))
1444
1445 /*
1446 * Ensure that the MR corresponds to userland memory and that it is
1447 * a currently valid memory region as well.
1448 */
1449 mutex_enter(&mr->mr_lock);
1450 if ((mr->mr_is_umem == 0) || (mr->mr_umemcookie == NULL)) {
1451 mutex_exit(&mr->mr_lock);
1452 return (IBT_MR_HDL_INVALID);
1453 }
1454
1455 /*
1456 * If it has passed all the above checks, then extract the callback
1457 * function and argument from the input structure. Copy them into
1458 * the MR handle. This function will be called only if the memory
1459 * corresponding to the MR handle gets a umem_lockmemory() callback.
1460 */
1461 mr->mr_umem_cbfunc = data->mr_func;
1462 mr->mr_umem_cbarg1 = data->mr_arg1;
1463 mr->mr_umem_cbarg2 = data->mr_arg2;
1464 mutex_exit(&mr->mr_lock);
1465
1466 return (DDI_SUCCESS);
1467 }
1468
1469
1470 /*
1471 * hermon_umap_ci_data_out()
1472 * Context: Can be called from user or kernel context.
1473 */
1474 /* ARGSUSED */
1475 ibt_status_t
1476 hermon_umap_ci_data_out(hermon_state_t *state, ibt_ci_data_flags_t flags,
1477 ibt_object_type_t object, void *hdl, void *data_p, size_t data_sz)
1478 {
1479 int status;
1480
1481 /*
1482 * Depending on the type of object about which additional information
1483 * is being requested (CQ or QP), we call the appropriate resource-
1484 * specific mapping function.
1485 */
1486 switch (object) {
1487 case IBT_HDL_CQ:
1488 status = hermon_umap_cq_data_out((hermon_cqhdl_t)hdl,
1489 (mlnx_umap_cq_data_out_t *)data_p, data_sz);
1490 if (status != DDI_SUCCESS) {
1491 return (status);
1492 }
1493 break;
1494
1495 case IBT_HDL_QP:
1496 status = hermon_umap_qp_data_out((hermon_qphdl_t)hdl,
1497 (mlnx_umap_qp_data_out_t *)data_p, data_sz);
1498 if (status != DDI_SUCCESS) {
1499 return (status);
1500 }
1501 break;
1502
1503 case IBT_HDL_SRQ:
1504 status = hermon_umap_srq_data_out((hermon_srqhdl_t)hdl,
1505 (mlnx_umap_srq_data_out_t *)data_p, data_sz);
1506 if (status != DDI_SUCCESS) {
1507 return (status);
1508 }
1509 break;
1510
1511 case IBT_HDL_PD:
1512 status = hermon_umap_pd_data_out((hermon_pdhdl_t)hdl,
1513 (mlnx_umap_pd_data_out_t *)data_p, data_sz);
1514 if (status != DDI_SUCCESS) {
1515 return (status);
1516 }
1517 break;
1518
1519 /*
1520 * For other possible valid IBT types, we return IBT_NOT_SUPPORTED,
1521 * since the Hermon driver does not support these.
1522 */
1523 case IBT_HDL_HCA:
1524 case IBT_HDL_MR:
1525 case IBT_HDL_MW:
1526 case IBT_HDL_AH:
1527 case IBT_HDL_SCHED:
1528 case IBT_HDL_EEC:
1529 case IBT_HDL_RDD:
1530 return (IBT_NOT_SUPPORTED);
1531
1532 /*
1533 * Any other types are invalid.
1534 */
1535 default:
1536 return (IBT_INVALID_PARAM);
1537 }
1538
1539 return (DDI_SUCCESS);
1540 }
1541
1542
1543 /*
1544 * hermon_umap_cq_data_out()
1545 * Context: Can be called from user or kernel context.
1546 */
1547 static ibt_status_t
1548 hermon_umap_cq_data_out(hermon_cqhdl_t cq, mlnx_umap_cq_data_out_t *data,
1549 size_t data_sz)
1550 {
1551 /* Check for valid CQ handle pointer */
1552 if (cq == NULL) {
1553 return (IBT_CQ_HDL_INVALID);
1554 }
1555
1556 /* Check for valid CQ mapping structure size */
1557 if (data_sz < sizeof (mlnx_umap_cq_data_out_t)) {
1558 return (IBT_INSUFF_RESOURCE);
1559 }
1560 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*data))
1561
1562 /* deal with cq_alloc() verses cq_resize() */
1563 if (cq->cq_resize_hdl) {
1564 data->mcq_maplen = cq->cq_resize_hdl->cq_cqinfo.qa_size;
1565 data->mcq_numcqe = cq->cq_resize_hdl->cq_bufsz;
1566 } else {
1567 data->mcq_maplen = cq->cq_cqinfo.qa_size;
1568 data->mcq_numcqe = cq->cq_bufsz;
1569 }
1570
1571 /*
1572 * If it has passed all the above checks, then fill in all the useful
1573 * mapping information (including the mapping offset that will be
1574 * passed back to the devmap() interface during a subsequent mmap()
1575 * call.
1576 *
1577 * The "offset" for CQ mmap()'s looks like this:
1578 * +----------------------------------------+--------+--------------+
1579 * | CQ Number | 0x33 | Reserved (0) |
1580 * +----------------------------------------+--------+--------------+
1581 * (64 - 8 - PAGESHIFT) bits 8 bits PAGESHIFT bits
1582 *
1583 * This returns information about the mapping offset, the length of
1584 * the CQ memory, the CQ number (for use in later CQ doorbells), the
1585 * number of CQEs the CQ memory can hold, and the size of each CQE.
1586 */
1587 data->mcq_rev = MLNX_UMAP_IF_VERSION;
1588 data->mcq_mapoffset = ((((uint64_t)cq->cq_cqnum <<
1589 MLNX_UMAP_RSRC_TYPE_SHIFT) | MLNX_UMAP_CQMEM_RSRC) << PAGESHIFT);
1590 data->mcq_cqnum = cq->cq_cqnum;
1591 data->mcq_cqesz = sizeof (hermon_hw_cqe_t);
1592
1593 /* doorbell record fields */
1594 data->mcq_polldbr_mapoffset = cq->cq_dbr_mapoffset;
1595 data->mcq_polldbr_maplen = PAGESIZE;
1596 data->mcq_polldbr_offset = (uintptr_t)cq->cq_arm_ci_vdbr &
1597 PAGEOFFSET;
1598 data->mcq_armdbr_mapoffset = cq->cq_dbr_mapoffset;
1599 data->mcq_armdbr_maplen = PAGESIZE;
1600 data->mcq_armdbr_offset = data->mcq_polldbr_offset + 4;
1601
1602 return (DDI_SUCCESS);
1603 }
1604
1605
1606 /*
1607 * hermon_umap_qp_data_out()
1608 * Context: Can be called from user or kernel context.
1609 */
1610 static ibt_status_t
1611 hermon_umap_qp_data_out(hermon_qphdl_t qp, mlnx_umap_qp_data_out_t *data,
1612 size_t data_sz)
1613 {
1614 /* Check for valid QP handle pointer */
1615 if (qp == NULL) {
1616 return (IBT_QP_HDL_INVALID);
1617 }
1618
1619 /* Check for valid QP mapping structure size */
1620 if (data_sz < sizeof (mlnx_umap_qp_data_out_t)) {
1621 return (IBT_INSUFF_RESOURCE);
1622 }
1623 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*data))
1624
1625 /*
1626 * If it has passed all the checks, then fill in all the useful
1627 * mapping information (including the mapping offset that will be
1628 * passed back to the devmap() interface during a subsequent mmap()
1629 * call.
1630 *
1631 * The "offset" for QP mmap()'s looks like this:
1632 * +----------------------------------------+--------+--------------+
1633 * | QP Number | 0x44 | Reserved (0) |
1634 * +----------------------------------------+--------+--------------+
1635 * (64 - 8 - PAGESHIFT) bits 8 bits PAGESHIFT bits
1636 *
1637 * This returns information about the mapping offset, the length of
1638 * the QP memory, and the QP number (for use in later send and recv
1639 * doorbells). It also returns the following information for both
1640 * the receive work queue and the send work queue, respectively: the
1641 * offset (from the base mapped address) of the start of the given
1642 * work queue, the 64-bit IB virtual address that corresponds to
1643 * the base mapped address (needed for posting WQEs though the
1644 * QP doorbells), the number of WQEs the given work queue can hold,
1645 * and the size of each WQE for the given work queue.
1646 */
1647 data->mqp_rev = MLNX_UMAP_IF_VERSION;
1648 data->mqp_mapoffset = ((((uint64_t)qp->qp_qpnum <<
1649 MLNX_UMAP_RSRC_TYPE_SHIFT) | MLNX_UMAP_QPMEM_RSRC) << PAGESHIFT);
1650 data->mqp_maplen = qp->qp_wqinfo.qa_size;
1651 data->mqp_qpnum = qp->qp_qpnum;
1652
1653 /*
1654 * If this QP is associated with a shared receive queue (SRQ),
1655 * then return invalid RecvQ parameters. Otherwise, return
1656 * the proper parameter values.
1657 */
1658 if (qp->qp_alloc_flags & IBT_QP_USES_SRQ) {
1659 data->mqp_rq_off = (uint32_t)qp->qp_wqinfo.qa_size;
1660 data->mqp_rq_desc_addr = (uint32_t)qp->qp_wqinfo.qa_size;
1661 data->mqp_rq_numwqe = 0;
1662 data->mqp_rq_wqesz = 0;
1663 data->mqp_rdbr_mapoffset = 0;
1664 data->mqp_rdbr_maplen = 0;
1665 data->mqp_rdbr_offset = 0;
1666 } else {
1667 data->mqp_rq_off = (uintptr_t)qp->qp_rq_buf -
1668 (uintptr_t)qp->qp_wqinfo.qa_buf_aligned;
1669 data->mqp_rq_desc_addr = (uint32_t)((uintptr_t)qp->qp_rq_buf -
1670 qp->qp_desc_off);
1671 data->mqp_rq_numwqe = qp->qp_rq_bufsz;
1672 data->mqp_rq_wqesz = (1 << qp->qp_rq_log_wqesz);
1673
1674 /* doorbell record fields */
1675 data->mqp_rdbr_mapoffset = qp->qp_rdbr_mapoffset;
1676 data->mqp_rdbr_maplen = PAGESIZE;
1677 data->mqp_rdbr_offset = (uintptr_t)qp->qp_rq_vdbr &
1678 PAGEOFFSET;
1679 }
1680 data->mqp_sq_off = (uintptr_t)qp->qp_sq_buf -
1681 (uintptr_t)qp->qp_wqinfo.qa_buf_aligned;
1682 data->mqp_sq_desc_addr = (uint32_t)((uintptr_t)qp->qp_sq_buf -
1683 qp->qp_desc_off);
1684 data->mqp_sq_numwqe = qp->qp_sq_bufsz;
1685 data->mqp_sq_wqesz = (1 << qp->qp_sq_log_wqesz);
1686 data->mqp_sq_headroomwqes = qp->qp_sq_hdrmwqes;
1687
1688 /* doorbell record fields */
1689 data->mqp_sdbr_mapoffset = 0;
1690 data->mqp_sdbr_maplen = 0;
1691 data->mqp_sdbr_offset = 0;
1692
1693 return (DDI_SUCCESS);
1694 }
1695
1696
1697 /*
1698 * hermon_umap_srq_data_out()
1699 * Context: Can be called from user or kernel context.
1700 */
1701 static ibt_status_t
1702 hermon_umap_srq_data_out(hermon_srqhdl_t srq, mlnx_umap_srq_data_out_t *data,
1703 size_t data_sz)
1704 {
1705 /* Check for valid SRQ handle pointer */
1706 if (srq == NULL) {
1707 return (IBT_SRQ_HDL_INVALID);
1708 }
1709
1710 /* Check for valid SRQ mapping structure size */
1711 if (data_sz < sizeof (mlnx_umap_srq_data_out_t)) {
1712 return (IBT_INSUFF_RESOURCE);
1713 }
1714 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*data))
1715
1716 /*
1717 * If it has passed all the checks, then fill in all the useful
1718 * mapping information (including the mapping offset that will be
1719 * passed back to the devmap() interface during a subsequent mmap()
1720 * call.
1721 *
1722 * The "offset" for SRQ mmap()'s looks like this:
1723 * +----------------------------------------+--------+--------------+
1724 * | SRQ Number | 0x66 | Reserved (0) |
1725 * +----------------------------------------+--------+--------------+
1726 * (64 - 8 - PAGESHIFT) bits 8 bits PAGESHIFT bits
1727 *
1728 * This returns information about the mapping offset, the length of the
1729 * SRQ memory, and the SRQ number (for use in later send and recv
1730 * doorbells). It also returns the following information for the
1731 * shared receive queue: the offset (from the base mapped address) of
1732 * the start of the given work queue, the 64-bit IB virtual address
1733 * that corresponds to the base mapped address (needed for posting WQEs
1734 * though the QP doorbells), the number of WQEs the given work queue
1735 * can hold, and the size of each WQE for the given work queue.
1736 */
1737 data->msrq_rev = MLNX_UMAP_IF_VERSION;
1738 data->msrq_mapoffset = ((((uint64_t)srq->srq_srqnum <<
1739 MLNX_UMAP_RSRC_TYPE_SHIFT) | MLNX_UMAP_SRQMEM_RSRC) << PAGESHIFT);
1740 data->msrq_maplen = srq->srq_wqinfo.qa_size;
1741 data->msrq_srqnum = srq->srq_srqnum;
1742
1743 data->msrq_desc_addr = (uint32_t)((uintptr_t)srq->srq_wq_buf -
1744 srq->srq_desc_off);
1745 data->msrq_numwqe = srq->srq_wq_bufsz;
1746 data->msrq_wqesz = (1 << srq->srq_wq_log_wqesz);
1747
1748 /* doorbell record fields */
1749 data->msrq_rdbr_mapoffset = srq->srq_rdbr_mapoffset;
1750 data->msrq_rdbr_maplen = PAGESIZE;
1751 data->msrq_rdbr_offset = (uintptr_t)srq->srq_wq_vdbr &
1752 PAGEOFFSET;
1753
1754 return (DDI_SUCCESS);
1755 }
1756
1757
1758 /*
1759 * hermon_umap_pd_data_out()
1760 * Context: Can be called from user or kernel context.
1761 */
1762 static ibt_status_t
1763 hermon_umap_pd_data_out(hermon_pdhdl_t pd, mlnx_umap_pd_data_out_t *data,
1764 size_t data_sz)
1765 {
1766 /* Check for valid PD handle pointer */
1767 if (pd == NULL) {
1768 return (IBT_PD_HDL_INVALID);
1769 }
1770
1771 /* Check for valid PD mapping structure size */
1772 if (data_sz < sizeof (mlnx_umap_pd_data_out_t)) {
1773 return (IBT_INSUFF_RESOURCE);
1774 }
1775 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*data))
1776
1777 /*
1778 * If it has passed all the checks, then fill the PD table index
1779 * (the PD table allocated index for the PD pd_pdnum).
1780 */
1781 data->mpd_rev = MLNX_UMAP_IF_VERSION;
1782 data->mpd_pdnum = pd->pd_pdnum;
1783
1784 return (DDI_SUCCESS);
1785 }
1786
1787
1788 /*
1789 * hermon_umap_db_init()
1790 * Context: Only called from attach() path context
1791 */
1792 void
1793 hermon_umap_db_init(void)
1794 {
1795 /*
1796 * Initialize the lock used by the Hermon "userland resources database"
1797 * This is used to ensure atomic access to add, remove, and find
1798 * entries in the database.
1799 */
1800 mutex_init(&hermon_userland_rsrc_db.hdl_umapdb_lock, NULL,
1801 MUTEX_DRIVER, NULL);
1802
1803 /*
1804 * Initialize the AVL tree used for the "userland resources
1805 * database". Using an AVL tree here provides the ability to
1806 * scale the database size to large numbers of resources. The
1807 * entries in the tree are "hermon_umap_db_entry_t" (see
1808 * hermon_umap.h). The tree is searched with the help of the
1809 * hermon_umap_db_compare() routine.
1810 */
1811 avl_create(&hermon_userland_rsrc_db.hdl_umapdb_avl,
1812 hermon_umap_db_compare, sizeof (hermon_umap_db_entry_t),
1813 offsetof(hermon_umap_db_entry_t, hdbe_avlnode));
1814 }
1815
1816
1817 /*
1818 * hermon_umap_db_fini()
1819 * Context: Only called from attach() and/or detach() path contexts
1820 */
1821 void
1822 hermon_umap_db_fini(void)
1823 {
1824 /* Destroy the AVL tree for the "userland resources database" */
1825 avl_destroy(&hermon_userland_rsrc_db.hdl_umapdb_avl);
1826
1827 /* Destroy the lock for the "userland resources database" */
1828 mutex_destroy(&hermon_userland_rsrc_db.hdl_umapdb_lock);
1829 }
1830
1831
1832 /*
1833 * hermon_umap_db_alloc()
1834 * Context: Can be called from user or kernel context.
1835 */
1836 hermon_umap_db_entry_t *
1837 hermon_umap_db_alloc(uint_t instance, uint64_t key, uint_t type, uint64_t value)
1838 {
1839 hermon_umap_db_entry_t *umapdb;
1840
1841 /* Allocate an entry to add to the "userland resources database" */
1842 umapdb = kmem_zalloc(sizeof (hermon_umap_db_entry_t), KM_NOSLEEP);
1843 if (umapdb == NULL) {
1844 return (NULL);
1845 }
1846 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*umapdb))
1847
1848 /* Fill in the fields in the database entry */
1849 umapdb->hdbe_common.hdb_instance = instance;
1850 umapdb->hdbe_common.hdb_type = type;
1851 umapdb->hdbe_common.hdb_key = key;
1852 umapdb->hdbe_common.hdb_value = value;
1853
1854 return (umapdb);
1855 }
1856
1857
1858 /*
1859 * hermon_umap_db_free()
1860 * Context: Can be called from user or kernel context.
1861 */
1862 void
1863 hermon_umap_db_free(hermon_umap_db_entry_t *umapdb)
1864 {
1865 /* Free the database entry */
1866 kmem_free(umapdb, sizeof (hermon_umap_db_entry_t));
1867 }
1868
1869
1870 /*
1871 * hermon_umap_db_add()
1872 * Context: Can be called from user or kernel context.
1873 */
1874 void
1875 hermon_umap_db_add(hermon_umap_db_entry_t *umapdb)
1876 {
1877 mutex_enter(&hermon_userland_rsrc_db.hdl_umapdb_lock);
1878 hermon_umap_db_add_nolock(umapdb);
1879 mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock);
1880 }
1881
1882
1883 /*
1884 * hermon_umap_db_add_nolock()
1885 * Context: Can be called from user or kernel context.
1886 */
1887 void
1888 hermon_umap_db_add_nolock(hermon_umap_db_entry_t *umapdb)
1889 {
1890 hermon_umap_db_query_t query;
1891 avl_index_t where;
1892
1893 ASSERT(MUTEX_HELD(&hermon_userland_rsrc_db.hdl_umapdb_lock));
1894
1895 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*umapdb))
1896
1897 /*
1898 * Copy the common portion of the "to-be-added" database entry
1899 * into the "hermon_umap_db_query_t" structure. We use this structure
1900 * (with no flags set) to find the appropriate location in the
1901 * "userland resources database" for the new entry to be added.
1902 *
1903 * Note: we expect that this entry should not be found in the
1904 * database (unless something bad has happened).
1905 */
1906 query.hqdb_common = umapdb->hdbe_common;
1907 query.hqdb_flags = 0;
1908 (void) avl_find(&hermon_userland_rsrc_db.hdl_umapdb_avl, &query,
1909 &where);
1910
1911 /*
1912 * Now, using the "where" field from the avl_find() operation
1913 * above, we will insert the new database entry ("umapdb").
1914 */
1915 avl_insert(&hermon_userland_rsrc_db.hdl_umapdb_avl, umapdb,
1916 where);
1917 }
1918
1919
1920 /*
1921 * hermon_umap_db_find()
1922 * Context: Can be called from user or kernel context.
1923 */
1924 int
1925 hermon_umap_db_find(uint_t instance, uint64_t key, uint_t type,
1926 uint64_t *value, uint_t flag, hermon_umap_db_entry_t **umapdb)
1927 {
1928 int status;
1929
1930 mutex_enter(&hermon_userland_rsrc_db.hdl_umapdb_lock);
1931 status = hermon_umap_db_find_nolock(instance, key, type, value, flag,
1932 umapdb);
1933 mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock);
1934
1935 return (status);
1936 }
1937
1938
1939 /*
1940 * hermon_umap_db_find_nolock()
1941 * Context: Can be called from user or kernel context.
1942 */
1943 int
1944 hermon_umap_db_find_nolock(uint_t instance, uint64_t key, uint_t type,
1945 uint64_t *value, uint_t flags, hermon_umap_db_entry_t **umapdb)
1946 {
1947 hermon_umap_db_query_t query;
1948 hermon_umap_db_entry_t *entry;
1949 avl_index_t where;
1950
1951 ASSERT(MUTEX_HELD(&hermon_userland_rsrc_db.hdl_umapdb_lock));
1952
1953 /*
1954 * Fill in key, type, instance, and flags values of the
1955 * hermon_umap_db_query_t in preparation for the database
1956 * lookup.
1957 */
1958 query.hqdb_flags = flags;
1959 query.hqdb_common.hdb_key = key;
1960 query.hqdb_common.hdb_type = type;
1961 query.hqdb_common.hdb_instance = instance;
1962
1963 /*
1964 * Perform the database query. If no entry is found, then
1965 * return failure, else continue.
1966 */
1967 entry = (hermon_umap_db_entry_t *)avl_find(
1968 &hermon_userland_rsrc_db.hdl_umapdb_avl, &query, &where);
1969 if (entry == NULL) {
1970 return (DDI_FAILURE);
1971 }
1972 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*entry))
1973
1974 /*
1975 * If the flags argument specifies that the entry should
1976 * be removed if found, then call avl_remove() to remove
1977 * the entry from the database.
1978 */
1979 if (flags & HERMON_UMAP_DB_REMOVE) {
1980
1981 avl_remove(&hermon_userland_rsrc_db.hdl_umapdb_avl, entry);
1982
1983 /*
1984 * The database entry is returned with the expectation
1985 * that the caller will use hermon_umap_db_free() to
1986 * free the entry's memory. ASSERT that this is non-NULL.
1987 * NULL pointer should never be passed for the
1988 * HERMON_UMAP_DB_REMOVE case.
1989 */
1990 ASSERT(umapdb != NULL);
1991 }
1992
1993 /*
1994 * If the caller would like visibility to the database entry
1995 * (indicated through the use of a non-NULL "umapdb" argument),
1996 * then fill it in.
1997 */
1998 if (umapdb != NULL) {
1999 *umapdb = entry;
2000 }
2001
2002 /* Extract value field from database entry and return success */
2003 *value = entry->hdbe_common.hdb_value;
2004
2005 return (DDI_SUCCESS);
2006 }
2007
2008
2009 /*
2010 * hermon_umap_umemlock_cb()
2011 * Context: Can be called from callback context.
2012 */
2013 void
2014 hermon_umap_umemlock_cb(ddi_umem_cookie_t *umem_cookie)
2015 {
2016 hermon_umap_db_entry_t *umapdb;
2017 hermon_state_t *state;
2018 hermon_rsrc_t *rsrcp;
2019 hermon_mrhdl_t mr;
2020 uint64_t value;
2021 uint_t instance;
2022 int status;
2023 void (*mr_callback)(void *, void *);
2024 void *mr_cbarg1, *mr_cbarg2;
2025
2026 /*
2027 * If this was userland memory, then we need to remove its entry
2028 * from the "userland resources database". Note: We use the
2029 * HERMON_UMAP_DB_IGNORE_INSTANCE flag here because we don't know
2030 * which instance was used when the entry was added (but we want
2031 * to know after the entry is found using the other search criteria).
2032 */
2033 status = hermon_umap_db_find(0, (uint64_t)(uintptr_t)umem_cookie,
2034 MLNX_UMAP_MRMEM_RSRC, &value, (HERMON_UMAP_DB_REMOVE |
2035 HERMON_UMAP_DB_IGNORE_INSTANCE), &umapdb);
2036 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*umapdb))
2037 if (status == DDI_SUCCESS) {
2038 instance = umapdb->hdbe_common.hdb_instance;
2039 state = ddi_get_soft_state(hermon_statep, instance);
2040 if (state == NULL) {
2041 cmn_err(CE_WARN, "Unable to match Hermon instance\n");
2042 return;
2043 }
2044
2045 /* Free the database entry */
2046 hermon_umap_db_free(umapdb);
2047
2048 /* Use "value" to convert to an MR handle */
2049 rsrcp = (hermon_rsrc_t *)(uintptr_t)value;
2050 mr = (hermon_mrhdl_t)rsrcp->hr_addr;
2051
2052 /*
2053 * If a callback has been provided, call it first. This
2054 * callback is expected to do any cleanup necessary to
2055 * guarantee that the subsequent MR deregister (below)
2056 * will succeed. Specifically, this means freeing up memory
2057 * windows which might have been associated with the MR.
2058 */
2059 mutex_enter(&mr->mr_lock);
2060 mr_callback = mr->mr_umem_cbfunc;
2061 mr_cbarg1 = mr->mr_umem_cbarg1;
2062 mr_cbarg2 = mr->mr_umem_cbarg2;
2063 mutex_exit(&mr->mr_lock);
2064 if (mr_callback != NULL) {
2065 mr_callback(mr_cbarg1, mr_cbarg2);
2066 }
2067
2068 /*
2069 * Then call hermon_mr_deregister() to release the resources
2070 * associated with the MR handle. Note: Because this routine
2071 * will also check for whether the ddi_umem_cookie_t is in the
2072 * database, it will take responsibility for disabling the
2073 * memory region and calling ddi_umem_unlock().
2074 */
2075 status = hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
2076 HERMON_SLEEP);
2077 if (status != DDI_SUCCESS) {
2078 HERMON_WARNING(state, "Unexpected failure in "
2079 "deregister from callback\n");
2080 }
2081 }
2082 }
2083
2084
2085 /*
2086 * hermon_umap_db_compare()
2087 * Context: Can be called from user or kernel context.
2088 */
2089 static int
2090 hermon_umap_db_compare(const void *q, const void *e)
2091 {
2092 hermon_umap_db_common_t *entry_common, *query_common;
2093 uint_t query_flags;
2094
2095 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*((hermon_umap_db_query_t *)q)))
2096
2097 entry_common = &((hermon_umap_db_entry_t *)e)->hdbe_common;
2098 query_common = &((hermon_umap_db_query_t *)q)->hqdb_common;
2099 query_flags = ((hermon_umap_db_query_t *)q)->hqdb_flags;
2100
2101 /*
2102 * The first comparison is done on the "key" value in "query"
2103 * and "entry". If they are not equal, then the appropriate
2104 * search direction is returned. Else, we continue by
2105 * comparing "type".
2106 */
2107 if (query_common->hdb_key < entry_common->hdb_key) {
2108 return (-1);
2109 } else if (query_common->hdb_key > entry_common->hdb_key) {
2110 return (+1);
2111 }
2112
2113 /*
2114 * If the search reaches this point, then "query" and "entry"
2115 * have equal key values. So we continue be comparing their
2116 * "type" values. Again, if they are not equal, then the
2117 * appropriate search direction is returned. Else, we continue
2118 * by comparing "instance".
2119 */
2120 if (query_common->hdb_type < entry_common->hdb_type) {
2121 return (-1);
2122 } else if (query_common->hdb_type > entry_common->hdb_type) {
2123 return (+1);
2124 }
2125
2126 /*
2127 * If the search reaches this point, then "query" and "entry"
2128 * have exactly the same key and type values. Now we consult
2129 * the "flags" field in the query to determine whether the
2130 * "instance" is relevant to the search. If the
2131 * HERMON_UMAP_DB_IGNORE_INSTANCE flags is set, then return
2132 * success (0) here. Otherwise, continue the search by comparing
2133 * instance values and returning the appropriate search direction.
2134 */
2135 if (query_flags & HERMON_UMAP_DB_IGNORE_INSTANCE) {
2136 return (0);
2137 }
2138
2139 /*
2140 * If the search has reached this point, then "query" and "entry"
2141 * can only be differentiated by their instance values. If these
2142 * are not equal, then return the appropriate search direction.
2143 * Else, we return success (0).
2144 */
2145 if (query_common->hdb_instance < entry_common->hdb_instance) {
2146 return (-1);
2147 } else if (query_common->hdb_instance > entry_common->hdb_instance) {
2148 return (+1);
2149 }
2150
2151 /* Everything matches... so return success */
2152 return (0);
2153 }
2154
2155
2156 /*
2157 * hermon_umap_db_set_onclose_cb()
2158 * Context: Can be called from user or kernel context.
2159 */
2160 int
2161 hermon_umap_db_set_onclose_cb(dev_t dev, uint64_t flag,
2162 int (*callback)(void *), void *arg)
2163 {
2164 hermon_umap_db_priv_t *priv;
2165 hermon_umap_db_entry_t *umapdb;
2166 minor_t instance;
2167 uint64_t value;
2168 int status;
2169
2170 instance = HERMON_DEV_INSTANCE(dev);
2171 if (instance == (minor_t)-1) {
2172 return (DDI_FAILURE);
2173 }
2174
2175 if (flag != HERMON_ONCLOSE_FLASH_INPROGRESS) {
2176 return (DDI_FAILURE);
2177 }
2178
2179 /*
2180 * Grab the lock for the "userland resources database" and find
2181 * the entry corresponding to this minor number. Once it's found,
2182 * allocate (if necessary) and add an entry (in the "hdb_priv"
2183 * field) to indicate that further processing may be needed during
2184 * Hermon's close() handling.
2185 */
2186 mutex_enter(&hermon_userland_rsrc_db.hdl_umapdb_lock);
2187 status = hermon_umap_db_find_nolock(instance, dev,
2188 MLNX_UMAP_PID_RSRC, &value, 0, &umapdb);
2189 if (status != DDI_SUCCESS) {
2190 mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock);
2191 return (DDI_FAILURE);
2192 }
2193
2194 priv = (hermon_umap_db_priv_t *)umapdb->hdbe_common.hdb_priv;
2195 if (priv == NULL) {
2196 priv = (hermon_umap_db_priv_t *)kmem_zalloc(
2197 sizeof (hermon_umap_db_priv_t), KM_NOSLEEP);
2198 if (priv == NULL) {
2199 mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock);
2200 return (DDI_FAILURE);
2201 }
2202 }
2203
2204 /*
2205 * Save away the callback and argument to be used during Hermon's
2206 * close() processing.
2207 */
2208 priv->hdp_cb = callback;
2209 priv->hdp_arg = arg;
2210
2211 umapdb->hdbe_common.hdb_priv = (void *)priv;
2212 mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock);
2213
2214 return (DDI_SUCCESS);
2215 }
2216
2217
2218 /*
2219 * hermon_umap_db_clear_onclose_cb()
2220 * Context: Can be called from user or kernel context.
2221 */
2222 int
2223 hermon_umap_db_clear_onclose_cb(dev_t dev, uint64_t flag)
2224 {
2225 hermon_umap_db_priv_t *priv;
2226 hermon_umap_db_entry_t *umapdb;
2227 minor_t instance;
2228 uint64_t value;
2229 int status;
2230
2231 instance = HERMON_DEV_INSTANCE(dev);
2232 if (instance == (minor_t)-1) {
2233 return (DDI_FAILURE);
2234 }
2235
2236 if (flag != HERMON_ONCLOSE_FLASH_INPROGRESS) {
2237 return (DDI_FAILURE);
2238 }
2239
2240 /*
2241 * Grab the lock for the "userland resources database" and find
2242 * the entry corresponding to this minor number. Once it's found,
2243 * remove the entry (in the "hdb_priv" field) that indicated the
2244 * need for further processing during Hermon's close(). Free the
2245 * entry, if appropriate.
2246 */
2247 mutex_enter(&hermon_userland_rsrc_db.hdl_umapdb_lock);
2248 status = hermon_umap_db_find_nolock(instance, dev,
2249 MLNX_UMAP_PID_RSRC, &value, 0, &umapdb);
2250 if (status != DDI_SUCCESS) {
2251 mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock);
2252 return (DDI_FAILURE);
2253 }
2254
2255 priv = (hermon_umap_db_priv_t *)umapdb->hdbe_common.hdb_priv;
2256 if (priv != NULL) {
2257 kmem_free(priv, sizeof (hermon_umap_db_priv_t));
2258 priv = NULL;
2259 }
2260
2261 umapdb->hdbe_common.hdb_priv = (void *)priv;
2262 mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock);
2263 return (DDI_SUCCESS);
2264 }
2265
2266
2267 /*
2268 * hermon_umap_db_clear_onclose_cb()
2269 * Context: Can be called from user or kernel context.
2270 */
2271 int
2272 hermon_umap_db_handle_onclose_cb(hermon_umap_db_priv_t *priv)
2273 {
2274 int (*callback)(void *);
2275
2276 ASSERT(MUTEX_HELD(&hermon_userland_rsrc_db.hdl_umapdb_lock));
2277
2278 /*
2279 * Call the callback.
2280 * Note: Currently there is only one callback (in "hdp_cb"), but
2281 * in the future there may be more, depending on what other types
2282 * of interaction there are between userland processes and the
2283 * driver.
2284 */
2285 callback = priv->hdp_cb;
2286 return (callback(priv->hdp_arg));
2287 }