473
474 /*
475 * The bottom bits of "offset" are undefined (number depends on
476 * system PAGESIZE). Shifting these off leaves us with a "key".
477 * The "key" is actually a combination of both a real key value
478 * (for the purpose of database lookup) and a "type" value. Although
479 * we are not going to do any database lookup per se, we do want
480 * to extract the "key" and the "type" (to enable faster lookup of
481 * the appropriate CQ or QP handle).
482 */
483 key = off >> PAGESHIFT;
484 type = key & MLNX_UMAP_RSRC_TYPE_MASK;
485 key = key >> MLNX_UMAP_RSRC_TYPE_SHIFT;
486
487 /*
488 * Allocate an entry to track the mapping and unmapping (specifically,
489 * partial unmapping) of this resource.
490 */
491 dvm_track = (hermon_devmap_track_t *)kmem_zalloc(
492 sizeof (hermon_devmap_track_t), KM_SLEEP);
493 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
494 dvm_track->hdt_offset = off;
495 dvm_track->hdt_state = state;
496 dvm_track->hdt_refcnt = 1;
497 mutex_init(&dvm_track->hdt_lock, NULL, MUTEX_DRIVER,
498 DDI_INTR_PRI(state->hs_intrmsi_pri));
499
500 /*
501 * Depending of the type of resource that has been mapped out, we
502 * need to update the QP or CQ handle to reflect that it has, in
503 * fact, been mapped. This allows the driver code which frees a QP
504 * or a CQ to know whether it is appropriate to do a
505 * devmap_devmem_remap() to invalidate the userland mapping for the
506 * corresponding queue's memory.
507 */
508 if (type == MLNX_UMAP_CQMEM_RSRC) {
509
510 /* Use "key" (CQ number) to do fast lookup of CQ handle */
511 cq = hermon_cqhdl_from_cqnum(state, key);
512
513 /*
589
590 /*
591 * hermon_devmap_umem_dup()
592 * Context: Can be called from kernel context.
593 */
594 /* ARGSUSED */
595 static int
596 hermon_devmap_umem_dup(devmap_cookie_t dhp, void *pvtp, devmap_cookie_t new_dhp,
597 void **new_pvtp)
598 {
599 hermon_state_t *state;
600 hermon_devmap_track_t *dvm_track, *new_dvm_track;
601 uint_t maxprot;
602 int status;
603
604 /*
605 * Extract the Hermon softstate pointer from "Hermon devmap tracking
606 * structure" (in "pvtp").
607 */
608 dvm_track = (hermon_devmap_track_t *)pvtp;
609 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
610 state = dvm_track->hdt_state;
611
612 /*
613 * Since this devmap_dup() entry point is generally called
614 * when a process does fork(2), it is incumbent upon the driver
615 * to insure that the child does not inherit a valid copy of
616 * the parent's QP or CQ resource. This is accomplished by using
617 * devmap_devmem_remap() to invalidate the child's mapping to the
618 * kernel memory.
619 */
620 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
621 status = devmap_devmem_remap(new_dhp, state->hs_dip, 0, 0,
622 dvm_track->hdt_size, maxprot, DEVMAP_MAPPING_INVALID, NULL);
623 if (status != DDI_SUCCESS) {
624 HERMON_WARNING(state, "failed in hermon_devmap_umem_dup()");
625 return (status);
626 }
627
628 /*
629 * Allocate a new entry to track the subsequent unmapping
630 * (specifically, all partial unmappings) of the child's newly
631 * invalidated resource. Note: Setting the "hdt_size" field to
632 * zero here is an indication to the devmap_unmap() entry point
633 * that this mapping is invalid, and that its subsequent unmapping
634 * should not affect any of the parent's CQ or QP resources.
635 */
636 new_dvm_track = (hermon_devmap_track_t *)kmem_zalloc(
637 sizeof (hermon_devmap_track_t), KM_SLEEP);
638 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*new_dvm_track))
639 new_dvm_track->hdt_offset = 0;
640 new_dvm_track->hdt_state = state;
641 new_dvm_track->hdt_refcnt = 1;
642 new_dvm_track->hdt_size = 0;
643 mutex_init(&new_dvm_track->hdt_lock, NULL, MUTEX_DRIVER,
644 DDI_INTR_PRI(state->hs_intrmsi_pri));
645 *new_pvtp = new_dvm_track;
646
647 return (DDI_SUCCESS);
648 }
649
650
651 /*
652 * hermon_devmap_umem_unmap()
653 * Context: Can be called from kernel context.
654 */
655 /* ARGSUSED */
656 static void
657 hermon_devmap_umem_unmap(devmap_cookie_t dhp, void *pvtp, offset_t off,
658 size_t len, devmap_cookie_t new_dhp1, void **pvtp1,
659 devmap_cookie_t new_dhp2, void **pvtp2)
660 {
661 hermon_state_t *state;
662 hermon_rsrc_t *rsrcp;
663 hermon_devmap_track_t *dvm_track;
664 hermon_cqhdl_t cq;
665 hermon_qphdl_t qp;
666 hermon_srqhdl_t srq;
667 uint64_t key, value;
668 uint_t type;
669 uint_t size;
670 int status;
671
672 /*
673 * Extract the Hermon softstate pointer from "Hermon devmap tracking
674 * structure" (in "pvtp").
675 */
676 dvm_track = (hermon_devmap_track_t *)pvtp;
677 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
678 state = dvm_track->hdt_state;
679
680 /*
681 * Extract the "offset" from the "Hermon devmap tracking structure".
682 * Note: The input argument "off" is ignored here because the
683 * Hermon mapping interfaces define a very specific meaning to
684 * each "logical offset". Also extract the "key" and "type" encoded
685 * in the logical offset.
686 */
687 key = dvm_track->hdt_offset >> PAGESHIFT;
688 type = key & MLNX_UMAP_RSRC_TYPE_MASK;
689 key = key >> MLNX_UMAP_RSRC_TYPE_SHIFT;
690
691 /*
692 * Extract the "size" of the mapping. If this size is determined
693 * to be zero, then it is an indication of a previously invalidated
694 * mapping, and no CQ or QP resources should be affected.
695 */
696 size = dvm_track->hdt_size;
697
878
879 /*
880 * The bottom bits of "offset" are undefined (number depends on
881 * system PAGESIZE). Shifting these off leaves us with a "key".
882 * The "key" is actually a combination of both a real key value
883 * (for the purpose of database lookup) and a "type" value. Although
884 * we are not going to do any database lookup per se, we do want
885 * to extract the "key" and the "type" (to enable faster lookup of
886 * the appropriate CQ or QP handle).
887 */
888 key = off >> PAGESHIFT;
889 type = key & MLNX_UMAP_RSRC_TYPE_MASK;
890 key = key >> MLNX_UMAP_RSRC_TYPE_SHIFT;
891
892 /*
893 * Allocate an entry to track the mapping and unmapping (specifically,
894 * partial unmapping) of this resource.
895 */
896 dvm_track = (hermon_devmap_track_t *)kmem_zalloc(
897 sizeof (hermon_devmap_track_t), KM_SLEEP);
898 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
899 dvm_track->hdt_offset = off;
900 dvm_track->hdt_state = state;
901 dvm_track->hdt_refcnt = 1;
902 mutex_init(&dvm_track->hdt_lock, NULL, MUTEX_DRIVER,
903 DDI_INTR_PRI(state->hs_intrmsi_pri));
904
905 /*
906 * Depending of the type of resource that has been mapped out, we
907 * need to update the QP or CQ handle to reflect that it has, in
908 * fact, been mapped. This allows the driver code which frees a QP
909 * or a CQ to know whether it is appropriate to do a
910 * devmap_devmem_remap() to invalidate the userland mapping for the
911 * corresponding queue's memory.
912 */
913 if (type == MLNX_UMAP_CQMEM_RSRC) {
914
915 /* Use "key" (CQ number) to do fast lookup of CQ handle */
916 cq = hermon_cqhdl_from_cqnum(state, key);
917
918 /*
988
989 /*
990 * hermon_devmap_dbrecmem_dup()
991 * Context: Can be called from kernel context.
992 */
993 /* ARGSUSED */
994 static int
995 hermon_devmap_dbrecmem_dup(devmap_cookie_t dhp, void *pvtp,
996 devmap_cookie_t new_dhp, void **new_pvtp)
997 {
998 hermon_state_t *state;
999 hermon_devmap_track_t *dvm_track, *new_dvm_track;
1000 uint_t maxprot;
1001 int status;
1002
1003 /*
1004 * Extract the Hermon softstate pointer from "Hermon devmap tracking
1005 * structure" (in "pvtp").
1006 */
1007 dvm_track = (hermon_devmap_track_t *)pvtp;
1008 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
1009 state = dvm_track->hdt_state;
1010
1011 /*
1012 * Since this devmap_dup() entry point is generally called
1013 * when a process does fork(2), it is incumbent upon the driver
1014 * to insure that the child does not inherit a valid copy of
1015 * the parent's QP or CQ resource. This is accomplished by using
1016 * devmap_devmem_remap() to invalidate the child's mapping to the
1017 * kernel memory.
1018 */
1019 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
1020 status = devmap_devmem_remap(new_dhp, state->hs_dip, 0, 0,
1021 dvm_track->hdt_size, maxprot, DEVMAP_MAPPING_INVALID, NULL);
1022 if (status != DDI_SUCCESS) {
1023 HERMON_WARNING(state, "failed in hermon_devmap_dbrecmem_dup()");
1024 return (status);
1025 }
1026
1027 /*
1028 * Allocate a new entry to track the subsequent unmapping
1029 * (specifically, all partial unmappings) of the child's newly
1030 * invalidated resource. Note: Setting the "hdt_size" field to
1031 * zero here is an indication to the devmap_unmap() entry point
1032 * that this mapping is invalid, and that its subsequent unmapping
1033 * should not affect any of the parent's CQ or QP resources.
1034 */
1035 new_dvm_track = (hermon_devmap_track_t *)kmem_zalloc(
1036 sizeof (hermon_devmap_track_t), KM_SLEEP);
1037 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*new_dvm_track))
1038 new_dvm_track->hdt_offset = 0;
1039 new_dvm_track->hdt_state = state;
1040 new_dvm_track->hdt_refcnt = 1;
1041 new_dvm_track->hdt_size = 0;
1042 mutex_init(&new_dvm_track->hdt_lock, NULL, MUTEX_DRIVER,
1043 DDI_INTR_PRI(state->hs_intrmsi_pri));
1044 *new_pvtp = new_dvm_track;
1045
1046 return (DDI_SUCCESS);
1047 }
1048
1049
1050 /*
1051 * hermon_devmap_dbrecmem_unmap()
1052 * Context: Can be called from kernel context.
1053 */
1054 /* ARGSUSED */
1055 static void
1056 hermon_devmap_dbrecmem_unmap(devmap_cookie_t dhp, void *pvtp, offset_t off,
1057 size_t len, devmap_cookie_t new_dhp1, void **pvtp1,
1058 devmap_cookie_t new_dhp2, void **pvtp2)
1059 {
1060 hermon_state_t *state;
1061 hermon_rsrc_t *rsrcp;
1062 hermon_devmap_track_t *dvm_track;
1063 hermon_cqhdl_t cq;
1064 hermon_qphdl_t qp;
1065 hermon_srqhdl_t srq;
1066 uint64_t key, value;
1067 uint_t type;
1068 uint_t size;
1069 int status;
1070
1071 /*
1072 * Extract the Hermon softstate pointer from "Hermon devmap tracking
1073 * structure" (in "pvtp").
1074 */
1075 dvm_track = (hermon_devmap_track_t *)pvtp;
1076 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
1077 state = dvm_track->hdt_state;
1078
1079 /*
1080 * Extract the "offset" from the "Hermon devmap tracking structure".
1081 * Note: The input argument "off" is ignored here because the
1082 * Hermon mapping interfaces define a very specific meaning to
1083 * each "logical offset". Also extract the "key" and "type" encoded
1084 * in the logical offset.
1085 */
1086 key = dvm_track->hdt_offset >> PAGESHIFT;
1087 type = key & MLNX_UMAP_RSRC_TYPE_MASK;
1088 key = key >> MLNX_UMAP_RSRC_TYPE_SHIFT;
1089
1090 /*
1091 * Extract the "size" of the mapping. If this size is determined
1092 * to be zero, then it is an indication of a previously invalidated
1093 * mapping, and no CQ or QP resources should be affected.
1094 */
1095 size = dvm_track->hdt_size;
1096
1249 hermon_devmap_track_t *dvm_track;
1250 minor_t instance;
1251
1252 /* Get Hermon softstate structure from instance */
1253 instance = HERMON_DEV_INSTANCE(dev);
1254 state = ddi_get_soft_state(hermon_statep, instance);
1255 if (state == NULL) {
1256 return (ENXIO);
1257 }
1258
1259 /*
1260 * Allocate an entry to track the mapping and unmapping of this
1261 * resource. Note: We don't need to initialize the "refcnt" or
1262 * "offset" fields here, nor do we need to initialize the mutex
1263 * used with the "refcnt". Since UAR pages are single pages, they
1264 * are not subject to "partial" unmappings. This makes these other
1265 * fields unnecessary.
1266 */
1267 dvm_track = (hermon_devmap_track_t *)kmem_zalloc(
1268 sizeof (hermon_devmap_track_t), KM_SLEEP);
1269 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
1270 dvm_track->hdt_state = state;
1271 dvm_track->hdt_size = (uint_t)PAGESIZE;
1272
1273 /*
1274 * Pass the private "Hermon devmap tracking structure" back. This
1275 * pointer will be returned in a subsequent "unmap" callback.
1276 */
1277 *pvtp = dvm_track;
1278
1279 return (DDI_SUCCESS);
1280 }
1281
1282
1283 /*
1284 * hermon_devmap_devmem_dup()
1285 * Context: Can be called from kernel context.
1286 */
1287 /* ARGSUSED */
1288 static int
1289 hermon_devmap_devmem_dup(devmap_cookie_t dhp, void *pvtp,
1290 devmap_cookie_t new_dhp, void **new_pvtp)
1291 {
1292 hermon_state_t *state;
1293 hermon_devmap_track_t *dvm_track;
1294 uint_t maxprot;
1295 int status;
1296
1297 /*
1298 * Extract the Hermon softstate pointer from "Hermon devmap tracking
1299 * structure" (in "pvtp"). Note: If the tracking structure is NULL
1300 * here, it means that the mapping corresponds to an invalid mapping.
1301 * In this case, it can be safely ignored ("new_pvtp" set to NULL).
1302 */
1303 dvm_track = (hermon_devmap_track_t *)pvtp;
1304 if (dvm_track == NULL) {
1305 *new_pvtp = NULL;
1306 return (DDI_SUCCESS);
1307 }
1308
1309 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
1310 state = dvm_track->hdt_state;
1311
1312 /*
1313 * Since this devmap_dup() entry point is generally called
1314 * when a process does fork(2), it is incumbent upon the driver
1315 * to insure that the child does not inherit a valid copy of
1316 * the parent's resource. This is accomplished by using
1317 * devmap_devmem_remap() to invalidate the child's mapping to the
1318 * kernel memory.
1319 */
1320 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
1321 status = devmap_devmem_remap(new_dhp, state->hs_dip, 0, 0,
1322 dvm_track->hdt_size, maxprot, DEVMAP_MAPPING_INVALID, NULL);
1323 if (status != DDI_SUCCESS) {
1324 HERMON_WARNING(state, "failed in hermon_devmap_devmem_dup()");
1325 return (status);
1326 }
1327
1328 /*
1329 * Since the region is invalid, there is no need for us to
1341 /*
1342 * hermon_devmap_devmem_unmap()
1343 * Context: Can be called from kernel context.
1344 */
1345 /* ARGSUSED */
1346 static void
1347 hermon_devmap_devmem_unmap(devmap_cookie_t dhp, void *pvtp, offset_t off,
1348 size_t len, devmap_cookie_t new_dhp1, void **pvtp1,
1349 devmap_cookie_t new_dhp2, void **pvtp2)
1350 {
1351 hermon_devmap_track_t *dvm_track;
1352
1353 /*
1354 * Free up the "Hermon devmap tracking structure" (in "pvtp").
1355 * There cannot be "partial" unmappings here because all UAR pages
1356 * are single pages. Note: If the tracking structure is NULL here,
1357 * it means that the mapping corresponds to an invalid mapping. In
1358 * this case, it can be safely ignored.
1359 */
1360 dvm_track = (hermon_devmap_track_t *)pvtp;
1361 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
1362 if (dvm_track == NULL) {
1363 return;
1364 }
1365
1366 kmem_free(dvm_track, sizeof (hermon_devmap_track_t));
1367 }
1368
1369
1370 /*
1371 * hermon_umap_ci_data_in()
1372 * Context: Can be called from user or kernel context.
1373 */
1374 /* ARGSUSED */
1375 ibt_status_t
1376 hermon_umap_ci_data_in(hermon_state_t *state, ibt_ci_data_flags_t flags,
1377 ibt_object_type_t object, void *hdl, void *data_p, size_t data_sz)
1378 {
1379 int status;
1380
1381 /*
1423 * hermon_umap_mr_data_in()
1424 * Context: Can be called from user or kernel context.
1425 */
1426 static ibt_status_t
1427 hermon_umap_mr_data_in(hermon_mrhdl_t mr, ibt_mr_data_in_t *data,
1428 size_t data_sz)
1429 {
1430 if (data->mr_rev != IBT_MR_DATA_IN_IF_VERSION) {
1431 return (IBT_NOT_SUPPORTED);
1432 }
1433
1434 /* Check for valid MR handle pointer */
1435 if (mr == NULL) {
1436 return (IBT_MR_HDL_INVALID);
1437 }
1438
1439 /* Check for valid MR input structure size */
1440 if (data_sz < sizeof (ibt_mr_data_in_t)) {
1441 return (IBT_INSUFF_RESOURCE);
1442 }
1443 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*data))
1444
1445 /*
1446 * Ensure that the MR corresponds to userland memory and that it is
1447 * a currently valid memory region as well.
1448 */
1449 mutex_enter(&mr->mr_lock);
1450 if ((mr->mr_is_umem == 0) || (mr->mr_umemcookie == NULL)) {
1451 mutex_exit(&mr->mr_lock);
1452 return (IBT_MR_HDL_INVALID);
1453 }
1454
1455 /*
1456 * If it has passed all the above checks, then extract the callback
1457 * function and argument from the input structure. Copy them into
1458 * the MR handle. This function will be called only if the memory
1459 * corresponding to the MR handle gets a umem_lockmemory() callback.
1460 */
1461 mr->mr_umem_cbfunc = data->mr_func;
1462 mr->mr_umem_cbarg1 = data->mr_arg1;
1463 mr->mr_umem_cbarg2 = data->mr_arg2;
1540 }
1541
1542
1543 /*
1544 * hermon_umap_cq_data_out()
1545 * Context: Can be called from user or kernel context.
1546 */
1547 static ibt_status_t
1548 hermon_umap_cq_data_out(hermon_cqhdl_t cq, mlnx_umap_cq_data_out_t *data,
1549 size_t data_sz)
1550 {
1551 /* Check for valid CQ handle pointer */
1552 if (cq == NULL) {
1553 return (IBT_CQ_HDL_INVALID);
1554 }
1555
1556 /* Check for valid CQ mapping structure size */
1557 if (data_sz < sizeof (mlnx_umap_cq_data_out_t)) {
1558 return (IBT_INSUFF_RESOURCE);
1559 }
1560 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*data))
1561
1562 /* deal with cq_alloc() verses cq_resize() */
1563 if (cq->cq_resize_hdl) {
1564 data->mcq_maplen = cq->cq_resize_hdl->cq_cqinfo.qa_size;
1565 data->mcq_numcqe = cq->cq_resize_hdl->cq_bufsz;
1566 } else {
1567 data->mcq_maplen = cq->cq_cqinfo.qa_size;
1568 data->mcq_numcqe = cq->cq_bufsz;
1569 }
1570
1571 /*
1572 * If it has passed all the above checks, then fill in all the useful
1573 * mapping information (including the mapping offset that will be
1574 * passed back to the devmap() interface during a subsequent mmap()
1575 * call.
1576 *
1577 * The "offset" for CQ mmap()'s looks like this:
1578 * +----------------------------------------+--------+--------------+
1579 * | CQ Number | 0x33 | Reserved (0) |
1580 * +----------------------------------------+--------+--------------+
1603 }
1604
1605
1606 /*
1607 * hermon_umap_qp_data_out()
1608 * Context: Can be called from user or kernel context.
1609 */
1610 static ibt_status_t
1611 hermon_umap_qp_data_out(hermon_qphdl_t qp, mlnx_umap_qp_data_out_t *data,
1612 size_t data_sz)
1613 {
1614 /* Check for valid QP handle pointer */
1615 if (qp == NULL) {
1616 return (IBT_QP_HDL_INVALID);
1617 }
1618
1619 /* Check for valid QP mapping structure size */
1620 if (data_sz < sizeof (mlnx_umap_qp_data_out_t)) {
1621 return (IBT_INSUFF_RESOURCE);
1622 }
1623 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*data))
1624
1625 /*
1626 * If it has passed all the checks, then fill in all the useful
1627 * mapping information (including the mapping offset that will be
1628 * passed back to the devmap() interface during a subsequent mmap()
1629 * call.
1630 *
1631 * The "offset" for QP mmap()'s looks like this:
1632 * +----------------------------------------+--------+--------------+
1633 * | QP Number | 0x44 | Reserved (0) |
1634 * +----------------------------------------+--------+--------------+
1635 * (64 - 8 - PAGESHIFT) bits 8 bits PAGESHIFT bits
1636 *
1637 * This returns information about the mapping offset, the length of
1638 * the QP memory, and the QP number (for use in later send and recv
1639 * doorbells). It also returns the following information for both
1640 * the receive work queue and the send work queue, respectively: the
1641 * offset (from the base mapped address) of the start of the given
1642 * work queue, the 64-bit IB virtual address that corresponds to
1643 * the base mapped address (needed for posting WQEs though the
1694 }
1695
1696
1697 /*
1698 * hermon_umap_srq_data_out()
1699 * Context: Can be called from user or kernel context.
1700 */
1701 static ibt_status_t
1702 hermon_umap_srq_data_out(hermon_srqhdl_t srq, mlnx_umap_srq_data_out_t *data,
1703 size_t data_sz)
1704 {
1705 /* Check for valid SRQ handle pointer */
1706 if (srq == NULL) {
1707 return (IBT_SRQ_HDL_INVALID);
1708 }
1709
1710 /* Check for valid SRQ mapping structure size */
1711 if (data_sz < sizeof (mlnx_umap_srq_data_out_t)) {
1712 return (IBT_INSUFF_RESOURCE);
1713 }
1714 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*data))
1715
1716 /*
1717 * If it has passed all the checks, then fill in all the useful
1718 * mapping information (including the mapping offset that will be
1719 * passed back to the devmap() interface during a subsequent mmap()
1720 * call.
1721 *
1722 * The "offset" for SRQ mmap()'s looks like this:
1723 * +----------------------------------------+--------+--------------+
1724 * | SRQ Number | 0x66 | Reserved (0) |
1725 * +----------------------------------------+--------+--------------+
1726 * (64 - 8 - PAGESHIFT) bits 8 bits PAGESHIFT bits
1727 *
1728 * This returns information about the mapping offset, the length of the
1729 * SRQ memory, and the SRQ number (for use in later send and recv
1730 * doorbells). It also returns the following information for the
1731 * shared receive queue: the offset (from the base mapped address) of
1732 * the start of the given work queue, the 64-bit IB virtual address
1733 * that corresponds to the base mapped address (needed for posting WQEs
1734 * though the QP doorbells), the number of WQEs the given work queue
1755 }
1756
1757
1758 /*
1759 * hermon_umap_pd_data_out()
1760 * Context: Can be called from user or kernel context.
1761 */
1762 static ibt_status_t
1763 hermon_umap_pd_data_out(hermon_pdhdl_t pd, mlnx_umap_pd_data_out_t *data,
1764 size_t data_sz)
1765 {
1766 /* Check for valid PD handle pointer */
1767 if (pd == NULL) {
1768 return (IBT_PD_HDL_INVALID);
1769 }
1770
1771 /* Check for valid PD mapping structure size */
1772 if (data_sz < sizeof (mlnx_umap_pd_data_out_t)) {
1773 return (IBT_INSUFF_RESOURCE);
1774 }
1775 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*data))
1776
1777 /*
1778 * If it has passed all the checks, then fill the PD table index
1779 * (the PD table allocated index for the PD pd_pdnum).
1780 */
1781 data->mpd_rev = MLNX_UMAP_IF_VERSION;
1782 data->mpd_pdnum = pd->pd_pdnum;
1783
1784 return (DDI_SUCCESS);
1785 }
1786
1787
1788 /*
1789 * hermon_umap_db_init()
1790 * Context: Only called from attach() path context
1791 */
1792 void
1793 hermon_umap_db_init(void)
1794 {
1795 /*
1826
1827 /* Destroy the lock for the "userland resources database" */
1828 mutex_destroy(&hermon_userland_rsrc_db.hdl_umapdb_lock);
1829 }
1830
1831
1832 /*
1833 * hermon_umap_db_alloc()
1834 * Context: Can be called from user or kernel context.
1835 */
1836 hermon_umap_db_entry_t *
1837 hermon_umap_db_alloc(uint_t instance, uint64_t key, uint_t type, uint64_t value)
1838 {
1839 hermon_umap_db_entry_t *umapdb;
1840
1841 /* Allocate an entry to add to the "userland resources database" */
1842 umapdb = kmem_zalloc(sizeof (hermon_umap_db_entry_t), KM_NOSLEEP);
1843 if (umapdb == NULL) {
1844 return (NULL);
1845 }
1846 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*umapdb))
1847
1848 /* Fill in the fields in the database entry */
1849 umapdb->hdbe_common.hdb_instance = instance;
1850 umapdb->hdbe_common.hdb_type = type;
1851 umapdb->hdbe_common.hdb_key = key;
1852 umapdb->hdbe_common.hdb_value = value;
1853
1854 return (umapdb);
1855 }
1856
1857
1858 /*
1859 * hermon_umap_db_free()
1860 * Context: Can be called from user or kernel context.
1861 */
1862 void
1863 hermon_umap_db_free(hermon_umap_db_entry_t *umapdb)
1864 {
1865 /* Free the database entry */
1866 kmem_free(umapdb, sizeof (hermon_umap_db_entry_t));
1875 hermon_umap_db_add(hermon_umap_db_entry_t *umapdb)
1876 {
1877 mutex_enter(&hermon_userland_rsrc_db.hdl_umapdb_lock);
1878 hermon_umap_db_add_nolock(umapdb);
1879 mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock);
1880 }
1881
1882
1883 /*
1884 * hermon_umap_db_add_nolock()
1885 * Context: Can be called from user or kernel context.
1886 */
1887 void
1888 hermon_umap_db_add_nolock(hermon_umap_db_entry_t *umapdb)
1889 {
1890 hermon_umap_db_query_t query;
1891 avl_index_t where;
1892
1893 ASSERT(MUTEX_HELD(&hermon_userland_rsrc_db.hdl_umapdb_lock));
1894
1895 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*umapdb))
1896
1897 /*
1898 * Copy the common portion of the "to-be-added" database entry
1899 * into the "hermon_umap_db_query_t" structure. We use this structure
1900 * (with no flags set) to find the appropriate location in the
1901 * "userland resources database" for the new entry to be added.
1902 *
1903 * Note: we expect that this entry should not be found in the
1904 * database (unless something bad has happened).
1905 */
1906 query.hqdb_common = umapdb->hdbe_common;
1907 query.hqdb_flags = 0;
1908 (void) avl_find(&hermon_userland_rsrc_db.hdl_umapdb_avl, &query,
1909 &where);
1910
1911 /*
1912 * Now, using the "where" field from the avl_find() operation
1913 * above, we will insert the new database entry ("umapdb").
1914 */
1915 avl_insert(&hermon_userland_rsrc_db.hdl_umapdb_avl, umapdb,
1916 where);
1952
1953 /*
1954 * Fill in key, type, instance, and flags values of the
1955 * hermon_umap_db_query_t in preparation for the database
1956 * lookup.
1957 */
1958 query.hqdb_flags = flags;
1959 query.hqdb_common.hdb_key = key;
1960 query.hqdb_common.hdb_type = type;
1961 query.hqdb_common.hdb_instance = instance;
1962
1963 /*
1964 * Perform the database query. If no entry is found, then
1965 * return failure, else continue.
1966 */
1967 entry = (hermon_umap_db_entry_t *)avl_find(
1968 &hermon_userland_rsrc_db.hdl_umapdb_avl, &query, &where);
1969 if (entry == NULL) {
1970 return (DDI_FAILURE);
1971 }
1972 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*entry))
1973
1974 /*
1975 * If the flags argument specifies that the entry should
1976 * be removed if found, then call avl_remove() to remove
1977 * the entry from the database.
1978 */
1979 if (flags & HERMON_UMAP_DB_REMOVE) {
1980
1981 avl_remove(&hermon_userland_rsrc_db.hdl_umapdb_avl, entry);
1982
1983 /*
1984 * The database entry is returned with the expectation
1985 * that the caller will use hermon_umap_db_free() to
1986 * free the entry's memory. ASSERT that this is non-NULL.
1987 * NULL pointer should never be passed for the
1988 * HERMON_UMAP_DB_REMOVE case.
1989 */
1990 ASSERT(umapdb != NULL);
1991 }
1992
2016 hermon_umap_db_entry_t *umapdb;
2017 hermon_state_t *state;
2018 hermon_rsrc_t *rsrcp;
2019 hermon_mrhdl_t mr;
2020 uint64_t value;
2021 uint_t instance;
2022 int status;
2023 void (*mr_callback)(void *, void *);
2024 void *mr_cbarg1, *mr_cbarg2;
2025
2026 /*
2027 * If this was userland memory, then we need to remove its entry
2028 * from the "userland resources database". Note: We use the
2029 * HERMON_UMAP_DB_IGNORE_INSTANCE flag here because we don't know
2030 * which instance was used when the entry was added (but we want
2031 * to know after the entry is found using the other search criteria).
2032 */
2033 status = hermon_umap_db_find(0, (uint64_t)(uintptr_t)umem_cookie,
2034 MLNX_UMAP_MRMEM_RSRC, &value, (HERMON_UMAP_DB_REMOVE |
2035 HERMON_UMAP_DB_IGNORE_INSTANCE), &umapdb);
2036 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*umapdb))
2037 if (status == DDI_SUCCESS) {
2038 instance = umapdb->hdbe_common.hdb_instance;
2039 state = ddi_get_soft_state(hermon_statep, instance);
2040 if (state == NULL) {
2041 cmn_err(CE_WARN, "Unable to match Hermon instance\n");
2042 return;
2043 }
2044
2045 /* Free the database entry */
2046 hermon_umap_db_free(umapdb);
2047
2048 /* Use "value" to convert to an MR handle */
2049 rsrcp = (hermon_rsrc_t *)(uintptr_t)value;
2050 mr = (hermon_mrhdl_t)rsrcp->hr_addr;
2051
2052 /*
2053 * If a callback has been provided, call it first. This
2054 * callback is expected to do any cleanup necessary to
2055 * guarantee that the subsequent MR deregister (below)
2056 * will succeed. Specifically, this means freeing up memory
2075 status = hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
2076 HERMON_SLEEP);
2077 if (status != DDI_SUCCESS) {
2078 HERMON_WARNING(state, "Unexpected failure in "
2079 "deregister from callback\n");
2080 }
2081 }
2082 }
2083
2084
2085 /*
2086 * hermon_umap_db_compare()
2087 * Context: Can be called from user or kernel context.
2088 */
2089 static int
2090 hermon_umap_db_compare(const void *q, const void *e)
2091 {
2092 hermon_umap_db_common_t *entry_common, *query_common;
2093 uint_t query_flags;
2094
2095 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*((hermon_umap_db_query_t *)q)))
2096
2097 entry_common = &((hermon_umap_db_entry_t *)e)->hdbe_common;
2098 query_common = &((hermon_umap_db_query_t *)q)->hqdb_common;
2099 query_flags = ((hermon_umap_db_query_t *)q)->hqdb_flags;
2100
2101 /*
2102 * The first comparison is done on the "key" value in "query"
2103 * and "entry". If they are not equal, then the appropriate
2104 * search direction is returned. Else, we continue by
2105 * comparing "type".
2106 */
2107 if (query_common->hdb_key < entry_common->hdb_key) {
2108 return (-1);
2109 } else if (query_common->hdb_key > entry_common->hdb_key) {
2110 return (+1);
2111 }
2112
2113 /*
2114 * If the search reaches this point, then "query" and "entry"
2115 * have equal key values. So we continue be comparing their
2116 * "type" values. Again, if they are not equal, then the
|
473
474 /*
475 * The bottom bits of "offset" are undefined (number depends on
476 * system PAGESIZE). Shifting these off leaves us with a "key".
477 * The "key" is actually a combination of both a real key value
478 * (for the purpose of database lookup) and a "type" value. Although
479 * we are not going to do any database lookup per se, we do want
480 * to extract the "key" and the "type" (to enable faster lookup of
481 * the appropriate CQ or QP handle).
482 */
483 key = off >> PAGESHIFT;
484 type = key & MLNX_UMAP_RSRC_TYPE_MASK;
485 key = key >> MLNX_UMAP_RSRC_TYPE_SHIFT;
486
487 /*
488 * Allocate an entry to track the mapping and unmapping (specifically,
489 * partial unmapping) of this resource.
490 */
491 dvm_track = (hermon_devmap_track_t *)kmem_zalloc(
492 sizeof (hermon_devmap_track_t), KM_SLEEP);
493 dvm_track->hdt_offset = off;
494 dvm_track->hdt_state = state;
495 dvm_track->hdt_refcnt = 1;
496 mutex_init(&dvm_track->hdt_lock, NULL, MUTEX_DRIVER,
497 DDI_INTR_PRI(state->hs_intrmsi_pri));
498
499 /*
500 * Depending of the type of resource that has been mapped out, we
501 * need to update the QP or CQ handle to reflect that it has, in
502 * fact, been mapped. This allows the driver code which frees a QP
503 * or a CQ to know whether it is appropriate to do a
504 * devmap_devmem_remap() to invalidate the userland mapping for the
505 * corresponding queue's memory.
506 */
507 if (type == MLNX_UMAP_CQMEM_RSRC) {
508
509 /* Use "key" (CQ number) to do fast lookup of CQ handle */
510 cq = hermon_cqhdl_from_cqnum(state, key);
511
512 /*
588
589 /*
590 * hermon_devmap_umem_dup()
591 * Context: Can be called from kernel context.
592 */
593 /* ARGSUSED */
594 static int
595 hermon_devmap_umem_dup(devmap_cookie_t dhp, void *pvtp, devmap_cookie_t new_dhp,
596 void **new_pvtp)
597 {
598 hermon_state_t *state;
599 hermon_devmap_track_t *dvm_track, *new_dvm_track;
600 uint_t maxprot;
601 int status;
602
603 /*
604 * Extract the Hermon softstate pointer from "Hermon devmap tracking
605 * structure" (in "pvtp").
606 */
607 dvm_track = (hermon_devmap_track_t *)pvtp;
608 state = dvm_track->hdt_state;
609
610 /*
611 * Since this devmap_dup() entry point is generally called
612 * when a process does fork(2), it is incumbent upon the driver
613 * to insure that the child does not inherit a valid copy of
614 * the parent's QP or CQ resource. This is accomplished by using
615 * devmap_devmem_remap() to invalidate the child's mapping to the
616 * kernel memory.
617 */
618 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
619 status = devmap_devmem_remap(new_dhp, state->hs_dip, 0, 0,
620 dvm_track->hdt_size, maxprot, DEVMAP_MAPPING_INVALID, NULL);
621 if (status != DDI_SUCCESS) {
622 HERMON_WARNING(state, "failed in hermon_devmap_umem_dup()");
623 return (status);
624 }
625
626 /*
627 * Allocate a new entry to track the subsequent unmapping
628 * (specifically, all partial unmappings) of the child's newly
629 * invalidated resource. Note: Setting the "hdt_size" field to
630 * zero here is an indication to the devmap_unmap() entry point
631 * that this mapping is invalid, and that its subsequent unmapping
632 * should not affect any of the parent's CQ or QP resources.
633 */
634 new_dvm_track = (hermon_devmap_track_t *)kmem_zalloc(
635 sizeof (hermon_devmap_track_t), KM_SLEEP);
636 new_dvm_track->hdt_offset = 0;
637 new_dvm_track->hdt_state = state;
638 new_dvm_track->hdt_refcnt = 1;
639 new_dvm_track->hdt_size = 0;
640 mutex_init(&new_dvm_track->hdt_lock, NULL, MUTEX_DRIVER,
641 DDI_INTR_PRI(state->hs_intrmsi_pri));
642 *new_pvtp = new_dvm_track;
643
644 return (DDI_SUCCESS);
645 }
646
647
648 /*
649 * hermon_devmap_umem_unmap()
650 * Context: Can be called from kernel context.
651 */
652 /* ARGSUSED */
653 static void
654 hermon_devmap_umem_unmap(devmap_cookie_t dhp, void *pvtp, offset_t off,
655 size_t len, devmap_cookie_t new_dhp1, void **pvtp1,
656 devmap_cookie_t new_dhp2, void **pvtp2)
657 {
658 hermon_state_t *state;
659 hermon_rsrc_t *rsrcp;
660 hermon_devmap_track_t *dvm_track;
661 hermon_cqhdl_t cq;
662 hermon_qphdl_t qp;
663 hermon_srqhdl_t srq;
664 uint64_t key, value;
665 uint_t type;
666 uint_t size;
667 int status;
668
669 /*
670 * Extract the Hermon softstate pointer from "Hermon devmap tracking
671 * structure" (in "pvtp").
672 */
673 dvm_track = (hermon_devmap_track_t *)pvtp;
674 state = dvm_track->hdt_state;
675
676 /*
677 * Extract the "offset" from the "Hermon devmap tracking structure".
678 * Note: The input argument "off" is ignored here because the
679 * Hermon mapping interfaces define a very specific meaning to
680 * each "logical offset". Also extract the "key" and "type" encoded
681 * in the logical offset.
682 */
683 key = dvm_track->hdt_offset >> PAGESHIFT;
684 type = key & MLNX_UMAP_RSRC_TYPE_MASK;
685 key = key >> MLNX_UMAP_RSRC_TYPE_SHIFT;
686
687 /*
688 * Extract the "size" of the mapping. If this size is determined
689 * to be zero, then it is an indication of a previously invalidated
690 * mapping, and no CQ or QP resources should be affected.
691 */
692 size = dvm_track->hdt_size;
693
874
875 /*
876 * The bottom bits of "offset" are undefined (number depends on
877 * system PAGESIZE). Shifting these off leaves us with a "key".
878 * The "key" is actually a combination of both a real key value
879 * (for the purpose of database lookup) and a "type" value. Although
880 * we are not going to do any database lookup per se, we do want
881 * to extract the "key" and the "type" (to enable faster lookup of
882 * the appropriate CQ or QP handle).
883 */
884 key = off >> PAGESHIFT;
885 type = key & MLNX_UMAP_RSRC_TYPE_MASK;
886 key = key >> MLNX_UMAP_RSRC_TYPE_SHIFT;
887
888 /*
889 * Allocate an entry to track the mapping and unmapping (specifically,
890 * partial unmapping) of this resource.
891 */
892 dvm_track = (hermon_devmap_track_t *)kmem_zalloc(
893 sizeof (hermon_devmap_track_t), KM_SLEEP);
894 dvm_track->hdt_offset = off;
895 dvm_track->hdt_state = state;
896 dvm_track->hdt_refcnt = 1;
897 mutex_init(&dvm_track->hdt_lock, NULL, MUTEX_DRIVER,
898 DDI_INTR_PRI(state->hs_intrmsi_pri));
899
900 /*
901 * Depending of the type of resource that has been mapped out, we
902 * need to update the QP or CQ handle to reflect that it has, in
903 * fact, been mapped. This allows the driver code which frees a QP
904 * or a CQ to know whether it is appropriate to do a
905 * devmap_devmem_remap() to invalidate the userland mapping for the
906 * corresponding queue's memory.
907 */
908 if (type == MLNX_UMAP_CQMEM_RSRC) {
909
910 /* Use "key" (CQ number) to do fast lookup of CQ handle */
911 cq = hermon_cqhdl_from_cqnum(state, key);
912
913 /*
983
984 /*
985 * hermon_devmap_dbrecmem_dup()
986 * Context: Can be called from kernel context.
987 */
988 /* ARGSUSED */
989 static int
990 hermon_devmap_dbrecmem_dup(devmap_cookie_t dhp, void *pvtp,
991 devmap_cookie_t new_dhp, void **new_pvtp)
992 {
993 hermon_state_t *state;
994 hermon_devmap_track_t *dvm_track, *new_dvm_track;
995 uint_t maxprot;
996 int status;
997
998 /*
999 * Extract the Hermon softstate pointer from "Hermon devmap tracking
1000 * structure" (in "pvtp").
1001 */
1002 dvm_track = (hermon_devmap_track_t *)pvtp;
1003 state = dvm_track->hdt_state;
1004
1005 /*
1006 * Since this devmap_dup() entry point is generally called
1007 * when a process does fork(2), it is incumbent upon the driver
1008 * to insure that the child does not inherit a valid copy of
1009 * the parent's QP or CQ resource. This is accomplished by using
1010 * devmap_devmem_remap() to invalidate the child's mapping to the
1011 * kernel memory.
1012 */
1013 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
1014 status = devmap_devmem_remap(new_dhp, state->hs_dip, 0, 0,
1015 dvm_track->hdt_size, maxprot, DEVMAP_MAPPING_INVALID, NULL);
1016 if (status != DDI_SUCCESS) {
1017 HERMON_WARNING(state, "failed in hermon_devmap_dbrecmem_dup()");
1018 return (status);
1019 }
1020
1021 /*
1022 * Allocate a new entry to track the subsequent unmapping
1023 * (specifically, all partial unmappings) of the child's newly
1024 * invalidated resource. Note: Setting the "hdt_size" field to
1025 * zero here is an indication to the devmap_unmap() entry point
1026 * that this mapping is invalid, and that its subsequent unmapping
1027 * should not affect any of the parent's CQ or QP resources.
1028 */
1029 new_dvm_track = (hermon_devmap_track_t *)kmem_zalloc(
1030 sizeof (hermon_devmap_track_t), KM_SLEEP);
1031 new_dvm_track->hdt_offset = 0;
1032 new_dvm_track->hdt_state = state;
1033 new_dvm_track->hdt_refcnt = 1;
1034 new_dvm_track->hdt_size = 0;
1035 mutex_init(&new_dvm_track->hdt_lock, NULL, MUTEX_DRIVER,
1036 DDI_INTR_PRI(state->hs_intrmsi_pri));
1037 *new_pvtp = new_dvm_track;
1038
1039 return (DDI_SUCCESS);
1040 }
1041
1042
1043 /*
1044 * hermon_devmap_dbrecmem_unmap()
1045 * Context: Can be called from kernel context.
1046 */
1047 /* ARGSUSED */
1048 static void
1049 hermon_devmap_dbrecmem_unmap(devmap_cookie_t dhp, void *pvtp, offset_t off,
1050 size_t len, devmap_cookie_t new_dhp1, void **pvtp1,
1051 devmap_cookie_t new_dhp2, void **pvtp2)
1052 {
1053 hermon_state_t *state;
1054 hermon_rsrc_t *rsrcp;
1055 hermon_devmap_track_t *dvm_track;
1056 hermon_cqhdl_t cq;
1057 hermon_qphdl_t qp;
1058 hermon_srqhdl_t srq;
1059 uint64_t key, value;
1060 uint_t type;
1061 uint_t size;
1062 int status;
1063
1064 /*
1065 * Extract the Hermon softstate pointer from "Hermon devmap tracking
1066 * structure" (in "pvtp").
1067 */
1068 dvm_track = (hermon_devmap_track_t *)pvtp;
1069 state = dvm_track->hdt_state;
1070
1071 /*
1072 * Extract the "offset" from the "Hermon devmap tracking structure".
1073 * Note: The input argument "off" is ignored here because the
1074 * Hermon mapping interfaces define a very specific meaning to
1075 * each "logical offset". Also extract the "key" and "type" encoded
1076 * in the logical offset.
1077 */
1078 key = dvm_track->hdt_offset >> PAGESHIFT;
1079 type = key & MLNX_UMAP_RSRC_TYPE_MASK;
1080 key = key >> MLNX_UMAP_RSRC_TYPE_SHIFT;
1081
1082 /*
1083 * Extract the "size" of the mapping. If this size is determined
1084 * to be zero, then it is an indication of a previously invalidated
1085 * mapping, and no CQ or QP resources should be affected.
1086 */
1087 size = dvm_track->hdt_size;
1088
1241 hermon_devmap_track_t *dvm_track;
1242 minor_t instance;
1243
1244 /* Get Hermon softstate structure from instance */
1245 instance = HERMON_DEV_INSTANCE(dev);
1246 state = ddi_get_soft_state(hermon_statep, instance);
1247 if (state == NULL) {
1248 return (ENXIO);
1249 }
1250
1251 /*
1252 * Allocate an entry to track the mapping and unmapping of this
1253 * resource. Note: We don't need to initialize the "refcnt" or
1254 * "offset" fields here, nor do we need to initialize the mutex
1255 * used with the "refcnt". Since UAR pages are single pages, they
1256 * are not subject to "partial" unmappings. This makes these other
1257 * fields unnecessary.
1258 */
1259 dvm_track = (hermon_devmap_track_t *)kmem_zalloc(
1260 sizeof (hermon_devmap_track_t), KM_SLEEP);
1261 dvm_track->hdt_state = state;
1262 dvm_track->hdt_size = (uint_t)PAGESIZE;
1263
1264 /*
1265 * Pass the private "Hermon devmap tracking structure" back. This
1266 * pointer will be returned in a subsequent "unmap" callback.
1267 */
1268 *pvtp = dvm_track;
1269
1270 return (DDI_SUCCESS);
1271 }
1272
1273
1274 /*
1275 * hermon_devmap_devmem_dup()
1276 * Context: Can be called from kernel context.
1277 */
1278 /* ARGSUSED */
1279 static int
1280 hermon_devmap_devmem_dup(devmap_cookie_t dhp, void *pvtp,
1281 devmap_cookie_t new_dhp, void **new_pvtp)
1282 {
1283 hermon_state_t *state;
1284 hermon_devmap_track_t *dvm_track;
1285 uint_t maxprot;
1286 int status;
1287
1288 /*
1289 * Extract the Hermon softstate pointer from "Hermon devmap tracking
1290 * structure" (in "pvtp"). Note: If the tracking structure is NULL
1291 * here, it means that the mapping corresponds to an invalid mapping.
1292 * In this case, it can be safely ignored ("new_pvtp" set to NULL).
1293 */
1294 dvm_track = (hermon_devmap_track_t *)pvtp;
1295 if (dvm_track == NULL) {
1296 *new_pvtp = NULL;
1297 return (DDI_SUCCESS);
1298 }
1299
1300 state = dvm_track->hdt_state;
1301
1302 /*
1303 * Since this devmap_dup() entry point is generally called
1304 * when a process does fork(2), it is incumbent upon the driver
1305 * to insure that the child does not inherit a valid copy of
1306 * the parent's resource. This is accomplished by using
1307 * devmap_devmem_remap() to invalidate the child's mapping to the
1308 * kernel memory.
1309 */
1310 maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
1311 status = devmap_devmem_remap(new_dhp, state->hs_dip, 0, 0,
1312 dvm_track->hdt_size, maxprot, DEVMAP_MAPPING_INVALID, NULL);
1313 if (status != DDI_SUCCESS) {
1314 HERMON_WARNING(state, "failed in hermon_devmap_devmem_dup()");
1315 return (status);
1316 }
1317
1318 /*
1319 * Since the region is invalid, there is no need for us to
1331 /*
1332 * hermon_devmap_devmem_unmap()
1333 * Context: Can be called from kernel context.
1334 */
1335 /* ARGSUSED */
1336 static void
1337 hermon_devmap_devmem_unmap(devmap_cookie_t dhp, void *pvtp, offset_t off,
1338 size_t len, devmap_cookie_t new_dhp1, void **pvtp1,
1339 devmap_cookie_t new_dhp2, void **pvtp2)
1340 {
1341 hermon_devmap_track_t *dvm_track;
1342
1343 /*
1344 * Free up the "Hermon devmap tracking structure" (in "pvtp").
1345 * There cannot be "partial" unmappings here because all UAR pages
1346 * are single pages. Note: If the tracking structure is NULL here,
1347 * it means that the mapping corresponds to an invalid mapping. In
1348 * this case, it can be safely ignored.
1349 */
1350 dvm_track = (hermon_devmap_track_t *)pvtp;
1351 if (dvm_track == NULL) {
1352 return;
1353 }
1354
1355 kmem_free(dvm_track, sizeof (hermon_devmap_track_t));
1356 }
1357
1358
1359 /*
1360 * hermon_umap_ci_data_in()
1361 * Context: Can be called from user or kernel context.
1362 */
1363 /* ARGSUSED */
1364 ibt_status_t
1365 hermon_umap_ci_data_in(hermon_state_t *state, ibt_ci_data_flags_t flags,
1366 ibt_object_type_t object, void *hdl, void *data_p, size_t data_sz)
1367 {
1368 int status;
1369
1370 /*
1412 * hermon_umap_mr_data_in()
1413 * Context: Can be called from user or kernel context.
1414 */
1415 static ibt_status_t
1416 hermon_umap_mr_data_in(hermon_mrhdl_t mr, ibt_mr_data_in_t *data,
1417 size_t data_sz)
1418 {
1419 if (data->mr_rev != IBT_MR_DATA_IN_IF_VERSION) {
1420 return (IBT_NOT_SUPPORTED);
1421 }
1422
1423 /* Check for valid MR handle pointer */
1424 if (mr == NULL) {
1425 return (IBT_MR_HDL_INVALID);
1426 }
1427
1428 /* Check for valid MR input structure size */
1429 if (data_sz < sizeof (ibt_mr_data_in_t)) {
1430 return (IBT_INSUFF_RESOURCE);
1431 }
1432
1433 /*
1434 * Ensure that the MR corresponds to userland memory and that it is
1435 * a currently valid memory region as well.
1436 */
1437 mutex_enter(&mr->mr_lock);
1438 if ((mr->mr_is_umem == 0) || (mr->mr_umemcookie == NULL)) {
1439 mutex_exit(&mr->mr_lock);
1440 return (IBT_MR_HDL_INVALID);
1441 }
1442
1443 /*
1444 * If it has passed all the above checks, then extract the callback
1445 * function and argument from the input structure. Copy them into
1446 * the MR handle. This function will be called only if the memory
1447 * corresponding to the MR handle gets a umem_lockmemory() callback.
1448 */
1449 mr->mr_umem_cbfunc = data->mr_func;
1450 mr->mr_umem_cbarg1 = data->mr_arg1;
1451 mr->mr_umem_cbarg2 = data->mr_arg2;
1528 }
1529
1530
1531 /*
1532 * hermon_umap_cq_data_out()
1533 * Context: Can be called from user or kernel context.
1534 */
1535 static ibt_status_t
1536 hermon_umap_cq_data_out(hermon_cqhdl_t cq, mlnx_umap_cq_data_out_t *data,
1537 size_t data_sz)
1538 {
1539 /* Check for valid CQ handle pointer */
1540 if (cq == NULL) {
1541 return (IBT_CQ_HDL_INVALID);
1542 }
1543
1544 /* Check for valid CQ mapping structure size */
1545 if (data_sz < sizeof (mlnx_umap_cq_data_out_t)) {
1546 return (IBT_INSUFF_RESOURCE);
1547 }
1548
1549 /* deal with cq_alloc() verses cq_resize() */
1550 if (cq->cq_resize_hdl) {
1551 data->mcq_maplen = cq->cq_resize_hdl->cq_cqinfo.qa_size;
1552 data->mcq_numcqe = cq->cq_resize_hdl->cq_bufsz;
1553 } else {
1554 data->mcq_maplen = cq->cq_cqinfo.qa_size;
1555 data->mcq_numcqe = cq->cq_bufsz;
1556 }
1557
1558 /*
1559 * If it has passed all the above checks, then fill in all the useful
1560 * mapping information (including the mapping offset that will be
1561 * passed back to the devmap() interface during a subsequent mmap()
1562 * call.
1563 *
1564 * The "offset" for CQ mmap()'s looks like this:
1565 * +----------------------------------------+--------+--------------+
1566 * | CQ Number | 0x33 | Reserved (0) |
1567 * +----------------------------------------+--------+--------------+
1590 }
1591
1592
1593 /*
1594 * hermon_umap_qp_data_out()
1595 * Context: Can be called from user or kernel context.
1596 */
1597 static ibt_status_t
1598 hermon_umap_qp_data_out(hermon_qphdl_t qp, mlnx_umap_qp_data_out_t *data,
1599 size_t data_sz)
1600 {
1601 /* Check for valid QP handle pointer */
1602 if (qp == NULL) {
1603 return (IBT_QP_HDL_INVALID);
1604 }
1605
1606 /* Check for valid QP mapping structure size */
1607 if (data_sz < sizeof (mlnx_umap_qp_data_out_t)) {
1608 return (IBT_INSUFF_RESOURCE);
1609 }
1610
1611 /*
1612 * If it has passed all the checks, then fill in all the useful
1613 * mapping information (including the mapping offset that will be
1614 * passed back to the devmap() interface during a subsequent mmap()
1615 * call.
1616 *
1617 * The "offset" for QP mmap()'s looks like this:
1618 * +----------------------------------------+--------+--------------+
1619 * | QP Number | 0x44 | Reserved (0) |
1620 * +----------------------------------------+--------+--------------+
1621 * (64 - 8 - PAGESHIFT) bits 8 bits PAGESHIFT bits
1622 *
1623 * This returns information about the mapping offset, the length of
1624 * the QP memory, and the QP number (for use in later send and recv
1625 * doorbells). It also returns the following information for both
1626 * the receive work queue and the send work queue, respectively: the
1627 * offset (from the base mapped address) of the start of the given
1628 * work queue, the 64-bit IB virtual address that corresponds to
1629 * the base mapped address (needed for posting WQEs though the
1680 }
1681
1682
1683 /*
1684 * hermon_umap_srq_data_out()
1685 * Context: Can be called from user or kernel context.
1686 */
1687 static ibt_status_t
1688 hermon_umap_srq_data_out(hermon_srqhdl_t srq, mlnx_umap_srq_data_out_t *data,
1689 size_t data_sz)
1690 {
1691 /* Check for valid SRQ handle pointer */
1692 if (srq == NULL) {
1693 return (IBT_SRQ_HDL_INVALID);
1694 }
1695
1696 /* Check for valid SRQ mapping structure size */
1697 if (data_sz < sizeof (mlnx_umap_srq_data_out_t)) {
1698 return (IBT_INSUFF_RESOURCE);
1699 }
1700
1701 /*
1702 * If it has passed all the checks, then fill in all the useful
1703 * mapping information (including the mapping offset that will be
1704 * passed back to the devmap() interface during a subsequent mmap()
1705 * call.
1706 *
1707 * The "offset" for SRQ mmap()'s looks like this:
1708 * +----------------------------------------+--------+--------------+
1709 * | SRQ Number | 0x66 | Reserved (0) |
1710 * +----------------------------------------+--------+--------------+
1711 * (64 - 8 - PAGESHIFT) bits 8 bits PAGESHIFT bits
1712 *
1713 * This returns information about the mapping offset, the length of the
1714 * SRQ memory, and the SRQ number (for use in later send and recv
1715 * doorbells). It also returns the following information for the
1716 * shared receive queue: the offset (from the base mapped address) of
1717 * the start of the given work queue, the 64-bit IB virtual address
1718 * that corresponds to the base mapped address (needed for posting WQEs
1719 * though the QP doorbells), the number of WQEs the given work queue
1740 }
1741
1742
1743 /*
1744 * hermon_umap_pd_data_out()
1745 * Context: Can be called from user or kernel context.
1746 */
1747 static ibt_status_t
1748 hermon_umap_pd_data_out(hermon_pdhdl_t pd, mlnx_umap_pd_data_out_t *data,
1749 size_t data_sz)
1750 {
1751 /* Check for valid PD handle pointer */
1752 if (pd == NULL) {
1753 return (IBT_PD_HDL_INVALID);
1754 }
1755
1756 /* Check for valid PD mapping structure size */
1757 if (data_sz < sizeof (mlnx_umap_pd_data_out_t)) {
1758 return (IBT_INSUFF_RESOURCE);
1759 }
1760
1761 /*
1762 * If it has passed all the checks, then fill the PD table index
1763 * (the PD table allocated index for the PD pd_pdnum).
1764 */
1765 data->mpd_rev = MLNX_UMAP_IF_VERSION;
1766 data->mpd_pdnum = pd->pd_pdnum;
1767
1768 return (DDI_SUCCESS);
1769 }
1770
1771
1772 /*
1773 * hermon_umap_db_init()
1774 * Context: Only called from attach() path context
1775 */
1776 void
1777 hermon_umap_db_init(void)
1778 {
1779 /*
1810
1811 /* Destroy the lock for the "userland resources database" */
1812 mutex_destroy(&hermon_userland_rsrc_db.hdl_umapdb_lock);
1813 }
1814
1815
1816 /*
1817 * hermon_umap_db_alloc()
1818 * Context: Can be called from user or kernel context.
1819 */
1820 hermon_umap_db_entry_t *
1821 hermon_umap_db_alloc(uint_t instance, uint64_t key, uint_t type, uint64_t value)
1822 {
1823 hermon_umap_db_entry_t *umapdb;
1824
1825 /* Allocate an entry to add to the "userland resources database" */
1826 umapdb = kmem_zalloc(sizeof (hermon_umap_db_entry_t), KM_NOSLEEP);
1827 if (umapdb == NULL) {
1828 return (NULL);
1829 }
1830
1831 /* Fill in the fields in the database entry */
1832 umapdb->hdbe_common.hdb_instance = instance;
1833 umapdb->hdbe_common.hdb_type = type;
1834 umapdb->hdbe_common.hdb_key = key;
1835 umapdb->hdbe_common.hdb_value = value;
1836
1837 return (umapdb);
1838 }
1839
1840
1841 /*
1842 * hermon_umap_db_free()
1843 * Context: Can be called from user or kernel context.
1844 */
1845 void
1846 hermon_umap_db_free(hermon_umap_db_entry_t *umapdb)
1847 {
1848 /* Free the database entry */
1849 kmem_free(umapdb, sizeof (hermon_umap_db_entry_t));
1858 hermon_umap_db_add(hermon_umap_db_entry_t *umapdb)
1859 {
1860 mutex_enter(&hermon_userland_rsrc_db.hdl_umapdb_lock);
1861 hermon_umap_db_add_nolock(umapdb);
1862 mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock);
1863 }
1864
1865
1866 /*
1867 * hermon_umap_db_add_nolock()
1868 * Context: Can be called from user or kernel context.
1869 */
1870 void
1871 hermon_umap_db_add_nolock(hermon_umap_db_entry_t *umapdb)
1872 {
1873 hermon_umap_db_query_t query;
1874 avl_index_t where;
1875
1876 ASSERT(MUTEX_HELD(&hermon_userland_rsrc_db.hdl_umapdb_lock));
1877
1878 /*
1879 * Copy the common portion of the "to-be-added" database entry
1880 * into the "hermon_umap_db_query_t" structure. We use this structure
1881 * (with no flags set) to find the appropriate location in the
1882 * "userland resources database" for the new entry to be added.
1883 *
1884 * Note: we expect that this entry should not be found in the
1885 * database (unless something bad has happened).
1886 */
1887 query.hqdb_common = umapdb->hdbe_common;
1888 query.hqdb_flags = 0;
1889 (void) avl_find(&hermon_userland_rsrc_db.hdl_umapdb_avl, &query,
1890 &where);
1891
1892 /*
1893 * Now, using the "where" field from the avl_find() operation
1894 * above, we will insert the new database entry ("umapdb").
1895 */
1896 avl_insert(&hermon_userland_rsrc_db.hdl_umapdb_avl, umapdb,
1897 where);
1933
1934 /*
1935 * Fill in key, type, instance, and flags values of the
1936 * hermon_umap_db_query_t in preparation for the database
1937 * lookup.
1938 */
1939 query.hqdb_flags = flags;
1940 query.hqdb_common.hdb_key = key;
1941 query.hqdb_common.hdb_type = type;
1942 query.hqdb_common.hdb_instance = instance;
1943
1944 /*
1945 * Perform the database query. If no entry is found, then
1946 * return failure, else continue.
1947 */
1948 entry = (hermon_umap_db_entry_t *)avl_find(
1949 &hermon_userland_rsrc_db.hdl_umapdb_avl, &query, &where);
1950 if (entry == NULL) {
1951 return (DDI_FAILURE);
1952 }
1953
1954 /*
1955 * If the flags argument specifies that the entry should
1956 * be removed if found, then call avl_remove() to remove
1957 * the entry from the database.
1958 */
1959 if (flags & HERMON_UMAP_DB_REMOVE) {
1960
1961 avl_remove(&hermon_userland_rsrc_db.hdl_umapdb_avl, entry);
1962
1963 /*
1964 * The database entry is returned with the expectation
1965 * that the caller will use hermon_umap_db_free() to
1966 * free the entry's memory. ASSERT that this is non-NULL.
1967 * NULL pointer should never be passed for the
1968 * HERMON_UMAP_DB_REMOVE case.
1969 */
1970 ASSERT(umapdb != NULL);
1971 }
1972
1996 hermon_umap_db_entry_t *umapdb;
1997 hermon_state_t *state;
1998 hermon_rsrc_t *rsrcp;
1999 hermon_mrhdl_t mr;
2000 uint64_t value;
2001 uint_t instance;
2002 int status;
2003 void (*mr_callback)(void *, void *);
2004 void *mr_cbarg1, *mr_cbarg2;
2005
2006 /*
2007 * If this was userland memory, then we need to remove its entry
2008 * from the "userland resources database". Note: We use the
2009 * HERMON_UMAP_DB_IGNORE_INSTANCE flag here because we don't know
2010 * which instance was used when the entry was added (but we want
2011 * to know after the entry is found using the other search criteria).
2012 */
2013 status = hermon_umap_db_find(0, (uint64_t)(uintptr_t)umem_cookie,
2014 MLNX_UMAP_MRMEM_RSRC, &value, (HERMON_UMAP_DB_REMOVE |
2015 HERMON_UMAP_DB_IGNORE_INSTANCE), &umapdb);
2016 if (status == DDI_SUCCESS) {
2017 instance = umapdb->hdbe_common.hdb_instance;
2018 state = ddi_get_soft_state(hermon_statep, instance);
2019 if (state == NULL) {
2020 cmn_err(CE_WARN, "Unable to match Hermon instance\n");
2021 return;
2022 }
2023
2024 /* Free the database entry */
2025 hermon_umap_db_free(umapdb);
2026
2027 /* Use "value" to convert to an MR handle */
2028 rsrcp = (hermon_rsrc_t *)(uintptr_t)value;
2029 mr = (hermon_mrhdl_t)rsrcp->hr_addr;
2030
2031 /*
2032 * If a callback has been provided, call it first. This
2033 * callback is expected to do any cleanup necessary to
2034 * guarantee that the subsequent MR deregister (below)
2035 * will succeed. Specifically, this means freeing up memory
2054 status = hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
2055 HERMON_SLEEP);
2056 if (status != DDI_SUCCESS) {
2057 HERMON_WARNING(state, "Unexpected failure in "
2058 "deregister from callback\n");
2059 }
2060 }
2061 }
2062
2063
2064 /*
2065 * hermon_umap_db_compare()
2066 * Context: Can be called from user or kernel context.
2067 */
2068 static int
2069 hermon_umap_db_compare(const void *q, const void *e)
2070 {
2071 hermon_umap_db_common_t *entry_common, *query_common;
2072 uint_t query_flags;
2073
2074 entry_common = &((hermon_umap_db_entry_t *)e)->hdbe_common;
2075 query_common = &((hermon_umap_db_query_t *)q)->hqdb_common;
2076 query_flags = ((hermon_umap_db_query_t *)q)->hqdb_flags;
2077
2078 /*
2079 * The first comparison is done on the "key" value in "query"
2080 * and "entry". If they are not equal, then the appropriate
2081 * search direction is returned. Else, we continue by
2082 * comparing "type".
2083 */
2084 if (query_common->hdb_key < entry_common->hdb_key) {
2085 return (-1);
2086 } else if (query_common->hdb_key > entry_common->hdb_key) {
2087 return (+1);
2088 }
2089
2090 /*
2091 * If the search reaches this point, then "query" and "entry"
2092 * have equal key values. So we continue be comparing their
2093 * "type" values. Again, if they are not equal, then the
|