495 /*
496 * Someday maybe the "ibt_adds_vect_t *attr_p" will be NULL to
497 * indicate that we wish to allocate an "invalid" (i.e. empty)
498 * address handle XXX
499 */
500
501 /* Validate that specified port number is legal */
502 if (!hermon_portnum_is_valid(state, attr_p->av_port_num)) {
503 return (IBT_HCA_PORT_INVALID);
504 }
505
506 /*
507 * Allocate the software structure for tracking the address handle
508 * (i.e. the Hermon Address Handle struct).
509 */
510 status = hermon_rsrc_alloc(state, HERMON_AHHDL, 1, sleepflag, &rsrc);
511 if (status != DDI_SUCCESS) {
512 return (IBT_INSUFF_RESOURCE);
513 }
514 ah = (hermon_ahhdl_t)rsrc->hr_addr;
515 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ah))
516
517 /* Increment the reference count on the protection domain (PD) */
518 hermon_pd_refcnt_inc(pd);
519
520 udav = (hermon_hw_udav_t *)kmem_zalloc(sizeof (hermon_hw_udav_t),
521 KM_SLEEP);
522 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*udav))
523
524 /*
525 * Fill in the UDAV data. We first zero out the UDAV, then populate
526 * it by then calling hermon_set_addr_path() to fill in the common
527 * portions that can be pulled from the "ibt_adds_vect_t" passed in
528 */
529 status = hermon_set_addr_path(state, attr_p,
530 (hermon_hw_addr_path_t *)udav, HERMON_ADDRPATH_UDAV);
531 if (status != DDI_SUCCESS) {
532 hermon_pd_refcnt_dec(pd);
533 hermon_rsrc_free(state, &rsrc);
534 return (status);
535 }
536 udav->pd = pd->pd_pdnum;
537 udav->sl = attr_p->av_srvl;
538
539 /*
540 * Fill in the rest of the Hermon Address Handle struct.
541 *
542 * NOTE: We are saving away a copy of the "av_dgid.gid_guid" field
560 * Context: Can be called only from user or kernel context.
561 */
562 /* ARGSUSED */
563 int
564 hermon_ah_free(hermon_state_t *state, hermon_ahhdl_t *ahhdl, uint_t sleepflag)
565 {
566 hermon_rsrc_t *rsrc;
567 hermon_pdhdl_t pd;
568 hermon_ahhdl_t ah;
569
570 /*
571 * Pull all the necessary information from the Hermon Address Handle
572 * struct. This is necessary here because the resource for the
573 * AH is going to be freed up as part of this operation.
574 */
575 ah = *ahhdl;
576 mutex_enter(&ah->ah_lock);
577 rsrc = ah->ah_rsrcp;
578 pd = ah->ah_pdhdl;
579 mutex_exit(&ah->ah_lock);
580 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ah))
581
582 /* Free the UDAV memory */
583 kmem_free(ah->ah_udav, sizeof (hermon_hw_udav_t));
584
585 /* Decrement the reference count on the protection domain (PD) */
586 hermon_pd_refcnt_dec(pd);
587
588 /* Free the Hermon Address Handle structure */
589 hermon_rsrc_free(state, &rsrc);
590
591 /* Set the ahhdl pointer to NULL and return success */
592 *ahhdl = NULL;
593
594 return (DDI_SUCCESS);
595 }
596
597
598 /*
599 * hermon_ah_query()
600 * Context: Can be called from interrupt or base context.
601 */
602 /* ARGSUSED */
603 int
604 hermon_ah_query(hermon_state_t *state, hermon_ahhdl_t ah, hermon_pdhdl_t *pd,
605 ibt_adds_vect_t *attr_p)
606 {
607 mutex_enter(&ah->ah_lock);
608 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attr_p))
609
610 /*
611 * Pull the PD and UDAV from the Hermon Address Handle structure
612 */
613 *pd = ah->ah_pdhdl;
614
615 /*
616 * Fill in "ibt_adds_vect_t". We call hermon_get_addr_path() to fill
617 * the common portions that can be pulled from the UDAV we pass in.
618 *
619 * NOTE: We will also fill the "av_dgid.gid_guid" field from the
620 * "ah_save_guid" field we have previously saved away. The reason
621 * for this is described in hermon_ah_alloc() and hermon_ah_modify().
622 */
623 hermon_get_addr_path(state, (hermon_hw_addr_path_t *)ah->ah_udav,
624 attr_p, HERMON_ADDRPATH_UDAV);
625
626 attr_p->av_dgid.gid_guid = ah->ah_save_guid;
627
628 mutex_exit(&ah->ah_lock);
655
656 /*
657 * Fill in the new UDAV with the caller's data, passed in via the
658 * "ibt_adds_vect_t" structure.
659 *
660 * NOTE: We also need to save away a copy of the "av_dgid.gid_guid"
661 * field here (just as we did during hermon_ah_alloc()) because we
662 * may need to return it later to the IBTF (as a result of a
663 * subsequent query operation). As explained in hermon_ah_alloc(),
664 * unlike the other UDAV parameters, the value of "av_dgid.gid_guid"
665 * is not always preserved. The reason for this is described in
666 * hermon_set_addr_path().
667 */
668 status = hermon_set_addr_path(state, attr_p,
669 (hermon_hw_addr_path_t *)ah->ah_udav, HERMON_ADDRPATH_UDAV);
670 if (status != DDI_SUCCESS) {
671 mutex_exit(&ah->ah_lock);
672 return (status);
673 }
674 ah->ah_save_guid = attr_p->av_dgid.gid_guid;
675 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*(ah->ah_udav)))
676 ah->ah_udav->sl = attr_p->av_srvl;
677
678 /*
679 * Copy changes into the new UDAV.
680 * Note: We copy in 64-bit chunks. For the first two of these
681 * chunks it is necessary to read the current contents of the
682 * UDAV, mask off the modifiable portions (maintaining any
683 * of the "reserved" portions), and then mask on the new data.
684 */
685 size = sizeof (hermon_hw_udav_t) >> 3;
686 for (i = 0; i < size; i++) {
687 data_old = ((uint64_t *)&old_udav)[i];
688
689 /*
690 * Apply mask to change only the relevant values.
691 */
692 if (i == 0) {
693 data_old = data_old & HERMON_UDAV_MODIFY_MASK0;
694 } else if (i == 1) {
695 data_old = data_old & HERMON_UDAV_MODIFY_MASK1;
1802 */
1803 int
1804 hermon_pd_alloc(hermon_state_t *state, hermon_pdhdl_t *pdhdl, uint_t sleepflag)
1805 {
1806 hermon_rsrc_t *rsrc;
1807 hermon_pdhdl_t pd;
1808 int status;
1809
1810 /*
1811 * Allocate the software structure for tracking the protection domain
1812 * (i.e. the Hermon Protection Domain handle). By default each PD
1813 * structure will have a unique PD number assigned to it. All that
1814 * is necessary is for software to initialize the PD reference count
1815 * (to zero) and return success.
1816 */
1817 status = hermon_rsrc_alloc(state, HERMON_PDHDL, 1, sleepflag, &rsrc);
1818 if (status != DDI_SUCCESS) {
1819 return (IBT_INSUFF_RESOURCE);
1820 }
1821 pd = (hermon_pdhdl_t)rsrc->hr_addr;
1822 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pd))
1823
1824 pd->pd_refcnt = 0;
1825 *pdhdl = pd;
1826
1827 return (DDI_SUCCESS);
1828 }
1829
1830
1831 /*
1832 * hermon_pd_free()
1833 * Context: Can be called only from user or kernel context.
1834 */
1835 int
1836 hermon_pd_free(hermon_state_t *state, hermon_pdhdl_t *pdhdl)
1837 {
1838 hermon_rsrc_t *rsrc;
1839 hermon_pdhdl_t pd;
1840
1841 /*
1842 * Pull all the necessary information from the Hermon Protection Domain
1843 * handle. This is necessary here because the resource for the
1844 * PD is going to be freed up as part of this operation.
1845 */
1846 pd = *pdhdl;
1847 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pd))
1848 rsrc = pd->pd_rsrcp;
1849
1850 /*
1851 * Check the PD reference count. If the reference count is non-zero,
1852 * then it means that this protection domain is still referenced by
1853 * some memory region, queue pair, address handle, or other IB object
1854 * If it is non-zero, then return an error. Otherwise, free the
1855 * Hermon resource and return success.
1856 */
1857 if (pd->pd_refcnt != 0) {
1858 return (IBT_PD_IN_USE);
1859 }
1860
1861 /* Free the Hermon Protection Domain handle */
1862 hermon_rsrc_free(state, &rsrc);
1863
1864 /* Set the pdhdl pointer to NULL and return success */
1865 *pdhdl = (hermon_pdhdl_t)NULL;
1866
1867 return (DDI_SUCCESS);
1891 atomic_dec_32(&pd->pd_refcnt);
1892 }
1893
1894
1895 /*
1896 * hermon_port_query()
1897 * Context: Can be called only from user or kernel context.
1898 */
1899 int
1900 hermon_port_query(hermon_state_t *state, uint_t port, ibt_hca_portinfo_t *pi)
1901 {
1902 sm_portinfo_t portinfo;
1903 sm_guidinfo_t guidinfo;
1904 sm_pkey_table_t pkeytable;
1905 ib_gid_t *sgid;
1906 uint_t sgid_max, pkey_max, tbl_size;
1907 int i, j, indx, status;
1908 ib_pkey_t *pkeyp;
1909 ib_guid_t *guidp;
1910
1911 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pi))
1912 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*state))
1913
1914 /* Validate that specified port number is legal */
1915 if (!hermon_portnum_is_valid(state, port)) {
1916 return (IBT_HCA_PORT_INVALID);
1917 }
1918 pkeyp = state->hs_pkey[port - 1];
1919 guidp = state->hs_guid[port - 1];
1920
1921 /*
1922 * We use the Hermon MAD_IFC command to post a GetPortInfo MAD
1923 * to the firmware (for the specified port number). This returns
1924 * a full PortInfo MAD (in "portinfo") which we subsequently
1925 * parse to fill in the "ibt_hca_portinfo_t" structure returned
1926 * to the IBTF.
1927 */
1928 status = hermon_getportinfo_cmd_post(state, port,
1929 HERMON_SLEEPFLAG_FOR_CONTEXT(), &portinfo);
1930 if (status != HERMON_CMD_SUCCESS) {
1931 cmn_err(CE_CONT, "Hermon: GetPortInfo (port %02d) command "
1932 "failed: %08x\n", port, status);
1933 if (status == HERMON_CMD_INVALID_STATUS) {
1991 * GetPortInfo above) to form the entire SGID table.
1992 */
1993 for (i = 0; i < pi->p_sgid_tbl_sz; i += 8) {
1994 status = hermon_getguidinfo_cmd_post(state, port, i >> 3,
1995 HERMON_SLEEPFLAG_FOR_CONTEXT(), &guidinfo);
1996 if (status != HERMON_CMD_SUCCESS) {
1997 cmn_err(CE_CONT, "Hermon: GetGUIDInfo (port %02d) "
1998 "command failed: %08x\n", port, status);
1999 if (status == HERMON_CMD_INVALID_STATUS) {
2000 hermon_fm_ereport(state, HCA_SYS_ERR,
2001 HCA_ERR_SRV_LOST);
2002 }
2003 return (ibc_get_ci_failure(0));
2004 }
2005
2006 /* Figure out how many of the entries are valid */
2007 sgid_max = min((pi->p_sgid_tbl_sz - i), 8);
2008 for (j = 0; j < sgid_max; j++) {
2009 indx = (i + j);
2010 sgid = &pi->p_sgid_tbl[indx];
2011 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sgid))
2012 sgid->gid_prefix = portinfo.GidPrefix;
2013 guidp[indx] = sgid->gid_guid =
2014 guidinfo.GUIDBlocks[j];
2015 }
2016 }
2017
2018 /*
2019 * Fill in the PKey table. Just as for the GID tables above, the
2020 * only access to the Hermon PKey tables is through the firmware's
2021 * MAD_IFC interface. We post as many GetPKeyTable MADs as necessary
2022 * to read in the entire contents of the PKey table (for the specified
2023 * port). Note: The GetPKeyTable command only gets 32 PKeys per
2024 * operation.
2025 */
2026 for (i = 0; i < pi->p_pkey_tbl_sz; i += 32) {
2027 status = hermon_getpkeytable_cmd_post(state, port, i,
2028 HERMON_SLEEPFLAG_FOR_CONTEXT(), &pkeytable);
2029 if (status != HERMON_CMD_SUCCESS) {
2030 cmn_err(CE_CONT, "Hermon: GetPKeyTable (port %02d) "
2031 "command failed: %08x\n", port, status);
2157 * hermon_set_addr_path()
2158 * Context: Can be called from interrupt or base context.
2159 *
2160 * Note: This routine is used for two purposes. It is used to fill in the
2161 * Hermon UDAV fields, and it is used to fill in the address path information
2162 * for QPs. Because the two Hermon structures are similar, common fields can
2163 * be filled in here. Because they are different, however, we pass
2164 * an additional flag to indicate which type is being filled and do each one
2165 * uniquely
2166 */
2167
2168 int hermon_srate_override = -1; /* allows ease of testing */
2169
2170 int
2171 hermon_set_addr_path(hermon_state_t *state, ibt_adds_vect_t *av,
2172 hermon_hw_addr_path_t *path, uint_t type)
2173 {
2174 uint_t gidtbl_sz;
2175 hermon_hw_udav_t *udav;
2176
2177 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*av))
2178 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*path))
2179
2180 udav = (hermon_hw_udav_t *)(void *)path;
2181 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*udav))
2182 path->mlid = av->av_src_path;
2183 path->rlid = av->av_dlid;
2184
2185 switch (av->av_srate) {
2186 case IBT_SRATE_2: /* 1xSDR-2.5Gb/s injection rate */
2187 path->max_stat_rate = 7; break;
2188 case IBT_SRATE_10: /* 4xSDR-10.0Gb/s injection rate */
2189 path->max_stat_rate = 8; break;
2190 case IBT_SRATE_30: /* 12xSDR-30Gb/s injection rate */
2191 path->max_stat_rate = 9; break;
2192 case IBT_SRATE_5: /* 1xDDR-5Gb/s injection rate */
2193 path->max_stat_rate = 10; break;
2194 case IBT_SRATE_20: /* 4xDDR-20Gb/s injection rate */
2195 path->max_stat_rate = 11; break;
2196 case IBT_SRATE_40: /* 4xQDR-40Gb/s injection rate */
2197 path->max_stat_rate = 12; break;
2198 case IBT_SRATE_60: /* 12xDDR-60Gb/s injection rate */
2199 path->max_stat_rate = 13; break;
2200 case IBT_SRATE_80: /* 8xQDR-80Gb/s injection rate */
2201 path->max_stat_rate = 14; break;
2272 }
2273
2274
2275 /*
2276 * hermon_get_addr_path()
2277 * Context: Can be called from interrupt or base context.
2278 *
2279 * Note: Just like hermon_set_addr_path() above, this routine is used for two
2280 * purposes. It is used to read in the Hermon UDAV fields, and it is used to
2281 * read in the address path information for QPs. Because the two Hermon
2282 * structures are similar, common fields can be read in here. But because
2283 * they are slightly different, we pass an additional flag to indicate which
2284 * type is being read.
2285 */
2286 void
2287 hermon_get_addr_path(hermon_state_t *state, hermon_hw_addr_path_t *path,
2288 ibt_adds_vect_t *av, uint_t type)
2289 {
2290 uint_t gidtbl_sz;
2291
2292 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*path))
2293 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*av))
2294
2295 av->av_src_path = path->mlid;
2296 av->av_dlid = path->rlid;
2297
2298 /* Set "av_ipd" value from max_stat_rate */
2299 switch (path->max_stat_rate) {
2300 case 7: /* 1xSDR-2.5Gb/s injection rate */
2301 av->av_srate = IBT_SRATE_2; break;
2302 case 8: /* 4xSDR-10.0Gb/s injection rate */
2303 av->av_srate = IBT_SRATE_10; break;
2304 case 9: /* 12xSDR-30Gb/s injection rate */
2305 av->av_srate = IBT_SRATE_30; break;
2306 case 10: /* 1xDDR-5Gb/s injection rate */
2307 av->av_srate = IBT_SRATE_5; break;
2308 case 11: /* 4xDDR-20Gb/s injection rate */
2309 av->av_srate = IBT_SRATE_20; break;
2310 case 12: /* xQDR-40Gb/s injection rate */
2311 av->av_srate = IBT_SRATE_40; break;
2312 case 13: /* 12xDDR-60Gb/s injection rate */
2313 av->av_srate = IBT_SRATE_60; break;
2314 case 14: /* 8xQDR-80Gb/s injection rate */
2383 return (1);
2384 } else {
2385 return (0);
2386 }
2387 }
2388
2389
2390 /*
2391 * hermon_queue_alloc()
2392 * Context: Can be called from interrupt or base context.
2393 */
2394 int
2395 hermon_queue_alloc(hermon_state_t *state, hermon_qalloc_info_t *qa_info,
2396 uint_t sleepflag)
2397 {
2398 ddi_dma_attr_t dma_attr;
2399 int (*callback)(caddr_t);
2400 uint64_t realsize, alloc_mask;
2401 int flag, status;
2402
2403 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qa_info))
2404
2405 /* Set the callback flag appropriately */
2406 callback = (sleepflag == HERMON_SLEEP) ? DDI_DMA_SLEEP :
2407 DDI_DMA_DONTWAIT;
2408
2409 /*
2410 * Initialize many of the default DMA attributes. Then set additional
2411 * alignment restrictions as necessary for the queue memory. Also
2412 * respect the configured value for IOMMU bypass
2413 */
2414 hermon_dma_attr_init(state, &dma_attr);
2415 dma_attr.dma_attr_align = qa_info->qa_bind_align;
2416 #ifdef __sparc
2417 if (state->hs_cfg_profile->cp_iommu_bypass == HERMON_BINDMEM_BYPASS) {
2418 dma_attr.dma_attr_flags = DDI_DMA_FORCE_PHYSICAL;
2419 }
2420 #endif
2421
2422 /* Allocate a DMA handle */
2423 status = ddi_dma_alloc_handle(state->hs_dip, &dma_attr, callback, NULL,
2424 &qa_info->qa_dmahdl);
2503 }
2504 /*
2505 * The last step is to figure out the offset of the start relative
2506 * to the first page of the region - will be used in the eqc/cqc
2507 * passed to the HW
2508 */
2509 qa_info->qa_pgoffs = (uint_t)((uintptr_t)
2510 qa_info->qa_buf_aligned & HERMON_PAGEOFFSET);
2511
2512 return (DDI_SUCCESS);
2513 }
2514
2515
2516 /*
2517 * hermon_queue_free()
2518 * Context: Can be called from interrupt or base context.
2519 */
2520 void
2521 hermon_queue_free(hermon_qalloc_info_t *qa_info)
2522 {
2523 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qa_info))
2524
2525 /*
2526 * Depending on how (i.e. from where) we allocated the memory for
2527 * this queue, we choose the appropriate method for releasing the
2528 * resources.
2529 */
2530 if (qa_info->qa_location == HERMON_QUEUE_LOCATION_NORMAL) {
2531
2532 ddi_dma_mem_free(&qa_info->qa_acchdl);
2533
2534 } else if (qa_info->qa_location == HERMON_QUEUE_LOCATION_USERLAND) {
2535
2536 ddi_umem_free(qa_info->qa_umemcookie);
2537
2538 }
2539
2540 /* Always free the dma handle */
2541 ddi_dma_free_handle(&qa_info->qa_dmahdl);
2542 }
2543
2544 /*
2552 {
2553 hermon_fmrhdl_t fmrpool;
2554 hermon_fmr_list_t *fmr, *fmr_next;
2555 hermon_mrhdl_t mr;
2556 int status;
2557 int sleep;
2558 int i;
2559
2560 sleep = (fmr_attr->fmr_flags & IBT_MR_SLEEP) ? HERMON_SLEEP :
2561 HERMON_NOSLEEP;
2562 if ((sleep == HERMON_SLEEP) &&
2563 (sleep != HERMON_SLEEPFLAG_FOR_CONTEXT())) {
2564 return (IBT_INVALID_PARAM);
2565 }
2566
2567 fmrpool = (hermon_fmrhdl_t)kmem_zalloc(sizeof (*fmrpool), sleep);
2568 if (fmrpool == NULL) {
2569 status = IBT_INSUFF_RESOURCE;
2570 goto fail;
2571 }
2572 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*fmrpool))
2573
2574 mutex_init(&fmrpool->fmr_lock, NULL, MUTEX_DRIVER,
2575 DDI_INTR_PRI(state->hs_intrmsi_pri));
2576 mutex_init(&fmrpool->remap_lock, NULL, MUTEX_DRIVER,
2577 DDI_INTR_PRI(state->hs_intrmsi_pri));
2578 mutex_init(&fmrpool->dirty_lock, NULL, MUTEX_DRIVER,
2579 DDI_INTR_PRI(state->hs_intrmsi_pri));
2580
2581 fmrpool->fmr_state = state;
2582 fmrpool->fmr_flush_function = fmr_attr->fmr_func_hdlr;
2583 fmrpool->fmr_flush_arg = fmr_attr->fmr_func_arg;
2584 fmrpool->fmr_pool_size = 0;
2585 fmrpool->fmr_max_pages = fmr_attr->fmr_max_pages_per_fmr;
2586 fmrpool->fmr_page_sz = fmr_attr->fmr_page_sz;
2587 fmrpool->fmr_dirty_watermark = fmr_attr->fmr_pool_size / 4;
2588 fmrpool->fmr_dirty_len = 0;
2589 fmrpool->fmr_remap_watermark = fmr_attr->fmr_pool_size / 32;
2590 fmrpool->fmr_remap_len = 0;
2591 fmrpool->fmr_flags = fmr_attr->fmr_flags;
2592 fmrpool->fmr_stat_register = 0;
2593 fmrpool->fmr_max_remaps = state->hs_cfg_profile->cp_fmr_max_remaps;
2594 fmrpool->fmr_remap_gen = 1;
2595
2596 fmrpool->fmr_free_list_tail = &fmrpool->fmr_free_list;
2597 fmrpool->fmr_dirty_list = NULL;
2598 fmrpool->fmr_dirty_list_tail = &fmrpool->fmr_dirty_list;
2599 fmrpool->fmr_remap_list = NULL;
2600 fmrpool->fmr_remap_list_tail = &fmrpool->fmr_remap_list;
2601 fmrpool->fmr_pool_size = fmrpool->fmr_free_len =
2602 fmr_attr->fmr_pool_size;
2603
2604 for (i = 0; i < fmr_attr->fmr_pool_size; i++) {
2605 status = hermon_mr_alloc_fmr(state, pd, fmrpool, &mr);
2606 if (status != DDI_SUCCESS) {
2607 goto fail2;
2608 }
2609
2610 fmr = (hermon_fmr_list_t *)kmem_zalloc(
2611 sizeof (hermon_fmr_list_t), sleep);
2612 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*fmr))
2613
2614 fmr->fmr = mr;
2615 fmr->fmr_remaps = 0;
2616 fmr->fmr_remap_gen = fmrpool->fmr_remap_gen;
2617 fmr->fmr_pool = fmrpool;
2618 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))
2619 mr->mr_fmr = fmr;
2620
2621 if (!i) /* address of last entry's link */
2622 fmrpool->fmr_free_list_tail = &fmr->fmr_next;
2623 fmr->fmr_next = fmrpool->fmr_free_list;
2624 fmrpool->fmr_free_list = fmr;
2625 }
2626
2627 /* Set to return pool */
2628 *fmrpoolp = fmrpool;
2629
2630 IBTF_DPRINTF_L2("fmr", "create_fmr_pool SUCCESS");
2631 return (IBT_SUCCESS);
2632 fail2:
2633 for (fmr = fmrpool->fmr_free_list; fmr != NULL; fmr = fmr_next) {
2634 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*fmr))
2635 fmr_next = fmr->fmr_next;
2636 (void) hermon_mr_dealloc_fmr(state, &fmr->fmr);
2637 kmem_free(fmr, sizeof (hermon_fmr_list_t));
2638 }
2639 kmem_free(fmrpool, sizeof (*fmrpool));
2640 fail:
2641 *fmrpoolp = NULL;
2642 IBTF_DPRINTF_L2("fmr", "create_fmr_pool FAILED");
2643 if (status == DDI_FAILURE) {
2644 return (ibc_get_ci_failure(0));
2645 } else {
2646 return (status);
2647 }
2648 }
2649
2650 /*
2651 * hermon_destroy_fmr_pool()
2652 * Destroy an FMR pool and free all associated resources.
2653 * Context: Can be called from kernel context only.
2654 */
2743 if (hermon_fmr_verbose & 2)
2744 IBTF_DPRINTF_L2("fmr", "register needs cleanup");
2745 hermon_fmr_cleanup(fmrpool);
2746 }
2747
2748 /* grab next free entry */
2749 fmr = fmrpool->fmr_free_list;
2750 if (fmr == NULL) {
2751 IBTF_DPRINTF_L2("fmr", "WARNING: no free fmr resource");
2752 cmn_err(CE_CONT, "no free fmr resource\n");
2753 mutex_exit(&fmrpool->fmr_lock);
2754 return (IBT_INSUFF_RESOURCE);
2755 }
2756
2757 if ((fmrpool->fmr_free_list = fmr->fmr_next) == NULL)
2758 fmrpool->fmr_free_list_tail = &fmrpool->fmr_free_list;
2759 fmr->fmr_next = NULL;
2760 fmrpool->fmr_stat_register++;
2761 mutex_exit(&fmrpool->fmr_lock);
2762
2763 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*fmr))
2764 status = hermon_mr_register_physical_fmr(state, mem_pattr, fmr->fmr,
2765 mem_desc_p);
2766 if (status != DDI_SUCCESS) {
2767 return (status);
2768 }
2769 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*fmr->fmr))
2770 if (hermon_rdma_debug & 0x4)
2771 IBTF_DPRINTF_L2("fmr", " reg: mr %p key %x",
2772 fmr->fmr, fmr->fmr->mr_rkey);
2773 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*fmr->fmr))
2774 if (fmr->fmr_remap_gen != fmrpool->fmr_remap_gen) {
2775 fmr->fmr_remap_gen = fmrpool->fmr_remap_gen;
2776 fmr->fmr_remaps = 0;
2777 }
2778
2779 fmr->fmr_remaps++;
2780
2781 *mr = (hermon_mrhdl_t)fmr->fmr;
2782
2783 return (DDI_SUCCESS);
2784 }
2785
2786 /*
2787 * hermon_deregister_fmr()
2788 * Unmap FMR
2789 * Context: Can be called from kernel context only.
2790 */
2791 int
2792 hermon_deregister_fmr(hermon_state_t *state, hermon_mrhdl_t mr)
2793 {
2794 hermon_fmrhdl_t fmrpool;
2795 hermon_fmr_list_t *fmr, **fmrlast;
2796 int len;
2797
2798 fmr = mr->mr_fmr;
2799 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*fmr))
2800 fmrpool = fmr->fmr_pool;
2801
2802 /* mark as owned by software */
2803 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*(fmr->fmr)))
2804 *(uint8_t *)(fmr->fmr->mr_mptrsrcp->hr_addr) = 0xF0;
2805
2806 if (fmr->fmr_remaps <
2807 state->hs_cfg_profile->cp_fmr_max_remaps) {
2808 /* add to remap list */
2809 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*(fmr->fmr)))
2810 if (hermon_rdma_debug & 0x4)
2811 IBTF_DPRINTF_L2("fmr", "dereg: mr %p key %x",
2812 fmr->fmr, fmr->fmr->mr_rkey);
2813 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*(fmr->fmr)))
2814 mutex_enter(&fmrpool->remap_lock);
2815 fmr->fmr_next = NULL;
2816 *(fmrpool->fmr_remap_list_tail) = fmr;
2817 fmrpool->fmr_remap_list_tail = &fmr->fmr_next;
2818 fmrpool->fmr_remap_len++;
2819
2820 /* conditionally add remap list back to free list */
2821 fmrlast = NULL;
2822 if (fmrpool->fmr_remap_len >=
2823 fmrpool->fmr_remap_watermark) {
2824 fmr = fmrpool->fmr_remap_list;
2825 fmrlast = fmrpool->fmr_remap_list_tail;
2826 len = fmrpool->fmr_remap_len;
2827 fmrpool->fmr_remap_len = 0;
2828 fmrpool->fmr_remap_list = NULL;
2829 fmrpool->fmr_remap_list_tail =
2830 &fmrpool->fmr_remap_list;
2831 }
2832 mutex_exit(&fmrpool->remap_lock);
2833 if (fmrlast) {
2834 mutex_enter(&fmrpool->fmr_lock);
2835 *(fmrpool->fmr_free_list_tail) = fmr;
2836 fmrpool->fmr_free_list_tail = fmrlast;
2837 fmrpool->fmr_free_len += len;
2838 mutex_exit(&fmrpool->fmr_lock);
2839 }
2840 } else {
2841 /* add to dirty list */
2842 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*(fmr->fmr)))
2843 if (hermon_rdma_debug & 0x4)
2844 IBTF_DPRINTF_L2("fmr", "dirty: mr %p key %x",
2845 fmr->fmr, fmr->fmr->mr_rkey);
2846 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*(fmr->fmr)))
2847
2848 mutex_enter(&fmrpool->dirty_lock);
2849 fmr->fmr_next = NULL;
2850 *(fmrpool->fmr_dirty_list_tail) = fmr;
2851 fmrpool->fmr_dirty_list_tail = &fmr->fmr_next;
2852 fmrpool->fmr_dirty_len++;
2853
2854 if (fmrpool->fmr_dirty_len >=
2855 fmrpool->fmr_dirty_watermark) {
2856 mutex_exit(&fmrpool->dirty_lock);
2857 mutex_enter(&fmrpool->fmr_lock);
2858 hermon_fmr_cleanup(fmrpool);
2859 mutex_exit(&fmrpool->fmr_lock);
2860 } else
2861 mutex_exit(&fmrpool->dirty_lock);
2862 }
2863 return (DDI_SUCCESS);
2864 }
2865
2866 /*
|
495 /*
496 * Someday maybe the "ibt_adds_vect_t *attr_p" will be NULL to
497 * indicate that we wish to allocate an "invalid" (i.e. empty)
498 * address handle XXX
499 */
500
501 /* Validate that specified port number is legal */
502 if (!hermon_portnum_is_valid(state, attr_p->av_port_num)) {
503 return (IBT_HCA_PORT_INVALID);
504 }
505
506 /*
507 * Allocate the software structure for tracking the address handle
508 * (i.e. the Hermon Address Handle struct).
509 */
510 status = hermon_rsrc_alloc(state, HERMON_AHHDL, 1, sleepflag, &rsrc);
511 if (status != DDI_SUCCESS) {
512 return (IBT_INSUFF_RESOURCE);
513 }
514 ah = (hermon_ahhdl_t)rsrc->hr_addr;
515
516 /* Increment the reference count on the protection domain (PD) */
517 hermon_pd_refcnt_inc(pd);
518
519 udav = (hermon_hw_udav_t *)kmem_zalloc(sizeof (hermon_hw_udav_t),
520 KM_SLEEP);
521
522 /*
523 * Fill in the UDAV data. We first zero out the UDAV, then populate
524 * it by then calling hermon_set_addr_path() to fill in the common
525 * portions that can be pulled from the "ibt_adds_vect_t" passed in
526 */
527 status = hermon_set_addr_path(state, attr_p,
528 (hermon_hw_addr_path_t *)udav, HERMON_ADDRPATH_UDAV);
529 if (status != DDI_SUCCESS) {
530 hermon_pd_refcnt_dec(pd);
531 hermon_rsrc_free(state, &rsrc);
532 return (status);
533 }
534 udav->pd = pd->pd_pdnum;
535 udav->sl = attr_p->av_srvl;
536
537 /*
538 * Fill in the rest of the Hermon Address Handle struct.
539 *
540 * NOTE: We are saving away a copy of the "av_dgid.gid_guid" field
558 * Context: Can be called only from user or kernel context.
559 */
560 /* ARGSUSED */
561 int
562 hermon_ah_free(hermon_state_t *state, hermon_ahhdl_t *ahhdl, uint_t sleepflag)
563 {
564 hermon_rsrc_t *rsrc;
565 hermon_pdhdl_t pd;
566 hermon_ahhdl_t ah;
567
568 /*
569 * Pull all the necessary information from the Hermon Address Handle
570 * struct. This is necessary here because the resource for the
571 * AH is going to be freed up as part of this operation.
572 */
573 ah = *ahhdl;
574 mutex_enter(&ah->ah_lock);
575 rsrc = ah->ah_rsrcp;
576 pd = ah->ah_pdhdl;
577 mutex_exit(&ah->ah_lock);
578
579 /* Free the UDAV memory */
580 kmem_free(ah->ah_udav, sizeof (hermon_hw_udav_t));
581
582 /* Decrement the reference count on the protection domain (PD) */
583 hermon_pd_refcnt_dec(pd);
584
585 /* Free the Hermon Address Handle structure */
586 hermon_rsrc_free(state, &rsrc);
587
588 /* Set the ahhdl pointer to NULL and return success */
589 *ahhdl = NULL;
590
591 return (DDI_SUCCESS);
592 }
593
594
595 /*
596 * hermon_ah_query()
597 * Context: Can be called from interrupt or base context.
598 */
599 /* ARGSUSED */
600 int
601 hermon_ah_query(hermon_state_t *state, hermon_ahhdl_t ah, hermon_pdhdl_t *pd,
602 ibt_adds_vect_t *attr_p)
603 {
604 mutex_enter(&ah->ah_lock);
605
606 /*
607 * Pull the PD and UDAV from the Hermon Address Handle structure
608 */
609 *pd = ah->ah_pdhdl;
610
611 /*
612 * Fill in "ibt_adds_vect_t". We call hermon_get_addr_path() to fill
613 * the common portions that can be pulled from the UDAV we pass in.
614 *
615 * NOTE: We will also fill the "av_dgid.gid_guid" field from the
616 * "ah_save_guid" field we have previously saved away. The reason
617 * for this is described in hermon_ah_alloc() and hermon_ah_modify().
618 */
619 hermon_get_addr_path(state, (hermon_hw_addr_path_t *)ah->ah_udav,
620 attr_p, HERMON_ADDRPATH_UDAV);
621
622 attr_p->av_dgid.gid_guid = ah->ah_save_guid;
623
624 mutex_exit(&ah->ah_lock);
651
652 /*
653 * Fill in the new UDAV with the caller's data, passed in via the
654 * "ibt_adds_vect_t" structure.
655 *
656 * NOTE: We also need to save away a copy of the "av_dgid.gid_guid"
657 * field here (just as we did during hermon_ah_alloc()) because we
658 * may need to return it later to the IBTF (as a result of a
659 * subsequent query operation). As explained in hermon_ah_alloc(),
660 * unlike the other UDAV parameters, the value of "av_dgid.gid_guid"
661 * is not always preserved. The reason for this is described in
662 * hermon_set_addr_path().
663 */
664 status = hermon_set_addr_path(state, attr_p,
665 (hermon_hw_addr_path_t *)ah->ah_udav, HERMON_ADDRPATH_UDAV);
666 if (status != DDI_SUCCESS) {
667 mutex_exit(&ah->ah_lock);
668 return (status);
669 }
670 ah->ah_save_guid = attr_p->av_dgid.gid_guid;
671 ah->ah_udav->sl = attr_p->av_srvl;
672
673 /*
674 * Copy changes into the new UDAV.
675 * Note: We copy in 64-bit chunks. For the first two of these
676 * chunks it is necessary to read the current contents of the
677 * UDAV, mask off the modifiable portions (maintaining any
678 * of the "reserved" portions), and then mask on the new data.
679 */
680 size = sizeof (hermon_hw_udav_t) >> 3;
681 for (i = 0; i < size; i++) {
682 data_old = ((uint64_t *)&old_udav)[i];
683
684 /*
685 * Apply mask to change only the relevant values.
686 */
687 if (i == 0) {
688 data_old = data_old & HERMON_UDAV_MODIFY_MASK0;
689 } else if (i == 1) {
690 data_old = data_old & HERMON_UDAV_MODIFY_MASK1;
1797 */
1798 int
1799 hermon_pd_alloc(hermon_state_t *state, hermon_pdhdl_t *pdhdl, uint_t sleepflag)
1800 {
1801 hermon_rsrc_t *rsrc;
1802 hermon_pdhdl_t pd;
1803 int status;
1804
1805 /*
1806 * Allocate the software structure for tracking the protection domain
1807 * (i.e. the Hermon Protection Domain handle). By default each PD
1808 * structure will have a unique PD number assigned to it. All that
1809 * is necessary is for software to initialize the PD reference count
1810 * (to zero) and return success.
1811 */
1812 status = hermon_rsrc_alloc(state, HERMON_PDHDL, 1, sleepflag, &rsrc);
1813 if (status != DDI_SUCCESS) {
1814 return (IBT_INSUFF_RESOURCE);
1815 }
1816 pd = (hermon_pdhdl_t)rsrc->hr_addr;
1817
1818 pd->pd_refcnt = 0;
1819 *pdhdl = pd;
1820
1821 return (DDI_SUCCESS);
1822 }
1823
1824
1825 /*
1826 * hermon_pd_free()
1827 * Context: Can be called only from user or kernel context.
1828 */
1829 int
1830 hermon_pd_free(hermon_state_t *state, hermon_pdhdl_t *pdhdl)
1831 {
1832 hermon_rsrc_t *rsrc;
1833 hermon_pdhdl_t pd;
1834
1835 /*
1836 * Pull all the necessary information from the Hermon Protection Domain
1837 * handle. This is necessary here because the resource for the
1838 * PD is going to be freed up as part of this operation.
1839 */
1840 pd = *pdhdl;
1841 rsrc = pd->pd_rsrcp;
1842
1843 /*
1844 * Check the PD reference count. If the reference count is non-zero,
1845 * then it means that this protection domain is still referenced by
1846 * some memory region, queue pair, address handle, or other IB object
1847 * If it is non-zero, then return an error. Otherwise, free the
1848 * Hermon resource and return success.
1849 */
1850 if (pd->pd_refcnt != 0) {
1851 return (IBT_PD_IN_USE);
1852 }
1853
1854 /* Free the Hermon Protection Domain handle */
1855 hermon_rsrc_free(state, &rsrc);
1856
1857 /* Set the pdhdl pointer to NULL and return success */
1858 *pdhdl = (hermon_pdhdl_t)NULL;
1859
1860 return (DDI_SUCCESS);
1884 atomic_dec_32(&pd->pd_refcnt);
1885 }
1886
1887
1888 /*
1889 * hermon_port_query()
1890 * Context: Can be called only from user or kernel context.
1891 */
1892 int
1893 hermon_port_query(hermon_state_t *state, uint_t port, ibt_hca_portinfo_t *pi)
1894 {
1895 sm_portinfo_t portinfo;
1896 sm_guidinfo_t guidinfo;
1897 sm_pkey_table_t pkeytable;
1898 ib_gid_t *sgid;
1899 uint_t sgid_max, pkey_max, tbl_size;
1900 int i, j, indx, status;
1901 ib_pkey_t *pkeyp;
1902 ib_guid_t *guidp;
1903
1904 /* Validate that specified port number is legal */
1905 if (!hermon_portnum_is_valid(state, port)) {
1906 return (IBT_HCA_PORT_INVALID);
1907 }
1908 pkeyp = state->hs_pkey[port - 1];
1909 guidp = state->hs_guid[port - 1];
1910
1911 /*
1912 * We use the Hermon MAD_IFC command to post a GetPortInfo MAD
1913 * to the firmware (for the specified port number). This returns
1914 * a full PortInfo MAD (in "portinfo") which we subsequently
1915 * parse to fill in the "ibt_hca_portinfo_t" structure returned
1916 * to the IBTF.
1917 */
1918 status = hermon_getportinfo_cmd_post(state, port,
1919 HERMON_SLEEPFLAG_FOR_CONTEXT(), &portinfo);
1920 if (status != HERMON_CMD_SUCCESS) {
1921 cmn_err(CE_CONT, "Hermon: GetPortInfo (port %02d) command "
1922 "failed: %08x\n", port, status);
1923 if (status == HERMON_CMD_INVALID_STATUS) {
1981 * GetPortInfo above) to form the entire SGID table.
1982 */
1983 for (i = 0; i < pi->p_sgid_tbl_sz; i += 8) {
1984 status = hermon_getguidinfo_cmd_post(state, port, i >> 3,
1985 HERMON_SLEEPFLAG_FOR_CONTEXT(), &guidinfo);
1986 if (status != HERMON_CMD_SUCCESS) {
1987 cmn_err(CE_CONT, "Hermon: GetGUIDInfo (port %02d) "
1988 "command failed: %08x\n", port, status);
1989 if (status == HERMON_CMD_INVALID_STATUS) {
1990 hermon_fm_ereport(state, HCA_SYS_ERR,
1991 HCA_ERR_SRV_LOST);
1992 }
1993 return (ibc_get_ci_failure(0));
1994 }
1995
1996 /* Figure out how many of the entries are valid */
1997 sgid_max = min((pi->p_sgid_tbl_sz - i), 8);
1998 for (j = 0; j < sgid_max; j++) {
1999 indx = (i + j);
2000 sgid = &pi->p_sgid_tbl[indx];
2001 sgid->gid_prefix = portinfo.GidPrefix;
2002 guidp[indx] = sgid->gid_guid =
2003 guidinfo.GUIDBlocks[j];
2004 }
2005 }
2006
2007 /*
2008 * Fill in the PKey table. Just as for the GID tables above, the
2009 * only access to the Hermon PKey tables is through the firmware's
2010 * MAD_IFC interface. We post as many GetPKeyTable MADs as necessary
2011 * to read in the entire contents of the PKey table (for the specified
2012 * port). Note: The GetPKeyTable command only gets 32 PKeys per
2013 * operation.
2014 */
2015 for (i = 0; i < pi->p_pkey_tbl_sz; i += 32) {
2016 status = hermon_getpkeytable_cmd_post(state, port, i,
2017 HERMON_SLEEPFLAG_FOR_CONTEXT(), &pkeytable);
2018 if (status != HERMON_CMD_SUCCESS) {
2019 cmn_err(CE_CONT, "Hermon: GetPKeyTable (port %02d) "
2020 "command failed: %08x\n", port, status);
2146 * hermon_set_addr_path()
2147 * Context: Can be called from interrupt or base context.
2148 *
2149 * Note: This routine is used for two purposes. It is used to fill in the
2150 * Hermon UDAV fields, and it is used to fill in the address path information
2151 * for QPs. Because the two Hermon structures are similar, common fields can
2152 * be filled in here. Because they are different, however, we pass
2153 * an additional flag to indicate which type is being filled and do each one
2154 * uniquely
2155 */
2156
2157 int hermon_srate_override = -1; /* allows ease of testing */
2158
2159 int
2160 hermon_set_addr_path(hermon_state_t *state, ibt_adds_vect_t *av,
2161 hermon_hw_addr_path_t *path, uint_t type)
2162 {
2163 uint_t gidtbl_sz;
2164 hermon_hw_udav_t *udav;
2165
2166 udav = (hermon_hw_udav_t *)(void *)path;
2167 path->mlid = av->av_src_path;
2168 path->rlid = av->av_dlid;
2169
2170 switch (av->av_srate) {
2171 case IBT_SRATE_2: /* 1xSDR-2.5Gb/s injection rate */
2172 path->max_stat_rate = 7; break;
2173 case IBT_SRATE_10: /* 4xSDR-10.0Gb/s injection rate */
2174 path->max_stat_rate = 8; break;
2175 case IBT_SRATE_30: /* 12xSDR-30Gb/s injection rate */
2176 path->max_stat_rate = 9; break;
2177 case IBT_SRATE_5: /* 1xDDR-5Gb/s injection rate */
2178 path->max_stat_rate = 10; break;
2179 case IBT_SRATE_20: /* 4xDDR-20Gb/s injection rate */
2180 path->max_stat_rate = 11; break;
2181 case IBT_SRATE_40: /* 4xQDR-40Gb/s injection rate */
2182 path->max_stat_rate = 12; break;
2183 case IBT_SRATE_60: /* 12xDDR-60Gb/s injection rate */
2184 path->max_stat_rate = 13; break;
2185 case IBT_SRATE_80: /* 8xQDR-80Gb/s injection rate */
2186 path->max_stat_rate = 14; break;
2257 }
2258
2259
2260 /*
2261 * hermon_get_addr_path()
2262 * Context: Can be called from interrupt or base context.
2263 *
2264 * Note: Just like hermon_set_addr_path() above, this routine is used for two
2265 * purposes. It is used to read in the Hermon UDAV fields, and it is used to
2266 * read in the address path information for QPs. Because the two Hermon
2267 * structures are similar, common fields can be read in here. But because
2268 * they are slightly different, we pass an additional flag to indicate which
2269 * type is being read.
2270 */
2271 void
2272 hermon_get_addr_path(hermon_state_t *state, hermon_hw_addr_path_t *path,
2273 ibt_adds_vect_t *av, uint_t type)
2274 {
2275 uint_t gidtbl_sz;
2276
2277 av->av_src_path = path->mlid;
2278 av->av_dlid = path->rlid;
2279
2280 /* Set "av_ipd" value from max_stat_rate */
2281 switch (path->max_stat_rate) {
2282 case 7: /* 1xSDR-2.5Gb/s injection rate */
2283 av->av_srate = IBT_SRATE_2; break;
2284 case 8: /* 4xSDR-10.0Gb/s injection rate */
2285 av->av_srate = IBT_SRATE_10; break;
2286 case 9: /* 12xSDR-30Gb/s injection rate */
2287 av->av_srate = IBT_SRATE_30; break;
2288 case 10: /* 1xDDR-5Gb/s injection rate */
2289 av->av_srate = IBT_SRATE_5; break;
2290 case 11: /* 4xDDR-20Gb/s injection rate */
2291 av->av_srate = IBT_SRATE_20; break;
2292 case 12: /* xQDR-40Gb/s injection rate */
2293 av->av_srate = IBT_SRATE_40; break;
2294 case 13: /* 12xDDR-60Gb/s injection rate */
2295 av->av_srate = IBT_SRATE_60; break;
2296 case 14: /* 8xQDR-80Gb/s injection rate */
2365 return (1);
2366 } else {
2367 return (0);
2368 }
2369 }
2370
2371
2372 /*
2373 * hermon_queue_alloc()
2374 * Context: Can be called from interrupt or base context.
2375 */
2376 int
2377 hermon_queue_alloc(hermon_state_t *state, hermon_qalloc_info_t *qa_info,
2378 uint_t sleepflag)
2379 {
2380 ddi_dma_attr_t dma_attr;
2381 int (*callback)(caddr_t);
2382 uint64_t realsize, alloc_mask;
2383 int flag, status;
2384
2385 /* Set the callback flag appropriately */
2386 callback = (sleepflag == HERMON_SLEEP) ? DDI_DMA_SLEEP :
2387 DDI_DMA_DONTWAIT;
2388
2389 /*
2390 * Initialize many of the default DMA attributes. Then set additional
2391 * alignment restrictions as necessary for the queue memory. Also
2392 * respect the configured value for IOMMU bypass
2393 */
2394 hermon_dma_attr_init(state, &dma_attr);
2395 dma_attr.dma_attr_align = qa_info->qa_bind_align;
2396 #ifdef __sparc
2397 if (state->hs_cfg_profile->cp_iommu_bypass == HERMON_BINDMEM_BYPASS) {
2398 dma_attr.dma_attr_flags = DDI_DMA_FORCE_PHYSICAL;
2399 }
2400 #endif
2401
2402 /* Allocate a DMA handle */
2403 status = ddi_dma_alloc_handle(state->hs_dip, &dma_attr, callback, NULL,
2404 &qa_info->qa_dmahdl);
2483 }
2484 /*
2485 * The last step is to figure out the offset of the start relative
2486 * to the first page of the region - will be used in the eqc/cqc
2487 * passed to the HW
2488 */
2489 qa_info->qa_pgoffs = (uint_t)((uintptr_t)
2490 qa_info->qa_buf_aligned & HERMON_PAGEOFFSET);
2491
2492 return (DDI_SUCCESS);
2493 }
2494
2495
2496 /*
2497 * hermon_queue_free()
2498 * Context: Can be called from interrupt or base context.
2499 */
2500 void
2501 hermon_queue_free(hermon_qalloc_info_t *qa_info)
2502 {
2503 /*
2504 * Depending on how (i.e. from where) we allocated the memory for
2505 * this queue, we choose the appropriate method for releasing the
2506 * resources.
2507 */
2508 if (qa_info->qa_location == HERMON_QUEUE_LOCATION_NORMAL) {
2509
2510 ddi_dma_mem_free(&qa_info->qa_acchdl);
2511
2512 } else if (qa_info->qa_location == HERMON_QUEUE_LOCATION_USERLAND) {
2513
2514 ddi_umem_free(qa_info->qa_umemcookie);
2515
2516 }
2517
2518 /* Always free the dma handle */
2519 ddi_dma_free_handle(&qa_info->qa_dmahdl);
2520 }
2521
2522 /*
2530 {
2531 hermon_fmrhdl_t fmrpool;
2532 hermon_fmr_list_t *fmr, *fmr_next;
2533 hermon_mrhdl_t mr;
2534 int status;
2535 int sleep;
2536 int i;
2537
2538 sleep = (fmr_attr->fmr_flags & IBT_MR_SLEEP) ? HERMON_SLEEP :
2539 HERMON_NOSLEEP;
2540 if ((sleep == HERMON_SLEEP) &&
2541 (sleep != HERMON_SLEEPFLAG_FOR_CONTEXT())) {
2542 return (IBT_INVALID_PARAM);
2543 }
2544
2545 fmrpool = (hermon_fmrhdl_t)kmem_zalloc(sizeof (*fmrpool), sleep);
2546 if (fmrpool == NULL) {
2547 status = IBT_INSUFF_RESOURCE;
2548 goto fail;
2549 }
2550
2551 mutex_init(&fmrpool->fmr_lock, NULL, MUTEX_DRIVER,
2552 DDI_INTR_PRI(state->hs_intrmsi_pri));
2553 mutex_init(&fmrpool->remap_lock, NULL, MUTEX_DRIVER,
2554 DDI_INTR_PRI(state->hs_intrmsi_pri));
2555 mutex_init(&fmrpool->dirty_lock, NULL, MUTEX_DRIVER,
2556 DDI_INTR_PRI(state->hs_intrmsi_pri));
2557
2558 fmrpool->fmr_state = state;
2559 fmrpool->fmr_flush_function = fmr_attr->fmr_func_hdlr;
2560 fmrpool->fmr_flush_arg = fmr_attr->fmr_func_arg;
2561 fmrpool->fmr_pool_size = 0;
2562 fmrpool->fmr_max_pages = fmr_attr->fmr_max_pages_per_fmr;
2563 fmrpool->fmr_page_sz = fmr_attr->fmr_page_sz;
2564 fmrpool->fmr_dirty_watermark = fmr_attr->fmr_pool_size / 4;
2565 fmrpool->fmr_dirty_len = 0;
2566 fmrpool->fmr_remap_watermark = fmr_attr->fmr_pool_size / 32;
2567 fmrpool->fmr_remap_len = 0;
2568 fmrpool->fmr_flags = fmr_attr->fmr_flags;
2569 fmrpool->fmr_stat_register = 0;
2570 fmrpool->fmr_max_remaps = state->hs_cfg_profile->cp_fmr_max_remaps;
2571 fmrpool->fmr_remap_gen = 1;
2572
2573 fmrpool->fmr_free_list_tail = &fmrpool->fmr_free_list;
2574 fmrpool->fmr_dirty_list = NULL;
2575 fmrpool->fmr_dirty_list_tail = &fmrpool->fmr_dirty_list;
2576 fmrpool->fmr_remap_list = NULL;
2577 fmrpool->fmr_remap_list_tail = &fmrpool->fmr_remap_list;
2578 fmrpool->fmr_pool_size = fmrpool->fmr_free_len =
2579 fmr_attr->fmr_pool_size;
2580
2581 for (i = 0; i < fmr_attr->fmr_pool_size; i++) {
2582 status = hermon_mr_alloc_fmr(state, pd, fmrpool, &mr);
2583 if (status != DDI_SUCCESS) {
2584 goto fail2;
2585 }
2586
2587 fmr = (hermon_fmr_list_t *)kmem_zalloc(
2588 sizeof (hermon_fmr_list_t), sleep);
2589
2590 fmr->fmr = mr;
2591 fmr->fmr_remaps = 0;
2592 fmr->fmr_remap_gen = fmrpool->fmr_remap_gen;
2593 fmr->fmr_pool = fmrpool;
2594 mr->mr_fmr = fmr;
2595
2596 if (!i) /* address of last entry's link */
2597 fmrpool->fmr_free_list_tail = &fmr->fmr_next;
2598 fmr->fmr_next = fmrpool->fmr_free_list;
2599 fmrpool->fmr_free_list = fmr;
2600 }
2601
2602 /* Set to return pool */
2603 *fmrpoolp = fmrpool;
2604
2605 IBTF_DPRINTF_L2("fmr", "create_fmr_pool SUCCESS");
2606 return (IBT_SUCCESS);
2607 fail2:
2608 for (fmr = fmrpool->fmr_free_list; fmr != NULL; fmr = fmr_next) {
2609 fmr_next = fmr->fmr_next;
2610 (void) hermon_mr_dealloc_fmr(state, &fmr->fmr);
2611 kmem_free(fmr, sizeof (hermon_fmr_list_t));
2612 }
2613 kmem_free(fmrpool, sizeof (*fmrpool));
2614 fail:
2615 *fmrpoolp = NULL;
2616 IBTF_DPRINTF_L2("fmr", "create_fmr_pool FAILED");
2617 if (status == DDI_FAILURE) {
2618 return (ibc_get_ci_failure(0));
2619 } else {
2620 return (status);
2621 }
2622 }
2623
2624 /*
2625 * hermon_destroy_fmr_pool()
2626 * Destroy an FMR pool and free all associated resources.
2627 * Context: Can be called from kernel context only.
2628 */
2717 if (hermon_fmr_verbose & 2)
2718 IBTF_DPRINTF_L2("fmr", "register needs cleanup");
2719 hermon_fmr_cleanup(fmrpool);
2720 }
2721
2722 /* grab next free entry */
2723 fmr = fmrpool->fmr_free_list;
2724 if (fmr == NULL) {
2725 IBTF_DPRINTF_L2("fmr", "WARNING: no free fmr resource");
2726 cmn_err(CE_CONT, "no free fmr resource\n");
2727 mutex_exit(&fmrpool->fmr_lock);
2728 return (IBT_INSUFF_RESOURCE);
2729 }
2730
2731 if ((fmrpool->fmr_free_list = fmr->fmr_next) == NULL)
2732 fmrpool->fmr_free_list_tail = &fmrpool->fmr_free_list;
2733 fmr->fmr_next = NULL;
2734 fmrpool->fmr_stat_register++;
2735 mutex_exit(&fmrpool->fmr_lock);
2736
2737 status = hermon_mr_register_physical_fmr(state, mem_pattr, fmr->fmr,
2738 mem_desc_p);
2739 if (status != DDI_SUCCESS) {
2740 return (status);
2741 }
2742 if (hermon_rdma_debug & 0x4)
2743 IBTF_DPRINTF_L2("fmr", " reg: mr %p key %x",
2744 fmr->fmr, fmr->fmr->mr_rkey);
2745 if (fmr->fmr_remap_gen != fmrpool->fmr_remap_gen) {
2746 fmr->fmr_remap_gen = fmrpool->fmr_remap_gen;
2747 fmr->fmr_remaps = 0;
2748 }
2749
2750 fmr->fmr_remaps++;
2751
2752 *mr = (hermon_mrhdl_t)fmr->fmr;
2753
2754 return (DDI_SUCCESS);
2755 }
2756
2757 /*
2758 * hermon_deregister_fmr()
2759 * Unmap FMR
2760 * Context: Can be called from kernel context only.
2761 */
2762 int
2763 hermon_deregister_fmr(hermon_state_t *state, hermon_mrhdl_t mr)
2764 {
2765 hermon_fmrhdl_t fmrpool;
2766 hermon_fmr_list_t *fmr, **fmrlast;
2767 int len;
2768
2769 fmr = mr->mr_fmr;
2770 fmrpool = fmr->fmr_pool;
2771
2772 /* mark as owned by software */
2773 *(uint8_t *)(fmr->fmr->mr_mptrsrcp->hr_addr) = 0xF0;
2774
2775 if (fmr->fmr_remaps <
2776 state->hs_cfg_profile->cp_fmr_max_remaps) {
2777 /* add to remap list */
2778 if (hermon_rdma_debug & 0x4)
2779 IBTF_DPRINTF_L2("fmr", "dereg: mr %p key %x",
2780 fmr->fmr, fmr->fmr->mr_rkey);
2781 mutex_enter(&fmrpool->remap_lock);
2782 fmr->fmr_next = NULL;
2783 *(fmrpool->fmr_remap_list_tail) = fmr;
2784 fmrpool->fmr_remap_list_tail = &fmr->fmr_next;
2785 fmrpool->fmr_remap_len++;
2786
2787 /* conditionally add remap list back to free list */
2788 fmrlast = NULL;
2789 if (fmrpool->fmr_remap_len >=
2790 fmrpool->fmr_remap_watermark) {
2791 fmr = fmrpool->fmr_remap_list;
2792 fmrlast = fmrpool->fmr_remap_list_tail;
2793 len = fmrpool->fmr_remap_len;
2794 fmrpool->fmr_remap_len = 0;
2795 fmrpool->fmr_remap_list = NULL;
2796 fmrpool->fmr_remap_list_tail =
2797 &fmrpool->fmr_remap_list;
2798 }
2799 mutex_exit(&fmrpool->remap_lock);
2800 if (fmrlast) {
2801 mutex_enter(&fmrpool->fmr_lock);
2802 *(fmrpool->fmr_free_list_tail) = fmr;
2803 fmrpool->fmr_free_list_tail = fmrlast;
2804 fmrpool->fmr_free_len += len;
2805 mutex_exit(&fmrpool->fmr_lock);
2806 }
2807 } else {
2808 /* add to dirty list */
2809 if (hermon_rdma_debug & 0x4)
2810 IBTF_DPRINTF_L2("fmr", "dirty: mr %p key %x",
2811 fmr->fmr, fmr->fmr->mr_rkey);
2812
2813 mutex_enter(&fmrpool->dirty_lock);
2814 fmr->fmr_next = NULL;
2815 *(fmrpool->fmr_dirty_list_tail) = fmr;
2816 fmrpool->fmr_dirty_list_tail = &fmr->fmr_next;
2817 fmrpool->fmr_dirty_len++;
2818
2819 if (fmrpool->fmr_dirty_len >=
2820 fmrpool->fmr_dirty_watermark) {
2821 mutex_exit(&fmrpool->dirty_lock);
2822 mutex_enter(&fmrpool->fmr_lock);
2823 hermon_fmr_cleanup(fmrpool);
2824 mutex_exit(&fmrpool->fmr_lock);
2825 } else
2826 mutex_exit(&fmrpool->dirty_lock);
2827 }
2828 return (DDI_SUCCESS);
2829 }
2830
2831 /*
|