108 */
109 status = tavor_rsrc_alloc(state, TAVOR_UDAV, 1, sleepflag, &udav);
110 if (status != DDI_SUCCESS) {
111 /* Set "status" and "errormsg" and goto failure */
112 TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed UDAV");
113 goto ahalloc_fail;
114 }
115
116 /*
117 * Allocate the software structure for tracking the address handle
118 * (i.e. the Tavor Address Handle struct). If we fail here, we must
119 * undo the previous resource allocation.
120 */
121 status = tavor_rsrc_alloc(state, TAVOR_AHHDL, 1, sleepflag, &rsrc);
122 if (status != DDI_SUCCESS) {
123 /* Set "status" and "errormsg" and goto failure */
124 TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed AH handler");
125 goto ahalloc_fail1;
126 }
127 ah = (tavor_ahhdl_t)rsrc->tr_addr;
128 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ah))
129
130 /* Increment the reference count on the protection domain (PD) */
131 tavor_pd_refcnt_inc(pd);
132
133 /*
134 * Fill in the UDAV entry. Note: We are only filling in a temporary
135 * copy here, which we will later copy into the actual entry in
136 * Tavor DDR memory. This starts be zeroing out the temporary copy
137 * and then calling tavor_set_addr_path() to fill in the common
138 * portions that can be pulled from the "ibt_adds_vect_t" passed in
139 */
140 bzero(&udav_entry, sizeof (tavor_hw_udav_t));
141 status = tavor_set_addr_path(state, attr_p,
142 (tavor_hw_addr_path_t *)&udav_entry, TAVOR_ADDRPATH_UDAV, NULL);
143 if (status != DDI_SUCCESS) {
144 tavor_pd_refcnt_dec(pd);
145 tavor_rsrc_free(state, &rsrc);
146 tavor_rsrc_free(state, &udav);
147 /* Set "status" and "errormsg" and goto failure */
148 TAVOR_TNF_FAIL(status, "failed in tavor_set_addr_path");
247 tavor_rsrc_t *udav, *rsrc;
248 tavor_pdhdl_t pd;
249 tavor_mrhdl_t mr;
250 tavor_ahhdl_t ah;
251 int status;
252
253 TAVOR_TNF_ENTER(tavor_ah_free);
254
255 /*
256 * Pull all the necessary information from the Tavor Address Handle
257 * struct. This is necessary here because the resource for the
258 * AH is going to be freed up as part of this operation.
259 */
260 ah = *ahhdl;
261 mutex_enter(&ah->ah_lock);
262 udav = ah->ah_udavrsrcp;
263 rsrc = ah->ah_rsrcp;
264 pd = ah->ah_pdhdl;
265 mr = ah->ah_mrhdl;
266 mutex_exit(&ah->ah_lock);
267 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ah))
268
269 /*
270 * Deregister the memory for the UDAV. If this fails for any reason,
271 * then it is an indication that something (either in HW or SW) has
272 * gone seriously wrong. So we print a warning message and return
273 * failure.
274 */
275 status = tavor_mr_deregister(state, &mr, TAVOR_MR_DEREG_ALL,
276 sleepflag);
277 if (status != DDI_SUCCESS) {
278 TNF_PROBE_0(tavor_ah_free_dereg_mr_fail, TAVOR_TNF_ERROR, "");
279 TAVOR_TNF_EXIT(tavor_ah_free);
280 return (ibc_get_ci_failure(0));
281 }
282
283 /*
284 * Write zero to the first 64-bit word in the UDAV entry. As
285 * described above (in tavor_ah_alloc), the PD number is stored in
286 * the first 64-bits of each UDAV and setting this to zero is
287 * guaranteed to invalidate the entry.
309
310
311 /*
312 * tavor_ah_query()
313 * Context: Can be called from interrupt or base context.
314 */
315 /* ARGSUSED */
316 int
317 tavor_ah_query(tavor_state_t *state, tavor_ahhdl_t ah, tavor_pdhdl_t *pd,
318 ibt_adds_vect_t *attr_p)
319 {
320 tavor_hw_udav_t udav_entry;
321 tavor_rsrc_t *udav;
322 uint64_t data;
323 uint32_t size;
324 int i;
325
326 TAVOR_TNF_ENTER(tavor_ah_query);
327
328 mutex_enter(&ah->ah_lock);
329 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attr_p))
330
331 /*
332 * Pull all the necessary information from the Tavor Address Handle
333 * structure
334 */
335 udav = ah->ah_udavrsrcp;
336 *pd = ah->ah_pdhdl;
337
338 /*
339 * Copy the UDAV entry into the temporary copy. Here we copy all
340 * the information from the UDAV entry in DDR memory into the
341 * temporary UDAV. Note: We don't need to sync the UDAV for
342 * reading by software because Tavor HW never modifies the entry.
343 */
344 size = sizeof (tavor_hw_udav_t) >> 3;
345 for (i = 0; i < size; i++) {
346 data = ddi_get64(udav->tr_acchdl,
347 ((uint64_t *)udav->tr_addr + i));
348 ((uint64_t *)&udav_entry)[i] = data;
349 }
500 mutex_exit(&ah->ah_lock);
501 TAVOR_TNF_EXIT(tavor_ah_modify);
502 return (DDI_SUCCESS);
503 }
504
505
506 /*
507 * tavor_udav_sync()
508 * Context: Can be called from interrupt or base context.
509 */
510 /* ARGSUSED */
511 static void
512 tavor_udav_sync(tavor_ahhdl_t ah, tavor_hw_udav_t *udav, uint_t flag)
513 {
514 ddi_dma_handle_t dmahdl;
515 off_t offset;
516 int status;
517
518 TAVOR_TNF_ENTER(tavor_udav_sync);
519
520 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ah))
521
522 /* Determine if AH needs to be synced or not */
523 if (ah->ah_sync == 0) {
524 TAVOR_TNF_EXIT(tavor_udav_sync);
525 return;
526 }
527
528 /* Get the DMA handle from AH handle */
529 dmahdl = ah->ah_mrhdl->mr_bindinfo.bi_dmahdl;
530
531 /* Calculate offset into address handle */
532 offset = (off_t)0;
533 status = ddi_dma_sync(dmahdl, offset, sizeof (tavor_hw_udav_t), flag);
534 if (status != DDI_SUCCESS) {
535 TNF_PROBE_0(tavor_udav_sync_getnextentry_fail,
536 TAVOR_TNF_ERROR, "");
537 TAVOR_TNF_EXIT(tavor_udav_sync);
538 return;
539 }
540
541 TAVOR_TNF_EXIT(tavor_udav_sync);
1715 tavor_rsrc_t *rsrc;
1716 tavor_pdhdl_t pd;
1717 int status;
1718
1719 TAVOR_TNF_ENTER(tavor_pd_alloc);
1720
1721 /*
1722 * Allocate the software structure for tracking the protection domain
1723 * (i.e. the Tavor Protection Domain handle). By default each PD
1724 * structure will have a unique PD number assigned to it. All that
1725 * is necessary is for software to initialize the PD reference count
1726 * (to zero) and return success.
1727 */
1728 status = tavor_rsrc_alloc(state, TAVOR_PDHDL, 1, sleepflag, &rsrc);
1729 if (status != DDI_SUCCESS) {
1730 TNF_PROBE_0(tavor_pd_alloc_rsrcalloc_fail, TAVOR_TNF_ERROR, "");
1731 TAVOR_TNF_EXIT(tavor_pd_alloc);
1732 return (IBT_INSUFF_RESOURCE);
1733 }
1734 pd = (tavor_pdhdl_t)rsrc->tr_addr;
1735 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pd))
1736
1737 pd->pd_refcnt = 0;
1738 *pdhdl = pd;
1739
1740 TAVOR_TNF_EXIT(tavor_pd_alloc);
1741 return (DDI_SUCCESS);
1742 }
1743
1744
1745 /*
1746 * tavor_pd_free()
1747 * Context: Can be called only from user or kernel context.
1748 */
1749 int
1750 tavor_pd_free(tavor_state_t *state, tavor_pdhdl_t *pdhdl)
1751 {
1752 tavor_rsrc_t *rsrc;
1753 tavor_pdhdl_t pd;
1754
1755 TAVOR_TNF_ENTER(tavor_pd_free);
1756
1757 /*
1758 * Pull all the necessary information from the Tavor Protection Domain
1759 * handle. This is necessary here because the resource for the
1760 * PD is going to be freed up as part of this operation.
1761 */
1762 pd = *pdhdl;
1763 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pd))
1764 rsrc = pd->pd_rsrcp;
1765
1766 /*
1767 * Check the PD reference count. If the reference count is non-zero,
1768 * then it means that this protection domain is still referenced by
1769 * some memory region, queue pair, address handle, or other IB object
1770 * If it is non-zero, then return an error. Otherwise, free the
1771 * Tavor resource and return success.
1772 */
1773 if (pd->pd_refcnt != 0) {
1774 TNF_PROBE_1(tavor_pd_free_refcnt_fail, TAVOR_TNF_ERROR, "",
1775 tnf_int, refcnt, pd->pd_refcnt);
1776 TAVOR_TNF_EXIT(tavor_pd_free);
1777 return (IBT_PD_IN_USE);
1778 }
1779
1780 /* Free the Tavor Protection Domain handle */
1781 tavor_rsrc_free(state, &rsrc);
1782
1783 /* Set the pdhdl pointer to NULL and return success */
1821
1822 }
1823
1824
1825 /*
1826 * tavor_port_query()
1827 * Context: Can be called only from user or kernel context.
1828 */
1829 int
1830 tavor_port_query(tavor_state_t *state, uint_t port, ibt_hca_portinfo_t *pi)
1831 {
1832 sm_portinfo_t portinfo;
1833 sm_guidinfo_t guidinfo;
1834 sm_pkey_table_t pkeytable;
1835 ib_gid_t *sgid;
1836 uint_t sgid_max, pkey_max, tbl_size;
1837 int i, j, indx, status;
1838
1839 TAVOR_TNF_ENTER(tavor_port_query);
1840
1841 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pi))
1842
1843 /* Validate that specified port number is legal */
1844 if (!tavor_portnum_is_valid(state, port)) {
1845 TNF_PROBE_1(tavor_port_query_inv_portnum_fail,
1846 TAVOR_TNF_ERROR, "", tnf_uint, port, port);
1847 TAVOR_TNF_EXIT(tavor_port_query);
1848 return (IBT_HCA_PORT_INVALID);
1849 }
1850
1851 /*
1852 * We use the Tavor MAD_IFC command to post a GetPortInfo MAD
1853 * to the firmware (for the specified port number). This returns
1854 * a full PortInfo MAD (in "portinfo") which we subsequently
1855 * parse to fill in the "ibt_hca_portinfo_t" structure returned
1856 * to the IBTF.
1857 */
1858 status = tavor_getportinfo_cmd_post(state, port,
1859 TAVOR_SLEEPFLAG_FOR_CONTEXT(), &portinfo);
1860 if (status != TAVOR_CMD_SUCCESS) {
1861 cmn_err(CE_CONT, "Tavor: GetPortInfo (port %02d) command "
1862 "failed: %08x\n", port, status);
1917 * GUIDs are then appended to the GID prefix for the port (from the
1918 * GetPortInfo above) to form the entire SGID table.
1919 */
1920 for (i = 0; i < pi->p_sgid_tbl_sz; i += 8) {
1921 status = tavor_getguidinfo_cmd_post(state, port, i >> 3,
1922 TAVOR_SLEEPFLAG_FOR_CONTEXT(), &guidinfo);
1923 if (status != TAVOR_CMD_SUCCESS) {
1924 cmn_err(CE_CONT, "Tavor: GetGUIDInfo (port %02d) "
1925 "command failed: %08x\n", port, status);
1926 TNF_PROBE_1(tavor_port_query_getguidinfo_cmd_fail,
1927 TAVOR_TNF_ERROR, "", tnf_uint, cmd_status, status);
1928 TAVOR_TNF_EXIT(tavor_port_query);
1929 return (ibc_get_ci_failure(0));
1930 }
1931
1932 /* Figure out how many of the entries are valid */
1933 sgid_max = min((pi->p_sgid_tbl_sz - i), 8);
1934 for (j = 0; j < sgid_max; j++) {
1935 indx = (i + j);
1936 sgid = &pi->p_sgid_tbl[indx];
1937 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sgid))
1938 sgid->gid_prefix = portinfo.GidPrefix;
1939 sgid->gid_guid = guidinfo.GUIDBlocks[j];
1940 }
1941 }
1942
1943 /*
1944 * Fill in the PKey table. Just as for the GID tables above, the
1945 * only access to the Tavor PKey tables is through the firmware's
1946 * MAD_IFC interface. We post as many GetPKeyTable MADs as necessary
1947 * to read in the entire contents of the PKey table (for the specified
1948 * port). Note: The GetPKeyTable command only gets 32 PKeys per
1949 * operation.
1950 */
1951 for (i = 0; i < pi->p_pkey_tbl_sz; i += 32) {
1952 status = tavor_getpkeytable_cmd_post(state, port, i,
1953 TAVOR_SLEEPFLAG_FOR_CONTEXT(), &pkeytable);
1954 if (status != TAVOR_CMD_SUCCESS) {
1955 cmn_err(CE_CONT, "Tavor: GetPKeyTable (port %02d) "
1956 "command failed: %08x\n", port, status);
1957 TNF_PROBE_1(tavor_port_query_getpkeytable_cmd_fail,
2081
2082
2083 /*
2084 * tavor_set_addr_path()
2085 * Context: Can be called from interrupt or base context.
2086 *
2087 * Note: This routine is used for two purposes. It is used to fill in the
2088 * Tavor UDAV fields, and it is used to fill in the address path information
2089 * for QPs. Because the two Tavor structures are similar, common fields can
2090 * be filled in here. Because they are slightly different, however, we pass
2091 * an additional flag to indicate which type is being filled.
2092 */
2093 int
2094 tavor_set_addr_path(tavor_state_t *state, ibt_adds_vect_t *av,
2095 tavor_hw_addr_path_t *path, uint_t type, tavor_qphdl_t qp)
2096 {
2097 uint_t gidtbl_sz;
2098
2099 TAVOR_TNF_ENTER(tavor_set_addr_path);
2100
2101 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*av))
2102 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*path))
2103
2104 path->ml_path = av->av_src_path;
2105 path->rlid = av->av_dlid;
2106 path->sl = av->av_srvl;
2107
2108 /* Port number only valid (in "av_port_num") if this is a UDAV */
2109 if (type == TAVOR_ADDRPATH_UDAV) {
2110 path->portnum = av->av_port_num;
2111 }
2112
2113 /*
2114 * Validate (and fill in) static rate.
2115 *
2116 * The stat_rate_sup is used to decide how to set the rate and
2117 * if it is zero, the driver uses the old interface.
2118 */
2119 if (state->ts_devlim.stat_rate_sup) {
2120 if (av->av_srate == IBT_SRATE_20) {
2121 path->max_stat_rate = 0; /* 4x@DDR injection rate */
2122 } else if (av->av_srate == IBT_SRATE_5) {
2123 path->max_stat_rate = 3; /* 1x@DDR injection rate */
2135 }
2136 } else {
2137 if (av->av_srate == IBT_SRATE_10) {
2138 path->max_stat_rate = 0; /* 4x@SDR injection rate */
2139 } else if (av->av_srate == IBT_SRATE_2) {
2140 path->max_stat_rate = 1; /* 1x@SDR injection rate */
2141 } else if (av->av_srate == IBT_SRATE_NOT_SPECIFIED) {
2142 path->max_stat_rate = 0; /* Max */
2143 } else {
2144 TNF_PROBE_1(tavor_set_addr_path_inv_srate_fail,
2145 TAVOR_TNF_ERROR, "", tnf_uint, srate, av->av_srate);
2146 TAVOR_TNF_EXIT(tavor_set_addr_path);
2147 return (IBT_STATIC_RATE_INVALID);
2148 }
2149 }
2150
2151 /*
2152 * If this is a QP operation save asoft copy.
2153 */
2154 if (qp) {
2155 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(qp->qp_save_srate))
2156 qp->qp_save_srate = av->av_srate;
2157 }
2158
2159 /* If "grh" flag is set, then check for valid SGID index too */
2160 gidtbl_sz = (1 << state->ts_devlim.log_max_gid);
2161 if ((av->av_send_grh) && (av->av_sgid_ix > gidtbl_sz)) {
2162 TNF_PROBE_1(tavor_set_addr_path_inv_sgid_ix_fail,
2163 TAVOR_TNF_ERROR, "", tnf_uint, sgid_ix, av->av_sgid_ix);
2164 TAVOR_TNF_EXIT(tavor_set_addr_path);
2165 return (IBT_SGID_INVALID);
2166 }
2167
2168 /*
2169 * Fill in all "global" values regardless of the value in the GRH
2170 * flag. Because "grh" is not set unless "av_send_grh" is set, the
2171 * hardware will ignore the other "global" values as necessary. Note:
2172 * SW does this here to enable later query operations to return
2173 * exactly the same params that were passed when the addr path was
2174 * last written.
2175 */
2212 }
2213
2214
2215 /*
2216 * tavor_get_addr_path()
2217 * Context: Can be called from interrupt or base context.
2218 *
2219 * Note: Just like tavor_set_addr_path() above, this routine is used for two
2220 * purposes. It is used to read in the Tavor UDAV fields, and it is used to
2221 * read in the address path information for QPs. Because the two Tavor
2222 * structures are similar, common fields can be read in here. But because
2223 * they are slightly different, we pass an additional flag to indicate which
2224 * type is being read.
2225 */
2226 void
2227 tavor_get_addr_path(tavor_state_t *state, tavor_hw_addr_path_t *path,
2228 ibt_adds_vect_t *av, uint_t type, tavor_qphdl_t qp)
2229 {
2230 uint_t gidtbl_sz;
2231
2232 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*path))
2233 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*av))
2234
2235 av->av_src_path = path->ml_path;
2236 av->av_port_num = path->portnum;
2237 av->av_dlid = path->rlid;
2238 av->av_srvl = path->sl;
2239
2240 /*
2241 * Set "av_ipd" value from max_stat_rate.
2242 */
2243 if (qp) {
2244 /*
2245 * If a QP operation use the soft copy
2246 */
2247 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(qp->qp_save_srate))
2248 av->av_srate = qp->qp_save_srate;
2249 } else {
2250 /*
2251 * The stat_rate_sup is used to decide how the srate value is
2252 * set and
2253 * if it is zero, the driver uses the old interface.
2254 */
2255 if (state->ts_devlim.stat_rate_sup) {
2256 if (path->max_stat_rate == 0) {
2257 av->av_srate = IBT_SRATE_20; /* 4x@DDR rate */
2258 } else if (path->max_stat_rate == 1) {
2259 av->av_srate = IBT_SRATE_2; /* 1x@SDR rate */
2260 } else if (path->max_stat_rate == 2) {
2261 av->av_srate = IBT_SRATE_10; /* 4x@SDR rate */
2262 } else if (path->max_stat_rate == 3) {
2263 av->av_srate = IBT_SRATE_5; /* 1xDDR rate */
2264 }
2265 } else {
2266 if (path->max_stat_rate == 0) {
2267 av->av_srate = IBT_SRATE_10; /* 4x@SDR rate */
2334 }
2335 }
2336
2337
2338 /*
2339 * tavor_queue_alloc()
2340 * Context: Can be called from interrupt or base context.
2341 */
2342 int
2343 tavor_queue_alloc(tavor_state_t *state, tavor_qalloc_info_t *qa_info,
2344 uint_t sleepflag)
2345 {
2346 ddi_dma_attr_t dma_attr;
2347 int (*callback)(caddr_t);
2348 uint64_t realsize, alloc_mask;
2349 uint_t dma_xfer_mode, type;
2350 int flag, status;
2351
2352 TAVOR_TNF_ENTER(tavor_queue_alloc);
2353
2354 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qa_info))
2355
2356 /* Set the callback flag appropriately */
2357 callback = (sleepflag == TAVOR_SLEEP) ? DDI_DMA_SLEEP :
2358 DDI_DMA_DONTWAIT;
2359
2360 /*
2361 * Initialize many of the default DMA attributes. Then set additional
2362 * alignment restrictions as necessary for the queue memory. Also
2363 * respect the configured value for IOMMU bypass
2364 */
2365 tavor_dma_attr_init(&dma_attr);
2366 dma_attr.dma_attr_align = qa_info->qa_bind_align;
2367 type = state->ts_cfg_profile->cp_iommu_bypass;
2368 if (type == TAVOR_BINDMEM_BYPASS) {
2369 dma_attr.dma_attr_flags = DDI_DMA_FORCE_PHYSICAL;
2370 }
2371
2372 /* Allocate a DMA handle */
2373 status = ddi_dma_alloc_handle(state->ts_dip, &dma_attr, callback, NULL,
2374 &qa_info->qa_dmahdl);
2375 if (status != DDI_SUCCESS) {
2490 */
2491 if (qa_info->qa_bind_align != qa_info->qa_alloc_align) {
2492 qa_info->qa_buf_aligned = (uint32_t *)(uintptr_t)(((uintptr_t)
2493 qa_info->qa_buf_aligned + alloc_mask) & ~alloc_mask);
2494 }
2495
2496 TAVOR_TNF_EXIT(tavor_queue_alloc);
2497 return (DDI_SUCCESS);
2498 }
2499
2500
2501 /*
2502 * tavor_queue_free()
2503 * Context: Can be called from interrupt or base context.
2504 */
2505 void
2506 tavor_queue_free(tavor_state_t *state, tavor_qalloc_info_t *qa_info)
2507 {
2508 TAVOR_TNF_ENTER(tavor_queue_free);
2509
2510 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qa_info))
2511
2512 /*
2513 * Depending on how (i.e. from where) we allocated the memory for
2514 * this queue, we choose the appropriate method for releasing the
2515 * resources.
2516 */
2517 if (qa_info->qa_location == TAVOR_QUEUE_LOCATION_NORMAL) {
2518
2519 ddi_dma_mem_free(&qa_info->qa_acchdl);
2520
2521 } else if (qa_info->qa_location == TAVOR_QUEUE_LOCATION_USERLAND) {
2522
2523 ddi_umem_free(qa_info->qa_umemcookie);
2524
2525 } else { /* TAVOR_QUEUE_LOCATION_INDDR */
2526
2527 vmem_xfree(state->ts_ddrvmem, qa_info->qa_buf_real,
2528 qa_info->qa_buf_realsz);
2529 }
2530
2531 /* Always free the dma handle */
2532 ddi_dma_free_handle(&qa_info->qa_dmahdl);
2533
2534 TAVOR_TNF_EXIT(tavor_queue_free);
2535 }
2536
2537
2538 /*
2539 * tavor_dmaattr_get()
2540 * Context: Can be called from interrupt or base context.
2541 */
2542 void
2543 tavor_dma_attr_init(ddi_dma_attr_t *dma_attr)
2544 {
2545 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dma_attr))
2546
2547 dma_attr->dma_attr_version = DMA_ATTR_V0;
2548 dma_attr->dma_attr_addr_lo = 0;
2549 dma_attr->dma_attr_addr_hi = 0xFFFFFFFFFFFFFFFFull;
2550 dma_attr->dma_attr_count_max = 0xFFFFFFFFFFFFFFFFull;
2551 dma_attr->dma_attr_align = 1;
2552 dma_attr->dma_attr_burstsizes = 0x3FF;
2553 dma_attr->dma_attr_minxfer = 1;
2554 dma_attr->dma_attr_maxxfer = 0xFFFFFFFFFFFFFFFFull;
2555 dma_attr->dma_attr_seg = 0xFFFFFFFFFFFFFFFFull;
2556 dma_attr->dma_attr_sgllen = 0x7FFFFFFF;
2557 dma_attr->dma_attr_granular = 1;
2558 dma_attr->dma_attr_flags = 0;
2559 }
|
108 */
109 status = tavor_rsrc_alloc(state, TAVOR_UDAV, 1, sleepflag, &udav);
110 if (status != DDI_SUCCESS) {
111 /* Set "status" and "errormsg" and goto failure */
112 TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed UDAV");
113 goto ahalloc_fail;
114 }
115
116 /*
117 * Allocate the software structure for tracking the address handle
118 * (i.e. the Tavor Address Handle struct). If we fail here, we must
119 * undo the previous resource allocation.
120 */
121 status = tavor_rsrc_alloc(state, TAVOR_AHHDL, 1, sleepflag, &rsrc);
122 if (status != DDI_SUCCESS) {
123 /* Set "status" and "errormsg" and goto failure */
124 TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed AH handler");
125 goto ahalloc_fail1;
126 }
127 ah = (tavor_ahhdl_t)rsrc->tr_addr;
128
129 /* Increment the reference count on the protection domain (PD) */
130 tavor_pd_refcnt_inc(pd);
131
132 /*
133 * Fill in the UDAV entry. Note: We are only filling in a temporary
134 * copy here, which we will later copy into the actual entry in
135 * Tavor DDR memory. This starts be zeroing out the temporary copy
136 * and then calling tavor_set_addr_path() to fill in the common
137 * portions that can be pulled from the "ibt_adds_vect_t" passed in
138 */
139 bzero(&udav_entry, sizeof (tavor_hw_udav_t));
140 status = tavor_set_addr_path(state, attr_p,
141 (tavor_hw_addr_path_t *)&udav_entry, TAVOR_ADDRPATH_UDAV, NULL);
142 if (status != DDI_SUCCESS) {
143 tavor_pd_refcnt_dec(pd);
144 tavor_rsrc_free(state, &rsrc);
145 tavor_rsrc_free(state, &udav);
146 /* Set "status" and "errormsg" and goto failure */
147 TAVOR_TNF_FAIL(status, "failed in tavor_set_addr_path");
246 tavor_rsrc_t *udav, *rsrc;
247 tavor_pdhdl_t pd;
248 tavor_mrhdl_t mr;
249 tavor_ahhdl_t ah;
250 int status;
251
252 TAVOR_TNF_ENTER(tavor_ah_free);
253
254 /*
255 * Pull all the necessary information from the Tavor Address Handle
256 * struct. This is necessary here because the resource for the
257 * AH is going to be freed up as part of this operation.
258 */
259 ah = *ahhdl;
260 mutex_enter(&ah->ah_lock);
261 udav = ah->ah_udavrsrcp;
262 rsrc = ah->ah_rsrcp;
263 pd = ah->ah_pdhdl;
264 mr = ah->ah_mrhdl;
265 mutex_exit(&ah->ah_lock);
266
267 /*
268 * Deregister the memory for the UDAV. If this fails for any reason,
269 * then it is an indication that something (either in HW or SW) has
270 * gone seriously wrong. So we print a warning message and return
271 * failure.
272 */
273 status = tavor_mr_deregister(state, &mr, TAVOR_MR_DEREG_ALL,
274 sleepflag);
275 if (status != DDI_SUCCESS) {
276 TNF_PROBE_0(tavor_ah_free_dereg_mr_fail, TAVOR_TNF_ERROR, "");
277 TAVOR_TNF_EXIT(tavor_ah_free);
278 return (ibc_get_ci_failure(0));
279 }
280
281 /*
282 * Write zero to the first 64-bit word in the UDAV entry. As
283 * described above (in tavor_ah_alloc), the PD number is stored in
284 * the first 64-bits of each UDAV and setting this to zero is
285 * guaranteed to invalidate the entry.
307
308
309 /*
310 * tavor_ah_query()
311 * Context: Can be called from interrupt or base context.
312 */
313 /* ARGSUSED */
314 int
315 tavor_ah_query(tavor_state_t *state, tavor_ahhdl_t ah, tavor_pdhdl_t *pd,
316 ibt_adds_vect_t *attr_p)
317 {
318 tavor_hw_udav_t udav_entry;
319 tavor_rsrc_t *udav;
320 uint64_t data;
321 uint32_t size;
322 int i;
323
324 TAVOR_TNF_ENTER(tavor_ah_query);
325
326 mutex_enter(&ah->ah_lock);
327
328 /*
329 * Pull all the necessary information from the Tavor Address Handle
330 * structure
331 */
332 udav = ah->ah_udavrsrcp;
333 *pd = ah->ah_pdhdl;
334
335 /*
336 * Copy the UDAV entry into the temporary copy. Here we copy all
337 * the information from the UDAV entry in DDR memory into the
338 * temporary UDAV. Note: We don't need to sync the UDAV for
339 * reading by software because Tavor HW never modifies the entry.
340 */
341 size = sizeof (tavor_hw_udav_t) >> 3;
342 for (i = 0; i < size; i++) {
343 data = ddi_get64(udav->tr_acchdl,
344 ((uint64_t *)udav->tr_addr + i));
345 ((uint64_t *)&udav_entry)[i] = data;
346 }
497 mutex_exit(&ah->ah_lock);
498 TAVOR_TNF_EXIT(tavor_ah_modify);
499 return (DDI_SUCCESS);
500 }
501
502
503 /*
504 * tavor_udav_sync()
505 * Context: Can be called from interrupt or base context.
506 */
507 /* ARGSUSED */
508 static void
509 tavor_udav_sync(tavor_ahhdl_t ah, tavor_hw_udav_t *udav, uint_t flag)
510 {
511 ddi_dma_handle_t dmahdl;
512 off_t offset;
513 int status;
514
515 TAVOR_TNF_ENTER(tavor_udav_sync);
516
517 /* Determine if AH needs to be synced or not */
518 if (ah->ah_sync == 0) {
519 TAVOR_TNF_EXIT(tavor_udav_sync);
520 return;
521 }
522
523 /* Get the DMA handle from AH handle */
524 dmahdl = ah->ah_mrhdl->mr_bindinfo.bi_dmahdl;
525
526 /* Calculate offset into address handle */
527 offset = (off_t)0;
528 status = ddi_dma_sync(dmahdl, offset, sizeof (tavor_hw_udav_t), flag);
529 if (status != DDI_SUCCESS) {
530 TNF_PROBE_0(tavor_udav_sync_getnextentry_fail,
531 TAVOR_TNF_ERROR, "");
532 TAVOR_TNF_EXIT(tavor_udav_sync);
533 return;
534 }
535
536 TAVOR_TNF_EXIT(tavor_udav_sync);
1710 tavor_rsrc_t *rsrc;
1711 tavor_pdhdl_t pd;
1712 int status;
1713
1714 TAVOR_TNF_ENTER(tavor_pd_alloc);
1715
1716 /*
1717 * Allocate the software structure for tracking the protection domain
1718 * (i.e. the Tavor Protection Domain handle). By default each PD
1719 * structure will have a unique PD number assigned to it. All that
1720 * is necessary is for software to initialize the PD reference count
1721 * (to zero) and return success.
1722 */
1723 status = tavor_rsrc_alloc(state, TAVOR_PDHDL, 1, sleepflag, &rsrc);
1724 if (status != DDI_SUCCESS) {
1725 TNF_PROBE_0(tavor_pd_alloc_rsrcalloc_fail, TAVOR_TNF_ERROR, "");
1726 TAVOR_TNF_EXIT(tavor_pd_alloc);
1727 return (IBT_INSUFF_RESOURCE);
1728 }
1729 pd = (tavor_pdhdl_t)rsrc->tr_addr;
1730
1731 pd->pd_refcnt = 0;
1732 *pdhdl = pd;
1733
1734 TAVOR_TNF_EXIT(tavor_pd_alloc);
1735 return (DDI_SUCCESS);
1736 }
1737
1738
1739 /*
1740 * tavor_pd_free()
1741 * Context: Can be called only from user or kernel context.
1742 */
1743 int
1744 tavor_pd_free(tavor_state_t *state, tavor_pdhdl_t *pdhdl)
1745 {
1746 tavor_rsrc_t *rsrc;
1747 tavor_pdhdl_t pd;
1748
1749 TAVOR_TNF_ENTER(tavor_pd_free);
1750
1751 /*
1752 * Pull all the necessary information from the Tavor Protection Domain
1753 * handle. This is necessary here because the resource for the
1754 * PD is going to be freed up as part of this operation.
1755 */
1756 pd = *pdhdl;
1757 rsrc = pd->pd_rsrcp;
1758
1759 /*
1760 * Check the PD reference count. If the reference count is non-zero,
1761 * then it means that this protection domain is still referenced by
1762 * some memory region, queue pair, address handle, or other IB object
1763 * If it is non-zero, then return an error. Otherwise, free the
1764 * Tavor resource and return success.
1765 */
1766 if (pd->pd_refcnt != 0) {
1767 TNF_PROBE_1(tavor_pd_free_refcnt_fail, TAVOR_TNF_ERROR, "",
1768 tnf_int, refcnt, pd->pd_refcnt);
1769 TAVOR_TNF_EXIT(tavor_pd_free);
1770 return (IBT_PD_IN_USE);
1771 }
1772
1773 /* Free the Tavor Protection Domain handle */
1774 tavor_rsrc_free(state, &rsrc);
1775
1776 /* Set the pdhdl pointer to NULL and return success */
1814
1815 }
1816
1817
1818 /*
1819 * tavor_port_query()
1820 * Context: Can be called only from user or kernel context.
1821 */
1822 int
1823 tavor_port_query(tavor_state_t *state, uint_t port, ibt_hca_portinfo_t *pi)
1824 {
1825 sm_portinfo_t portinfo;
1826 sm_guidinfo_t guidinfo;
1827 sm_pkey_table_t pkeytable;
1828 ib_gid_t *sgid;
1829 uint_t sgid_max, pkey_max, tbl_size;
1830 int i, j, indx, status;
1831
1832 TAVOR_TNF_ENTER(tavor_port_query);
1833
1834 /* Validate that specified port number is legal */
1835 if (!tavor_portnum_is_valid(state, port)) {
1836 TNF_PROBE_1(tavor_port_query_inv_portnum_fail,
1837 TAVOR_TNF_ERROR, "", tnf_uint, port, port);
1838 TAVOR_TNF_EXIT(tavor_port_query);
1839 return (IBT_HCA_PORT_INVALID);
1840 }
1841
1842 /*
1843 * We use the Tavor MAD_IFC command to post a GetPortInfo MAD
1844 * to the firmware (for the specified port number). This returns
1845 * a full PortInfo MAD (in "portinfo") which we subsequently
1846 * parse to fill in the "ibt_hca_portinfo_t" structure returned
1847 * to the IBTF.
1848 */
1849 status = tavor_getportinfo_cmd_post(state, port,
1850 TAVOR_SLEEPFLAG_FOR_CONTEXT(), &portinfo);
1851 if (status != TAVOR_CMD_SUCCESS) {
1852 cmn_err(CE_CONT, "Tavor: GetPortInfo (port %02d) command "
1853 "failed: %08x\n", port, status);
1908 * GUIDs are then appended to the GID prefix for the port (from the
1909 * GetPortInfo above) to form the entire SGID table.
1910 */
1911 for (i = 0; i < pi->p_sgid_tbl_sz; i += 8) {
1912 status = tavor_getguidinfo_cmd_post(state, port, i >> 3,
1913 TAVOR_SLEEPFLAG_FOR_CONTEXT(), &guidinfo);
1914 if (status != TAVOR_CMD_SUCCESS) {
1915 cmn_err(CE_CONT, "Tavor: GetGUIDInfo (port %02d) "
1916 "command failed: %08x\n", port, status);
1917 TNF_PROBE_1(tavor_port_query_getguidinfo_cmd_fail,
1918 TAVOR_TNF_ERROR, "", tnf_uint, cmd_status, status);
1919 TAVOR_TNF_EXIT(tavor_port_query);
1920 return (ibc_get_ci_failure(0));
1921 }
1922
1923 /* Figure out how many of the entries are valid */
1924 sgid_max = min((pi->p_sgid_tbl_sz - i), 8);
1925 for (j = 0; j < sgid_max; j++) {
1926 indx = (i + j);
1927 sgid = &pi->p_sgid_tbl[indx];
1928 sgid->gid_prefix = portinfo.GidPrefix;
1929 sgid->gid_guid = guidinfo.GUIDBlocks[j];
1930 }
1931 }
1932
1933 /*
1934 * Fill in the PKey table. Just as for the GID tables above, the
1935 * only access to the Tavor PKey tables is through the firmware's
1936 * MAD_IFC interface. We post as many GetPKeyTable MADs as necessary
1937 * to read in the entire contents of the PKey table (for the specified
1938 * port). Note: The GetPKeyTable command only gets 32 PKeys per
1939 * operation.
1940 */
1941 for (i = 0; i < pi->p_pkey_tbl_sz; i += 32) {
1942 status = tavor_getpkeytable_cmd_post(state, port, i,
1943 TAVOR_SLEEPFLAG_FOR_CONTEXT(), &pkeytable);
1944 if (status != TAVOR_CMD_SUCCESS) {
1945 cmn_err(CE_CONT, "Tavor: GetPKeyTable (port %02d) "
1946 "command failed: %08x\n", port, status);
1947 TNF_PROBE_1(tavor_port_query_getpkeytable_cmd_fail,
2071
2072
2073 /*
2074 * tavor_set_addr_path()
2075 * Context: Can be called from interrupt or base context.
2076 *
2077 * Note: This routine is used for two purposes. It is used to fill in the
2078 * Tavor UDAV fields, and it is used to fill in the address path information
2079 * for QPs. Because the two Tavor structures are similar, common fields can
2080 * be filled in here. Because they are slightly different, however, we pass
2081 * an additional flag to indicate which type is being filled.
2082 */
2083 int
2084 tavor_set_addr_path(tavor_state_t *state, ibt_adds_vect_t *av,
2085 tavor_hw_addr_path_t *path, uint_t type, tavor_qphdl_t qp)
2086 {
2087 uint_t gidtbl_sz;
2088
2089 TAVOR_TNF_ENTER(tavor_set_addr_path);
2090
2091 path->ml_path = av->av_src_path;
2092 path->rlid = av->av_dlid;
2093 path->sl = av->av_srvl;
2094
2095 /* Port number only valid (in "av_port_num") if this is a UDAV */
2096 if (type == TAVOR_ADDRPATH_UDAV) {
2097 path->portnum = av->av_port_num;
2098 }
2099
2100 /*
2101 * Validate (and fill in) static rate.
2102 *
2103 * The stat_rate_sup is used to decide how to set the rate and
2104 * if it is zero, the driver uses the old interface.
2105 */
2106 if (state->ts_devlim.stat_rate_sup) {
2107 if (av->av_srate == IBT_SRATE_20) {
2108 path->max_stat_rate = 0; /* 4x@DDR injection rate */
2109 } else if (av->av_srate == IBT_SRATE_5) {
2110 path->max_stat_rate = 3; /* 1x@DDR injection rate */
2122 }
2123 } else {
2124 if (av->av_srate == IBT_SRATE_10) {
2125 path->max_stat_rate = 0; /* 4x@SDR injection rate */
2126 } else if (av->av_srate == IBT_SRATE_2) {
2127 path->max_stat_rate = 1; /* 1x@SDR injection rate */
2128 } else if (av->av_srate == IBT_SRATE_NOT_SPECIFIED) {
2129 path->max_stat_rate = 0; /* Max */
2130 } else {
2131 TNF_PROBE_1(tavor_set_addr_path_inv_srate_fail,
2132 TAVOR_TNF_ERROR, "", tnf_uint, srate, av->av_srate);
2133 TAVOR_TNF_EXIT(tavor_set_addr_path);
2134 return (IBT_STATIC_RATE_INVALID);
2135 }
2136 }
2137
2138 /*
2139 * If this is a QP operation save asoft copy.
2140 */
2141 if (qp) {
2142 qp->qp_save_srate = av->av_srate;
2143 }
2144
2145 /* If "grh" flag is set, then check for valid SGID index too */
2146 gidtbl_sz = (1 << state->ts_devlim.log_max_gid);
2147 if ((av->av_send_grh) && (av->av_sgid_ix > gidtbl_sz)) {
2148 TNF_PROBE_1(tavor_set_addr_path_inv_sgid_ix_fail,
2149 TAVOR_TNF_ERROR, "", tnf_uint, sgid_ix, av->av_sgid_ix);
2150 TAVOR_TNF_EXIT(tavor_set_addr_path);
2151 return (IBT_SGID_INVALID);
2152 }
2153
2154 /*
2155 * Fill in all "global" values regardless of the value in the GRH
2156 * flag. Because "grh" is not set unless "av_send_grh" is set, the
2157 * hardware will ignore the other "global" values as necessary. Note:
2158 * SW does this here to enable later query operations to return
2159 * exactly the same params that were passed when the addr path was
2160 * last written.
2161 */
2198 }
2199
2200
2201 /*
2202 * tavor_get_addr_path()
2203 * Context: Can be called from interrupt or base context.
2204 *
2205 * Note: Just like tavor_set_addr_path() above, this routine is used for two
2206 * purposes. It is used to read in the Tavor UDAV fields, and it is used to
2207 * read in the address path information for QPs. Because the two Tavor
2208 * structures are similar, common fields can be read in here. But because
2209 * they are slightly different, we pass an additional flag to indicate which
2210 * type is being read.
2211 */
2212 void
2213 tavor_get_addr_path(tavor_state_t *state, tavor_hw_addr_path_t *path,
2214 ibt_adds_vect_t *av, uint_t type, tavor_qphdl_t qp)
2215 {
2216 uint_t gidtbl_sz;
2217
2218 av->av_src_path = path->ml_path;
2219 av->av_port_num = path->portnum;
2220 av->av_dlid = path->rlid;
2221 av->av_srvl = path->sl;
2222
2223 /*
2224 * Set "av_ipd" value from max_stat_rate.
2225 */
2226 if (qp) {
2227 /*
2228 * If a QP operation use the soft copy
2229 */
2230 av->av_srate = qp->qp_save_srate;
2231 } else {
2232 /*
2233 * The stat_rate_sup is used to decide how the srate value is
2234 * set and
2235 * if it is zero, the driver uses the old interface.
2236 */
2237 if (state->ts_devlim.stat_rate_sup) {
2238 if (path->max_stat_rate == 0) {
2239 av->av_srate = IBT_SRATE_20; /* 4x@DDR rate */
2240 } else if (path->max_stat_rate == 1) {
2241 av->av_srate = IBT_SRATE_2; /* 1x@SDR rate */
2242 } else if (path->max_stat_rate == 2) {
2243 av->av_srate = IBT_SRATE_10; /* 4x@SDR rate */
2244 } else if (path->max_stat_rate == 3) {
2245 av->av_srate = IBT_SRATE_5; /* 1xDDR rate */
2246 }
2247 } else {
2248 if (path->max_stat_rate == 0) {
2249 av->av_srate = IBT_SRATE_10; /* 4x@SDR rate */
2316 }
2317 }
2318
2319
2320 /*
2321 * tavor_queue_alloc()
2322 * Context: Can be called from interrupt or base context.
2323 */
2324 int
2325 tavor_queue_alloc(tavor_state_t *state, tavor_qalloc_info_t *qa_info,
2326 uint_t sleepflag)
2327 {
2328 ddi_dma_attr_t dma_attr;
2329 int (*callback)(caddr_t);
2330 uint64_t realsize, alloc_mask;
2331 uint_t dma_xfer_mode, type;
2332 int flag, status;
2333
2334 TAVOR_TNF_ENTER(tavor_queue_alloc);
2335
2336 /* Set the callback flag appropriately */
2337 callback = (sleepflag == TAVOR_SLEEP) ? DDI_DMA_SLEEP :
2338 DDI_DMA_DONTWAIT;
2339
2340 /*
2341 * Initialize many of the default DMA attributes. Then set additional
2342 * alignment restrictions as necessary for the queue memory. Also
2343 * respect the configured value for IOMMU bypass
2344 */
2345 tavor_dma_attr_init(&dma_attr);
2346 dma_attr.dma_attr_align = qa_info->qa_bind_align;
2347 type = state->ts_cfg_profile->cp_iommu_bypass;
2348 if (type == TAVOR_BINDMEM_BYPASS) {
2349 dma_attr.dma_attr_flags = DDI_DMA_FORCE_PHYSICAL;
2350 }
2351
2352 /* Allocate a DMA handle */
2353 status = ddi_dma_alloc_handle(state->ts_dip, &dma_attr, callback, NULL,
2354 &qa_info->qa_dmahdl);
2355 if (status != DDI_SUCCESS) {
2470 */
2471 if (qa_info->qa_bind_align != qa_info->qa_alloc_align) {
2472 qa_info->qa_buf_aligned = (uint32_t *)(uintptr_t)(((uintptr_t)
2473 qa_info->qa_buf_aligned + alloc_mask) & ~alloc_mask);
2474 }
2475
2476 TAVOR_TNF_EXIT(tavor_queue_alloc);
2477 return (DDI_SUCCESS);
2478 }
2479
2480
2481 /*
2482 * tavor_queue_free()
2483 * Context: Can be called from interrupt or base context.
2484 */
2485 void
2486 tavor_queue_free(tavor_state_t *state, tavor_qalloc_info_t *qa_info)
2487 {
2488 TAVOR_TNF_ENTER(tavor_queue_free);
2489
2490 /*
2491 * Depending on how (i.e. from where) we allocated the memory for
2492 * this queue, we choose the appropriate method for releasing the
2493 * resources.
2494 */
2495 if (qa_info->qa_location == TAVOR_QUEUE_LOCATION_NORMAL) {
2496
2497 ddi_dma_mem_free(&qa_info->qa_acchdl);
2498
2499 } else if (qa_info->qa_location == TAVOR_QUEUE_LOCATION_USERLAND) {
2500
2501 ddi_umem_free(qa_info->qa_umemcookie);
2502
2503 } else { /* TAVOR_QUEUE_LOCATION_INDDR */
2504
2505 vmem_xfree(state->ts_ddrvmem, qa_info->qa_buf_real,
2506 qa_info->qa_buf_realsz);
2507 }
2508
2509 /* Always free the dma handle */
2510 ddi_dma_free_handle(&qa_info->qa_dmahdl);
2511
2512 TAVOR_TNF_EXIT(tavor_queue_free);
2513 }
2514
2515
2516 /*
2517 * tavor_dmaattr_get()
2518 * Context: Can be called from interrupt or base context.
2519 */
2520 void
2521 tavor_dma_attr_init(ddi_dma_attr_t *dma_attr)
2522 {
2523 dma_attr->dma_attr_version = DMA_ATTR_V0;
2524 dma_attr->dma_attr_addr_lo = 0;
2525 dma_attr->dma_attr_addr_hi = 0xFFFFFFFFFFFFFFFFull;
2526 dma_attr->dma_attr_count_max = 0xFFFFFFFFFFFFFFFFull;
2527 dma_attr->dma_attr_align = 1;
2528 dma_attr->dma_attr_burstsizes = 0x3FF;
2529 dma_attr->dma_attr_minxfer = 1;
2530 dma_attr->dma_attr_maxxfer = 0xFFFFFFFFFFFFFFFFull;
2531 dma_attr->dma_attr_seg = 0xFFFFFFFFFFFFFFFFull;
2532 dma_attr->dma_attr_sgllen = 0x7FFFFFFF;
2533 dma_attr->dma_attr_granular = 1;
2534 dma_attr->dma_attr_flags = 0;
2535 }
|