Print this page 
8368 remove warlock leftovers from usr/src/uts
Split Close
Expand all
Collapse all
          --- old/usr/src/uts/common/io/ib/adapters/tavor/tavor_ci.c
          +++ new/usr/src/uts/common/io/ib/adapters/tavor/tavor_ci.c
↓ open down ↓ 820 lines elided ↑ open up ↑ 
 821  821      ibt_qp_type_t type, ibt_qp_alloc_attr_t *attr_p,
 822  822      ibt_chan_sizes_t *queue_sizes_p, ib_qpn_t *qpn, ibc_qp_hdl_t *qp_p)
 823  823  {
 824  824          tavor_state_t           *state;
 825  825          tavor_qp_info_t         qpinfo;
 826  826          tavor_qp_options_t      op;
 827  827          int                     status;
 828  828  
 829  829          TAVOR_TNF_ENTER(tavor_ci_alloc_qp);
 830  830  
 831      -        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attr_p))
 832      -        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*queue_sizes_p))
 833      -
 834  831          /* Check for valid HCA handle */
 835  832          if (hca == NULL) {
 836  833                  TNF_PROBE_0(tavor_ci_alloc_qp_invhca_fail,
 837  834                      TAVOR_TNF_ERROR, "");
 838  835                  TAVOR_TNF_EXIT(tavor_ci_alloc_qp);
 839  836                  return (IBT_HCA_HDL_INVALID);
 840  837          }
 841  838  
 842  839          /* Grab the Tavor softstate pointer */
 843  840          state = (tavor_state_t *)hca;
↓ open down ↓ 32 lines elided ↑ open up ↑ 
 876  873      ibt_qp_alloc_attr_t *attr_p, ibt_chan_sizes_t *queue_sizes_p,
 877  874      ibc_qp_hdl_t *qp_p)
 878  875  {
 879  876          tavor_state_t           *state;
 880  877          tavor_qp_info_t         qpinfo;
 881  878          tavor_qp_options_t      op;
 882  879          int                     status;
 883  880  
 884  881          TAVOR_TNF_ENTER(tavor_ci_alloc_special_qp);
 885  882  
 886      -        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attr_p))
 887      -        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*queue_sizes_p))
 888      -
 889  883          /* Check for valid HCA handle */
 890  884          if (hca == NULL) {
 891  885                  TNF_PROBE_0(tavor_ci_alloc_special_qp_invhca_fail,
 892  886                      TAVOR_TNF_ERROR, "");
 893  887                  TAVOR_TNF_EXIT(tavor_ci_alloc_special_qp);
 894  888                  return (IBT_HCA_HDL_INVALID);
 895  889          }
 896  890  
 897  891          /* Grab the Tavor softstate pointer */
 898  892          state = (tavor_state_t *)hca;
↓ open down ↓ 569 lines elided ↑ open up ↑ 
1468 1462      ibt_mr_desc_t *mr_desc)
1469 1463  {
1470 1464          tavor_mr_options_t      op;
1471 1465          tavor_state_t           *state;
1472 1466          tavor_pdhdl_t           pdhdl;
1473 1467          tavor_mrhdl_t           mrhdl;
1474 1468          int                     status;
1475 1469  
1476 1470          TAVOR_TNF_ENTER(tavor_ci_register_mr);
1477 1471  
1478      -        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr_desc))
1479      -
1480 1472          ASSERT(mr_attr != NULL);
1481 1473          ASSERT(mr_p != NULL);
1482 1474          ASSERT(mr_desc != NULL);
1483 1475  
1484 1476          /* Check for valid HCA handle */
1485 1477          if (hca == NULL) {
1486 1478                  TNF_PROBE_0(tavor_ci_register_mr_invhca_fail,
1487 1479                      TAVOR_TNF_ERROR, "");
1488 1480                  TAVOR_TNF_EXIT(tavor_ci_register_mr);
1489 1481                  return (IBT_HCA_HDL_INVALID);
↓ open down ↓ 28 lines elided ↑ open up ↑ 
1518 1510          op.mro_bind_type   = state->ts_cfg_profile->cp_iommu_bypass;
1519 1511          op.mro_bind_dmahdl = NULL;
1520 1512          op.mro_bind_override_addr = 0;
1521 1513          status = tavor_mr_register(state, pdhdl, mr_attr, &mrhdl, &op);
1522 1514          if (status != DDI_SUCCESS) {
1523 1515                  TNF_PROBE_1(tavor_ci_register_mr_fail, TAVOR_TNF_ERROR, "",
1524 1516                      tnf_uint, status, status);
1525 1517                  TAVOR_TNF_EXIT(tavor_ci_register_mr);
1526 1518                  return (status);
1527 1519          }
1528      -        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mrhdl))
1529 1520  
1530 1521          /* Fill in the mr_desc structure */
1531 1522          mr_desc->md_vaddr = mrhdl->mr_bindinfo.bi_addr;
1532 1523          mr_desc->md_lkey  = mrhdl->mr_lkey;
1533 1524          /* Only set RKey if remote access was requested */
1534 1525          if ((mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC) ||
1535 1526              (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
1536 1527              (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_READ)) {
1537 1528                  mr_desc->md_rkey = mrhdl->mr_rkey;
1538 1529          }
↓ open down ↓ 26 lines elided ↑ open up ↑ 
1565 1556  {
1566 1557          tavor_mr_options_t      op;
1567 1558          tavor_state_t           *state;
1568 1559          tavor_pdhdl_t           pdhdl;
1569 1560          tavor_mrhdl_t           mrhdl;
1570 1561          int                     status;
1571 1562          ibt_mr_flags_t          flags = attrp->mr_flags;
1572 1563  
1573 1564          TAVOR_TNF_ENTER(tavor_ci_register_buf);
1574 1565  
1575      -        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr_desc))
1576      -
1577 1566          ASSERT(mr_p != NULL);
1578 1567          ASSERT(mr_desc != NULL);
1579 1568  
1580 1569          /* Check for valid HCA handle */
1581 1570          if (hca == NULL) {
1582 1571                  TNF_PROBE_0(tavor_ci_register_buf_invhca_fail,
1583 1572                      TAVOR_TNF_ERROR, "");
1584 1573                  TAVOR_TNF_EXIT(tavor_ci_register_buf);
1585 1574                  return (IBT_HCA_HDL_INVALID);
1586 1575          }
↓ open down ↓ 27 lines elided ↑ open up ↑ 
1614 1603          op.mro_bind_type   = state->ts_cfg_profile->cp_iommu_bypass;
1615 1604          op.mro_bind_dmahdl = NULL;
1616 1605          op.mro_bind_override_addr = 0;
1617 1606          status = tavor_mr_register_buf(state, pdhdl, attrp, buf, &mrhdl, &op);
1618 1607          if (status != DDI_SUCCESS) {
1619 1608                  TNF_PROBE_1(tavor_ci_register_mr_fail, TAVOR_TNF_ERROR, "",
1620 1609                      tnf_uint, status, status);
1621 1610                  TAVOR_TNF_EXIT(tavor_ci_register_mr);
1622 1611                  return (status);
1623 1612          }
1624      -        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mrhdl))
1625 1613  
1626 1614          /* Fill in the mr_desc structure */
1627 1615          mr_desc->md_vaddr = mrhdl->mr_bindinfo.bi_addr;
1628 1616          mr_desc->md_lkey  = mrhdl->mr_lkey;
1629 1617          /* Only set RKey if remote access was requested */
1630 1618          if ((flags & IBT_MR_ENABLE_REMOTE_ATOMIC) ||
1631 1619              (flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
1632 1620              (flags & IBT_MR_ENABLE_REMOTE_READ)) {
1633 1621                  mr_desc->md_rkey = mrhdl->mr_rkey;
1634 1622          }
↓ open down ↓ 126 lines elided ↑ open up ↑ 
1761 1749      ibc_pd_hdl_t pd, ibt_smr_attr_t *mr_attr, void *ibtl_reserved,
1762 1750      ibc_mr_hdl_t *mr_p, ibt_mr_desc_t *mr_desc)
1763 1751  {
1764 1752          tavor_state_t           *state;
1765 1753          tavor_pdhdl_t           pdhdl;
1766 1754          tavor_mrhdl_t           mrhdl, mrhdl_new;
1767 1755          int                     status;
1768 1756  
1769 1757          TAVOR_TNF_ENTER(tavor_ci_register_shared_mr);
1770 1758  
1771      -        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr_desc))
1772      -
1773 1759          ASSERT(mr_attr != NULL);
1774 1760          ASSERT(mr_p != NULL);
1775 1761          ASSERT(mr_desc != NULL);
1776 1762  
1777 1763          /* Check for valid HCA handle */
1778 1764          if (hca == NULL) {
1779 1765                  TNF_PROBE_0(tavor_ci_register_shared_mr_invhca_fail,
1780 1766                      TAVOR_TNF_ERROR, "");
1781 1767                  TAVOR_TNF_EXIT(tavor_ci_register_shared_mr);
1782 1768                  return (IBT_HCA_HDL_INVALID);
↓ open down ↓ 34 lines elided ↑ open up ↑ 
1817 1803  
1818 1804          /* Register the shared memory region */
1819 1805          status = tavor_mr_register_shared(state, mrhdl, pdhdl, mr_attr,
1820 1806              &mrhdl_new);
1821 1807          if (status != DDI_SUCCESS) {
1822 1808                  TNF_PROBE_1(tavor_ci_register_shared_mr_fail, TAVOR_TNF_ERROR,
1823 1809                      "", tnf_uint, status, status);
1824 1810                  TAVOR_TNF_EXIT(tavor_ci_register_shared_mr);
1825 1811                  return (status);
1826 1812          }
1827      -        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mrhdl_new))
1828 1813  
1829 1814          /* Fill in the mr_desc structure */
1830 1815          mr_desc->md_vaddr = mrhdl_new->mr_bindinfo.bi_addr;
1831 1816          mr_desc->md_lkey  = mrhdl_new->mr_lkey;
1832 1817          /* Only set RKey if remote access was requested */
1833 1818          if ((mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC) ||
1834 1819              (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
1835 1820              (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_READ)) {
1836 1821                  mr_desc->md_rkey = mrhdl_new->mr_rkey;
1837 1822          }
↓ open down ↓ 25 lines elided ↑ open up ↑ 
1863 1848      ibt_mr_desc_t *mr_desc)
1864 1849  {
1865 1850          tavor_mr_options_t      op;
1866 1851          tavor_state_t           *state;
1867 1852          tavor_pdhdl_t           pdhdl;
1868 1853          tavor_mrhdl_t           mrhdl, mrhdl_new;
1869 1854          int                     status;
1870 1855  
1871 1856          TAVOR_TNF_ENTER(tavor_ci_reregister_mr);
1872 1857  
1873      -        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr_desc))
1874      -
1875 1858          ASSERT(mr_attr != NULL);
1876 1859          ASSERT(mr_new != NULL);
1877 1860          ASSERT(mr_desc != NULL);
1878 1861  
1879 1862          /* Check for valid HCA handle */
1880 1863          if (hca == NULL) {
1881 1864                  TNF_PROBE_0(tavor_ci_reregister_mr_hca_inv, TAVOR_TNF_ERROR,
1882 1865                      "");
1883 1866                  TAVOR_TNF_EXIT(tavor_ci_reregister_mr);
1884 1867                  return (IBT_HCA_HDL_INVALID);
↓ open down ↓ 15 lines elided ↑ open up ↑ 
1900 1883          /* Reregister the memory region */
1901 1884          op.mro_bind_type = state->ts_cfg_profile->cp_iommu_bypass;
1902 1885          status = tavor_mr_reregister(state, mrhdl, pdhdl, mr_attr,
1903 1886              &mrhdl_new, &op);
1904 1887          if (status != DDI_SUCCESS) {
1905 1888                  TNF_PROBE_1(tavor_ci_reregister_mr_fail, TAVOR_TNF_ERROR, "",
1906 1889                      tnf_uint, status, status);
1907 1890                  TAVOR_TNF_EXIT(tavor_ci_reregister_mr);
1908 1891                  return (status);
1909 1892          }
1910      -        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mrhdl_new))
1911 1893  
1912 1894          /* Fill in the mr_desc structure */
1913 1895          mr_desc->md_vaddr = mrhdl_new->mr_bindinfo.bi_addr;
1914 1896          mr_desc->md_lkey  = mrhdl_new->mr_lkey;
1915 1897          /* Only set RKey if remote access was requested */
1916 1898          if ((mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC) ||
1917 1899              (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
1918 1900              (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_READ)) {
1919 1901                  mr_desc->md_rkey = mrhdl_new->mr_rkey;
1920 1902          }
↓ open down ↓ 26 lines elided ↑ open up ↑ 
1947 1929  {
1948 1930          tavor_mr_options_t      op;
1949 1931          tavor_state_t           *state;
1950 1932          tavor_pdhdl_t           pdhdl;
1951 1933          tavor_mrhdl_t           mrhdl, mrhdl_new;
1952 1934          int                     status;
1953 1935          ibt_mr_flags_t          flags = attrp->mr_flags;
1954 1936  
1955 1937          TAVOR_TNF_ENTER(tavor_ci_reregister_buf);
1956 1938  
1957      -        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr_desc))
1958      -
1959 1939          ASSERT(mr_new != NULL);
1960 1940          ASSERT(mr_desc != NULL);
1961 1941  
1962 1942          /* Check for valid HCA handle */
1963 1943          if (hca == NULL) {
1964 1944                  TNF_PROBE_0(tavor_ci_reregister_buf_hca_inv, TAVOR_TNF_ERROR,
1965 1945                      "");
1966 1946                  TAVOR_TNF_EXIT(tavor_ci_reregister_buf);
1967 1947                  return (IBT_HCA_HDL_INVALID);
1968 1948          }
↓ open down ↓ 14 lines elided ↑ open up ↑ 
1983 1963          /* Reregister the memory region */
1984 1964          op.mro_bind_type = state->ts_cfg_profile->cp_iommu_bypass;
1985 1965          status = tavor_mr_reregister_buf(state, mrhdl, pdhdl, attrp, buf,
1986 1966              &mrhdl_new, &op);
1987 1967          if (status != DDI_SUCCESS) {
1988 1968                  TNF_PROBE_1(tavor_ci_reregister_buf_fail, TAVOR_TNF_ERROR, "",
1989 1969                      tnf_uint, status, status);
1990 1970                  TAVOR_TNF_EXIT(tavor_ci_reregister_buf);
1991 1971                  return (status);
1992 1972          }
1993      -        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mrhdl_new))
1994 1973  
1995 1974          /* Fill in the mr_desc structure */
1996 1975          mr_desc->md_vaddr = mrhdl_new->mr_bindinfo.bi_addr;
1997 1976          mr_desc->md_lkey  = mrhdl_new->mr_lkey;
1998 1977          /* Only set RKey if remote access was requested */
1999 1978          if ((flags & IBT_MR_ENABLE_REMOTE_ATOMIC) ||
2000 1979              (flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
2001 1980              (flags & IBT_MR_ENABLE_REMOTE_READ)) {
2002 1981                  mr_desc->md_rkey = mrhdl_new->mr_rkey;
2003 1982          }
↓ open down ↓ 92 lines elided ↑ open up ↑ 
2096 2075          pdhdl = (tavor_pdhdl_t)pd;
2097 2076  
2098 2077          /* Allocate the memory window */
2099 2078          status = tavor_mw_alloc(state, pdhdl, flags, &mwhdl);
2100 2079          if (status != DDI_SUCCESS) {
2101 2080                  TNF_PROBE_1(tavor_ci_alloc_mw_fail, TAVOR_TNF_ERROR, "",
2102 2081                      tnf_uint, status, status);
2103 2082                  TAVOR_TNF_EXIT(tavor_ci_alloc_mw);
2104 2083                  return (status);
2105 2084          }
2106      -        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mwhdl))
2107 2085  
2108 2086          /* Return the MW handle and RKey */
2109 2087          *mw_p = (ibc_mw_hdl_t)mwhdl;
2110 2088          *rkey_p = mwhdl->mr_rkey;
2111 2089  
2112 2090          TAVOR_TNF_EXIT(tavor_ci_alloc_mw);
2113 2091          return (IBT_SUCCESS);
2114 2092  }
2115 2093  
2116 2094
↓ open down ↓ 92 lines elided ↑ open up ↑ 
2209 2187  static ibt_status_t
2210 2188  tavor_ci_register_dma_mr(ibc_hca_hdl_t hca, ibc_pd_hdl_t pd,
2211 2189      ibt_dmr_attr_t *mr_attr, void *ibtl_reserved, ibc_mr_hdl_t *mr_p,
2212 2190      ibt_mr_desc_t *mr_desc)
2213 2191  {
2214 2192          tavor_state_t           *state;
2215 2193          tavor_pdhdl_t           pdhdl;
2216 2194          tavor_mrhdl_t           mrhdl;
2217 2195          int                     status;
2218 2196  
2219      -        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr_desc))
2220      -
2221 2197          ASSERT(mr_attr != NULL);
2222 2198          ASSERT(mr_p != NULL);
2223 2199          ASSERT(mr_desc != NULL);
2224 2200  
2225 2201          /* Check for valid HCA handle */
2226 2202          if (hca == NULL) {
2227 2203                  return (IBT_HCA_HDL_INVALID);
2228 2204          }
2229 2205  
2230 2206          /* Check for valid PD handle pointer */
↓ open down ↓ 12 lines elided ↑ open up ↑ 
2243 2219          }
2244 2220  
2245 2221          /* Grab the Tavor softstate pointer and PD handle */
2246 2222          state = (tavor_state_t *)hca;
2247 2223          pdhdl = (tavor_pdhdl_t)pd;
2248 2224  
2249 2225          status = tavor_dma_mr_register(state, pdhdl, mr_attr, &mrhdl);
2250 2226          if (status != DDI_SUCCESS) {
2251 2227                  return (status);
2252 2228          }
2253      -        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mrhdl))
2254 2229  
2255 2230          /* Fill in the mr_desc structure */
2256 2231          mr_desc->md_vaddr = mr_attr->dmr_paddr;
2257 2232          mr_desc->md_lkey  = mrhdl->mr_lkey;
2258 2233          /* Only set RKey if remote access was requested */
2259 2234          if ((mr_attr->dmr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC) ||
2260 2235              (mr_attr->dmr_flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
2261 2236              (mr_attr->dmr_flags & IBT_MR_ENABLE_REMOTE_READ)) {
2262 2237                  mr_desc->md_rkey = mrhdl->mr_rkey;
2263 2238          }
↓ open down ↓ 802 lines elided ↑ open up ↑ 
3066 3041  static ibt_status_t
3067 3042  tavor_ci_unmap_mem_area(ibc_hca_hdl_t hca, ibc_ma_hdl_t ma_hdl)
3068 3043  {
3069 3044          return (IBT_NOT_SUPPORTED);
3070 3045  }
3071 3046  
3072 3047  struct ibc_mi_s {
3073 3048          int                     imh_len;
3074 3049          ddi_dma_handle_t        imh_dmahandle[1];
3075 3050  };
3076      -_NOTE(SCHEME_PROTECTS_DATA("safe sharing",
3077      -    ibc_mi_s::imh_len
3078      -    ibc_mi_s::imh_dmahandle))
3079 3051  
3080      -
3081 3052  /*
3082 3053   * tavor_ci_map_mem_iov()
3083 3054   * Map the memory
3084 3055   *    Context: Can be called from interrupt or base context.
3085 3056   */
3086 3057  /* ARGSUSED */
3087 3058  static ibt_status_t
3088 3059  tavor_ci_map_mem_iov(ibc_hca_hdl_t hca, ibt_iov_attr_t *iov_attr,
3089 3060      ibt_all_wr_t *wr, ibc_mi_hdl_t *mi_hdl_p)
3090 3061  {
↓ open down ↓ 5 lines elided ↑ open up ↑ 
3096 3067          ddi_dma_cookie_t        dmacookie;
3097 3068          ddi_dma_attr_t          dma_attr;
3098 3069          uint_t                  cookie_cnt;
3099 3070          ibc_mi_hdl_t            mi_hdl;
3100 3071          ibt_lkey_t              rsvd_lkey;
3101 3072          ibt_wr_ds_t             *sgl;
3102 3073          tavor_state_t           *state;
3103 3074          int                     kmflag;
3104 3075          int                     (*callback)(caddr_t);
3105 3076  
3106      -        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*wr))
3107      -
3108 3077          if (mi_hdl_p == NULL)
3109 3078                  return (IBT_MI_HDL_INVALID);
3110 3079  
3111 3080          /* Check for valid HCA handle */
3112 3081          if (hca == NULL)
3113 3082                  return (IBT_HCA_HDL_INVALID);
3114 3083  
3115 3084          /* Tavor does not allow the default "use reserved lkey" */
3116 3085          if ((iov_attr->iov_flags & IBT_IOV_ALT_LKEY) == 0)
3117 3086                  return (IBT_INVALID_PARAM);
↓ open down ↓ 18 lines elided ↑ open up ↑ 
3136 3105          } else {
3137 3106                  kmflag = KM_NOSLEEP;
3138 3107                  callback = DDI_DMA_DONTWAIT;
3139 3108          }
3140 3109  
3141 3110          if (iov_attr->iov_flags & IBT_IOV_BUF) {
3142 3111                  mi_hdl = kmem_alloc(sizeof (*mi_hdl), kmflag);
3143 3112                  if (mi_hdl == NULL)
3144 3113                          return (IBT_INSUFF_RESOURCE);
3145 3114                  sgl = wr->send.wr_sgl;
3146      -                _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sgl))
3147 3115  
3148 3116                  status = ddi_dma_alloc_handle(state->ts_dip, &dma_attr,
3149 3117                      callback, NULL, &dmahdl);
3150 3118                  if (status != DDI_SUCCESS) {
3151 3119                          kmem_free(mi_hdl, sizeof (*mi_hdl));
3152 3120                          return (IBT_INSUFF_RESOURCE);
3153 3121                  }
3154 3122                  status = ddi_dma_buf_bind_handle(dmahdl, iov_attr->iov_buf,
3155 3123                      DDI_DMA_RDWR | DDI_DMA_CONSISTENT, callback, NULL,
3156 3124                      &dmacookie, &cookie_cnt);
↓ open down ↓ 19 lines elided ↑ open up ↑ 
3176 3144                  mi_hdl->imh_len = 1;
3177 3145                  mi_hdl->imh_dmahandle[0] = dmahdl;
3178 3146                  *mi_hdl_p = mi_hdl;
3179 3147                  return (IBT_SUCCESS);
3180 3148          }
3181 3149  
3182 3150          if (iov_attr->iov_flags & IBT_IOV_RECV)
3183 3151                  sgl = wr->recv.wr_sgl;
3184 3152          else
3185 3153                  sgl = wr->send.wr_sgl;
3186      -        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sgl))
3187 3154  
3188 3155          len = iov_attr->iov_list_len;
3189 3156          for (i = 0, j = 0; j < len; j++) {
3190 3157                  if (iov_attr->iov[j].iov_len == 0)
3191 3158                          continue;
3192 3159                  i++;
3193 3160          }
3194 3161          mi_hdl = kmem_alloc(sizeof (*mi_hdl) +
3195 3162              (i - 1) * sizeof (ddi_dma_handle_t), kmflag);
3196 3163          if (mi_hdl == NULL)
↓ open down ↓ 264 lines elided ↑ open up ↑ 
3461 3428          if (mem_alloc_hdl == NULL) {
3462 3429                  TNF_PROBE_0(tavor_ci_free_io_mem_hdl_fail,
3463 3430                      TAVOR_TNF_ERROR, "");
3464 3431                  TAVOR_TNF_EXIT(tavor_ci_free_io_mem);
3465 3432                  return (IBT_MEM_ALLOC_HDL_INVALID);
3466 3433          }
3467 3434  
3468 3435          memhdl = (tavor_mem_alloc_hdl_t)mem_alloc_hdl;
3469 3436  
3470 3437          /* free the memory */
3471      -        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*memhdl))
3472 3438          ddi_dma_mem_free(&memhdl->tavor_acc_hdl);
3473 3439          ddi_dma_free_handle(&memhdl->tavor_dma_hdl);
3474 3440  
3475 3441          kmem_free(memhdl, sizeof (*memhdl));
3476 3442          TAVOR_TNF_EXIT(tavor_dma_free);
3477 3443          return (IBT_SUCCESS);
3478 3444  }
3479 3445  
3480 3446  
3481 3447  int
↓ open down ↓ 41 lines elided ↑ open up ↑ 
3523 3489          /* Package the tavor_dma_info contents and return */
3524 3490          *mem_hdl = kmem_alloc(sizeof (**mem_hdl),
3525 3491              flags & IBT_MR_NOSLEEP ? KM_NOSLEEP : KM_SLEEP);
3526 3492          if (*mem_hdl == NULL) {
3527 3493                  ddi_dma_mem_free(&acc_hdl);
3528 3494                  ddi_dma_free_handle(&dma_hdl);
3529 3495                  TNF_PROBE_0(tavor_dma_alloc_memory_fail, TAVOR_TNF_ERROR, "");
3530 3496                  TAVOR_TNF_EXIT(tavor_mem_alloc);
3531 3497                  return (DDI_FAILURE);
3532 3498          }
3533      -        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(**mem_hdl))
3534 3499          (*mem_hdl)->tavor_dma_hdl = dma_hdl;
3535 3500          (*mem_hdl)->tavor_acc_hdl = acc_hdl;
3536 3501  
3537 3502          TAVOR_TNF_EXIT(tavor_mem_alloc);
3538 3503          return (DDI_SUCCESS);
3539 3504  }
    
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX