892 return (EFAULT);
893 }
894
895 /*
896 * Check ioctl revision
897 */
898 if (info.tp_revision != TAVOR_VTS_IOCTL_REVISION) {
899 TNF_PROBE_0(tavor_ioctl_ports_bad_rev, TAVOR_TNF_ERROR, "");
900 TAVOR_TNF_EXIT(tavor_ioctl_ports);
901 return (EINVAL);
902 }
903
904 /* Allocate space for temporary GID table/PKey table */
905 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_gidtbl);
906 sgid_tbl = (ib_gid_t *)kmem_zalloc(tbl_size * sizeof (ib_gid_t),
907 KM_SLEEP);
908 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_pkeytbl);
909 pkey_tbl = (ib_pkey_t *)kmem_zalloc(tbl_size * sizeof (ib_pkey_t),
910 KM_SLEEP);
911
912 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sgid_tbl, *pkey_tbl))
913
914 /*
915 * Setup the number of ports, then loop through all ports and
916 * query properties of each.
917 */
918 info.tp_num_ports = (uint8_t)state->ts_cfg_profile->cp_num_ports;
919 for (i = 0; i < info.tp_num_ports; i++) {
920 /*
921 * Get portstate information from the device. If
922 * tavor_port_query() fails, leave zeroes in user
923 * struct port entry and continue.
924 */
925 bzero(&pi, sizeof (ibt_hca_portinfo_t));
926 pi.p_sgid_tbl = sgid_tbl;
927 pi.p_pkey_tbl = pkey_tbl;
928 if (tavor_port_query(state, i + 1, &pi) != 0) {
929 TNF_PROBE_0(tavor_ioctl_ports_query_failed,
930 TAVOR_TNF_ERROR, "");
931 }
932
933 portstat.tsp_port_num = pi.p_port_num;
982 TAVOR_TNF_EXIT(tavor_ioctl_ports);
983 return (0);
984 }
985
986 /*
987 * tavor_ioctl_loopback()
988 */
989 static int
990 tavor_ioctl_loopback(tavor_state_t *state, intptr_t arg, int mode)
991 {
992 tavor_loopback_ioctl_t lb;
993 tavor_loopback_state_t lstate;
994 ibt_hca_portinfo_t pi;
995 uint_t tbl_size, loopmax, max_usec;
996 ib_gid_t *sgid_tbl;
997 ib_pkey_t *pkey_tbl;
998 int j, iter, ret;
999
1000 TAVOR_TNF_ENTER(tavor_ioctl_loopback);
1001
1002 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(lstate))
1003
1004 /*
1005 * Access to Tavor VTS ioctls is not allowed in "maintenance mode".
1006 */
1007 if (state->ts_operational_mode == TAVOR_MAINTENANCE_MODE) {
1008 TNF_PROBE_0(tavor_ioctl_loopback_maintenance_mode_fail,
1009 TAVOR_TNF_ERROR, "");
1010 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1011 return (EFAULT);
1012 }
1013
1014 /* copyin the user struct to kernel */
1015 #ifdef _MULTI_DATAMODEL
1016 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
1017 tavor_loopback_ioctl32_t lb32;
1018
1019 if (ddi_copyin((void *)arg, &lb32,
1020 sizeof (tavor_loopback_ioctl32_t), mode) != 0) {
1021 TNF_PROBE_0(tavor_ioctl_loopback_copyin_fail,
1022 TAVOR_TNF_ERROR, "");
1023 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1934 return (status);
1935 }
1936
1937 /*
1938 * tavor_flash_bank()
1939 */
1940 static void
1941 tavor_flash_bank(tavor_state_t *state, uint32_t addr)
1942 {
1943 ddi_acc_handle_t hdl;
1944 uint32_t bank;
1945
1946 TAVOR_TNF_ENTER(tavor_flash_bank);
1947
1948 /* Set handle */
1949 hdl = state->ts_pci_cfghdl;
1950
1951 /* Determine the bank setting from the address */
1952 bank = addr & TAVOR_HW_FLASH_BANK_MASK;
1953
1954 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->ts_fw_flashbank))
1955
1956 /*
1957 * If the bank is different from the currently set bank, we need to
1958 * change it. Also, if an 'addr' of 0 is given, this allows the
1959 * capability to force the flash bank to 0. This is useful at init
1960 * time to initially set the bank value
1961 */
1962 if (state->ts_fw_flashbank != bank || addr == 0) {
1963 /* Set bank using the GPIO settings */
1964 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_DATACLEAR, 0x70);
1965 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_DATASET,
1966 (bank >> 15) & 0x70);
1967
1968 /* Save the bank state */
1969 state->ts_fw_flashbank = bank;
1970 }
1971
1972 TAVOR_TNF_EXIT(tavor_flash_bank);
1973 }
1974
1975 /*
2316 }
2317
2318 static void
2319 tavor_flash_cfi_dword(uint32_t *dword, uint8_t *ch, int i)
2320 {
2321 *dword = (uint32_t)
2322 ((uint32_t)ch[i] << 24 |
2323 (uint32_t)ch[i+1] << 16 |
2324 (uint32_t)ch[i+2] << 8 |
2325 (uint32_t)ch[i+3]);
2326 }
2327
2328 /*
2329 * tavor_loopback_free_qps
2330 */
2331 static void
2332 tavor_loopback_free_qps(tavor_loopback_state_t *lstate)
2333 {
2334 int i;
2335
2336 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*lstate))
2337
2338 if (lstate->tls_tx.tlc_qp_hdl != NULL) {
2339 (void) tavor_qp_free(lstate->tls_state,
2340 &lstate->tls_tx.tlc_qp_hdl, IBC_FREE_QP_AND_QPN, NULL,
2341 TAVOR_NOSLEEP);
2342 }
2343 if (lstate->tls_rx.tlc_qp_hdl != NULL) {
2344 (void) tavor_qp_free(lstate->tls_state,
2345 &lstate->tls_rx.tlc_qp_hdl, IBC_FREE_QP_AND_QPN, NULL,
2346 TAVOR_NOSLEEP);
2347 }
2348 lstate->tls_tx.tlc_qp_hdl = NULL;
2349 lstate->tls_rx.tlc_qp_hdl = NULL;
2350 for (i = 0; i < 2; i++) {
2351 if (lstate->tls_tx.tlc_cqhdl[i] != NULL) {
2352 (void) tavor_cq_free(lstate->tls_state,
2353 &lstate->tls_tx.tlc_cqhdl[i], TAVOR_NOSLEEP);
2354 }
2355 if (lstate->tls_rx.tlc_cqhdl[i] != NULL) {
2356 (void) tavor_cq_free(lstate->tls_state,
2357 &lstate->tls_rx.tlc_cqhdl[i], TAVOR_NOSLEEP);
2379 TAVOR_NOSLEEP);
2380 }
2381 if (lstate->tls_pd_hdl != NULL) {
2382 (void) tavor_pd_free(lstate->tls_state, &lstate->tls_pd_hdl);
2383 }
2384 if (lstate->tls_tx.tlc_buf != NULL) {
2385 kmem_free(lstate->tls_tx.tlc_buf, lstate->tls_tx.tlc_buf_sz);
2386 }
2387 if (lstate->tls_rx.tlc_buf != NULL) {
2388 kmem_free(lstate->tls_rx.tlc_buf, lstate->tls_rx.tlc_buf_sz);
2389 }
2390 bzero(lstate, sizeof (tavor_loopback_state_t));
2391 }
2392
2393 /*
2394 * tavor_loopback_init
2395 */
2396 static int
2397 tavor_loopback_init(tavor_state_t *state, tavor_loopback_state_t *lstate)
2398 {
2399 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*lstate))
2400
2401 lstate->tls_hca_hdl = (ibc_hca_hdl_t)state;
2402 lstate->tls_status = tavor_pd_alloc(lstate->tls_state,
2403 &lstate->tls_pd_hdl, TAVOR_NOSLEEP);
2404 if (lstate->tls_status != IBT_SUCCESS) {
2405 lstate->tls_err = TAVOR_LOOPBACK_PROT_DOMAIN_ALLOC_FAIL;
2406 TNF_PROBE_0(tavor_ioctl_loopback_alloc_pd_fail,
2407 TAVOR_TNF_ERROR, "");
2408 return (EFAULT);
2409 }
2410
2411 return (0);
2412 }
2413
2414 /*
2415 * tavor_loopback_init_qp_info
2416 */
2417 static void
2418 tavor_loopback_init_qp_info(tavor_loopback_state_t *lstate,
2419 tavor_loopback_comm_t *comm)
2420 {
2445 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_adds_vect.av_send_grh = 0;
2446 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_adds_vect.av_dlid =
2447 lstate->tls_lid;
2448 comm->tlc_qp_info.qp_transport.rc.rc_retry_cnt = lstate->tls_retry;
2449 comm->tlc_qp_info.qp_transport.rc.rc_sq_psn = 0;
2450 comm->tlc_qp_info.qp_transport.rc.rc_rq_psn = 0;
2451 comm->tlc_qp_info.qp_transport.rc.rc_rdma_ra_in = 4;
2452 comm->tlc_qp_info.qp_transport.rc.rc_rdma_ra_out = 4;
2453 comm->tlc_qp_info.qp_transport.rc.rc_dst_qpn = 0;
2454 comm->tlc_qp_info.qp_transport.rc.rc_min_rnr_nak = IBT_RNR_NAK_655ms;
2455 comm->tlc_qp_info.qp_transport.rc.rc_path_mtu = IB_MTU_1K;
2456 }
2457
2458 /*
2459 * tavor_loopback_alloc_mem
2460 */
2461 static int
2462 tavor_loopback_alloc_mem(tavor_loopback_state_t *lstate,
2463 tavor_loopback_comm_t *comm, int sz)
2464 {
2465 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*comm))
2466
2467 /* Allocate buffer of specified size */
2468 comm->tlc_buf_sz = sz;
2469 comm->tlc_buf = kmem_zalloc(sz, KM_NOSLEEP);
2470 if (comm->tlc_buf == NULL) {
2471 return (EFAULT);
2472 }
2473
2474 /* Register the buffer as a memory region */
2475 comm->tlc_memattr.mr_vaddr = (uint64_t)(uintptr_t)comm->tlc_buf;
2476 comm->tlc_memattr.mr_len = (ib_msglen_t)sz;
2477 comm->tlc_memattr.mr_as = NULL;
2478 comm->tlc_memattr.mr_flags = IBT_MR_NOSLEEP |
2479 IBT_MR_ENABLE_REMOTE_WRITE | IBT_MR_ENABLE_LOCAL_WRITE;
2480
2481 comm->tlc_status = tavor_mr_register(lstate->tls_state,
2482 lstate->tls_pd_hdl, &comm->tlc_memattr, &comm->tlc_mrhdl, NULL);
2483
2484 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*comm->tlc_mrhdl))
2485
2486 comm->tlc_mrdesc.md_vaddr = comm->tlc_mrhdl->mr_bindinfo.bi_addr;
2487 comm->tlc_mrdesc.md_lkey = comm->tlc_mrhdl->mr_lkey;
2488 comm->tlc_mrdesc.md_rkey = comm->tlc_mrhdl->mr_rkey;
2489 if (comm->tlc_status != IBT_SUCCESS) {
2490 return (EFAULT);
2491 }
2492 return (0);
2493 }
2494
2495 /*
2496 * tavor_loopback_alloc_qps
2497 */
2498 static int
2499 tavor_loopback_alloc_qps(tavor_loopback_state_t *lstate,
2500 tavor_loopback_comm_t *comm)
2501 {
2502 uint32_t i, real_size;
2503 tavor_qp_info_t qpinfo;
2504
2505 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*comm))
2506 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*lstate))
2507
2508 /* Allocate send and recv CQs */
2509 for (i = 0; i < 2; i++) {
2510 bzero(&comm->tlc_cq_attr, sizeof (ibt_cq_attr_t));
2511 comm->tlc_cq_attr.cq_size = 128;
2512 comm->tlc_status = tavor_cq_alloc(lstate->tls_state,
2513 (ibt_cq_hdl_t)NULL, &comm->tlc_cq_attr, &real_size,
2514 &comm->tlc_cqhdl[i], TAVOR_NOSLEEP);
2515 if (comm->tlc_status != IBT_SUCCESS) {
2516 lstate->tls_err += i;
2517 return (EFAULT);
2518 }
2519 }
2520
2521 /* Allocate the QP */
2522 tavor_loopback_init_qp_info(lstate, comm);
2523 comm->tlc_qp_attr.qp_pd_hdl = (ibt_pd_hdl_t)lstate->tls_pd_hdl;
2524 comm->tlc_qp_attr.qp_scq_hdl = (ibt_cq_hdl_t)comm->tlc_cqhdl[0];
2525 comm->tlc_qp_attr.qp_rcq_hdl = (ibt_cq_hdl_t)comm->tlc_cqhdl[1];
2526 comm->tlc_qp_attr.qp_ibc_scq_hdl = (ibt_opaque1_t)comm->tlc_cqhdl[0];
2527 comm->tlc_qp_attr.qp_ibc_rcq_hdl = (ibt_opaque1_t)comm->tlc_cqhdl[1];
2533 comm->tlc_status = tavor_qp_alloc(lstate->tls_state, &qpinfo,
2534 TAVOR_NOSLEEP, NULL);
2535 if (comm->tlc_status == DDI_SUCCESS) {
2536 comm->tlc_qp_hdl = qpinfo.qpi_qphdl;
2537 }
2538
2539 if (comm->tlc_status != IBT_SUCCESS) {
2540 lstate->tls_err += 2;
2541 return (EFAULT);
2542 }
2543 return (0);
2544 }
2545
2546 /*
2547 * tavor_loopback_modify_qp
2548 */
2549 static int
2550 tavor_loopback_modify_qp(tavor_loopback_state_t *lstate,
2551 tavor_loopback_comm_t *comm, uint_t qp_num)
2552 {
2553 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*comm))
2554 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*lstate))
2555
2556 /* Modify QP to INIT */
2557 tavor_loopback_init_qp_info(lstate, comm);
2558 comm->tlc_qp_info.qp_state = IBT_STATE_INIT;
2559 comm->tlc_status = tavor_qp_modify(lstate->tls_state, comm->tlc_qp_hdl,
2560 IBT_CEP_SET_STATE, &comm->tlc_qp_info, &comm->tlc_queue_sizes);
2561 if (comm->tlc_status != IBT_SUCCESS) {
2562 return (EFAULT);
2563 }
2564
2565 /*
2566 * Modify QP to RTR (set destination LID and QP number to local
2567 * LID and QP number)
2568 */
2569 comm->tlc_qp_info.qp_state = IBT_STATE_RTR;
2570 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_adds_vect.av_dlid
2571 = lstate->tls_lid;
2572 comm->tlc_qp_info.qp_transport.rc.rc_dst_qpn = qp_num;
2573 comm->tlc_status = tavor_qp_modify(lstate->tls_state, comm->tlc_qp_hdl,
2574 IBT_CEP_SET_STATE, &comm->tlc_qp_info, &comm->tlc_queue_sizes);
2575 if (comm->tlc_status != IBT_SUCCESS) {
2621 } else
2622 #endif /* _MULTI_DATAMODEL */
2623 if (ddi_copyout(lb, (void *)arg, sizeof (tavor_loopback_ioctl_t),
2624 mode) != 0) {
2625 TNF_PROBE_0(tavor_ioctl_loopback_copyout_fail,
2626 TAVOR_TNF_ERROR, "");
2627 return (EFAULT);
2628 }
2629 return (0);
2630 }
2631
2632 /*
2633 * tavor_loopback_post_send
2634 */
2635 static int
2636 tavor_loopback_post_send(tavor_loopback_state_t *lstate,
2637 tavor_loopback_comm_t *tx, tavor_loopback_comm_t *rx)
2638 {
2639 int ret;
2640
2641 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*tx))
2642
2643 bzero(&tx->tlc_sgl, sizeof (ibt_wr_ds_t));
2644 bzero(&tx->tlc_wr, sizeof (ibt_send_wr_t));
2645
2646 /* Initialize local address for TX buffer */
2647 tx->tlc_sgl.ds_va = tx->tlc_mrdesc.md_vaddr;
2648 tx->tlc_sgl.ds_key = tx->tlc_mrdesc.md_lkey;
2649 tx->tlc_sgl.ds_len = tx->tlc_buf_sz;
2650
2651 /* Initialize the remaining details of the work request */
2652 tx->tlc_wr.wr_id = tx->tlc_wrid++;
2653 tx->tlc_wr.wr_flags = IBT_WR_SEND_SIGNAL;
2654 tx->tlc_wr.wr_nds = 1;
2655 tx->tlc_wr.wr_sgl = &tx->tlc_sgl;
2656 tx->tlc_wr.wr_opcode = IBT_WRC_RDMAW;
2657 tx->tlc_wr.wr_trans = IBT_RC_SRV;
2658
2659 /* Initialize the remote address for RX buffer */
2660 tx->tlc_wr.wr.rc.rcwr.rdma.rdma_raddr = rx->tlc_mrdesc.md_vaddr;
2661 tx->tlc_wr.wr.rc.rcwr.rdma.rdma_rkey = rx->tlc_mrdesc.md_rkey;
2662 tx->tlc_complete = 0;
2663 ret = tavor_post_send(lstate->tls_state, tx->tlc_qp_hdl, &tx->tlc_wr,
2664 1, NULL);
2665 if (ret != IBT_SUCCESS) {
2666 return (EFAULT);
2667 }
2668 return (0);
2669 }
2670
2671 /*
2672 * tavor_loopback_poll_cq
2673 */
2674 static int
2675 tavor_loopback_poll_cq(tavor_loopback_state_t *lstate,
2676 tavor_loopback_comm_t *comm)
2677 {
2678 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*comm))
2679
2680 comm->tlc_wc.wc_status = 0;
2681 comm->tlc_num_polled = 0;
2682 comm->tlc_status = tavor_cq_poll(lstate->tls_state,
2683 comm->tlc_cqhdl[0], &comm->tlc_wc, 1, &comm->tlc_num_polled);
2684 if ((comm->tlc_status == IBT_SUCCESS) &&
2685 (comm->tlc_wc.wc_status != IBT_WC_SUCCESS)) {
2686 comm->tlc_status = ibc_get_ci_failure(0);
2687 }
2688 return (comm->tlc_status);
2689 }
|
892 return (EFAULT);
893 }
894
895 /*
896 * Check ioctl revision
897 */
898 if (info.tp_revision != TAVOR_VTS_IOCTL_REVISION) {
899 TNF_PROBE_0(tavor_ioctl_ports_bad_rev, TAVOR_TNF_ERROR, "");
900 TAVOR_TNF_EXIT(tavor_ioctl_ports);
901 return (EINVAL);
902 }
903
904 /* Allocate space for temporary GID table/PKey table */
905 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_gidtbl);
906 sgid_tbl = (ib_gid_t *)kmem_zalloc(tbl_size * sizeof (ib_gid_t),
907 KM_SLEEP);
908 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_pkeytbl);
909 pkey_tbl = (ib_pkey_t *)kmem_zalloc(tbl_size * sizeof (ib_pkey_t),
910 KM_SLEEP);
911
912 /*
913 * Setup the number of ports, then loop through all ports and
914 * query properties of each.
915 */
916 info.tp_num_ports = (uint8_t)state->ts_cfg_profile->cp_num_ports;
917 for (i = 0; i < info.tp_num_ports; i++) {
918 /*
919 * Get portstate information from the device. If
920 * tavor_port_query() fails, leave zeroes in user
921 * struct port entry and continue.
922 */
923 bzero(&pi, sizeof (ibt_hca_portinfo_t));
924 pi.p_sgid_tbl = sgid_tbl;
925 pi.p_pkey_tbl = pkey_tbl;
926 if (tavor_port_query(state, i + 1, &pi) != 0) {
927 TNF_PROBE_0(tavor_ioctl_ports_query_failed,
928 TAVOR_TNF_ERROR, "");
929 }
930
931 portstat.tsp_port_num = pi.p_port_num;
980 TAVOR_TNF_EXIT(tavor_ioctl_ports);
981 return (0);
982 }
983
984 /*
985 * tavor_ioctl_loopback()
986 */
987 static int
988 tavor_ioctl_loopback(tavor_state_t *state, intptr_t arg, int mode)
989 {
990 tavor_loopback_ioctl_t lb;
991 tavor_loopback_state_t lstate;
992 ibt_hca_portinfo_t pi;
993 uint_t tbl_size, loopmax, max_usec;
994 ib_gid_t *sgid_tbl;
995 ib_pkey_t *pkey_tbl;
996 int j, iter, ret;
997
998 TAVOR_TNF_ENTER(tavor_ioctl_loopback);
999
1000 /*
1001 * Access to Tavor VTS ioctls is not allowed in "maintenance mode".
1002 */
1003 if (state->ts_operational_mode == TAVOR_MAINTENANCE_MODE) {
1004 TNF_PROBE_0(tavor_ioctl_loopback_maintenance_mode_fail,
1005 TAVOR_TNF_ERROR, "");
1006 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1007 return (EFAULT);
1008 }
1009
1010 /* copyin the user struct to kernel */
1011 #ifdef _MULTI_DATAMODEL
1012 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
1013 tavor_loopback_ioctl32_t lb32;
1014
1015 if (ddi_copyin((void *)arg, &lb32,
1016 sizeof (tavor_loopback_ioctl32_t), mode) != 0) {
1017 TNF_PROBE_0(tavor_ioctl_loopback_copyin_fail,
1018 TAVOR_TNF_ERROR, "");
1019 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1930 return (status);
1931 }
1932
1933 /*
1934 * tavor_flash_bank()
1935 */
1936 static void
1937 tavor_flash_bank(tavor_state_t *state, uint32_t addr)
1938 {
1939 ddi_acc_handle_t hdl;
1940 uint32_t bank;
1941
1942 TAVOR_TNF_ENTER(tavor_flash_bank);
1943
1944 /* Set handle */
1945 hdl = state->ts_pci_cfghdl;
1946
1947 /* Determine the bank setting from the address */
1948 bank = addr & TAVOR_HW_FLASH_BANK_MASK;
1949
1950 /*
1951 * If the bank is different from the currently set bank, we need to
1952 * change it. Also, if an 'addr' of 0 is given, this allows the
1953 * capability to force the flash bank to 0. This is useful at init
1954 * time to initially set the bank value
1955 */
1956 if (state->ts_fw_flashbank != bank || addr == 0) {
1957 /* Set bank using the GPIO settings */
1958 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_DATACLEAR, 0x70);
1959 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_DATASET,
1960 (bank >> 15) & 0x70);
1961
1962 /* Save the bank state */
1963 state->ts_fw_flashbank = bank;
1964 }
1965
1966 TAVOR_TNF_EXIT(tavor_flash_bank);
1967 }
1968
1969 /*
2310 }
2311
2312 static void
2313 tavor_flash_cfi_dword(uint32_t *dword, uint8_t *ch, int i)
2314 {
2315 *dword = (uint32_t)
2316 ((uint32_t)ch[i] << 24 |
2317 (uint32_t)ch[i+1] << 16 |
2318 (uint32_t)ch[i+2] << 8 |
2319 (uint32_t)ch[i+3]);
2320 }
2321
2322 /*
2323 * tavor_loopback_free_qps
2324 */
2325 static void
2326 tavor_loopback_free_qps(tavor_loopback_state_t *lstate)
2327 {
2328 int i;
2329
2330 if (lstate->tls_tx.tlc_qp_hdl != NULL) {
2331 (void) tavor_qp_free(lstate->tls_state,
2332 &lstate->tls_tx.tlc_qp_hdl, IBC_FREE_QP_AND_QPN, NULL,
2333 TAVOR_NOSLEEP);
2334 }
2335 if (lstate->tls_rx.tlc_qp_hdl != NULL) {
2336 (void) tavor_qp_free(lstate->tls_state,
2337 &lstate->tls_rx.tlc_qp_hdl, IBC_FREE_QP_AND_QPN, NULL,
2338 TAVOR_NOSLEEP);
2339 }
2340 lstate->tls_tx.tlc_qp_hdl = NULL;
2341 lstate->tls_rx.tlc_qp_hdl = NULL;
2342 for (i = 0; i < 2; i++) {
2343 if (lstate->tls_tx.tlc_cqhdl[i] != NULL) {
2344 (void) tavor_cq_free(lstate->tls_state,
2345 &lstate->tls_tx.tlc_cqhdl[i], TAVOR_NOSLEEP);
2346 }
2347 if (lstate->tls_rx.tlc_cqhdl[i] != NULL) {
2348 (void) tavor_cq_free(lstate->tls_state,
2349 &lstate->tls_rx.tlc_cqhdl[i], TAVOR_NOSLEEP);
2371 TAVOR_NOSLEEP);
2372 }
2373 if (lstate->tls_pd_hdl != NULL) {
2374 (void) tavor_pd_free(lstate->tls_state, &lstate->tls_pd_hdl);
2375 }
2376 if (lstate->tls_tx.tlc_buf != NULL) {
2377 kmem_free(lstate->tls_tx.tlc_buf, lstate->tls_tx.tlc_buf_sz);
2378 }
2379 if (lstate->tls_rx.tlc_buf != NULL) {
2380 kmem_free(lstate->tls_rx.tlc_buf, lstate->tls_rx.tlc_buf_sz);
2381 }
2382 bzero(lstate, sizeof (tavor_loopback_state_t));
2383 }
2384
2385 /*
2386 * tavor_loopback_init
2387 */
2388 static int
2389 tavor_loopback_init(tavor_state_t *state, tavor_loopback_state_t *lstate)
2390 {
2391 lstate->tls_hca_hdl = (ibc_hca_hdl_t)state;
2392 lstate->tls_status = tavor_pd_alloc(lstate->tls_state,
2393 &lstate->tls_pd_hdl, TAVOR_NOSLEEP);
2394 if (lstate->tls_status != IBT_SUCCESS) {
2395 lstate->tls_err = TAVOR_LOOPBACK_PROT_DOMAIN_ALLOC_FAIL;
2396 TNF_PROBE_0(tavor_ioctl_loopback_alloc_pd_fail,
2397 TAVOR_TNF_ERROR, "");
2398 return (EFAULT);
2399 }
2400
2401 return (0);
2402 }
2403
2404 /*
2405 * tavor_loopback_init_qp_info
2406 */
2407 static void
2408 tavor_loopback_init_qp_info(tavor_loopback_state_t *lstate,
2409 tavor_loopback_comm_t *comm)
2410 {
2435 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_adds_vect.av_send_grh = 0;
2436 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_adds_vect.av_dlid =
2437 lstate->tls_lid;
2438 comm->tlc_qp_info.qp_transport.rc.rc_retry_cnt = lstate->tls_retry;
2439 comm->tlc_qp_info.qp_transport.rc.rc_sq_psn = 0;
2440 comm->tlc_qp_info.qp_transport.rc.rc_rq_psn = 0;
2441 comm->tlc_qp_info.qp_transport.rc.rc_rdma_ra_in = 4;
2442 comm->tlc_qp_info.qp_transport.rc.rc_rdma_ra_out = 4;
2443 comm->tlc_qp_info.qp_transport.rc.rc_dst_qpn = 0;
2444 comm->tlc_qp_info.qp_transport.rc.rc_min_rnr_nak = IBT_RNR_NAK_655ms;
2445 comm->tlc_qp_info.qp_transport.rc.rc_path_mtu = IB_MTU_1K;
2446 }
2447
2448 /*
2449 * tavor_loopback_alloc_mem
2450 */
2451 static int
2452 tavor_loopback_alloc_mem(tavor_loopback_state_t *lstate,
2453 tavor_loopback_comm_t *comm, int sz)
2454 {
2455 /* Allocate buffer of specified size */
2456 comm->tlc_buf_sz = sz;
2457 comm->tlc_buf = kmem_zalloc(sz, KM_NOSLEEP);
2458 if (comm->tlc_buf == NULL) {
2459 return (EFAULT);
2460 }
2461
2462 /* Register the buffer as a memory region */
2463 comm->tlc_memattr.mr_vaddr = (uint64_t)(uintptr_t)comm->tlc_buf;
2464 comm->tlc_memattr.mr_len = (ib_msglen_t)sz;
2465 comm->tlc_memattr.mr_as = NULL;
2466 comm->tlc_memattr.mr_flags = IBT_MR_NOSLEEP |
2467 IBT_MR_ENABLE_REMOTE_WRITE | IBT_MR_ENABLE_LOCAL_WRITE;
2468
2469 comm->tlc_status = tavor_mr_register(lstate->tls_state,
2470 lstate->tls_pd_hdl, &comm->tlc_memattr, &comm->tlc_mrhdl, NULL);
2471
2472 comm->tlc_mrdesc.md_vaddr = comm->tlc_mrhdl->mr_bindinfo.bi_addr;
2473 comm->tlc_mrdesc.md_lkey = comm->tlc_mrhdl->mr_lkey;
2474 comm->tlc_mrdesc.md_rkey = comm->tlc_mrhdl->mr_rkey;
2475 if (comm->tlc_status != IBT_SUCCESS) {
2476 return (EFAULT);
2477 }
2478 return (0);
2479 }
2480
2481 /*
2482 * tavor_loopback_alloc_qps
2483 */
2484 static int
2485 tavor_loopback_alloc_qps(tavor_loopback_state_t *lstate,
2486 tavor_loopback_comm_t *comm)
2487 {
2488 uint32_t i, real_size;
2489 tavor_qp_info_t qpinfo;
2490
2491 /* Allocate send and recv CQs */
2492 for (i = 0; i < 2; i++) {
2493 bzero(&comm->tlc_cq_attr, sizeof (ibt_cq_attr_t));
2494 comm->tlc_cq_attr.cq_size = 128;
2495 comm->tlc_status = tavor_cq_alloc(lstate->tls_state,
2496 (ibt_cq_hdl_t)NULL, &comm->tlc_cq_attr, &real_size,
2497 &comm->tlc_cqhdl[i], TAVOR_NOSLEEP);
2498 if (comm->tlc_status != IBT_SUCCESS) {
2499 lstate->tls_err += i;
2500 return (EFAULT);
2501 }
2502 }
2503
2504 /* Allocate the QP */
2505 tavor_loopback_init_qp_info(lstate, comm);
2506 comm->tlc_qp_attr.qp_pd_hdl = (ibt_pd_hdl_t)lstate->tls_pd_hdl;
2507 comm->tlc_qp_attr.qp_scq_hdl = (ibt_cq_hdl_t)comm->tlc_cqhdl[0];
2508 comm->tlc_qp_attr.qp_rcq_hdl = (ibt_cq_hdl_t)comm->tlc_cqhdl[1];
2509 comm->tlc_qp_attr.qp_ibc_scq_hdl = (ibt_opaque1_t)comm->tlc_cqhdl[0];
2510 comm->tlc_qp_attr.qp_ibc_rcq_hdl = (ibt_opaque1_t)comm->tlc_cqhdl[1];
2516 comm->tlc_status = tavor_qp_alloc(lstate->tls_state, &qpinfo,
2517 TAVOR_NOSLEEP, NULL);
2518 if (comm->tlc_status == DDI_SUCCESS) {
2519 comm->tlc_qp_hdl = qpinfo.qpi_qphdl;
2520 }
2521
2522 if (comm->tlc_status != IBT_SUCCESS) {
2523 lstate->tls_err += 2;
2524 return (EFAULT);
2525 }
2526 return (0);
2527 }
2528
2529 /*
2530 * tavor_loopback_modify_qp
2531 */
2532 static int
2533 tavor_loopback_modify_qp(tavor_loopback_state_t *lstate,
2534 tavor_loopback_comm_t *comm, uint_t qp_num)
2535 {
2536 /* Modify QP to INIT */
2537 tavor_loopback_init_qp_info(lstate, comm);
2538 comm->tlc_qp_info.qp_state = IBT_STATE_INIT;
2539 comm->tlc_status = tavor_qp_modify(lstate->tls_state, comm->tlc_qp_hdl,
2540 IBT_CEP_SET_STATE, &comm->tlc_qp_info, &comm->tlc_queue_sizes);
2541 if (comm->tlc_status != IBT_SUCCESS) {
2542 return (EFAULT);
2543 }
2544
2545 /*
2546 * Modify QP to RTR (set destination LID and QP number to local
2547 * LID and QP number)
2548 */
2549 comm->tlc_qp_info.qp_state = IBT_STATE_RTR;
2550 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_adds_vect.av_dlid
2551 = lstate->tls_lid;
2552 comm->tlc_qp_info.qp_transport.rc.rc_dst_qpn = qp_num;
2553 comm->tlc_status = tavor_qp_modify(lstate->tls_state, comm->tlc_qp_hdl,
2554 IBT_CEP_SET_STATE, &comm->tlc_qp_info, &comm->tlc_queue_sizes);
2555 if (comm->tlc_status != IBT_SUCCESS) {
2601 } else
2602 #endif /* _MULTI_DATAMODEL */
2603 if (ddi_copyout(lb, (void *)arg, sizeof (tavor_loopback_ioctl_t),
2604 mode) != 0) {
2605 TNF_PROBE_0(tavor_ioctl_loopback_copyout_fail,
2606 TAVOR_TNF_ERROR, "");
2607 return (EFAULT);
2608 }
2609 return (0);
2610 }
2611
2612 /*
2613 * tavor_loopback_post_send
2614 */
2615 static int
2616 tavor_loopback_post_send(tavor_loopback_state_t *lstate,
2617 tavor_loopback_comm_t *tx, tavor_loopback_comm_t *rx)
2618 {
2619 int ret;
2620
2621 bzero(&tx->tlc_sgl, sizeof (ibt_wr_ds_t));
2622 bzero(&tx->tlc_wr, sizeof (ibt_send_wr_t));
2623
2624 /* Initialize local address for TX buffer */
2625 tx->tlc_sgl.ds_va = tx->tlc_mrdesc.md_vaddr;
2626 tx->tlc_sgl.ds_key = tx->tlc_mrdesc.md_lkey;
2627 tx->tlc_sgl.ds_len = tx->tlc_buf_sz;
2628
2629 /* Initialize the remaining details of the work request */
2630 tx->tlc_wr.wr_id = tx->tlc_wrid++;
2631 tx->tlc_wr.wr_flags = IBT_WR_SEND_SIGNAL;
2632 tx->tlc_wr.wr_nds = 1;
2633 tx->tlc_wr.wr_sgl = &tx->tlc_sgl;
2634 tx->tlc_wr.wr_opcode = IBT_WRC_RDMAW;
2635 tx->tlc_wr.wr_trans = IBT_RC_SRV;
2636
2637 /* Initialize the remote address for RX buffer */
2638 tx->tlc_wr.wr.rc.rcwr.rdma.rdma_raddr = rx->tlc_mrdesc.md_vaddr;
2639 tx->tlc_wr.wr.rc.rcwr.rdma.rdma_rkey = rx->tlc_mrdesc.md_rkey;
2640 tx->tlc_complete = 0;
2641 ret = tavor_post_send(lstate->tls_state, tx->tlc_qp_hdl, &tx->tlc_wr,
2642 1, NULL);
2643 if (ret != IBT_SUCCESS) {
2644 return (EFAULT);
2645 }
2646 return (0);
2647 }
2648
2649 /*
2650 * tavor_loopback_poll_cq
2651 */
2652 static int
2653 tavor_loopback_poll_cq(tavor_loopback_state_t *lstate,
2654 tavor_loopback_comm_t *comm)
2655 {
2656 comm->tlc_wc.wc_status = 0;
2657 comm->tlc_num_polled = 0;
2658 comm->tlc_status = tavor_cq_poll(lstate->tls_state,
2659 comm->tlc_cqhdl[0], &comm->tlc_wc, 1, &comm->tlc_num_polled);
2660 if ((comm->tlc_status == IBT_SUCCESS) &&
2661 (comm->tlc_wc.wc_status != IBT_WC_SUCCESS)) {
2662 comm->tlc_status = ibc_get_ci_failure(0);
2663 }
2664 return (comm->tlc_status);
2665 }
|