58 static void tavor_mbox_sync(tavor_mbox_t *mbox, uint_t offset,
59 uint_t length, uint_t flag);
60
61 /*
62 * tavor_cmd_post()
63 * Context: Can be called from interrupt or base context.
64 *
65 * The "cp_flags" field in cmdpost
66 * is used to determine whether to wait for an available
67 * outstanding command (if necessary) or to return error.
68 */
69 int
70 tavor_cmd_post(tavor_state_t *state, tavor_cmd_post_t *cmdpost)
71 {
72 tavor_cmd_t *cmdptr;
73 int status;
74 uint16_t token;
75
76 TAVOR_TNF_ENTER(tavor_cmd_post);
77
78 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cmdpost))
79
80 /* Determine if we are going to spin until completion */
81 if (cmdpost->cp_flags == TAVOR_CMD_NOSLEEP_SPIN) {
82
83 TNF_PROBE_0_DEBUG(tavor_cmd_post_spin, TAVOR_TNF_TRACE, "");
84
85 /* Write the command to the HCR */
86 status = tavor_write_hcr(state, cmdpost, 0);
87 if (status != TAVOR_CMD_SUCCESS) {
88 TNF_PROBE_0(tavor_cmd_post_fail,
89 TAVOR_TNF_ERROR, "");
90 TAVOR_TNF_EXIT(tavor_cmd_post);
91 return (status);
92 }
93
94 TAVOR_TNF_EXIT(tavor_cmd_post);
95 return (TAVOR_CMD_SUCCESS);
96
97 } else { /* "TAVOR_CMD_SLEEP_NOSPIN" */
98
99 TNF_PROBE_0_DEBUG(tavor_cmd_post_nospin, TAVOR_TNF_TRACE, "");
100
101 ASSERT(TAVOR_SLEEPFLAG_FOR_CONTEXT() != TAVOR_NOSLEEP);
102
103 /* NOTE: Expect threads to be waiting in here */
104 status = tavor_outstanding_cmd_alloc(state, &cmdptr,
105 cmdpost->cp_flags);
106 if (status != TAVOR_CMD_SUCCESS) {
107 TNF_PROBE_0(tavor_cmd_alloc_fail, TAVOR_TNF_ERROR, "");
108 TAVOR_TNF_EXIT(tavor_cmd_post);
109 return (status);
110 }
111 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cmdptr))
112
113 /*
114 * Set status to "TAVOR_CMD_INVALID_STATUS". It is
115 * appropriate to do this here without the "cmd_comp_lock"
116 * because this register is overloaded. Later it will be
117 * used to indicate - through a change from this invalid
118 * value to some other value - that the condition variable
119 * has been signaled. Once it has, status will then contain
120 * the _real_ completion status
121 */
122 cmdptr->cmd_status = TAVOR_CMD_INVALID_STATUS;
123
124 /* Write the command to the HCR */
125 token = (uint16_t)cmdptr->cmd_indx;
126 status = tavor_write_hcr(state, cmdpost, token);
127 if (status != TAVOR_CMD_SUCCESS) {
128 tavor_outstanding_cmd_free(state, &cmdptr);
129 TNF_PROBE_0(tavor_cmd_post_fail, TAVOR_TNF_ERROR, "");
130 TAVOR_TNF_EXIT(tavor_cmd_post);
131 return (status);
132 }
133
134 /*
135 * cv_wait() on the "command_complete" condition variable.
136 * Note: We have the "__lock_lint" here to workaround warlock.
137 * Since warlock doesn't know that other parts of the Tavor
138 * may occasionally call this routine while holding their own
139 * locks, it complains about this cv_wait. In reality,
140 * however, the rest of the driver never calls this routine
141 * with a lock held unless they pass TAVOR_CMD_NOSLEEP.
142 */
143 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*cmdptr))
144 mutex_enter(&cmdptr->cmd_comp_lock);
145 while (cmdptr->cmd_status == TAVOR_CMD_INVALID_STATUS) {
146 #ifndef __lock_lint
147 cv_wait(&cmdptr->cmd_comp_cv, &cmdptr->cmd_comp_lock);
148 /* NOTE: EXPECT SEVERAL THREADS TO BE WAITING HERE */
149 #endif
150 }
151 mutex_exit(&cmdptr->cmd_comp_lock);
152 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cmdptr))
153
154 /*
155 * Wake up after command completes (cv_signal). Read status
156 * from the command (success, fail, etc.). It is appropriate
157 * here (as above) to read the status field without the
158 * "cmd_comp_lock" because it is no longer being used to
159 * indicate whether the condition variable has been signaled
160 * (i.e. at this point we are certain that it already has).
161 */
162 status = cmdptr->cmd_status;
163
164 /* Save the "outparam" values into the cmdpost struct */
165 cmdpost->cp_outparm = cmdptr->cmd_outparm;
166
167 /*
168 * Add the command back to the "outstanding commands list".
169 * Signal the "cmd_list" condition variable, if necessary.
170 */
171 tavor_outstanding_cmd_free(state, &cmdptr);
172
234 } else {
235 /* NOTE: Expect threads to be waiting in here */
236 status = tavor_impl_mbox_alloc(state,
237 &state->ts_out_mblist, &mbox_info->mbi_out,
238 mbox_wait);
239 if (status != TAVOR_CMD_SUCCESS) {
240 /* If we allocated an "In" mailbox, free it */
241 if (mbox_info->mbi_alloc_flags &
242 TAVOR_ALLOC_INMBOX) {
243 tavor_impl_mbox_free(
244 &state->ts_in_mblist,
245 &mbox_info->mbi_in);
246 }
247 TAVOR_TNF_EXIT(tavor_mbox_alloc);
248 return (status);
249 }
250 }
251 }
252
253 /* Store appropriate context in mbox_info */
254 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(mbox_info->mbi_sleep_context))
255 mbox_info->mbi_sleep_context = sleep_context;
256
257 TAVOR_TNF_EXIT(tavor_mbox_alloc);
258 return (TAVOR_CMD_SUCCESS);
259 }
260
261
262 /*
263 * tavor_mbox_free()
264 * Context: Can be called from interrupt or base context.
265 */
266 void
267 tavor_mbox_free(tavor_state_t *state, tavor_mbox_info_t *mbox_info)
268 {
269 TAVOR_TNF_ENTER(tavor_mbox_free);
270
271 /*
272 * The mailbox has to be freed in the same context from which it was
273 * allocated. The context is stored in the mbox_info at
274 * tavor_mbox_alloc() time. We check the stored context against the
275 * current context here.
276 */
277 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(mbox_info->mbi_sleep_context))
278 ASSERT(mbox_info->mbi_sleep_context == TAVOR_SLEEPFLAG_FOR_CONTEXT());
279
280 /* Determine correct mboxlist based on calling context */
281 if (mbox_info->mbi_sleep_context == TAVOR_NOSLEEP) {
282 /* Free the intr "In" mailbox */
283 if (mbox_info->mbi_alloc_flags & TAVOR_ALLOC_INMBOX) {
284 tavor_impl_mbox_free(&state->ts_in_intr_mblist,
285 &mbox_info->mbi_in);
286 }
287
288 /* Free the intr "Out" mailbox */
289 if (mbox_info->mbi_alloc_flags & TAVOR_ALLOC_OUTMBOX) {
290 tavor_impl_mbox_free(&state->ts_out_intr_mblist,
291 &mbox_info->mbi_out);
292 }
293 } else {
294 /* Free the "In" mailbox */
295 if (mbox_info->mbi_alloc_flags & TAVOR_ALLOC_INMBOX) {
296 tavor_impl_mbox_free(&state->ts_in_mblist,
297 &mbox_info->mbi_in);
562 TNF_PROBE_0(tavor_impl_mbox_alloc_fail,
563 TAVOR_TNF_ERROR, "");
564 TAVOR_TNF_EXIT(tavor_impl_mbox_alloc);
565 return (TAVOR_CMD_INSUFF_RSRC);
566 }
567
568 /* Delay before polling for mailbox again */
569 drv_usecwait(state->ts_cfg_profile->cp_cmd_poll_delay);
570 mutex_enter(&mblist->mbl_lock);
571 }
572 mblist->mbl_pollers--;
573
574 /* TAVOR_SLEEP */
575 } else {
576 /*
577 * Grab lock here as we prepare to cv_wait if needed.
578 */
579 mutex_enter(&mblist->mbl_lock);
580 while (mblist->mbl_entries_free == 0) {
581 /*
582 * Wait (on cv) for a mailbox to become free. Note:
583 * Just as we do above in tavor_cmd_post(), we also
584 * have the "__lock_lint" here to workaround warlock.
585 * Warlock doesn't know that other parts of the Tavor
586 * may occasionally call this routine while holding
587 * their own locks, so it complains about this cv_wait.
588 * In reality, however, the rest of the driver never
589 * calls this routine with a lock held unless they pass
590 * TAVOR_CMD_NOSLEEP.
591 */
592 mblist->mbl_waiters++;
593 #ifndef __lock_lint
594 cv_wait(&mblist->mbl_cv, &mblist->mbl_lock);
595 #endif
596 }
597 }
598
599 /* Grab the next available mailbox from list */
600 mbox_ptr = mblist->mbl_mbox;
601 index = mblist->mbl_head_indx;
602 next = mbox_ptr[index].mb_next;
603 prev = mbox_ptr[index].mb_prev;
604
605 /* Remove it from the mailbox list */
606 mblist->mbl_mbox[next].mb_prev = prev;
607 mblist->mbl_mbox[prev].mb_next = next;
608 mblist->mbl_head_indx = next;
609
610 /* Update the "free" count and return the mailbox pointer */
611 mblist->mbl_entries_free--;
612 *mb = &mbox_ptr[index];
613
614 mutex_exit(&mblist->mbl_lock);
615
870
871 /* Ensure that outstanding commands are supported */
872 ASSERT(cmd_list->cml_num_alloc != 0);
873
874 /*
875 * If the outstanding command list is empty, then wait (if
876 * appropriate in the current context). Otherwise, grab the
877 * next available command.
878 */
879 while (cmd_list->cml_entries_free == 0) {
880 /* No free commands */
881 if (cmd_wait == TAVOR_NOSLEEP) {
882 mutex_exit(&cmd_list->cml_lock);
883 TNF_PROBE_0(tavor_outstanding_cmd_alloc_fail,
884 TAVOR_TNF_ERROR, "");
885 TAVOR_TNF_EXIT(tavor_outstanding_cmd_alloc);
886 return (TAVOR_CMD_INSUFF_RSRC);
887 }
888
889 /*
890 * Wait (on cv) for a command to become free. Note: Just
891 * as we do above in tavor_cmd_post(), we also have the
892 * "__lock_lint" here to workaround warlock. Warlock doesn't
893 * know that other parts of the Tavor may occasionally call
894 * this routine while holding their own locks, so it complains
895 * about this cv_wait. In reality, however, the rest of the
896 * driver never calls this routine with a lock held unless
897 * they pass TAVOR_CMD_NOSLEEP.
898 */
899 cmd_list->cml_waiters++;
900 #ifndef __lock_lint
901 cv_wait(&cmd_list->cml_cv, &cmd_list->cml_lock);
902 #endif
903 }
904
905 /* Grab the next available command from the list */
906 head = cmd_list->cml_head_indx;
907 *cmd_ptr = &cmd_list->cml_cmd[head];
908 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(**cmd_ptr))
909 next = (*cmd_ptr)->cmd_next;
910 prev = (*cmd_ptr)->cmd_prev;
911 (*cmd_ptr)->cmd_status = TAVOR_CMD_INVALID_STATUS;
912
913 /* Remove it from the command list */
914 cmd_list->cml_cmd[next].cmd_prev = prev;
915 cmd_list->cml_cmd[prev].cmd_next = next;
916 cmd_list->cml_head_indx = next;
917
918 /* Update the "free" count and return */
919 cmd_list->cml_entries_free--;
920
921 mutex_exit(&cmd_list->cml_lock);
922
923 TAVOR_TNF_EXIT(tavor_outstanding_cmd_alloc);
924 return (TAVOR_CMD_SUCCESS);
925 }
926
927
928 /*
988 /*
989 * tavor_write_hcr()
990 * Context: Can be called from interrupt or base context.
991 */
992 static int
993 tavor_write_hcr(tavor_state_t *state, tavor_cmd_post_t *cmdpost,
994 uint16_t token)
995 {
996 tavor_hw_hcr_t *hcr;
997 uint_t status, count, countmax;
998 uint64_t hcrreg;
999
1000 TAVOR_TNF_ENTER(tavor_write_hcr);
1001
1002 /*
1003 * Grab the "HCR access" lock if the driver is not in
1004 * fastreboot. In fastreboot, this function is called
1005 * with the single thread but in high interrupt context
1006 * (so that this mutex lock cannot be used).
1007 */
1008 #ifdef __lock_lint
1009 mutex_enter(&state->ts_cmd_regs.hcr_lock);
1010 #else
1011 if (!TAVOR_IN_FASTREBOOT(state)) {
1012 mutex_enter(&state->ts_cmd_regs.hcr_lock);
1013 }
1014 #endif
1015
1016 hcr = state->ts_cmd_regs.hcr;
1017
1018 /*
1019 * First, check the "go" bit to see if the previous hcr usage is
1020 * complete. As long as it is set then we must continue to poll.
1021 */
1022 count = 0;
1023 countmax = state->ts_cfg_profile->cp_cmd_poll_max;
1024 for (;;) {
1025 hcrreg = ddi_get32(state->ts_reg_cmdhdl, &hcr->cmd);
1026
1027 /* If "go" bit is clear, then done */
1028 if ((hcrreg & TAVOR_HCR_CMD_GO_MASK) == 0) {
1029 TNF_PROBE_1_DEBUG(tavor_write_hcr_loop_count,
1030 TAVOR_TNF_ERROR, "", tnf_uint, nospinloopcount,
1031 count);
1032 break;
1033 }
1034 /* Delay before polling the "go" bit again */
1035 drv_usecwait(state->ts_cfg_profile->cp_cmd_poll_delay);
1036
1037 /*
1038 * If we poll more than the maximum number of times, then
1039 * return a "timeout" error.
1040 */
1041 if (++count > countmax) {
1042 #ifdef __lock_lint
1043 mutex_exit(&state->ts_cmd_regs.hcr_lock);
1044 #else
1045 if (!TAVOR_IN_FASTREBOOT(state)) {
1046 mutex_exit(&state->ts_cmd_regs.hcr_lock);
1047 }
1048 #endif
1049 TNF_PROBE_0(tavor_write_hcr_timeout1, TAVOR_TNF_ERROR,
1050 "");
1051 TAVOR_TNF_EXIT(tavor_write_hcr);
1052 return (TAVOR_CMD_TIMEOUT);
1053 }
1054 }
1055
1056 /* Write "inparam" as a 64-bit quantity */
1057 ddi_put64(state->ts_reg_cmdhdl, (uint64_t *)&hcr->in_param0,
1058 cmdpost->cp_inparm);
1059
1060 /* Write "inmod" and 32-bits of "outparam" as 64-bit */
1061 hcrreg = ((uint64_t)cmdpost->cp_inmod << 32);
1062 hcrreg = hcrreg | (cmdpost->cp_outparm >> 32);
1063 ddi_put64(state->ts_reg_cmdhdl, (uint64_t *)&hcr->input_modifier,
1064 hcrreg);
1065
1066 /* Write the other 32-bits of "outparam" and "token" as 64-bit */
1067 hcrreg = (cmdpost->cp_outparm << 32);
1068 hcrreg = hcrreg | ((uint32_t)token << TAVOR_HCR_TOKEN_SHIFT);
1089 countmax = state->ts_cfg_profile->cp_cmd_poll_max;
1090
1091 for (;;) {
1092 hcrreg = ddi_get32(state->ts_reg_cmdhdl, &hcr->cmd);
1093
1094 /* If "go" bit is clear, then done */
1095 if ((hcrreg & TAVOR_HCR_CMD_GO_MASK) == 0) {
1096 TNF_PROBE_1_DEBUG(tavor_write_hcr_loop_count,
1097 TAVOR_TNF_ERROR, "", tnf_uint,
1098 spinloopcount, count);
1099 break;
1100 }
1101 /* Delay before polling the "go" bit again */
1102 drv_usecwait(state->ts_cfg_profile->cp_cmd_poll_delay);
1103
1104 /*
1105 * If we poll more than the maximum number of times,
1106 * then return a "timeout" error.
1107 */
1108 if (++count > countmax) {
1109 #ifdef __lock_lint
1110 mutex_exit(&state-> ts_cmd_regs.hcr_lock);
1111 #else
1112 if (!TAVOR_IN_FASTREBOOT(state)) {
1113 mutex_exit(&state->
1114 ts_cmd_regs.hcr_lock);
1115 }
1116 #endif
1117 TNF_PROBE_0(tavor_write_hcr_timeout2,
1118 TAVOR_TNF_ERROR, "");
1119 TAVOR_TNF_EXIT(tavor_write_hcr);
1120 return (TAVOR_CMD_TIMEOUT);
1121 }
1122 }
1123
1124 /* Pull out the "status" bits from the HCR */
1125 status = (hcrreg >> TAVOR_HCR_CMD_STATUS_SHFT);
1126
1127 /*
1128 * Read the "outparam" value. Note: we have to read "outparam"
1129 * as two separate 32-bit reads because the field in the HCR is
1130 * not 64-bit aligned.
1131 */
1132 hcrreg = ddi_get32(state->ts_reg_cmdhdl, &hcr->out_param0);
1133 cmdpost->cp_outparm = hcrreg << 32;
1134 hcrreg = ddi_get32(state->ts_reg_cmdhdl, &hcr->out_param1);
1135 cmdpost->cp_outparm |= hcrreg;
1136
1137 /* NOSPIN */
1138 } else {
1139 status = TAVOR_CMD_SUCCESS;
1140 }
1141
1142 /* Drop the "HCR access" lock */
1143 #ifdef __lock_lint
1144 mutex_exit(&state->ts_cmd_regs.hcr_lock);
1145 #else
1146 if (!TAVOR_IN_FASTREBOOT(state)) {
1147 mutex_exit(&state->ts_cmd_regs.hcr_lock);
1148 }
1149 #endif
1150
1151 TAVOR_TNF_EXIT(tavor_write_hcr);
1152 return (status);
1153 }
1154
1155
1156 /*
1157 * tavor_outstanding_cmdlist_init()
1158 * Context: Only called from attach() path context
1159 */
1160 int
1161 tavor_outstanding_cmdlist_init(tavor_state_t *state)
1162 {
1163 uint_t num_outstanding_cmds, head, tail;
1164 int i;
1165
1166 TAVOR_TNF_ENTER(tavor_outstanding_cmdlist_init);
1167
1168 /*
1169 * Determine the number of the outstanding commands supported
1789 cmd.cp_opcode = MAD_IFC;
1790 cmd.cp_opmod = TAVOR_CMD_MKEY_DONTCHECK; /* No MKey checking */
1791 cmd.cp_flags = sleepflag;
1792 status = tavor_cmd_post(state, &cmd);
1793 if (status != TAVOR_CMD_SUCCESS) {
1794 TNF_PROBE_0(tavor_getportinfo_cmd_post_fail,
1795 TAVOR_TNF_ERROR, "");
1796 goto getportinfo_fail;
1797 }
1798
1799 /* Sync the mailbox to read the results */
1800 size = sizeof (sm_portinfo_t);
1801 tavor_mbox_sync(mbox_info.mbi_out, TAVOR_CMD_MADDATA_OFFSET,
1802 size, DDI_DMA_SYNC_FORCPU);
1803
1804 /*
1805 * Copy GetPortInfo response MAD into "portinfo". Do any endian
1806 * swapping that may be necessary to flip any of the "portinfo"
1807 * fields
1808 */
1809 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*portinfo))
1810 bcopy((void *)((uintptr_t)mbox_info.mbi_out->mb_addr +
1811 TAVOR_CMD_MADDATA_OFFSET), portinfo, size);
1812 TAVOR_GETPORTINFO_SWAP(portinfo);
1813 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*portinfo))
1814
1815 getportinfo_fail:
1816 /* Free the mailbox */
1817 tavor_mbox_free(state, &mbox_info);
1818
1819 TAVOR_TNF_EXIT(tavor_getportinfo_cmd_post);
1820 return (status);
1821 }
1822
1823
1824 /*
1825 * tavor_getnodeinfo_cmd_post()
1826 * Context: Can be called from interrupt or base context.
1827 * (Currently called only from attach() and detach() path contexts)
1828 */
1829 int
1830 tavor_getnodeinfo_cmd_post(tavor_state_t *state, uint_t sleepflag,
1831 sm_nodeinfo_t *nodeinfo)
1832 {
1833 tavor_mbox_info_t mbox_info;
2023 cmd.cp_inmod = port;
2024 cmd.cp_opcode = MAD_IFC;
2025 cmd.cp_opmod = TAVOR_CMD_MKEY_DONTCHECK; /* No MKey checking */
2026 cmd.cp_flags = sleepflag;
2027 status = tavor_cmd_post(state, &cmd);
2028 if (status != TAVOR_CMD_SUCCESS) {
2029 TNF_PROBE_0(tavor_getguidinfo_cmd_post_fail,
2030 TAVOR_TNF_ERROR, "");
2031 goto getguidinfo_fail;
2032 }
2033
2034 /* Sync the mailbox to read the results */
2035 size = sizeof (sm_guidinfo_t);
2036 tavor_mbox_sync(mbox_info.mbi_out, TAVOR_CMD_MADDATA_OFFSET,
2037 size, DDI_DMA_SYNC_FORCPU);
2038
2039 /*
2040 * Copy GetGUIDInfo response MAD into "guidinfo". Do any endian
2041 * swapping that may be necessary to flip the "guidinfo" fields
2042 */
2043 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*guidinfo))
2044 bcopy((void *)((uintptr_t)mbox_info.mbi_out->mb_addr +
2045 TAVOR_CMD_MADDATA_OFFSET), guidinfo, size);
2046 TAVOR_GETGUIDINFO_SWAP(guidinfo);
2047 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*guidinfo))
2048
2049 getguidinfo_fail:
2050 /* Free the mailbox */
2051 tavor_mbox_free(state, &mbox_info);
2052
2053 TAVOR_TNF_EXIT(tavor_getguidinfo_cmd_post);
2054 return (status);
2055 }
2056
2057
2058 /*
2059 * tavor_getpkeytable_cmd_post()
2060 * Context: Can be called from interrupt or base context.
2061 */
2062 int
2063 tavor_getpkeytable_cmd_post(tavor_state_t *state, uint_t port,
2064 uint_t pkeyblock, uint_t sleepflag, sm_pkey_table_t *pkeytable)
2065 {
2066 tavor_mbox_info_t mbox_info;
2067 tavor_cmd_post_t cmd;
2102 cmd.cp_inmod = port;
2103 cmd.cp_opcode = MAD_IFC;
2104 cmd.cp_opmod = TAVOR_CMD_MKEY_DONTCHECK; /* No MKey checking */
2105 cmd.cp_flags = sleepflag;
2106 status = tavor_cmd_post(state, &cmd);
2107 if (status != TAVOR_CMD_SUCCESS) {
2108 TNF_PROBE_0(tavor_getpkeytable_cmd_post_fail,
2109 TAVOR_TNF_ERROR, "");
2110 goto getpkeytable_fail;
2111 }
2112
2113 /* Sync the mailbox to read the results */
2114 size = sizeof (sm_pkey_table_t);
2115 tavor_mbox_sync(mbox_info.mbi_out, TAVOR_CMD_MADDATA_OFFSET,
2116 size, DDI_DMA_SYNC_FORCPU);
2117
2118 /*
2119 * Copy GetPKeyTable response MAD into "pkeytable". Do any endian
2120 * swapping that may be necessary to flip the "pkeytable" fields
2121 */
2122 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pkeytable))
2123 bcopy((void *)((uintptr_t)mbox_info.mbi_out->mb_addr +
2124 TAVOR_CMD_MADDATA_OFFSET), pkeytable, size);
2125 TAVOR_GETPKEYTABLE_SWAP(pkeytable);
2126 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*pkeytable))
2127
2128 getpkeytable_fail:
2129 /* Free the mailbox */
2130 tavor_mbox_free(state, &mbox_info);
2131
2132 TAVOR_TNF_EXIT(tavor_getpkeytable_cmd_post);
2133 return (status);
2134 }
2135
2136
2137 /*
2138 * tavor_write_mtt_cmd_post()
2139 * Context: Can be called from interrupt or base context.
2140 */
2141 int
2142 tavor_write_mtt_cmd_post(tavor_state_t *state, tavor_mbox_info_t *mbox_info,
2143 uint_t num_mtt, uint_t sleepflag)
2144 {
2145 tavor_cmd_post_t cmd;
2146 uint_t size;
|
58 static void tavor_mbox_sync(tavor_mbox_t *mbox, uint_t offset,
59 uint_t length, uint_t flag);
60
61 /*
62 * tavor_cmd_post()
63 * Context: Can be called from interrupt or base context.
64 *
65 * The "cp_flags" field in cmdpost
66 * is used to determine whether to wait for an available
67 * outstanding command (if necessary) or to return error.
68 */
69 int
70 tavor_cmd_post(tavor_state_t *state, tavor_cmd_post_t *cmdpost)
71 {
72 tavor_cmd_t *cmdptr;
73 int status;
74 uint16_t token;
75
76 TAVOR_TNF_ENTER(tavor_cmd_post);
77
78 /* Determine if we are going to spin until completion */
79 if (cmdpost->cp_flags == TAVOR_CMD_NOSLEEP_SPIN) {
80
81 TNF_PROBE_0_DEBUG(tavor_cmd_post_spin, TAVOR_TNF_TRACE, "");
82
83 /* Write the command to the HCR */
84 status = tavor_write_hcr(state, cmdpost, 0);
85 if (status != TAVOR_CMD_SUCCESS) {
86 TNF_PROBE_0(tavor_cmd_post_fail,
87 TAVOR_TNF_ERROR, "");
88 TAVOR_TNF_EXIT(tavor_cmd_post);
89 return (status);
90 }
91
92 TAVOR_TNF_EXIT(tavor_cmd_post);
93 return (TAVOR_CMD_SUCCESS);
94
95 } else { /* "TAVOR_CMD_SLEEP_NOSPIN" */
96
97 TNF_PROBE_0_DEBUG(tavor_cmd_post_nospin, TAVOR_TNF_TRACE, "");
98
99 ASSERT(TAVOR_SLEEPFLAG_FOR_CONTEXT() != TAVOR_NOSLEEP);
100
101 /* NOTE: Expect threads to be waiting in here */
102 status = tavor_outstanding_cmd_alloc(state, &cmdptr,
103 cmdpost->cp_flags);
104 if (status != TAVOR_CMD_SUCCESS) {
105 TNF_PROBE_0(tavor_cmd_alloc_fail, TAVOR_TNF_ERROR, "");
106 TAVOR_TNF_EXIT(tavor_cmd_post);
107 return (status);
108 }
109
110 /*
111 * Set status to "TAVOR_CMD_INVALID_STATUS". It is
112 * appropriate to do this here without the "cmd_comp_lock"
113 * because this register is overloaded. Later it will be
114 * used to indicate - through a change from this invalid
115 * value to some other value - that the condition variable
116 * has been signaled. Once it has, status will then contain
117 * the _real_ completion status
118 */
119 cmdptr->cmd_status = TAVOR_CMD_INVALID_STATUS;
120
121 /* Write the command to the HCR */
122 token = (uint16_t)cmdptr->cmd_indx;
123 status = tavor_write_hcr(state, cmdpost, token);
124 if (status != TAVOR_CMD_SUCCESS) {
125 tavor_outstanding_cmd_free(state, &cmdptr);
126 TNF_PROBE_0(tavor_cmd_post_fail, TAVOR_TNF_ERROR, "");
127 TAVOR_TNF_EXIT(tavor_cmd_post);
128 return (status);
129 }
130
131 /*
132 * cv_wait() on the "command_complete" condition variable.
133 */
134 mutex_enter(&cmdptr->cmd_comp_lock);
135 while (cmdptr->cmd_status == TAVOR_CMD_INVALID_STATUS) {
136 /* NOTE: EXPECT SEVERAL THREADS TO BE WAITING HERE */
137 cv_wait(&cmdptr->cmd_comp_cv, &cmdptr->cmd_comp_lock);
138 }
139 mutex_exit(&cmdptr->cmd_comp_lock);
140
141 /*
142 * Wake up after command completes (cv_signal). Read status
143 * from the command (success, fail, etc.). It is appropriate
144 * here (as above) to read the status field without the
145 * "cmd_comp_lock" because it is no longer being used to
146 * indicate whether the condition variable has been signaled
147 * (i.e. at this point we are certain that it already has).
148 */
149 status = cmdptr->cmd_status;
150
151 /* Save the "outparam" values into the cmdpost struct */
152 cmdpost->cp_outparm = cmdptr->cmd_outparm;
153
154 /*
155 * Add the command back to the "outstanding commands list".
156 * Signal the "cmd_list" condition variable, if necessary.
157 */
158 tavor_outstanding_cmd_free(state, &cmdptr);
159
221 } else {
222 /* NOTE: Expect threads to be waiting in here */
223 status = tavor_impl_mbox_alloc(state,
224 &state->ts_out_mblist, &mbox_info->mbi_out,
225 mbox_wait);
226 if (status != TAVOR_CMD_SUCCESS) {
227 /* If we allocated an "In" mailbox, free it */
228 if (mbox_info->mbi_alloc_flags &
229 TAVOR_ALLOC_INMBOX) {
230 tavor_impl_mbox_free(
231 &state->ts_in_mblist,
232 &mbox_info->mbi_in);
233 }
234 TAVOR_TNF_EXIT(tavor_mbox_alloc);
235 return (status);
236 }
237 }
238 }
239
240 /* Store appropriate context in mbox_info */
241 mbox_info->mbi_sleep_context = sleep_context;
242
243 TAVOR_TNF_EXIT(tavor_mbox_alloc);
244 return (TAVOR_CMD_SUCCESS);
245 }
246
247
248 /*
249 * tavor_mbox_free()
250 * Context: Can be called from interrupt or base context.
251 */
252 void
253 tavor_mbox_free(tavor_state_t *state, tavor_mbox_info_t *mbox_info)
254 {
255 TAVOR_TNF_ENTER(tavor_mbox_free);
256
257 /*
258 * The mailbox has to be freed in the same context from which it was
259 * allocated. The context is stored in the mbox_info at
260 * tavor_mbox_alloc() time. We check the stored context against the
261 * current context here.
262 */
263 ASSERT(mbox_info->mbi_sleep_context == TAVOR_SLEEPFLAG_FOR_CONTEXT());
264
265 /* Determine correct mboxlist based on calling context */
266 if (mbox_info->mbi_sleep_context == TAVOR_NOSLEEP) {
267 /* Free the intr "In" mailbox */
268 if (mbox_info->mbi_alloc_flags & TAVOR_ALLOC_INMBOX) {
269 tavor_impl_mbox_free(&state->ts_in_intr_mblist,
270 &mbox_info->mbi_in);
271 }
272
273 /* Free the intr "Out" mailbox */
274 if (mbox_info->mbi_alloc_flags & TAVOR_ALLOC_OUTMBOX) {
275 tavor_impl_mbox_free(&state->ts_out_intr_mblist,
276 &mbox_info->mbi_out);
277 }
278 } else {
279 /* Free the "In" mailbox */
280 if (mbox_info->mbi_alloc_flags & TAVOR_ALLOC_INMBOX) {
281 tavor_impl_mbox_free(&state->ts_in_mblist,
282 &mbox_info->mbi_in);
547 TNF_PROBE_0(tavor_impl_mbox_alloc_fail,
548 TAVOR_TNF_ERROR, "");
549 TAVOR_TNF_EXIT(tavor_impl_mbox_alloc);
550 return (TAVOR_CMD_INSUFF_RSRC);
551 }
552
553 /* Delay before polling for mailbox again */
554 drv_usecwait(state->ts_cfg_profile->cp_cmd_poll_delay);
555 mutex_enter(&mblist->mbl_lock);
556 }
557 mblist->mbl_pollers--;
558
559 /* TAVOR_SLEEP */
560 } else {
561 /*
562 * Grab lock here as we prepare to cv_wait if needed.
563 */
564 mutex_enter(&mblist->mbl_lock);
565 while (mblist->mbl_entries_free == 0) {
566 /*
567 * Wait (on cv) for a mailbox to become free.
568 */
569 mblist->mbl_waiters++;
570 cv_wait(&mblist->mbl_cv, &mblist->mbl_lock);
571 }
572 }
573
574 /* Grab the next available mailbox from list */
575 mbox_ptr = mblist->mbl_mbox;
576 index = mblist->mbl_head_indx;
577 next = mbox_ptr[index].mb_next;
578 prev = mbox_ptr[index].mb_prev;
579
580 /* Remove it from the mailbox list */
581 mblist->mbl_mbox[next].mb_prev = prev;
582 mblist->mbl_mbox[prev].mb_next = next;
583 mblist->mbl_head_indx = next;
584
585 /* Update the "free" count and return the mailbox pointer */
586 mblist->mbl_entries_free--;
587 *mb = &mbox_ptr[index];
588
589 mutex_exit(&mblist->mbl_lock);
590
845
846 /* Ensure that outstanding commands are supported */
847 ASSERT(cmd_list->cml_num_alloc != 0);
848
849 /*
850 * If the outstanding command list is empty, then wait (if
851 * appropriate in the current context). Otherwise, grab the
852 * next available command.
853 */
854 while (cmd_list->cml_entries_free == 0) {
855 /* No free commands */
856 if (cmd_wait == TAVOR_NOSLEEP) {
857 mutex_exit(&cmd_list->cml_lock);
858 TNF_PROBE_0(tavor_outstanding_cmd_alloc_fail,
859 TAVOR_TNF_ERROR, "");
860 TAVOR_TNF_EXIT(tavor_outstanding_cmd_alloc);
861 return (TAVOR_CMD_INSUFF_RSRC);
862 }
863
864 /*
865 * Wait (on cv) for a command to become free.
866 */
867 cmd_list->cml_waiters++;
868 cv_wait(&cmd_list->cml_cv, &cmd_list->cml_lock);
869 }
870
871 /* Grab the next available command from the list */
872 head = cmd_list->cml_head_indx;
873 *cmd_ptr = &cmd_list->cml_cmd[head];
874 next = (*cmd_ptr)->cmd_next;
875 prev = (*cmd_ptr)->cmd_prev;
876 (*cmd_ptr)->cmd_status = TAVOR_CMD_INVALID_STATUS;
877
878 /* Remove it from the command list */
879 cmd_list->cml_cmd[next].cmd_prev = prev;
880 cmd_list->cml_cmd[prev].cmd_next = next;
881 cmd_list->cml_head_indx = next;
882
883 /* Update the "free" count and return */
884 cmd_list->cml_entries_free--;
885
886 mutex_exit(&cmd_list->cml_lock);
887
888 TAVOR_TNF_EXIT(tavor_outstanding_cmd_alloc);
889 return (TAVOR_CMD_SUCCESS);
890 }
891
892
893 /*
953 /*
954 * tavor_write_hcr()
955 * Context: Can be called from interrupt or base context.
956 */
957 static int
958 tavor_write_hcr(tavor_state_t *state, tavor_cmd_post_t *cmdpost,
959 uint16_t token)
960 {
961 tavor_hw_hcr_t *hcr;
962 uint_t status, count, countmax;
963 uint64_t hcrreg;
964
965 TAVOR_TNF_ENTER(tavor_write_hcr);
966
967 /*
968 * Grab the "HCR access" lock if the driver is not in
969 * fastreboot. In fastreboot, this function is called
970 * with the single thread but in high interrupt context
971 * (so that this mutex lock cannot be used).
972 */
973 if (!TAVOR_IN_FASTREBOOT(state)) {
974 mutex_enter(&state->ts_cmd_regs.hcr_lock);
975 }
976
977 hcr = state->ts_cmd_regs.hcr;
978
979 /*
980 * First, check the "go" bit to see if the previous hcr usage is
981 * complete. As long as it is set then we must continue to poll.
982 */
983 count = 0;
984 countmax = state->ts_cfg_profile->cp_cmd_poll_max;
985 for (;;) {
986 hcrreg = ddi_get32(state->ts_reg_cmdhdl, &hcr->cmd);
987
988 /* If "go" bit is clear, then done */
989 if ((hcrreg & TAVOR_HCR_CMD_GO_MASK) == 0) {
990 TNF_PROBE_1_DEBUG(tavor_write_hcr_loop_count,
991 TAVOR_TNF_ERROR, "", tnf_uint, nospinloopcount,
992 count);
993 break;
994 }
995 /* Delay before polling the "go" bit again */
996 drv_usecwait(state->ts_cfg_profile->cp_cmd_poll_delay);
997
998 /*
999 * If we poll more than the maximum number of times, then
1000 * return a "timeout" error.
1001 */
1002 if (++count > countmax) {
1003 if (!TAVOR_IN_FASTREBOOT(state)) {
1004 mutex_exit(&state->ts_cmd_regs.hcr_lock);
1005 }
1006 TNF_PROBE_0(tavor_write_hcr_timeout1, TAVOR_TNF_ERROR,
1007 "");
1008 TAVOR_TNF_EXIT(tavor_write_hcr);
1009 return (TAVOR_CMD_TIMEOUT);
1010 }
1011 }
1012
1013 /* Write "inparam" as a 64-bit quantity */
1014 ddi_put64(state->ts_reg_cmdhdl, (uint64_t *)&hcr->in_param0,
1015 cmdpost->cp_inparm);
1016
1017 /* Write "inmod" and 32-bits of "outparam" as 64-bit */
1018 hcrreg = ((uint64_t)cmdpost->cp_inmod << 32);
1019 hcrreg = hcrreg | (cmdpost->cp_outparm >> 32);
1020 ddi_put64(state->ts_reg_cmdhdl, (uint64_t *)&hcr->input_modifier,
1021 hcrreg);
1022
1023 /* Write the other 32-bits of "outparam" and "token" as 64-bit */
1024 hcrreg = (cmdpost->cp_outparm << 32);
1025 hcrreg = hcrreg | ((uint32_t)token << TAVOR_HCR_TOKEN_SHIFT);
1046 countmax = state->ts_cfg_profile->cp_cmd_poll_max;
1047
1048 for (;;) {
1049 hcrreg = ddi_get32(state->ts_reg_cmdhdl, &hcr->cmd);
1050
1051 /* If "go" bit is clear, then done */
1052 if ((hcrreg & TAVOR_HCR_CMD_GO_MASK) == 0) {
1053 TNF_PROBE_1_DEBUG(tavor_write_hcr_loop_count,
1054 TAVOR_TNF_ERROR, "", tnf_uint,
1055 spinloopcount, count);
1056 break;
1057 }
1058 /* Delay before polling the "go" bit again */
1059 drv_usecwait(state->ts_cfg_profile->cp_cmd_poll_delay);
1060
1061 /*
1062 * If we poll more than the maximum number of times,
1063 * then return a "timeout" error.
1064 */
1065 if (++count > countmax) {
1066 if (!TAVOR_IN_FASTREBOOT(state)) {
1067 mutex_exit(&state->
1068 ts_cmd_regs.hcr_lock);
1069 }
1070 TNF_PROBE_0(tavor_write_hcr_timeout2,
1071 TAVOR_TNF_ERROR, "");
1072 TAVOR_TNF_EXIT(tavor_write_hcr);
1073 return (TAVOR_CMD_TIMEOUT);
1074 }
1075 }
1076
1077 /* Pull out the "status" bits from the HCR */
1078 status = (hcrreg >> TAVOR_HCR_CMD_STATUS_SHFT);
1079
1080 /*
1081 * Read the "outparam" value. Note: we have to read "outparam"
1082 * as two separate 32-bit reads because the field in the HCR is
1083 * not 64-bit aligned.
1084 */
1085 hcrreg = ddi_get32(state->ts_reg_cmdhdl, &hcr->out_param0);
1086 cmdpost->cp_outparm = hcrreg << 32;
1087 hcrreg = ddi_get32(state->ts_reg_cmdhdl, &hcr->out_param1);
1088 cmdpost->cp_outparm |= hcrreg;
1089
1090 /* NOSPIN */
1091 } else {
1092 status = TAVOR_CMD_SUCCESS;
1093 }
1094
1095 /* Drop the "HCR access" lock */
1096 if (!TAVOR_IN_FASTREBOOT(state)) {
1097 mutex_exit(&state->ts_cmd_regs.hcr_lock);
1098 }
1099
1100 TAVOR_TNF_EXIT(tavor_write_hcr);
1101 return (status);
1102 }
1103
1104
1105 /*
1106 * tavor_outstanding_cmdlist_init()
1107 * Context: Only called from attach() path context
1108 */
1109 int
1110 tavor_outstanding_cmdlist_init(tavor_state_t *state)
1111 {
1112 uint_t num_outstanding_cmds, head, tail;
1113 int i;
1114
1115 TAVOR_TNF_ENTER(tavor_outstanding_cmdlist_init);
1116
1117 /*
1118 * Determine the number of the outstanding commands supported
1738 cmd.cp_opcode = MAD_IFC;
1739 cmd.cp_opmod = TAVOR_CMD_MKEY_DONTCHECK; /* No MKey checking */
1740 cmd.cp_flags = sleepflag;
1741 status = tavor_cmd_post(state, &cmd);
1742 if (status != TAVOR_CMD_SUCCESS) {
1743 TNF_PROBE_0(tavor_getportinfo_cmd_post_fail,
1744 TAVOR_TNF_ERROR, "");
1745 goto getportinfo_fail;
1746 }
1747
1748 /* Sync the mailbox to read the results */
1749 size = sizeof (sm_portinfo_t);
1750 tavor_mbox_sync(mbox_info.mbi_out, TAVOR_CMD_MADDATA_OFFSET,
1751 size, DDI_DMA_SYNC_FORCPU);
1752
1753 /*
1754 * Copy GetPortInfo response MAD into "portinfo". Do any endian
1755 * swapping that may be necessary to flip any of the "portinfo"
1756 * fields
1757 */
1758 bcopy((void *)((uintptr_t)mbox_info.mbi_out->mb_addr +
1759 TAVOR_CMD_MADDATA_OFFSET), portinfo, size);
1760 TAVOR_GETPORTINFO_SWAP(portinfo);
1761
1762 getportinfo_fail:
1763 /* Free the mailbox */
1764 tavor_mbox_free(state, &mbox_info);
1765
1766 TAVOR_TNF_EXIT(tavor_getportinfo_cmd_post);
1767 return (status);
1768 }
1769
1770
1771 /*
1772 * tavor_getnodeinfo_cmd_post()
1773 * Context: Can be called from interrupt or base context.
1774 * (Currently called only from attach() and detach() path contexts)
1775 */
1776 int
1777 tavor_getnodeinfo_cmd_post(tavor_state_t *state, uint_t sleepflag,
1778 sm_nodeinfo_t *nodeinfo)
1779 {
1780 tavor_mbox_info_t mbox_info;
1970 cmd.cp_inmod = port;
1971 cmd.cp_opcode = MAD_IFC;
1972 cmd.cp_opmod = TAVOR_CMD_MKEY_DONTCHECK; /* No MKey checking */
1973 cmd.cp_flags = sleepflag;
1974 status = tavor_cmd_post(state, &cmd);
1975 if (status != TAVOR_CMD_SUCCESS) {
1976 TNF_PROBE_0(tavor_getguidinfo_cmd_post_fail,
1977 TAVOR_TNF_ERROR, "");
1978 goto getguidinfo_fail;
1979 }
1980
1981 /* Sync the mailbox to read the results */
1982 size = sizeof (sm_guidinfo_t);
1983 tavor_mbox_sync(mbox_info.mbi_out, TAVOR_CMD_MADDATA_OFFSET,
1984 size, DDI_DMA_SYNC_FORCPU);
1985
1986 /*
1987 * Copy GetGUIDInfo response MAD into "guidinfo". Do any endian
1988 * swapping that may be necessary to flip the "guidinfo" fields
1989 */
1990 bcopy((void *)((uintptr_t)mbox_info.mbi_out->mb_addr +
1991 TAVOR_CMD_MADDATA_OFFSET), guidinfo, size);
1992 TAVOR_GETGUIDINFO_SWAP(guidinfo);
1993
1994 getguidinfo_fail:
1995 /* Free the mailbox */
1996 tavor_mbox_free(state, &mbox_info);
1997
1998 TAVOR_TNF_EXIT(tavor_getguidinfo_cmd_post);
1999 return (status);
2000 }
2001
2002
2003 /*
2004 * tavor_getpkeytable_cmd_post()
2005 * Context: Can be called from interrupt or base context.
2006 */
2007 int
2008 tavor_getpkeytable_cmd_post(tavor_state_t *state, uint_t port,
2009 uint_t pkeyblock, uint_t sleepflag, sm_pkey_table_t *pkeytable)
2010 {
2011 tavor_mbox_info_t mbox_info;
2012 tavor_cmd_post_t cmd;
2047 cmd.cp_inmod = port;
2048 cmd.cp_opcode = MAD_IFC;
2049 cmd.cp_opmod = TAVOR_CMD_MKEY_DONTCHECK; /* No MKey checking */
2050 cmd.cp_flags = sleepflag;
2051 status = tavor_cmd_post(state, &cmd);
2052 if (status != TAVOR_CMD_SUCCESS) {
2053 TNF_PROBE_0(tavor_getpkeytable_cmd_post_fail,
2054 TAVOR_TNF_ERROR, "");
2055 goto getpkeytable_fail;
2056 }
2057
2058 /* Sync the mailbox to read the results */
2059 size = sizeof (sm_pkey_table_t);
2060 tavor_mbox_sync(mbox_info.mbi_out, TAVOR_CMD_MADDATA_OFFSET,
2061 size, DDI_DMA_SYNC_FORCPU);
2062
2063 /*
2064 * Copy GetPKeyTable response MAD into "pkeytable". Do any endian
2065 * swapping that may be necessary to flip the "pkeytable" fields
2066 */
2067 bcopy((void *)((uintptr_t)mbox_info.mbi_out->mb_addr +
2068 TAVOR_CMD_MADDATA_OFFSET), pkeytable, size);
2069 TAVOR_GETPKEYTABLE_SWAP(pkeytable);
2070
2071 getpkeytable_fail:
2072 /* Free the mailbox */
2073 tavor_mbox_free(state, &mbox_info);
2074
2075 TAVOR_TNF_EXIT(tavor_getpkeytable_cmd_post);
2076 return (status);
2077 }
2078
2079
2080 /*
2081 * tavor_write_mtt_cmd_post()
2082 * Context: Can be called from interrupt or base context.
2083 */
2084 int
2085 tavor_write_mtt_cmd_post(tavor_state_t *state, tavor_mbox_info_t *mbox_info,
2086 uint_t num_mtt, uint_t sleepflag)
2087 {
2088 tavor_cmd_post_t cmd;
2089 uint_t size;
|