1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /*
28 * tavor_ioctl.c
29 * Tavor IOCTL Routines
30 *
31 * Implements all ioctl access into the driver. This includes all routines
32 * necessary for updating firmware, accessing the tavor flash device, and
33 * providing interfaces for VTS.
34 */
35
36 #include <sys/types.h>
37 #include <sys/conf.h>
38 #include <sys/ddi.h>
39 #include <sys/sunddi.h>
40 #include <sys/modctl.h>
41 #include <sys/file.h>
42
43 #include <sys/ib/adapters/tavor/tavor.h>
44
45 /* Tavor HCA state pointer (extern) */
46 extern void *tavor_statep;
47
48 /*
49 * The ioctl declarations (for firmware flash burning, register read/write
50 * (DEBUG-only), and VTS interfaces)
51 */
52 static int tavor_ioctl_flash_read(tavor_state_t *state, dev_t dev,
53 intptr_t arg, int mode);
54 static int tavor_ioctl_flash_write(tavor_state_t *state, dev_t dev,
55 intptr_t arg, int mode);
56 static int tavor_ioctl_flash_erase(tavor_state_t *state, dev_t dev,
57 intptr_t arg, int mode);
58 static int tavor_ioctl_flash_init(tavor_state_t *state, dev_t dev,
59 intptr_t arg, int mode);
60 static int tavor_ioctl_flash_fini(tavor_state_t *state, dev_t dev);
61 static void tavor_ioctl_flash_cleanup(tavor_state_t *state);
62 static void tavor_ioctl_flash_cleanup_nolock(tavor_state_t *state);
63 #ifdef DEBUG
64 static int tavor_ioctl_reg_write(tavor_state_t *state, intptr_t arg,
65 int mode);
66 static int tavor_ioctl_reg_read(tavor_state_t *state, intptr_t arg,
67 int mode);
68 #endif /* DEBUG */
69 static int tavor_ioctl_info(tavor_state_t *state, dev_t dev,
70 intptr_t arg, int mode);
71 static int tavor_ioctl_ports(tavor_state_t *state, intptr_t arg,
72 int mode);
73 static int tavor_ioctl_loopback(tavor_state_t *state, intptr_t arg,
74 int mode);
75 static int tavor_ioctl_ddr_read(tavor_state_t *state, intptr_t arg,
76 int mode);
77
78 /* Tavor Flash Functions */
79 static void tavor_flash_read_sector(tavor_state_t *state, uint32_t sector_num);
80 static void tavor_flash_read_quadlet(tavor_state_t *state, uint32_t *data,
81 uint32_t addr);
82 static int tavor_flash_write_sector(tavor_state_t *state, uint32_t sector_num);
83 static int tavor_flash_write_byte(tavor_state_t *state, uint32_t addr,
84 uchar_t data);
85 static int tavor_flash_erase_sector(tavor_state_t *state, uint32_t sector_num);
86 static int tavor_flash_erase_chip(tavor_state_t *state);
87 static void tavor_flash_bank(tavor_state_t *state, uint32_t addr);
88 static uint32_t tavor_flash_read(tavor_state_t *state, uint32_t addr);
89 static void tavor_flash_write(tavor_state_t *state, uint32_t addr,
90 uchar_t data);
91 static void tavor_flash_init(tavor_state_t *state);
92 static void tavor_flash_cfi_init(tavor_state_t *state, uint32_t *cfi_info,
93 int *intel_xcmd);
94 static void tavor_flash_fini(tavor_state_t *state);
95 static void tavor_flash_reset(tavor_state_t *state);
96 static uint32_t tavor_flash_read_cfg(ddi_acc_handle_t pci_config_hdl,
97 uint32_t addr);
98 static void tavor_flash_write_cfg(ddi_acc_handle_t pci_config_hdl,
99 uint32_t addr, uint32_t data);
100 static void tavor_flash_cfi_byte(uint8_t *ch, uint32_t dword, int i);
101 static void tavor_flash_cfi_dword(uint32_t *dword, uint8_t *ch, int i);
102
103 /* Tavor loopback test functions */
104 static void tavor_loopback_free_qps(tavor_loopback_state_t *lstate);
105 static void tavor_loopback_free_state(tavor_loopback_state_t *lstate);
106 static int tavor_loopback_init(tavor_state_t *state,
107 tavor_loopback_state_t *lstate);
108 static void tavor_loopback_init_qp_info(tavor_loopback_state_t *lstate,
109 tavor_loopback_comm_t *comm);
110 static int tavor_loopback_alloc_mem(tavor_loopback_state_t *lstate,
111 tavor_loopback_comm_t *comm, int sz);
112 static int tavor_loopback_alloc_qps(tavor_loopback_state_t *lstate,
113 tavor_loopback_comm_t *comm);
114 static int tavor_loopback_modify_qp(tavor_loopback_state_t *lstate,
115 tavor_loopback_comm_t *comm, uint_t qp_num);
116 static int tavor_loopback_copyout(tavor_loopback_ioctl_t *lb,
117 intptr_t arg, int mode);
118 static int tavor_loopback_post_send(tavor_loopback_state_t *lstate,
119 tavor_loopback_comm_t *tx, tavor_loopback_comm_t *rx);
120 static int tavor_loopback_poll_cq(tavor_loopback_state_t *lstate,
121 tavor_loopback_comm_t *comm);
122
123 /* Patchable timeout values for flash operations */
124 int tavor_hw_flash_timeout_gpio_sema = TAVOR_HW_FLASH_TIMEOUT_GPIO_SEMA;
125 int tavor_hw_flash_timeout_config = TAVOR_HW_FLASH_TIMEOUT_CONFIG;
126 int tavor_hw_flash_timeout_write = TAVOR_HW_FLASH_TIMEOUT_WRITE;
127 int tavor_hw_flash_timeout_erase = TAVOR_HW_FLASH_TIMEOUT_ERASE;
128
129 /*
130 * tavor_ioctl()
131 */
132 /* ARGSUSED */
133 int
134 tavor_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp,
135 int *rvalp)
136 {
137 tavor_state_t *state;
138 minor_t instance;
139 int status;
140
141 TAVOR_TNF_ENTER(tavor_ioctl);
142
143 if (drv_priv(credp) != 0) {
144 TNF_PROBE_0(tavor_ioctl_priv_fail, TAVOR_TNF_ERROR, "");
145 TAVOR_TNF_EXIT(tavor_ioctl);
146 return (EPERM);
147 }
148
149 instance = TAVOR_DEV_INSTANCE(dev);
150 if (instance == -1) {
151 TNF_PROBE_0(tavor_ioctl_inst_fail, TAVOR_TNF_ERROR, "");
152 TAVOR_TNF_EXIT(tavor_ioctl);
153 return (EBADF);
154 }
155
156 state = ddi_get_soft_state(tavor_statep, instance);
157 if (state == NULL) {
158 TNF_PROBE_0(tavor_ioctl_gss_fail, TAVOR_TNF_ERROR, "");
159 TAVOR_TNF_EXIT(tavor_ioctl);
160 return (EBADF);
161 }
162
163 status = 0;
164
165 switch (cmd) {
166 case TAVOR_IOCTL_FLASH_READ:
167 status = tavor_ioctl_flash_read(state, dev, arg, mode);
168 break;
169
170 case TAVOR_IOCTL_FLASH_WRITE:
171 status = tavor_ioctl_flash_write(state, dev, arg, mode);
172 break;
173
174 case TAVOR_IOCTL_FLASH_ERASE:
175 status = tavor_ioctl_flash_erase(state, dev, arg, mode);
176 break;
177
178 case TAVOR_IOCTL_FLASH_INIT:
179 status = tavor_ioctl_flash_init(state, dev, arg, mode);
180 break;
181
182 case TAVOR_IOCTL_FLASH_FINI:
183 status = tavor_ioctl_flash_fini(state, dev);
184 break;
185
186 case TAVOR_IOCTL_INFO:
187 status = tavor_ioctl_info(state, dev, arg, mode);
188 break;
189
190 case TAVOR_IOCTL_PORTS:
191 status = tavor_ioctl_ports(state, arg, mode);
192 break;
193
194 case TAVOR_IOCTL_DDR_READ:
195 status = tavor_ioctl_ddr_read(state, arg, mode);
196 break;
197
198 case TAVOR_IOCTL_LOOPBACK:
199 status = tavor_ioctl_loopback(state, arg, mode);
200 break;
201
202 #ifdef DEBUG
203 case TAVOR_IOCTL_REG_WRITE:
204 status = tavor_ioctl_reg_write(state, arg, mode);
205 break;
206
207 case TAVOR_IOCTL_REG_READ:
208 status = tavor_ioctl_reg_read(state, arg, mode);
209 break;
210 #endif /* DEBUG */
211
212 default:
213 status = ENOTTY;
214 TNF_PROBE_0(tavor_ioctl_default_fail, TAVOR_TNF_ERROR, "");
215 break;
216 }
217 *rvalp = status;
218
219 TAVOR_TNF_EXIT(tavor_ioctl);
220 return (status);
221 }
222
223 /*
224 * tavor_ioctl_flash_read()
225 */
226 static int
227 tavor_ioctl_flash_read(tavor_state_t *state, dev_t dev, intptr_t arg, int mode)
228 {
229 tavor_flash_ioctl_t ioctl_info;
230 int status = 0;
231
232 TAVOR_TNF_ENTER(tavor_ioctl_flash_read);
233
234 /*
235 * Check that flash init ioctl has been called first. And check
236 * that the same dev_t that called init is the one calling read now.
237 */
238 mutex_enter(&state->ts_fw_flashlock);
239 if ((state->ts_fw_flashdev != dev) ||
240 (state->ts_fw_flashstarted == 0)) {
241 mutex_exit(&state->ts_fw_flashlock);
242 TNF_PROBE_0(tavor_flash_bad_state, TAVOR_TNF_ERROR, "");
243 TAVOR_TNF_EXIT(tavor_ioctl_flash_read);
244 return (EIO);
245 }
246
247 /* copy user struct to kernel */
248 #ifdef _MULTI_DATAMODEL
249 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
250 tavor_flash_ioctl32_t info32;
251
252 if (ddi_copyin((void *)arg, &info32,
253 sizeof (tavor_flash_ioctl32_t), mode) != 0) {
254 mutex_exit(&state->ts_fw_flashlock);
255 TNF_PROBE_0(tavor_ioctl_flash_read_copyin_fail,
256 TAVOR_TNF_ERROR, "");
257 TAVOR_TNF_EXIT(tavor_ioctl_flash_read);
258 return (EFAULT);
259 }
260 ioctl_info.tf_type = info32.tf_type;
261 ioctl_info.tf_sector = (caddr_t)(uintptr_t)info32.tf_sector;
262 ioctl_info.tf_sector_num = info32.tf_sector_num;
263 ioctl_info.tf_addr = info32.tf_addr;
264 } else
265 #endif /* _MULTI_DATAMODEL */
266 if (ddi_copyin((void *)arg, &ioctl_info, sizeof (tavor_flash_ioctl_t),
267 mode) != 0) {
268 mutex_exit(&state->ts_fw_flashlock);
269 TNF_PROBE_0(tavor_ioctl_flash_read_copyin_fail,
270 TAVOR_TNF_ERROR, "");
271 TAVOR_TNF_EXIT(tavor_ioctl_flash_read);
272 return (EFAULT);
273 }
274
275 /*
276 * Determine type of READ ioctl
277 */
278 switch (ioctl_info.tf_type) {
279 case TAVOR_FLASH_READ_SECTOR:
280 /* Check if sector num is too large for flash device */
281 if (ioctl_info.tf_sector_num >=
282 (state->ts_fw_device_sz >> state->ts_fw_log_sector_sz)) {
283 mutex_exit(&state->ts_fw_flashlock);
284 TNF_PROBE_0(tavor_flash_read_sector_num_too_large,
285 TAVOR_TNF_ERROR, "");
286 TAVOR_TNF_EXIT(tavor_ioctl_flash_read);
287 return (EFAULT);
288 }
289
290 /* Perform the Sector Read */
291 tavor_flash_reset(state);
292 tavor_flash_read_sector(state, ioctl_info.tf_sector_num);
293
294 /* copyout the firmware sector image data */
295 if (ddi_copyout(&state->ts_fw_sector[0],
296 &ioctl_info.tf_sector[0], 1 << state->ts_fw_log_sector_sz,
297 mode) != 0) {
298 mutex_exit(&state->ts_fw_flashlock);
299 TNF_PROBE_0(tavor_flash_read_copyout_fail,
300 TAVOR_TNF_ERROR, "");
301 TAVOR_TNF_EXIT(tavor_ioctl_flash_read);
302 return (EFAULT);
303 }
304 break;
305
306 case TAVOR_FLASH_READ_QUADLET:
307 /* Check if addr is too large for flash device */
308 if (ioctl_info.tf_addr >= state->ts_fw_device_sz) {
309 mutex_exit(&state->ts_fw_flashlock);
310 TNF_PROBE_0(tavor_flash_read_quad_addr_too_large,
311 TAVOR_TNF_ERROR, "");
312 TAVOR_TNF_EXIT(tavor_ioctl_flash_read);
313 return (EFAULT);
314 }
315
316 /* Perform the Quadlet Read */
317 tavor_flash_reset(state);
318 tavor_flash_read_quadlet(state, &ioctl_info.tf_quadlet,
319 ioctl_info.tf_addr);
320 break;
321
322 default:
323 TNF_PROBE_0(tavor_ioctl_flash_read_invalid_type,
324 TAVOR_TNF_ERROR, "");
325 status = EIO;
326 break;
327 }
328
329 /* copy results back to userland */
330 #ifdef _MULTI_DATAMODEL
331 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
332 tavor_flash_ioctl32_t info32;
333
334 info32.tf_quadlet = ioctl_info.tf_quadlet;
335 info32.tf_type = ioctl_info.tf_type;
336 info32.tf_sector_num = ioctl_info.tf_sector_num;
337 info32.tf_sector = (caddr32_t)(uintptr_t)ioctl_info.tf_sector;
338 info32.tf_addr = ioctl_info.tf_addr;
339
340 if (ddi_copyout(&info32, (void *)arg,
341 sizeof (tavor_flash_ioctl32_t), mode) != 0) {
342 mutex_exit(&state->ts_fw_flashlock);
343 TNF_PROBE_0(tavor_flash_read_copyout_fail,
344 TAVOR_TNF_ERROR, "");
345 TAVOR_TNF_EXIT(tavor_ioctl_flash_read);
346 return (EFAULT);
347 }
348 } else
349 #endif /* _MULTI_DATAMODEL */
350 if (ddi_copyout(&ioctl_info, (void *)arg,
351 sizeof (tavor_flash_ioctl_t), mode) != 0) {
352 mutex_exit(&state->ts_fw_flashlock);
353 TNF_PROBE_0(tavor_flash_read_copyout_fail,
354 TAVOR_TNF_ERROR, "");
355 TAVOR_TNF_EXIT(tavor_ioctl_flash_read);
356 return (EFAULT);
357 }
358
359 mutex_exit(&state->ts_fw_flashlock);
360 TAVOR_TNF_EXIT(tavor_ioctl_flash_read);
361 return (status);
362 }
363
364 /*
365 * tavor_ioctl_flash_write()
366 */
367 static int
368 tavor_ioctl_flash_write(tavor_state_t *state, dev_t dev, intptr_t arg, int mode)
369 {
370 tavor_flash_ioctl_t ioctl_info;
371 int status = 0;
372
373 TAVOR_TNF_ENTER(tavor_ioctl_flash_write);
374
375 /*
376 * Check that flash init ioctl has been called first. And check
377 * that the same dev_t that called init is the one calling write now.
378 */
379 mutex_enter(&state->ts_fw_flashlock);
380 if ((state->ts_fw_flashdev != dev) ||
381 (state->ts_fw_flashstarted == 0)) {
382 mutex_exit(&state->ts_fw_flashlock);
383 TNF_PROBE_0(tavor_flash_bad_state, TAVOR_TNF_ERROR, "");
384 TAVOR_TNF_EXIT(tavor_ioctl_flash_write);
385 return (EIO);
386 }
387
388 /* copy user struct to kernel */
389 #ifdef _MULTI_DATAMODEL
390 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
391 tavor_flash_ioctl32_t info32;
392
393 if (ddi_copyin((void *)arg, &info32,
394 sizeof (tavor_flash_ioctl32_t), mode) != 0) {
395 mutex_exit(&state->ts_fw_flashlock);
396 TNF_PROBE_0(tavor_ioctl_flash_write_copyin_fail,
397 TAVOR_TNF_ERROR, "");
398 TAVOR_TNF_EXIT(tavor_ioctl_flash_write);
399 return (EFAULT);
400 }
401 ioctl_info.tf_type = info32.tf_type;
402 ioctl_info.tf_sector = (caddr_t)(uintptr_t)info32.tf_sector;
403 ioctl_info.tf_sector_num = info32.tf_sector_num;
404 ioctl_info.tf_addr = info32.tf_addr;
405 ioctl_info.tf_byte = info32.tf_byte;
406 } else
407 #endif /* _MULTI_DATAMODEL */
408 if (ddi_copyin((void *)arg, &ioctl_info,
409 sizeof (tavor_flash_ioctl_t), mode) != 0) {
410 mutex_exit(&state->ts_fw_flashlock);
411 TNF_PROBE_0(tavor_ioctl_flash_write_ci_fail,
412 TAVOR_TNF_ERROR, "");
413 TAVOR_TNF_EXIT(tavor_ioctl_flash_write);
414 return (EFAULT);
415 }
416
417 /*
418 * Determine type of WRITE ioctl
419 */
420 switch (ioctl_info.tf_type) {
421 case TAVOR_FLASH_WRITE_SECTOR:
422 /* Check if sector num is too large for flash device */
423 if (ioctl_info.tf_sector_num >=
424 (state->ts_fw_device_sz >> state->ts_fw_log_sector_sz)) {
425 mutex_exit(&state->ts_fw_flashlock);
426 TNF_PROBE_0(tavor_flash_write_sector_num_too_large,
427 TAVOR_TNF_ERROR, "");
428 TAVOR_TNF_EXIT(tavor_ioctl_flash_write);
429 return (EFAULT);
430 }
431
432 /* copy in fw sector image data */
433 if (ddi_copyin(&ioctl_info.tf_sector[0],
434 &state->ts_fw_sector[0], 1 << state->ts_fw_log_sector_sz,
435 mode) != 0) {
436 mutex_exit(&state->ts_fw_flashlock);
437 TNF_PROBE_0(tavor_ioctl_flash_write_fw_sector_ci_fail,
438 TAVOR_TNF_ERROR, "");
439 TAVOR_TNF_EXIT(tavor_ioctl_flash_write);
440 return (EFAULT);
441 }
442
443 /* Perform Write Sector */
444 status = tavor_flash_write_sector(state,
445 ioctl_info.tf_sector_num);
446 break;
447
448 case TAVOR_FLASH_WRITE_BYTE:
449 /* Check if addr is too large for flash device */
450 if (ioctl_info.tf_addr >= state->ts_fw_device_sz) {
451 mutex_exit(&state->ts_fw_flashlock);
452 TNF_PROBE_0(tavor_flash_write_byte_addr_too_large,
453 TAVOR_TNF_ERROR, "");
454 TAVOR_TNF_EXIT(tavor_ioctl_flash_write);
455 return (EFAULT);
456 }
457
458 /* Perform Write Byte */
459 tavor_flash_bank(state, ioctl_info.tf_addr);
460 tavor_flash_reset(state);
461 status = tavor_flash_write_byte(state, ioctl_info.tf_addr,
462 ioctl_info.tf_byte);
463 tavor_flash_reset(state);
464 break;
465
466 default:
467 TNF_PROBE_0(tavor_ioctl_flash_write_invalid_type,
468 TAVOR_TNF_ERROR, "");
469 status = EIO;
470 break;
471 }
472
473 mutex_exit(&state->ts_fw_flashlock);
474 TAVOR_TNF_EXIT(tavor_ioctl_flash_write);
475 return (status);
476 }
477
478 /*
479 * tavor_ioctl_flash_erase()
480 */
481 static int
482 tavor_ioctl_flash_erase(tavor_state_t *state, dev_t dev, intptr_t arg, int mode)
483 {
484 tavor_flash_ioctl_t ioctl_info;
485 int status = 0;
486
487 TAVOR_TNF_ENTER(tavor_ioctl_flash_erase);
488
489 /*
490 * Check that flash init ioctl has been called first. And check
491 * that the same dev_t that called init is the one calling erase now.
492 */
493 mutex_enter(&state->ts_fw_flashlock);
494 if ((state->ts_fw_flashdev != dev) ||
495 (state->ts_fw_flashstarted == 0)) {
496 mutex_exit(&state->ts_fw_flashlock);
497 TNF_PROBE_0(tavor_flash_bad_state, TAVOR_TNF_ERROR, "");
498 TAVOR_TNF_EXIT(tavor_ioctl_flash_erase);
499 return (EIO);
500 }
501
502 /* copy user struct to kernel */
503 #ifdef _MULTI_DATAMODEL
504 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
505 tavor_flash_ioctl32_t info32;
506
507 if (ddi_copyin((void *)arg, &info32,
508 sizeof (tavor_flash_ioctl32_t), mode) != 0) {
509 mutex_exit(&state->ts_fw_flashlock);
510 TNF_PROBE_0(tavor_ioctl_flash_read_copyin_fail,
511 TAVOR_TNF_ERROR, "");
512 TAVOR_TNF_EXIT(tavor_ioctl_flash_erase);
513 return (EFAULT);
514 }
515 ioctl_info.tf_type = info32.tf_type;
516 ioctl_info.tf_sector_num = info32.tf_sector_num;
517 } else
518 #endif /* _MULTI_DATAMODEL */
519 if (ddi_copyin((void *)arg, &ioctl_info, sizeof (tavor_flash_ioctl_t),
520 mode) != 0) {
521 mutex_exit(&state->ts_fw_flashlock);
522 TNF_PROBE_0(tavor_ioctl_flash_erase_ci_fail,
523 TAVOR_TNF_ERROR, "");
524 TAVOR_TNF_EXIT(tavor_ioctl_flash_erase);
525 return (EFAULT);
526 }
527
528 /*
529 * Determine type of ERASE ioctl
530 */
531 switch (ioctl_info.tf_type) {
532 case TAVOR_FLASH_ERASE_SECTOR:
533 /* Check if sector num is too large for flash device */
534 if (ioctl_info.tf_sector_num >=
535 (state->ts_fw_device_sz >> state->ts_fw_log_sector_sz)) {
536 mutex_exit(&state->ts_fw_flashlock);
537 TNF_PROBE_0(tavor_flash_erase_sector_num_too_large,
538 TAVOR_TNF_ERROR, "");
539 TAVOR_TNF_EXIT(tavor_ioctl_flash_write);
540 return (EFAULT);
541 }
542
543 /* Perform Sector Erase */
544 status = tavor_flash_erase_sector(state,
545 ioctl_info.tf_sector_num);
546 break;
547
548 case TAVOR_FLASH_ERASE_CHIP:
549 /* Perform Chip Erase */
550 status = tavor_flash_erase_chip(state);
551 break;
552
553 default:
554 TNF_PROBE_0(tavor_ioctl_flash_erase_invalid_type,
555 TAVOR_TNF_ERROR, "");
556 status = EIO;
557 break;
558 }
559
560 mutex_exit(&state->ts_fw_flashlock);
561 TAVOR_TNF_EXIT(tavor_ioctl_flash_erase);
562 return (status);
563 }
564
565 /*
566 * tavor_ioctl_flash_init()
567 */
568 static int
569 tavor_ioctl_flash_init(tavor_state_t *state, dev_t dev, intptr_t arg, int mode)
570 {
571 tavor_flash_init_ioctl_t init_info;
572 int ret;
573 int intel_xcmd = 0;
574
575 TAVOR_TNF_ENTER(tavor_ioctl_flash_init);
576
577 /*
578 * init cannot be called more than once. If we have already init'd the
579 * flash, return directly.
580 */
581 mutex_enter(&state->ts_fw_flashlock);
582 if (state->ts_fw_flashstarted == 1) {
583 mutex_exit(&state->ts_fw_flashlock);
584 TNF_PROBE_0(tavor_ioctl_flash_init_already_started,
585 TAVOR_TNF_ERROR, "");
586 TAVOR_TNF_EXIT(tavor_ioctl_flash_init);
587 return (EIO);
588 }
589
590 /* copyin the user struct to kernel */
591 if (ddi_copyin((void *)arg, &init_info,
592 sizeof (tavor_flash_init_ioctl_t), mode) != 0) {
593 mutex_exit(&state->ts_fw_flashlock);
594 TNF_PROBE_0(tavor_flash_init_ioctl_copyin_fail,
595 TAVOR_TNF_ERROR, "");
596 TAVOR_TNF_EXIT(tavor_ioctl_flash_init);
597 return (EFAULT);
598 }
599
600 /* Init Flash */
601 tavor_flash_init(state);
602
603 /* Read CFI info */
604 tavor_flash_cfi_init(state, &init_info.tf_cfi_info[0], &intel_xcmd);
605
606 /*
607 * Return error if the command set is unknown.
608 */
609 if (state->ts_fw_cmdset == TAVOR_FLASH_UNKNOWN_CMDSET) {
610 mutex_exit(&state->ts_fw_flashlock);
611 TNF_PROBE_1(tavor_ioctl_flash_init_cmdset_fail,
612 TAVOR_TNF_ERROR, "", tnf_string, errmsg,
613 "UNKNOWN flash command set");
614 TAVOR_TNF_EXIT(tavor_ioctl_flash_init);
615 return (EFAULT);
616 }
617
618 /* Read HWREV - least significant 8 bits is revision ID */
619 init_info.tf_hwrev = pci_config_get32(state->ts_pci_cfghdl,
620 TAVOR_HW_FLASH_CFG_HWREV) & 0xFF;
621
622 /* Fill in the firmwate revision numbers */
623 init_info.tf_fwrev.tfi_maj = state->ts_fw.fw_rev_major;
624 init_info.tf_fwrev.tfi_min = state->ts_fw.fw_rev_minor;
625 init_info.tf_fwrev.tfi_sub = state->ts_fw.fw_rev_subminor;
626
627 /* Alloc flash mem for one sector size */
628 state->ts_fw_sector = (uint32_t *)kmem_zalloc(1 <<
629 state->ts_fw_log_sector_sz, KM_SLEEP);
630
631 /* Set HW part number and length */
632 init_info.tf_pn_len = state->ts_hca_pn_len;
633 if (state->ts_hca_pn_len != 0) {
634 (void) memcpy(init_info.tf_hwpn, state->ts_hca_pn,
635 state->ts_hca_pn_len);
636 }
637
638 /* Copy ioctl results back to userland */
639 if (ddi_copyout(&init_info, (void *)arg,
640 sizeof (tavor_flash_init_ioctl_t), mode) != 0) {
641
642 tavor_ioctl_flash_cleanup_nolock(state);
643
644 mutex_exit(&state->ts_fw_flashlock);
645 TNF_PROBE_0(tavor_ioctl_flash_init_copyout_fail,
646 TAVOR_TNF_ERROR, "");
647 TAVOR_TNF_EXIT(tavor_ioctl_flash_init);
648 return (EFAULT);
649 }
650
651 /* Set flash state to started */
652 state->ts_fw_flashstarted = 1;
653 state->ts_fw_flashdev = dev;
654
655 mutex_exit(&state->ts_fw_flashlock);
656
657 /*
658 * If "flash init" is successful, add an "on close" callback to the
659 * current dev node to ensure that "flash fini" gets called later
660 * even if the userland process prematurely exits.
661 */
662 ret = tavor_umap_db_set_onclose_cb(dev,
663 TAVOR_ONCLOSE_FLASH_INPROGRESS,
664 (void (*)(void *))tavor_ioctl_flash_cleanup, state);
665 if (ret != DDI_SUCCESS) {
666 (void) tavor_ioctl_flash_fini(state, dev);
667
668 TNF_PROBE_0(tavor_ioctl_flash_init_set_cb_fail,
669 TAVOR_TNF_ERROR, "");
670 TAVOR_TNF_EXIT(tavor_ioctl_flash_init);
671 return (EFAULT);
672 }
673
674 TAVOR_TNF_EXIT(tavor_ioctl_flash_init);
675 return (0);
676 }
677
678 /*
679 * tavor_ioctl_flash_fini()
680 */
681 static int
682 tavor_ioctl_flash_fini(tavor_state_t *state, dev_t dev)
683 {
684 int ret;
685
686 TAVOR_TNF_ENTER(tavor_ioctl_flash_fini);
687
688 /*
689 * Check that flash init ioctl has been called first. And check
690 * that the same dev_t that called init is the one calling fini now.
691 */
692 mutex_enter(&state->ts_fw_flashlock);
693 if ((state->ts_fw_flashdev != dev) ||
694 (state->ts_fw_flashstarted == 0)) {
695 mutex_exit(&state->ts_fw_flashlock);
696 TNF_PROBE_0(tavor_flash_bad_state, TAVOR_TNF_ERROR, "");
697 TAVOR_TNF_EXIT(tavor_ioctl_flash_fini);
698 return (EIO);
699 }
700
701 tavor_ioctl_flash_cleanup_nolock(state);
702
703 mutex_exit(&state->ts_fw_flashlock);
704
705 /*
706 * If "flash fini" is successful, remove the "on close" callback
707 * that was setup during "flash init".
708 */
709 ret = tavor_umap_db_clear_onclose_cb(dev,
710 TAVOR_ONCLOSE_FLASH_INPROGRESS);
711 if (ret != DDI_SUCCESS) {
712 TNF_PROBE_0(tavor_flash_fini_clear_cb_fail, TAVOR_TNF_ERROR,
713 "");
714 TAVOR_TNF_EXIT(tavor_ioctl_flash_fini);
715 return (EFAULT);
716 }
717
718 TAVOR_TNF_EXIT(tavor_ioctl_flash_fini);
719 return (0);
720 }
721
722
723 /*
724 * tavor_ioctl_flash_cleanup()
725 */
726 static void
727 tavor_ioctl_flash_cleanup(tavor_state_t *state)
728 {
729 TAVOR_TNF_ENTER(tavor_ioctl_flash_cleanup);
730
731 mutex_enter(&state->ts_fw_flashlock);
732 tavor_ioctl_flash_cleanup_nolock(state);
733 mutex_exit(&state->ts_fw_flashlock);
734
735 TAVOR_TNF_EXIT(tavor_ioctl_flash_cleanup);
736 }
737
738
739 /*
740 * tavor_ioctl_flash_cleanup_nolock()
741 */
742 static void
743 tavor_ioctl_flash_cleanup_nolock(tavor_state_t *state)
744 {
745 TAVOR_TNF_ENTER(tavor_ioctl_flash_cleanup_nolock);
746
747 ASSERT(MUTEX_HELD(&state->ts_fw_flashlock));
748
749 /* free flash mem */
750 kmem_free(state->ts_fw_sector, 1 << state->ts_fw_log_sector_sz);
751
752 /* Fini the Flash */
753 tavor_flash_fini(state);
754
755 /* Set flash state to fini */
756 state->ts_fw_flashstarted = 0;
757 state->ts_fw_flashdev = 0;
758
759 TAVOR_TNF_EXIT(tavor_ioctl_flash_cleanup_nolock);
760 }
761
762
763 /*
764 * tavor_ioctl_info()
765 */
766 static int
767 tavor_ioctl_info(tavor_state_t *state, dev_t dev, intptr_t arg, int mode)
768 {
769 tavor_info_ioctl_t info;
770 tavor_flash_init_ioctl_t init_info;
771
772 TAVOR_TNF_ENTER(tavor_ioctl_info);
773
774 /*
775 * Access to Tavor VTS ioctls is not allowed in "maintenance mode".
776 */
777 if (state->ts_operational_mode == TAVOR_MAINTENANCE_MODE) {
778 TNF_PROBE_0(tavor_ioctl_info_maintenance_mode_fail,
779 TAVOR_TNF_ERROR, "");
780 TAVOR_TNF_EXIT(tavor_ioctl_info);
781 return (EFAULT);
782 }
783
784 /* copyin the user struct to kernel */
785 if (ddi_copyin((void *)arg, &info, sizeof (tavor_info_ioctl_t),
786 mode) != 0) {
787 TNF_PROBE_0(tavor_ioctl_info_copyin_fail, TAVOR_TNF_ERROR, "");
788 TAVOR_TNF_EXIT(tavor_ioctl_info);
789 return (EFAULT);
790 }
791
792 /*
793 * Check ioctl revision
794 */
795 if (info.ti_revision != TAVOR_VTS_IOCTL_REVISION) {
796 TNF_PROBE_0(tavor_ioctl_info_bad_rev, TAVOR_TNF_ERROR, "");
797 TAVOR_TNF_EXIT(tavor_ioctl_info);
798 return (EINVAL);
799 }
800
801 /*
802 * If the 'fw_device_sz' has not been initialized yet, we initialize it
803 * here. This is done by leveraging the
804 * tavor_ioctl_flash_init()/fini() calls. We also hold our own mutex
805 * around this operation in case we have multiple VTS threads in
806 * process at the same time.
807 */
808 mutex_enter(&state->ts_info_lock);
809 if (state->ts_fw_device_sz == 0) {
810 if (tavor_ioctl_flash_init(state, dev, (intptr_t)&init_info,
811 (FKIOCTL | mode)) != 0) {
812 mutex_exit(&state->ts_info_lock);
813 TNF_PROBE_0(tavor_ioctl_info_flash_init_fail,
814 TAVOR_TNF_ERROR, "");
815 TAVOR_TNF_EXIT(tavor_ioctl_info);
816 return (EFAULT);
817 }
818 (void) tavor_ioctl_flash_fini(state, dev);
819 }
820 mutex_exit(&state->ts_info_lock);
821
822 info.ti_hw_rev = state->ts_adapter.rev_id;
823 info.ti_flash_sz = state->ts_fw_device_sz;
824 info.ti_fw_rev.tfi_maj = state->ts_fw.fw_rev_major;
825 info.ti_fw_rev.tfi_min = state->ts_fw.fw_rev_minor;
826 info.ti_fw_rev.tfi_sub = state->ts_fw.fw_rev_subminor;
827 info.ti_mem_start_offset = 0;
828 info.ti_mem_end_offset = state->ts_ddr.ddr_endaddr -
829 state->ts_ddr.ddr_baseaddr;
830
831 /* Copy ioctl results back to user struct */
832 if (ddi_copyout(&info, (void *)arg, sizeof (tavor_info_ioctl_t),
833 mode) != 0) {
834 TNF_PROBE_0(tavor_ioctl_info_copyout_fail, TAVOR_TNF_ERROR, "");
835 TAVOR_TNF_EXIT(tavor_ioctl_info);
836 return (EFAULT);
837 }
838
839 TAVOR_TNF_EXIT(tavor_ioctl_info);
840 return (0);
841 }
842
843 /*
844 * tavor_ioctl_ports()
845 */
846 static int
847 tavor_ioctl_ports(tavor_state_t *state, intptr_t arg, int mode)
848 {
849 tavor_ports_ioctl_t info;
850 tavor_stat_port_ioctl_t portstat;
851 ibt_hca_portinfo_t pi;
852 uint_t tbl_size;
853 ib_gid_t *sgid_tbl;
854 ib_pkey_t *pkey_tbl;
855 int i;
856
857 TAVOR_TNF_ENTER(tavor_ioctl_ports);
858
859 /*
860 * Access to Tavor VTS ioctls is not allowed in "maintenance mode".
861 */
862 if (state->ts_operational_mode == TAVOR_MAINTENANCE_MODE) {
863 TNF_PROBE_0(tavor_ioctl_ports_maintenance_mode_fail,
864 TAVOR_TNF_ERROR, "");
865 TAVOR_TNF_EXIT(tavor_ioctl_ports);
866 return (EFAULT);
867 }
868
869 /* copyin the user struct to kernel */
870 #ifdef _MULTI_DATAMODEL
871 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
872 tavor_ports_ioctl32_t info32;
873
874 if (ddi_copyin((void *)arg, &info32,
875 sizeof (tavor_ports_ioctl32_t), mode) != 0) {
876 TNF_PROBE_0(tavor_ioctl_ports_copyin_fail,
877 TAVOR_TNF_ERROR, "");
878 TAVOR_TNF_EXIT(tavor_ioctl_ports);
879 return (EFAULT);
880 }
881 info.tp_revision = info32.tp_revision;
882 info.tp_ports =
883 (tavor_stat_port_ioctl_t *)(uintptr_t)info32.tp_ports;
884 info.tp_num_ports = info32.tp_num_ports;
885
886 } else
887 #endif /* _MULTI_DATAMODEL */
888 if (ddi_copyin((void *)arg, &info, sizeof (tavor_ports_ioctl_t),
889 mode) != 0) {
890 TNF_PROBE_0(tavor_ioctl_ports_copyin_fail, TAVOR_TNF_ERROR, "");
891 TAVOR_TNF_EXIT(tavor_ioctl_ports);
892 return (EFAULT);
893 }
894
895 /*
896 * Check ioctl revision
897 */
898 if (info.tp_revision != TAVOR_VTS_IOCTL_REVISION) {
899 TNF_PROBE_0(tavor_ioctl_ports_bad_rev, TAVOR_TNF_ERROR, "");
900 TAVOR_TNF_EXIT(tavor_ioctl_ports);
901 return (EINVAL);
902 }
903
904 /* Allocate space for temporary GID table/PKey table */
905 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_gidtbl);
906 sgid_tbl = (ib_gid_t *)kmem_zalloc(tbl_size * sizeof (ib_gid_t),
907 KM_SLEEP);
908 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_pkeytbl);
909 pkey_tbl = (ib_pkey_t *)kmem_zalloc(tbl_size * sizeof (ib_pkey_t),
910 KM_SLEEP);
911
912 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sgid_tbl, *pkey_tbl))
913
914 /*
915 * Setup the number of ports, then loop through all ports and
916 * query properties of each.
917 */
918 info.tp_num_ports = (uint8_t)state->ts_cfg_profile->cp_num_ports;
919 for (i = 0; i < info.tp_num_ports; i++) {
920 /*
921 * Get portstate information from the device. If
922 * tavor_port_query() fails, leave zeroes in user
923 * struct port entry and continue.
924 */
925 bzero(&pi, sizeof (ibt_hca_portinfo_t));
926 pi.p_sgid_tbl = sgid_tbl;
927 pi.p_pkey_tbl = pkey_tbl;
928 if (tavor_port_query(state, i + 1, &pi) != 0) {
929 TNF_PROBE_0(tavor_ioctl_ports_query_failed,
930 TAVOR_TNF_ERROR, "");
931 }
932
933 portstat.tsp_port_num = pi.p_port_num;
934 portstat.tsp_state = pi.p_linkstate;
935 portstat.tsp_guid = pi.p_sgid_tbl[0].gid_guid;
936
937 /*
938 * Copy queried port results back to user struct. If
939 * this fails, then break out of loop, attempt to copy
940 * out remaining info to user struct, and return (without
941 * error).
942 */
943 if (ddi_copyout(&portstat,
944 &(((tavor_stat_port_ioctl_t *)info.tp_ports)[i]),
945 sizeof (tavor_stat_port_ioctl_t), mode) != 0) {
946 break;
947 }
948 }
949
950 /* Free the temporary space used for GID table/PKey table */
951 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_gidtbl);
952 kmem_free(sgid_tbl, tbl_size * sizeof (ib_gid_t));
953 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_pkeytbl);
954 kmem_free(pkey_tbl, tbl_size * sizeof (ib_pkey_t));
955
956 /* Copy ioctl results back to user struct */
957 #ifdef _MULTI_DATAMODEL
958 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
959 tavor_ports_ioctl32_t info32;
960
961 info32.tp_revision = info.tp_revision;
962 info32.tp_ports = (caddr32_t)(uintptr_t)info.tp_ports;
963 info32.tp_num_ports = info.tp_num_ports;
964
965 if (ddi_copyout(&info32, (void *)arg,
966 sizeof (tavor_ports_ioctl32_t), mode) != 0) {
967 TNF_PROBE_0(tavor_ioctl_ports_copyout_fail,
968 TAVOR_TNF_ERROR, "");
969 TAVOR_TNF_EXIT(tavor_ioctl_ports);
970 return (EFAULT);
971 }
972 } else
973 #endif /* _MULTI_DATAMODEL */
974 if (ddi_copyout(&info, (void *)arg, sizeof (tavor_ports_ioctl_t),
975 mode) != 0) {
976 TNF_PROBE_0(tavor_ioctl_ports_copyout_fail,
977 TAVOR_TNF_ERROR, "");
978 TAVOR_TNF_EXIT(tavor_ioctl_ports);
979 return (EFAULT);
980 }
981
982 TAVOR_TNF_EXIT(tavor_ioctl_ports);
983 return (0);
984 }
985
986 /*
987 * tavor_ioctl_loopback()
988 */
989 static int
990 tavor_ioctl_loopback(tavor_state_t *state, intptr_t arg, int mode)
991 {
992 tavor_loopback_ioctl_t lb;
993 tavor_loopback_state_t lstate;
994 ibt_hca_portinfo_t pi;
995 uint_t tbl_size, loopmax, max_usec;
996 ib_gid_t *sgid_tbl;
997 ib_pkey_t *pkey_tbl;
998 int j, iter, ret;
999
1000 TAVOR_TNF_ENTER(tavor_ioctl_loopback);
1001
1002 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(lstate))
1003
1004 /*
1005 * Access to Tavor VTS ioctls is not allowed in "maintenance mode".
1006 */
1007 if (state->ts_operational_mode == TAVOR_MAINTENANCE_MODE) {
1008 TNF_PROBE_0(tavor_ioctl_loopback_maintenance_mode_fail,
1009 TAVOR_TNF_ERROR, "");
1010 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1011 return (EFAULT);
1012 }
1013
1014 /* copyin the user struct to kernel */
1015 #ifdef _MULTI_DATAMODEL
1016 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
1017 tavor_loopback_ioctl32_t lb32;
1018
1019 if (ddi_copyin((void *)arg, &lb32,
1020 sizeof (tavor_loopback_ioctl32_t), mode) != 0) {
1021 TNF_PROBE_0(tavor_ioctl_loopback_copyin_fail,
1022 TAVOR_TNF_ERROR, "");
1023 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1024 return (EFAULT);
1025 }
1026 lb.tlb_revision = lb32.tlb_revision;
1027 lb.tlb_send_buf = (caddr_t)(uintptr_t)lb32.tlb_send_buf;
1028 lb.tlb_fail_buf = (caddr_t)(uintptr_t)lb32.tlb_fail_buf;
1029 lb.tlb_buf_sz = lb32.tlb_buf_sz;
1030 lb.tlb_num_iter = lb32.tlb_num_iter;
1031 lb.tlb_pass_done = lb32.tlb_pass_done;
1032 lb.tlb_timeout = lb32.tlb_timeout;
1033 lb.tlb_error_type = lb32.tlb_error_type;
1034 lb.tlb_port_num = lb32.tlb_port_num;
1035 lb.tlb_num_retry = lb32.tlb_num_retry;
1036 } else
1037 #endif /* _MULTI_DATAMODEL */
1038 if (ddi_copyin((void *)arg, &lb, sizeof (tavor_loopback_ioctl_t),
1039 mode) != 0) {
1040 TNF_PROBE_0(tavor_ioctl_loopback_copyin_fail,
1041 TAVOR_TNF_ERROR, "");
1042 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1043 return (EFAULT);
1044 }
1045
1046 /* Initialize the internal loopback test state structure */
1047 bzero(&lstate, sizeof (tavor_loopback_state_t));
1048
1049 /*
1050 * Check ioctl revision
1051 */
1052 if (lb.tlb_revision != TAVOR_VTS_IOCTL_REVISION) {
1053 lb.tlb_error_type = TAVOR_LOOPBACK_INVALID_REVISION;
1054 (void) tavor_loopback_copyout(&lb, arg, mode);
1055 TNF_PROBE_0(tavor_ioctl_loopback_bad_rev,
1056 TAVOR_TNF_ERROR, "");
1057 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1058 return (EINVAL);
1059 }
1060
1061 /* Validate that specified port number is legal */
1062 if (!tavor_portnum_is_valid(state, lb.tlb_port_num)) {
1063 lb.tlb_error_type = TAVOR_LOOPBACK_INVALID_PORT;
1064 (void) tavor_loopback_copyout(&lb, arg, mode);
1065 TNF_PROBE_0(tavor_ioctl_loopback_inv_port,
1066 TAVOR_TNF_ERROR, "");
1067 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1068 return (EINVAL);
1069 }
1070
1071 /* Allocate space for temporary GID table/PKey table */
1072 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_gidtbl);
1073 sgid_tbl = (ib_gid_t *)kmem_zalloc(tbl_size * sizeof (ib_gid_t),
1074 KM_SLEEP);
1075 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_pkeytbl);
1076 pkey_tbl = (ib_pkey_t *)kmem_zalloc(tbl_size * sizeof (ib_pkey_t),
1077 KM_SLEEP);
1078
1079 /*
1080 * Get portstate information from specific port on device
1081 */
1082 bzero(&pi, sizeof (ibt_hca_portinfo_t));
1083 pi.p_sgid_tbl = sgid_tbl;
1084 pi.p_pkey_tbl = pkey_tbl;
1085 if (tavor_port_query(state, lb.tlb_port_num, &pi) != 0) {
1086 /* Free the temporary space used for GID table/PKey table */
1087 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_gidtbl);
1088 kmem_free(sgid_tbl, tbl_size * sizeof (ib_gid_t));
1089 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_pkeytbl);
1090 kmem_free(pkey_tbl, tbl_size * sizeof (ib_pkey_t));
1091
1092 lb.tlb_error_type = TAVOR_LOOPBACK_INVALID_PORT;
1093 (void) tavor_loopback_copyout(&lb, arg, mode);
1094 tavor_loopback_free_state(&lstate);
1095 TNF_PROBE_0(tavor_ioctl_loopback_bad_port,
1096 TAVOR_TNF_ERROR, "");
1097 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1098 return (EINVAL);
1099 }
1100
1101 lstate.tls_port = pi.p_port_num;
1102 lstate.tls_lid = pi.p_base_lid;
1103 lstate.tls_pkey_ix = (pi.p_linkstate == TAVOR_PORT_LINK_ACTIVE) ? 1 : 0;
1104 lstate.tls_state = state;
1105 lstate.tls_retry = lb.tlb_num_retry;
1106
1107 /* Free the temporary space used for GID table/PKey table */
1108 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_gidtbl);
1109 kmem_free(sgid_tbl, tbl_size * sizeof (ib_gid_t));
1110 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_pkeytbl);
1111 kmem_free(pkey_tbl, tbl_size * sizeof (ib_pkey_t));
1112
1113 /*
1114 * Compute the timeout duration in usec per the formula:
1115 * to_usec_per_retry = 4.096us * (2 ^ supplied_timeout)
1116 * (plus we add a little fudge-factor here too)
1117 */
1118 lstate.tls_timeout = lb.tlb_timeout;
1119 max_usec = (4096 * (1 << lstate.tls_timeout)) / 1000;
1120 max_usec = max_usec * (lstate.tls_retry + 1);
1121 max_usec = max_usec + 10000;
1122
1123 /*
1124 * Determine how many times we should loop before declaring a
1125 * timeout failure.
1126 */
1127 loopmax = max_usec/TAVOR_VTS_LOOPBACK_MIN_WAIT_DUR;
1128 if ((max_usec % TAVOR_VTS_LOOPBACK_MIN_WAIT_DUR) != 0) {
1129 loopmax++;
1130 }
1131
1132 if (lb.tlb_send_buf == NULL || lb.tlb_buf_sz == 0) {
1133 lb.tlb_error_type = TAVOR_LOOPBACK_SEND_BUF_INVALID;
1134 (void) tavor_loopback_copyout(&lb, arg, mode);
1135 tavor_loopback_free_state(&lstate);
1136 TNF_PROBE_0(tavor_ioctl_loopback_buf_null,
1137 TAVOR_TNF_ERROR, "");
1138 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1139 return (EINVAL);
1140 }
1141
1142 /* Allocate protection domain (PD) */
1143 if (tavor_loopback_init(state, &lstate) != 0) {
1144 lb.tlb_error_type = lstate.tls_err;
1145 (void) tavor_loopback_copyout(&lb, arg, mode);
1146 tavor_loopback_free_state(&lstate);
1147 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1148 return (EFAULT);
1149 }
1150
1151 /* Allocate and register a TX buffer */
1152 if (tavor_loopback_alloc_mem(&lstate, &lstate.tls_tx,
1153 lb.tlb_buf_sz) != 0) {
1154 lb.tlb_error_type =
1155 TAVOR_LOOPBACK_SEND_BUF_MEM_REGION_ALLOC_FAIL;
1156 (void) tavor_loopback_copyout(&lb, arg, mode);
1157 tavor_loopback_free_state(&lstate);
1158 TNF_PROBE_0(tavor_ioctl_loopback_txbuf_alloc_fail,
1159 TAVOR_TNF_ERROR, "");
1160 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1161 return (EFAULT);
1162 }
1163
1164 /* Allocate and register an RX buffer */
1165 if (tavor_loopback_alloc_mem(&lstate, &lstate.tls_rx,
1166 lb.tlb_buf_sz) != 0) {
1167 lb.tlb_error_type =
1168 TAVOR_LOOPBACK_RECV_BUF_MEM_REGION_ALLOC_FAIL;
1169 (void) tavor_loopback_copyout(&lb, arg, mode);
1170 tavor_loopback_free_state(&lstate);
1171 TNF_PROBE_0(tavor_ioctl_loopback_rxbuf_alloc_fail,
1172 TAVOR_TNF_ERROR, "");
1173 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1174 return (EFAULT);
1175 }
1176
1177 /* Copy in the transmit buffer data */
1178 if (ddi_copyin((void *)lb.tlb_send_buf, lstate.tls_tx.tlc_buf,
1179 lb.tlb_buf_sz, mode) != 0) {
1180 lb.tlb_error_type = TAVOR_LOOPBACK_SEND_BUF_COPY_FAIL;
1181 (void) tavor_loopback_copyout(&lb, arg, mode);
1182 tavor_loopback_free_state(&lstate);
1183 TNF_PROBE_0(tavor_ioctl_loopback_tx_copyin_fail,
1184 TAVOR_TNF_ERROR, "");
1185 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1186 return (EFAULT);
1187 }
1188
1189 /* Allocate the transmit QP and CQs */
1190 lstate.tls_err = TAVOR_LOOPBACK_XMIT_SEND_CQ_ALLOC_FAIL;
1191 if (tavor_loopback_alloc_qps(&lstate, &lstate.tls_tx) != 0) {
1192 lb.tlb_error_type = lstate.tls_err;
1193 (void) tavor_loopback_copyout(&lb, arg, mode);
1194 tavor_loopback_free_state(&lstate);
1195 TNF_PROBE_0(tavor_ioctl_loopback_txqp_alloc_fail,
1196 TAVOR_TNF_ERROR, "");
1197 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1198 return (EFAULT);
1199 }
1200
1201 /* Allocate the receive QP and CQs */
1202 lstate.tls_err = TAVOR_LOOPBACK_RECV_SEND_CQ_ALLOC_FAIL;
1203 if (tavor_loopback_alloc_qps(&lstate, &lstate.tls_rx) != 0) {
1204 lb.tlb_error_type = lstate.tls_err;
1205 (void) tavor_loopback_copyout(&lb, arg, mode);
1206 tavor_loopback_free_state(&lstate);
1207 TNF_PROBE_0(tavor_ioctl_loopback_rxqp_alloc_fail,
1208 TAVOR_TNF_ERROR, "");
1209 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1210 return (EFAULT);
1211 }
1212
1213 /* Activate the TX QP (connect to RX QP) */
1214 lstate.tls_err = TAVOR_LOOPBACK_XMIT_QP_INIT_FAIL;
1215 if (tavor_loopback_modify_qp(&lstate, &lstate.tls_tx,
1216 lstate.tls_rx.tlc_qp_num) != 0) {
1217 lb.tlb_error_type = lstate.tls_err;
1218 (void) tavor_loopback_copyout(&lb, arg, mode);
1219 tavor_loopback_free_state(&lstate);
1220 TNF_PROBE_0(tavor_ioctl_loopback_txqp_init_fail,
1221 TAVOR_TNF_ERROR, "");
1222 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1223 return (EFAULT);
1224 }
1225
1226 /* Activate the RX QP (connect to TX QP) */
1227 lstate.tls_err = TAVOR_LOOPBACK_RECV_QP_INIT_FAIL;
1228 if (tavor_loopback_modify_qp(&lstate, &lstate.tls_rx,
1229 lstate.tls_tx.tlc_qp_num) != 0) {
1230 lb.tlb_error_type = lstate.tls_err;
1231 (void) tavor_loopback_copyout(&lb, arg, mode);
1232 tavor_loopback_free_state(&lstate);
1233 TNF_PROBE_0(tavor_ioctl_loopback_rxqp_init_fail,
1234 TAVOR_TNF_ERROR, "");
1235 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1236 return (EFAULT);
1237 }
1238
1239 /* Run the loopback test (for specified number of iterations) */
1240 lb.tlb_pass_done = 0;
1241 for (iter = 0; iter < lb.tlb_num_iter; iter++) {
1242 lstate.tls_err = 0;
1243 bzero(lstate.tls_rx.tlc_buf, lb.tlb_buf_sz);
1244
1245 /* Post RDMA Write work request */
1246 if (tavor_loopback_post_send(&lstate, &lstate.tls_tx,
1247 &lstate.tls_rx) != IBT_SUCCESS) {
1248 lb.tlb_error_type = TAVOR_LOOPBACK_WQE_POST_FAIL;
1249 (void) tavor_loopback_copyout(&lb, arg, mode);
1250 tavor_loopback_free_state(&lstate);
1251 TNF_PROBE_0(tavor_ioctl_loopback_wqe_post_fail,
1252 TAVOR_TNF_ERROR, "");
1253 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1254 return (EFAULT);
1255 }
1256
1257 /* Poll the TX CQ for a completion every few ticks */
1258 for (j = 0; j < loopmax; j++) {
1259 delay(drv_usectohz(TAVOR_VTS_LOOPBACK_MIN_WAIT_DUR));
1260
1261 ret = tavor_loopback_poll_cq(&lstate, &lstate.tls_tx);
1262 if (((ret != IBT_SUCCESS) && (ret != IBT_CQ_EMPTY)) ||
1263 ((ret == IBT_CQ_EMPTY) && (j == loopmax - 1))) {
1264 lb.tlb_error_type = TAVOR_LOOPBACK_CQ_POLL_FAIL;
1265 if (ddi_copyout(lstate.tls_rx.tlc_buf,
1266 lb.tlb_fail_buf, lstate.tls_tx.tlc_buf_sz,
1267 mode) != 0) {
1268 TNF_PROBE_0(
1269 tavor_ioctl_loopback_xfer_co_fail,
1270 TAVOR_TNF_ERROR, "");
1271 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1272 return (EFAULT);
1273 }
1274 (void) tavor_loopback_copyout(&lb, arg, mode);
1275 tavor_loopback_free_state(&lstate);
1276 TNF_PROBE_0(tavor_ioctl_loopback_xfer_fail,
1277 TAVOR_TNF_ERROR, "");
1278 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1279 return (EFAULT);
1280 } else if (ret == IBT_CQ_EMPTY) {
1281 continue;
1282 }
1283
1284 /* Compare the data buffers */
1285 if (bcmp(lstate.tls_tx.tlc_buf, lstate.tls_rx.tlc_buf,
1286 lb.tlb_buf_sz) == 0) {
1287 break;
1288 } else {
1289 lb.tlb_error_type =
1290 TAVOR_LOOPBACK_SEND_RECV_COMPARE_FAIL;
1291 if (ddi_copyout(lstate.tls_rx.tlc_buf,
1292 lb.tlb_fail_buf, lstate.tls_tx.tlc_buf_sz,
1293 mode) != 0) {
1294 TNF_PROBE_0(
1295 tavor_ioctl_loopback_bcmp_co_fail,
1296 TAVOR_TNF_ERROR, "");
1297 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1298 return (EFAULT);
1299 }
1300 (void) tavor_loopback_copyout(&lb, arg, mode);
1301 tavor_loopback_free_state(&lstate);
1302 TNF_PROBE_0(tavor_ioctl_loopback_bcmp_fail,
1303 TAVOR_TNF_ERROR, "");
1304 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1305 return (EFAULT);
1306 }
1307 }
1308
1309 lstate.tls_err = TAVOR_LOOPBACK_SUCCESS;
1310 lb.tlb_pass_done = iter + 1;
1311 }
1312
1313 lb.tlb_error_type = TAVOR_LOOPBACK_SUCCESS;
1314
1315 /* Copy ioctl results back to user struct */
1316 ret = tavor_loopback_copyout(&lb, arg, mode);
1317
1318 /* Free up everything and release all consumed resources */
1319 tavor_loopback_free_state(&lstate);
1320
1321 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1322 return (ret);
1323 }
1324
1325 /*
1326 * tavor_ioctl_ddr_read()
1327 */
1328 static int
1329 tavor_ioctl_ddr_read(tavor_state_t *state, intptr_t arg, int mode)
1330 {
1331 tavor_ddr_read_ioctl_t rdreg;
1332 uint32_t *addr;
1333 uintptr_t baseaddr;
1334 uint64_t ddr_size;
1335
1336 TAVOR_TNF_ENTER(tavor_ioctl_ddr_read);
1337
1338 /*
1339 * Access to Tavor VTS ioctls is not allowed in "maintenance mode".
1340 */
1341 if (state->ts_operational_mode == TAVOR_MAINTENANCE_MODE) {
1342 TNF_PROBE_0(tavor_ioctl_ddr_read_maintenance_mode_fail,
1343 TAVOR_TNF_ERROR, "");
1344 TAVOR_TNF_EXIT(tavor_ioctl_ddr_read);
1345 return (EFAULT);
1346 }
1347
1348 /* copyin the user struct to kernel */
1349 if (ddi_copyin((void *)arg, &rdreg, sizeof (tavor_ddr_read_ioctl_t),
1350 mode) != 0) {
1351 TNF_PROBE_0(tavor_ioctl_ddr_read_copyin_fail,
1352 TAVOR_TNF_ERROR, "");
1353 TAVOR_TNF_EXIT(tavor_ioctl_ddr_read);
1354 return (EFAULT);
1355 }
1356
1357 /*
1358 * Check ioctl revision
1359 */
1360 if (rdreg.tdr_revision != TAVOR_VTS_IOCTL_REVISION) {
1361 TNF_PROBE_0(tavor_ioctl_ddr_read_bad_rev, TAVOR_TNF_ERROR, "");
1362 TAVOR_TNF_EXIT(tavor_ioctl_ddr_read);
1363 return (EINVAL);
1364 }
1365
1366 /*
1367 * Check for valid offset
1368 */
1369 ddr_size = (state->ts_ddr.ddr_endaddr - state->ts_ddr.ddr_baseaddr + 1);
1370 if ((uint64_t)rdreg.tdr_offset >= ddr_size) {
1371 TNF_PROBE_0(tavor_ioctl_ddr_read_bad_offset,
1372 TAVOR_TNF_ERROR, "");
1373 TAVOR_TNF_EXIT(tavor_ioctl_ddr_read);
1374 return (EINVAL);
1375 }
1376
1377 /* Determine base address for requested register read */
1378 baseaddr = (uintptr_t)state->ts_reg_ddr_baseaddr;
1379
1380 /* Ensure that address is properly-aligned */
1381 addr = (uint32_t *)((baseaddr + rdreg.tdr_offset) & ~0x3);
1382
1383 /* Read the register pointed to by addr */
1384 rdreg.tdr_data = ddi_get32(state->ts_reg_cmdhdl, addr);
1385
1386 /* Copy ioctl results back to user struct */
1387 if (ddi_copyout(&rdreg, (void *)arg, sizeof (tavor_ddr_read_ioctl_t),
1388 mode) != 0) {
1389 TNF_PROBE_0(tavor_ioctl_ddr_read_copyout_fail,
1390 TAVOR_TNF_ERROR, "");
1391 TAVOR_TNF_EXIT(tavor_ioctl_ddr_read);
1392 return (EFAULT);
1393 }
1394
1395 TAVOR_TNF_EXIT(tavor_ioctl_ddr_read);
1396 return (0);
1397 }
1398
1399
1400 #ifdef DEBUG
1401 /*
1402 * tavor_ioctl_reg_read()
1403 */
1404 static int
1405 tavor_ioctl_reg_read(tavor_state_t *state, intptr_t arg, int mode)
1406 {
1407 tavor_reg_ioctl_t rdreg;
1408 uint32_t *addr;
1409 uintptr_t baseaddr;
1410 int status;
1411
1412 TAVOR_TNF_ENTER(tavor_ioctl_reg_read);
1413
1414 /*
1415 * Access to Tavor registers is not allowed in "maintenance mode".
1416 * This is primarily because the device may not have BARs to access
1417 */
1418 if (state->ts_operational_mode == TAVOR_MAINTENANCE_MODE) {
1419 TNF_PROBE_0(tavor_ioctl_reg_read_maintence_mode_fail,
1420 TAVOR_TNF_ERROR, "");
1421 TAVOR_TNF_EXIT(tavor_ioctl_reg_read);
1422 return (EFAULT);
1423 }
1424
1425 /* Copy in the tavor_reg_ioctl_t structure */
1426 status = ddi_copyin((void *)arg, &rdreg, sizeof (tavor_reg_ioctl_t),
1427 mode);
1428 if (status != 0) {
1429 TNF_PROBE_0(tavor_ioctl_reg_read_copyin_fail,
1430 TAVOR_TNF_ERROR, "");
1431 TAVOR_TNF_EXIT(tavor_ioctl_reg_read);
1432 return (EFAULT);
1433 }
1434
1435 /* Determine base address for requested register set */
1436 switch (rdreg.trg_reg_set) {
1437 case TAVOR_CMD_BAR:
1438 baseaddr = (uintptr_t)state->ts_reg_cmd_baseaddr;
1439 break;
1440
1441 case TAVOR_UAR_BAR:
1442 baseaddr = (uintptr_t)state->ts_reg_uar_baseaddr;
1443 break;
1444
1445 case TAVOR_DDR_BAR:
1446 baseaddr = (uintptr_t)state->ts_reg_ddr_baseaddr;
1447 break;
1448
1449 default:
1450 TNF_PROBE_0(tavor_ioctl_reg_read_invregset_fail,
1451 TAVOR_TNF_ERROR, "");
1452 TAVOR_TNF_EXIT(tavor_ioctl_reg_read);
1453 return (EFAULT);
1454 }
1455
1456 /* Ensure that address is properly-aligned */
1457 addr = (uint32_t *)((baseaddr + rdreg.trg_offset) & ~0x3);
1458
1459 /* Read the register pointed to by addr */
1460 rdreg.trg_data = ddi_get32(state->ts_reg_cmdhdl, addr);
1461
1462 /* Copy in the result into the tavor_reg_ioctl_t structure */
1463 status = ddi_copyout(&rdreg, (void *)arg, sizeof (tavor_reg_ioctl_t),
1464 mode);
1465 if (status != 0) {
1466 TNF_PROBE_0(tavor_ioctl_reg_read_copyout_fail,
1467 TAVOR_TNF_ERROR, "");
1468 TAVOR_TNF_EXIT(tavor_ioctl_reg_read);
1469 return (EFAULT);
1470 }
1471
1472 TAVOR_TNF_EXIT(tavor_ioctl_reg_read);
1473 return (0);
1474 }
1475
1476
1477 /*
1478 * tavor_ioctl_reg_write()
1479 */
1480 static int
1481 tavor_ioctl_reg_write(tavor_state_t *state, intptr_t arg, int mode)
1482 {
1483 tavor_reg_ioctl_t wrreg;
1484 uint32_t *addr;
1485 uintptr_t baseaddr;
1486 int status;
1487
1488 TAVOR_TNF_ENTER(tavor_ioctl_reg_write);
1489
1490 /*
1491 * Access to Tavor registers is not allowed in "maintenance mode".
1492 * This is primarily because the device may not have BARs to access
1493 */
1494 if (state->ts_operational_mode == TAVOR_MAINTENANCE_MODE) {
1495 TNF_PROBE_0(tavor_ioctl_reg_write_maintence_mode_fail,
1496 TAVOR_TNF_ERROR, "");
1497 TAVOR_TNF_EXIT(tavor_ioctl_reg_write);
1498 return (EFAULT);
1499 }
1500
1501 /* Copy in the tavor_reg_ioctl_t structure */
1502 status = ddi_copyin((void *)arg, &wrreg, sizeof (tavor_reg_ioctl_t),
1503 mode);
1504 if (status != 0) {
1505 TNF_PROBE_0(tavor_ioctl_reg_write_copyin_fail,
1506 TAVOR_TNF_ERROR, "");
1507 TAVOR_TNF_EXIT(tavor_ioctl_reg_write);
1508 return (EFAULT);
1509 }
1510
1511 /* Determine base address for requested register set */
1512 switch (wrreg.trg_reg_set) {
1513 case TAVOR_CMD_BAR:
1514 baseaddr = (uintptr_t)state->ts_reg_cmd_baseaddr;
1515 break;
1516
1517 case TAVOR_UAR_BAR:
1518 baseaddr = (uintptr_t)state->ts_reg_uar_baseaddr;
1519 break;
1520
1521 case TAVOR_DDR_BAR:
1522 baseaddr = (uintptr_t)state->ts_reg_ddr_baseaddr;
1523 break;
1524
1525 default:
1526 TNF_PROBE_0(tavor_ioctl_reg_write_invregset_fail,
1527 TAVOR_TNF_ERROR, "");
1528 TAVOR_TNF_EXIT(tavor_ioctl_reg_write);
1529 return (EFAULT);
1530 }
1531
1532 /* Ensure that address is properly-aligned */
1533 addr = (uint32_t *)((baseaddr + wrreg.trg_offset) & ~0x3);
1534
1535 /* Write the data to the register pointed to by addr */
1536 ddi_put32(state->ts_reg_cmdhdl, addr, wrreg.trg_data);
1537
1538 TAVOR_TNF_EXIT(tavor_ioctl_reg_write);
1539 return (0);
1540 }
1541 #endif /* DEBUG */
1542
1543 /*
1544 * tavor_flash_reset()
1545 */
1546 static void
1547 tavor_flash_reset(tavor_state_t *state)
1548 {
1549 TAVOR_TNF_ENTER(tavor_flash_reset);
1550
1551 /*
1552 * Performs a reset to the flash device. After a reset the flash will
1553 * be operating in normal mode (capable of read/write, etc.).
1554 */
1555 switch (state->ts_fw_cmdset) {
1556 case TAVOR_FLASH_AMD_CMDSET:
1557 tavor_flash_write(state, 0x555, TAVOR_HW_FLASH_RESET_AMD);
1558 break;
1559
1560 case TAVOR_FLASH_INTEL_CMDSET:
1561 tavor_flash_write(state, 0x555, TAVOR_HW_FLASH_RESET_INTEL);
1562 break;
1563
1564 default:
1565 break;
1566 }
1567
1568 TAVOR_TNF_EXIT(tavor_flash_reset);
1569 }
1570
1571 /*
1572 * tavor_flash_read_sector()
1573 */
1574 static void
1575 tavor_flash_read_sector(tavor_state_t *state, uint32_t sector_num)
1576 {
1577 uint32_t addr;
1578 uint32_t end_addr;
1579 uint32_t *image;
1580 int i;
1581
1582 TAVOR_TNF_ENTER(tavor_flash_read_sector);
1583
1584 image = (uint32_t *)&state->ts_fw_sector[0];
1585
1586 /*
1587 * Calculate the start and end address of the sector, based on the
1588 * sector number passed in.
1589 */
1590 addr = sector_num << state->ts_fw_log_sector_sz;
1591 end_addr = addr + (1 << state->ts_fw_log_sector_sz);
1592
1593 /* Set the flash bank correctly for the given address */
1594 tavor_flash_bank(state, addr);
1595
1596 /* Read the entire sector, one quadlet at a time */
1597 for (i = 0; addr < end_addr; i++, addr += 4) {
1598 image[i] = tavor_flash_read(state, addr);
1599 }
1600
1601 TAVOR_TNF_EXIT(tavor_flash_read_sector);
1602 }
1603
1604 /*
1605 * tavor_flash_read_quadlet()
1606 */
1607 static void
1608 tavor_flash_read_quadlet(tavor_state_t *state, uint32_t *data,
1609 uint32_t addr)
1610 {
1611 TAVOR_TNF_ENTER(tavor_flash_read_quadlet);
1612
1613 /* Set the flash bank correctly for the given address */
1614 tavor_flash_bank(state, addr);
1615
1616 /* Read one quadlet of data */
1617 *data = tavor_flash_read(state, addr);
1618
1619 TAVOR_TNF_EXIT(tavor_flash_read_quadlet);
1620 }
1621
1622 /*
1623 * tavor_flash_write_sector()
1624 */
1625 static int
1626 tavor_flash_write_sector(tavor_state_t *state, uint32_t sector_num)
1627 {
1628 uint32_t addr;
1629 uint32_t end_addr;
1630 uchar_t *sector;
1631 int status = 0;
1632 int i;
1633
1634 TAVOR_TNF_ENTER(tavor_flash_write_sector);
1635
1636 sector = (uchar_t *)&state->ts_fw_sector[0];
1637
1638 /*
1639 * Calculate the start and end address of the sector, based on the
1640 * sector number passed in.
1641 */
1642 addr = sector_num << state->ts_fw_log_sector_sz;
1643 end_addr = addr + (1 << state->ts_fw_log_sector_sz);
1644
1645 /* Set the flash bank correctly for the given address */
1646 tavor_flash_bank(state, addr);
1647
1648 /* Erase the sector before writing */
1649 tavor_flash_reset(state);
1650 status = tavor_flash_erase_sector(state, sector_num);
1651 if (status != 0) {
1652 TAVOR_TNF_EXIT(tavor_flash_write_sector);
1653 return (status);
1654 }
1655
1656 /* Write the entire sector, one byte at a time */
1657 for (i = 0; addr < end_addr; i++, addr++) {
1658 status = tavor_flash_write_byte(state, addr, sector[i]);
1659 if (status != 0) {
1660 break;
1661 }
1662 }
1663
1664 tavor_flash_reset(state);
1665 TAVOR_TNF_EXIT(tavor_flash_write_sector);
1666 return (status);
1667 }
1668
1669 /*
1670 * tavor_flash_write_byte()
1671 */
1672 static int
1673 tavor_flash_write_byte(tavor_state_t *state, uint32_t addr, uchar_t data)
1674 {
1675 uint32_t stat;
1676 int status = 0;
1677 int i;
1678
1679 TAVOR_TNF_ENTER(tavor_flash_write_byte);
1680
1681 switch (state->ts_fw_cmdset) {
1682 case TAVOR_FLASH_AMD_CMDSET:
1683 /* Issue Flash Byte program command */
1684 tavor_flash_write(state, addr, 0xAA);
1685 tavor_flash_write(state, addr, 0x55);
1686 tavor_flash_write(state, addr, 0xA0);
1687 tavor_flash_write(state, addr, data);
1688
1689 /*
1690 * Wait for Write Byte to Complete:
1691 * 1) Wait 1usec
1692 * 2) Read status of the write operation
1693 * 3) Determine if we have timed out the write operation
1694 * 4) Compare correct data value to the status value that
1695 * was read from the same address.
1696 */
1697 i = 0;
1698 do {
1699 drv_usecwait(1);
1700 stat = tavor_flash_read(state, addr & ~3);
1701
1702 if (i == tavor_hw_flash_timeout_write) {
1703 cmn_err(CE_WARN,
1704 "tavor_flash_write_byte: ACS write "
1705 "timeout: addr: 0x%x, data: 0x%x\n",
1706 addr, data);
1707 status = EIO;
1708 break;
1709 }
1710
1711 i++;
1712 } while (data != ((stat >> ((3 - (addr & 3)) << 3)) & 0xFF));
1713 break;
1714
1715 case TAVOR_FLASH_INTEL_CMDSET:
1716 /* Issue Flash Byte program command */
1717 tavor_flash_write(state, addr, TAVOR_HW_FLASH_ICS_WRITE);
1718 tavor_flash_write(state, addr, data);
1719
1720 /* wait for completion */
1721 i = 0;
1722 do {
1723 drv_usecwait(1);
1724 stat = tavor_flash_read(state, addr & ~3);
1725
1726 if (i == tavor_hw_flash_timeout_write) {
1727 cmn_err(CE_WARN,
1728 "tavor_flash_write_byte: ICS write "
1729 "timeout: addr: %x, data: %x\n",
1730 addr, data);
1731 status = EIO;
1732 break;
1733 }
1734
1735 i++;
1736 } while ((stat & TAVOR_HW_FLASH_ICS_READY) == 0);
1737
1738 if (stat & TAVOR_HW_FLASH_ICS_ERROR) {
1739 cmn_err(CE_WARN,
1740 "tavor_flash_write_byte: ICS write cmd error: "
1741 "addr: %x, data: %x\n",
1742 addr, data);
1743 status = EIO;
1744 }
1745 break;
1746
1747 default:
1748 cmn_err(CE_WARN,
1749 "tavor_flash_write_byte: unknown cmd set: 0x%x\n",
1750 state->ts_fw_cmdset);
1751 status = EIO;
1752 break;
1753 }
1754
1755 TAVOR_TNF_EXIT(tavor_flash_write_byte);
1756 return (status);
1757 }
1758
1759 /*
1760 * tavor_flash_erase_sector()
1761 */
1762 static int
1763 tavor_flash_erase_sector(tavor_state_t *state, uint32_t sector_num)
1764 {
1765 uint32_t addr;
1766 uint32_t stat;
1767 int status = 0;
1768 int i;
1769
1770 TAVOR_TNF_ENTER(tavor_flash_erase_sector);
1771
1772 /* Get address from sector num */
1773 addr = sector_num << state->ts_fw_log_sector_sz;
1774
1775 switch (state->ts_fw_cmdset) {
1776 case TAVOR_FLASH_AMD_CMDSET:
1777 /* Issue Flash Sector Erase Command */
1778 tavor_flash_write(state, addr, 0xAA);
1779 tavor_flash_write(state, addr, 0x55);
1780 tavor_flash_write(state, addr, 0x80);
1781 tavor_flash_write(state, addr, 0xAA);
1782 tavor_flash_write(state, addr, 0x55);
1783 tavor_flash_write(state, addr, 0x30);
1784
1785 /*
1786 * Wait for Sector Erase to Complete
1787 * 1) Wait 1usec
1788 * 2) read the status at the base addr of the sector
1789 * 3) Determine if we have timed out
1790 * 4) Compare status of address with the value of a fully
1791 * erased quadlet. If these are equal, the sector
1792 * has been erased.
1793 */
1794 i = 0;
1795 do {
1796 /* wait 1usec */
1797 drv_usecwait(1);
1798 stat = tavor_flash_read(state, addr);
1799
1800 if (i == tavor_hw_flash_timeout_erase) {
1801 cmn_err(CE_WARN,
1802 "tavor_flash_erase_sector: "
1803 "ACS erase timeout\n");
1804 status = EIO;
1805 break;
1806 }
1807
1808 i++;
1809 } while (stat != 0xFFFFFFFF);
1810 break;
1811
1812 case TAVOR_FLASH_INTEL_CMDSET:
1813 /* Issue Erase Command */
1814 tavor_flash_write(state, addr, TAVOR_HW_FLASH_ICS_ERASE);
1815 tavor_flash_write(state, addr, TAVOR_HW_FLASH_ICS_CONFIRM);
1816
1817 /* wait for completion */
1818 i = 0;
1819 do {
1820 drv_usecwait(1);
1821 stat = tavor_flash_read(state, addr & ~3);
1822
1823 if (i == tavor_hw_flash_timeout_erase) {
1824 cmn_err(CE_WARN,
1825 "tavor_flash_erase_sector: "
1826 "ICS erase timeout\n");
1827 status = EIO;
1828 break;
1829 }
1830
1831 i++;
1832 } while ((stat & TAVOR_HW_FLASH_ICS_READY) == 0);
1833
1834 if (stat & TAVOR_HW_FLASH_ICS_ERROR) {
1835 cmn_err(CE_WARN,
1836 "tavor_flash_erase_sector: "
1837 "ICS erase cmd error\n");
1838 status = EIO;
1839 }
1840 break;
1841
1842 default:
1843 cmn_err(CE_WARN,
1844 "tavor_flash_erase_sector: unknown cmd set: 0x%x\n",
1845 state->ts_fw_cmdset);
1846 status = EIO;
1847 break;
1848 }
1849
1850 tavor_flash_reset(state);
1851
1852 TAVOR_TNF_EXIT(tavor_flash_erase_sector);
1853 return (status);
1854 }
1855
1856 /*
1857 * tavor_flash_erase_chip()
1858 */
1859 static int
1860 tavor_flash_erase_chip(tavor_state_t *state)
1861 {
1862 uint_t size;
1863 uint32_t stat;
1864 int status = 0;
1865 int num_sect;
1866 int i;
1867
1868 TAVOR_TNF_ENTER(tavor_flash_erase_chip);
1869
1870 switch (state->ts_fw_cmdset) {
1871 case TAVOR_FLASH_AMD_CMDSET:
1872 /* Issue Flash Chip Erase Command */
1873 tavor_flash_write(state, 0, 0xAA);
1874 tavor_flash_write(state, 0, 0x55);
1875 tavor_flash_write(state, 0, 0x80);
1876 tavor_flash_write(state, 0, 0xAA);
1877 tavor_flash_write(state, 0, 0x55);
1878 tavor_flash_write(state, 0, 0x10);
1879
1880 /*
1881 * Wait for Chip Erase to Complete
1882 * 1) Wait 1usec
1883 * 2) read the status at the base addr of the sector
1884 * 3) Determine if we have timed out
1885 * 4) Compare status of address with the value of a
1886 * fully erased quadlet. If these are equal, the
1887 * chip has been erased.
1888 */
1889 i = 0;
1890 do {
1891 /* wait 1usec */
1892 drv_usecwait(1);
1893 stat = tavor_flash_read(state, 0);
1894
1895 if (i == tavor_hw_flash_timeout_erase) {
1896 cmn_err(CE_WARN,
1897 "tavor_flash_erase_chip: erase timeout\n");
1898 status = EIO;
1899 break;
1900 }
1901
1902 i++;
1903 } while (stat != 0xFFFFFFFF);
1904 break;
1905
1906 case TAVOR_FLASH_INTEL_CMDSET:
1907 /*
1908 * The Intel chip doesn't have a chip erase command, so erase
1909 * all blocks one at a time.
1910 */
1911 size = (0x1 << state->ts_fw_log_sector_sz);
1912 num_sect = state->ts_fw_device_sz / size;
1913
1914 for (i = 0; i < num_sect; i++) {
1915 status = tavor_flash_erase_sector(state, i);
1916 if (status != 0) {
1917 cmn_err(CE_WARN,
1918 "tavor_flash_erase_chip: "
1919 "ICS sector %d erase error\n", i);
1920 status = EIO;
1921 break;
1922 }
1923 }
1924 break;
1925
1926 default:
1927 cmn_err(CE_WARN, "tavor_flash_erase_chip: "
1928 "unknown cmd set: 0x%x\n", state->ts_fw_cmdset);
1929 status = EIO;
1930 break;
1931 }
1932
1933 TAVOR_TNF_EXIT(tavor_flash_erase_chip);
1934 return (status);
1935 }
1936
1937 /*
1938 * tavor_flash_bank()
1939 */
1940 static void
1941 tavor_flash_bank(tavor_state_t *state, uint32_t addr)
1942 {
1943 ddi_acc_handle_t hdl;
1944 uint32_t bank;
1945
1946 TAVOR_TNF_ENTER(tavor_flash_bank);
1947
1948 /* Set handle */
1949 hdl = state->ts_pci_cfghdl;
1950
1951 /* Determine the bank setting from the address */
1952 bank = addr & TAVOR_HW_FLASH_BANK_MASK;
1953
1954 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->ts_fw_flashbank))
1955
1956 /*
1957 * If the bank is different from the currently set bank, we need to
1958 * change it. Also, if an 'addr' of 0 is given, this allows the
1959 * capability to force the flash bank to 0. This is useful at init
1960 * time to initially set the bank value
1961 */
1962 if (state->ts_fw_flashbank != bank || addr == 0) {
1963 /* Set bank using the GPIO settings */
1964 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_DATACLEAR, 0x70);
1965 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_DATASET,
1966 (bank >> 15) & 0x70);
1967
1968 /* Save the bank state */
1969 state->ts_fw_flashbank = bank;
1970 }
1971
1972 TAVOR_TNF_EXIT(tavor_flash_bank);
1973 }
1974
1975 /*
1976 * tavor_flash_read()
1977 */
1978 static uint32_t
1979 tavor_flash_read(tavor_state_t *state, uint32_t addr)
1980 {
1981 ddi_acc_handle_t hdl;
1982 uint32_t data;
1983 int timeout;
1984
1985 TAVOR_TNF_ENTER(tavor_flash_read);
1986
1987 /* Set handle */
1988 hdl = state->ts_pci_cfghdl;
1989
1990 /*
1991 * The Read operation does the following:
1992 * 1) Write the masked address to the TAVOR_FLASH_ADDR register.
1993 * Only the least significant 19 bits are valid.
1994 * 2) Read back the register until the command has completed.
1995 * 3) Read the data retrieved from the address at the TAVOR_FLASH_DATA
1996 * register.
1997 */
1998 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_ADDR,
1999 (addr & TAVOR_HW_FLASH_ADDR_MASK) | (1 << 29));
2000
2001 timeout = 0;
2002 do {
2003 data = tavor_flash_read_cfg(hdl, TAVOR_HW_FLASH_ADDR);
2004 timeout++;
2005 } while ((data & TAVOR_HW_FLASH_CMD_MASK) &&
2006 (timeout < tavor_hw_flash_timeout_config));
2007
2008 if (timeout == tavor_hw_flash_timeout_config) {
2009 cmn_err(CE_WARN, "tavor_flash_read: config command timeout.\n");
2010 }
2011
2012 data = tavor_flash_read_cfg(hdl, TAVOR_HW_FLASH_DATA);
2013
2014 TAVOR_TNF_EXIT(tavor_flash_read);
2015 return (data);
2016 }
2017
2018 /*
2019 * tavor_flash_write()
2020 */
2021 static void
2022 tavor_flash_write(tavor_state_t *state, uint32_t addr, uchar_t data)
2023 {
2024 ddi_acc_handle_t hdl;
2025 int cmd;
2026 int timeout;
2027
2028 TAVOR_TNF_ENTER(tavor_flash_write);
2029
2030 /* Set handle */
2031 hdl = state->ts_pci_cfghdl;
2032
2033 /*
2034 * The Write operation does the following:
2035 * 1) Write the data to be written to the TAVOR_FLASH_DATA offset.
2036 * 2) Write the address to write the data to to the TAVOR_FLASH_ADDR
2037 * offset.
2038 * 3) Wait until the write completes.
2039 */
2040 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_DATA, data << 24);
2041 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_ADDR,
2042 (addr & 0x7FFFF) | (2 << 29));
2043
2044 timeout = 0;
2045 do {
2046 cmd = tavor_flash_read_cfg(hdl, TAVOR_HW_FLASH_ADDR);
2047 timeout++;
2048 } while ((cmd & TAVOR_HW_FLASH_CMD_MASK) &&
2049 (timeout < tavor_hw_flash_timeout_config));
2050
2051 if (timeout == tavor_hw_flash_timeout_config) {
2052 cmn_err(CE_WARN, "tavor_flash_write: config cmd timeout.\n");
2053 }
2054
2055 TAVOR_TNF_EXIT(tavor_flash_write);
2056 }
2057
2058 /*
2059 * tavor_flash_init()
2060 */
2061 static void
2062 tavor_flash_init(tavor_state_t *state)
2063 {
2064 uint32_t word;
2065 ddi_acc_handle_t hdl;
2066 int sema_cnt;
2067 int gpio;
2068
2069 TAVOR_TNF_ENTER(tavor_flash_init);
2070
2071 /* Set handle */
2072 hdl = state->ts_pci_cfghdl;
2073
2074 /* Init the flash */
2075
2076 /*
2077 * Grab the GPIO semaphore. This allows us exclusive access to the
2078 * GPIO settings on the Tavor for the duration of the flash burning
2079 * procedure.
2080 */
2081 sema_cnt = 0;
2082 do {
2083 word = tavor_flash_read_cfg(hdl, TAVOR_HW_FLASH_GPIO_SEMA);
2084 if (word == 0) {
2085 break;
2086 }
2087
2088 sema_cnt++;
2089 drv_usecwait(1);
2090 } while (sema_cnt < tavor_hw_flash_timeout_gpio_sema);
2091
2092 /*
2093 * Determine if we timed out trying to grab the GPIO semaphore
2094 */
2095 if (sema_cnt == tavor_hw_flash_timeout_gpio_sema) {
2096 cmn_err(CE_WARN, "tavor_flash_init: GPIO SEMA timeout\n");
2097 }
2098
2099 /* Save away original GPIO Values */
2100 state->ts_fw_gpio[0] = tavor_flash_read_cfg(hdl,
2101 TAVOR_HW_FLASH_GPIO_DIR);
2102 state->ts_fw_gpio[1] = tavor_flash_read_cfg(hdl,
2103 TAVOR_HW_FLASH_GPIO_POL);
2104 state->ts_fw_gpio[2] = tavor_flash_read_cfg(hdl,
2105 TAVOR_HW_FLASH_GPIO_MOD);
2106 state->ts_fw_gpio[3] = tavor_flash_read_cfg(hdl,
2107 TAVOR_HW_FLASH_GPIO_DAT);
2108
2109 /* Set New GPIO Values */
2110 gpio = state->ts_fw_gpio[0] | 0x70;
2111 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_DIR, gpio);
2112
2113 gpio = state->ts_fw_gpio[1] & ~0x70;
2114 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_POL, gpio);
2115
2116 gpio = state->ts_fw_gpio[2] & ~0x70;
2117 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_MOD, gpio);
2118
2119 /* Set CPUMODE to enable tavor to access the flash device */
2120 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_CPUMODE,
2121 1 << TAVOR_HW_FLASH_CPU_SHIFT);
2122
2123 /* Initialize to bank 0 */
2124 tavor_flash_bank(state, 0);
2125
2126 TAVOR_TNF_EXIT(tavor_flash_init);
2127 }
2128
2129 /*
2130 * tavor_flash_cfi_init
2131 * Implements access to the CFI (Common Flash Interface) data
2132 */
2133 static void
2134 tavor_flash_cfi_init(tavor_state_t *state, uint32_t *cfi_info, int *intel_xcmd)
2135 {
2136 uint32_t data;
2137 uint32_t sector_sz_bytes;
2138 uint32_t bit_count;
2139 uint8_t cfi_ch_info[TAVOR_CFI_INFO_SIZE];
2140 uint32_t cfi_dw_info[TAVOR_CFI_INFO_QSIZE];
2141 int i;
2142
2143 TAVOR_TNF_ENTER(tavor_flash_cfi_init);
2144
2145 /*
2146 * Determine if the user command supports the Intel Extended
2147 * Command Set. The query string is contained in the fourth
2148 * quad word.
2149 */
2150 tavor_flash_cfi_byte(cfi_ch_info, cfi_info[0x04], 0x10);
2151 if (cfi_ch_info[0x10] == 'M' &&
2152 cfi_ch_info[0x11] == 'X' &&
2153 cfi_ch_info[0x12] == '2') {
2154 *intel_xcmd = 1; /* support is there */
2155 }
2156
2157 /* CFI QUERY */
2158 tavor_flash_write(state, 0x55, TAVOR_FLASH_CFI_INIT);
2159
2160 /* Read in CFI data */
2161 for (i = 0; i < TAVOR_CFI_INFO_SIZE; i += 4) {
2162 data = tavor_flash_read(state, i);
2163 cfi_dw_info[i >> 2] = data;
2164 tavor_flash_cfi_byte(cfi_ch_info, data, i);
2165 }
2166
2167 /* Determine chip set */
2168 state->ts_fw_cmdset = TAVOR_FLASH_UNKNOWN_CMDSET;
2169 if (cfi_ch_info[0x20] == 'Q' &&
2170 cfi_ch_info[0x22] == 'R' &&
2171 cfi_ch_info[0x24] == 'Y') {
2172 /*
2173 * Mode: x16 working in x8 mode (Intel).
2174 * Pack data - skip spacing bytes.
2175 */
2176 for (i = 0; i < TAVOR_CFI_INFO_SIZE; i += 2) {
2177 cfi_ch_info[i/2] = cfi_ch_info[i];
2178 }
2179 }
2180 state->ts_fw_cmdset = cfi_ch_info[0x13];
2181 if (state->ts_fw_cmdset != TAVOR_FLASH_INTEL_CMDSET &&
2182 state->ts_fw_cmdset != TAVOR_FLASH_AMD_CMDSET) {
2183 cmn_err(CE_WARN,
2184 "tavor_flash_cfi_init: UNKNOWN chip cmd set\n");
2185 state->ts_fw_cmdset = TAVOR_FLASH_UNKNOWN_CMDSET;
2186 goto out;
2187 }
2188
2189 /* Determine total bytes in one sector size */
2190 sector_sz_bytes = ((cfi_ch_info[0x30] << 8) | cfi_ch_info[0x2F]) << 8;
2191
2192 /* Calculate equivalent of log2 (n) */
2193 for (bit_count = 0; sector_sz_bytes > 1; bit_count++) {
2194 sector_sz_bytes >>= 1;
2195 }
2196
2197 /* Set sector size */
2198 state->ts_fw_log_sector_sz = bit_count;
2199
2200 /* Set flash size */
2201 state->ts_fw_device_sz = 0x1 << cfi_ch_info[0x27];
2202
2203 /* Reset to turn off CFI mode */
2204 tavor_flash_reset(state);
2205
2206 /*
2207 * Pass CFI data back to user command.
2208 */
2209 for (i = 0; i < TAVOR_FLASH_CFI_SIZE_QUADLET; i++) {
2210 tavor_flash_cfi_dword(&cfi_info[i], cfi_ch_info, i << 2);
2211 }
2212
2213 if (*intel_xcmd == 1) {
2214 /*
2215 * Inform the user cmd that this driver does support the
2216 * Intel Extended Command Set.
2217 */
2218 cfi_ch_info[0x10] = 'M';
2219 cfi_ch_info[0x11] = 'X';
2220 cfi_ch_info[0x12] = '2';
2221 } else {
2222 cfi_ch_info[0x10] = 'Q';
2223 cfi_ch_info[0x11] = 'R';
2224 cfi_ch_info[0x12] = 'Y';
2225 }
2226 cfi_ch_info[0x13] = state->ts_fw_cmdset;
2227 tavor_flash_cfi_dword(&cfi_info[0x4], cfi_ch_info, 0x10);
2228 out:
2229 TAVOR_TNF_EXIT(tavor_flash_cfi_init);
2230 }
2231
2232 /*
2233 * tavor_flash_fini()
2234 */
2235 static void
2236 tavor_flash_fini(tavor_state_t *state)
2237 {
2238 ddi_acc_handle_t hdl;
2239
2240 TAVOR_TNF_ENTER(tavor_flash_fini);
2241
2242 /* Set handle */
2243 hdl = state->ts_pci_cfghdl;
2244
2245 /* Restore original GPIO Values */
2246 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_DIR,
2247 state->ts_fw_gpio[0]);
2248 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_POL,
2249 state->ts_fw_gpio[1]);
2250 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_MOD,
2251 state->ts_fw_gpio[2]);
2252 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_DAT,
2253 state->ts_fw_gpio[3]);
2254
2255 /* Give up semaphore */
2256 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_SEMA, 0);
2257
2258 TAVOR_TNF_EXIT(tavor_flash_fini);
2259 }
2260
2261 /*
2262 * tavor_flash_read_cfg
2263 */
2264 static uint32_t
2265 tavor_flash_read_cfg(ddi_acc_handle_t pci_config_hdl, uint32_t addr)
2266 {
2267 uint32_t read;
2268
2269 TAVOR_TNF_ENTER(tavor_flash_read_cfg);
2270
2271 /*
2272 * Perform flash read operation:
2273 * 1) Place addr to read from on the TAVOR_HW_FLASH_CFG_ADDR register
2274 * 2) Read data at that addr from the TAVOR_HW_FLASH_CFG_DATA register
2275 */
2276 pci_config_put32(pci_config_hdl, TAVOR_HW_FLASH_CFG_ADDR, addr);
2277 read = pci_config_get32(pci_config_hdl, TAVOR_HW_FLASH_CFG_DATA);
2278
2279 TAVOR_TNF_EXIT(tavor_flash_read_cfg);
2280
2281 return (read);
2282 }
2283
2284 /*
2285 * tavor_flash_write_cfg
2286 */
2287 static void
2288 tavor_flash_write_cfg(ddi_acc_handle_t pci_config_hdl, uint32_t addr,
2289 uint32_t data)
2290 {
2291 TAVOR_TNF_ENTER(tavor_flash_write_cfg);
2292
2293 /*
2294 * Perform flash write operation:
2295 * 1) Place addr to write to on the TAVOR_HW_FLASH_CFG_ADDR register
2296 * 2) Place data to write on to the TAVOR_HW_FLASH_CFG_DATA register
2297 */
2298 pci_config_put32(pci_config_hdl, TAVOR_HW_FLASH_CFG_ADDR, addr);
2299 pci_config_put32(pci_config_hdl, TAVOR_HW_FLASH_CFG_DATA, data);
2300
2301 TAVOR_TNF_EXIT(tavor_flash_write_cfg);
2302 }
2303
2304 /*
2305 * Support routines to convert Common Flash Interface (CFI) data
2306 * from a 32 bit word to a char array, and from a char array to
2307 * a 32 bit word.
2308 */
2309 static void
2310 tavor_flash_cfi_byte(uint8_t *ch, uint32_t dword, int i)
2311 {
2312 ch[i] = (uint8_t)((dword & 0xFF000000) >> 24);
2313 ch[i+1] = (uint8_t)((dword & 0x00FF0000) >> 16);
2314 ch[i+2] = (uint8_t)((dword & 0x0000FF00) >> 8);
2315 ch[i+3] = (uint8_t)((dword & 0x000000FF));
2316 }
2317
2318 static void
2319 tavor_flash_cfi_dword(uint32_t *dword, uint8_t *ch, int i)
2320 {
2321 *dword = (uint32_t)
2322 ((uint32_t)ch[i] << 24 |
2323 (uint32_t)ch[i+1] << 16 |
2324 (uint32_t)ch[i+2] << 8 |
2325 (uint32_t)ch[i+3]);
2326 }
2327
2328 /*
2329 * tavor_loopback_free_qps
2330 */
2331 static void
2332 tavor_loopback_free_qps(tavor_loopback_state_t *lstate)
2333 {
2334 int i;
2335
2336 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*lstate))
2337
2338 if (lstate->tls_tx.tlc_qp_hdl != NULL) {
2339 (void) tavor_qp_free(lstate->tls_state,
2340 &lstate->tls_tx.tlc_qp_hdl, IBC_FREE_QP_AND_QPN, NULL,
2341 TAVOR_NOSLEEP);
2342 }
2343 if (lstate->tls_rx.tlc_qp_hdl != NULL) {
2344 (void) tavor_qp_free(lstate->tls_state,
2345 &lstate->tls_rx.tlc_qp_hdl, IBC_FREE_QP_AND_QPN, NULL,
2346 TAVOR_NOSLEEP);
2347 }
2348 lstate->tls_tx.tlc_qp_hdl = NULL;
2349 lstate->tls_rx.tlc_qp_hdl = NULL;
2350 for (i = 0; i < 2; i++) {
2351 if (lstate->tls_tx.tlc_cqhdl[i] != NULL) {
2352 (void) tavor_cq_free(lstate->tls_state,
2353 &lstate->tls_tx.tlc_cqhdl[i], TAVOR_NOSLEEP);
2354 }
2355 if (lstate->tls_rx.tlc_cqhdl[i] != NULL) {
2356 (void) tavor_cq_free(lstate->tls_state,
2357 &lstate->tls_rx.tlc_cqhdl[i], TAVOR_NOSLEEP);
2358 }
2359 lstate->tls_tx.tlc_cqhdl[i] = NULL;
2360 lstate->tls_rx.tlc_cqhdl[i] = NULL;
2361 }
2362 }
2363
2364 /*
2365 * tavor_loopback_free_state
2366 */
2367 static void
2368 tavor_loopback_free_state(tavor_loopback_state_t *lstate)
2369 {
2370 tavor_loopback_free_qps(lstate);
2371 if (lstate->tls_tx.tlc_mrhdl != NULL) {
2372 (void) tavor_mr_deregister(lstate->tls_state,
2373 &lstate->tls_tx.tlc_mrhdl, TAVOR_MR_DEREG_ALL,
2374 TAVOR_NOSLEEP);
2375 }
2376 if (lstate->tls_rx.tlc_mrhdl != NULL) {
2377 (void) tavor_mr_deregister(lstate->tls_state,
2378 &lstate->tls_rx.tlc_mrhdl, TAVOR_MR_DEREG_ALL,
2379 TAVOR_NOSLEEP);
2380 }
2381 if (lstate->tls_pd_hdl != NULL) {
2382 (void) tavor_pd_free(lstate->tls_state, &lstate->tls_pd_hdl);
2383 }
2384 if (lstate->tls_tx.tlc_buf != NULL) {
2385 kmem_free(lstate->tls_tx.tlc_buf, lstate->tls_tx.tlc_buf_sz);
2386 }
2387 if (lstate->tls_rx.tlc_buf != NULL) {
2388 kmem_free(lstate->tls_rx.tlc_buf, lstate->tls_rx.tlc_buf_sz);
2389 }
2390 bzero(lstate, sizeof (tavor_loopback_state_t));
2391 }
2392
2393 /*
2394 * tavor_loopback_init
2395 */
2396 static int
2397 tavor_loopback_init(tavor_state_t *state, tavor_loopback_state_t *lstate)
2398 {
2399 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*lstate))
2400
2401 lstate->tls_hca_hdl = (ibc_hca_hdl_t)state;
2402 lstate->tls_status = tavor_pd_alloc(lstate->tls_state,
2403 &lstate->tls_pd_hdl, TAVOR_NOSLEEP);
2404 if (lstate->tls_status != IBT_SUCCESS) {
2405 lstate->tls_err = TAVOR_LOOPBACK_PROT_DOMAIN_ALLOC_FAIL;
2406 TNF_PROBE_0(tavor_ioctl_loopback_alloc_pd_fail,
2407 TAVOR_TNF_ERROR, "");
2408 return (EFAULT);
2409 }
2410
2411 return (0);
2412 }
2413
2414 /*
2415 * tavor_loopback_init_qp_info
2416 */
2417 static void
2418 tavor_loopback_init_qp_info(tavor_loopback_state_t *lstate,
2419 tavor_loopback_comm_t *comm)
2420 {
2421 bzero(&comm->tlc_cq_attr, sizeof (ibt_cq_attr_t));
2422 bzero(&comm->tlc_qp_attr, sizeof (ibt_qp_alloc_attr_t));
2423 bzero(&comm->tlc_qp_info, sizeof (ibt_qp_info_t));
2424
2425 comm->tlc_wrid = 1;
2426 comm->tlc_cq_attr.cq_size = 128;
2427 comm->tlc_qp_attr.qp_sizes.cs_sq_sgl = 3;
2428 comm->tlc_qp_attr.qp_sizes.cs_rq_sgl = 3;
2429 comm->tlc_qp_attr.qp_sizes.cs_sq = 16;
2430 comm->tlc_qp_attr.qp_sizes.cs_rq = 16;
2431 comm->tlc_qp_attr.qp_flags = IBT_WR_SIGNALED;
2432
2433 comm->tlc_qp_info.qp_state = IBT_STATE_RESET;
2434 comm->tlc_qp_info.qp_trans = IBT_RC_SRV;
2435 comm->tlc_qp_info.qp_flags = IBT_CEP_RDMA_RD | IBT_CEP_RDMA_WR;
2436 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_hca_port_num =
2437 lstate->tls_port;
2438 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_pkey_ix =
2439 lstate->tls_pkey_ix;
2440 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_timeout =
2441 lstate->tls_timeout;
2442 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_adds_vect.av_srvl = 0;
2443 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_adds_vect.av_srate =
2444 IBT_SRATE_4X;
2445 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_adds_vect.av_send_grh = 0;
2446 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_adds_vect.av_dlid =
2447 lstate->tls_lid;
2448 comm->tlc_qp_info.qp_transport.rc.rc_retry_cnt = lstate->tls_retry;
2449 comm->tlc_qp_info.qp_transport.rc.rc_sq_psn = 0;
2450 comm->tlc_qp_info.qp_transport.rc.rc_rq_psn = 0;
2451 comm->tlc_qp_info.qp_transport.rc.rc_rdma_ra_in = 4;
2452 comm->tlc_qp_info.qp_transport.rc.rc_rdma_ra_out = 4;
2453 comm->tlc_qp_info.qp_transport.rc.rc_dst_qpn = 0;
2454 comm->tlc_qp_info.qp_transport.rc.rc_min_rnr_nak = IBT_RNR_NAK_655ms;
2455 comm->tlc_qp_info.qp_transport.rc.rc_path_mtu = IB_MTU_1K;
2456 }
2457
2458 /*
2459 * tavor_loopback_alloc_mem
2460 */
2461 static int
2462 tavor_loopback_alloc_mem(tavor_loopback_state_t *lstate,
2463 tavor_loopback_comm_t *comm, int sz)
2464 {
2465 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*comm))
2466
2467 /* Allocate buffer of specified size */
2468 comm->tlc_buf_sz = sz;
2469 comm->tlc_buf = kmem_zalloc(sz, KM_NOSLEEP);
2470 if (comm->tlc_buf == NULL) {
2471 return (EFAULT);
2472 }
2473
2474 /* Register the buffer as a memory region */
2475 comm->tlc_memattr.mr_vaddr = (uint64_t)(uintptr_t)comm->tlc_buf;
2476 comm->tlc_memattr.mr_len = (ib_msglen_t)sz;
2477 comm->tlc_memattr.mr_as = NULL;
2478 comm->tlc_memattr.mr_flags = IBT_MR_NOSLEEP |
2479 IBT_MR_ENABLE_REMOTE_WRITE | IBT_MR_ENABLE_LOCAL_WRITE;
2480
2481 comm->tlc_status = tavor_mr_register(lstate->tls_state,
2482 lstate->tls_pd_hdl, &comm->tlc_memattr, &comm->tlc_mrhdl, NULL);
2483
2484 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*comm->tlc_mrhdl))
2485
2486 comm->tlc_mrdesc.md_vaddr = comm->tlc_mrhdl->mr_bindinfo.bi_addr;
2487 comm->tlc_mrdesc.md_lkey = comm->tlc_mrhdl->mr_lkey;
2488 comm->tlc_mrdesc.md_rkey = comm->tlc_mrhdl->mr_rkey;
2489 if (comm->tlc_status != IBT_SUCCESS) {
2490 return (EFAULT);
2491 }
2492 return (0);
2493 }
2494
2495 /*
2496 * tavor_loopback_alloc_qps
2497 */
2498 static int
2499 tavor_loopback_alloc_qps(tavor_loopback_state_t *lstate,
2500 tavor_loopback_comm_t *comm)
2501 {
2502 uint32_t i, real_size;
2503 tavor_qp_info_t qpinfo;
2504
2505 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*comm))
2506 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*lstate))
2507
2508 /* Allocate send and recv CQs */
2509 for (i = 0; i < 2; i++) {
2510 bzero(&comm->tlc_cq_attr, sizeof (ibt_cq_attr_t));
2511 comm->tlc_cq_attr.cq_size = 128;
2512 comm->tlc_status = tavor_cq_alloc(lstate->tls_state,
2513 (ibt_cq_hdl_t)NULL, &comm->tlc_cq_attr, &real_size,
2514 &comm->tlc_cqhdl[i], TAVOR_NOSLEEP);
2515 if (comm->tlc_status != IBT_SUCCESS) {
2516 lstate->tls_err += i;
2517 return (EFAULT);
2518 }
2519 }
2520
2521 /* Allocate the QP */
2522 tavor_loopback_init_qp_info(lstate, comm);
2523 comm->tlc_qp_attr.qp_pd_hdl = (ibt_pd_hdl_t)lstate->tls_pd_hdl;
2524 comm->tlc_qp_attr.qp_scq_hdl = (ibt_cq_hdl_t)comm->tlc_cqhdl[0];
2525 comm->tlc_qp_attr.qp_rcq_hdl = (ibt_cq_hdl_t)comm->tlc_cqhdl[1];
2526 comm->tlc_qp_attr.qp_ibc_scq_hdl = (ibt_opaque1_t)comm->tlc_cqhdl[0];
2527 comm->tlc_qp_attr.qp_ibc_rcq_hdl = (ibt_opaque1_t)comm->tlc_cqhdl[1];
2528 qpinfo.qpi_attrp = &comm->tlc_qp_attr;
2529 qpinfo.qpi_type = IBT_RC_RQP;
2530 qpinfo.qpi_ibt_qphdl = NULL;
2531 qpinfo.qpi_queueszp = &comm->tlc_chan_sizes;
2532 qpinfo.qpi_qpn = &comm->tlc_qp_num;
2533 comm->tlc_status = tavor_qp_alloc(lstate->tls_state, &qpinfo,
2534 TAVOR_NOSLEEP, NULL);
2535 if (comm->tlc_status == DDI_SUCCESS) {
2536 comm->tlc_qp_hdl = qpinfo.qpi_qphdl;
2537 }
2538
2539 if (comm->tlc_status != IBT_SUCCESS) {
2540 lstate->tls_err += 2;
2541 return (EFAULT);
2542 }
2543 return (0);
2544 }
2545
2546 /*
2547 * tavor_loopback_modify_qp
2548 */
2549 static int
2550 tavor_loopback_modify_qp(tavor_loopback_state_t *lstate,
2551 tavor_loopback_comm_t *comm, uint_t qp_num)
2552 {
2553 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*comm))
2554 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*lstate))
2555
2556 /* Modify QP to INIT */
2557 tavor_loopback_init_qp_info(lstate, comm);
2558 comm->tlc_qp_info.qp_state = IBT_STATE_INIT;
2559 comm->tlc_status = tavor_qp_modify(lstate->tls_state, comm->tlc_qp_hdl,
2560 IBT_CEP_SET_STATE, &comm->tlc_qp_info, &comm->tlc_queue_sizes);
2561 if (comm->tlc_status != IBT_SUCCESS) {
2562 return (EFAULT);
2563 }
2564
2565 /*
2566 * Modify QP to RTR (set destination LID and QP number to local
2567 * LID and QP number)
2568 */
2569 comm->tlc_qp_info.qp_state = IBT_STATE_RTR;
2570 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_adds_vect.av_dlid
2571 = lstate->tls_lid;
2572 comm->tlc_qp_info.qp_transport.rc.rc_dst_qpn = qp_num;
2573 comm->tlc_status = tavor_qp_modify(lstate->tls_state, comm->tlc_qp_hdl,
2574 IBT_CEP_SET_STATE, &comm->tlc_qp_info, &comm->tlc_queue_sizes);
2575 if (comm->tlc_status != IBT_SUCCESS) {
2576 lstate->tls_err += 1;
2577 return (EFAULT);
2578 }
2579
2580 /* Modify QP to RTS */
2581 comm->tlc_qp_info.qp_current_state = IBT_STATE_RTR;
2582 comm->tlc_qp_info.qp_state = IBT_STATE_RTS;
2583 comm->tlc_status = tavor_qp_modify(lstate->tls_state, comm->tlc_qp_hdl,
2584 IBT_CEP_SET_STATE, &comm->tlc_qp_info, &comm->tlc_queue_sizes);
2585 if (comm->tlc_status != IBT_SUCCESS) {
2586 lstate->tls_err += 2;
2587 return (EFAULT);
2588 }
2589 return (0);
2590 }
2591
2592 /*
2593 * tavor_loopback_copyout
2594 */
2595 static int
2596 tavor_loopback_copyout(tavor_loopback_ioctl_t *lb, intptr_t arg, int mode)
2597 {
2598 #ifdef _MULTI_DATAMODEL
2599 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
2600 tavor_loopback_ioctl32_t lb32;
2601
2602 lb32.tlb_revision = lb->tlb_revision;
2603 lb32.tlb_send_buf =
2604 (caddr32_t)(uintptr_t)lb->tlb_send_buf;
2605 lb32.tlb_fail_buf =
2606 (caddr32_t)(uintptr_t)lb->tlb_fail_buf;
2607 lb32.tlb_buf_sz = lb->tlb_buf_sz;
2608 lb32.tlb_num_iter = lb->tlb_num_iter;
2609 lb32.tlb_pass_done = lb->tlb_pass_done;
2610 lb32.tlb_timeout = lb->tlb_timeout;
2611 lb32.tlb_error_type = lb->tlb_error_type;
2612 lb32.tlb_port_num = lb->tlb_port_num;
2613 lb32.tlb_num_retry = lb->tlb_num_retry;
2614
2615 if (ddi_copyout(&lb32, (void *)arg,
2616 sizeof (tavor_loopback_ioctl32_t), mode) != 0) {
2617 TNF_PROBE_0(tavor_ioctl_loopback_copyout_fail,
2618 TAVOR_TNF_ERROR, "");
2619 return (EFAULT);
2620 }
2621 } else
2622 #endif /* _MULTI_DATAMODEL */
2623 if (ddi_copyout(lb, (void *)arg, sizeof (tavor_loopback_ioctl_t),
2624 mode) != 0) {
2625 TNF_PROBE_0(tavor_ioctl_loopback_copyout_fail,
2626 TAVOR_TNF_ERROR, "");
2627 return (EFAULT);
2628 }
2629 return (0);
2630 }
2631
2632 /*
2633 * tavor_loopback_post_send
2634 */
2635 static int
2636 tavor_loopback_post_send(tavor_loopback_state_t *lstate,
2637 tavor_loopback_comm_t *tx, tavor_loopback_comm_t *rx)
2638 {
2639 int ret;
2640
2641 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*tx))
2642
2643 bzero(&tx->tlc_sgl, sizeof (ibt_wr_ds_t));
2644 bzero(&tx->tlc_wr, sizeof (ibt_send_wr_t));
2645
2646 /* Initialize local address for TX buffer */
2647 tx->tlc_sgl.ds_va = tx->tlc_mrdesc.md_vaddr;
2648 tx->tlc_sgl.ds_key = tx->tlc_mrdesc.md_lkey;
2649 tx->tlc_sgl.ds_len = tx->tlc_buf_sz;
2650
2651 /* Initialize the remaining details of the work request */
2652 tx->tlc_wr.wr_id = tx->tlc_wrid++;
2653 tx->tlc_wr.wr_flags = IBT_WR_SEND_SIGNAL;
2654 tx->tlc_wr.wr_nds = 1;
2655 tx->tlc_wr.wr_sgl = &tx->tlc_sgl;
2656 tx->tlc_wr.wr_opcode = IBT_WRC_RDMAW;
2657 tx->tlc_wr.wr_trans = IBT_RC_SRV;
2658
2659 /* Initialize the remote address for RX buffer */
2660 tx->tlc_wr.wr.rc.rcwr.rdma.rdma_raddr = rx->tlc_mrdesc.md_vaddr;
2661 tx->tlc_wr.wr.rc.rcwr.rdma.rdma_rkey = rx->tlc_mrdesc.md_rkey;
2662 tx->tlc_complete = 0;
2663 ret = tavor_post_send(lstate->tls_state, tx->tlc_qp_hdl, &tx->tlc_wr,
2664 1, NULL);
2665 if (ret != IBT_SUCCESS) {
2666 return (EFAULT);
2667 }
2668 return (0);
2669 }
2670
2671 /*
2672 * tavor_loopback_poll_cq
2673 */
2674 static int
2675 tavor_loopback_poll_cq(tavor_loopback_state_t *lstate,
2676 tavor_loopback_comm_t *comm)
2677 {
2678 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*comm))
2679
2680 comm->tlc_wc.wc_status = 0;
2681 comm->tlc_num_polled = 0;
2682 comm->tlc_status = tavor_cq_poll(lstate->tls_state,
2683 comm->tlc_cqhdl[0], &comm->tlc_wc, 1, &comm->tlc_num_polled);
2684 if ((comm->tlc_status == IBT_SUCCESS) &&
2685 (comm->tlc_wc.wc_status != IBT_WC_SUCCESS)) {
2686 comm->tlc_status = ibc_get_ci_failure(0);
2687 }
2688 return (comm->tlc_status);
2689 }