1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /*
28 * tavor_ioctl.c
29 * Tavor IOCTL Routines
30 *
31 * Implements all ioctl access into the driver. This includes all routines
32 * necessary for updating firmware, accessing the tavor flash device, and
33 * providing interfaces for VTS.
34 */
35
36 #include <sys/types.h>
37 #include <sys/conf.h>
38 #include <sys/ddi.h>
39 #include <sys/sunddi.h>
40 #include <sys/modctl.h>
41 #include <sys/file.h>
42
43 #include <sys/ib/adapters/tavor/tavor.h>
44
45 /* Tavor HCA state pointer (extern) */
46 extern void *tavor_statep;
47
48 /*
49 * The ioctl declarations (for firmware flash burning, register read/write
50 * (DEBUG-only), and VTS interfaces)
51 */
52 static int tavor_ioctl_flash_read(tavor_state_t *state, dev_t dev,
53 intptr_t arg, int mode);
54 static int tavor_ioctl_flash_write(tavor_state_t *state, dev_t dev,
55 intptr_t arg, int mode);
56 static int tavor_ioctl_flash_erase(tavor_state_t *state, dev_t dev,
57 intptr_t arg, int mode);
58 static int tavor_ioctl_flash_init(tavor_state_t *state, dev_t dev,
59 intptr_t arg, int mode);
60 static int tavor_ioctl_flash_fini(tavor_state_t *state, dev_t dev);
61 static void tavor_ioctl_flash_cleanup(tavor_state_t *state);
62 static void tavor_ioctl_flash_cleanup_nolock(tavor_state_t *state);
63 #ifdef DEBUG
64 static int tavor_ioctl_reg_write(tavor_state_t *state, intptr_t arg,
65 int mode);
66 static int tavor_ioctl_reg_read(tavor_state_t *state, intptr_t arg,
67 int mode);
68 #endif /* DEBUG */
69 static int tavor_ioctl_info(tavor_state_t *state, dev_t dev,
70 intptr_t arg, int mode);
71 static int tavor_ioctl_ports(tavor_state_t *state, intptr_t arg,
72 int mode);
73 static int tavor_ioctl_loopback(tavor_state_t *state, intptr_t arg,
74 int mode);
75 static int tavor_ioctl_ddr_read(tavor_state_t *state, intptr_t arg,
76 int mode);
77
78 /* Tavor Flash Functions */
79 static void tavor_flash_read_sector(tavor_state_t *state, uint32_t sector_num);
80 static void tavor_flash_read_quadlet(tavor_state_t *state, uint32_t *data,
81 uint32_t addr);
82 static int tavor_flash_write_sector(tavor_state_t *state, uint32_t sector_num);
83 static int tavor_flash_write_byte(tavor_state_t *state, uint32_t addr,
84 uchar_t data);
85 static int tavor_flash_erase_sector(tavor_state_t *state, uint32_t sector_num);
86 static int tavor_flash_erase_chip(tavor_state_t *state);
87 static void tavor_flash_bank(tavor_state_t *state, uint32_t addr);
88 static uint32_t tavor_flash_read(tavor_state_t *state, uint32_t addr);
89 static void tavor_flash_write(tavor_state_t *state, uint32_t addr,
90 uchar_t data);
91 static void tavor_flash_init(tavor_state_t *state);
92 static void tavor_flash_cfi_init(tavor_state_t *state, uint32_t *cfi_info,
93 int *intel_xcmd);
94 static void tavor_flash_fini(tavor_state_t *state);
95 static void tavor_flash_reset(tavor_state_t *state);
96 static uint32_t tavor_flash_read_cfg(ddi_acc_handle_t pci_config_hdl,
97 uint32_t addr);
98 static void tavor_flash_write_cfg(ddi_acc_handle_t pci_config_hdl,
99 uint32_t addr, uint32_t data);
100 static void tavor_flash_cfi_byte(uint8_t *ch, uint32_t dword, int i);
101 static void tavor_flash_cfi_dword(uint32_t *dword, uint8_t *ch, int i);
102
103 /* Tavor loopback test functions */
104 static void tavor_loopback_free_qps(tavor_loopback_state_t *lstate);
105 static void tavor_loopback_free_state(tavor_loopback_state_t *lstate);
106 static int tavor_loopback_init(tavor_state_t *state,
107 tavor_loopback_state_t *lstate);
108 static void tavor_loopback_init_qp_info(tavor_loopback_state_t *lstate,
109 tavor_loopback_comm_t *comm);
110 static int tavor_loopback_alloc_mem(tavor_loopback_state_t *lstate,
111 tavor_loopback_comm_t *comm, int sz);
112 static int tavor_loopback_alloc_qps(tavor_loopback_state_t *lstate,
113 tavor_loopback_comm_t *comm);
114 static int tavor_loopback_modify_qp(tavor_loopback_state_t *lstate,
115 tavor_loopback_comm_t *comm, uint_t qp_num);
116 static int tavor_loopback_copyout(tavor_loopback_ioctl_t *lb,
117 intptr_t arg, int mode);
118 static int tavor_loopback_post_send(tavor_loopback_state_t *lstate,
119 tavor_loopback_comm_t *tx, tavor_loopback_comm_t *rx);
120 static int tavor_loopback_poll_cq(tavor_loopback_state_t *lstate,
121 tavor_loopback_comm_t *comm);
122
123 /* Patchable timeout values for flash operations */
124 int tavor_hw_flash_timeout_gpio_sema = TAVOR_HW_FLASH_TIMEOUT_GPIO_SEMA;
125 int tavor_hw_flash_timeout_config = TAVOR_HW_FLASH_TIMEOUT_CONFIG;
126 int tavor_hw_flash_timeout_write = TAVOR_HW_FLASH_TIMEOUT_WRITE;
127 int tavor_hw_flash_timeout_erase = TAVOR_HW_FLASH_TIMEOUT_ERASE;
128
129 /*
130 * tavor_ioctl()
131 */
132 /* ARGSUSED */
133 int
134 tavor_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp,
135 int *rvalp)
136 {
137 tavor_state_t *state;
138 minor_t instance;
139 int status;
140
141 TAVOR_TNF_ENTER(tavor_ioctl);
142
143 if (drv_priv(credp) != 0) {
144 TNF_PROBE_0(tavor_ioctl_priv_fail, TAVOR_TNF_ERROR, "");
145 TAVOR_TNF_EXIT(tavor_ioctl);
146 return (EPERM);
147 }
148
149 instance = TAVOR_DEV_INSTANCE(dev);
150 if (instance == -1) {
151 TNF_PROBE_0(tavor_ioctl_inst_fail, TAVOR_TNF_ERROR, "");
152 TAVOR_TNF_EXIT(tavor_ioctl);
153 return (EBADF);
154 }
155
156 state = ddi_get_soft_state(tavor_statep, instance);
157 if (state == NULL) {
158 TNF_PROBE_0(tavor_ioctl_gss_fail, TAVOR_TNF_ERROR, "");
159 TAVOR_TNF_EXIT(tavor_ioctl);
160 return (EBADF);
161 }
162
163 status = 0;
164
165 switch (cmd) {
166 case TAVOR_IOCTL_FLASH_READ:
167 status = tavor_ioctl_flash_read(state, dev, arg, mode);
168 break;
169
170 case TAVOR_IOCTL_FLASH_WRITE:
171 status = tavor_ioctl_flash_write(state, dev, arg, mode);
172 break;
173
174 case TAVOR_IOCTL_FLASH_ERASE:
175 status = tavor_ioctl_flash_erase(state, dev, arg, mode);
176 break;
177
178 case TAVOR_IOCTL_FLASH_INIT:
179 status = tavor_ioctl_flash_init(state, dev, arg, mode);
180 break;
181
182 case TAVOR_IOCTL_FLASH_FINI:
183 status = tavor_ioctl_flash_fini(state, dev);
184 break;
185
186 case TAVOR_IOCTL_INFO:
187 status = tavor_ioctl_info(state, dev, arg, mode);
188 break;
189
190 case TAVOR_IOCTL_PORTS:
191 status = tavor_ioctl_ports(state, arg, mode);
192 break;
193
194 case TAVOR_IOCTL_DDR_READ:
195 status = tavor_ioctl_ddr_read(state, arg, mode);
196 break;
197
198 case TAVOR_IOCTL_LOOPBACK:
199 status = tavor_ioctl_loopback(state, arg, mode);
200 break;
201
202 #ifdef DEBUG
203 case TAVOR_IOCTL_REG_WRITE:
204 status = tavor_ioctl_reg_write(state, arg, mode);
205 break;
206
207 case TAVOR_IOCTL_REG_READ:
208 status = tavor_ioctl_reg_read(state, arg, mode);
209 break;
210 #endif /* DEBUG */
211
212 default:
213 status = ENOTTY;
214 TNF_PROBE_0(tavor_ioctl_default_fail, TAVOR_TNF_ERROR, "");
215 break;
216 }
217 *rvalp = status;
218
219 TAVOR_TNF_EXIT(tavor_ioctl);
220 return (status);
221 }
222
223 /*
224 * tavor_ioctl_flash_read()
225 */
226 static int
227 tavor_ioctl_flash_read(tavor_state_t *state, dev_t dev, intptr_t arg, int mode)
228 {
229 tavor_flash_ioctl_t ioctl_info;
230 int status = 0;
231
232 TAVOR_TNF_ENTER(tavor_ioctl_flash_read);
233
234 /*
235 * Check that flash init ioctl has been called first. And check
236 * that the same dev_t that called init is the one calling read now.
237 */
238 mutex_enter(&state->ts_fw_flashlock);
239 if ((state->ts_fw_flashdev != dev) ||
240 (state->ts_fw_flashstarted == 0)) {
241 mutex_exit(&state->ts_fw_flashlock);
242 TNF_PROBE_0(tavor_flash_bad_state, TAVOR_TNF_ERROR, "");
243 TAVOR_TNF_EXIT(tavor_ioctl_flash_read);
244 return (EIO);
245 }
246
247 /* copy user struct to kernel */
248 #ifdef _MULTI_DATAMODEL
249 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
250 tavor_flash_ioctl32_t info32;
251
252 if (ddi_copyin((void *)arg, &info32,
253 sizeof (tavor_flash_ioctl32_t), mode) != 0) {
254 mutex_exit(&state->ts_fw_flashlock);
255 TNF_PROBE_0(tavor_ioctl_flash_read_copyin_fail,
256 TAVOR_TNF_ERROR, "");
257 TAVOR_TNF_EXIT(tavor_ioctl_flash_read);
258 return (EFAULT);
259 }
260 ioctl_info.tf_type = info32.tf_type;
261 ioctl_info.tf_sector = (caddr_t)(uintptr_t)info32.tf_sector;
262 ioctl_info.tf_sector_num = info32.tf_sector_num;
263 ioctl_info.tf_addr = info32.tf_addr;
264 } else
265 #endif /* _MULTI_DATAMODEL */
266 if (ddi_copyin((void *)arg, &ioctl_info, sizeof (tavor_flash_ioctl_t),
267 mode) != 0) {
268 mutex_exit(&state->ts_fw_flashlock);
269 TNF_PROBE_0(tavor_ioctl_flash_read_copyin_fail,
270 TAVOR_TNF_ERROR, "");
271 TAVOR_TNF_EXIT(tavor_ioctl_flash_read);
272 return (EFAULT);
273 }
274
275 /*
276 * Determine type of READ ioctl
277 */
278 switch (ioctl_info.tf_type) {
279 case TAVOR_FLASH_READ_SECTOR:
280 /* Check if sector num is too large for flash device */
281 if (ioctl_info.tf_sector_num >=
282 (state->ts_fw_device_sz >> state->ts_fw_log_sector_sz)) {
283 mutex_exit(&state->ts_fw_flashlock);
284 TNF_PROBE_0(tavor_flash_read_sector_num_too_large,
285 TAVOR_TNF_ERROR, "");
286 TAVOR_TNF_EXIT(tavor_ioctl_flash_read);
287 return (EFAULT);
288 }
289
290 /* Perform the Sector Read */
291 tavor_flash_reset(state);
292 tavor_flash_read_sector(state, ioctl_info.tf_sector_num);
293
294 /* copyout the firmware sector image data */
295 if (ddi_copyout(&state->ts_fw_sector[0],
296 &ioctl_info.tf_sector[0], 1 << state->ts_fw_log_sector_sz,
297 mode) != 0) {
298 mutex_exit(&state->ts_fw_flashlock);
299 TNF_PROBE_0(tavor_flash_read_copyout_fail,
300 TAVOR_TNF_ERROR, "");
301 TAVOR_TNF_EXIT(tavor_ioctl_flash_read);
302 return (EFAULT);
303 }
304 break;
305
306 case TAVOR_FLASH_READ_QUADLET:
307 /* Check if addr is too large for flash device */
308 if (ioctl_info.tf_addr >= state->ts_fw_device_sz) {
309 mutex_exit(&state->ts_fw_flashlock);
310 TNF_PROBE_0(tavor_flash_read_quad_addr_too_large,
311 TAVOR_TNF_ERROR, "");
312 TAVOR_TNF_EXIT(tavor_ioctl_flash_read);
313 return (EFAULT);
314 }
315
316 /* Perform the Quadlet Read */
317 tavor_flash_reset(state);
318 tavor_flash_read_quadlet(state, &ioctl_info.tf_quadlet,
319 ioctl_info.tf_addr);
320 break;
321
322 default:
323 TNF_PROBE_0(tavor_ioctl_flash_read_invalid_type,
324 TAVOR_TNF_ERROR, "");
325 status = EIO;
326 break;
327 }
328
329 /* copy results back to userland */
330 #ifdef _MULTI_DATAMODEL
331 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
332 tavor_flash_ioctl32_t info32;
333
334 info32.tf_quadlet = ioctl_info.tf_quadlet;
335 info32.tf_type = ioctl_info.tf_type;
336 info32.tf_sector_num = ioctl_info.tf_sector_num;
337 info32.tf_sector = (caddr32_t)(uintptr_t)ioctl_info.tf_sector;
338 info32.tf_addr = ioctl_info.tf_addr;
339
340 if (ddi_copyout(&info32, (void *)arg,
341 sizeof (tavor_flash_ioctl32_t), mode) != 0) {
342 mutex_exit(&state->ts_fw_flashlock);
343 TNF_PROBE_0(tavor_flash_read_copyout_fail,
344 TAVOR_TNF_ERROR, "");
345 TAVOR_TNF_EXIT(tavor_ioctl_flash_read);
346 return (EFAULT);
347 }
348 } else
349 #endif /* _MULTI_DATAMODEL */
350 if (ddi_copyout(&ioctl_info, (void *)arg,
351 sizeof (tavor_flash_ioctl_t), mode) != 0) {
352 mutex_exit(&state->ts_fw_flashlock);
353 TNF_PROBE_0(tavor_flash_read_copyout_fail,
354 TAVOR_TNF_ERROR, "");
355 TAVOR_TNF_EXIT(tavor_ioctl_flash_read);
356 return (EFAULT);
357 }
358
359 mutex_exit(&state->ts_fw_flashlock);
360 TAVOR_TNF_EXIT(tavor_ioctl_flash_read);
361 return (status);
362 }
363
364 /*
365 * tavor_ioctl_flash_write()
366 */
367 static int
368 tavor_ioctl_flash_write(tavor_state_t *state, dev_t dev, intptr_t arg, int mode)
369 {
370 tavor_flash_ioctl_t ioctl_info;
371 int status = 0;
372
373 TAVOR_TNF_ENTER(tavor_ioctl_flash_write);
374
375 /*
376 * Check that flash init ioctl has been called first. And check
377 * that the same dev_t that called init is the one calling write now.
378 */
379 mutex_enter(&state->ts_fw_flashlock);
380 if ((state->ts_fw_flashdev != dev) ||
381 (state->ts_fw_flashstarted == 0)) {
382 mutex_exit(&state->ts_fw_flashlock);
383 TNF_PROBE_0(tavor_flash_bad_state, TAVOR_TNF_ERROR, "");
384 TAVOR_TNF_EXIT(tavor_ioctl_flash_write);
385 return (EIO);
386 }
387
388 /* copy user struct to kernel */
389 #ifdef _MULTI_DATAMODEL
390 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
391 tavor_flash_ioctl32_t info32;
392
393 if (ddi_copyin((void *)arg, &info32,
394 sizeof (tavor_flash_ioctl32_t), mode) != 0) {
395 mutex_exit(&state->ts_fw_flashlock);
396 TNF_PROBE_0(tavor_ioctl_flash_write_copyin_fail,
397 TAVOR_TNF_ERROR, "");
398 TAVOR_TNF_EXIT(tavor_ioctl_flash_write);
399 return (EFAULT);
400 }
401 ioctl_info.tf_type = info32.tf_type;
402 ioctl_info.tf_sector = (caddr_t)(uintptr_t)info32.tf_sector;
403 ioctl_info.tf_sector_num = info32.tf_sector_num;
404 ioctl_info.tf_addr = info32.tf_addr;
405 ioctl_info.tf_byte = info32.tf_byte;
406 } else
407 #endif /* _MULTI_DATAMODEL */
408 if (ddi_copyin((void *)arg, &ioctl_info,
409 sizeof (tavor_flash_ioctl_t), mode) != 0) {
410 mutex_exit(&state->ts_fw_flashlock);
411 TNF_PROBE_0(tavor_ioctl_flash_write_ci_fail,
412 TAVOR_TNF_ERROR, "");
413 TAVOR_TNF_EXIT(tavor_ioctl_flash_write);
414 return (EFAULT);
415 }
416
417 /*
418 * Determine type of WRITE ioctl
419 */
420 switch (ioctl_info.tf_type) {
421 case TAVOR_FLASH_WRITE_SECTOR:
422 /* Check if sector num is too large for flash device */
423 if (ioctl_info.tf_sector_num >=
424 (state->ts_fw_device_sz >> state->ts_fw_log_sector_sz)) {
425 mutex_exit(&state->ts_fw_flashlock);
426 TNF_PROBE_0(tavor_flash_write_sector_num_too_large,
427 TAVOR_TNF_ERROR, "");
428 TAVOR_TNF_EXIT(tavor_ioctl_flash_write);
429 return (EFAULT);
430 }
431
432 /* copy in fw sector image data */
433 if (ddi_copyin(&ioctl_info.tf_sector[0],
434 &state->ts_fw_sector[0], 1 << state->ts_fw_log_sector_sz,
435 mode) != 0) {
436 mutex_exit(&state->ts_fw_flashlock);
437 TNF_PROBE_0(tavor_ioctl_flash_write_fw_sector_ci_fail,
438 TAVOR_TNF_ERROR, "");
439 TAVOR_TNF_EXIT(tavor_ioctl_flash_write);
440 return (EFAULT);
441 }
442
443 /* Perform Write Sector */
444 status = tavor_flash_write_sector(state,
445 ioctl_info.tf_sector_num);
446 break;
447
448 case TAVOR_FLASH_WRITE_BYTE:
449 /* Check if addr is too large for flash device */
450 if (ioctl_info.tf_addr >= state->ts_fw_device_sz) {
451 mutex_exit(&state->ts_fw_flashlock);
452 TNF_PROBE_0(tavor_flash_write_byte_addr_too_large,
453 TAVOR_TNF_ERROR, "");
454 TAVOR_TNF_EXIT(tavor_ioctl_flash_write);
455 return (EFAULT);
456 }
457
458 /* Perform Write Byte */
459 tavor_flash_bank(state, ioctl_info.tf_addr);
460 tavor_flash_reset(state);
461 status = tavor_flash_write_byte(state, ioctl_info.tf_addr,
462 ioctl_info.tf_byte);
463 tavor_flash_reset(state);
464 break;
465
466 default:
467 TNF_PROBE_0(tavor_ioctl_flash_write_invalid_type,
468 TAVOR_TNF_ERROR, "");
469 status = EIO;
470 break;
471 }
472
473 mutex_exit(&state->ts_fw_flashlock);
474 TAVOR_TNF_EXIT(tavor_ioctl_flash_write);
475 return (status);
476 }
477
478 /*
479 * tavor_ioctl_flash_erase()
480 */
481 static int
482 tavor_ioctl_flash_erase(tavor_state_t *state, dev_t dev, intptr_t arg, int mode)
483 {
484 tavor_flash_ioctl_t ioctl_info;
485 int status = 0;
486
487 TAVOR_TNF_ENTER(tavor_ioctl_flash_erase);
488
489 /*
490 * Check that flash init ioctl has been called first. And check
491 * that the same dev_t that called init is the one calling erase now.
492 */
493 mutex_enter(&state->ts_fw_flashlock);
494 if ((state->ts_fw_flashdev != dev) ||
495 (state->ts_fw_flashstarted == 0)) {
496 mutex_exit(&state->ts_fw_flashlock);
497 TNF_PROBE_0(tavor_flash_bad_state, TAVOR_TNF_ERROR, "");
498 TAVOR_TNF_EXIT(tavor_ioctl_flash_erase);
499 return (EIO);
500 }
501
502 /* copy user struct to kernel */
503 #ifdef _MULTI_DATAMODEL
504 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
505 tavor_flash_ioctl32_t info32;
506
507 if (ddi_copyin((void *)arg, &info32,
508 sizeof (tavor_flash_ioctl32_t), mode) != 0) {
509 mutex_exit(&state->ts_fw_flashlock);
510 TNF_PROBE_0(tavor_ioctl_flash_read_copyin_fail,
511 TAVOR_TNF_ERROR, "");
512 TAVOR_TNF_EXIT(tavor_ioctl_flash_erase);
513 return (EFAULT);
514 }
515 ioctl_info.tf_type = info32.tf_type;
516 ioctl_info.tf_sector_num = info32.tf_sector_num;
517 } else
518 #endif /* _MULTI_DATAMODEL */
519 if (ddi_copyin((void *)arg, &ioctl_info, sizeof (tavor_flash_ioctl_t),
520 mode) != 0) {
521 mutex_exit(&state->ts_fw_flashlock);
522 TNF_PROBE_0(tavor_ioctl_flash_erase_ci_fail,
523 TAVOR_TNF_ERROR, "");
524 TAVOR_TNF_EXIT(tavor_ioctl_flash_erase);
525 return (EFAULT);
526 }
527
528 /*
529 * Determine type of ERASE ioctl
530 */
531 switch (ioctl_info.tf_type) {
532 case TAVOR_FLASH_ERASE_SECTOR:
533 /* Check if sector num is too large for flash device */
534 if (ioctl_info.tf_sector_num >=
535 (state->ts_fw_device_sz >> state->ts_fw_log_sector_sz)) {
536 mutex_exit(&state->ts_fw_flashlock);
537 TNF_PROBE_0(tavor_flash_erase_sector_num_too_large,
538 TAVOR_TNF_ERROR, "");
539 TAVOR_TNF_EXIT(tavor_ioctl_flash_write);
540 return (EFAULT);
541 }
542
543 /* Perform Sector Erase */
544 status = tavor_flash_erase_sector(state,
545 ioctl_info.tf_sector_num);
546 break;
547
548 case TAVOR_FLASH_ERASE_CHIP:
549 /* Perform Chip Erase */
550 status = tavor_flash_erase_chip(state);
551 break;
552
553 default:
554 TNF_PROBE_0(tavor_ioctl_flash_erase_invalid_type,
555 TAVOR_TNF_ERROR, "");
556 status = EIO;
557 break;
558 }
559
560 mutex_exit(&state->ts_fw_flashlock);
561 TAVOR_TNF_EXIT(tavor_ioctl_flash_erase);
562 return (status);
563 }
564
565 /*
566 * tavor_ioctl_flash_init()
567 */
568 static int
569 tavor_ioctl_flash_init(tavor_state_t *state, dev_t dev, intptr_t arg, int mode)
570 {
571 tavor_flash_init_ioctl_t init_info;
572 int ret;
573 int intel_xcmd = 0;
574
575 TAVOR_TNF_ENTER(tavor_ioctl_flash_init);
576
577 /*
578 * init cannot be called more than once. If we have already init'd the
579 * flash, return directly.
580 */
581 mutex_enter(&state->ts_fw_flashlock);
582 if (state->ts_fw_flashstarted == 1) {
583 mutex_exit(&state->ts_fw_flashlock);
584 TNF_PROBE_0(tavor_ioctl_flash_init_already_started,
585 TAVOR_TNF_ERROR, "");
586 TAVOR_TNF_EXIT(tavor_ioctl_flash_init);
587 return (EIO);
588 }
589
590 /* copyin the user struct to kernel */
591 if (ddi_copyin((void *)arg, &init_info,
592 sizeof (tavor_flash_init_ioctl_t), mode) != 0) {
593 mutex_exit(&state->ts_fw_flashlock);
594 TNF_PROBE_0(tavor_flash_init_ioctl_copyin_fail,
595 TAVOR_TNF_ERROR, "");
596 TAVOR_TNF_EXIT(tavor_ioctl_flash_init);
597 return (EFAULT);
598 }
599
600 /* Init Flash */
601 tavor_flash_init(state);
602
603 /* Read CFI info */
604 tavor_flash_cfi_init(state, &init_info.tf_cfi_info[0], &intel_xcmd);
605
606 /*
607 * Return error if the command set is unknown.
608 */
609 if (state->ts_fw_cmdset == TAVOR_FLASH_UNKNOWN_CMDSET) {
610 mutex_exit(&state->ts_fw_flashlock);
611 TNF_PROBE_1(tavor_ioctl_flash_init_cmdset_fail,
612 TAVOR_TNF_ERROR, "", tnf_string, errmsg,
613 "UNKNOWN flash command set");
614 TAVOR_TNF_EXIT(tavor_ioctl_flash_init);
615 return (EFAULT);
616 }
617
618 /* Read HWREV - least significant 8 bits is revision ID */
619 init_info.tf_hwrev = pci_config_get32(state->ts_pci_cfghdl,
620 TAVOR_HW_FLASH_CFG_HWREV) & 0xFF;
621
622 /* Fill in the firmwate revision numbers */
623 init_info.tf_fwrev.tfi_maj = state->ts_fw.fw_rev_major;
624 init_info.tf_fwrev.tfi_min = state->ts_fw.fw_rev_minor;
625 init_info.tf_fwrev.tfi_sub = state->ts_fw.fw_rev_subminor;
626
627 /* Alloc flash mem for one sector size */
628 state->ts_fw_sector = (uint32_t *)kmem_zalloc(1 <<
629 state->ts_fw_log_sector_sz, KM_SLEEP);
630
631 /* Set HW part number and length */
632 init_info.tf_pn_len = state->ts_hca_pn_len;
633 if (state->ts_hca_pn_len != 0) {
634 (void) memcpy(init_info.tf_hwpn, state->ts_hca_pn,
635 state->ts_hca_pn_len);
636 }
637
638 /* Copy ioctl results back to userland */
639 if (ddi_copyout(&init_info, (void *)arg,
640 sizeof (tavor_flash_init_ioctl_t), mode) != 0) {
641
642 tavor_ioctl_flash_cleanup_nolock(state);
643
644 mutex_exit(&state->ts_fw_flashlock);
645 TNF_PROBE_0(tavor_ioctl_flash_init_copyout_fail,
646 TAVOR_TNF_ERROR, "");
647 TAVOR_TNF_EXIT(tavor_ioctl_flash_init);
648 return (EFAULT);
649 }
650
651 /* Set flash state to started */
652 state->ts_fw_flashstarted = 1;
653 state->ts_fw_flashdev = dev;
654
655 mutex_exit(&state->ts_fw_flashlock);
656
657 /*
658 * If "flash init" is successful, add an "on close" callback to the
659 * current dev node to ensure that "flash fini" gets called later
660 * even if the userland process prematurely exits.
661 */
662 ret = tavor_umap_db_set_onclose_cb(dev,
663 TAVOR_ONCLOSE_FLASH_INPROGRESS,
664 (void (*)(void *))tavor_ioctl_flash_cleanup, state);
665 if (ret != DDI_SUCCESS) {
666 (void) tavor_ioctl_flash_fini(state, dev);
667
668 TNF_PROBE_0(tavor_ioctl_flash_init_set_cb_fail,
669 TAVOR_TNF_ERROR, "");
670 TAVOR_TNF_EXIT(tavor_ioctl_flash_init);
671 return (EFAULT);
672 }
673
674 TAVOR_TNF_EXIT(tavor_ioctl_flash_init);
675 return (0);
676 }
677
678 /*
679 * tavor_ioctl_flash_fini()
680 */
681 static int
682 tavor_ioctl_flash_fini(tavor_state_t *state, dev_t dev)
683 {
684 int ret;
685
686 TAVOR_TNF_ENTER(tavor_ioctl_flash_fini);
687
688 /*
689 * Check that flash init ioctl has been called first. And check
690 * that the same dev_t that called init is the one calling fini now.
691 */
692 mutex_enter(&state->ts_fw_flashlock);
693 if ((state->ts_fw_flashdev != dev) ||
694 (state->ts_fw_flashstarted == 0)) {
695 mutex_exit(&state->ts_fw_flashlock);
696 TNF_PROBE_0(tavor_flash_bad_state, TAVOR_TNF_ERROR, "");
697 TAVOR_TNF_EXIT(tavor_ioctl_flash_fini);
698 return (EIO);
699 }
700
701 tavor_ioctl_flash_cleanup_nolock(state);
702
703 mutex_exit(&state->ts_fw_flashlock);
704
705 /*
706 * If "flash fini" is successful, remove the "on close" callback
707 * that was setup during "flash init".
708 */
709 ret = tavor_umap_db_clear_onclose_cb(dev,
710 TAVOR_ONCLOSE_FLASH_INPROGRESS);
711 if (ret != DDI_SUCCESS) {
712 TNF_PROBE_0(tavor_flash_fini_clear_cb_fail, TAVOR_TNF_ERROR,
713 "");
714 TAVOR_TNF_EXIT(tavor_ioctl_flash_fini);
715 return (EFAULT);
716 }
717
718 TAVOR_TNF_EXIT(tavor_ioctl_flash_fini);
719 return (0);
720 }
721
722
723 /*
724 * tavor_ioctl_flash_cleanup()
725 */
726 static void
727 tavor_ioctl_flash_cleanup(tavor_state_t *state)
728 {
729 TAVOR_TNF_ENTER(tavor_ioctl_flash_cleanup);
730
731 mutex_enter(&state->ts_fw_flashlock);
732 tavor_ioctl_flash_cleanup_nolock(state);
733 mutex_exit(&state->ts_fw_flashlock);
734
735 TAVOR_TNF_EXIT(tavor_ioctl_flash_cleanup);
736 }
737
738
739 /*
740 * tavor_ioctl_flash_cleanup_nolock()
741 */
742 static void
743 tavor_ioctl_flash_cleanup_nolock(tavor_state_t *state)
744 {
745 TAVOR_TNF_ENTER(tavor_ioctl_flash_cleanup_nolock);
746
747 ASSERT(MUTEX_HELD(&state->ts_fw_flashlock));
748
749 /* free flash mem */
750 kmem_free(state->ts_fw_sector, 1 << state->ts_fw_log_sector_sz);
751
752 /* Fini the Flash */
753 tavor_flash_fini(state);
754
755 /* Set flash state to fini */
756 state->ts_fw_flashstarted = 0;
757 state->ts_fw_flashdev = 0;
758
759 TAVOR_TNF_EXIT(tavor_ioctl_flash_cleanup_nolock);
760 }
761
762
763 /*
764 * tavor_ioctl_info()
765 */
766 static int
767 tavor_ioctl_info(tavor_state_t *state, dev_t dev, intptr_t arg, int mode)
768 {
769 tavor_info_ioctl_t info;
770 tavor_flash_init_ioctl_t init_info;
771
772 TAVOR_TNF_ENTER(tavor_ioctl_info);
773
774 /*
775 * Access to Tavor VTS ioctls is not allowed in "maintenance mode".
776 */
777 if (state->ts_operational_mode == TAVOR_MAINTENANCE_MODE) {
778 TNF_PROBE_0(tavor_ioctl_info_maintenance_mode_fail,
779 TAVOR_TNF_ERROR, "");
780 TAVOR_TNF_EXIT(tavor_ioctl_info);
781 return (EFAULT);
782 }
783
784 /* copyin the user struct to kernel */
785 if (ddi_copyin((void *)arg, &info, sizeof (tavor_info_ioctl_t),
786 mode) != 0) {
787 TNF_PROBE_0(tavor_ioctl_info_copyin_fail, TAVOR_TNF_ERROR, "");
788 TAVOR_TNF_EXIT(tavor_ioctl_info);
789 return (EFAULT);
790 }
791
792 /*
793 * Check ioctl revision
794 */
795 if (info.ti_revision != TAVOR_VTS_IOCTL_REVISION) {
796 TNF_PROBE_0(tavor_ioctl_info_bad_rev, TAVOR_TNF_ERROR, "");
797 TAVOR_TNF_EXIT(tavor_ioctl_info);
798 return (EINVAL);
799 }
800
801 /*
802 * If the 'fw_device_sz' has not been initialized yet, we initialize it
803 * here. This is done by leveraging the
804 * tavor_ioctl_flash_init()/fini() calls. We also hold our own mutex
805 * around this operation in case we have multiple VTS threads in
806 * process at the same time.
807 */
808 mutex_enter(&state->ts_info_lock);
809 if (state->ts_fw_device_sz == 0) {
810 if (tavor_ioctl_flash_init(state, dev, (intptr_t)&init_info,
811 (FKIOCTL | mode)) != 0) {
812 mutex_exit(&state->ts_info_lock);
813 TNF_PROBE_0(tavor_ioctl_info_flash_init_fail,
814 TAVOR_TNF_ERROR, "");
815 TAVOR_TNF_EXIT(tavor_ioctl_info);
816 return (EFAULT);
817 }
818 (void) tavor_ioctl_flash_fini(state, dev);
819 }
820 mutex_exit(&state->ts_info_lock);
821
822 info.ti_hw_rev = state->ts_adapter.rev_id;
823 info.ti_flash_sz = state->ts_fw_device_sz;
824 info.ti_fw_rev.tfi_maj = state->ts_fw.fw_rev_major;
825 info.ti_fw_rev.tfi_min = state->ts_fw.fw_rev_minor;
826 info.ti_fw_rev.tfi_sub = state->ts_fw.fw_rev_subminor;
827 info.ti_mem_start_offset = 0;
828 info.ti_mem_end_offset = state->ts_ddr.ddr_endaddr -
829 state->ts_ddr.ddr_baseaddr;
830
831 /* Copy ioctl results back to user struct */
832 if (ddi_copyout(&info, (void *)arg, sizeof (tavor_info_ioctl_t),
833 mode) != 0) {
834 TNF_PROBE_0(tavor_ioctl_info_copyout_fail, TAVOR_TNF_ERROR, "");
835 TAVOR_TNF_EXIT(tavor_ioctl_info);
836 return (EFAULT);
837 }
838
839 TAVOR_TNF_EXIT(tavor_ioctl_info);
840 return (0);
841 }
842
843 /*
844 * tavor_ioctl_ports()
845 */
846 static int
847 tavor_ioctl_ports(tavor_state_t *state, intptr_t arg, int mode)
848 {
849 tavor_ports_ioctl_t info;
850 tavor_stat_port_ioctl_t portstat;
851 ibt_hca_portinfo_t pi;
852 uint_t tbl_size;
853 ib_gid_t *sgid_tbl;
854 ib_pkey_t *pkey_tbl;
855 int i;
856
857 TAVOR_TNF_ENTER(tavor_ioctl_ports);
858
859 /*
860 * Access to Tavor VTS ioctls is not allowed in "maintenance mode".
861 */
862 if (state->ts_operational_mode == TAVOR_MAINTENANCE_MODE) {
863 TNF_PROBE_0(tavor_ioctl_ports_maintenance_mode_fail,
864 TAVOR_TNF_ERROR, "");
865 TAVOR_TNF_EXIT(tavor_ioctl_ports);
866 return (EFAULT);
867 }
868
869 /* copyin the user struct to kernel */
870 #ifdef _MULTI_DATAMODEL
871 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
872 tavor_ports_ioctl32_t info32;
873
874 if (ddi_copyin((void *)arg, &info32,
875 sizeof (tavor_ports_ioctl32_t), mode) != 0) {
876 TNF_PROBE_0(tavor_ioctl_ports_copyin_fail,
877 TAVOR_TNF_ERROR, "");
878 TAVOR_TNF_EXIT(tavor_ioctl_ports);
879 return (EFAULT);
880 }
881 info.tp_revision = info32.tp_revision;
882 info.tp_ports =
883 (tavor_stat_port_ioctl_t *)(uintptr_t)info32.tp_ports;
884 info.tp_num_ports = info32.tp_num_ports;
885
886 } else
887 #endif /* _MULTI_DATAMODEL */
888 if (ddi_copyin((void *)arg, &info, sizeof (tavor_ports_ioctl_t),
889 mode) != 0) {
890 TNF_PROBE_0(tavor_ioctl_ports_copyin_fail, TAVOR_TNF_ERROR, "");
891 TAVOR_TNF_EXIT(tavor_ioctl_ports);
892 return (EFAULT);
893 }
894
895 /*
896 * Check ioctl revision
897 */
898 if (info.tp_revision != TAVOR_VTS_IOCTL_REVISION) {
899 TNF_PROBE_0(tavor_ioctl_ports_bad_rev, TAVOR_TNF_ERROR, "");
900 TAVOR_TNF_EXIT(tavor_ioctl_ports);
901 return (EINVAL);
902 }
903
904 /* Allocate space for temporary GID table/PKey table */
905 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_gidtbl);
906 sgid_tbl = (ib_gid_t *)kmem_zalloc(tbl_size * sizeof (ib_gid_t),
907 KM_SLEEP);
908 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_pkeytbl);
909 pkey_tbl = (ib_pkey_t *)kmem_zalloc(tbl_size * sizeof (ib_pkey_t),
910 KM_SLEEP);
911
912 /*
913 * Setup the number of ports, then loop through all ports and
914 * query properties of each.
915 */
916 info.tp_num_ports = (uint8_t)state->ts_cfg_profile->cp_num_ports;
917 for (i = 0; i < info.tp_num_ports; i++) {
918 /*
919 * Get portstate information from the device. If
920 * tavor_port_query() fails, leave zeroes in user
921 * struct port entry and continue.
922 */
923 bzero(&pi, sizeof (ibt_hca_portinfo_t));
924 pi.p_sgid_tbl = sgid_tbl;
925 pi.p_pkey_tbl = pkey_tbl;
926 if (tavor_port_query(state, i + 1, &pi) != 0) {
927 TNF_PROBE_0(tavor_ioctl_ports_query_failed,
928 TAVOR_TNF_ERROR, "");
929 }
930
931 portstat.tsp_port_num = pi.p_port_num;
932 portstat.tsp_state = pi.p_linkstate;
933 portstat.tsp_guid = pi.p_sgid_tbl[0].gid_guid;
934
935 /*
936 * Copy queried port results back to user struct. If
937 * this fails, then break out of loop, attempt to copy
938 * out remaining info to user struct, and return (without
939 * error).
940 */
941 if (ddi_copyout(&portstat,
942 &(((tavor_stat_port_ioctl_t *)info.tp_ports)[i]),
943 sizeof (tavor_stat_port_ioctl_t), mode) != 0) {
944 break;
945 }
946 }
947
948 /* Free the temporary space used for GID table/PKey table */
949 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_gidtbl);
950 kmem_free(sgid_tbl, tbl_size * sizeof (ib_gid_t));
951 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_pkeytbl);
952 kmem_free(pkey_tbl, tbl_size * sizeof (ib_pkey_t));
953
954 /* Copy ioctl results back to user struct */
955 #ifdef _MULTI_DATAMODEL
956 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
957 tavor_ports_ioctl32_t info32;
958
959 info32.tp_revision = info.tp_revision;
960 info32.tp_ports = (caddr32_t)(uintptr_t)info.tp_ports;
961 info32.tp_num_ports = info.tp_num_ports;
962
963 if (ddi_copyout(&info32, (void *)arg,
964 sizeof (tavor_ports_ioctl32_t), mode) != 0) {
965 TNF_PROBE_0(tavor_ioctl_ports_copyout_fail,
966 TAVOR_TNF_ERROR, "");
967 TAVOR_TNF_EXIT(tavor_ioctl_ports);
968 return (EFAULT);
969 }
970 } else
971 #endif /* _MULTI_DATAMODEL */
972 if (ddi_copyout(&info, (void *)arg, sizeof (tavor_ports_ioctl_t),
973 mode) != 0) {
974 TNF_PROBE_0(tavor_ioctl_ports_copyout_fail,
975 TAVOR_TNF_ERROR, "");
976 TAVOR_TNF_EXIT(tavor_ioctl_ports);
977 return (EFAULT);
978 }
979
980 TAVOR_TNF_EXIT(tavor_ioctl_ports);
981 return (0);
982 }
983
984 /*
985 * tavor_ioctl_loopback()
986 */
987 static int
988 tavor_ioctl_loopback(tavor_state_t *state, intptr_t arg, int mode)
989 {
990 tavor_loopback_ioctl_t lb;
991 tavor_loopback_state_t lstate;
992 ibt_hca_portinfo_t pi;
993 uint_t tbl_size, loopmax, max_usec;
994 ib_gid_t *sgid_tbl;
995 ib_pkey_t *pkey_tbl;
996 int j, iter, ret;
997
998 TAVOR_TNF_ENTER(tavor_ioctl_loopback);
999
1000 /*
1001 * Access to Tavor VTS ioctls is not allowed in "maintenance mode".
1002 */
1003 if (state->ts_operational_mode == TAVOR_MAINTENANCE_MODE) {
1004 TNF_PROBE_0(tavor_ioctl_loopback_maintenance_mode_fail,
1005 TAVOR_TNF_ERROR, "");
1006 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1007 return (EFAULT);
1008 }
1009
1010 /* copyin the user struct to kernel */
1011 #ifdef _MULTI_DATAMODEL
1012 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
1013 tavor_loopback_ioctl32_t lb32;
1014
1015 if (ddi_copyin((void *)arg, &lb32,
1016 sizeof (tavor_loopback_ioctl32_t), mode) != 0) {
1017 TNF_PROBE_0(tavor_ioctl_loopback_copyin_fail,
1018 TAVOR_TNF_ERROR, "");
1019 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1020 return (EFAULT);
1021 }
1022 lb.tlb_revision = lb32.tlb_revision;
1023 lb.tlb_send_buf = (caddr_t)(uintptr_t)lb32.tlb_send_buf;
1024 lb.tlb_fail_buf = (caddr_t)(uintptr_t)lb32.tlb_fail_buf;
1025 lb.tlb_buf_sz = lb32.tlb_buf_sz;
1026 lb.tlb_num_iter = lb32.tlb_num_iter;
1027 lb.tlb_pass_done = lb32.tlb_pass_done;
1028 lb.tlb_timeout = lb32.tlb_timeout;
1029 lb.tlb_error_type = lb32.tlb_error_type;
1030 lb.tlb_port_num = lb32.tlb_port_num;
1031 lb.tlb_num_retry = lb32.tlb_num_retry;
1032 } else
1033 #endif /* _MULTI_DATAMODEL */
1034 if (ddi_copyin((void *)arg, &lb, sizeof (tavor_loopback_ioctl_t),
1035 mode) != 0) {
1036 TNF_PROBE_0(tavor_ioctl_loopback_copyin_fail,
1037 TAVOR_TNF_ERROR, "");
1038 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1039 return (EFAULT);
1040 }
1041
1042 /* Initialize the internal loopback test state structure */
1043 bzero(&lstate, sizeof (tavor_loopback_state_t));
1044
1045 /*
1046 * Check ioctl revision
1047 */
1048 if (lb.tlb_revision != TAVOR_VTS_IOCTL_REVISION) {
1049 lb.tlb_error_type = TAVOR_LOOPBACK_INVALID_REVISION;
1050 (void) tavor_loopback_copyout(&lb, arg, mode);
1051 TNF_PROBE_0(tavor_ioctl_loopback_bad_rev,
1052 TAVOR_TNF_ERROR, "");
1053 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1054 return (EINVAL);
1055 }
1056
1057 /* Validate that specified port number is legal */
1058 if (!tavor_portnum_is_valid(state, lb.tlb_port_num)) {
1059 lb.tlb_error_type = TAVOR_LOOPBACK_INVALID_PORT;
1060 (void) tavor_loopback_copyout(&lb, arg, mode);
1061 TNF_PROBE_0(tavor_ioctl_loopback_inv_port,
1062 TAVOR_TNF_ERROR, "");
1063 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1064 return (EINVAL);
1065 }
1066
1067 /* Allocate space for temporary GID table/PKey table */
1068 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_gidtbl);
1069 sgid_tbl = (ib_gid_t *)kmem_zalloc(tbl_size * sizeof (ib_gid_t),
1070 KM_SLEEP);
1071 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_pkeytbl);
1072 pkey_tbl = (ib_pkey_t *)kmem_zalloc(tbl_size * sizeof (ib_pkey_t),
1073 KM_SLEEP);
1074
1075 /*
1076 * Get portstate information from specific port on device
1077 */
1078 bzero(&pi, sizeof (ibt_hca_portinfo_t));
1079 pi.p_sgid_tbl = sgid_tbl;
1080 pi.p_pkey_tbl = pkey_tbl;
1081 if (tavor_port_query(state, lb.tlb_port_num, &pi) != 0) {
1082 /* Free the temporary space used for GID table/PKey table */
1083 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_gidtbl);
1084 kmem_free(sgid_tbl, tbl_size * sizeof (ib_gid_t));
1085 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_pkeytbl);
1086 kmem_free(pkey_tbl, tbl_size * sizeof (ib_pkey_t));
1087
1088 lb.tlb_error_type = TAVOR_LOOPBACK_INVALID_PORT;
1089 (void) tavor_loopback_copyout(&lb, arg, mode);
1090 tavor_loopback_free_state(&lstate);
1091 TNF_PROBE_0(tavor_ioctl_loopback_bad_port,
1092 TAVOR_TNF_ERROR, "");
1093 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1094 return (EINVAL);
1095 }
1096
1097 lstate.tls_port = pi.p_port_num;
1098 lstate.tls_lid = pi.p_base_lid;
1099 lstate.tls_pkey_ix = (pi.p_linkstate == TAVOR_PORT_LINK_ACTIVE) ? 1 : 0;
1100 lstate.tls_state = state;
1101 lstate.tls_retry = lb.tlb_num_retry;
1102
1103 /* Free the temporary space used for GID table/PKey table */
1104 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_gidtbl);
1105 kmem_free(sgid_tbl, tbl_size * sizeof (ib_gid_t));
1106 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_pkeytbl);
1107 kmem_free(pkey_tbl, tbl_size * sizeof (ib_pkey_t));
1108
1109 /*
1110 * Compute the timeout duration in usec per the formula:
1111 * to_usec_per_retry = 4.096us * (2 ^ supplied_timeout)
1112 * (plus we add a little fudge-factor here too)
1113 */
1114 lstate.tls_timeout = lb.tlb_timeout;
1115 max_usec = (4096 * (1 << lstate.tls_timeout)) / 1000;
1116 max_usec = max_usec * (lstate.tls_retry + 1);
1117 max_usec = max_usec + 10000;
1118
1119 /*
1120 * Determine how many times we should loop before declaring a
1121 * timeout failure.
1122 */
1123 loopmax = max_usec/TAVOR_VTS_LOOPBACK_MIN_WAIT_DUR;
1124 if ((max_usec % TAVOR_VTS_LOOPBACK_MIN_WAIT_DUR) != 0) {
1125 loopmax++;
1126 }
1127
1128 if (lb.tlb_send_buf == NULL || lb.tlb_buf_sz == 0) {
1129 lb.tlb_error_type = TAVOR_LOOPBACK_SEND_BUF_INVALID;
1130 (void) tavor_loopback_copyout(&lb, arg, mode);
1131 tavor_loopback_free_state(&lstate);
1132 TNF_PROBE_0(tavor_ioctl_loopback_buf_null,
1133 TAVOR_TNF_ERROR, "");
1134 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1135 return (EINVAL);
1136 }
1137
1138 /* Allocate protection domain (PD) */
1139 if (tavor_loopback_init(state, &lstate) != 0) {
1140 lb.tlb_error_type = lstate.tls_err;
1141 (void) tavor_loopback_copyout(&lb, arg, mode);
1142 tavor_loopback_free_state(&lstate);
1143 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1144 return (EFAULT);
1145 }
1146
1147 /* Allocate and register a TX buffer */
1148 if (tavor_loopback_alloc_mem(&lstate, &lstate.tls_tx,
1149 lb.tlb_buf_sz) != 0) {
1150 lb.tlb_error_type =
1151 TAVOR_LOOPBACK_SEND_BUF_MEM_REGION_ALLOC_FAIL;
1152 (void) tavor_loopback_copyout(&lb, arg, mode);
1153 tavor_loopback_free_state(&lstate);
1154 TNF_PROBE_0(tavor_ioctl_loopback_txbuf_alloc_fail,
1155 TAVOR_TNF_ERROR, "");
1156 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1157 return (EFAULT);
1158 }
1159
1160 /* Allocate and register an RX buffer */
1161 if (tavor_loopback_alloc_mem(&lstate, &lstate.tls_rx,
1162 lb.tlb_buf_sz) != 0) {
1163 lb.tlb_error_type =
1164 TAVOR_LOOPBACK_RECV_BUF_MEM_REGION_ALLOC_FAIL;
1165 (void) tavor_loopback_copyout(&lb, arg, mode);
1166 tavor_loopback_free_state(&lstate);
1167 TNF_PROBE_0(tavor_ioctl_loopback_rxbuf_alloc_fail,
1168 TAVOR_TNF_ERROR, "");
1169 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1170 return (EFAULT);
1171 }
1172
1173 /* Copy in the transmit buffer data */
1174 if (ddi_copyin((void *)lb.tlb_send_buf, lstate.tls_tx.tlc_buf,
1175 lb.tlb_buf_sz, mode) != 0) {
1176 lb.tlb_error_type = TAVOR_LOOPBACK_SEND_BUF_COPY_FAIL;
1177 (void) tavor_loopback_copyout(&lb, arg, mode);
1178 tavor_loopback_free_state(&lstate);
1179 TNF_PROBE_0(tavor_ioctl_loopback_tx_copyin_fail,
1180 TAVOR_TNF_ERROR, "");
1181 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1182 return (EFAULT);
1183 }
1184
1185 /* Allocate the transmit QP and CQs */
1186 lstate.tls_err = TAVOR_LOOPBACK_XMIT_SEND_CQ_ALLOC_FAIL;
1187 if (tavor_loopback_alloc_qps(&lstate, &lstate.tls_tx) != 0) {
1188 lb.tlb_error_type = lstate.tls_err;
1189 (void) tavor_loopback_copyout(&lb, arg, mode);
1190 tavor_loopback_free_state(&lstate);
1191 TNF_PROBE_0(tavor_ioctl_loopback_txqp_alloc_fail,
1192 TAVOR_TNF_ERROR, "");
1193 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1194 return (EFAULT);
1195 }
1196
1197 /* Allocate the receive QP and CQs */
1198 lstate.tls_err = TAVOR_LOOPBACK_RECV_SEND_CQ_ALLOC_FAIL;
1199 if (tavor_loopback_alloc_qps(&lstate, &lstate.tls_rx) != 0) {
1200 lb.tlb_error_type = lstate.tls_err;
1201 (void) tavor_loopback_copyout(&lb, arg, mode);
1202 tavor_loopback_free_state(&lstate);
1203 TNF_PROBE_0(tavor_ioctl_loopback_rxqp_alloc_fail,
1204 TAVOR_TNF_ERROR, "");
1205 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1206 return (EFAULT);
1207 }
1208
1209 /* Activate the TX QP (connect to RX QP) */
1210 lstate.tls_err = TAVOR_LOOPBACK_XMIT_QP_INIT_FAIL;
1211 if (tavor_loopback_modify_qp(&lstate, &lstate.tls_tx,
1212 lstate.tls_rx.tlc_qp_num) != 0) {
1213 lb.tlb_error_type = lstate.tls_err;
1214 (void) tavor_loopback_copyout(&lb, arg, mode);
1215 tavor_loopback_free_state(&lstate);
1216 TNF_PROBE_0(tavor_ioctl_loopback_txqp_init_fail,
1217 TAVOR_TNF_ERROR, "");
1218 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1219 return (EFAULT);
1220 }
1221
1222 /* Activate the RX QP (connect to TX QP) */
1223 lstate.tls_err = TAVOR_LOOPBACK_RECV_QP_INIT_FAIL;
1224 if (tavor_loopback_modify_qp(&lstate, &lstate.tls_rx,
1225 lstate.tls_tx.tlc_qp_num) != 0) {
1226 lb.tlb_error_type = lstate.tls_err;
1227 (void) tavor_loopback_copyout(&lb, arg, mode);
1228 tavor_loopback_free_state(&lstate);
1229 TNF_PROBE_0(tavor_ioctl_loopback_rxqp_init_fail,
1230 TAVOR_TNF_ERROR, "");
1231 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1232 return (EFAULT);
1233 }
1234
1235 /* Run the loopback test (for specified number of iterations) */
1236 lb.tlb_pass_done = 0;
1237 for (iter = 0; iter < lb.tlb_num_iter; iter++) {
1238 lstate.tls_err = 0;
1239 bzero(lstate.tls_rx.tlc_buf, lb.tlb_buf_sz);
1240
1241 /* Post RDMA Write work request */
1242 if (tavor_loopback_post_send(&lstate, &lstate.tls_tx,
1243 &lstate.tls_rx) != IBT_SUCCESS) {
1244 lb.tlb_error_type = TAVOR_LOOPBACK_WQE_POST_FAIL;
1245 (void) tavor_loopback_copyout(&lb, arg, mode);
1246 tavor_loopback_free_state(&lstate);
1247 TNF_PROBE_0(tavor_ioctl_loopback_wqe_post_fail,
1248 TAVOR_TNF_ERROR, "");
1249 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1250 return (EFAULT);
1251 }
1252
1253 /* Poll the TX CQ for a completion every few ticks */
1254 for (j = 0; j < loopmax; j++) {
1255 delay(drv_usectohz(TAVOR_VTS_LOOPBACK_MIN_WAIT_DUR));
1256
1257 ret = tavor_loopback_poll_cq(&lstate, &lstate.tls_tx);
1258 if (((ret != IBT_SUCCESS) && (ret != IBT_CQ_EMPTY)) ||
1259 ((ret == IBT_CQ_EMPTY) && (j == loopmax - 1))) {
1260 lb.tlb_error_type = TAVOR_LOOPBACK_CQ_POLL_FAIL;
1261 if (ddi_copyout(lstate.tls_rx.tlc_buf,
1262 lb.tlb_fail_buf, lstate.tls_tx.tlc_buf_sz,
1263 mode) != 0) {
1264 TNF_PROBE_0(
1265 tavor_ioctl_loopback_xfer_co_fail,
1266 TAVOR_TNF_ERROR, "");
1267 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1268 return (EFAULT);
1269 }
1270 (void) tavor_loopback_copyout(&lb, arg, mode);
1271 tavor_loopback_free_state(&lstate);
1272 TNF_PROBE_0(tavor_ioctl_loopback_xfer_fail,
1273 TAVOR_TNF_ERROR, "");
1274 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1275 return (EFAULT);
1276 } else if (ret == IBT_CQ_EMPTY) {
1277 continue;
1278 }
1279
1280 /* Compare the data buffers */
1281 if (bcmp(lstate.tls_tx.tlc_buf, lstate.tls_rx.tlc_buf,
1282 lb.tlb_buf_sz) == 0) {
1283 break;
1284 } else {
1285 lb.tlb_error_type =
1286 TAVOR_LOOPBACK_SEND_RECV_COMPARE_FAIL;
1287 if (ddi_copyout(lstate.tls_rx.tlc_buf,
1288 lb.tlb_fail_buf, lstate.tls_tx.tlc_buf_sz,
1289 mode) != 0) {
1290 TNF_PROBE_0(
1291 tavor_ioctl_loopback_bcmp_co_fail,
1292 TAVOR_TNF_ERROR, "");
1293 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1294 return (EFAULT);
1295 }
1296 (void) tavor_loopback_copyout(&lb, arg, mode);
1297 tavor_loopback_free_state(&lstate);
1298 TNF_PROBE_0(tavor_ioctl_loopback_bcmp_fail,
1299 TAVOR_TNF_ERROR, "");
1300 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1301 return (EFAULT);
1302 }
1303 }
1304
1305 lstate.tls_err = TAVOR_LOOPBACK_SUCCESS;
1306 lb.tlb_pass_done = iter + 1;
1307 }
1308
1309 lb.tlb_error_type = TAVOR_LOOPBACK_SUCCESS;
1310
1311 /* Copy ioctl results back to user struct */
1312 ret = tavor_loopback_copyout(&lb, arg, mode);
1313
1314 /* Free up everything and release all consumed resources */
1315 tavor_loopback_free_state(&lstate);
1316
1317 TAVOR_TNF_EXIT(tavor_ioctl_loopback);
1318 return (ret);
1319 }
1320
1321 /*
1322 * tavor_ioctl_ddr_read()
1323 */
1324 static int
1325 tavor_ioctl_ddr_read(tavor_state_t *state, intptr_t arg, int mode)
1326 {
1327 tavor_ddr_read_ioctl_t rdreg;
1328 uint32_t *addr;
1329 uintptr_t baseaddr;
1330 uint64_t ddr_size;
1331
1332 TAVOR_TNF_ENTER(tavor_ioctl_ddr_read);
1333
1334 /*
1335 * Access to Tavor VTS ioctls is not allowed in "maintenance mode".
1336 */
1337 if (state->ts_operational_mode == TAVOR_MAINTENANCE_MODE) {
1338 TNF_PROBE_0(tavor_ioctl_ddr_read_maintenance_mode_fail,
1339 TAVOR_TNF_ERROR, "");
1340 TAVOR_TNF_EXIT(tavor_ioctl_ddr_read);
1341 return (EFAULT);
1342 }
1343
1344 /* copyin the user struct to kernel */
1345 if (ddi_copyin((void *)arg, &rdreg, sizeof (tavor_ddr_read_ioctl_t),
1346 mode) != 0) {
1347 TNF_PROBE_0(tavor_ioctl_ddr_read_copyin_fail,
1348 TAVOR_TNF_ERROR, "");
1349 TAVOR_TNF_EXIT(tavor_ioctl_ddr_read);
1350 return (EFAULT);
1351 }
1352
1353 /*
1354 * Check ioctl revision
1355 */
1356 if (rdreg.tdr_revision != TAVOR_VTS_IOCTL_REVISION) {
1357 TNF_PROBE_0(tavor_ioctl_ddr_read_bad_rev, TAVOR_TNF_ERROR, "");
1358 TAVOR_TNF_EXIT(tavor_ioctl_ddr_read);
1359 return (EINVAL);
1360 }
1361
1362 /*
1363 * Check for valid offset
1364 */
1365 ddr_size = (state->ts_ddr.ddr_endaddr - state->ts_ddr.ddr_baseaddr + 1);
1366 if ((uint64_t)rdreg.tdr_offset >= ddr_size) {
1367 TNF_PROBE_0(tavor_ioctl_ddr_read_bad_offset,
1368 TAVOR_TNF_ERROR, "");
1369 TAVOR_TNF_EXIT(tavor_ioctl_ddr_read);
1370 return (EINVAL);
1371 }
1372
1373 /* Determine base address for requested register read */
1374 baseaddr = (uintptr_t)state->ts_reg_ddr_baseaddr;
1375
1376 /* Ensure that address is properly-aligned */
1377 addr = (uint32_t *)((baseaddr + rdreg.tdr_offset) & ~0x3);
1378
1379 /* Read the register pointed to by addr */
1380 rdreg.tdr_data = ddi_get32(state->ts_reg_cmdhdl, addr);
1381
1382 /* Copy ioctl results back to user struct */
1383 if (ddi_copyout(&rdreg, (void *)arg, sizeof (tavor_ddr_read_ioctl_t),
1384 mode) != 0) {
1385 TNF_PROBE_0(tavor_ioctl_ddr_read_copyout_fail,
1386 TAVOR_TNF_ERROR, "");
1387 TAVOR_TNF_EXIT(tavor_ioctl_ddr_read);
1388 return (EFAULT);
1389 }
1390
1391 TAVOR_TNF_EXIT(tavor_ioctl_ddr_read);
1392 return (0);
1393 }
1394
1395
1396 #ifdef DEBUG
1397 /*
1398 * tavor_ioctl_reg_read()
1399 */
1400 static int
1401 tavor_ioctl_reg_read(tavor_state_t *state, intptr_t arg, int mode)
1402 {
1403 tavor_reg_ioctl_t rdreg;
1404 uint32_t *addr;
1405 uintptr_t baseaddr;
1406 int status;
1407
1408 TAVOR_TNF_ENTER(tavor_ioctl_reg_read);
1409
1410 /*
1411 * Access to Tavor registers is not allowed in "maintenance mode".
1412 * This is primarily because the device may not have BARs to access
1413 */
1414 if (state->ts_operational_mode == TAVOR_MAINTENANCE_MODE) {
1415 TNF_PROBE_0(tavor_ioctl_reg_read_maintence_mode_fail,
1416 TAVOR_TNF_ERROR, "");
1417 TAVOR_TNF_EXIT(tavor_ioctl_reg_read);
1418 return (EFAULT);
1419 }
1420
1421 /* Copy in the tavor_reg_ioctl_t structure */
1422 status = ddi_copyin((void *)arg, &rdreg, sizeof (tavor_reg_ioctl_t),
1423 mode);
1424 if (status != 0) {
1425 TNF_PROBE_0(tavor_ioctl_reg_read_copyin_fail,
1426 TAVOR_TNF_ERROR, "");
1427 TAVOR_TNF_EXIT(tavor_ioctl_reg_read);
1428 return (EFAULT);
1429 }
1430
1431 /* Determine base address for requested register set */
1432 switch (rdreg.trg_reg_set) {
1433 case TAVOR_CMD_BAR:
1434 baseaddr = (uintptr_t)state->ts_reg_cmd_baseaddr;
1435 break;
1436
1437 case TAVOR_UAR_BAR:
1438 baseaddr = (uintptr_t)state->ts_reg_uar_baseaddr;
1439 break;
1440
1441 case TAVOR_DDR_BAR:
1442 baseaddr = (uintptr_t)state->ts_reg_ddr_baseaddr;
1443 break;
1444
1445 default:
1446 TNF_PROBE_0(tavor_ioctl_reg_read_invregset_fail,
1447 TAVOR_TNF_ERROR, "");
1448 TAVOR_TNF_EXIT(tavor_ioctl_reg_read);
1449 return (EFAULT);
1450 }
1451
1452 /* Ensure that address is properly-aligned */
1453 addr = (uint32_t *)((baseaddr + rdreg.trg_offset) & ~0x3);
1454
1455 /* Read the register pointed to by addr */
1456 rdreg.trg_data = ddi_get32(state->ts_reg_cmdhdl, addr);
1457
1458 /* Copy in the result into the tavor_reg_ioctl_t structure */
1459 status = ddi_copyout(&rdreg, (void *)arg, sizeof (tavor_reg_ioctl_t),
1460 mode);
1461 if (status != 0) {
1462 TNF_PROBE_0(tavor_ioctl_reg_read_copyout_fail,
1463 TAVOR_TNF_ERROR, "");
1464 TAVOR_TNF_EXIT(tavor_ioctl_reg_read);
1465 return (EFAULT);
1466 }
1467
1468 TAVOR_TNF_EXIT(tavor_ioctl_reg_read);
1469 return (0);
1470 }
1471
1472
1473 /*
1474 * tavor_ioctl_reg_write()
1475 */
1476 static int
1477 tavor_ioctl_reg_write(tavor_state_t *state, intptr_t arg, int mode)
1478 {
1479 tavor_reg_ioctl_t wrreg;
1480 uint32_t *addr;
1481 uintptr_t baseaddr;
1482 int status;
1483
1484 TAVOR_TNF_ENTER(tavor_ioctl_reg_write);
1485
1486 /*
1487 * Access to Tavor registers is not allowed in "maintenance mode".
1488 * This is primarily because the device may not have BARs to access
1489 */
1490 if (state->ts_operational_mode == TAVOR_MAINTENANCE_MODE) {
1491 TNF_PROBE_0(tavor_ioctl_reg_write_maintence_mode_fail,
1492 TAVOR_TNF_ERROR, "");
1493 TAVOR_TNF_EXIT(tavor_ioctl_reg_write);
1494 return (EFAULT);
1495 }
1496
1497 /* Copy in the tavor_reg_ioctl_t structure */
1498 status = ddi_copyin((void *)arg, &wrreg, sizeof (tavor_reg_ioctl_t),
1499 mode);
1500 if (status != 0) {
1501 TNF_PROBE_0(tavor_ioctl_reg_write_copyin_fail,
1502 TAVOR_TNF_ERROR, "");
1503 TAVOR_TNF_EXIT(tavor_ioctl_reg_write);
1504 return (EFAULT);
1505 }
1506
1507 /* Determine base address for requested register set */
1508 switch (wrreg.trg_reg_set) {
1509 case TAVOR_CMD_BAR:
1510 baseaddr = (uintptr_t)state->ts_reg_cmd_baseaddr;
1511 break;
1512
1513 case TAVOR_UAR_BAR:
1514 baseaddr = (uintptr_t)state->ts_reg_uar_baseaddr;
1515 break;
1516
1517 case TAVOR_DDR_BAR:
1518 baseaddr = (uintptr_t)state->ts_reg_ddr_baseaddr;
1519 break;
1520
1521 default:
1522 TNF_PROBE_0(tavor_ioctl_reg_write_invregset_fail,
1523 TAVOR_TNF_ERROR, "");
1524 TAVOR_TNF_EXIT(tavor_ioctl_reg_write);
1525 return (EFAULT);
1526 }
1527
1528 /* Ensure that address is properly-aligned */
1529 addr = (uint32_t *)((baseaddr + wrreg.trg_offset) & ~0x3);
1530
1531 /* Write the data to the register pointed to by addr */
1532 ddi_put32(state->ts_reg_cmdhdl, addr, wrreg.trg_data);
1533
1534 TAVOR_TNF_EXIT(tavor_ioctl_reg_write);
1535 return (0);
1536 }
1537 #endif /* DEBUG */
1538
1539 /*
1540 * tavor_flash_reset()
1541 */
1542 static void
1543 tavor_flash_reset(tavor_state_t *state)
1544 {
1545 TAVOR_TNF_ENTER(tavor_flash_reset);
1546
1547 /*
1548 * Performs a reset to the flash device. After a reset the flash will
1549 * be operating in normal mode (capable of read/write, etc.).
1550 */
1551 switch (state->ts_fw_cmdset) {
1552 case TAVOR_FLASH_AMD_CMDSET:
1553 tavor_flash_write(state, 0x555, TAVOR_HW_FLASH_RESET_AMD);
1554 break;
1555
1556 case TAVOR_FLASH_INTEL_CMDSET:
1557 tavor_flash_write(state, 0x555, TAVOR_HW_FLASH_RESET_INTEL);
1558 break;
1559
1560 default:
1561 break;
1562 }
1563
1564 TAVOR_TNF_EXIT(tavor_flash_reset);
1565 }
1566
1567 /*
1568 * tavor_flash_read_sector()
1569 */
1570 static void
1571 tavor_flash_read_sector(tavor_state_t *state, uint32_t sector_num)
1572 {
1573 uint32_t addr;
1574 uint32_t end_addr;
1575 uint32_t *image;
1576 int i;
1577
1578 TAVOR_TNF_ENTER(tavor_flash_read_sector);
1579
1580 image = (uint32_t *)&state->ts_fw_sector[0];
1581
1582 /*
1583 * Calculate the start and end address of the sector, based on the
1584 * sector number passed in.
1585 */
1586 addr = sector_num << state->ts_fw_log_sector_sz;
1587 end_addr = addr + (1 << state->ts_fw_log_sector_sz);
1588
1589 /* Set the flash bank correctly for the given address */
1590 tavor_flash_bank(state, addr);
1591
1592 /* Read the entire sector, one quadlet at a time */
1593 for (i = 0; addr < end_addr; i++, addr += 4) {
1594 image[i] = tavor_flash_read(state, addr);
1595 }
1596
1597 TAVOR_TNF_EXIT(tavor_flash_read_sector);
1598 }
1599
1600 /*
1601 * tavor_flash_read_quadlet()
1602 */
1603 static void
1604 tavor_flash_read_quadlet(tavor_state_t *state, uint32_t *data,
1605 uint32_t addr)
1606 {
1607 TAVOR_TNF_ENTER(tavor_flash_read_quadlet);
1608
1609 /* Set the flash bank correctly for the given address */
1610 tavor_flash_bank(state, addr);
1611
1612 /* Read one quadlet of data */
1613 *data = tavor_flash_read(state, addr);
1614
1615 TAVOR_TNF_EXIT(tavor_flash_read_quadlet);
1616 }
1617
1618 /*
1619 * tavor_flash_write_sector()
1620 */
1621 static int
1622 tavor_flash_write_sector(tavor_state_t *state, uint32_t sector_num)
1623 {
1624 uint32_t addr;
1625 uint32_t end_addr;
1626 uchar_t *sector;
1627 int status = 0;
1628 int i;
1629
1630 TAVOR_TNF_ENTER(tavor_flash_write_sector);
1631
1632 sector = (uchar_t *)&state->ts_fw_sector[0];
1633
1634 /*
1635 * Calculate the start and end address of the sector, based on the
1636 * sector number passed in.
1637 */
1638 addr = sector_num << state->ts_fw_log_sector_sz;
1639 end_addr = addr + (1 << state->ts_fw_log_sector_sz);
1640
1641 /* Set the flash bank correctly for the given address */
1642 tavor_flash_bank(state, addr);
1643
1644 /* Erase the sector before writing */
1645 tavor_flash_reset(state);
1646 status = tavor_flash_erase_sector(state, sector_num);
1647 if (status != 0) {
1648 TAVOR_TNF_EXIT(tavor_flash_write_sector);
1649 return (status);
1650 }
1651
1652 /* Write the entire sector, one byte at a time */
1653 for (i = 0; addr < end_addr; i++, addr++) {
1654 status = tavor_flash_write_byte(state, addr, sector[i]);
1655 if (status != 0) {
1656 break;
1657 }
1658 }
1659
1660 tavor_flash_reset(state);
1661 TAVOR_TNF_EXIT(tavor_flash_write_sector);
1662 return (status);
1663 }
1664
1665 /*
1666 * tavor_flash_write_byte()
1667 */
1668 static int
1669 tavor_flash_write_byte(tavor_state_t *state, uint32_t addr, uchar_t data)
1670 {
1671 uint32_t stat;
1672 int status = 0;
1673 int i;
1674
1675 TAVOR_TNF_ENTER(tavor_flash_write_byte);
1676
1677 switch (state->ts_fw_cmdset) {
1678 case TAVOR_FLASH_AMD_CMDSET:
1679 /* Issue Flash Byte program command */
1680 tavor_flash_write(state, addr, 0xAA);
1681 tavor_flash_write(state, addr, 0x55);
1682 tavor_flash_write(state, addr, 0xA0);
1683 tavor_flash_write(state, addr, data);
1684
1685 /*
1686 * Wait for Write Byte to Complete:
1687 * 1) Wait 1usec
1688 * 2) Read status of the write operation
1689 * 3) Determine if we have timed out the write operation
1690 * 4) Compare correct data value to the status value that
1691 * was read from the same address.
1692 */
1693 i = 0;
1694 do {
1695 drv_usecwait(1);
1696 stat = tavor_flash_read(state, addr & ~3);
1697
1698 if (i == tavor_hw_flash_timeout_write) {
1699 cmn_err(CE_WARN,
1700 "tavor_flash_write_byte: ACS write "
1701 "timeout: addr: 0x%x, data: 0x%x\n",
1702 addr, data);
1703 status = EIO;
1704 break;
1705 }
1706
1707 i++;
1708 } while (data != ((stat >> ((3 - (addr & 3)) << 3)) & 0xFF));
1709 break;
1710
1711 case TAVOR_FLASH_INTEL_CMDSET:
1712 /* Issue Flash Byte program command */
1713 tavor_flash_write(state, addr, TAVOR_HW_FLASH_ICS_WRITE);
1714 tavor_flash_write(state, addr, data);
1715
1716 /* wait for completion */
1717 i = 0;
1718 do {
1719 drv_usecwait(1);
1720 stat = tavor_flash_read(state, addr & ~3);
1721
1722 if (i == tavor_hw_flash_timeout_write) {
1723 cmn_err(CE_WARN,
1724 "tavor_flash_write_byte: ICS write "
1725 "timeout: addr: %x, data: %x\n",
1726 addr, data);
1727 status = EIO;
1728 break;
1729 }
1730
1731 i++;
1732 } while ((stat & TAVOR_HW_FLASH_ICS_READY) == 0);
1733
1734 if (stat & TAVOR_HW_FLASH_ICS_ERROR) {
1735 cmn_err(CE_WARN,
1736 "tavor_flash_write_byte: ICS write cmd error: "
1737 "addr: %x, data: %x\n",
1738 addr, data);
1739 status = EIO;
1740 }
1741 break;
1742
1743 default:
1744 cmn_err(CE_WARN,
1745 "tavor_flash_write_byte: unknown cmd set: 0x%x\n",
1746 state->ts_fw_cmdset);
1747 status = EIO;
1748 break;
1749 }
1750
1751 TAVOR_TNF_EXIT(tavor_flash_write_byte);
1752 return (status);
1753 }
1754
1755 /*
1756 * tavor_flash_erase_sector()
1757 */
1758 static int
1759 tavor_flash_erase_sector(tavor_state_t *state, uint32_t sector_num)
1760 {
1761 uint32_t addr;
1762 uint32_t stat;
1763 int status = 0;
1764 int i;
1765
1766 TAVOR_TNF_ENTER(tavor_flash_erase_sector);
1767
1768 /* Get address from sector num */
1769 addr = sector_num << state->ts_fw_log_sector_sz;
1770
1771 switch (state->ts_fw_cmdset) {
1772 case TAVOR_FLASH_AMD_CMDSET:
1773 /* Issue Flash Sector Erase Command */
1774 tavor_flash_write(state, addr, 0xAA);
1775 tavor_flash_write(state, addr, 0x55);
1776 tavor_flash_write(state, addr, 0x80);
1777 tavor_flash_write(state, addr, 0xAA);
1778 tavor_flash_write(state, addr, 0x55);
1779 tavor_flash_write(state, addr, 0x30);
1780
1781 /*
1782 * Wait for Sector Erase to Complete
1783 * 1) Wait 1usec
1784 * 2) read the status at the base addr of the sector
1785 * 3) Determine if we have timed out
1786 * 4) Compare status of address with the value of a fully
1787 * erased quadlet. If these are equal, the sector
1788 * has been erased.
1789 */
1790 i = 0;
1791 do {
1792 /* wait 1usec */
1793 drv_usecwait(1);
1794 stat = tavor_flash_read(state, addr);
1795
1796 if (i == tavor_hw_flash_timeout_erase) {
1797 cmn_err(CE_WARN,
1798 "tavor_flash_erase_sector: "
1799 "ACS erase timeout\n");
1800 status = EIO;
1801 break;
1802 }
1803
1804 i++;
1805 } while (stat != 0xFFFFFFFF);
1806 break;
1807
1808 case TAVOR_FLASH_INTEL_CMDSET:
1809 /* Issue Erase Command */
1810 tavor_flash_write(state, addr, TAVOR_HW_FLASH_ICS_ERASE);
1811 tavor_flash_write(state, addr, TAVOR_HW_FLASH_ICS_CONFIRM);
1812
1813 /* wait for completion */
1814 i = 0;
1815 do {
1816 drv_usecwait(1);
1817 stat = tavor_flash_read(state, addr & ~3);
1818
1819 if (i == tavor_hw_flash_timeout_erase) {
1820 cmn_err(CE_WARN,
1821 "tavor_flash_erase_sector: "
1822 "ICS erase timeout\n");
1823 status = EIO;
1824 break;
1825 }
1826
1827 i++;
1828 } while ((stat & TAVOR_HW_FLASH_ICS_READY) == 0);
1829
1830 if (stat & TAVOR_HW_FLASH_ICS_ERROR) {
1831 cmn_err(CE_WARN,
1832 "tavor_flash_erase_sector: "
1833 "ICS erase cmd error\n");
1834 status = EIO;
1835 }
1836 break;
1837
1838 default:
1839 cmn_err(CE_WARN,
1840 "tavor_flash_erase_sector: unknown cmd set: 0x%x\n",
1841 state->ts_fw_cmdset);
1842 status = EIO;
1843 break;
1844 }
1845
1846 tavor_flash_reset(state);
1847
1848 TAVOR_TNF_EXIT(tavor_flash_erase_sector);
1849 return (status);
1850 }
1851
1852 /*
1853 * tavor_flash_erase_chip()
1854 */
1855 static int
1856 tavor_flash_erase_chip(tavor_state_t *state)
1857 {
1858 uint_t size;
1859 uint32_t stat;
1860 int status = 0;
1861 int num_sect;
1862 int i;
1863
1864 TAVOR_TNF_ENTER(tavor_flash_erase_chip);
1865
1866 switch (state->ts_fw_cmdset) {
1867 case TAVOR_FLASH_AMD_CMDSET:
1868 /* Issue Flash Chip Erase Command */
1869 tavor_flash_write(state, 0, 0xAA);
1870 tavor_flash_write(state, 0, 0x55);
1871 tavor_flash_write(state, 0, 0x80);
1872 tavor_flash_write(state, 0, 0xAA);
1873 tavor_flash_write(state, 0, 0x55);
1874 tavor_flash_write(state, 0, 0x10);
1875
1876 /*
1877 * Wait for Chip Erase to Complete
1878 * 1) Wait 1usec
1879 * 2) read the status at the base addr of the sector
1880 * 3) Determine if we have timed out
1881 * 4) Compare status of address with the value of a
1882 * fully erased quadlet. If these are equal, the
1883 * chip has been erased.
1884 */
1885 i = 0;
1886 do {
1887 /* wait 1usec */
1888 drv_usecwait(1);
1889 stat = tavor_flash_read(state, 0);
1890
1891 if (i == tavor_hw_flash_timeout_erase) {
1892 cmn_err(CE_WARN,
1893 "tavor_flash_erase_chip: erase timeout\n");
1894 status = EIO;
1895 break;
1896 }
1897
1898 i++;
1899 } while (stat != 0xFFFFFFFF);
1900 break;
1901
1902 case TAVOR_FLASH_INTEL_CMDSET:
1903 /*
1904 * The Intel chip doesn't have a chip erase command, so erase
1905 * all blocks one at a time.
1906 */
1907 size = (0x1 << state->ts_fw_log_sector_sz);
1908 num_sect = state->ts_fw_device_sz / size;
1909
1910 for (i = 0; i < num_sect; i++) {
1911 status = tavor_flash_erase_sector(state, i);
1912 if (status != 0) {
1913 cmn_err(CE_WARN,
1914 "tavor_flash_erase_chip: "
1915 "ICS sector %d erase error\n", i);
1916 status = EIO;
1917 break;
1918 }
1919 }
1920 break;
1921
1922 default:
1923 cmn_err(CE_WARN, "tavor_flash_erase_chip: "
1924 "unknown cmd set: 0x%x\n", state->ts_fw_cmdset);
1925 status = EIO;
1926 break;
1927 }
1928
1929 TAVOR_TNF_EXIT(tavor_flash_erase_chip);
1930 return (status);
1931 }
1932
1933 /*
1934 * tavor_flash_bank()
1935 */
1936 static void
1937 tavor_flash_bank(tavor_state_t *state, uint32_t addr)
1938 {
1939 ddi_acc_handle_t hdl;
1940 uint32_t bank;
1941
1942 TAVOR_TNF_ENTER(tavor_flash_bank);
1943
1944 /* Set handle */
1945 hdl = state->ts_pci_cfghdl;
1946
1947 /* Determine the bank setting from the address */
1948 bank = addr & TAVOR_HW_FLASH_BANK_MASK;
1949
1950 /*
1951 * If the bank is different from the currently set bank, we need to
1952 * change it. Also, if an 'addr' of 0 is given, this allows the
1953 * capability to force the flash bank to 0. This is useful at init
1954 * time to initially set the bank value
1955 */
1956 if (state->ts_fw_flashbank != bank || addr == 0) {
1957 /* Set bank using the GPIO settings */
1958 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_DATACLEAR, 0x70);
1959 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_DATASET,
1960 (bank >> 15) & 0x70);
1961
1962 /* Save the bank state */
1963 state->ts_fw_flashbank = bank;
1964 }
1965
1966 TAVOR_TNF_EXIT(tavor_flash_bank);
1967 }
1968
1969 /*
1970 * tavor_flash_read()
1971 */
1972 static uint32_t
1973 tavor_flash_read(tavor_state_t *state, uint32_t addr)
1974 {
1975 ddi_acc_handle_t hdl;
1976 uint32_t data;
1977 int timeout;
1978
1979 TAVOR_TNF_ENTER(tavor_flash_read);
1980
1981 /* Set handle */
1982 hdl = state->ts_pci_cfghdl;
1983
1984 /*
1985 * The Read operation does the following:
1986 * 1) Write the masked address to the TAVOR_FLASH_ADDR register.
1987 * Only the least significant 19 bits are valid.
1988 * 2) Read back the register until the command has completed.
1989 * 3) Read the data retrieved from the address at the TAVOR_FLASH_DATA
1990 * register.
1991 */
1992 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_ADDR,
1993 (addr & TAVOR_HW_FLASH_ADDR_MASK) | (1 << 29));
1994
1995 timeout = 0;
1996 do {
1997 data = tavor_flash_read_cfg(hdl, TAVOR_HW_FLASH_ADDR);
1998 timeout++;
1999 } while ((data & TAVOR_HW_FLASH_CMD_MASK) &&
2000 (timeout < tavor_hw_flash_timeout_config));
2001
2002 if (timeout == tavor_hw_flash_timeout_config) {
2003 cmn_err(CE_WARN, "tavor_flash_read: config command timeout.\n");
2004 }
2005
2006 data = tavor_flash_read_cfg(hdl, TAVOR_HW_FLASH_DATA);
2007
2008 TAVOR_TNF_EXIT(tavor_flash_read);
2009 return (data);
2010 }
2011
2012 /*
2013 * tavor_flash_write()
2014 */
2015 static void
2016 tavor_flash_write(tavor_state_t *state, uint32_t addr, uchar_t data)
2017 {
2018 ddi_acc_handle_t hdl;
2019 int cmd;
2020 int timeout;
2021
2022 TAVOR_TNF_ENTER(tavor_flash_write);
2023
2024 /* Set handle */
2025 hdl = state->ts_pci_cfghdl;
2026
2027 /*
2028 * The Write operation does the following:
2029 * 1) Write the data to be written to the TAVOR_FLASH_DATA offset.
2030 * 2) Write the address to write the data to to the TAVOR_FLASH_ADDR
2031 * offset.
2032 * 3) Wait until the write completes.
2033 */
2034 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_DATA, data << 24);
2035 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_ADDR,
2036 (addr & 0x7FFFF) | (2 << 29));
2037
2038 timeout = 0;
2039 do {
2040 cmd = tavor_flash_read_cfg(hdl, TAVOR_HW_FLASH_ADDR);
2041 timeout++;
2042 } while ((cmd & TAVOR_HW_FLASH_CMD_MASK) &&
2043 (timeout < tavor_hw_flash_timeout_config));
2044
2045 if (timeout == tavor_hw_flash_timeout_config) {
2046 cmn_err(CE_WARN, "tavor_flash_write: config cmd timeout.\n");
2047 }
2048
2049 TAVOR_TNF_EXIT(tavor_flash_write);
2050 }
2051
2052 /*
2053 * tavor_flash_init()
2054 */
2055 static void
2056 tavor_flash_init(tavor_state_t *state)
2057 {
2058 uint32_t word;
2059 ddi_acc_handle_t hdl;
2060 int sema_cnt;
2061 int gpio;
2062
2063 TAVOR_TNF_ENTER(tavor_flash_init);
2064
2065 /* Set handle */
2066 hdl = state->ts_pci_cfghdl;
2067
2068 /* Init the flash */
2069
2070 /*
2071 * Grab the GPIO semaphore. This allows us exclusive access to the
2072 * GPIO settings on the Tavor for the duration of the flash burning
2073 * procedure.
2074 */
2075 sema_cnt = 0;
2076 do {
2077 word = tavor_flash_read_cfg(hdl, TAVOR_HW_FLASH_GPIO_SEMA);
2078 if (word == 0) {
2079 break;
2080 }
2081
2082 sema_cnt++;
2083 drv_usecwait(1);
2084 } while (sema_cnt < tavor_hw_flash_timeout_gpio_sema);
2085
2086 /*
2087 * Determine if we timed out trying to grab the GPIO semaphore
2088 */
2089 if (sema_cnt == tavor_hw_flash_timeout_gpio_sema) {
2090 cmn_err(CE_WARN, "tavor_flash_init: GPIO SEMA timeout\n");
2091 }
2092
2093 /* Save away original GPIO Values */
2094 state->ts_fw_gpio[0] = tavor_flash_read_cfg(hdl,
2095 TAVOR_HW_FLASH_GPIO_DIR);
2096 state->ts_fw_gpio[1] = tavor_flash_read_cfg(hdl,
2097 TAVOR_HW_FLASH_GPIO_POL);
2098 state->ts_fw_gpio[2] = tavor_flash_read_cfg(hdl,
2099 TAVOR_HW_FLASH_GPIO_MOD);
2100 state->ts_fw_gpio[3] = tavor_flash_read_cfg(hdl,
2101 TAVOR_HW_FLASH_GPIO_DAT);
2102
2103 /* Set New GPIO Values */
2104 gpio = state->ts_fw_gpio[0] | 0x70;
2105 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_DIR, gpio);
2106
2107 gpio = state->ts_fw_gpio[1] & ~0x70;
2108 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_POL, gpio);
2109
2110 gpio = state->ts_fw_gpio[2] & ~0x70;
2111 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_MOD, gpio);
2112
2113 /* Set CPUMODE to enable tavor to access the flash device */
2114 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_CPUMODE,
2115 1 << TAVOR_HW_FLASH_CPU_SHIFT);
2116
2117 /* Initialize to bank 0 */
2118 tavor_flash_bank(state, 0);
2119
2120 TAVOR_TNF_EXIT(tavor_flash_init);
2121 }
2122
2123 /*
2124 * tavor_flash_cfi_init
2125 * Implements access to the CFI (Common Flash Interface) data
2126 */
2127 static void
2128 tavor_flash_cfi_init(tavor_state_t *state, uint32_t *cfi_info, int *intel_xcmd)
2129 {
2130 uint32_t data;
2131 uint32_t sector_sz_bytes;
2132 uint32_t bit_count;
2133 uint8_t cfi_ch_info[TAVOR_CFI_INFO_SIZE];
2134 uint32_t cfi_dw_info[TAVOR_CFI_INFO_QSIZE];
2135 int i;
2136
2137 TAVOR_TNF_ENTER(tavor_flash_cfi_init);
2138
2139 /*
2140 * Determine if the user command supports the Intel Extended
2141 * Command Set. The query string is contained in the fourth
2142 * quad word.
2143 */
2144 tavor_flash_cfi_byte(cfi_ch_info, cfi_info[0x04], 0x10);
2145 if (cfi_ch_info[0x10] == 'M' &&
2146 cfi_ch_info[0x11] == 'X' &&
2147 cfi_ch_info[0x12] == '2') {
2148 *intel_xcmd = 1; /* support is there */
2149 }
2150
2151 /* CFI QUERY */
2152 tavor_flash_write(state, 0x55, TAVOR_FLASH_CFI_INIT);
2153
2154 /* Read in CFI data */
2155 for (i = 0; i < TAVOR_CFI_INFO_SIZE; i += 4) {
2156 data = tavor_flash_read(state, i);
2157 cfi_dw_info[i >> 2] = data;
2158 tavor_flash_cfi_byte(cfi_ch_info, data, i);
2159 }
2160
2161 /* Determine chip set */
2162 state->ts_fw_cmdset = TAVOR_FLASH_UNKNOWN_CMDSET;
2163 if (cfi_ch_info[0x20] == 'Q' &&
2164 cfi_ch_info[0x22] == 'R' &&
2165 cfi_ch_info[0x24] == 'Y') {
2166 /*
2167 * Mode: x16 working in x8 mode (Intel).
2168 * Pack data - skip spacing bytes.
2169 */
2170 for (i = 0; i < TAVOR_CFI_INFO_SIZE; i += 2) {
2171 cfi_ch_info[i/2] = cfi_ch_info[i];
2172 }
2173 }
2174 state->ts_fw_cmdset = cfi_ch_info[0x13];
2175 if (state->ts_fw_cmdset != TAVOR_FLASH_INTEL_CMDSET &&
2176 state->ts_fw_cmdset != TAVOR_FLASH_AMD_CMDSET) {
2177 cmn_err(CE_WARN,
2178 "tavor_flash_cfi_init: UNKNOWN chip cmd set\n");
2179 state->ts_fw_cmdset = TAVOR_FLASH_UNKNOWN_CMDSET;
2180 goto out;
2181 }
2182
2183 /* Determine total bytes in one sector size */
2184 sector_sz_bytes = ((cfi_ch_info[0x30] << 8) | cfi_ch_info[0x2F]) << 8;
2185
2186 /* Calculate equivalent of log2 (n) */
2187 for (bit_count = 0; sector_sz_bytes > 1; bit_count++) {
2188 sector_sz_bytes >>= 1;
2189 }
2190
2191 /* Set sector size */
2192 state->ts_fw_log_sector_sz = bit_count;
2193
2194 /* Set flash size */
2195 state->ts_fw_device_sz = 0x1 << cfi_ch_info[0x27];
2196
2197 /* Reset to turn off CFI mode */
2198 tavor_flash_reset(state);
2199
2200 /*
2201 * Pass CFI data back to user command.
2202 */
2203 for (i = 0; i < TAVOR_FLASH_CFI_SIZE_QUADLET; i++) {
2204 tavor_flash_cfi_dword(&cfi_info[i], cfi_ch_info, i << 2);
2205 }
2206
2207 if (*intel_xcmd == 1) {
2208 /*
2209 * Inform the user cmd that this driver does support the
2210 * Intel Extended Command Set.
2211 */
2212 cfi_ch_info[0x10] = 'M';
2213 cfi_ch_info[0x11] = 'X';
2214 cfi_ch_info[0x12] = '2';
2215 } else {
2216 cfi_ch_info[0x10] = 'Q';
2217 cfi_ch_info[0x11] = 'R';
2218 cfi_ch_info[0x12] = 'Y';
2219 }
2220 cfi_ch_info[0x13] = state->ts_fw_cmdset;
2221 tavor_flash_cfi_dword(&cfi_info[0x4], cfi_ch_info, 0x10);
2222 out:
2223 TAVOR_TNF_EXIT(tavor_flash_cfi_init);
2224 }
2225
2226 /*
2227 * tavor_flash_fini()
2228 */
2229 static void
2230 tavor_flash_fini(tavor_state_t *state)
2231 {
2232 ddi_acc_handle_t hdl;
2233
2234 TAVOR_TNF_ENTER(tavor_flash_fini);
2235
2236 /* Set handle */
2237 hdl = state->ts_pci_cfghdl;
2238
2239 /* Restore original GPIO Values */
2240 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_DIR,
2241 state->ts_fw_gpio[0]);
2242 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_POL,
2243 state->ts_fw_gpio[1]);
2244 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_MOD,
2245 state->ts_fw_gpio[2]);
2246 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_DAT,
2247 state->ts_fw_gpio[3]);
2248
2249 /* Give up semaphore */
2250 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_SEMA, 0);
2251
2252 TAVOR_TNF_EXIT(tavor_flash_fini);
2253 }
2254
2255 /*
2256 * tavor_flash_read_cfg
2257 */
2258 static uint32_t
2259 tavor_flash_read_cfg(ddi_acc_handle_t pci_config_hdl, uint32_t addr)
2260 {
2261 uint32_t read;
2262
2263 TAVOR_TNF_ENTER(tavor_flash_read_cfg);
2264
2265 /*
2266 * Perform flash read operation:
2267 * 1) Place addr to read from on the TAVOR_HW_FLASH_CFG_ADDR register
2268 * 2) Read data at that addr from the TAVOR_HW_FLASH_CFG_DATA register
2269 */
2270 pci_config_put32(pci_config_hdl, TAVOR_HW_FLASH_CFG_ADDR, addr);
2271 read = pci_config_get32(pci_config_hdl, TAVOR_HW_FLASH_CFG_DATA);
2272
2273 TAVOR_TNF_EXIT(tavor_flash_read_cfg);
2274
2275 return (read);
2276 }
2277
2278 /*
2279 * tavor_flash_write_cfg
2280 */
2281 static void
2282 tavor_flash_write_cfg(ddi_acc_handle_t pci_config_hdl, uint32_t addr,
2283 uint32_t data)
2284 {
2285 TAVOR_TNF_ENTER(tavor_flash_write_cfg);
2286
2287 /*
2288 * Perform flash write operation:
2289 * 1) Place addr to write to on the TAVOR_HW_FLASH_CFG_ADDR register
2290 * 2) Place data to write on to the TAVOR_HW_FLASH_CFG_DATA register
2291 */
2292 pci_config_put32(pci_config_hdl, TAVOR_HW_FLASH_CFG_ADDR, addr);
2293 pci_config_put32(pci_config_hdl, TAVOR_HW_FLASH_CFG_DATA, data);
2294
2295 TAVOR_TNF_EXIT(tavor_flash_write_cfg);
2296 }
2297
2298 /*
2299 * Support routines to convert Common Flash Interface (CFI) data
2300 * from a 32 bit word to a char array, and from a char array to
2301 * a 32 bit word.
2302 */
2303 static void
2304 tavor_flash_cfi_byte(uint8_t *ch, uint32_t dword, int i)
2305 {
2306 ch[i] = (uint8_t)((dword & 0xFF000000) >> 24);
2307 ch[i+1] = (uint8_t)((dword & 0x00FF0000) >> 16);
2308 ch[i+2] = (uint8_t)((dword & 0x0000FF00) >> 8);
2309 ch[i+3] = (uint8_t)((dword & 0x000000FF));
2310 }
2311
2312 static void
2313 tavor_flash_cfi_dword(uint32_t *dword, uint8_t *ch, int i)
2314 {
2315 *dword = (uint32_t)
2316 ((uint32_t)ch[i] << 24 |
2317 (uint32_t)ch[i+1] << 16 |
2318 (uint32_t)ch[i+2] << 8 |
2319 (uint32_t)ch[i+3]);
2320 }
2321
2322 /*
2323 * tavor_loopback_free_qps
2324 */
2325 static void
2326 tavor_loopback_free_qps(tavor_loopback_state_t *lstate)
2327 {
2328 int i;
2329
2330 if (lstate->tls_tx.tlc_qp_hdl != NULL) {
2331 (void) tavor_qp_free(lstate->tls_state,
2332 &lstate->tls_tx.tlc_qp_hdl, IBC_FREE_QP_AND_QPN, NULL,
2333 TAVOR_NOSLEEP);
2334 }
2335 if (lstate->tls_rx.tlc_qp_hdl != NULL) {
2336 (void) tavor_qp_free(lstate->tls_state,
2337 &lstate->tls_rx.tlc_qp_hdl, IBC_FREE_QP_AND_QPN, NULL,
2338 TAVOR_NOSLEEP);
2339 }
2340 lstate->tls_tx.tlc_qp_hdl = NULL;
2341 lstate->tls_rx.tlc_qp_hdl = NULL;
2342 for (i = 0; i < 2; i++) {
2343 if (lstate->tls_tx.tlc_cqhdl[i] != NULL) {
2344 (void) tavor_cq_free(lstate->tls_state,
2345 &lstate->tls_tx.tlc_cqhdl[i], TAVOR_NOSLEEP);
2346 }
2347 if (lstate->tls_rx.tlc_cqhdl[i] != NULL) {
2348 (void) tavor_cq_free(lstate->tls_state,
2349 &lstate->tls_rx.tlc_cqhdl[i], TAVOR_NOSLEEP);
2350 }
2351 lstate->tls_tx.tlc_cqhdl[i] = NULL;
2352 lstate->tls_rx.tlc_cqhdl[i] = NULL;
2353 }
2354 }
2355
2356 /*
2357 * tavor_loopback_free_state
2358 */
2359 static void
2360 tavor_loopback_free_state(tavor_loopback_state_t *lstate)
2361 {
2362 tavor_loopback_free_qps(lstate);
2363 if (lstate->tls_tx.tlc_mrhdl != NULL) {
2364 (void) tavor_mr_deregister(lstate->tls_state,
2365 &lstate->tls_tx.tlc_mrhdl, TAVOR_MR_DEREG_ALL,
2366 TAVOR_NOSLEEP);
2367 }
2368 if (lstate->tls_rx.tlc_mrhdl != NULL) {
2369 (void) tavor_mr_deregister(lstate->tls_state,
2370 &lstate->tls_rx.tlc_mrhdl, TAVOR_MR_DEREG_ALL,
2371 TAVOR_NOSLEEP);
2372 }
2373 if (lstate->tls_pd_hdl != NULL) {
2374 (void) tavor_pd_free(lstate->tls_state, &lstate->tls_pd_hdl);
2375 }
2376 if (lstate->tls_tx.tlc_buf != NULL) {
2377 kmem_free(lstate->tls_tx.tlc_buf, lstate->tls_tx.tlc_buf_sz);
2378 }
2379 if (lstate->tls_rx.tlc_buf != NULL) {
2380 kmem_free(lstate->tls_rx.tlc_buf, lstate->tls_rx.tlc_buf_sz);
2381 }
2382 bzero(lstate, sizeof (tavor_loopback_state_t));
2383 }
2384
2385 /*
2386 * tavor_loopback_init
2387 */
2388 static int
2389 tavor_loopback_init(tavor_state_t *state, tavor_loopback_state_t *lstate)
2390 {
2391 lstate->tls_hca_hdl = (ibc_hca_hdl_t)state;
2392 lstate->tls_status = tavor_pd_alloc(lstate->tls_state,
2393 &lstate->tls_pd_hdl, TAVOR_NOSLEEP);
2394 if (lstate->tls_status != IBT_SUCCESS) {
2395 lstate->tls_err = TAVOR_LOOPBACK_PROT_DOMAIN_ALLOC_FAIL;
2396 TNF_PROBE_0(tavor_ioctl_loopback_alloc_pd_fail,
2397 TAVOR_TNF_ERROR, "");
2398 return (EFAULT);
2399 }
2400
2401 return (0);
2402 }
2403
2404 /*
2405 * tavor_loopback_init_qp_info
2406 */
2407 static void
2408 tavor_loopback_init_qp_info(tavor_loopback_state_t *lstate,
2409 tavor_loopback_comm_t *comm)
2410 {
2411 bzero(&comm->tlc_cq_attr, sizeof (ibt_cq_attr_t));
2412 bzero(&comm->tlc_qp_attr, sizeof (ibt_qp_alloc_attr_t));
2413 bzero(&comm->tlc_qp_info, sizeof (ibt_qp_info_t));
2414
2415 comm->tlc_wrid = 1;
2416 comm->tlc_cq_attr.cq_size = 128;
2417 comm->tlc_qp_attr.qp_sizes.cs_sq_sgl = 3;
2418 comm->tlc_qp_attr.qp_sizes.cs_rq_sgl = 3;
2419 comm->tlc_qp_attr.qp_sizes.cs_sq = 16;
2420 comm->tlc_qp_attr.qp_sizes.cs_rq = 16;
2421 comm->tlc_qp_attr.qp_flags = IBT_WR_SIGNALED;
2422
2423 comm->tlc_qp_info.qp_state = IBT_STATE_RESET;
2424 comm->tlc_qp_info.qp_trans = IBT_RC_SRV;
2425 comm->tlc_qp_info.qp_flags = IBT_CEP_RDMA_RD | IBT_CEP_RDMA_WR;
2426 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_hca_port_num =
2427 lstate->tls_port;
2428 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_pkey_ix =
2429 lstate->tls_pkey_ix;
2430 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_timeout =
2431 lstate->tls_timeout;
2432 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_adds_vect.av_srvl = 0;
2433 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_adds_vect.av_srate =
2434 IBT_SRATE_4X;
2435 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_adds_vect.av_send_grh = 0;
2436 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_adds_vect.av_dlid =
2437 lstate->tls_lid;
2438 comm->tlc_qp_info.qp_transport.rc.rc_retry_cnt = lstate->tls_retry;
2439 comm->tlc_qp_info.qp_transport.rc.rc_sq_psn = 0;
2440 comm->tlc_qp_info.qp_transport.rc.rc_rq_psn = 0;
2441 comm->tlc_qp_info.qp_transport.rc.rc_rdma_ra_in = 4;
2442 comm->tlc_qp_info.qp_transport.rc.rc_rdma_ra_out = 4;
2443 comm->tlc_qp_info.qp_transport.rc.rc_dst_qpn = 0;
2444 comm->tlc_qp_info.qp_transport.rc.rc_min_rnr_nak = IBT_RNR_NAK_655ms;
2445 comm->tlc_qp_info.qp_transport.rc.rc_path_mtu = IB_MTU_1K;
2446 }
2447
2448 /*
2449 * tavor_loopback_alloc_mem
2450 */
2451 static int
2452 tavor_loopback_alloc_mem(tavor_loopback_state_t *lstate,
2453 tavor_loopback_comm_t *comm, int sz)
2454 {
2455 /* Allocate buffer of specified size */
2456 comm->tlc_buf_sz = sz;
2457 comm->tlc_buf = kmem_zalloc(sz, KM_NOSLEEP);
2458 if (comm->tlc_buf == NULL) {
2459 return (EFAULT);
2460 }
2461
2462 /* Register the buffer as a memory region */
2463 comm->tlc_memattr.mr_vaddr = (uint64_t)(uintptr_t)comm->tlc_buf;
2464 comm->tlc_memattr.mr_len = (ib_msglen_t)sz;
2465 comm->tlc_memattr.mr_as = NULL;
2466 comm->tlc_memattr.mr_flags = IBT_MR_NOSLEEP |
2467 IBT_MR_ENABLE_REMOTE_WRITE | IBT_MR_ENABLE_LOCAL_WRITE;
2468
2469 comm->tlc_status = tavor_mr_register(lstate->tls_state,
2470 lstate->tls_pd_hdl, &comm->tlc_memattr, &comm->tlc_mrhdl, NULL);
2471
2472 comm->tlc_mrdesc.md_vaddr = comm->tlc_mrhdl->mr_bindinfo.bi_addr;
2473 comm->tlc_mrdesc.md_lkey = comm->tlc_mrhdl->mr_lkey;
2474 comm->tlc_mrdesc.md_rkey = comm->tlc_mrhdl->mr_rkey;
2475 if (comm->tlc_status != IBT_SUCCESS) {
2476 return (EFAULT);
2477 }
2478 return (0);
2479 }
2480
2481 /*
2482 * tavor_loopback_alloc_qps
2483 */
2484 static int
2485 tavor_loopback_alloc_qps(tavor_loopback_state_t *lstate,
2486 tavor_loopback_comm_t *comm)
2487 {
2488 uint32_t i, real_size;
2489 tavor_qp_info_t qpinfo;
2490
2491 /* Allocate send and recv CQs */
2492 for (i = 0; i < 2; i++) {
2493 bzero(&comm->tlc_cq_attr, sizeof (ibt_cq_attr_t));
2494 comm->tlc_cq_attr.cq_size = 128;
2495 comm->tlc_status = tavor_cq_alloc(lstate->tls_state,
2496 (ibt_cq_hdl_t)NULL, &comm->tlc_cq_attr, &real_size,
2497 &comm->tlc_cqhdl[i], TAVOR_NOSLEEP);
2498 if (comm->tlc_status != IBT_SUCCESS) {
2499 lstate->tls_err += i;
2500 return (EFAULT);
2501 }
2502 }
2503
2504 /* Allocate the QP */
2505 tavor_loopback_init_qp_info(lstate, comm);
2506 comm->tlc_qp_attr.qp_pd_hdl = (ibt_pd_hdl_t)lstate->tls_pd_hdl;
2507 comm->tlc_qp_attr.qp_scq_hdl = (ibt_cq_hdl_t)comm->tlc_cqhdl[0];
2508 comm->tlc_qp_attr.qp_rcq_hdl = (ibt_cq_hdl_t)comm->tlc_cqhdl[1];
2509 comm->tlc_qp_attr.qp_ibc_scq_hdl = (ibt_opaque1_t)comm->tlc_cqhdl[0];
2510 comm->tlc_qp_attr.qp_ibc_rcq_hdl = (ibt_opaque1_t)comm->tlc_cqhdl[1];
2511 qpinfo.qpi_attrp = &comm->tlc_qp_attr;
2512 qpinfo.qpi_type = IBT_RC_RQP;
2513 qpinfo.qpi_ibt_qphdl = NULL;
2514 qpinfo.qpi_queueszp = &comm->tlc_chan_sizes;
2515 qpinfo.qpi_qpn = &comm->tlc_qp_num;
2516 comm->tlc_status = tavor_qp_alloc(lstate->tls_state, &qpinfo,
2517 TAVOR_NOSLEEP, NULL);
2518 if (comm->tlc_status == DDI_SUCCESS) {
2519 comm->tlc_qp_hdl = qpinfo.qpi_qphdl;
2520 }
2521
2522 if (comm->tlc_status != IBT_SUCCESS) {
2523 lstate->tls_err += 2;
2524 return (EFAULT);
2525 }
2526 return (0);
2527 }
2528
2529 /*
2530 * tavor_loopback_modify_qp
2531 */
2532 static int
2533 tavor_loopback_modify_qp(tavor_loopback_state_t *lstate,
2534 tavor_loopback_comm_t *comm, uint_t qp_num)
2535 {
2536 /* Modify QP to INIT */
2537 tavor_loopback_init_qp_info(lstate, comm);
2538 comm->tlc_qp_info.qp_state = IBT_STATE_INIT;
2539 comm->tlc_status = tavor_qp_modify(lstate->tls_state, comm->tlc_qp_hdl,
2540 IBT_CEP_SET_STATE, &comm->tlc_qp_info, &comm->tlc_queue_sizes);
2541 if (comm->tlc_status != IBT_SUCCESS) {
2542 return (EFAULT);
2543 }
2544
2545 /*
2546 * Modify QP to RTR (set destination LID and QP number to local
2547 * LID and QP number)
2548 */
2549 comm->tlc_qp_info.qp_state = IBT_STATE_RTR;
2550 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_adds_vect.av_dlid
2551 = lstate->tls_lid;
2552 comm->tlc_qp_info.qp_transport.rc.rc_dst_qpn = qp_num;
2553 comm->tlc_status = tavor_qp_modify(lstate->tls_state, comm->tlc_qp_hdl,
2554 IBT_CEP_SET_STATE, &comm->tlc_qp_info, &comm->tlc_queue_sizes);
2555 if (comm->tlc_status != IBT_SUCCESS) {
2556 lstate->tls_err += 1;
2557 return (EFAULT);
2558 }
2559
2560 /* Modify QP to RTS */
2561 comm->tlc_qp_info.qp_current_state = IBT_STATE_RTR;
2562 comm->tlc_qp_info.qp_state = IBT_STATE_RTS;
2563 comm->tlc_status = tavor_qp_modify(lstate->tls_state, comm->tlc_qp_hdl,
2564 IBT_CEP_SET_STATE, &comm->tlc_qp_info, &comm->tlc_queue_sizes);
2565 if (comm->tlc_status != IBT_SUCCESS) {
2566 lstate->tls_err += 2;
2567 return (EFAULT);
2568 }
2569 return (0);
2570 }
2571
2572 /*
2573 * tavor_loopback_copyout
2574 */
2575 static int
2576 tavor_loopback_copyout(tavor_loopback_ioctl_t *lb, intptr_t arg, int mode)
2577 {
2578 #ifdef _MULTI_DATAMODEL
2579 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
2580 tavor_loopback_ioctl32_t lb32;
2581
2582 lb32.tlb_revision = lb->tlb_revision;
2583 lb32.tlb_send_buf =
2584 (caddr32_t)(uintptr_t)lb->tlb_send_buf;
2585 lb32.tlb_fail_buf =
2586 (caddr32_t)(uintptr_t)lb->tlb_fail_buf;
2587 lb32.tlb_buf_sz = lb->tlb_buf_sz;
2588 lb32.tlb_num_iter = lb->tlb_num_iter;
2589 lb32.tlb_pass_done = lb->tlb_pass_done;
2590 lb32.tlb_timeout = lb->tlb_timeout;
2591 lb32.tlb_error_type = lb->tlb_error_type;
2592 lb32.tlb_port_num = lb->tlb_port_num;
2593 lb32.tlb_num_retry = lb->tlb_num_retry;
2594
2595 if (ddi_copyout(&lb32, (void *)arg,
2596 sizeof (tavor_loopback_ioctl32_t), mode) != 0) {
2597 TNF_PROBE_0(tavor_ioctl_loopback_copyout_fail,
2598 TAVOR_TNF_ERROR, "");
2599 return (EFAULT);
2600 }
2601 } else
2602 #endif /* _MULTI_DATAMODEL */
2603 if (ddi_copyout(lb, (void *)arg, sizeof (tavor_loopback_ioctl_t),
2604 mode) != 0) {
2605 TNF_PROBE_0(tavor_ioctl_loopback_copyout_fail,
2606 TAVOR_TNF_ERROR, "");
2607 return (EFAULT);
2608 }
2609 return (0);
2610 }
2611
2612 /*
2613 * tavor_loopback_post_send
2614 */
2615 static int
2616 tavor_loopback_post_send(tavor_loopback_state_t *lstate,
2617 tavor_loopback_comm_t *tx, tavor_loopback_comm_t *rx)
2618 {
2619 int ret;
2620
2621 bzero(&tx->tlc_sgl, sizeof (ibt_wr_ds_t));
2622 bzero(&tx->tlc_wr, sizeof (ibt_send_wr_t));
2623
2624 /* Initialize local address for TX buffer */
2625 tx->tlc_sgl.ds_va = tx->tlc_mrdesc.md_vaddr;
2626 tx->tlc_sgl.ds_key = tx->tlc_mrdesc.md_lkey;
2627 tx->tlc_sgl.ds_len = tx->tlc_buf_sz;
2628
2629 /* Initialize the remaining details of the work request */
2630 tx->tlc_wr.wr_id = tx->tlc_wrid++;
2631 tx->tlc_wr.wr_flags = IBT_WR_SEND_SIGNAL;
2632 tx->tlc_wr.wr_nds = 1;
2633 tx->tlc_wr.wr_sgl = &tx->tlc_sgl;
2634 tx->tlc_wr.wr_opcode = IBT_WRC_RDMAW;
2635 tx->tlc_wr.wr_trans = IBT_RC_SRV;
2636
2637 /* Initialize the remote address for RX buffer */
2638 tx->tlc_wr.wr.rc.rcwr.rdma.rdma_raddr = rx->tlc_mrdesc.md_vaddr;
2639 tx->tlc_wr.wr.rc.rcwr.rdma.rdma_rkey = rx->tlc_mrdesc.md_rkey;
2640 tx->tlc_complete = 0;
2641 ret = tavor_post_send(lstate->tls_state, tx->tlc_qp_hdl, &tx->tlc_wr,
2642 1, NULL);
2643 if (ret != IBT_SUCCESS) {
2644 return (EFAULT);
2645 }
2646 return (0);
2647 }
2648
2649 /*
2650 * tavor_loopback_poll_cq
2651 */
2652 static int
2653 tavor_loopback_poll_cq(tavor_loopback_state_t *lstate,
2654 tavor_loopback_comm_t *comm)
2655 {
2656 comm->tlc_wc.wc_status = 0;
2657 comm->tlc_num_polled = 0;
2658 comm->tlc_status = tavor_cq_poll(lstate->tls_state,
2659 comm->tlc_cqhdl[0], &comm->tlc_wc, 1, &comm->tlc_num_polled);
2660 if ((comm->tlc_status == IBT_SUCCESS) &&
2661 (comm->tlc_wc.wc_status != IBT_WC_SUCCESS)) {
2662 comm->tlc_status = ibc_get_ci_failure(0);
2663 }
2664 return (comm->tlc_status);
2665 }