1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * This file is part of the Chelsio T4 support code.
14 *
15 * Copyright (C) 2010-2013 Chelsio Communications. All rights reserved.
16 *
17 * This program is distributed in the hope that it will be useful, but WITHOUT
18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
19 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this
20 * release for licensing terms and conditions.
21 */
22
23 #include <sys/ddi.h>
24 #include <sys/sunddi.h>
25 #include <sys/sunndi.h>
26 #include <sys/modctl.h>
27 #include <sys/conf.h>
28 #include <sys/devops.h>
29 #include <sys/pci.h>
30 #include <sys/atomic.h>
31 #include <sys/types.h>
32 #include <sys/file.h>
33 #include <sys/errno.h>
34 #include <sys/open.h>
35 #include <sys/cred.h>
36 #include <sys/stat.h>
37 #include <sys/mkdev.h>
38 #include <sys/queue.h>
39 #include <sys/containerof.h>
40
41 #include "version.h"
42 #include "common/common.h"
43 #include "common/t4_msg.h"
44 #include "common/t4_regs.h"
45 #include "firmware/t4_fw.h"
46 #include "firmware/t4_cfg.h"
47 #include "firmware/t5_fw.h"
48 #include "firmware/t5_cfg.h"
49 #include "firmware/t6_fw.h"
50 #include "firmware/t6_cfg.h"
51 #include "t4_l2t.h"
52
53 static int t4_cb_open(dev_t *devp, int flag, int otyp, cred_t *credp);
54 static int t4_cb_close(dev_t dev, int flag, int otyp, cred_t *credp);
55 static int t4_cb_ioctl(dev_t dev, int cmd, intptr_t d, int mode, cred_t *credp,
56 int *rp);
57 struct cb_ops t4_cb_ops = {
58 .cb_open = t4_cb_open,
59 .cb_close = t4_cb_close,
60 .cb_strategy = nodev,
61 .cb_print = nodev,
62 .cb_dump = nodev,
63 .cb_read = nodev,
64 .cb_write = nodev,
65 .cb_ioctl = t4_cb_ioctl,
66 .cb_devmap = nodev,
67 .cb_mmap = nodev,
68 .cb_segmap = nodev,
69 .cb_chpoll = nochpoll,
70 .cb_prop_op = ddi_prop_op,
71 .cb_flag = D_MP,
72 .cb_rev = CB_REV,
73 .cb_aread = nodev,
74 .cb_awrite = nodev
75 };
76
77 static int t4_bus_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t op,
78 void *arg, void *result);
79 static int t4_bus_config(dev_info_t *dip, uint_t flags, ddi_bus_config_op_t op,
80 void *arg, dev_info_t **cdipp);
81 static int t4_bus_unconfig(dev_info_t *dip, uint_t flags,
82 ddi_bus_config_op_t op, void *arg);
83 struct bus_ops t4_bus_ops = {
84 .busops_rev = BUSO_REV,
85 .bus_ctl = t4_bus_ctl,
86 .bus_prop_op = ddi_bus_prop_op,
87 .bus_config = t4_bus_config,
88 .bus_unconfig = t4_bus_unconfig,
89 };
90
91 static int t4_devo_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg,
92 void **rp);
93 static int t4_devo_probe(dev_info_t *dip);
94 static int t4_devo_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
95 static int t4_devo_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
96 static int t4_devo_quiesce(dev_info_t *dip);
97 struct dev_ops t4_dev_ops = {
98 .devo_rev = DEVO_REV,
99 .devo_getinfo = t4_devo_getinfo,
100 .devo_identify = nulldev,
101 .devo_probe = t4_devo_probe,
102 .devo_attach = t4_devo_attach,
103 .devo_detach = t4_devo_detach,
104 .devo_reset = nodev,
105 .devo_cb_ops = &t4_cb_ops,
106 .devo_bus_ops = &t4_bus_ops,
107 .devo_quiesce = &t4_devo_quiesce,
108 };
109
110 static struct modldrv modldrv = {
111 .drv_modops = &mod_driverops,
112 .drv_linkinfo = "Chelsio T4 nexus " DRV_VERSION,
113 .drv_dev_ops = &t4_dev_ops
114 };
115
116 static struct modlinkage modlinkage = {
117 .ml_rev = MODREV_1,
118 .ml_linkage = {&modldrv, NULL},
119 };
120
121 void *t4_list;
122
123 struct intrs_and_queues {
124 int intr_type; /* DDI_INTR_TYPE_* */
125 int nirq; /* Number of vectors */
126 int intr_fwd; /* Interrupts forwarded */
127 int ntxq10g; /* # of NIC txq's for each 10G port */
128 int nrxq10g; /* # of NIC rxq's for each 10G port */
129 int ntxq1g; /* # of NIC txq's for each 1G port */
130 int nrxq1g; /* # of NIC rxq's for each 1G port */
131 #ifdef TCP_OFFLOAD_ENABLE
132 int nofldtxq10g; /* # of TOE txq's for each 10G port */
133 int nofldrxq10g; /* # of TOE rxq's for each 10G port */
134 int nofldtxq1g; /* # of TOE txq's for each 1G port */
135 int nofldrxq1g; /* # of TOE rxq's for each 1G port */
136 #endif
137 };
138
139 struct fw_info fi[3];
140
141 static int cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss,
142 mblk_t *m);
143 static int fw_msg_not_handled(struct adapter *, const __be64 *);
144 int t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h);
145 static unsigned int getpf(struct adapter *sc);
146 static int prep_firmware(struct adapter *sc);
147 static int upload_config_file(struct adapter *sc, uint32_t *mt, uint32_t *ma);
148 static int partition_resources(struct adapter *sc);
149 static int adap__pre_init_tweaks(struct adapter *sc);
150 static int get_params__pre_init(struct adapter *sc);
151 static int get_params__post_init(struct adapter *sc);
152 static int set_params__post_init(struct adapter *);
153 static void setup_memwin(struct adapter *sc);
154 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
155 uint32_t *);
156 void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
157 uint32_t position_memwin(struct adapter *, int, uint32_t);
158 static int prop_lookup_int_array(struct adapter *sc, char *name, int *data,
159 uint_t count);
160 static int prop_lookup_int_array(struct adapter *sc, char *name, int *data,
161 uint_t count);
162 static int init_driver_props(struct adapter *sc, struct driver_properties *p);
163 static int remove_extra_props(struct adapter *sc, int n10g, int n1g);
164 static int cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
165 struct intrs_and_queues *iaq);
166 static int add_child_node(struct adapter *sc, int idx);
167 static int remove_child_node(struct adapter *sc, int idx);
168 static kstat_t *setup_kstats(struct adapter *sc);
169 static kstat_t *setup_wc_kstats(struct adapter *);
170 static int update_wc_kstats(kstat_t *, int);
171 #ifdef TCP_OFFLOAD_ENABLE
172 static int toe_capability(struct port_info *pi, int enable);
173 static int activate_uld(struct adapter *sc, int id, struct uld_softc *usc);
174 static int deactivate_uld(struct uld_softc *usc);
175 #endif
176 static kmutex_t t4_adapter_list_lock;
177 static SLIST_HEAD(, adapter) t4_adapter_list;
178 #ifdef TCP_OFFLOAD_ENABLE
179 static kmutex_t t4_uld_list_lock;
180 static SLIST_HEAD(, uld_info) t4_uld_list;
181 #endif
182
183 int
184 _init(void)
185 {
186 int rc;
187
188 rc = ddi_soft_state_init(&t4_list, sizeof (struct adapter), 0);
189 if (rc != 0)
190 return (rc);
191
192 rc = mod_install(&modlinkage);
193 if (rc != 0)
194 ddi_soft_state_fini(&t4_list);
195
196 mutex_init(&t4_adapter_list_lock, NULL, MUTEX_DRIVER, NULL);
197 SLIST_INIT(&t4_adapter_list);
198
199 #ifdef TCP_OFFLOAD_ENABLE
200 mutex_init(&t4_uld_list_lock, NULL, MUTEX_DRIVER, NULL);
201 SLIST_INIT(&t4_uld_list);
202 #endif
203
204 return (rc);
205 }
206
207 int
208 _fini(void)
209 {
210 int rc;
211
212 rc = mod_remove(&modlinkage);
213 if (rc != 0)
214 return (rc);
215
216 ddi_soft_state_fini(&t4_list);
217 return (0);
218 }
219
220 int
221 _info(struct modinfo *mi)
222 {
223 return (mod_info(&modlinkage, mi));
224 }
225
226 /* ARGSUSED */
227 static int
228 t4_devo_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **rp)
229 {
230 struct adapter *sc;
231 minor_t minor;
232
233 minor = getminor((dev_t)arg); /* same as instance# in our case */
234
235 if (cmd == DDI_INFO_DEVT2DEVINFO) {
236 sc = ddi_get_soft_state(t4_list, minor);
237 if (sc == NULL)
238 return (DDI_FAILURE);
239
240 ASSERT(sc->dev == (dev_t)arg);
241 *rp = (void *)sc->dip;
242 } else if (cmd == DDI_INFO_DEVT2INSTANCE)
243 *rp = (void *) (unsigned long) minor;
244 else
245 ASSERT(0);
246
247 return (DDI_SUCCESS);
248 }
249
250 static int
251 t4_devo_probe(dev_info_t *dip)
252 {
253 int rc, id, *reg;
254 uint_t n, pf;
255
256 id = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
257 "device-id", 0xffff);
258 if (id == 0xffff)
259 return (DDI_PROBE_DONTCARE);
260
261 rc = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
262 "reg", ®, &n);
263 if (rc != DDI_SUCCESS)
264 return (DDI_PROBE_DONTCARE);
265
266 pf = PCI_REG_FUNC_G(reg[0]);
267 ddi_prop_free(reg);
268
269 /* Prevent driver attachment on any PF except 0 on the FPGA */
270 if (id == 0xa000 && pf != 0)
271 return (DDI_PROBE_FAILURE);
272
273 return (DDI_PROBE_DONTCARE);
274 }
275
276 static int
277 t4_devo_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
278 {
279 struct adapter *sc = NULL;
280 struct sge *s;
281 int i, instance, rc = DDI_SUCCESS, rqidx, tqidx, q;
282 int irq = 0, nxg, n100g, n40g, n25g, n10g, n1g;
283 #ifdef TCP_OFFLOAD_ENABLE
284 int ofld_rqidx, ofld_tqidx;
285 #endif
286 char name[16];
287 struct driver_properties *prp;
288 struct intrs_and_queues iaq;
289 ddi_device_acc_attr_t da = {
290 .devacc_attr_version = DDI_DEVICE_ATTR_V0,
291 .devacc_attr_endian_flags = DDI_STRUCTURE_LE_ACC,
292 .devacc_attr_dataorder = DDI_UNORDERED_OK_ACC
293 };
294 ddi_device_acc_attr_t da1 = {
295 .devacc_attr_version = DDI_DEVICE_ATTR_V0,
296 .devacc_attr_endian_flags = DDI_STRUCTURE_LE_ACC,
297 .devacc_attr_dataorder = DDI_MERGING_OK_ACC
298 };
299
300 if (cmd != DDI_ATTACH)
301 return (DDI_FAILURE);
302
303 /*
304 * Allocate space for soft state.
305 */
306 instance = ddi_get_instance(dip);
307 rc = ddi_soft_state_zalloc(t4_list, instance);
308 if (rc != DDI_SUCCESS) {
309 cxgb_printf(dip, CE_WARN,
310 "failed to allocate soft state: %d", rc);
311 return (DDI_FAILURE);
312 }
313
314 sc = ddi_get_soft_state(t4_list, instance);
315 sc->dip = dip;
316 sc->dev = makedevice(ddi_driver_major(dip), instance);
317 mutex_init(&sc->lock, NULL, MUTEX_DRIVER, NULL);
318 cv_init(&sc->cv, NULL, CV_DRIVER, NULL);
319 mutex_init(&sc->sfl_lock, NULL, MUTEX_DRIVER, NULL);
320
321 mutex_enter(&t4_adapter_list_lock);
322 SLIST_INSERT_HEAD(&t4_adapter_list, sc, link);
323 mutex_exit(&t4_adapter_list_lock);
324
325 sc->pf = getpf(sc);
326 if (sc->pf > 8) {
327 rc = EINVAL;
328 cxgb_printf(dip, CE_WARN,
329 "failed to determine PCI PF# of device");
330 goto done;
331 }
332 sc->mbox = sc->pf;
333
334 /* Initialize the driver properties */
335 prp = &sc->props;
336 (void)init_driver_props(sc, prp);
337
338 /*
339 * Enable access to the PCI config space.
340 */
341 rc = pci_config_setup(dip, &sc->pci_regh);
342 if (rc != DDI_SUCCESS) {
343 cxgb_printf(dip, CE_WARN,
344 "failed to enable PCI config space access: %d", rc);
345 goto done;
346 }
347
348 /* TODO: Set max read request to 4K */
349
350 /*
351 * Enable MMIO access.
352 */
353 rc = ddi_regs_map_setup(dip, 1, &sc->regp, 0, 0, &da, &sc->regh);
354 if (rc != DDI_SUCCESS) {
355 cxgb_printf(dip, CE_WARN,
356 "failed to map device registers: %d", rc);
357 goto done;
358 }
359
360 (void) memset(sc->chan_map, 0xff, sizeof (sc->chan_map));
361
362 /*
363 * Initialize cpl handler.
364 */
365 for (i = 0; i < ARRAY_SIZE(sc->cpl_handler); i++) {
366 sc->cpl_handler[i] = cpl_not_handled;
367 }
368
369 for (i = 0; i < ARRAY_SIZE(sc->fw_msg_handler); i++) {
370 sc->fw_msg_handler[i] = fw_msg_not_handled;
371 }
372
373 for (i = 0; i < NCHAN; i++) {
374 (void) snprintf(name, sizeof (name), "%s-%d",
375 "reclaim", i);
376 sc->tq[i] = ddi_taskq_create(sc->dip,
377 name, 1, TASKQ_DEFAULTPRI, 0);
378
379 if (sc->tq[i] == NULL) {
380 cxgb_printf(dip, CE_WARN,
381 "failed to create task queues");
382 rc = DDI_FAILURE;
383 goto done;
384 }
385 }
386
387 /*
388 * Prepare the adapter for operation.
389 */
390 rc = -t4_prep_adapter(sc, false);
391 if (rc != 0) {
392 cxgb_printf(dip, CE_WARN, "failed to prepare adapter: %d", rc);
393 goto done;
394 }
395
396 /*
397 * Enable BAR1 access.
398 */
399 sc->doorbells |= DOORBELL_KDB;
400 rc = ddi_regs_map_setup(dip, 2, &sc->reg1p, 0, 0, &da1, &sc->reg1h);
401 if (rc != DDI_SUCCESS) {
402 cxgb_printf(dip, CE_WARN,
403 "failed to map BAR1 device registers: %d", rc);
404 goto done;
405 } else {
406 if (is_t5(sc->params.chip)) {
407 sc->doorbells |= DOORBELL_UDB;
408 if (prp->wc) {
409 /*
410 * Enable write combining on BAR2. This is the
411 * userspace doorbell BAR and is split into 128B
412 * (UDBS_SEG_SIZE) doorbell regions, each associated
413 * with an egress queue. The first 64B has the doorbell
414 * and the second 64B can be used to submit a tx work
415 * request with an implicit doorbell.
416 */
417 sc->doorbells &= ~DOORBELL_UDB;
418 sc->doorbells |= (DOORBELL_WCWR |
419 DOORBELL_UDBWC);
420 t4_write_reg(sc, A_SGE_STAT_CFG,
421 V_STATSOURCE_T5(7) | V_STATMODE(0));
422 }
423 }
424 }
425
426 /*
427 * Do this really early. Note that minor number = instance.
428 */
429 (void) snprintf(name, sizeof (name), "%s,%d", T4_NEXUS_NAME, instance);
430 rc = ddi_create_minor_node(dip, name, S_IFCHR, instance,
431 DDI_NT_NEXUS, 0);
432 if (rc != DDI_SUCCESS) {
433 cxgb_printf(dip, CE_WARN,
434 "failed to create device node: %d", rc);
435 rc = DDI_SUCCESS; /* carry on */
436 }
437
438 /* Do this early. Memory window is required for loading config file. */
439 setup_memwin(sc);
440
441 /* Prepare the firmware for operation */
442 rc = prep_firmware(sc);
443 if (rc != 0)
444 goto done; /* error message displayed already */
445
446 rc = adap__pre_init_tweaks(sc);
447 if (rc != 0)
448 goto done;
449
450 rc = get_params__pre_init(sc);
451 if (rc != 0)
452 goto done; /* error message displayed already */
453
454 t4_sge_init(sc);
455
456 if (sc->flags & MASTER_PF) {
457 /* get basic stuff going */
458 rc = -t4_fw_initialize(sc, sc->mbox);
459 if (rc != 0) {
460 cxgb_printf(sc->dip, CE_WARN,
461 "early init failed: %d.\n", rc);
462 goto done;
463 }
464 }
465
466 rc = get_params__post_init(sc);
467 if (rc != 0)
468 goto done; /* error message displayed already */
469
470 rc = set_params__post_init(sc);
471 if (rc != 0)
472 goto done; /* error message displayed already */
473
474 /*
475 * TODO: This is the place to call t4_set_filter_mode()
476 */
477
478 /* tweak some settings */
479 t4_write_reg(sc, A_TP_SHIFT_CNT, V_SYNSHIFTMAX(6) | V_RXTSHIFTMAXR1(4) |
480 V_RXTSHIFTMAXR2(15) | V_PERSHIFTBACKOFFMAX(8) | V_PERSHIFTMAX(8) |
481 V_KEEPALIVEMAXR1(4) | V_KEEPALIVEMAXR2(9));
482 t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
483
484 /*
485 * Work-around for bug 2619
486 * Set DisableVlan field in TP_RSS_CONFIG_VRT register so that the
487 * VLAN tag extraction is disabled.
488 */
489 t4_set_reg_field(sc, A_TP_RSS_CONFIG_VRT, F_DISABLEVLAN, F_DISABLEVLAN);
490
491 /* Store filter mode */
492 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &sc->filter_mode, 1,
493 A_TP_VLAN_PRI_MAP);
494
495 /*
496 * First pass over all the ports - allocate VIs and initialize some
497 * basic parameters like mac address, port type, etc. We also figure
498 * out whether a port is 10G or 1G and use that information when
499 * calculating how many interrupts to attempt to allocate.
500 */
501 n100g = n40g = n25g = n10g = n1g = 0;
502 for_each_port(sc, i) {
503 struct port_info *pi;
504
505 pi = kmem_zalloc(sizeof (*pi), KM_SLEEP);
506 sc->port[i] = pi;
507
508 /* These must be set before t4_port_init */
509 pi->adapter = sc;
510 /* LINTED: E_ASSIGN_NARROW_CONV */
511 pi->port_id = i;
512 }
513
514 /* Allocate the vi and initialize parameters like mac addr */
515 rc = -t4_port_init(sc, sc->mbox, sc->pf, 0);
516 if (rc) {
517 cxgb_printf(dip, CE_WARN,
518 "unable to initialize port: %d", rc);
519 goto done;
520 }
521
522 for_each_port(sc, i) {
523 struct port_info *pi = sc->port[i];
524
525 mutex_init(&pi->lock, NULL, MUTEX_DRIVER, NULL);
526 pi->mtu = ETHERMTU;
527
528 if (is_100G_port(pi)) {
529 n100g++;
530 pi->tmr_idx = prp->tmr_idx_10g;
531 pi->pktc_idx = prp->pktc_idx_10g;
532 } else if (is_40G_port(pi)) {
533 n40g++;
534 pi->tmr_idx = prp->tmr_idx_10g;
535 pi->pktc_idx = prp->pktc_idx_10g;
536 } else if (is_25G_port(pi)) {
537 n25g++;
538 pi->tmr_idx = prp->tmr_idx_10g;
539 pi->pktc_idx = prp->pktc_idx_10g;
540 } else if (is_10G_port(pi)) {
541 n10g++;
542 pi->tmr_idx = prp->tmr_idx_10g;
543 pi->pktc_idx = prp->pktc_idx_10g;
544 } else {
545 n1g++;
546 pi->tmr_idx = prp->tmr_idx_1g;
547 pi->pktc_idx = prp->pktc_idx_1g;
548 }
549
550 pi->xact_addr_filt = -1;
551 t4_mc_init(pi);
552
553 setbit(&sc->registered_device_map, i);
554 }
555
556 nxg = n10g + n25g + n40g + n100g;
557 (void) remove_extra_props(sc, nxg, n1g);
558
559 if (sc->registered_device_map == 0) {
560 cxgb_printf(dip, CE_WARN, "no usable ports");
561 rc = DDI_FAILURE;
562 goto done;
563 }
564
565 rc = cfg_itype_and_nqueues(sc, nxg, n1g, &iaq);
566 if (rc != 0)
567 goto done; /* error message displayed already */
568
569 sc->intr_type = iaq.intr_type;
570 sc->intr_count = iaq.nirq;
571
572 if (sc->props.multi_rings && (sc->intr_type != DDI_INTR_TYPE_MSIX)) {
573 sc->props.multi_rings = 0;
574 cxgb_printf(dip, CE_WARN,
575 "Multiple rings disabled as interrupt type is not MSI-X");
576 }
577
578 if (sc->props.multi_rings && iaq.intr_fwd) {
579 sc->props.multi_rings = 0;
580 cxgb_printf(dip, CE_WARN,
581 "Multiple rings disabled as interrupts are forwarded");
582 }
583
584 if (!sc->props.multi_rings) {
585 iaq.ntxq10g = 1;
586 iaq.ntxq1g = 1;
587 }
588 s = &sc->sge;
589 s->nrxq = nxg * iaq.nrxq10g + n1g * iaq.nrxq1g;
590 s->ntxq = nxg * iaq.ntxq10g + n1g * iaq.ntxq1g;
591 s->neq = s->ntxq + s->nrxq; /* the fl in an rxq is an eq */
592 #ifdef TCP_OFFLOAD_ENABLE
593 /* control queues, 1 per port + 1 mgmtq */
594 s->neq += sc->params.nports + 1;
595 #endif
596 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */
597 if (iaq.intr_fwd != 0)
598 sc->flags |= INTR_FWD;
599 #ifdef TCP_OFFLOAD_ENABLE
600 if (is_offload(sc) != 0) {
601
602 s->nofldrxq = nxg * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
603 s->nofldtxq = nxg * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
604 s->neq += s->nofldtxq + s->nofldrxq;
605 s->niq += s->nofldrxq;
606
607 s->ofld_rxq = kmem_zalloc(s->nofldrxq *
608 sizeof (struct sge_ofld_rxq), KM_SLEEP);
609 s->ofld_txq = kmem_zalloc(s->nofldtxq *
610 sizeof (struct sge_wrq), KM_SLEEP);
611 s->ctrlq = kmem_zalloc(sc->params.nports *
612 sizeof (struct sge_wrq), KM_SLEEP);
613
614 }
615 #endif
616 s->rxq = kmem_zalloc(s->nrxq * sizeof (struct sge_rxq), KM_SLEEP);
617 s->txq = kmem_zalloc(s->ntxq * sizeof (struct sge_txq), KM_SLEEP);
618 s->iqmap = kmem_zalloc(s->niq * sizeof (struct sge_iq *), KM_SLEEP);
619 s->eqmap = kmem_zalloc(s->neq * sizeof (struct sge_eq *), KM_SLEEP);
620
621 sc->intr_handle = kmem_zalloc(sc->intr_count *
622 sizeof (ddi_intr_handle_t), KM_SLEEP);
623
624 /*
625 * Second pass over the ports. This time we know the number of rx and
626 * tx queues that each port should get.
627 */
628 rqidx = tqidx = 0;
629 #ifdef TCP_OFFLOAD_ENABLE
630 ofld_rqidx = ofld_tqidx = 0;
631 #endif
632 for_each_port(sc, i) {
633 struct port_info *pi = sc->port[i];
634
635 if (pi == NULL)
636 continue;
637
638 t4_mc_cb_init(pi);
639 /* LINTED: E_ASSIGN_NARROW_CONV */
640 pi->first_rxq = rqidx;
641 /* LINTED: E_ASSIGN_NARROW_CONV */
642 pi->nrxq = (is_10XG_port(pi)) ? iaq.nrxq10g
643 : iaq.nrxq1g;
644 /* LINTED: E_ASSIGN_NARROW_CONV */
645 pi->first_txq = tqidx;
646 /* LINTED: E_ASSIGN_NARROW_CONV */
647 pi->ntxq = (is_10XG_port(pi)) ? iaq.ntxq10g
648 : iaq.ntxq1g;
649
650 rqidx += pi->nrxq;
651 tqidx += pi->ntxq;
652
653 #ifdef TCP_OFFLOAD_ENABLE
654 if (is_offload(sc) != 0) {
655 /* LINTED: E_ASSIGN_NARROW_CONV */
656 pi->first_ofld_rxq = ofld_rqidx;
657 pi->nofldrxq = max(1, pi->nrxq / 4);
658
659 /* LINTED: E_ASSIGN_NARROW_CONV */
660 pi->first_ofld_txq = ofld_tqidx;
661 pi->nofldtxq = max(1, pi->ntxq / 2);
662
663 ofld_rqidx += pi->nofldrxq;
664 ofld_tqidx += pi->nofldtxq;
665 }
666 #endif
667
668 /*
669 * Enable hw checksumming and LSO for all ports by default.
670 * They can be disabled using ndd (hw_csum and hw_lso).
671 */
672 pi->features |= (CXGBE_HW_CSUM | CXGBE_HW_LSO);
673 }
674
675 #ifdef TCP_OFFLOAD_ENABLE
676 sc->l2t = t4_init_l2t(sc);
677 #endif
678
679 /*
680 * Setup Interrupts.
681 */
682
683 i = 0;
684 rc = ddi_intr_alloc(dip, sc->intr_handle, sc->intr_type, 0,
685 sc->intr_count, &i, DDI_INTR_ALLOC_STRICT);
686 if (rc != DDI_SUCCESS) {
687 cxgb_printf(dip, CE_WARN,
688 "failed to allocate %d interrupt(s) of type %d: %d, %d",
689 sc->intr_count, sc->intr_type, rc, i);
690 goto done;
691 }
692 ASSERT(sc->intr_count == i); /* allocation was STRICT */
693 (void) ddi_intr_get_cap(sc->intr_handle[0], &sc->intr_cap);
694 (void) ddi_intr_get_pri(sc->intr_handle[0], &sc->intr_pri);
695 if (sc->intr_count == 1) {
696 ASSERT(sc->flags & INTR_FWD);
697 (void) ddi_intr_add_handler(sc->intr_handle[0], t4_intr_all, sc,
698 &s->fwq);
699 } else {
700 /* Multiple interrupts. The first one is always error intr */
701 (void) ddi_intr_add_handler(sc->intr_handle[0], t4_intr_err, sc,
702 NULL);
703 irq++;
704
705 /* The second one is always the firmware event queue */
706 (void) ddi_intr_add_handler(sc->intr_handle[1], t4_intr, sc,
707 &s->fwq);
708 irq++;
709 /*
710 * Note that if INTR_FWD is set then either the NIC rx
711 * queues or (exclusive or) the TOE rx queueus will be taking
712 * direct interrupts.
713 *
714 * There is no need to check for is_offload(sc) as nofldrxq
715 * will be 0 if offload is disabled.
716 */
717 for_each_port(sc, i) {
718 struct port_info *pi = sc->port[i];
719 struct sge_rxq *rxq;
720 #ifdef TCP_OFFLOAD_ENABLE
721 struct sge_ofld_rxq *ofld_rxq;
722
723 /*
724 * Skip over the NIC queues if they aren't taking direct
725 * interrupts.
726 */
727 if ((sc->flags & INTR_FWD) &&
728 pi->nofldrxq > pi->nrxq)
729 goto ofld_queues;
730 #endif
731 rxq = &s->rxq[pi->first_rxq];
732 for (q = 0; q < pi->nrxq; q++, rxq++) {
733 (void) ddi_intr_add_handler(
734 sc->intr_handle[irq], t4_intr, sc,
735 &rxq->iq);
736 irq++;
737 }
738
739 #ifdef TCP_OFFLOAD_ENABLE
740 /*
741 * Skip over the offload queues if they aren't taking
742 * direct interrupts.
743 */
744 if ((sc->flags & INTR_FWD))
745 continue;
746 ofld_queues:
747 ofld_rxq = &s->ofld_rxq[pi->first_ofld_rxq];
748 for (q = 0; q < pi->nofldrxq; q++, ofld_rxq++) {
749 (void) ddi_intr_add_handler(
750 sc->intr_handle[irq], t4_intr, sc,
751 &ofld_rxq->iq);
752 irq++;
753 }
754 #endif
755 }
756
757 }
758 sc->flags |= INTR_ALLOCATED;
759
760 ASSERT(rc == DDI_SUCCESS);
761 ddi_report_dev(dip);
762
763 /*
764 * Hardware/Firmware/etc. Version/Revision IDs.
765 */
766 t4_dump_version_info(sc);
767
768 if (n100g) {
769 cxgb_printf(dip, CE_NOTE,
770 "%dx100G (%d rxq, %d txq total) %d %s.",
771 n100g, rqidx, tqidx, sc->intr_count,
772 sc->intr_type == DDI_INTR_TYPE_MSIX ? "MSI-X interrupts" :
773 sc->intr_type == DDI_INTR_TYPE_MSI ? "MSI interrupts" :
774 "fixed interrupt");
775 } else if (n40g) {
776 cxgb_printf(dip, CE_NOTE,
777 "%dx40G (%d rxq, %d txq total) %d %s.",
778 n40g, rqidx, tqidx, sc->intr_count,
779 sc->intr_type == DDI_INTR_TYPE_MSIX ? "MSI-X interrupts" :
780 sc->intr_type == DDI_INTR_TYPE_MSI ? "MSI interrupts" :
781 "fixed interrupt");
782 } else if (n25g) {
783 cxgb_printf(dip, CE_NOTE,
784 "%dx25G (%d rxq, %d txq total) %d %s.",
785 n25g, rqidx, tqidx, sc->intr_count,
786 sc->intr_type == DDI_INTR_TYPE_MSIX ? "MSI-X interrupts" :
787 sc->intr_type == DDI_INTR_TYPE_MSI ? "MSI interrupts" :
788 "fixed interrupt");
789 } else if (n10g && n1g) {
790 cxgb_printf(dip, CE_NOTE,
791 "%dx10G %dx1G (%d rxq, %d txq total) %d %s.",
792 n10g, n1g, rqidx, tqidx, sc->intr_count,
793 sc->intr_type == DDI_INTR_TYPE_MSIX ? "MSI-X interrupts" :
794 sc->intr_type == DDI_INTR_TYPE_MSI ? "MSI interrupts" :
795 "fixed interrupt");
796 } else {
797 cxgb_printf(dip, CE_NOTE,
798 "%dx%sG (%d rxq, %d txq per port) %d %s.",
799 n10g ? n10g : n1g,
800 n10g ? "10" : "1",
801 n10g ? iaq.nrxq10g : iaq.nrxq1g,
802 n10g ? iaq.ntxq10g : iaq.ntxq1g,
803 sc->intr_count,
804 sc->intr_type == DDI_INTR_TYPE_MSIX ? "MSI-X interrupts" :
805 sc->intr_type == DDI_INTR_TYPE_MSI ? "MSI interrupts" :
806 "fixed interrupt");
807 }
808
809 sc->ksp = setup_kstats(sc);
810 sc->ksp_stat = setup_wc_kstats(sc);
811 sc->params.drv_memwin = MEMWIN_NIC;
812
813 done:
814 if (rc != DDI_SUCCESS) {
815 (void) t4_devo_detach(dip, DDI_DETACH);
816
817 /* rc may have errno style errors or DDI errors */
818 rc = DDI_FAILURE;
819 }
820
821 return (rc);
822 }
823
824 static int
825 t4_devo_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
826 {
827 int instance, i;
828 struct adapter *sc;
829 struct port_info *pi;
830 struct sge *s;
831
832 if (cmd != DDI_DETACH)
833 return (DDI_FAILURE);
834
835 instance = ddi_get_instance(dip);
836 sc = ddi_get_soft_state(t4_list, instance);
837 if (sc == NULL)
838 return (DDI_SUCCESS);
839
840 if (sc->flags & FULL_INIT_DONE) {
841 t4_intr_disable(sc);
842 for_each_port(sc, i) {
843 pi = sc->port[i];
844 if (pi && pi->flags & PORT_INIT_DONE)
845 (void) port_full_uninit(pi);
846 }
847 (void) adapter_full_uninit(sc);
848 }
849
850 /* Safe to call no matter what */
851 ddi_prop_remove_all(dip);
852 ddi_remove_minor_node(dip, NULL);
853
854 for (i = 0; i < NCHAN; i++) {
855 if (sc->tq[i]) {
856 ddi_taskq_wait(sc->tq[i]);
857 ddi_taskq_destroy(sc->tq[i]);
858 }
859 }
860
861 if (sc->ksp != NULL)
862 kstat_delete(sc->ksp);
863 if (sc->ksp_stat != NULL)
864 kstat_delete(sc->ksp_stat);
865
866 s = &sc->sge;
867 if (s->rxq != NULL)
868 kmem_free(s->rxq, s->nrxq * sizeof (struct sge_rxq));
869 #ifdef TCP_OFFLOAD_ENABLE
870 if (s->ofld_txq != NULL)
871 kmem_free(s->ofld_txq, s->nofldtxq * sizeof (struct sge_wrq));
872 if (s->ofld_rxq != NULL)
873 kmem_free(s->ofld_rxq,
874 s->nofldrxq * sizeof (struct sge_ofld_rxq));
875 if (s->ctrlq != NULL)
876 kmem_free(s->ctrlq,
877 sc->params.nports * sizeof (struct sge_wrq));
878 #endif
879 if (s->txq != NULL)
880 kmem_free(s->txq, s->ntxq * sizeof (struct sge_txq));
881 if (s->iqmap != NULL)
882 kmem_free(s->iqmap, s->niq * sizeof (struct sge_iq *));
883 if (s->eqmap != NULL)
884 kmem_free(s->eqmap, s->neq * sizeof (struct sge_eq *));
885
886 if (s->rxbuf_cache != NULL)
887 rxbuf_cache_destroy(s->rxbuf_cache);
888
889 if (sc->flags & INTR_ALLOCATED) {
890 for (i = 0; i < sc->intr_count; i++) {
891 (void) ddi_intr_remove_handler(sc->intr_handle[i]);
892 (void) ddi_intr_free(sc->intr_handle[i]);
893 }
894 sc->flags &= ~INTR_ALLOCATED;
895 }
896
897 if (sc->intr_handle != NULL) {
898 kmem_free(sc->intr_handle,
899 sc->intr_count * sizeof (*sc->intr_handle));
900 }
901
902 for_each_port(sc, i) {
903 pi = sc->port[i];
904 if (pi != NULL) {
905 mutex_destroy(&pi->lock);
906 kmem_free(pi, sizeof (*pi));
907 clrbit(&sc->registered_device_map, i);
908 }
909 }
910
911 if (sc->flags & FW_OK)
912 (void) t4_fw_bye(sc, sc->mbox);
913
914 if (sc->reg1h != NULL)
915 ddi_regs_map_free(&sc->reg1h);
916
917 if (sc->regh != NULL)
918 ddi_regs_map_free(&sc->regh);
919
920 if (sc->pci_regh != NULL)
921 pci_config_teardown(&sc->pci_regh);
922
923 mutex_enter(&t4_adapter_list_lock);
924 SLIST_REMOVE_HEAD(&t4_adapter_list, link);
925 mutex_exit(&t4_adapter_list_lock);
926
927 mutex_destroy(&sc->lock);
928 cv_destroy(&sc->cv);
929 mutex_destroy(&sc->sfl_lock);
930
931 #ifdef DEBUG
932 bzero(sc, sizeof (*sc));
933 #endif
934 ddi_soft_state_free(t4_list, instance);
935
936 return (DDI_SUCCESS);
937 }
938
939 static int
940 t4_devo_quiesce(dev_info_t *dip)
941 {
942 int instance;
943 struct adapter *sc;
944
945 instance = ddi_get_instance(dip);
946 sc = ddi_get_soft_state(t4_list, instance);
947 if (sc == NULL)
948 return (DDI_SUCCESS);
949
950 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
951 t4_intr_disable(sc);
952 t4_write_reg(sc, A_PL_RST, F_PIORSTMODE | F_PIORST);
953
954 return (DDI_SUCCESS);
955 }
956
957 static int
958 t4_bus_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t op, void *arg,
959 void *result)
960 {
961 char s[4];
962 struct port_info *pi;
963 dev_info_t *child = (dev_info_t *)arg;
964
965 switch (op) {
966 case DDI_CTLOPS_REPORTDEV:
967 pi = ddi_get_parent_data(rdip);
968 pi->instance = ddi_get_instance(dip);
969 pi->child_inst = ddi_get_instance(rdip);
970 cmn_err(CE_CONT, "?%s%d is port %s on %s%d\n",
971 ddi_node_name(rdip), ddi_get_instance(rdip),
972 ddi_get_name_addr(rdip), ddi_driver_name(dip),
973 ddi_get_instance(dip));
974 return (DDI_SUCCESS);
975
976 case DDI_CTLOPS_INITCHILD:
977 pi = ddi_get_parent_data(child);
978 if (pi == NULL)
979 return (DDI_NOT_WELL_FORMED);
980 (void) snprintf(s, sizeof (s), "%d", pi->port_id);
981 ddi_set_name_addr(child, s);
982 return (DDI_SUCCESS);
983
984 case DDI_CTLOPS_UNINITCHILD:
985 ddi_set_name_addr(child, NULL);
986 return (DDI_SUCCESS);
987
988 case DDI_CTLOPS_ATTACH:
989 case DDI_CTLOPS_DETACH:
990 return (DDI_SUCCESS);
991
992 default:
993 return (ddi_ctlops(dip, rdip, op, arg, result));
994 }
995 }
996
997 static int
998 t4_bus_config(dev_info_t *dip, uint_t flags, ddi_bus_config_op_t op, void *arg,
999 dev_info_t **cdipp)
1000 {
1001 int instance, i;
1002 struct adapter *sc;
1003
1004 instance = ddi_get_instance(dip);
1005 sc = ddi_get_soft_state(t4_list, instance);
1006
1007 if (op == BUS_CONFIG_ONE) {
1008 char *c;
1009
1010 /*
1011 * arg is something like "cxgb@0" where 0 is the port_id hanging
1012 * off this nexus.
1013 */
1014
1015 c = arg;
1016 while (*(c + 1))
1017 c++;
1018
1019 /* There should be exactly 1 digit after '@' */
1020 if (*(c - 1) != '@')
1021 return (NDI_FAILURE);
1022
1023 i = *c - '0';
1024
1025 if (add_child_node(sc, i) != 0)
1026 return (NDI_FAILURE);
1027
1028 flags |= NDI_ONLINE_ATTACH;
1029
1030 } else if (op == BUS_CONFIG_ALL || op == BUS_CONFIG_DRIVER) {
1031 /* Allocate and bind all child device nodes */
1032 for_each_port(sc, i)
1033 (void) add_child_node(sc, i);
1034 flags |= NDI_ONLINE_ATTACH;
1035 }
1036
1037 return (ndi_busop_bus_config(dip, flags, op, arg, cdipp, 0));
1038 }
1039
1040 static int
1041 t4_bus_unconfig(dev_info_t *dip, uint_t flags, ddi_bus_config_op_t op,
1042 void *arg)
1043 {
1044 int instance, i, rc;
1045 struct adapter *sc;
1046
1047 instance = ddi_get_instance(dip);
1048 sc = ddi_get_soft_state(t4_list, instance);
1049
1050 if (op == BUS_CONFIG_ONE || op == BUS_UNCONFIG_ALL ||
1051 op == BUS_UNCONFIG_DRIVER)
1052 flags |= NDI_UNCONFIG;
1053
1054 rc = ndi_busop_bus_unconfig(dip, flags, op, arg);
1055 if (rc != 0)
1056 return (rc);
1057
1058 if (op == BUS_UNCONFIG_ONE) {
1059 char *c;
1060
1061 c = arg;
1062 while (*(c + 1))
1063 c++;
1064
1065 if (*(c - 1) != '@')
1066 return (NDI_SUCCESS);
1067
1068 i = *c - '0';
1069
1070 rc = remove_child_node(sc, i);
1071
1072 } else if (op == BUS_UNCONFIG_ALL || op == BUS_UNCONFIG_DRIVER) {
1073
1074 for_each_port(sc, i)
1075 (void) remove_child_node(sc, i);
1076 }
1077
1078 return (rc);
1079 }
1080
1081 /* ARGSUSED */
1082 static int
1083 t4_cb_open(dev_t *devp, int flag, int otyp, cred_t *credp)
1084 {
1085 struct adapter *sc;
1086
1087 if (otyp != OTYP_CHR)
1088 return (EINVAL);
1089
1090 sc = ddi_get_soft_state(t4_list, getminor(*devp));
1091 if (sc == NULL)
1092 return (ENXIO);
1093
1094 return (atomic_cas_uint(&sc->open, 0, EBUSY));
1095 }
1096
1097 /* ARGSUSED */
1098 static int
1099 t4_cb_close(dev_t dev, int flag, int otyp, cred_t *credp)
1100 {
1101 struct adapter *sc;
1102
1103 sc = ddi_get_soft_state(t4_list, getminor(dev));
1104 if (sc == NULL)
1105 return (EINVAL);
1106
1107 (void) atomic_swap_uint(&sc->open, 0);
1108 return (0);
1109 }
1110
1111 /* ARGSUSED */
1112 static int
1113 t4_cb_ioctl(dev_t dev, int cmd, intptr_t d, int mode, cred_t *credp, int *rp)
1114 {
1115 int instance;
1116 struct adapter *sc;
1117 void *data = (void *)d;
1118
1119 if (crgetuid(credp) != 0)
1120 return (EPERM);
1121
1122 instance = getminor(dev);
1123 sc = ddi_get_soft_state(t4_list, instance);
1124 if (sc == NULL)
1125 return (EINVAL);
1126
1127 return (t4_ioctl(sc, cmd, data, mode));
1128 }
1129
1130 static unsigned int
1131 getpf(struct adapter *sc)
1132 {
1133 int rc, *data;
1134 uint_t n, pf;
1135
1136 rc = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, sc->dip,
1137 DDI_PROP_DONTPASS, "reg", &data, &n);
1138 if (rc != DDI_SUCCESS) {
1139 cxgb_printf(sc->dip, CE_WARN,
1140 "failed to lookup \"reg\" property: %d", rc);
1141 return (0xff);
1142 }
1143
1144 pf = PCI_REG_FUNC_G(data[0]);
1145 ddi_prop_free(data);
1146
1147 return (pf);
1148 }
1149
1150
1151 static struct fw_info *
1152 find_fw_info(int chip)
1153 {
1154 u32 i;
1155
1156 fi[0].chip = CHELSIO_T4;
1157 fi[0].fw_hdr.chip = FW_HDR_CHIP_T4;
1158 fi[0].fw_hdr.fw_ver = cpu_to_be32(FW_VERSION(T4));
1159 fi[0].fw_hdr.intfver_nic = FW_INTFVER(T4, NIC);
1160 fi[0].fw_hdr.intfver_vnic = FW_INTFVER(T4, VNIC);
1161 fi[0].fw_hdr.intfver_ofld = FW_INTFVER(T4, OFLD);
1162 fi[0].fw_hdr.intfver_ri = FW_INTFVER(T4, RI);
1163 fi[0].fw_hdr.intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU);
1164 fi[0].fw_hdr.intfver_iscsi = FW_INTFVER(T4, ISCSI);
1165 fi[0].fw_hdr.intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU);
1166 fi[0].fw_hdr.intfver_fcoe = FW_INTFVER(T4, FCOE);
1167
1168 fi[1].chip = CHELSIO_T5;
1169 fi[1].fw_hdr.chip = FW_HDR_CHIP_T5;
1170 fi[1].fw_hdr.fw_ver = cpu_to_be32(FW_VERSION(T5));
1171 fi[1].fw_hdr.intfver_nic = FW_INTFVER(T5, NIC);
1172 fi[1].fw_hdr.intfver_vnic = FW_INTFVER(T5, VNIC);
1173 fi[1].fw_hdr.intfver_ofld = FW_INTFVER(T5, OFLD);
1174 fi[1].fw_hdr.intfver_ri = FW_INTFVER(T5, RI);
1175 fi[1].fw_hdr.intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU);
1176 fi[1].fw_hdr.intfver_iscsi = FW_INTFVER(T5, ISCSI);
1177 fi[1].fw_hdr.intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU);
1178 fi[1].fw_hdr.intfver_fcoe = FW_INTFVER(T5, FCOE);
1179
1180 fi[2].chip = CHELSIO_T6;
1181 fi[2].fw_hdr.chip = FW_HDR_CHIP_T6;
1182 fi[2].fw_hdr.fw_ver = cpu_to_be32(FW_VERSION(T6));
1183 fi[2].fw_hdr.intfver_nic = FW_INTFVER(T6, NIC);
1184 fi[2].fw_hdr.intfver_vnic = FW_INTFVER(T6, VNIC);
1185 fi[2].fw_hdr.intfver_ofld = FW_INTFVER(T6, OFLD);
1186 fi[2].fw_hdr.intfver_ri = FW_INTFVER(T6, RI);
1187 fi[2].fw_hdr.intfver_iscsipdu = FW_INTFVER(T6, ISCSIPDU);
1188 fi[2].fw_hdr.intfver_iscsi = FW_INTFVER(T6, ISCSI);
1189 fi[2].fw_hdr.intfver_fcoepdu = FW_INTFVER(T6, FCOEPDU);
1190 fi[2].fw_hdr.intfver_fcoe = FW_INTFVER(T6, FCOE);
1191
1192 for (i = 0; i < ARRAY_SIZE(fi); i++) {
1193 if (fi[i].chip == chip)
1194 return &fi[i];
1195 }
1196
1197 return NULL;
1198 }
1199
1200 /*
1201 * Install a compatible firmware (if required), establish contact with it,
1202 * become the master, and reset the device.
1203 */
1204 static int
1205 prep_firmware(struct adapter *sc)
1206 {
1207 int rc;
1208 int fw_size;
1209 int reset = 1;
1210 enum dev_state state;
1211 unsigned char *fw_data;
1212 struct fw_info *fw_info;
1213 struct fw_hdr *card_fw;
1214
1215 struct driver_properties *p = &sc->props;
1216
1217 /* Contact firmware, request master */
1218 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MUST, &state);
1219 if (rc < 0) {
1220 rc = -rc;
1221 cxgb_printf(sc->dip, CE_WARN,
1222 "failed to connect to the firmware: %d.", rc);
1223 return (rc);
1224 }
1225
1226 if (rc == sc->mbox)
1227 sc->flags |= MASTER_PF;
1228
1229 /* We may need FW version info for later reporting */
1230 t4_get_version_info(sc);
1231 fw_info = find_fw_info(CHELSIO_CHIP_VERSION(sc->params.chip));
1232 /* allocate memory to read the header of the firmware on the
1233 * card
1234 */
1235 if (!fw_info) {
1236 cxgb_printf(sc->dip, CE_WARN,
1237 "unable to look up firmware information for chip %d.\n",
1238 CHELSIO_CHIP_VERSION(sc->params.chip));
1239 return EINVAL;
1240 }
1241 card_fw = kmem_zalloc(sizeof(*card_fw), KM_SLEEP);
1242 if(!card_fw) {
1243 cxgb_printf(sc->dip, CE_WARN,
1244 "Memory allocation for card FW header failed\n");
1245 return ENOMEM;
1246 }
1247 switch(CHELSIO_CHIP_VERSION(sc->params.chip)) {
1248 case CHELSIO_T4:
1249 fw_data = t4fw_data;
1250 fw_size = t4fw_size;
1251 break;
1252 case CHELSIO_T5:
1253 fw_data = t5fw_data;
1254 fw_size = t5fw_size;
1255 break;
1256 case CHELSIO_T6:
1257 fw_data = t6fw_data;
1258 fw_size = t6fw_size;
1259 break;
1260 default:
1261 cxgb_printf(sc->dip, CE_WARN, "Adapter type not supported\n");
1262 kmem_free(card_fw, sizeof(*card_fw));
1263 return EINVAL;
1264 }
1265
1266 rc = -t4_prep_fw(sc, fw_info, fw_data, fw_size, card_fw,
1267 p->t4_fw_install, state, &reset);
1268
1269 kmem_free(card_fw, sizeof(*card_fw));
1270
1271 if (rc != 0) {
1272 cxgb_printf(sc->dip, CE_WARN,
1273 "failed to install firmware: %d", rc);
1274 return (rc);
1275 } else {
1276 /* refresh */
1277 (void) t4_check_fw_version(sc);
1278 }
1279
1280 /* Reset device */
1281 rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST);
1282 if (rc != 0) {
1283 cxgb_printf(sc->dip, CE_WARN,
1284 "firmware reset failed: %d.", rc);
1285 if (rc != ETIMEDOUT && rc != EIO)
1286 (void) t4_fw_bye(sc, sc->mbox);
1287 return (rc);
1288 }
1289
1290 /* Partition adapter resources as specified in the config file. */
1291 if (sc->flags & MASTER_PF) {
1292 /* Handle default vs special T4 config file */
1293
1294 rc = partition_resources(sc);
1295 if (rc != 0)
1296 goto err; /* error message displayed already */
1297 }
1298
1299 sc->flags |= FW_OK;
1300 return (0);
1301 err:
1302 return (rc);
1303
1304 }
1305
1306 static const struct memwin t4_memwin[] = {
1307 { MEMWIN0_BASE, MEMWIN0_APERTURE },
1308 { MEMWIN1_BASE, MEMWIN1_APERTURE },
1309 { MEMWIN2_BASE, MEMWIN2_APERTURE }
1310 };
1311
1312 static const struct memwin t5_memwin[] = {
1313 { MEMWIN0_BASE, MEMWIN0_APERTURE },
1314 { MEMWIN1_BASE, MEMWIN1_APERTURE },
1315 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1316 };
1317
1318 #define FW_PARAM_DEV(param) \
1319 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
1320 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
1321 #define FW_PARAM_PFVF(param) \
1322 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
1323 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
1324
1325 /*
1326 * Verify that the memory range specified by the memtype/offset/len pair is
1327 * valid and lies entirely within the memtype specified. The global address of
1328 * the start of the range is returned in addr.
1329 */
1330 int
1331 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1332 uint32_t *addr)
1333 {
1334 uint32_t em, addr_len, maddr, mlen;
1335
1336 /* Memory can only be accessed in naturally aligned 4 byte units */
1337 if (off & 3 || len & 3 || len == 0)
1338 return (EINVAL);
1339
1340 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1341 switch (mtype) {
1342 case MEM_EDC0:
1343 if (!(em & F_EDRAM0_ENABLE))
1344 return (EINVAL);
1345 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1346 maddr = G_EDRAM0_BASE(addr_len) << 20;
1347 mlen = G_EDRAM0_SIZE(addr_len) << 20;
1348 break;
1349 case MEM_EDC1:
1350 if (!(em & F_EDRAM1_ENABLE))
1351 return (EINVAL);
1352 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1353 maddr = G_EDRAM1_BASE(addr_len) << 20;
1354 mlen = G_EDRAM1_SIZE(addr_len) << 20;
1355 break;
1356 case MEM_MC:
1357 if (!(em & F_EXT_MEM_ENABLE))
1358 return (EINVAL);
1359 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1360 maddr = G_EXT_MEM_BASE(addr_len) << 20;
1361 mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1362 break;
1363 case MEM_MC1:
1364 if (is_t4(sc->params.chip) || !(em & F_EXT_MEM1_ENABLE))
1365 return (EINVAL);
1366 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1367 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1368 mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1369 break;
1370 default:
1371 return (EINVAL);
1372 }
1373
1374 if (mlen > 0 && off < mlen && off + len <= mlen) {
1375 *addr = maddr + off; /* global address */
1376 return (0);
1377 }
1378
1379 return (EFAULT);
1380 }
1381
1382 void
1383 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1384 {
1385 const struct memwin *mw;
1386
1387 if (is_t4(sc->params.chip)) {
1388 mw = &t4_memwin[win];
1389 } else {
1390 mw = &t5_memwin[win];
1391 }
1392
1393 if (base != NULL)
1394 *base = mw->base;
1395 if (aperture != NULL)
1396 *aperture = mw->aperture;
1397 }
1398
1399 /*
1400 * Upload configuration file to card's memory.
1401 */
1402 static int
1403 upload_config_file(struct adapter *sc, uint32_t *mt, uint32_t *ma)
1404 {
1405 int rc = 0, cflen;
1406 u_int i, n;
1407 uint32_t param, val, addr, mtype, maddr;
1408 uint32_t off, mw_base, mw_aperture;
1409 const uint32_t *cfdata;
1410
1411 /* Figure out where the firmware wants us to upload it. */
1412 param = FW_PARAM_DEV(CF);
1413 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
1414 if (rc != 0) {
1415 /* Firmwares without config file support will fail this way */
1416 cxgb_printf(sc->dip, CE_WARN,
1417 "failed to query config file location: %d.\n", rc);
1418 return (rc);
1419 }
1420 *mt = mtype = G_FW_PARAMS_PARAM_Y(val);
1421 *ma = maddr = G_FW_PARAMS_PARAM_Z(val) << 16;
1422
1423 switch (CHELSIO_CHIP_VERSION(sc->params.chip)) {
1424 case CHELSIO_T4:
1425 cflen = t4cfg_size & ~3;
1426 /* LINTED: E_BAD_PTR_CAST_ALIGN */
1427 cfdata = (const uint32_t *)t4cfg_data;
1428 break;
1429 case CHELSIO_T5:
1430 cflen = t5cfg_size & ~3;
1431 /* LINTED: E_BAD_PTR_CAST_ALIGN */
1432 cfdata = (const uint32_t *)t5cfg_data;
1433 break;
1434 case CHELSIO_T6:
1435 cflen = t6cfg_size & ~3;
1436 /* LINTED: E_BAD_PTR_CAST_ALIGN */
1437 cfdata = (const uint32_t *)t6cfg_data;
1438 break;
1439 default:
1440 cxgb_printf(sc->dip, CE_WARN,
1441 "Invalid Adapter detected\n");
1442 return EINVAL;
1443 }
1444
1445 if (cflen > FLASH_CFG_MAX_SIZE) {
1446 cxgb_printf(sc->dip, CE_WARN,
1447 "config file too long (%d, max allowed is %d). ",
1448 cflen, FLASH_CFG_MAX_SIZE);
1449 return (EFBIG);
1450 }
1451
1452 rc = validate_mt_off_len(sc, mtype, maddr, cflen, &addr);
1453 if (rc != 0) {
1454
1455 cxgb_printf(sc->dip, CE_WARN,
1456 "%s: addr (%d/0x%x) or len %d is not valid: %d. "
1457 "Will try to use the config on the card, if any.\n",
1458 __func__, mtype, maddr, cflen, rc);
1459 return (EFAULT);
1460 }
1461
1462 memwin_info(sc, 2, &mw_base, &mw_aperture);
1463 while (cflen) {
1464 off = position_memwin(sc, 2, addr);
1465 n = min(cflen, mw_aperture - off);
1466 for (i = 0; i < n; i += 4)
1467 t4_write_reg(sc, mw_base + off + i, *cfdata++);
1468 cflen -= n;
1469 addr += n;
1470 }
1471
1472 return (rc);
1473 }
1474
1475 /*
1476 * Partition chip resources for use between various PFs, VFs, etc. This is done
1477 * by uploading the firmware configuration file to the adapter and instructing
1478 * the firmware to process it.
1479 */
1480 static int
1481 partition_resources(struct adapter *sc)
1482 {
1483 int rc;
1484 struct fw_caps_config_cmd caps;
1485 uint32_t mtype, maddr, finicsum, cfcsum;
1486
1487 rc = upload_config_file(sc, &mtype, &maddr);
1488 if (rc != 0) {
1489 mtype = FW_MEMTYPE_CF_FLASH;
1490 maddr = t4_flash_cfg_addr(sc);
1491 }
1492
1493 bzero(&caps, sizeof (caps));
1494 caps.op_to_write = BE_32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1495 F_FW_CMD_REQUEST | F_FW_CMD_READ);
1496 caps.cfvalid_to_len16 = BE_32(F_FW_CAPS_CONFIG_CMD_CFVALID |
1497 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
1498 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) | FW_LEN16(caps));
1499 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof (caps), &caps);
1500 if (rc != 0) {
1501 cxgb_printf(sc->dip, CE_WARN,
1502 "failed to pre-process config file: %d.\n", rc);
1503 return (rc);
1504 }
1505
1506 finicsum = ntohl(caps.finicsum);
1507 cfcsum = ntohl(caps.cfcsum);
1508 if (finicsum != cfcsum) {
1509 cxgb_printf(sc->dip, CE_WARN,
1510 "WARNING: config file checksum mismatch: %08x %08x\n",
1511 finicsum, cfcsum);
1512 }
1513 sc->cfcsum = cfcsum;
1514
1515 /* TODO: Need to configure this correctly */
1516 caps.toecaps = htons(FW_CAPS_CONFIG_TOE);
1517 caps.iscsicaps = 0;
1518 caps.rdmacaps = 0;
1519 caps.fcoecaps = 0;
1520 /* TODO: Disable VNIC cap for now */
1521 caps.niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
1522
1523 caps.op_to_write = htonl(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1524 F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
1525 caps.cfvalid_to_len16 = htonl(FW_LEN16(caps));
1526 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof (caps), NULL);
1527 if (rc != 0) {
1528 cxgb_printf(sc->dip, CE_WARN,
1529 "failed to process config file: %d.\n", rc);
1530 return (rc);
1531 }
1532
1533 return (0);
1534 }
1535
1536 /*
1537 * Tweak configuration based on module parameters, etc. Most of these have
1538 * defaults assigned to them by Firmware Configuration Files (if we're using
1539 * them) but need to be explicitly set if we're using hard-coded
1540 * initialization. But even in the case of using Firmware Configuration
1541 * Files, we'd like to expose the ability to change these via module
1542 * parameters so these are essentially common tweaks/settings for
1543 * Configuration Files and hard-coded initialization ...
1544 */
1545 static int
1546 adap__pre_init_tweaks(struct adapter *sc)
1547 {
1548 int rx_dma_offset = 2; /* Offset of RX packets into DMA buffers */
1549
1550 /*
1551 * Fix up various Host-Dependent Parameters like Page Size, Cache
1552 * Line Size, etc. The firmware default is for a 4KB Page Size and
1553 * 64B Cache Line Size ...
1554 */
1555 (void) t4_fixup_host_params_compat(sc, PAGE_SIZE, CACHE_LINE, T5_LAST_REV);
1556
1557 t4_set_reg_field(sc, A_SGE_CONTROL,
1558 V_PKTSHIFT(M_PKTSHIFT), V_PKTSHIFT(rx_dma_offset));
1559
1560 return 0;
1561 }
1562 /*
1563 * Retrieve parameters that are needed (or nice to have) prior to calling
1564 * t4_sge_init and t4_fw_initialize.
1565 */
1566 static int
1567 get_params__pre_init(struct adapter *sc)
1568 {
1569 int rc;
1570 uint32_t param[2], val[2];
1571 struct fw_devlog_cmd cmd;
1572 struct devlog_params *dlog = &sc->params.devlog;
1573
1574 /*
1575 * Grab the raw VPD parameters.
1576 */
1577 rc = -t4_get_raw_vpd_params(sc, &sc->params.vpd);
1578 if (rc != 0) {
1579 cxgb_printf(sc->dip, CE_WARN,
1580 "failed to query VPD parameters (pre_init): %d.\n", rc);
1581 return (rc);
1582 }
1583
1584 param[0] = FW_PARAM_DEV(PORTVEC);
1585 param[1] = FW_PARAM_DEV(CCLK);
1586 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
1587 if (rc != 0) {
1588 cxgb_printf(sc->dip, CE_WARN,
1589 "failed to query parameters (pre_init): %d.\n", rc);
1590 return (rc);
1591 }
1592
1593 sc->params.portvec = val[0];
1594 sc->params.nports = 0;
1595 while (val[0]) {
1596 sc->params.nports++;
1597 val[0] &= val[0] - 1;
1598 }
1599
1600 sc->params.vpd.cclk = val[1];
1601
1602 /* Read device log parameters. */
1603 bzero(&cmd, sizeof (cmd));
1604 cmd.op_to_write = htonl(V_FW_CMD_OP(FW_DEVLOG_CMD) |
1605 F_FW_CMD_REQUEST | F_FW_CMD_READ);
1606 cmd.retval_len16 = htonl(FW_LEN16(cmd));
1607 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof (cmd), &cmd);
1608 if (rc != 0) {
1609 cxgb_printf(sc->dip, CE_WARN,
1610 "failed to get devlog parameters: %d.\n", rc);
1611 bzero(dlog, sizeof (*dlog));
1612 rc = 0; /* devlog isn't critical for device operation */
1613 } else {
1614 val[0] = ntohl(cmd.memtype_devlog_memaddr16_devlog);
1615 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
1616 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
1617 dlog->size = ntohl(cmd.memsize_devlog);
1618 }
1619
1620 return (rc);
1621 }
1622
1623 /*
1624 * Retrieve various parameters that are of interest to the driver. The device
1625 * has been initialized by the firmware at this point.
1626 */
1627 static int
1628 get_params__post_init(struct adapter *sc)
1629 {
1630 int rc;
1631 uint32_t param[7], val[7];
1632 struct fw_caps_config_cmd caps;
1633
1634 param[0] = FW_PARAM_PFVF(IQFLINT_START);
1635 param[1] = FW_PARAM_PFVF(EQ_START);
1636 param[2] = FW_PARAM_PFVF(FILTER_START);
1637 param[3] = FW_PARAM_PFVF(FILTER_END);
1638 param[4] = FW_PARAM_PFVF(L2T_START);
1639 param[5] = FW_PARAM_PFVF(L2T_END);
1640 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
1641 if (rc != 0) {
1642 cxgb_printf(sc->dip, CE_WARN,
1643 "failed to query parameters (post_init): %d.\n", rc);
1644 return (rc);
1645 }
1646
1647 /* LINTED: E_ASSIGN_NARROW_CONV */
1648 sc->sge.iq_start = val[0];
1649 sc->sge.eq_start = val[1];
1650 sc->tids.ftid_base = val[2];
1651 sc->tids.nftids = val[3] - val[2] + 1;
1652 sc->vres.l2t.start = val[4];
1653 sc->vres.l2t.size = val[5] - val[4] + 1;
1654
1655 /* get capabilites */
1656 bzero(&caps, sizeof (caps));
1657 caps.op_to_write = htonl(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1658 F_FW_CMD_REQUEST | F_FW_CMD_READ);
1659 caps.cfvalid_to_len16 = htonl(FW_LEN16(caps));
1660 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof (caps), &caps);
1661 if (rc != 0) {
1662 cxgb_printf(sc->dip, CE_WARN,
1663 "failed to get card capabilities: %d.\n", rc);
1664 return (rc);
1665 }
1666
1667 if (caps.toecaps != 0) {
1668 /* query offload-related parameters */
1669 param[0] = FW_PARAM_DEV(NTID);
1670 param[1] = FW_PARAM_PFVF(SERVER_START);
1671 param[2] = FW_PARAM_PFVF(SERVER_END);
1672 param[3] = FW_PARAM_PFVF(TDDP_START);
1673 param[4] = FW_PARAM_PFVF(TDDP_END);
1674 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
1675 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
1676 if (rc != 0) {
1677 cxgb_printf(sc->dip, CE_WARN,
1678 "failed to query TOE parameters: %d.\n", rc);
1679 return (rc);
1680 }
1681 sc->tids.ntids = val[0];
1682 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
1683 sc->tids.stid_base = val[1];
1684 sc->tids.nstids = val[2] - val[1] + 1;
1685 sc->vres.ddp.start = val[3];
1686 sc->vres.ddp.size = val[4] - val[3] + 1;
1687 sc->params.ofldq_wr_cred = val[5];
1688 sc->params.offload = 1;
1689 }
1690
1691 /* These are finalized by FW initialization, load their values now */
1692 val[0] = t4_read_reg(sc, A_TP_TIMER_RESOLUTION);
1693 sc->params.tp.tre = G_TIMERRESOLUTION(val[0]);
1694 sc->params.tp.dack_re = G_DELAYEDACKRESOLUTION(val[0]);
1695 t4_read_mtu_tbl(sc, sc->params.mtus, NULL);
1696
1697 return (rc);
1698 }
1699
1700 static int
1701 set_params__post_init(struct adapter *sc)
1702 {
1703 uint32_t param, val;
1704
1705 /* ask for encapsulated CPLs */
1706 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
1707 val = 1;
1708 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
1709
1710 return (0);
1711 }
1712
1713 /* TODO: verify */
1714 static void
1715 setup_memwin(struct adapter *sc)
1716 {
1717 pci_regspec_t *data;
1718 int rc;
1719 uint_t n;
1720 uintptr_t bar0;
1721 uintptr_t mem_win0_base, mem_win1_base, mem_win2_base;
1722 uintptr_t mem_win2_aperture;
1723
1724 rc = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, sc->dip,
1725 DDI_PROP_DONTPASS, "assigned-addresses", (int **)&data, &n);
1726 if (rc != DDI_SUCCESS) {
1727 cxgb_printf(sc->dip, CE_WARN,
1728 "failed to lookup \"assigned-addresses\" property: %d", rc);
1729 return;
1730 }
1731 n /= sizeof (*data);
1732
1733 bar0 = ((uint64_t)data[0].pci_phys_mid << 32) | data[0].pci_phys_low;
1734 ddi_prop_free(data);
1735
1736 if (is_t4(sc->params.chip)) {
1737 mem_win0_base = bar0 + MEMWIN0_BASE;
1738 mem_win1_base = bar0 + MEMWIN1_BASE;
1739 mem_win2_base = bar0 + MEMWIN2_BASE;
1740 mem_win2_aperture = MEMWIN2_APERTURE;
1741 } else {
1742 /* For T5, only relative offset inside the PCIe BAR is passed */
1743 mem_win0_base = MEMWIN0_BASE;
1744 mem_win1_base = MEMWIN1_BASE;
1745 mem_win2_base = MEMWIN2_BASE_T5;
1746 mem_win2_aperture = MEMWIN2_APERTURE_T5;
1747 }
1748
1749 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 0),
1750 mem_win0_base | V_BIR(0) |
1751 V_WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
1752
1753 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 1),
1754 mem_win1_base | V_BIR(0) |
1755 V_WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
1756
1757 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2),
1758 mem_win2_base | V_BIR(0) |
1759 V_WINDOW(ilog2(mem_win2_aperture) - 10));
1760
1761 /* flush */
1762 (void)t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1763 }
1764
1765 /*
1766 * Positions the memory window such that it can be used to access the specified
1767 * address in the chip's address space. The return value is the offset of addr
1768 * from the start of the window.
1769 */
1770 uint32_t
1771 position_memwin(struct adapter *sc, int n, uint32_t addr)
1772 {
1773 uint32_t start, pf;
1774 uint32_t reg;
1775
1776 if (addr & 3) {
1777 cxgb_printf(sc->dip, CE_WARN,
1778 "addr (0x%x) is not at a 4B boundary.\n", addr);
1779 return (EFAULT);
1780 }
1781
1782 if (is_t4(sc->params.chip)) {
1783 pf = 0;
1784 start = addr & ~0xf; /* start must be 16B aligned */
1785 } else {
1786 pf = V_PFNUM(sc->pf);
1787 start = addr & ~0x7f; /* start must be 128B aligned */
1788 }
1789 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1790
1791 t4_write_reg(sc, reg, start | pf);
1792 (void) t4_read_reg(sc, reg);
1793
1794 return (addr - start);
1795 }
1796
1797
1798 /*
1799 * Reads the named property and fills up the "data" array (which has at least
1800 * "count" elements). We first try and lookup the property for our dev_t and
1801 * then retry with DDI_DEV_T_ANY if it's not found.
1802 *
1803 * Returns non-zero if the property was found and "data" has been updated.
1804 */
1805 static int
1806 prop_lookup_int_array(struct adapter *sc, char *name, int *data, uint_t count)
1807 {
1808 dev_info_t *dip = sc->dip;
1809 dev_t dev = sc->dev;
1810 int rc, *d;
1811 uint_t i, n;
1812
1813 rc = ddi_prop_lookup_int_array(dev, dip, DDI_PROP_DONTPASS,
1814 name, &d, &n);
1815 if (rc == DDI_PROP_SUCCESS)
1816 goto found;
1817
1818 if (rc != DDI_PROP_NOT_FOUND) {
1819 cxgb_printf(dip, CE_WARN,
1820 "failed to lookup property %s for minor %d: %d.",
1821 name, getminor(dev), rc);
1822 return (0);
1823 }
1824
1825 rc = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
1826 name, &d, &n);
1827 if (rc == DDI_PROP_SUCCESS)
1828 goto found;
1829
1830 if (rc != DDI_PROP_NOT_FOUND) {
1831 cxgb_printf(dip, CE_WARN,
1832 "failed to lookup property %s: %d.", name, rc);
1833 return (0);
1834 }
1835
1836 return (0);
1837
1838 found:
1839 if (n > count) {
1840 cxgb_printf(dip, CE_NOTE,
1841 "property %s has too many elements (%d), ignoring extras",
1842 name, n);
1843 }
1844
1845 for (i = 0; i < n && i < count; i++)
1846 data[i] = d[i];
1847 ddi_prop_free(d);
1848
1849 return (1);
1850 }
1851
1852 static int
1853 prop_lookup_int(struct adapter *sc, char *name, int defval)
1854 {
1855 int rc;
1856
1857 rc = ddi_prop_get_int(sc->dev, sc->dip, DDI_PROP_DONTPASS, name, -1);
1858 if (rc != -1)
1859 return (rc);
1860
1861 return (ddi_prop_get_int(DDI_DEV_T_ANY, sc->dip, DDI_PROP_DONTPASS,
1862 name, defval));
1863 }
1864
1865 static int
1866 init_driver_props(struct adapter *sc, struct driver_properties *p)
1867 {
1868 dev_t dev = sc->dev;
1869 dev_info_t *dip = sc->dip;
1870 int i, *data;
1871 uint_t tmr[SGE_NTIMERS] = {5, 10, 20, 50, 100, 200};
1872 uint_t cnt[SGE_NCOUNTERS] = {1, 8, 16, 32}; /* 63 max */
1873
1874 /*
1875 * Holdoff timer
1876 */
1877 data = &p->timer_val[0];
1878 for (i = 0; i < SGE_NTIMERS; i++)
1879 data[i] = tmr[i];
1880 (void) prop_lookup_int_array(sc, "holdoff-timer-values", data,
1881 SGE_NTIMERS);
1882 for (i = 0; i < SGE_NTIMERS; i++) {
1883 int limit = 200U;
1884 if (data[i] > limit) {
1885 cxgb_printf(dip, CE_WARN,
1886 "holdoff timer %d is too high (%d), lowered to %d.",
1887 i, data[i], limit);
1888 data[i] = limit;
1889 }
1890 }
1891 (void) ddi_prop_update_int_array(dev, dip, "holdoff-timer-values",
1892 data, SGE_NTIMERS);
1893
1894 /*
1895 * Holdoff packet counter
1896 */
1897 data = &p->counter_val[0];
1898 for (i = 0; i < SGE_NCOUNTERS; i++)
1899 data[i] = cnt[i];
1900 (void) prop_lookup_int_array(sc, "holdoff-pkt-counter-values", data,
1901 SGE_NCOUNTERS);
1902 for (i = 0; i < SGE_NCOUNTERS; i++) {
1903 int limit = M_THRESHOLD_0;
1904 if (data[i] > limit) {
1905 cxgb_printf(dip, CE_WARN,
1906 "holdoff pkt-counter %d is too high (%d), "
1907 "lowered to %d.", i, data[i], limit);
1908 data[i] = limit;
1909 }
1910 }
1911 (void) ddi_prop_update_int_array(dev, dip, "holdoff-pkt-counter-values",
1912 data, SGE_NCOUNTERS);
1913
1914 /*
1915 * Maximum # of tx and rx queues to use for each
1916 * 100G, 40G, 25G, 10G and 1G port.
1917 */
1918 p->max_ntxq_10g = prop_lookup_int(sc, "max-ntxq-10G-port", 8);
1919 (void) ddi_prop_update_int(dev, dip, "max-ntxq-10G-port",
1920 p->max_ntxq_10g);
1921
1922 p->max_nrxq_10g = prop_lookup_int(sc, "max-nrxq-10G-port", 8);
1923 (void) ddi_prop_update_int(dev, dip, "max-nrxq-10G-port",
1924 p->max_nrxq_10g);
1925
1926 p->max_ntxq_1g = prop_lookup_int(sc, "max-ntxq-1G-port", 2);
1927 (void) ddi_prop_update_int(dev, dip, "max-ntxq-1G-port",
1928 p->max_ntxq_1g);
1929
1930 p->max_nrxq_1g = prop_lookup_int(sc, "max-nrxq-1G-port", 2);
1931 (void) ddi_prop_update_int(dev, dip, "max-nrxq-1G-port",
1932 p->max_nrxq_1g);
1933
1934 #ifdef TCP_OFFLOAD_ENABLE
1935 p->max_nofldtxq_10g = prop_lookup_int(sc, "max-nofldtxq-10G-port", 8);
1936 (void) ddi_prop_update_int(dev, dip, "max-ntxq-10G-port",
1937 p->max_nofldtxq_10g);
1938
1939 p->max_nofldrxq_10g = prop_lookup_int(sc, "max-nofldrxq-10G-port", 2);
1940 (void) ddi_prop_update_int(dev, dip, "max-nrxq-10G-port",
1941 p->max_nofldrxq_10g);
1942
1943 p->max_nofldtxq_1g = prop_lookup_int(sc, "max-nofldtxq-1G-port", 2);
1944 (void) ddi_prop_update_int(dev, dip, "max-ntxq-1G-port",
1945 p->max_nofldtxq_1g);
1946
1947 p->max_nofldrxq_1g = prop_lookup_int(sc, "max-nofldrxq-1G-port", 1);
1948 (void) ddi_prop_update_int(dev, dip, "max-nrxq-1G-port",
1949 p->max_nofldrxq_1g);
1950 #endif
1951
1952 /*
1953 * Holdoff parameters for 10G and 1G ports.
1954 */
1955 p->tmr_idx_10g = prop_lookup_int(sc, "holdoff-timer-idx-10G", 0);
1956 (void) ddi_prop_update_int(dev, dip, "holdoff-timer-idx-10G",
1957 p->tmr_idx_10g);
1958
1959 p->pktc_idx_10g = prop_lookup_int(sc, "holdoff-pktc-idx-10G", 2);
1960 (void) ddi_prop_update_int(dev, dip, "holdoff-pktc-idx-10G",
1961 p->pktc_idx_10g);
1962
1963 p->tmr_idx_1g = prop_lookup_int(sc, "holdoff-timer-idx-1G", 0);
1964 (void) ddi_prop_update_int(dev, dip, "holdoff-timer-idx-1G",
1965 p->tmr_idx_1g);
1966
1967 p->pktc_idx_1g = prop_lookup_int(sc, "holdoff-pktc-idx-1G", 2);
1968 (void) ddi_prop_update_int(dev, dip, "holdoff-pktc-idx-1G",
1969 p->pktc_idx_1g);
1970
1971 /*
1972 * Size (number of entries) of each tx and rx queue.
1973 */
1974 i = prop_lookup_int(sc, "qsize-txq", TX_EQ_QSIZE);
1975 p->qsize_txq = max(i, 128);
1976 if (p->qsize_txq != i) {
1977 cxgb_printf(dip, CE_WARN,
1978 "using %d instead of %d as the tx queue size",
1979 p->qsize_txq, i);
1980 }
1981 (void) ddi_prop_update_int(dev, dip, "qsize-txq", p->qsize_txq);
1982
1983 i = prop_lookup_int(sc, "qsize-rxq", RX_IQ_QSIZE);
1984 p->qsize_rxq = max(i, 128);
1985 while (p->qsize_rxq & 7)
1986 p->qsize_rxq--;
1987 if (p->qsize_rxq != i) {
1988 cxgb_printf(dip, CE_WARN,
1989 "using %d instead of %d as the rx queue size",
1990 p->qsize_rxq, i);
1991 }
1992 (void) ddi_prop_update_int(dev, dip, "qsize-rxq", p->qsize_rxq);
1993
1994 /*
1995 * Interrupt types allowed.
1996 * Bits 0, 1, 2 = INTx, MSI, MSI-X respectively. See sys/ddi_intr.h
1997 */
1998 p->intr_types = prop_lookup_int(sc, "interrupt-types",
1999 DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI | DDI_INTR_TYPE_FIXED);
2000 (void) ddi_prop_update_int(dev, dip, "interrupt-types", p->intr_types);
2001
2002 /*
2003 * Forwarded interrupt queues. Create this property to force the driver
2004 * to use forwarded interrupt queues.
2005 */
2006 if (ddi_prop_exists(dev, dip, DDI_PROP_DONTPASS,
2007 "interrupt-forwarding") != 0 ||
2008 ddi_prop_exists(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
2009 "interrupt-forwarding") != 0) {
2010 UNIMPLEMENTED();
2011 (void) ddi_prop_create(dev, dip, DDI_PROP_CANSLEEP,
2012 "interrupt-forwarding", NULL, 0);
2013 }
2014
2015 /*
2016 * Write combining
2017 * 0 to disable, 1 to enable
2018 */
2019 p->wc = prop_lookup_int(sc, "write-combine", 1);
2020 cxgb_printf(dip, CE_WARN, "write-combine: using of %d", p->wc);
2021 if (p->wc != 0 && p->wc != 1) {
2022 cxgb_printf(dip, CE_WARN,
2023 "write-combine: using 1 instead of %d", p->wc);
2024 p->wc = 1;
2025 }
2026 (void) ddi_prop_update_int(dev, dip, "write-combine", p->wc);
2027
2028 p->t4_fw_install = prop_lookup_int(sc, "t4_fw_install", 1);
2029 if (p->t4_fw_install != 0 && p->t4_fw_install != 2)
2030 p->t4_fw_install = 1;
2031 (void) ddi_prop_update_int(dev, dip, "t4_fw_install", p->t4_fw_install);
2032
2033 /* Multiple Rings */
2034 p->multi_rings = prop_lookup_int(sc, "multi-rings", 1);
2035 if (p->multi_rings != 0 && p->multi_rings != 1) {
2036 cxgb_printf(dip, CE_NOTE,
2037 "multi-rings: using value 1 instead of %d", p->multi_rings);
2038 p->multi_rings = 1;
2039 }
2040
2041 (void) ddi_prop_update_int(dev, dip, "multi-rings", p->multi_rings);
2042
2043 return (0);
2044 }
2045
2046 static int
2047 remove_extra_props(struct adapter *sc, int n10g, int n1g)
2048 {
2049 if (n10g == 0) {
2050 (void) ddi_prop_remove(sc->dev, sc->dip, "max-ntxq-10G-port");
2051 (void) ddi_prop_remove(sc->dev, sc->dip, "max-nrxq-10G-port");
2052 (void) ddi_prop_remove(sc->dev, sc->dip,
2053 "holdoff-timer-idx-10G");
2054 (void) ddi_prop_remove(sc->dev, sc->dip,
2055 "holdoff-pktc-idx-10G");
2056 }
2057
2058 if (n1g == 0) {
2059 (void) ddi_prop_remove(sc->dev, sc->dip, "max-ntxq-1G-port");
2060 (void) ddi_prop_remove(sc->dev, sc->dip, "max-nrxq-1G-port");
2061 (void) ddi_prop_remove(sc->dev, sc->dip,
2062 "holdoff-timer-idx-1G");
2063 (void) ddi_prop_remove(sc->dev, sc->dip, "holdoff-pktc-idx-1G");
2064 }
2065
2066 return (0);
2067 }
2068
2069 static int
2070 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
2071 struct intrs_and_queues *iaq)
2072 {
2073 struct driver_properties *p = &sc->props;
2074 int rc, itype, itypes, navail, nc, nrxq10g, nrxq1g, n;
2075 int nofldrxq10g = 0, nofldrxq1g = 0;
2076
2077 bzero(iaq, sizeof (*iaq));
2078 nc = ncpus; /* our snapshot of the number of CPUs */
2079 iaq->ntxq10g = min(nc, p->max_ntxq_10g);
2080 iaq->ntxq1g = min(nc, p->max_ntxq_1g);
2081 iaq->nrxq10g = nrxq10g = min(nc, p->max_nrxq_10g);
2082 iaq->nrxq1g = nrxq1g = min(nc, p->max_nrxq_1g);
2083 #ifdef TCP_OFFLOAD_ENABLE
2084 iaq->nofldtxq10g = min(nc, p->max_nofldtxq_10g);
2085 iaq->nofldtxq1g = min(nc, p->max_nofldtxq_1g);
2086 iaq->nofldrxq10g = nofldrxq10g = min(nc, p->max_nofldrxq_10g);
2087 iaq->nofldrxq1g = nofldrxq1g = min(nc, p->max_nofldrxq_1g);
2088 #endif
2089
2090 rc = ddi_intr_get_supported_types(sc->dip, &itypes);
2091 if (rc != DDI_SUCCESS) {
2092 cxgb_printf(sc->dip, CE_WARN,
2093 "failed to determine supported interrupt types: %d", rc);
2094 return (rc);
2095 }
2096
2097 for (itype = DDI_INTR_TYPE_MSIX; itype; itype >>= 1) {
2098 ASSERT(itype == DDI_INTR_TYPE_MSIX ||
2099 itype == DDI_INTR_TYPE_MSI ||
2100 itype == DDI_INTR_TYPE_FIXED);
2101
2102 if ((itype & itypes & p->intr_types) == 0)
2103 continue; /* not supported or not allowed */
2104
2105 navail = 0;
2106 rc = ddi_intr_get_navail(sc->dip, itype, &navail);
2107 if (rc != DDI_SUCCESS || navail == 0) {
2108 cxgb_printf(sc->dip, CE_WARN,
2109 "failed to get # of interrupts for type %d: %d",
2110 itype, rc);
2111 continue; /* carry on */
2112 }
2113
2114 iaq->intr_type = itype;
2115 if (navail == 0)
2116 continue;
2117
2118 /*
2119 * Best option: an interrupt vector for errors, one for the
2120 * firmware event queue, and one each for each rxq (NIC as well
2121 * as offload).
2122 */
2123 iaq->nirq = T4_EXTRA_INTR;
2124 iaq->nirq += n10g * (nrxq10g + nofldrxq10g);
2125 iaq->nirq += n1g * (nrxq1g + nofldrxq1g);
2126
2127 if (iaq->nirq <= navail &&
2128 (itype != DDI_INTR_TYPE_MSI || ISP2(iaq->nirq))) {
2129 iaq->intr_fwd = 0;
2130 goto allocate;
2131 }
2132
2133 /*
2134 * Second best option: an interrupt vector for errors, one for
2135 * the firmware event queue, and one each for either NIC or
2136 * offload rxq's.
2137 */
2138 iaq->nirq = T4_EXTRA_INTR;
2139 iaq->nirq += n10g * max(nrxq10g, nofldrxq10g);
2140 iaq->nirq += n1g * max(nrxq1g, nofldrxq1g);
2141 if (iaq->nirq <= navail &&
2142 (itype != DDI_INTR_TYPE_MSI || ISP2(iaq->nirq))) {
2143 iaq->intr_fwd = 1;
2144 goto allocate;
2145 }
2146
2147 /*
2148 * Next best option: an interrupt vector for errors, one for the
2149 * firmware event queue, and at least one per port. At this
2150 * point we know we'll have to downsize nrxq or nofldrxq to fit
2151 * what's available to us.
2152 */
2153 iaq->nirq = T4_EXTRA_INTR;
2154 iaq->nirq += n10g + n1g;
2155 if (iaq->nirq <= navail) {
2156 int leftover = navail - iaq->nirq;
2157
2158 if (n10g > 0) {
2159 int target = max(nrxq10g, nofldrxq10g);
2160
2161 n = 1;
2162 while (n < target && leftover >= n10g) {
2163 leftover -= n10g;
2164 iaq->nirq += n10g;
2165 n++;
2166 }
2167 iaq->nrxq10g = min(n, nrxq10g);
2168 #ifdef TCP_OFFLOAD_ENABLE
2169 iaq->nofldrxq10g = min(n, nofldrxq10g);
2170 #endif
2171 }
2172
2173 if (n1g > 0) {
2174 int target = max(nrxq1g, nofldrxq1g);
2175
2176 n = 1;
2177 while (n < target && leftover >= n1g) {
2178 leftover -= n1g;
2179 iaq->nirq += n1g;
2180 n++;
2181 }
2182 iaq->nrxq1g = min(n, nrxq1g);
2183 #ifdef TCP_OFFLOAD_ENABLE
2184 iaq->nofldrxq1g = min(n, nofldrxq1g);
2185 #endif
2186 }
2187
2188 /* We have arrived at a minimum value required to enable
2189 * per queue irq(either NIC or offload). Thus for non-
2190 * offload case, we will get a vector per queue, while
2191 * offload case, we will get a vector per offload/NIC q.
2192 * Hence enable Interrupt forwarding only for offload
2193 * case.
2194 */
2195 #ifdef TCP_OFFLOAD_ENABLE
2196 if (itype != DDI_INTR_TYPE_MSI || ISP2(iaq->nirq)) {
2197 iaq->intr_fwd = 1;
2198 #else
2199 if (itype != DDI_INTR_TYPE_MSI) {
2200 #endif
2201 goto allocate;
2202 }
2203 }
2204
2205 /*
2206 * Least desirable option: one interrupt vector for everything.
2207 */
2208 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
2209 #ifdef TCP_OFFLOAD_ENABLE
2210 iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2211 #endif
2212 iaq->intr_fwd = 1;
2213
2214 allocate:
2215 return (0);
2216 }
2217
2218 cxgb_printf(sc->dip, CE_WARN,
2219 "failed to find a usable interrupt type. supported=%d, allowed=%d",
2220 itypes, p->intr_types);
2221 return (DDI_FAILURE);
2222 }
2223
2224 static int
2225 add_child_node(struct adapter *sc, int idx)
2226 {
2227 int rc;
2228 struct port_info *pi;
2229
2230 if (idx < 0 || idx >= sc->params.nports)
2231 return (EINVAL);
2232
2233 pi = sc->port[idx];
2234 if (pi == NULL)
2235 return (ENODEV); /* t4_port_init failed earlier */
2236
2237 PORT_LOCK(pi);
2238 if (pi->dip != NULL) {
2239 rc = 0; /* EEXIST really, but then bus_config fails */
2240 goto done;
2241 }
2242
2243 rc = ndi_devi_alloc(sc->dip, T4_PORT_NAME, DEVI_SID_NODEID, &pi->dip);
2244 if (rc != DDI_SUCCESS || pi->dip == NULL) {
2245 rc = ENOMEM;
2246 goto done;
2247 }
2248
2249 (void) ddi_set_parent_data(pi->dip, pi);
2250 (void) ndi_devi_bind_driver(pi->dip, 0);
2251 rc = 0;
2252 done:
2253 PORT_UNLOCK(pi);
2254 return (rc);
2255 }
2256
2257 static int
2258 remove_child_node(struct adapter *sc, int idx)
2259 {
2260 int rc;
2261 struct port_info *pi;
2262
2263 if (idx < 0 || idx >= sc->params.nports)
2264 return (EINVAL);
2265
2266 pi = sc->port[idx];
2267 if (pi == NULL)
2268 return (ENODEV);
2269
2270 PORT_LOCK(pi);
2271 if (pi->dip == NULL) {
2272 rc = ENODEV;
2273 goto done;
2274 }
2275
2276 rc = ndi_devi_free(pi->dip);
2277 if (rc == 0)
2278 pi->dip = NULL;
2279 done:
2280 PORT_UNLOCK(pi);
2281 return (rc);
2282 }
2283
2284 #define KS_UINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_ULONG)
2285 #define KS_CINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_CHAR)
2286 #define KS_U_SET(x, y) kstatp->x.value.ul = (y)
2287 #define KS_C_SET(x, ...) \
2288 (void) snprintf(kstatp->x.value.c, 16, __VA_ARGS__)
2289
2290 /*
2291 * t4nex:X:config
2292 */
2293 struct t4_kstats {
2294 kstat_named_t chip_ver;
2295 kstat_named_t fw_vers;
2296 kstat_named_t tp_vers;
2297 kstat_named_t driver_version;
2298 kstat_named_t serial_number;
2299 kstat_named_t ec_level;
2300 kstat_named_t id;
2301 kstat_named_t bus_type;
2302 kstat_named_t bus_width;
2303 kstat_named_t bus_speed;
2304 kstat_named_t core_clock;
2305 kstat_named_t port_cnt;
2306 kstat_named_t port_type;
2307 kstat_named_t pci_vendor_id;
2308 kstat_named_t pci_device_id;
2309 };
2310 static kstat_t *
2311 setup_kstats(struct adapter *sc)
2312 {
2313 kstat_t *ksp;
2314 struct t4_kstats *kstatp;
2315 int ndata;
2316 struct pci_params *p = &sc->params.pci;
2317 struct vpd_params *v = &sc->params.vpd;
2318 uint16_t pci_vendor, pci_device;
2319
2320 ndata = sizeof (struct t4_kstats) / sizeof (kstat_named_t);
2321
2322 ksp = kstat_create(T4_NEXUS_NAME, ddi_get_instance(sc->dip), "config",
2323 "nexus", KSTAT_TYPE_NAMED, ndata, 0);
2324 if (ksp == NULL) {
2325 cxgb_printf(sc->dip, CE_WARN, "failed to initialize kstats.");
2326 return (NULL);
2327 }
2328
2329 kstatp = (struct t4_kstats *)ksp->ks_data;
2330
2331 KS_UINIT(chip_ver);
2332 KS_CINIT(fw_vers);
2333 KS_CINIT(tp_vers);
2334 KS_CINIT(driver_version);
2335 KS_CINIT(serial_number);
2336 KS_CINIT(ec_level);
2337 KS_CINIT(id);
2338 KS_CINIT(bus_type);
2339 KS_CINIT(bus_width);
2340 KS_CINIT(bus_speed);
2341 KS_UINIT(core_clock);
2342 KS_UINIT(port_cnt);
2343 KS_CINIT(port_type);
2344 KS_CINIT(pci_vendor_id);
2345 KS_CINIT(pci_device_id);
2346
2347 KS_U_SET(chip_ver, sc->params.chip);
2348 KS_C_SET(fw_vers, "%d.%d.%d.%d",
2349 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2350 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2351 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2352 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2353 KS_C_SET(tp_vers, "%d.%d.%d.%d",
2354 G_FW_HDR_FW_VER_MAJOR(sc->params.tp_vers),
2355 G_FW_HDR_FW_VER_MINOR(sc->params.tp_vers),
2356 G_FW_HDR_FW_VER_MICRO(sc->params.tp_vers),
2357 G_FW_HDR_FW_VER_BUILD(sc->params.tp_vers));
2358 KS_C_SET(driver_version, DRV_VERSION);
2359 KS_C_SET(serial_number, "%s", v->sn);
2360 KS_C_SET(ec_level, "%s", v->ec);
2361 KS_C_SET(id, "%s", v->id);
2362 KS_C_SET(bus_type, "pci-express");
2363 KS_C_SET(bus_width, "x%d lanes", p->width);
2364 KS_C_SET(bus_speed, "%d", p->speed);
2365 KS_U_SET(core_clock, v->cclk);
2366 KS_U_SET(port_cnt, sc->params.nports);
2367
2368 t4_os_pci_read_cfg2(sc, PCI_CONF_VENID, &pci_vendor);
2369 KS_C_SET(pci_vendor_id, "0x%x", pci_vendor);
2370
2371 t4_os_pci_read_cfg2(sc, PCI_CONF_DEVID, &pci_device);
2372 KS_C_SET(pci_device_id, "0x%x", pci_device);
2373
2374 KS_C_SET(port_type, "%s/%s/%s/%s",
2375 print_port_speed(sc->port[0]),
2376 print_port_speed(sc->port[1]),
2377 print_port_speed(sc->port[2]),
2378 print_port_speed(sc->port[3]));
2379
2380 /* Do NOT set ksp->ks_update. These kstats do not change. */
2381
2382 /* Install the kstat */
2383 ksp->ks_private = (void *)sc;
2384 kstat_install(ksp);
2385
2386 return (ksp);
2387 }
2388
2389 /*
2390 * t4nex:X:stat
2391 */
2392 struct t4_wc_kstats {
2393 kstat_named_t write_coal_success;
2394 kstat_named_t write_coal_failure;
2395 };
2396 static kstat_t *
2397 setup_wc_kstats(struct adapter *sc)
2398 {
2399 kstat_t *ksp;
2400 struct t4_wc_kstats *kstatp;
2401 int ndata;
2402
2403 ndata = sizeof(struct t4_wc_kstats) / sizeof(kstat_named_t);
2404 ksp = kstat_create(T4_NEXUS_NAME, ddi_get_instance(sc->dip), "stats",
2405 "nexus", KSTAT_TYPE_NAMED, ndata, 0);
2406 if (ksp == NULL) {
2407 cxgb_printf(sc->dip, CE_WARN, "failed to initialize kstats.");
2408 return (NULL);
2409 }
2410
2411 kstatp = (struct t4_wc_kstats *)ksp->ks_data;
2412
2413 KS_UINIT(write_coal_success);
2414 KS_UINIT(write_coal_failure);
2415
2416 ksp->ks_update = update_wc_kstats;
2417 /* Install the kstat */
2418 ksp->ks_private = (void *)sc;
2419 kstat_install(ksp);
2420
2421 return (ksp);
2422 }
2423
2424 static int
2425 update_wc_kstats(kstat_t *ksp, int rw)
2426 {
2427 struct t4_wc_kstats *kstatp = (struct t4_wc_kstats *)ksp->ks_data;
2428 struct adapter *sc = ksp->ks_private;
2429 uint32_t wc_total, wc_success, wc_failure;
2430
2431 if (rw == KSTAT_WRITE)
2432 return (0);
2433
2434 if (is_t5(sc->params.chip)) {
2435 wc_total = t4_read_reg(sc, A_SGE_STAT_TOTAL);
2436 wc_failure = t4_read_reg(sc, A_SGE_STAT_MATCH);
2437 wc_success = wc_total - wc_failure;
2438 } else {
2439 wc_success = 0;
2440 wc_failure = 0;
2441 }
2442
2443 KS_U_SET(write_coal_success, wc_success);
2444 KS_U_SET(write_coal_failure, wc_failure);
2445
2446 return (0);
2447 }
2448
2449 int
2450 adapter_full_init(struct adapter *sc)
2451 {
2452 int i, rc = 0;
2453
2454 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2455
2456 rc = t4_setup_adapter_queues(sc);
2457 if (rc != 0)
2458 goto done;
2459
2460 if (sc->intr_cap & DDI_INTR_FLAG_BLOCK)
2461 (void) ddi_intr_block_enable(sc->intr_handle, sc->intr_count);
2462 else {
2463 for (i = 0; i < sc->intr_count; i++)
2464 (void) ddi_intr_enable(sc->intr_handle[i]);
2465 }
2466 t4_intr_enable(sc);
2467 sc->flags |= FULL_INIT_DONE;
2468
2469 #ifdef TCP_OFFLOAD_ENABLE
2470 /* TODO: wrong place to enable TOE capability */
2471 if (is_offload(sc) != 0) {
2472 for_each_port(sc, i) {
2473 struct port_info *pi = sc->port[i];
2474 rc = toe_capability(pi, 1);
2475 if (rc != 0) {
2476 cxgb_printf(pi->dip, CE_WARN,
2477 "Failed to activate toe capability: %d",
2478 rc);
2479 rc = 0; /* not a fatal error */
2480 }
2481 }
2482 }
2483 #endif
2484
2485 done:
2486 if (rc != 0)
2487 (void) adapter_full_uninit(sc);
2488
2489 return (rc);
2490 }
2491
2492 int
2493 adapter_full_uninit(struct adapter *sc)
2494 {
2495 int i, rc = 0;
2496
2497 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2498
2499 if (sc->intr_cap & DDI_INTR_FLAG_BLOCK)
2500 (void) ddi_intr_block_disable(sc->intr_handle, sc->intr_count);
2501 else {
2502 for (i = 0; i < sc->intr_count; i++)
2503 (void) ddi_intr_disable(sc->intr_handle[i]);
2504 }
2505
2506 rc = t4_teardown_adapter_queues(sc);
2507 if (rc != 0)
2508 return (rc);
2509
2510 sc->flags &= ~FULL_INIT_DONE;
2511
2512 return (0);
2513 }
2514
2515 int
2516 port_full_init(struct port_info *pi)
2517 {
2518 struct adapter *sc = pi->adapter;
2519 uint16_t *rss;
2520 struct sge_rxq *rxq;
2521 int rc, i;
2522
2523 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2524 ASSERT((pi->flags & PORT_INIT_DONE) == 0);
2525
2526 /*
2527 * Allocate tx/rx/fl queues for this port.
2528 */
2529 rc = t4_setup_port_queues(pi);
2530 if (rc != 0)
2531 goto done; /* error message displayed already */
2532
2533 /*
2534 * Setup RSS for this port.
2535 */
2536 rss = kmem_zalloc(pi->nrxq * sizeof (*rss), KM_SLEEP);
2537 for_each_rxq(pi, i, rxq) {
2538 rss[i] = rxq->iq.abs_id;
2539 }
2540 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0,
2541 pi->rss_size, rss, pi->nrxq);
2542 kmem_free(rss, pi->nrxq * sizeof (*rss));
2543 if (rc != 0) {
2544 cxgb_printf(pi->dip, CE_WARN, "rss_config failed: %d", rc);
2545 goto done;
2546 }
2547
2548 pi->flags |= PORT_INIT_DONE;
2549 done:
2550 if (rc != 0)
2551 (void) port_full_uninit(pi);
2552
2553 return (rc);
2554 }
2555
2556 /*
2557 * Idempotent.
2558 */
2559 int
2560 port_full_uninit(struct port_info *pi)
2561 {
2562
2563 ASSERT(pi->flags & PORT_INIT_DONE);
2564
2565 (void) t4_teardown_port_queues(pi);
2566 pi->flags &= ~PORT_INIT_DONE;
2567
2568 return (0);
2569 }
2570
2571 void
2572 enable_port_queues(struct port_info *pi)
2573 {
2574 struct adapter *sc = pi->adapter;
2575 int i;
2576 struct sge_iq *iq;
2577 struct sge_rxq *rxq;
2578 #ifdef TCP_OFFLOAD_ENABLE
2579 struct sge_ofld_rxq *ofld_rxq;
2580 #endif
2581
2582 ASSERT(pi->flags & PORT_INIT_DONE);
2583
2584 /*
2585 * TODO: whatever was queued up after we set iq->state to IQS_DISABLED
2586 * back in disable_port_queues will be processed now, after an unbounded
2587 * delay. This can't be good.
2588 */
2589
2590 #ifdef TCP_OFFLOAD_ENABLE
2591 for_each_ofld_rxq(pi, i, ofld_rxq) {
2592 iq = &ofld_rxq->iq;
2593 if (atomic_cas_uint(&iq->state, IQS_DISABLED, IQS_IDLE) !=
2594 IQS_DISABLED)
2595 panic("%s: iq %p wasn't disabled", __func__,
2596 (void *)iq);
2597 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
2598 V_SEINTARM(iq->intr_params) | V_INGRESSQID(iq->cntxt_id));
2599 }
2600 #endif
2601
2602 for_each_rxq(pi, i, rxq) {
2603 iq = &rxq->iq;
2604 if (atomic_cas_uint(&iq->state, IQS_DISABLED, IQS_IDLE) !=
2605 IQS_DISABLED)
2606 panic("%s: iq %p wasn't disabled", __func__,
2607 (void *) iq);
2608 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
2609 V_SEINTARM(iq->intr_params) | V_INGRESSQID(iq->cntxt_id));
2610 }
2611 }
2612
2613 void
2614 disable_port_queues(struct port_info *pi)
2615 {
2616 int i;
2617 struct adapter *sc = pi->adapter;
2618 struct sge_rxq *rxq;
2619 #ifdef TCP_OFFLOAD_ENABLE
2620 struct sge_ofld_rxq *ofld_rxq;
2621 #endif
2622
2623 ASSERT(pi->flags & PORT_INIT_DONE);
2624
2625 /*
2626 * TODO: need proper implementation for all tx queues (ctrl, eth, ofld).
2627 */
2628
2629 #ifdef TCP_OFFLOAD_ENABLE
2630 for_each_ofld_rxq(pi, i, ofld_rxq) {
2631 while (atomic_cas_uint(&ofld_rxq->iq.state, IQS_IDLE,
2632 IQS_DISABLED) != IQS_IDLE)
2633 msleep(1);
2634 }
2635 #endif
2636
2637 for_each_rxq(pi, i, rxq) {
2638 while (atomic_cas_uint(&rxq->iq.state, IQS_IDLE,
2639 IQS_DISABLED) != IQS_IDLE)
2640 msleep(1);
2641 }
2642
2643 mutex_enter(&sc->sfl_lock);
2644 #ifdef TCP_OFFLOAD_ENABLE
2645 for_each_ofld_rxq(pi, i, ofld_rxq)
2646 ofld_rxq->fl.flags |= FL_DOOMED;
2647 #endif
2648 for_each_rxq(pi, i, rxq)
2649 rxq->fl.flags |= FL_DOOMED;
2650 mutex_exit(&sc->sfl_lock);
2651 /* TODO: need to wait for all fl's to be removed from sc->sfl */
2652 }
2653
2654 void
2655 t4_fatal_err(struct adapter *sc)
2656 {
2657 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
2658 t4_intr_disable(sc);
2659 cxgb_printf(sc->dip, CE_WARN,
2660 "encountered fatal error, adapter stopped.");
2661 }
2662
2663 int
2664 t4_os_find_pci_capability(struct adapter *sc, int cap)
2665 {
2666 uint16_t stat;
2667 uint8_t cap_ptr, cap_id;
2668
2669 t4_os_pci_read_cfg2(sc, PCI_CONF_STAT, &stat);
2670 if ((stat & PCI_STAT_CAP) == 0)
2671 return (0); /* does not implement capabilities */
2672
2673 t4_os_pci_read_cfg1(sc, PCI_CONF_CAP_PTR, &cap_ptr);
2674 while (cap_ptr) {
2675 t4_os_pci_read_cfg1(sc, cap_ptr + PCI_CAP_ID, &cap_id);
2676 if (cap_id == cap)
2677 return (cap_ptr); /* found */
2678 t4_os_pci_read_cfg1(sc, cap_ptr + PCI_CAP_NEXT_PTR, &cap_ptr);
2679 }
2680
2681 return (0); /* not found */
2682 }
2683
2684 void
2685 t4_os_portmod_changed(const struct adapter *sc, int idx)
2686 {
2687 static const char *mod_str[] = {
2688 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
2689 };
2690 const struct port_info *pi = sc->port[idx];
2691
2692 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
2693 cxgb_printf(pi->dip, CE_NOTE, "transceiver unplugged.");
2694 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
2695 cxgb_printf(pi->dip, CE_NOTE,
2696 "unknown transceiver inserted.\n");
2697 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
2698 cxgb_printf(pi->dip, CE_NOTE,
2699 "unsupported transceiver inserted.\n");
2700 else if (pi->mod_type > 0 && pi->mod_type < ARRAY_SIZE(mod_str))
2701 cxgb_printf(pi->dip, CE_NOTE, "%s transceiver inserted.\n",
2702 mod_str[pi->mod_type]);
2703 else
2704 cxgb_printf(pi->dip, CE_NOTE, "transceiver (type %d) inserted.",
2705 pi->mod_type);
2706 }
2707
2708 /* ARGSUSED */
2709 static int
2710 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m)
2711 {
2712 if (m != NULL)
2713 freemsg(m);
2714 return (0);
2715 }
2716
2717 int
2718 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
2719 {
2720 uint_t *loc, new;
2721
2722 if (opcode >= ARRAY_SIZE(sc->cpl_handler))
2723 return (EINVAL);
2724
2725 new = (uint_t)(unsigned long) (h ? h : cpl_not_handled);
2726 loc = (uint_t *)&sc->cpl_handler[opcode];
2727 (void) atomic_swap_uint(loc, new);
2728
2729 return (0);
2730 }
2731
2732 static int
2733 fw_msg_not_handled(struct adapter *sc, const __be64 *data)
2734 {
2735 struct cpl_fw6_msg *cpl;
2736
2737 cpl = __containerof((void *)data, struct cpl_fw6_msg, data);
2738
2739 cxgb_printf(sc->dip, CE_WARN, "%s fw_msg type %d", __func__, cpl->type);
2740 return (0);
2741 }
2742
2743 int
2744 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
2745 {
2746 fw_msg_handler_t *loc, new;
2747
2748 if (type >= ARRAY_SIZE(sc->fw_msg_handler))
2749 return (EINVAL);
2750
2751 /*
2752 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
2753 * handler dispatch table. Reject any attempt to install a handler for
2754 * this subtype.
2755 */
2756 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
2757 return (EINVAL);
2758
2759 new = h ? h : fw_msg_not_handled;
2760 loc = &sc->fw_msg_handler[type];
2761 (void)atomic_swap_ptr(loc, (void *)new);
2762
2763 return (0);
2764 }
2765
2766 #ifdef TCP_OFFLOAD_ENABLE
2767 static int
2768 toe_capability(struct port_info *pi, int enable)
2769 {
2770 int rc;
2771 struct adapter *sc = pi->adapter;
2772
2773 if (!is_offload(sc))
2774 return (ENODEV);
2775
2776 if (enable != 0) {
2777 if (isset(&sc->offload_map, pi->port_id) != 0)
2778 return (0);
2779
2780 if (sc->offload_map == 0) {
2781 rc = activate_uld(sc, ULD_TOM, &sc->tom);
2782 if (rc != 0)
2783 return (rc);
2784 }
2785
2786 setbit(&sc->offload_map, pi->port_id);
2787 } else {
2788 if (!isset(&sc->offload_map, pi->port_id))
2789 return (0);
2790
2791 clrbit(&sc->offload_map, pi->port_id);
2792
2793 if (sc->offload_map == 0) {
2794 rc = deactivate_uld(&sc->tom);
2795 if (rc != 0) {
2796 setbit(&sc->offload_map, pi->port_id);
2797 return (rc);
2798 }
2799 }
2800 }
2801
2802 return (0);
2803 }
2804
2805 /*
2806 * Add an upper layer driver to the global list.
2807 */
2808 int
2809 t4_register_uld(struct uld_info *ui)
2810 {
2811 int rc = 0;
2812 struct uld_info *u;
2813
2814 mutex_enter(&t4_uld_list_lock);
2815 SLIST_FOREACH(u, &t4_uld_list, link) {
2816 if (u->uld_id == ui->uld_id) {
2817 rc = EEXIST;
2818 goto done;
2819 }
2820 }
2821
2822 SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
2823 ui->refcount = 0;
2824 done:
2825 mutex_exit(&t4_uld_list_lock);
2826 return (rc);
2827 }
2828
2829 int
2830 t4_unregister_uld(struct uld_info *ui)
2831 {
2832 int rc = EINVAL;
2833 struct uld_info *u;
2834
2835 mutex_enter(&t4_uld_list_lock);
2836
2837 SLIST_FOREACH(u, &t4_uld_list, link) {
2838 if (u == ui) {
2839 if (ui->refcount > 0) {
2840 rc = EBUSY;
2841 goto done;
2842 }
2843
2844 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
2845 rc = 0;
2846 goto done;
2847 }
2848 }
2849 done:
2850 mutex_exit(&t4_uld_list_lock);
2851 return (rc);
2852 }
2853
2854 static int
2855 activate_uld(struct adapter *sc, int id, struct uld_softc *usc)
2856 {
2857 int rc = EAGAIN;
2858 struct uld_info *ui;
2859
2860 mutex_enter(&t4_uld_list_lock);
2861
2862 SLIST_FOREACH(ui, &t4_uld_list, link) {
2863 if (ui->uld_id == id) {
2864 rc = ui->attach(sc, &usc->softc);
2865 if (rc == 0) {
2866 ASSERT(usc->softc != NULL);
2867 ui->refcount++;
2868 usc->uld = ui;
2869 }
2870 goto done;
2871 }
2872 }
2873 done:
2874 mutex_exit(&t4_uld_list_lock);
2875
2876 return (rc);
2877 }
2878
2879 static int
2880 deactivate_uld(struct uld_softc *usc)
2881 {
2882 int rc;
2883
2884 mutex_enter(&t4_uld_list_lock);
2885
2886 if (usc->uld == NULL || usc->softc == NULL) {
2887 rc = EINVAL;
2888 goto done;
2889 }
2890
2891 rc = usc->uld->detach(usc->softc);
2892 if (rc == 0) {
2893 ASSERT(usc->uld->refcount > 0);
2894 usc->uld->refcount--;
2895 usc->uld = NULL;
2896 usc->softc = NULL;
2897 }
2898 done:
2899 mutex_exit(&t4_uld_list_lock);
2900
2901 return (rc);
2902 }
2903
2904 void
2905 t4_iterate(void (*func)(int, void *), void *arg)
2906 {
2907 struct adapter *sc;
2908
2909 mutex_enter(&t4_adapter_list_lock);
2910 SLIST_FOREACH(sc, &t4_adapter_list, link) {
2911 /*
2912 * func should not make any assumptions about what state sc is
2913 * in - the only guarantee is that sc->sc_lock is a valid lock.
2914 */
2915 func(ddi_get_instance(sc->dip), arg);
2916 }
2917 mutex_exit(&t4_adapter_list_lock);
2918 }
2919
2920 #endif