1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright(c) 2007-2010 Intel Corporation. All rights reserved.
24 */
25
26 /*
27 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
28 * Copyright (c) 2012, Joyent, Inc. All rights reserved.
29 * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
30 * Copyright (c) 2013 Saso Kiselkov. All rights reserved.
31 * Copyright (c) 2013 OSN Online Service Nuernberg GmbH. All rights reserved.
32 * Copyright 2016 OmniTI Computer Consulting, Inc. All rights reserved.
33 */
34
35 #include "ixgbe_sw.h"
36
37 static char ixgbe_ident[] = "Intel 10Gb Ethernet";
38 /* LINTED E_STATIC_UNUSED */
39 static char ixgbe_version[] = "ixgbe 1.1.7";
40
41 /*
42 * Local function protoypes
43 */
44 static int ixgbe_register_mac(ixgbe_t *);
45 static int ixgbe_identify_hardware(ixgbe_t *);
46 static int ixgbe_regs_map(ixgbe_t *);
47 static void ixgbe_init_properties(ixgbe_t *);
48 static int ixgbe_init_driver_settings(ixgbe_t *);
49 static void ixgbe_init_locks(ixgbe_t *);
50 static void ixgbe_destroy_locks(ixgbe_t *);
51 static int ixgbe_init(ixgbe_t *);
52 static int ixgbe_chip_start(ixgbe_t *);
53 static void ixgbe_chip_stop(ixgbe_t *);
54 static int ixgbe_reset(ixgbe_t *);
55 static void ixgbe_tx_clean(ixgbe_t *);
56 static boolean_t ixgbe_tx_drain(ixgbe_t *);
57 static boolean_t ixgbe_rx_drain(ixgbe_t *);
58 static int ixgbe_alloc_rings(ixgbe_t *);
59 static void ixgbe_free_rings(ixgbe_t *);
60 static int ixgbe_alloc_rx_data(ixgbe_t *);
61 static void ixgbe_free_rx_data(ixgbe_t *);
62 static void ixgbe_setup_rings(ixgbe_t *);
63 static void ixgbe_setup_rx(ixgbe_t *);
64 static void ixgbe_setup_tx(ixgbe_t *);
65 static void ixgbe_setup_rx_ring(ixgbe_rx_ring_t *);
66 static void ixgbe_setup_tx_ring(ixgbe_tx_ring_t *);
67 static void ixgbe_setup_rss(ixgbe_t *);
68 static void ixgbe_setup_vmdq(ixgbe_t *);
69 static void ixgbe_setup_vmdq_rss(ixgbe_t *);
70 static void ixgbe_init_unicst(ixgbe_t *);
71 static int ixgbe_unicst_find(ixgbe_t *, const uint8_t *);
72 static void ixgbe_setup_multicst(ixgbe_t *);
73 static void ixgbe_get_hw_state(ixgbe_t *);
74 static void ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe);
75 static void ixgbe_get_conf(ixgbe_t *);
76 static void ixgbe_init_params(ixgbe_t *);
77 static int ixgbe_get_prop(ixgbe_t *, char *, int, int, int);
78 static void ixgbe_driver_link_check(ixgbe_t *);
79 static void ixgbe_sfp_check(void *);
80 static void ixgbe_overtemp_check(void *);
81 static void ixgbe_link_timer(void *);
82 static void ixgbe_local_timer(void *);
83 static void ixgbe_arm_watchdog_timer(ixgbe_t *);
84 static void ixgbe_restart_watchdog_timer(ixgbe_t *);
85 static void ixgbe_disable_adapter_interrupts(ixgbe_t *);
86 static void ixgbe_enable_adapter_interrupts(ixgbe_t *);
87 static boolean_t is_valid_mac_addr(uint8_t *);
88 static boolean_t ixgbe_stall_check(ixgbe_t *);
89 static boolean_t ixgbe_set_loopback_mode(ixgbe_t *, uint32_t);
90 static void ixgbe_set_internal_mac_loopback(ixgbe_t *);
91 static boolean_t ixgbe_find_mac_address(ixgbe_t *);
92 static int ixgbe_alloc_intrs(ixgbe_t *);
93 static int ixgbe_alloc_intr_handles(ixgbe_t *, int);
94 static int ixgbe_add_intr_handlers(ixgbe_t *);
95 static void ixgbe_map_rxring_to_vector(ixgbe_t *, int, int);
96 static void ixgbe_map_txring_to_vector(ixgbe_t *, int, int);
97 static void ixgbe_setup_ivar(ixgbe_t *, uint16_t, uint8_t, int8_t);
98 static void ixgbe_enable_ivar(ixgbe_t *, uint16_t, int8_t);
99 static void ixgbe_disable_ivar(ixgbe_t *, uint16_t, int8_t);
100 static uint32_t ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index);
101 static int ixgbe_map_intrs_to_vectors(ixgbe_t *);
102 static void ixgbe_setup_adapter_vector(ixgbe_t *);
103 static void ixgbe_rem_intr_handlers(ixgbe_t *);
104 static void ixgbe_rem_intrs(ixgbe_t *);
105 static int ixgbe_enable_intrs(ixgbe_t *);
106 static int ixgbe_disable_intrs(ixgbe_t *);
107 static uint_t ixgbe_intr_legacy(void *, void *);
108 static uint_t ixgbe_intr_msi(void *, void *);
109 static uint_t ixgbe_intr_msix(void *, void *);
110 static void ixgbe_intr_rx_work(ixgbe_rx_ring_t *);
111 static void ixgbe_intr_tx_work(ixgbe_tx_ring_t *);
112 static void ixgbe_intr_other_work(ixgbe_t *, uint32_t);
113 static void ixgbe_get_driver_control(struct ixgbe_hw *);
114 static int ixgbe_addmac(void *, const uint8_t *);
115 static int ixgbe_remmac(void *, const uint8_t *);
116 static void ixgbe_release_driver_control(struct ixgbe_hw *);
117
118 static int ixgbe_attach(dev_info_t *, ddi_attach_cmd_t);
119 static int ixgbe_detach(dev_info_t *, ddi_detach_cmd_t);
120 static int ixgbe_resume(dev_info_t *);
121 static int ixgbe_suspend(dev_info_t *);
122 static int ixgbe_quiesce(dev_info_t *);
123 static void ixgbe_unconfigure(dev_info_t *, ixgbe_t *);
124 static uint8_t *ixgbe_mc_table_itr(struct ixgbe_hw *, uint8_t **, uint32_t *);
125 static int ixgbe_cbfunc(dev_info_t *, ddi_cb_action_t, void *, void *, void *);
126 static int ixgbe_intr_cb_register(ixgbe_t *);
127 static int ixgbe_intr_adjust(ixgbe_t *, ddi_cb_action_t, int);
128
129 static int ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err,
130 const void *impl_data);
131 static void ixgbe_fm_init(ixgbe_t *);
132 static void ixgbe_fm_fini(ixgbe_t *);
133
134 char *ixgbe_priv_props[] = {
135 "_tx_copy_thresh",
136 "_tx_recycle_thresh",
137 "_tx_overload_thresh",
138 "_tx_resched_thresh",
139 "_rx_copy_thresh",
140 "_rx_limit_per_intr",
141 "_intr_throttling",
142 "_adv_pause_cap",
143 "_adv_asym_pause_cap",
144 NULL
145 };
146
147 #define IXGBE_MAX_PRIV_PROPS \
148 (sizeof (ixgbe_priv_props) / sizeof (mac_priv_prop_t))
149
150 static struct cb_ops ixgbe_cb_ops = {
151 nulldev, /* cb_open */
152 nulldev, /* cb_close */
153 nodev, /* cb_strategy */
154 nodev, /* cb_print */
155 nodev, /* cb_dump */
156 nodev, /* cb_read */
157 nodev, /* cb_write */
158 nodev, /* cb_ioctl */
159 nodev, /* cb_devmap */
160 nodev, /* cb_mmap */
161 nodev, /* cb_segmap */
162 nochpoll, /* cb_chpoll */
163 ddi_prop_op, /* cb_prop_op */
164 NULL, /* cb_stream */
165 D_MP | D_HOTPLUG, /* cb_flag */
166 CB_REV, /* cb_rev */
167 nodev, /* cb_aread */
168 nodev /* cb_awrite */
169 };
170
171 static struct dev_ops ixgbe_dev_ops = {
172 DEVO_REV, /* devo_rev */
173 0, /* devo_refcnt */
174 NULL, /* devo_getinfo */
175 nulldev, /* devo_identify */
176 nulldev, /* devo_probe */
177 ixgbe_attach, /* devo_attach */
178 ixgbe_detach, /* devo_detach */
179 nodev, /* devo_reset */
180 &ixgbe_cb_ops, /* devo_cb_ops */
181 NULL, /* devo_bus_ops */
182 ddi_power, /* devo_power */
183 ixgbe_quiesce, /* devo_quiesce */
184 };
185
186 static struct modldrv ixgbe_modldrv = {
187 &mod_driverops, /* Type of module. This one is a driver */
188 ixgbe_ident, /* Discription string */
189 &ixgbe_dev_ops /* driver ops */
190 };
191
192 static struct modlinkage ixgbe_modlinkage = {
193 MODREV_1, &ixgbe_modldrv, NULL
194 };
195
196 /*
197 * Access attributes for register mapping
198 */
199 ddi_device_acc_attr_t ixgbe_regs_acc_attr = {
200 DDI_DEVICE_ATTR_V1,
201 DDI_STRUCTURE_LE_ACC,
202 DDI_STRICTORDER_ACC,
203 DDI_FLAGERR_ACC
204 };
205
206 /*
207 * Loopback property
208 */
209 static lb_property_t lb_normal = {
210 normal, "normal", IXGBE_LB_NONE
211 };
212
213 static lb_property_t lb_mac = {
214 internal, "MAC", IXGBE_LB_INTERNAL_MAC
215 };
216
217 static lb_property_t lb_external = {
218 external, "External", IXGBE_LB_EXTERNAL
219 };
220
221 #define IXGBE_M_CALLBACK_FLAGS \
222 (MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO)
223
224 static mac_callbacks_t ixgbe_m_callbacks = {
225 IXGBE_M_CALLBACK_FLAGS,
226 ixgbe_m_stat,
227 ixgbe_m_start,
228 ixgbe_m_stop,
229 ixgbe_m_promisc,
230 ixgbe_m_multicst,
231 NULL,
232 NULL,
233 NULL,
234 ixgbe_m_ioctl,
235 ixgbe_m_getcapab,
236 NULL,
237 NULL,
238 ixgbe_m_setprop,
239 ixgbe_m_getprop,
240 ixgbe_m_propinfo
241 };
242
243 /*
244 * Initialize capabilities of each supported adapter type
245 */
246 static adapter_info_t ixgbe_82598eb_cap = {
247 64, /* maximum number of rx queues */
248 1, /* minimum number of rx queues */
249 64, /* default number of rx queues */
250 16, /* maximum number of rx groups */
251 1, /* minimum number of rx groups */
252 1, /* default number of rx groups */
253 32, /* maximum number of tx queues */
254 1, /* minimum number of tx queues */
255 8, /* default number of tx queues */
256 16366, /* maximum MTU size */
257 0xFFFF, /* maximum interrupt throttle rate */
258 0, /* minimum interrupt throttle rate */
259 200, /* default interrupt throttle rate */
260 18, /* maximum total msix vectors */
261 16, /* maximum number of ring vectors */
262 2, /* maximum number of other vectors */
263 IXGBE_EICR_LSC, /* "other" interrupt types handled */
264 0, /* "other" interrupt types enable mask */
265 (IXGBE_FLAG_DCA_CAPABLE /* capability flags */
266 | IXGBE_FLAG_RSS_CAPABLE
267 | IXGBE_FLAG_VMDQ_CAPABLE)
268 };
269
270 static adapter_info_t ixgbe_82599eb_cap = {
271 128, /* maximum number of rx queues */
272 1, /* minimum number of rx queues */
273 128, /* default number of rx queues */
274 64, /* maximum number of rx groups */
275 1, /* minimum number of rx groups */
276 1, /* default number of rx groups */
277 128, /* maximum number of tx queues */
278 1, /* minimum number of tx queues */
279 8, /* default number of tx queues */
280 15500, /* maximum MTU size */
281 0xFF8, /* maximum interrupt throttle rate */
282 0, /* minimum interrupt throttle rate */
283 200, /* default interrupt throttle rate */
284 64, /* maximum total msix vectors */
285 16, /* maximum number of ring vectors */
286 2, /* maximum number of other vectors */
287 (IXGBE_EICR_LSC
288 | IXGBE_EICR_GPI_SDP1
289 | IXGBE_EICR_GPI_SDP2), /* "other" interrupt types handled */
290
291 (IXGBE_SDP1_GPIEN
292 | IXGBE_SDP2_GPIEN), /* "other" interrupt types enable mask */
293
294 (IXGBE_FLAG_DCA_CAPABLE
295 | IXGBE_FLAG_RSS_CAPABLE
296 | IXGBE_FLAG_VMDQ_CAPABLE
297 | IXGBE_FLAG_RSC_CAPABLE
298 | IXGBE_FLAG_SFP_PLUG_CAPABLE) /* capability flags */
299 };
300
301 static adapter_info_t ixgbe_X540_cap = {
302 128, /* maximum number of rx queues */
303 1, /* minimum number of rx queues */
304 128, /* default number of rx queues */
305 64, /* maximum number of rx groups */
306 1, /* minimum number of rx groups */
307 1, /* default number of rx groups */
308 128, /* maximum number of tx queues */
309 1, /* minimum number of tx queues */
310 8, /* default number of tx queues */
311 15500, /* maximum MTU size */
312 0xFF8, /* maximum interrupt throttle rate */
313 0, /* minimum interrupt throttle rate */
314 200, /* default interrupt throttle rate */
315 64, /* maximum total msix vectors */
316 16, /* maximum number of ring vectors */
317 2, /* maximum number of other vectors */
318 (IXGBE_EICR_LSC
319 | IXGBE_EICR_GPI_SDP1_X540
320 | IXGBE_EICR_GPI_SDP2_X540), /* "other" interrupt types handled */
321
322 (IXGBE_SDP1_GPIEN_X540
323 | IXGBE_SDP2_GPIEN_X540), /* "other" interrupt types enable mask */
324
325 (IXGBE_FLAG_DCA_CAPABLE
326 | IXGBE_FLAG_RSS_CAPABLE
327 | IXGBE_FLAG_VMDQ_CAPABLE
328 | IXGBE_FLAG_RSC_CAPABLE) /* capability flags */
329 };
330
331 static adapter_info_t ixgbe_X550_cap = {
332 128, /* maximum number of rx queues */
333 1, /* minimum number of rx queues */
334 128, /* default number of rx queues */
335 64, /* maximum number of rx groups */
336 1, /* minimum number of rx groups */
337 1, /* default number of rx groups */
338 128, /* maximum number of tx queues */
339 1, /* minimum number of tx queues */
340 8, /* default number of tx queues */
341 15500, /* maximum MTU size */
342 0xFF8, /* maximum interrupt throttle rate */
343 0, /* minimum interrupt throttle rate */
344 200, /* default interrupt throttle rate */
345 64, /* maximum total msix vectors */
346 16, /* maximum number of ring vectors */
347 2, /* maximum number of other vectors */
348 (IXGBE_EICR_LSC
349 | IXGBE_SDP1_GPIEN_X550
350 | IXGBE_SDP2_GPIEN_X550), /* "other" interrupt types handled */
351
352 (IXGBE_SDP1_GPIEN_X550
353 | IXGBE_SDP2_GPIEN_X550), /* "other" interrupt types enable mask */
354
355 (IXGBE_FLAG_RSS_CAPABLE
356 | IXGBE_FLAG_VMDQ_CAPABLE
357 | IXGBE_FLAG_RSC_CAPABLE) /* capability flags */
358 };
359
360 /*
361 * Module Initialization Functions.
362 */
363
364 int
365 _init(void)
366 {
367 int status;
368
369 mac_init_ops(&ixgbe_dev_ops, MODULE_NAME);
370
371 status = mod_install(&ixgbe_modlinkage);
372
373 if (status != DDI_SUCCESS) {
374 mac_fini_ops(&ixgbe_dev_ops);
375 }
376
377 return (status);
378 }
379
380 int
381 _fini(void)
382 {
383 int status;
384
385 status = mod_remove(&ixgbe_modlinkage);
386
387 if (status == DDI_SUCCESS) {
388 mac_fini_ops(&ixgbe_dev_ops);
389 }
390
391 return (status);
392 }
393
394 int
395 _info(struct modinfo *modinfop)
396 {
397 int status;
398
399 status = mod_info(&ixgbe_modlinkage, modinfop);
400
401 return (status);
402 }
403
404 /*
405 * ixgbe_attach - Driver attach.
406 *
407 * This function is the device specific initialization entry
408 * point. This entry point is required and must be written.
409 * The DDI_ATTACH command must be provided in the attach entry
410 * point. When attach() is called with cmd set to DDI_ATTACH,
411 * all normal kernel services (such as kmem_alloc(9F)) are
412 * available for use by the driver.
413 *
414 * The attach() function will be called once for each instance
415 * of the device on the system with cmd set to DDI_ATTACH.
416 * Until attach() succeeds, the only driver entry points which
417 * may be called are open(9E) and getinfo(9E).
418 */
419 static int
420 ixgbe_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd)
421 {
422 ixgbe_t *ixgbe;
423 struct ixgbe_osdep *osdep;
424 struct ixgbe_hw *hw;
425 int instance;
426 char taskqname[32];
427
428 /*
429 * Check the command and perform corresponding operations
430 */
431 switch (cmd) {
432 default:
433 return (DDI_FAILURE);
434
435 case DDI_RESUME:
436 return (ixgbe_resume(devinfo));
437
438 case DDI_ATTACH:
439 break;
440 }
441
442 /* Get the device instance */
443 instance = ddi_get_instance(devinfo);
444
445 /* Allocate memory for the instance data structure */
446 ixgbe = kmem_zalloc(sizeof (ixgbe_t), KM_SLEEP);
447
448 ixgbe->dip = devinfo;
449 ixgbe->instance = instance;
450
451 hw = &ixgbe->hw;
452 osdep = &ixgbe->osdep;
453 hw->back = osdep;
454 osdep->ixgbe = ixgbe;
455
456 /* Attach the instance pointer to the dev_info data structure */
457 ddi_set_driver_private(devinfo, ixgbe);
458
459 /*
460 * Initialize for fma support
461 */
462 ixgbe->fm_capabilities = ixgbe_get_prop(ixgbe, PROP_FM_CAPABLE,
463 0, 0x0f, DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
464 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
465 ixgbe_fm_init(ixgbe);
466 ixgbe->attach_progress |= ATTACH_PROGRESS_FM_INIT;
467
468 /*
469 * Map PCI config space registers
470 */
471 if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) {
472 ixgbe_error(ixgbe, "Failed to map PCI configurations");
473 goto attach_fail;
474 }
475 ixgbe->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG;
476
477 /*
478 * Identify the chipset family
479 */
480 if (ixgbe_identify_hardware(ixgbe) != IXGBE_SUCCESS) {
481 ixgbe_error(ixgbe, "Failed to identify hardware");
482 goto attach_fail;
483 }
484
485 /*
486 * Map device registers
487 */
488 if (ixgbe_regs_map(ixgbe) != IXGBE_SUCCESS) {
489 ixgbe_error(ixgbe, "Failed to map device registers");
490 goto attach_fail;
491 }
492 ixgbe->attach_progress |= ATTACH_PROGRESS_REGS_MAP;
493
494 /*
495 * Initialize driver parameters
496 */
497 ixgbe_init_properties(ixgbe);
498 ixgbe->attach_progress |= ATTACH_PROGRESS_PROPS;
499
500 /*
501 * Register interrupt callback
502 */
503 if (ixgbe_intr_cb_register(ixgbe) != IXGBE_SUCCESS) {
504 ixgbe_error(ixgbe, "Failed to register interrupt callback");
505 goto attach_fail;
506 }
507
508 /*
509 * Allocate interrupts
510 */
511 if (ixgbe_alloc_intrs(ixgbe) != IXGBE_SUCCESS) {
512 ixgbe_error(ixgbe, "Failed to allocate interrupts");
513 goto attach_fail;
514 }
515 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
516
517 /*
518 * Allocate rx/tx rings based on the ring numbers.
519 * The actual numbers of rx/tx rings are decided by the number of
520 * allocated interrupt vectors, so we should allocate the rings after
521 * interrupts are allocated.
522 */
523 if (ixgbe_alloc_rings(ixgbe) != IXGBE_SUCCESS) {
524 ixgbe_error(ixgbe, "Failed to allocate rx and tx rings");
525 goto attach_fail;
526 }
527 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS;
528
529 /*
530 * Map rings to interrupt vectors
531 */
532 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) {
533 ixgbe_error(ixgbe, "Failed to map interrupts to vectors");
534 goto attach_fail;
535 }
536
537 /*
538 * Add interrupt handlers
539 */
540 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) {
541 ixgbe_error(ixgbe, "Failed to add interrupt handlers");
542 goto attach_fail;
543 }
544 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
545
546 /*
547 * Create a taskq for sfp-change
548 */
549 (void) sprintf(taskqname, "ixgbe%d_sfp_taskq", instance);
550 if ((ixgbe->sfp_taskq = ddi_taskq_create(devinfo, taskqname,
551 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
552 ixgbe_error(ixgbe, "sfp_taskq create failed");
553 goto attach_fail;
554 }
555 ixgbe->attach_progress |= ATTACH_PROGRESS_SFP_TASKQ;
556
557 /*
558 * Create a taskq for over-temp
559 */
560 (void) sprintf(taskqname, "ixgbe%d_overtemp_taskq", instance);
561 if ((ixgbe->overtemp_taskq = ddi_taskq_create(devinfo, taskqname,
562 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
563 ixgbe_error(ixgbe, "overtemp_taskq create failed");
564 goto attach_fail;
565 }
566 ixgbe->attach_progress |= ATTACH_PROGRESS_OVERTEMP_TASKQ;
567
568 /*
569 * Initialize driver parameters
570 */
571 if (ixgbe_init_driver_settings(ixgbe) != IXGBE_SUCCESS) {
572 ixgbe_error(ixgbe, "Failed to initialize driver settings");
573 goto attach_fail;
574 }
575
576 /*
577 * Initialize mutexes for this device.
578 * Do this before enabling the interrupt handler and
579 * register the softint to avoid the condition where
580 * interrupt handler can try using uninitialized mutex.
581 */
582 ixgbe_init_locks(ixgbe);
583 ixgbe->attach_progress |= ATTACH_PROGRESS_LOCKS;
584
585 /*
586 * Initialize chipset hardware
587 */
588 if (ixgbe_init(ixgbe) != IXGBE_SUCCESS) {
589 ixgbe_error(ixgbe, "Failed to initialize adapter");
590 goto attach_fail;
591 }
592 ixgbe->link_check_complete = B_FALSE;
593 ixgbe->link_check_hrtime = gethrtime() +
594 (IXGBE_LINK_UP_TIME * 100000000ULL);
595 ixgbe->attach_progress |= ATTACH_PROGRESS_INIT;
596
597 if (ixgbe_check_acc_handle(ixgbe->osdep.cfg_handle) != DDI_FM_OK) {
598 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
599 goto attach_fail;
600 }
601
602 /*
603 * Initialize statistics
604 */
605 if (ixgbe_init_stats(ixgbe) != IXGBE_SUCCESS) {
606 ixgbe_error(ixgbe, "Failed to initialize statistics");
607 goto attach_fail;
608 }
609 ixgbe->attach_progress |= ATTACH_PROGRESS_STATS;
610
611 /*
612 * Register the driver to the MAC
613 */
614 if (ixgbe_register_mac(ixgbe) != IXGBE_SUCCESS) {
615 ixgbe_error(ixgbe, "Failed to register MAC");
616 goto attach_fail;
617 }
618 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN);
619 ixgbe->attach_progress |= ATTACH_PROGRESS_MAC;
620
621 ixgbe->periodic_id = ddi_periodic_add(ixgbe_link_timer, ixgbe,
622 IXGBE_CYCLIC_PERIOD, DDI_IPL_0);
623 if (ixgbe->periodic_id == 0) {
624 ixgbe_error(ixgbe, "Failed to add the link check timer");
625 goto attach_fail;
626 }
627 ixgbe->attach_progress |= ATTACH_PROGRESS_LINK_TIMER;
628
629 /*
630 * Now that mutex locks are initialized, and the chip is also
631 * initialized, enable interrupts.
632 */
633 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) {
634 ixgbe_error(ixgbe, "Failed to enable DDI interrupts");
635 goto attach_fail;
636 }
637 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
638
639 ixgbe_log(ixgbe, "%s", ixgbe_ident);
640 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_INITIALIZED);
641
642 return (DDI_SUCCESS);
643
644 attach_fail:
645 ixgbe_unconfigure(devinfo, ixgbe);
646 return (DDI_FAILURE);
647 }
648
649 /*
650 * ixgbe_detach - Driver detach.
651 *
652 * The detach() function is the complement of the attach routine.
653 * If cmd is set to DDI_DETACH, detach() is used to remove the
654 * state associated with a given instance of a device node
655 * prior to the removal of that instance from the system.
656 *
657 * The detach() function will be called once for each instance
658 * of the device for which there has been a successful attach()
659 * once there are no longer any opens on the device.
660 *
661 * Interrupts routine are disabled, All memory allocated by this
662 * driver are freed.
663 */
664 static int
665 ixgbe_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd)
666 {
667 ixgbe_t *ixgbe;
668
669 /*
670 * Check detach command
671 */
672 switch (cmd) {
673 default:
674 return (DDI_FAILURE);
675
676 case DDI_SUSPEND:
677 return (ixgbe_suspend(devinfo));
678
679 case DDI_DETACH:
680 break;
681 }
682
683 /*
684 * Get the pointer to the driver private data structure
685 */
686 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
687 if (ixgbe == NULL)
688 return (DDI_FAILURE);
689
690 /*
691 * If the device is still running, it needs to be stopped first.
692 * This check is necessary because under some specific circumstances,
693 * the detach routine can be called without stopping the interface
694 * first.
695 */
696 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
697 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED);
698 mutex_enter(&ixgbe->gen_lock);
699 ixgbe_stop(ixgbe, B_TRUE);
700 mutex_exit(&ixgbe->gen_lock);
701 /* Disable and stop the watchdog timer */
702 ixgbe_disable_watchdog_timer(ixgbe);
703 }
704
705 /*
706 * Check if there are still rx buffers held by the upper layer.
707 * If so, fail the detach.
708 */
709 if (!ixgbe_rx_drain(ixgbe))
710 return (DDI_FAILURE);
711
712 /*
713 * Do the remaining unconfigure routines
714 */
715 ixgbe_unconfigure(devinfo, ixgbe);
716
717 return (DDI_SUCCESS);
718 }
719
720 /*
721 * quiesce(9E) entry point.
722 *
723 * This function is called when the system is single-threaded at high
724 * PIL with preemption disabled. Therefore, this function must not be
725 * blocked.
726 *
727 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
728 * DDI_FAILURE indicates an error condition and should almost never happen.
729 */
730 static int
731 ixgbe_quiesce(dev_info_t *devinfo)
732 {
733 ixgbe_t *ixgbe;
734 struct ixgbe_hw *hw;
735
736 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
737
738 if (ixgbe == NULL)
739 return (DDI_FAILURE);
740
741 hw = &ixgbe->hw;
742
743 /*
744 * Disable the adapter interrupts
745 */
746 ixgbe_disable_adapter_interrupts(ixgbe);
747
748 /*
749 * Tell firmware driver is no longer in control
750 */
751 ixgbe_release_driver_control(hw);
752
753 /*
754 * Reset the chipset
755 */
756 (void) ixgbe_reset_hw(hw);
757
758 /*
759 * Reset PHY
760 */
761 (void) ixgbe_reset_phy(hw);
762
763 return (DDI_SUCCESS);
764 }
765
766 static void
767 ixgbe_unconfigure(dev_info_t *devinfo, ixgbe_t *ixgbe)
768 {
769 /*
770 * Disable interrupt
771 */
772 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
773 (void) ixgbe_disable_intrs(ixgbe);
774 }
775
776 /*
777 * remove the link check timer
778 */
779 if (ixgbe->attach_progress & ATTACH_PROGRESS_LINK_TIMER) {
780 if (ixgbe->periodic_id != NULL) {
781 ddi_periodic_delete(ixgbe->periodic_id);
782 ixgbe->periodic_id = NULL;
783 }
784 }
785
786 /*
787 * Unregister MAC
788 */
789 if (ixgbe->attach_progress & ATTACH_PROGRESS_MAC) {
790 (void) mac_unregister(ixgbe->mac_hdl);
791 }
792
793 /*
794 * Free statistics
795 */
796 if (ixgbe->attach_progress & ATTACH_PROGRESS_STATS) {
797 kstat_delete((kstat_t *)ixgbe->ixgbe_ks);
798 }
799
800 /*
801 * Remove interrupt handlers
802 */
803 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
804 ixgbe_rem_intr_handlers(ixgbe);
805 }
806
807 /*
808 * Remove taskq for sfp-status-change
809 */
810 if (ixgbe->attach_progress & ATTACH_PROGRESS_SFP_TASKQ) {
811 ddi_taskq_destroy(ixgbe->sfp_taskq);
812 }
813
814 /*
815 * Remove taskq for over-temp
816 */
817 if (ixgbe->attach_progress & ATTACH_PROGRESS_OVERTEMP_TASKQ) {
818 ddi_taskq_destroy(ixgbe->overtemp_taskq);
819 }
820
821 /*
822 * Remove interrupts
823 */
824 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) {
825 ixgbe_rem_intrs(ixgbe);
826 }
827
828 /*
829 * Unregister interrupt callback handler
830 */
831 (void) ddi_cb_unregister(ixgbe->cb_hdl);
832
833 /*
834 * Remove driver properties
835 */
836 if (ixgbe->attach_progress & ATTACH_PROGRESS_PROPS) {
837 (void) ddi_prop_remove_all(devinfo);
838 }
839
840 /*
841 * Stop the chipset
842 */
843 if (ixgbe->attach_progress & ATTACH_PROGRESS_INIT) {
844 mutex_enter(&ixgbe->gen_lock);
845 ixgbe_chip_stop(ixgbe);
846 mutex_exit(&ixgbe->gen_lock);
847 }
848
849 /*
850 * Free register handle
851 */
852 if (ixgbe->attach_progress & ATTACH_PROGRESS_REGS_MAP) {
853 if (ixgbe->osdep.reg_handle != NULL)
854 ddi_regs_map_free(&ixgbe->osdep.reg_handle);
855 }
856
857 /*
858 * Free PCI config handle
859 */
860 if (ixgbe->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) {
861 if (ixgbe->osdep.cfg_handle != NULL)
862 pci_config_teardown(&ixgbe->osdep.cfg_handle);
863 }
864
865 /*
866 * Free locks
867 */
868 if (ixgbe->attach_progress & ATTACH_PROGRESS_LOCKS) {
869 ixgbe_destroy_locks(ixgbe);
870 }
871
872 /*
873 * Free the rx/tx rings
874 */
875 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) {
876 ixgbe_free_rings(ixgbe);
877 }
878
879 /*
880 * Unregister FMA capabilities
881 */
882 if (ixgbe->attach_progress & ATTACH_PROGRESS_FM_INIT) {
883 ixgbe_fm_fini(ixgbe);
884 }
885
886 /*
887 * Free the driver data structure
888 */
889 kmem_free(ixgbe, sizeof (ixgbe_t));
890
891 ddi_set_driver_private(devinfo, NULL);
892 }
893
894 /*
895 * ixgbe_register_mac - Register the driver and its function pointers with
896 * the GLD interface.
897 */
898 static int
899 ixgbe_register_mac(ixgbe_t *ixgbe)
900 {
901 struct ixgbe_hw *hw = &ixgbe->hw;
902 mac_register_t *mac;
903 int status;
904
905 if ((mac = mac_alloc(MAC_VERSION)) == NULL)
906 return (IXGBE_FAILURE);
907
908 mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER;
909 mac->m_driver = ixgbe;
910 mac->m_dip = ixgbe->dip;
911 mac->m_src_addr = hw->mac.addr;
912 mac->m_callbacks = &ixgbe_m_callbacks;
913 mac->m_min_sdu = 0;
914 mac->m_max_sdu = ixgbe->default_mtu;
915 mac->m_margin = VLAN_TAGSZ;
916 mac->m_priv_props = ixgbe_priv_props;
917 mac->m_v12n = MAC_VIRT_LEVEL1;
918
919 status = mac_register(mac, &ixgbe->mac_hdl);
920
921 mac_free(mac);
922
923 return ((status == 0) ? IXGBE_SUCCESS : IXGBE_FAILURE);
924 }
925
926 /*
927 * ixgbe_identify_hardware - Identify the type of the chipset.
928 */
929 static int
930 ixgbe_identify_hardware(ixgbe_t *ixgbe)
931 {
932 struct ixgbe_hw *hw = &ixgbe->hw;
933 struct ixgbe_osdep *osdep = &ixgbe->osdep;
934
935 /*
936 * Get the device id
937 */
938 hw->vendor_id =
939 pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID);
940 hw->device_id =
941 pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID);
942 hw->revision_id =
943 pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID);
944 hw->subsystem_device_id =
945 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID);
946 hw->subsystem_vendor_id =
947 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID);
948
949 /*
950 * Set the mac type of the adapter based on the device id
951 */
952 if (ixgbe_set_mac_type(hw) != IXGBE_SUCCESS) {
953 return (IXGBE_FAILURE);
954 }
955
956 /*
957 * Install adapter capabilities
958 */
959 switch (hw->mac.type) {
960 case ixgbe_mac_82598EB:
961 IXGBE_DEBUGLOG_0(ixgbe, "identify 82598 adapter\n");
962 ixgbe->capab = &ixgbe_82598eb_cap;
963
964 if (ixgbe_get_media_type(hw) == ixgbe_media_type_copper) {
965 ixgbe->capab->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE;
966 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP1;
967 ixgbe->capab->other_gpie |= IXGBE_SDP1_GPIEN;
968 }
969 break;
970
971 case ixgbe_mac_82599EB:
972 IXGBE_DEBUGLOG_0(ixgbe, "identify 82599 adapter\n");
973 ixgbe->capab = &ixgbe_82599eb_cap;
974
975 if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM) {
976 ixgbe->capab->flags |= IXGBE_FLAG_TEMP_SENSOR_CAPABLE;
977 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP0;
978 ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN;
979 }
980 break;
981
982 case ixgbe_mac_X540:
983 IXGBE_DEBUGLOG_0(ixgbe, "identify X540 adapter\n");
984 ixgbe->capab = &ixgbe_X540_cap;
985 /*
986 * For now, X540 is all set in its capab structure.
987 * As other X540 variants show up, things can change here.
988 */
989 break;
990
991 case ixgbe_mac_X550:
992 case ixgbe_mac_X550EM_x:
993 IXGBE_DEBUGLOG_0(ixgbe, "identify X550 adapter\n");
994 ixgbe->capab = &ixgbe_X550_cap;
995
996 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP)
997 ixgbe->capab->flags |= IXGBE_FLAG_SFP_PLUG_CAPABLE;
998
999 break;
1000
1001 default:
1002 IXGBE_DEBUGLOG_1(ixgbe,
1003 "adapter not supported in ixgbe_identify_hardware(): %d\n",
1004 hw->mac.type);
1005 return (IXGBE_FAILURE);
1006 }
1007
1008 return (IXGBE_SUCCESS);
1009 }
1010
1011 /*
1012 * ixgbe_regs_map - Map the device registers.
1013 *
1014 */
1015 static int
1016 ixgbe_regs_map(ixgbe_t *ixgbe)
1017 {
1018 dev_info_t *devinfo = ixgbe->dip;
1019 struct ixgbe_hw *hw = &ixgbe->hw;
1020 struct ixgbe_osdep *osdep = &ixgbe->osdep;
1021 off_t mem_size;
1022
1023 /*
1024 * First get the size of device registers to be mapped.
1025 */
1026 if (ddi_dev_regsize(devinfo, IXGBE_ADAPTER_REGSET, &mem_size)
1027 != DDI_SUCCESS) {
1028 return (IXGBE_FAILURE);
1029 }
1030
1031 /*
1032 * Call ddi_regs_map_setup() to map registers
1033 */
1034 if ((ddi_regs_map_setup(devinfo, IXGBE_ADAPTER_REGSET,
1035 (caddr_t *)&hw->hw_addr, 0,
1036 mem_size, &ixgbe_regs_acc_attr,
1037 &osdep->reg_handle)) != DDI_SUCCESS) {
1038 return (IXGBE_FAILURE);
1039 }
1040
1041 return (IXGBE_SUCCESS);
1042 }
1043
1044 /*
1045 * ixgbe_init_properties - Initialize driver properties.
1046 */
1047 static void
1048 ixgbe_init_properties(ixgbe_t *ixgbe)
1049 {
1050 /*
1051 * Get conf file properties, including link settings
1052 * jumbo frames, ring number, descriptor number, etc.
1053 */
1054 ixgbe_get_conf(ixgbe);
1055
1056 ixgbe_init_params(ixgbe);
1057 }
1058
1059 /*
1060 * ixgbe_init_driver_settings - Initialize driver settings.
1061 *
1062 * The settings include hardware function pointers, bus information,
1063 * rx/tx rings settings, link state, and any other parameters that
1064 * need to be setup during driver initialization.
1065 */
1066 static int
1067 ixgbe_init_driver_settings(ixgbe_t *ixgbe)
1068 {
1069 struct ixgbe_hw *hw = &ixgbe->hw;
1070 dev_info_t *devinfo = ixgbe->dip;
1071 ixgbe_rx_ring_t *rx_ring;
1072 ixgbe_rx_group_t *rx_group;
1073 ixgbe_tx_ring_t *tx_ring;
1074 uint32_t rx_size;
1075 uint32_t tx_size;
1076 uint32_t ring_per_group;
1077 int i;
1078
1079 /*
1080 * Initialize chipset specific hardware function pointers
1081 */
1082 if (ixgbe_init_shared_code(hw) != IXGBE_SUCCESS) {
1083 return (IXGBE_FAILURE);
1084 }
1085
1086 /*
1087 * Get the system page size
1088 */
1089 ixgbe->sys_page_size = ddi_ptob(devinfo, (ulong_t)1);
1090
1091 /*
1092 * Set rx buffer size
1093 *
1094 * The IP header alignment room is counted in the calculation.
1095 * The rx buffer size is in unit of 1K that is required by the
1096 * chipset hardware.
1097 */
1098 rx_size = ixgbe->max_frame_size + IPHDR_ALIGN_ROOM;
1099 ixgbe->rx_buf_size = ((rx_size >> 10) +
1100 ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1101
1102 /*
1103 * Set tx buffer size
1104 */
1105 tx_size = ixgbe->max_frame_size;
1106 ixgbe->tx_buf_size = ((tx_size >> 10) +
1107 ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1108
1109 /*
1110 * Initialize rx/tx rings/groups parameters
1111 */
1112 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
1113 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1114 rx_ring = &ixgbe->rx_rings[i];
1115 rx_ring->index = i;
1116 rx_ring->ixgbe = ixgbe;
1117 rx_ring->group_index = i / ring_per_group;
1118 rx_ring->hw_index = ixgbe_get_hw_rx_index(ixgbe, i);
1119 }
1120
1121 for (i = 0; i < ixgbe->num_rx_groups; i++) {
1122 rx_group = &ixgbe->rx_groups[i];
1123 rx_group->index = i;
1124 rx_group->ixgbe = ixgbe;
1125 }
1126
1127 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1128 tx_ring = &ixgbe->tx_rings[i];
1129 tx_ring->index = i;
1130 tx_ring->ixgbe = ixgbe;
1131 if (ixgbe->tx_head_wb_enable)
1132 tx_ring->tx_recycle = ixgbe_tx_recycle_head_wb;
1133 else
1134 tx_ring->tx_recycle = ixgbe_tx_recycle_legacy;
1135
1136 tx_ring->ring_size = ixgbe->tx_ring_size;
1137 tx_ring->free_list_size = ixgbe->tx_ring_size +
1138 (ixgbe->tx_ring_size >> 1);
1139 }
1140
1141 /*
1142 * Initialize values of interrupt throttling rate
1143 */
1144 for (i = 1; i < MAX_INTR_VECTOR; i++)
1145 ixgbe->intr_throttling[i] = ixgbe->intr_throttling[0];
1146
1147 /*
1148 * The initial link state should be "unknown"
1149 */
1150 ixgbe->link_state = LINK_STATE_UNKNOWN;
1151
1152 return (IXGBE_SUCCESS);
1153 }
1154
1155 /*
1156 * ixgbe_init_locks - Initialize locks.
1157 */
1158 static void
1159 ixgbe_init_locks(ixgbe_t *ixgbe)
1160 {
1161 ixgbe_rx_ring_t *rx_ring;
1162 ixgbe_tx_ring_t *tx_ring;
1163 int i;
1164
1165 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1166 rx_ring = &ixgbe->rx_rings[i];
1167 mutex_init(&rx_ring->rx_lock, NULL,
1168 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1169 }
1170
1171 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1172 tx_ring = &ixgbe->tx_rings[i];
1173 mutex_init(&tx_ring->tx_lock, NULL,
1174 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1175 mutex_init(&tx_ring->recycle_lock, NULL,
1176 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1177 mutex_init(&tx_ring->tcb_head_lock, NULL,
1178 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1179 mutex_init(&tx_ring->tcb_tail_lock, NULL,
1180 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1181 }
1182
1183 mutex_init(&ixgbe->gen_lock, NULL,
1184 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1185
1186 mutex_init(&ixgbe->watchdog_lock, NULL,
1187 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1188 }
1189
1190 /*
1191 * ixgbe_destroy_locks - Destroy locks.
1192 */
1193 static void
1194 ixgbe_destroy_locks(ixgbe_t *ixgbe)
1195 {
1196 ixgbe_rx_ring_t *rx_ring;
1197 ixgbe_tx_ring_t *tx_ring;
1198 int i;
1199
1200 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1201 rx_ring = &ixgbe->rx_rings[i];
1202 mutex_destroy(&rx_ring->rx_lock);
1203 }
1204
1205 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1206 tx_ring = &ixgbe->tx_rings[i];
1207 mutex_destroy(&tx_ring->tx_lock);
1208 mutex_destroy(&tx_ring->recycle_lock);
1209 mutex_destroy(&tx_ring->tcb_head_lock);
1210 mutex_destroy(&tx_ring->tcb_tail_lock);
1211 }
1212
1213 mutex_destroy(&ixgbe->gen_lock);
1214 mutex_destroy(&ixgbe->watchdog_lock);
1215 }
1216
1217 static int
1218 ixgbe_resume(dev_info_t *devinfo)
1219 {
1220 ixgbe_t *ixgbe;
1221 int i;
1222
1223 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
1224 if (ixgbe == NULL)
1225 return (DDI_FAILURE);
1226
1227 mutex_enter(&ixgbe->gen_lock);
1228
1229 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
1230 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1231 mutex_exit(&ixgbe->gen_lock);
1232 return (DDI_FAILURE);
1233 }
1234
1235 /*
1236 * Enable and start the watchdog timer
1237 */
1238 ixgbe_enable_watchdog_timer(ixgbe);
1239 }
1240
1241 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_SUSPENDED);
1242
1243 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
1244 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1245 mac_tx_ring_update(ixgbe->mac_hdl,
1246 ixgbe->tx_rings[i].ring_handle);
1247 }
1248 }
1249
1250 mutex_exit(&ixgbe->gen_lock);
1251
1252 return (DDI_SUCCESS);
1253 }
1254
1255 static int
1256 ixgbe_suspend(dev_info_t *devinfo)
1257 {
1258 ixgbe_t *ixgbe;
1259
1260 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
1261 if (ixgbe == NULL)
1262 return (DDI_FAILURE);
1263
1264 mutex_enter(&ixgbe->gen_lock);
1265
1266 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_SUSPENDED);
1267 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) {
1268 mutex_exit(&ixgbe->gen_lock);
1269 return (DDI_SUCCESS);
1270 }
1271 ixgbe_stop(ixgbe, B_FALSE);
1272
1273 mutex_exit(&ixgbe->gen_lock);
1274
1275 /*
1276 * Disable and stop the watchdog timer
1277 */
1278 ixgbe_disable_watchdog_timer(ixgbe);
1279
1280 return (DDI_SUCCESS);
1281 }
1282
1283 /*
1284 * ixgbe_init - Initialize the device.
1285 */
1286 static int
1287 ixgbe_init(ixgbe_t *ixgbe)
1288 {
1289 struct ixgbe_hw *hw = &ixgbe->hw;
1290 u8 pbanum[IXGBE_PBANUM_LENGTH];
1291
1292 mutex_enter(&ixgbe->gen_lock);
1293
1294 /*
1295 * Reset chipset to put the hardware in a known state
1296 * before we try to do anything with the eeprom.
1297 */
1298 if (ixgbe_reset_hw(hw) != IXGBE_SUCCESS) {
1299 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1300 goto init_fail;
1301 }
1302
1303 /*
1304 * Need to init eeprom before validating the checksum.
1305 */
1306 if (ixgbe_init_eeprom_params(hw) < 0) {
1307 ixgbe_error(ixgbe,
1308 "Unable to intitialize the eeprom interface.");
1309 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1310 goto init_fail;
1311 }
1312
1313 /*
1314 * NVM validation
1315 */
1316 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1317 /*
1318 * Some PCI-E parts fail the first check due to
1319 * the link being in sleep state. Call it again,
1320 * if it fails a second time it's a real issue.
1321 */
1322 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1323 ixgbe_error(ixgbe,
1324 "Invalid NVM checksum. Please contact "
1325 "the vendor to update the NVM.");
1326 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1327 goto init_fail;
1328 }
1329 }
1330
1331 /*
1332 * Setup default flow control thresholds - enable/disable
1333 * & flow control type is controlled by ixgbe.conf
1334 */
1335 hw->fc.high_water[0] = DEFAULT_FCRTH;
1336 hw->fc.low_water[0] = DEFAULT_FCRTL;
1337 hw->fc.pause_time = DEFAULT_FCPAUSE;
1338 hw->fc.send_xon = B_TRUE;
1339
1340 /*
1341 * Initialize link settings
1342 */
1343 (void) ixgbe_driver_setup_link(ixgbe, B_FALSE);
1344
1345 /*
1346 * Initialize the chipset hardware
1347 */
1348 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1349 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1350 goto init_fail;
1351 }
1352
1353 /*
1354 * Read identifying information and place in devinfo.
1355 */
1356 pbanum[0] = '\0';
1357 (void) ixgbe_read_pba_string(hw, pbanum, sizeof (pbanum));
1358 if (*pbanum != '\0') {
1359 (void) ddi_prop_update_string(DDI_DEV_T_NONE, ixgbe->dip,
1360 "printed-board-assembly", (char *)pbanum);
1361 }
1362
1363 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1364 goto init_fail;
1365 }
1366
1367 mutex_exit(&ixgbe->gen_lock);
1368 return (IXGBE_SUCCESS);
1369
1370 init_fail:
1371 /*
1372 * Reset PHY
1373 */
1374 (void) ixgbe_reset_phy(hw);
1375
1376 mutex_exit(&ixgbe->gen_lock);
1377 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1378 return (IXGBE_FAILURE);
1379 }
1380
1381 /*
1382 * ixgbe_chip_start - Initialize and start the chipset hardware.
1383 */
1384 static int
1385 ixgbe_chip_start(ixgbe_t *ixgbe)
1386 {
1387 struct ixgbe_hw *hw = &ixgbe->hw;
1388 int ret_val, i;
1389
1390 ASSERT(mutex_owned(&ixgbe->gen_lock));
1391
1392 /*
1393 * Get the mac address
1394 * This function should handle SPARC case correctly.
1395 */
1396 if (!ixgbe_find_mac_address(ixgbe)) {
1397 ixgbe_error(ixgbe, "Failed to get the mac address");
1398 return (IXGBE_FAILURE);
1399 }
1400
1401 /*
1402 * Validate the mac address
1403 */
1404 (void) ixgbe_init_rx_addrs(hw);
1405 if (!is_valid_mac_addr(hw->mac.addr)) {
1406 ixgbe_error(ixgbe, "Invalid mac address");
1407 return (IXGBE_FAILURE);
1408 }
1409
1410 /*
1411 * Configure/Initialize hardware
1412 */
1413 ret_val = ixgbe_init_hw(hw);
1414 if (ret_val != IXGBE_SUCCESS) {
1415 if (ret_val == IXGBE_ERR_EEPROM_VERSION) {
1416 ixgbe_error(ixgbe,
1417 "This 82599 device is pre-release and contains"
1418 " outdated firmware, please contact your hardware"
1419 " vendor for a replacement.");
1420 } else {
1421 ixgbe_error(ixgbe, "Failed to initialize hardware");
1422 return (IXGBE_FAILURE);
1423 }
1424 }
1425
1426 /*
1427 * Re-enable relaxed ordering for performance. It is disabled
1428 * by default in the hardware init.
1429 */
1430 if (ixgbe->relax_order_enable == B_TRUE)
1431 ixgbe_enable_relaxed_ordering(hw);
1432
1433 /*
1434 * Setup adapter interrupt vectors
1435 */
1436 ixgbe_setup_adapter_vector(ixgbe);
1437
1438 /*
1439 * Initialize unicast addresses.
1440 */
1441 ixgbe_init_unicst(ixgbe);
1442
1443 /*
1444 * Setup and initialize the mctable structures.
1445 */
1446 ixgbe_setup_multicst(ixgbe);
1447
1448 /*
1449 * Set interrupt throttling rate
1450 */
1451 for (i = 0; i < ixgbe->intr_cnt; i++) {
1452 IXGBE_WRITE_REG(hw, IXGBE_EITR(i), ixgbe->intr_throttling[i]);
1453 }
1454
1455 /*
1456 * Save the state of the phy
1457 */
1458 ixgbe_get_hw_state(ixgbe);
1459
1460 /*
1461 * Make sure driver has control
1462 */
1463 ixgbe_get_driver_control(hw);
1464
1465 return (IXGBE_SUCCESS);
1466 }
1467
1468 /*
1469 * ixgbe_chip_stop - Stop the chipset hardware
1470 */
1471 static void
1472 ixgbe_chip_stop(ixgbe_t *ixgbe)
1473 {
1474 struct ixgbe_hw *hw = &ixgbe->hw;
1475
1476 ASSERT(mutex_owned(&ixgbe->gen_lock));
1477
1478 /*
1479 * Tell firmware driver is no longer in control
1480 */
1481 ixgbe_release_driver_control(hw);
1482
1483 /*
1484 * Reset the chipset
1485 */
1486 (void) ixgbe_reset_hw(hw);
1487
1488 /*
1489 * Reset PHY
1490 */
1491 (void) ixgbe_reset_phy(hw);
1492 }
1493
1494 /*
1495 * ixgbe_reset - Reset the chipset and re-start the driver.
1496 *
1497 * It involves stopping and re-starting the chipset,
1498 * and re-configuring the rx/tx rings.
1499 */
1500 static int
1501 ixgbe_reset(ixgbe_t *ixgbe)
1502 {
1503 int i;
1504
1505 /*
1506 * Disable and stop the watchdog timer
1507 */
1508 ixgbe_disable_watchdog_timer(ixgbe);
1509
1510 mutex_enter(&ixgbe->gen_lock);
1511
1512 ASSERT(ixgbe->ixgbe_state & IXGBE_STARTED);
1513 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED);
1514
1515 ixgbe_stop(ixgbe, B_FALSE);
1516
1517 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1518 mutex_exit(&ixgbe->gen_lock);
1519 return (IXGBE_FAILURE);
1520 }
1521
1522 /*
1523 * After resetting, need to recheck the link status.
1524 */
1525 ixgbe->link_check_complete = B_FALSE;
1526 ixgbe->link_check_hrtime = gethrtime() +
1527 (IXGBE_LINK_UP_TIME * 100000000ULL);
1528
1529 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STARTED);
1530
1531 if (!(ixgbe->ixgbe_state & IXGBE_SUSPENDED)) {
1532 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1533 mac_tx_ring_update(ixgbe->mac_hdl,
1534 ixgbe->tx_rings[i].ring_handle);
1535 }
1536 }
1537
1538 mutex_exit(&ixgbe->gen_lock);
1539
1540 /*
1541 * Enable and start the watchdog timer
1542 */
1543 ixgbe_enable_watchdog_timer(ixgbe);
1544
1545 return (IXGBE_SUCCESS);
1546 }
1547
1548 /*
1549 * ixgbe_tx_clean - Clean the pending transmit packets and DMA resources.
1550 */
1551 static void
1552 ixgbe_tx_clean(ixgbe_t *ixgbe)
1553 {
1554 ixgbe_tx_ring_t *tx_ring;
1555 tx_control_block_t *tcb;
1556 link_list_t pending_list;
1557 uint32_t desc_num;
1558 int i, j;
1559
1560 LINK_LIST_INIT(&pending_list);
1561
1562 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1563 tx_ring = &ixgbe->tx_rings[i];
1564
1565 mutex_enter(&tx_ring->recycle_lock);
1566
1567 /*
1568 * Clean the pending tx data - the pending packets in the
1569 * work_list that have no chances to be transmitted again.
1570 *
1571 * We must ensure the chipset is stopped or the link is down
1572 * before cleaning the transmit packets.
1573 */
1574 desc_num = 0;
1575 for (j = 0; j < tx_ring->ring_size; j++) {
1576 tcb = tx_ring->work_list[j];
1577 if (tcb != NULL) {
1578 desc_num += tcb->desc_num;
1579
1580 tx_ring->work_list[j] = NULL;
1581
1582 ixgbe_free_tcb(tcb);
1583
1584 LIST_PUSH_TAIL(&pending_list, &tcb->link);
1585 }
1586 }
1587
1588 if (desc_num > 0) {
1589 atomic_add_32(&tx_ring->tbd_free, desc_num);
1590 ASSERT(tx_ring->tbd_free == tx_ring->ring_size);
1591
1592 /*
1593 * Reset the head and tail pointers of the tbd ring;
1594 * Reset the writeback head if it's enable.
1595 */
1596 tx_ring->tbd_head = 0;
1597 tx_ring->tbd_tail = 0;
1598 if (ixgbe->tx_head_wb_enable)
1599 *tx_ring->tbd_head_wb = 0;
1600
1601 IXGBE_WRITE_REG(&ixgbe->hw,
1602 IXGBE_TDH(tx_ring->index), 0);
1603 IXGBE_WRITE_REG(&ixgbe->hw,
1604 IXGBE_TDT(tx_ring->index), 0);
1605 }
1606
1607 mutex_exit(&tx_ring->recycle_lock);
1608
1609 /*
1610 * Add the tx control blocks in the pending list to
1611 * the free list.
1612 */
1613 ixgbe_put_free_list(tx_ring, &pending_list);
1614 }
1615 }
1616
1617 /*
1618 * ixgbe_tx_drain - Drain the tx rings to allow pending packets to be
1619 * transmitted.
1620 */
1621 static boolean_t
1622 ixgbe_tx_drain(ixgbe_t *ixgbe)
1623 {
1624 ixgbe_tx_ring_t *tx_ring;
1625 boolean_t done;
1626 int i, j;
1627
1628 /*
1629 * Wait for a specific time to allow pending tx packets
1630 * to be transmitted.
1631 *
1632 * Check the counter tbd_free to see if transmission is done.
1633 * No lock protection is needed here.
1634 *
1635 * Return B_TRUE if all pending packets have been transmitted;
1636 * Otherwise return B_FALSE;
1637 */
1638 for (i = 0; i < TX_DRAIN_TIME; i++) {
1639
1640 done = B_TRUE;
1641 for (j = 0; j < ixgbe->num_tx_rings; j++) {
1642 tx_ring = &ixgbe->tx_rings[j];
1643 done = done &&
1644 (tx_ring->tbd_free == tx_ring->ring_size);
1645 }
1646
1647 if (done)
1648 break;
1649
1650 msec_delay(1);
1651 }
1652
1653 return (done);
1654 }
1655
1656 /*
1657 * ixgbe_rx_drain - Wait for all rx buffers to be released by upper layer.
1658 */
1659 static boolean_t
1660 ixgbe_rx_drain(ixgbe_t *ixgbe)
1661 {
1662 boolean_t done = B_TRUE;
1663 int i;
1664
1665 /*
1666 * Polling the rx free list to check if those rx buffers held by
1667 * the upper layer are released.
1668 *
1669 * Check the counter rcb_free to see if all pending buffers are
1670 * released. No lock protection is needed here.
1671 *
1672 * Return B_TRUE if all pending buffers have been released;
1673 * Otherwise return B_FALSE;
1674 */
1675 for (i = 0; i < RX_DRAIN_TIME; i++) {
1676 done = (ixgbe->rcb_pending == 0);
1677
1678 if (done)
1679 break;
1680
1681 msec_delay(1);
1682 }
1683
1684 return (done);
1685 }
1686
1687 /*
1688 * ixgbe_start - Start the driver/chipset.
1689 */
1690 int
1691 ixgbe_start(ixgbe_t *ixgbe, boolean_t alloc_buffer)
1692 {
1693 int i;
1694
1695 ASSERT(mutex_owned(&ixgbe->gen_lock));
1696
1697 if (alloc_buffer) {
1698 if (ixgbe_alloc_rx_data(ixgbe) != IXGBE_SUCCESS) {
1699 ixgbe_error(ixgbe,
1700 "Failed to allocate software receive rings");
1701 return (IXGBE_FAILURE);
1702 }
1703
1704 /* Allocate buffers for all the rx/tx rings */
1705 if (ixgbe_alloc_dma(ixgbe) != IXGBE_SUCCESS) {
1706 ixgbe_error(ixgbe, "Failed to allocate DMA resource");
1707 return (IXGBE_FAILURE);
1708 }
1709
1710 ixgbe->tx_ring_init = B_TRUE;
1711 } else {
1712 ixgbe->tx_ring_init = B_FALSE;
1713 }
1714
1715 for (i = 0; i < ixgbe->num_rx_rings; i++)
1716 mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1717 for (i = 0; i < ixgbe->num_tx_rings; i++)
1718 mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1719
1720 /*
1721 * Start the chipset hardware
1722 */
1723 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1724 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1725 goto start_failure;
1726 }
1727
1728 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1729 goto start_failure;
1730 }
1731
1732 /*
1733 * Setup the rx/tx rings
1734 */
1735 ixgbe_setup_rings(ixgbe);
1736
1737 /*
1738 * ixgbe_start() will be called when resetting, however if reset
1739 * happens, we need to clear the ERROR, STALL and OVERTEMP flags
1740 * before enabling the interrupts.
1741 */
1742 atomic_and_32(&ixgbe->ixgbe_state, ~(IXGBE_ERROR
1743 | IXGBE_STALL| IXGBE_OVERTEMP));
1744
1745 /*
1746 * Enable adapter interrupts
1747 * The interrupts must be enabled after the driver state is START
1748 */
1749 ixgbe_enable_adapter_interrupts(ixgbe);
1750
1751 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1752 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1753 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1754 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1755
1756 return (IXGBE_SUCCESS);
1757
1758 start_failure:
1759 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1760 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1761 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1762 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1763
1764 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1765
1766 return (IXGBE_FAILURE);
1767 }
1768
1769 /*
1770 * ixgbe_stop - Stop the driver/chipset.
1771 */
1772 void
1773 ixgbe_stop(ixgbe_t *ixgbe, boolean_t free_buffer)
1774 {
1775 int i;
1776
1777 ASSERT(mutex_owned(&ixgbe->gen_lock));
1778
1779 /*
1780 * Disable the adapter interrupts
1781 */
1782 ixgbe_disable_adapter_interrupts(ixgbe);
1783
1784 /*
1785 * Drain the pending tx packets
1786 */
1787 (void) ixgbe_tx_drain(ixgbe);
1788
1789 for (i = 0; i < ixgbe->num_rx_rings; i++)
1790 mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1791 for (i = 0; i < ixgbe->num_tx_rings; i++)
1792 mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1793
1794 /*
1795 * Stop the chipset hardware
1796 */
1797 ixgbe_chip_stop(ixgbe);
1798
1799 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1800 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1801 }
1802
1803 /*
1804 * Clean the pending tx data/resources
1805 */
1806 ixgbe_tx_clean(ixgbe);
1807
1808 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1809 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1810 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1811 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1812
1813 if (ixgbe->link_state == LINK_STATE_UP) {
1814 ixgbe->link_state = LINK_STATE_UNKNOWN;
1815 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
1816 }
1817
1818 if (free_buffer) {
1819 /*
1820 * Release the DMA/memory resources of rx/tx rings
1821 */
1822 ixgbe_free_dma(ixgbe);
1823 ixgbe_free_rx_data(ixgbe);
1824 }
1825 }
1826
1827 /*
1828 * ixgbe_cbfunc - Driver interface for generic DDI callbacks
1829 */
1830 /* ARGSUSED */
1831 static int
1832 ixgbe_cbfunc(dev_info_t *dip, ddi_cb_action_t cbaction, void *cbarg,
1833 void *arg1, void *arg2)
1834 {
1835 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
1836
1837 switch (cbaction) {
1838 /* IRM callback */
1839 int count;
1840 case DDI_CB_INTR_ADD:
1841 case DDI_CB_INTR_REMOVE:
1842 count = (int)(uintptr_t)cbarg;
1843 ASSERT(ixgbe->intr_type == DDI_INTR_TYPE_MSIX);
1844 DTRACE_PROBE2(ixgbe__irm__callback, int, count,
1845 int, ixgbe->intr_cnt);
1846 if (ixgbe_intr_adjust(ixgbe, cbaction, count) !=
1847 DDI_SUCCESS) {
1848 ixgbe_error(ixgbe,
1849 "IRM CB: Failed to adjust interrupts");
1850 goto cb_fail;
1851 }
1852 break;
1853 default:
1854 IXGBE_DEBUGLOG_1(ixgbe, "DDI CB: action 0x%x NOT supported",
1855 cbaction);
1856 return (DDI_ENOTSUP);
1857 }
1858 return (DDI_SUCCESS);
1859 cb_fail:
1860 return (DDI_FAILURE);
1861 }
1862
1863 /*
1864 * ixgbe_intr_adjust - Adjust interrupt to respond to IRM request.
1865 */
1866 static int
1867 ixgbe_intr_adjust(ixgbe_t *ixgbe, ddi_cb_action_t cbaction, int count)
1868 {
1869 int i, rc, actual;
1870
1871 if (count == 0)
1872 return (DDI_SUCCESS);
1873
1874 if ((cbaction == DDI_CB_INTR_ADD &&
1875 ixgbe->intr_cnt + count > ixgbe->intr_cnt_max) ||
1876 (cbaction == DDI_CB_INTR_REMOVE &&
1877 ixgbe->intr_cnt - count < ixgbe->intr_cnt_min))
1878 return (DDI_FAILURE);
1879
1880 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) {
1881 return (DDI_FAILURE);
1882 }
1883
1884 for (i = 0; i < ixgbe->num_rx_rings; i++)
1885 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle, NULL);
1886 for (i = 0; i < ixgbe->num_tx_rings; i++)
1887 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle, NULL);
1888
1889 mutex_enter(&ixgbe->gen_lock);
1890 ixgbe->ixgbe_state &= ~IXGBE_STARTED;
1891 ixgbe->ixgbe_state |= IXGBE_INTR_ADJUST;
1892 ixgbe->ixgbe_state |= IXGBE_SUSPENDED;
1893 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN);
1894
1895 ixgbe_stop(ixgbe, B_FALSE);
1896 /*
1897 * Disable interrupts
1898 */
1899 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
1900 rc = ixgbe_disable_intrs(ixgbe);
1901 ASSERT(rc == IXGBE_SUCCESS);
1902 }
1903 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ENABLE_INTR;
1904
1905 /*
1906 * Remove interrupt handlers
1907 */
1908 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
1909 ixgbe_rem_intr_handlers(ixgbe);
1910 }
1911 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ADD_INTR;
1912
1913 /*
1914 * Clear vect_map
1915 */
1916 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map));
1917 switch (cbaction) {
1918 case DDI_CB_INTR_ADD:
1919 rc = ddi_intr_alloc(ixgbe->dip, ixgbe->htable,
1920 DDI_INTR_TYPE_MSIX, ixgbe->intr_cnt, count, &actual,
1921 DDI_INTR_ALLOC_NORMAL);
1922 if (rc != DDI_SUCCESS || actual != count) {
1923 ixgbe_log(ixgbe, "Adjust interrupts failed."
1924 "return: %d, irm cb size: %d, actual: %d",
1925 rc, count, actual);
1926 goto intr_adjust_fail;
1927 }
1928 ixgbe->intr_cnt += count;
1929 break;
1930
1931 case DDI_CB_INTR_REMOVE:
1932 for (i = ixgbe->intr_cnt - count;
1933 i < ixgbe->intr_cnt; i ++) {
1934 rc = ddi_intr_free(ixgbe->htable[i]);
1935 ixgbe->htable[i] = NULL;
1936 if (rc != DDI_SUCCESS) {
1937 ixgbe_log(ixgbe, "Adjust interrupts failed."
1938 "return: %d, irm cb size: %d, actual: %d",
1939 rc, count, actual);
1940 goto intr_adjust_fail;
1941 }
1942 }
1943 ixgbe->intr_cnt -= count;
1944 break;
1945 }
1946
1947 /*
1948 * Get priority for first vector, assume remaining are all the same
1949 */
1950 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri);
1951 if (rc != DDI_SUCCESS) {
1952 ixgbe_log(ixgbe,
1953 "Get interrupt priority failed: %d", rc);
1954 goto intr_adjust_fail;
1955 }
1956 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap);
1957 if (rc != DDI_SUCCESS) {
1958 ixgbe_log(ixgbe, "Get interrupt cap failed: %d", rc);
1959 goto intr_adjust_fail;
1960 }
1961 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
1962
1963 /*
1964 * Map rings to interrupt vectors
1965 */
1966 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) {
1967 ixgbe_error(ixgbe,
1968 "IRM CB: Failed to map interrupts to vectors");
1969 goto intr_adjust_fail;
1970 }
1971
1972 /*
1973 * Add interrupt handlers
1974 */
1975 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) {
1976 ixgbe_error(ixgbe, "IRM CB: Failed to add interrupt handlers");
1977 goto intr_adjust_fail;
1978 }
1979 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
1980
1981 /*
1982 * Now that mutex locks are initialized, and the chip is also
1983 * initialized, enable interrupts.
1984 */
1985 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) {
1986 ixgbe_error(ixgbe, "IRM CB: Failed to enable DDI interrupts");
1987 goto intr_adjust_fail;
1988 }
1989 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
1990 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1991 ixgbe_error(ixgbe, "IRM CB: Failed to start");
1992 goto intr_adjust_fail;
1993 }
1994 ixgbe->ixgbe_state &= ~IXGBE_INTR_ADJUST;
1995 ixgbe->ixgbe_state &= ~IXGBE_SUSPENDED;
1996 ixgbe->ixgbe_state |= IXGBE_STARTED;
1997 mutex_exit(&ixgbe->gen_lock);
1998
1999 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2000 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle,
2001 ixgbe->htable[ixgbe->rx_rings[i].intr_vector]);
2002 }
2003 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2004 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle,
2005 ixgbe->htable[ixgbe->tx_rings[i].intr_vector]);
2006 }
2007
2008 /* Wakeup all Tx rings */
2009 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2010 mac_tx_ring_update(ixgbe->mac_hdl,
2011 ixgbe->tx_rings[i].ring_handle);
2012 }
2013
2014 IXGBE_DEBUGLOG_3(ixgbe,
2015 "IRM CB: interrupts new value: 0x%x(0x%x:0x%x).",
2016 ixgbe->intr_cnt, ixgbe->intr_cnt_min, ixgbe->intr_cnt_max);
2017 return (DDI_SUCCESS);
2018
2019 intr_adjust_fail:
2020 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
2021 mutex_exit(&ixgbe->gen_lock);
2022 return (DDI_FAILURE);
2023 }
2024
2025 /*
2026 * ixgbe_intr_cb_register - Register interrupt callback function.
2027 */
2028 static int
2029 ixgbe_intr_cb_register(ixgbe_t *ixgbe)
2030 {
2031 if (ddi_cb_register(ixgbe->dip, DDI_CB_FLAG_INTR, ixgbe_cbfunc,
2032 ixgbe, NULL, &ixgbe->cb_hdl) != DDI_SUCCESS) {
2033 return (IXGBE_FAILURE);
2034 }
2035 IXGBE_DEBUGLOG_0(ixgbe, "Interrupt callback function registered.");
2036 return (IXGBE_SUCCESS);
2037 }
2038
2039 /*
2040 * ixgbe_alloc_rings - Allocate memory space for rx/tx rings.
2041 */
2042 static int
2043 ixgbe_alloc_rings(ixgbe_t *ixgbe)
2044 {
2045 /*
2046 * Allocate memory space for rx rings
2047 */
2048 ixgbe->rx_rings = kmem_zalloc(
2049 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings,
2050 KM_NOSLEEP);
2051
2052 if (ixgbe->rx_rings == NULL) {
2053 return (IXGBE_FAILURE);
2054 }
2055
2056 /*
2057 * Allocate memory space for tx rings
2058 */
2059 ixgbe->tx_rings = kmem_zalloc(
2060 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings,
2061 KM_NOSLEEP);
2062
2063 if (ixgbe->tx_rings == NULL) {
2064 kmem_free(ixgbe->rx_rings,
2065 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2066 ixgbe->rx_rings = NULL;
2067 return (IXGBE_FAILURE);
2068 }
2069
2070 /*
2071 * Allocate memory space for rx ring groups
2072 */
2073 ixgbe->rx_groups = kmem_zalloc(
2074 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups,
2075 KM_NOSLEEP);
2076
2077 if (ixgbe->rx_groups == NULL) {
2078 kmem_free(ixgbe->rx_rings,
2079 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2080 kmem_free(ixgbe->tx_rings,
2081 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings);
2082 ixgbe->rx_rings = NULL;
2083 ixgbe->tx_rings = NULL;
2084 return (IXGBE_FAILURE);
2085 }
2086
2087 return (IXGBE_SUCCESS);
2088 }
2089
2090 /*
2091 * ixgbe_free_rings - Free the memory space of rx/tx rings.
2092 */
2093 static void
2094 ixgbe_free_rings(ixgbe_t *ixgbe)
2095 {
2096 if (ixgbe->rx_rings != NULL) {
2097 kmem_free(ixgbe->rx_rings,
2098 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2099 ixgbe->rx_rings = NULL;
2100 }
2101
2102 if (ixgbe->tx_rings != NULL) {
2103 kmem_free(ixgbe->tx_rings,
2104 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings);
2105 ixgbe->tx_rings = NULL;
2106 }
2107
2108 if (ixgbe->rx_groups != NULL) {
2109 kmem_free(ixgbe->rx_groups,
2110 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups);
2111 ixgbe->rx_groups = NULL;
2112 }
2113 }
2114
2115 static int
2116 ixgbe_alloc_rx_data(ixgbe_t *ixgbe)
2117 {
2118 ixgbe_rx_ring_t *rx_ring;
2119 int i;
2120
2121 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2122 rx_ring = &ixgbe->rx_rings[i];
2123 if (ixgbe_alloc_rx_ring_data(rx_ring) != IXGBE_SUCCESS)
2124 goto alloc_rx_rings_failure;
2125 }
2126 return (IXGBE_SUCCESS);
2127
2128 alloc_rx_rings_failure:
2129 ixgbe_free_rx_data(ixgbe);
2130 return (IXGBE_FAILURE);
2131 }
2132
2133 static void
2134 ixgbe_free_rx_data(ixgbe_t *ixgbe)
2135 {
2136 ixgbe_rx_ring_t *rx_ring;
2137 ixgbe_rx_data_t *rx_data;
2138 int i;
2139
2140 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2141 rx_ring = &ixgbe->rx_rings[i];
2142
2143 mutex_enter(&ixgbe->rx_pending_lock);
2144 rx_data = rx_ring->rx_data;
2145
2146 if (rx_data != NULL) {
2147 rx_data->flag |= IXGBE_RX_STOPPED;
2148
2149 if (rx_data->rcb_pending == 0) {
2150 ixgbe_free_rx_ring_data(rx_data);
2151 rx_ring->rx_data = NULL;
2152 }
2153 }
2154
2155 mutex_exit(&ixgbe->rx_pending_lock);
2156 }
2157 }
2158
2159 /*
2160 * ixgbe_setup_rings - Setup rx/tx rings.
2161 */
2162 static void
2163 ixgbe_setup_rings(ixgbe_t *ixgbe)
2164 {
2165 /*
2166 * Setup the rx/tx rings, including the following:
2167 *
2168 * 1. Setup the descriptor ring and the control block buffers;
2169 * 2. Initialize necessary registers for receive/transmit;
2170 * 3. Initialize software pointers/parameters for receive/transmit;
2171 */
2172 ixgbe_setup_rx(ixgbe);
2173
2174 ixgbe_setup_tx(ixgbe);
2175 }
2176
2177 static void
2178 ixgbe_setup_rx_ring(ixgbe_rx_ring_t *rx_ring)
2179 {
2180 ixgbe_t *ixgbe = rx_ring->ixgbe;
2181 ixgbe_rx_data_t *rx_data = rx_ring->rx_data;
2182 struct ixgbe_hw *hw = &ixgbe->hw;
2183 rx_control_block_t *rcb;
2184 union ixgbe_adv_rx_desc *rbd;
2185 uint32_t size;
2186 uint32_t buf_low;
2187 uint32_t buf_high;
2188 uint32_t reg_val;
2189 int i;
2190
2191 ASSERT(mutex_owned(&rx_ring->rx_lock));
2192 ASSERT(mutex_owned(&ixgbe->gen_lock));
2193
2194 for (i = 0; i < ixgbe->rx_ring_size; i++) {
2195 rcb = rx_data->work_list[i];
2196 rbd = &rx_data->rbd_ring[i];
2197
2198 rbd->read.pkt_addr = rcb->rx_buf.dma_address;
2199 rbd->read.hdr_addr = NULL;
2200 }
2201
2202 /*
2203 * Initialize the length register
2204 */
2205 size = rx_data->ring_size * sizeof (union ixgbe_adv_rx_desc);
2206 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rx_ring->hw_index), size);
2207
2208 /*
2209 * Initialize the base address registers
2210 */
2211 buf_low = (uint32_t)rx_data->rbd_area.dma_address;
2212 buf_high = (uint32_t)(rx_data->rbd_area.dma_address >> 32);
2213 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rx_ring->hw_index), buf_high);
2214 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rx_ring->hw_index), buf_low);
2215
2216 /*
2217 * Setup head & tail pointers
2218 */
2219 IXGBE_WRITE_REG(hw, IXGBE_RDT(rx_ring->hw_index),
2220 rx_data->ring_size - 1);
2221 IXGBE_WRITE_REG(hw, IXGBE_RDH(rx_ring->hw_index), 0);
2222
2223 rx_data->rbd_next = 0;
2224 rx_data->lro_first = 0;
2225
2226 /*
2227 * Setup the Receive Descriptor Control Register (RXDCTL)
2228 * PTHRESH=32 descriptors (half the internal cache)
2229 * HTHRESH=0 descriptors (to minimize latency on fetch)
2230 * WTHRESH defaults to 1 (writeback each descriptor)
2231 */
2232 reg_val = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index));
2233 reg_val |= IXGBE_RXDCTL_ENABLE; /* enable queue */
2234
2235 /* Not a valid value for 82599, X540 or X550 */
2236 if (hw->mac.type == ixgbe_mac_82598EB) {
2237 reg_val |= 0x0020; /* pthresh */
2238 }
2239 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index), reg_val);
2240
2241 if (hw->mac.type == ixgbe_mac_82599EB ||
2242 hw->mac.type == ixgbe_mac_X540 ||
2243 hw->mac.type == ixgbe_mac_X550 ||
2244 hw->mac.type == ixgbe_mac_X550EM_x) {
2245 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2246 reg_val |= (IXGBE_RDRXCTL_CRCSTRIP | IXGBE_RDRXCTL_AGGDIS);
2247 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val);
2248 }
2249
2250 /*
2251 * Setup the Split and Replication Receive Control Register.
2252 * Set the rx buffer size and the advanced descriptor type.
2253 */
2254 reg_val = (ixgbe->rx_buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) |
2255 IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
2256 reg_val |= IXGBE_SRRCTL_DROP_EN;
2257 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rx_ring->hw_index), reg_val);
2258 }
2259
2260 static void
2261 ixgbe_setup_rx(ixgbe_t *ixgbe)
2262 {
2263 ixgbe_rx_ring_t *rx_ring;
2264 struct ixgbe_hw *hw = &ixgbe->hw;
2265 uint32_t reg_val;
2266 uint32_t ring_mapping;
2267 uint32_t i, index;
2268 uint32_t psrtype_rss_bit;
2269
2270 /* PSRTYPE must be configured for 82599 */
2271 if (ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ &&
2272 ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ_RSS) {
2273 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2274 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2275 reg_val |= IXGBE_PSRTYPE_L2HDR;
2276 reg_val |= 0x80000000;
2277 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), reg_val);
2278 } else {
2279 if (ixgbe->num_rx_groups > 32) {
2280 psrtype_rss_bit = 0x20000000;
2281 } else {
2282 psrtype_rss_bit = 0x40000000;
2283 }
2284 for (i = 0; i < ixgbe->capab->max_rx_grp_num; i++) {
2285 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2286 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2287 reg_val |= IXGBE_PSRTYPE_L2HDR;
2288 reg_val |= psrtype_rss_bit;
2289 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(i), reg_val);
2290 }
2291 }
2292
2293 /*
2294 * Set filter control in FCTRL to accept broadcast packets and do
2295 * not pass pause frames to host. Flow control settings are already
2296 * in this register, so preserve them.
2297 */
2298 reg_val = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2299 reg_val |= IXGBE_FCTRL_BAM; /* broadcast accept mode */
2300 reg_val |= IXGBE_FCTRL_DPF; /* discard pause frames */
2301 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, reg_val);
2302
2303 /*
2304 * Hardware checksum settings
2305 */
2306 if (ixgbe->rx_hcksum_enable) {
2307 reg_val = IXGBE_RXCSUM_IPPCSE; /* IP checksum */
2308 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, reg_val);
2309 }
2310
2311 /*
2312 * Setup VMDq and RSS for multiple receive queues
2313 */
2314 switch (ixgbe->classify_mode) {
2315 case IXGBE_CLASSIFY_RSS:
2316 /*
2317 * One group, only RSS is needed when more than
2318 * one ring enabled.
2319 */
2320 ixgbe_setup_rss(ixgbe);
2321 break;
2322
2323 case IXGBE_CLASSIFY_VMDQ:
2324 /*
2325 * Multiple groups, each group has one ring,
2326 * only VMDq is needed.
2327 */
2328 ixgbe_setup_vmdq(ixgbe);
2329 break;
2330
2331 case IXGBE_CLASSIFY_VMDQ_RSS:
2332 /*
2333 * Multiple groups and multiple rings, both
2334 * VMDq and RSS are needed.
2335 */
2336 ixgbe_setup_vmdq_rss(ixgbe);
2337 break;
2338
2339 default:
2340 break;
2341 }
2342
2343 /*
2344 * Enable the receive unit. This must be done after filter
2345 * control is set in FCTRL.
2346 */
2347 reg_val = (IXGBE_RXCTRL_RXEN /* Enable Receive Unit */
2348 | IXGBE_RXCTRL_DMBYPS); /* descriptor monitor bypass */
2349 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, reg_val);
2350
2351 /*
2352 * ixgbe_setup_rx_ring must be called after configuring RXCTRL
2353 */
2354 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2355 rx_ring = &ixgbe->rx_rings[i];
2356 ixgbe_setup_rx_ring(rx_ring);
2357 }
2358
2359 /*
2360 * Setup the per-ring statistics mapping.
2361 */
2362 ring_mapping = 0;
2363 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2364 index = ixgbe->rx_rings[i].hw_index;
2365 ring_mapping = IXGBE_READ_REG(hw, IXGBE_RQSMR(index >> 2));
2366 ring_mapping |= (i & 0xF) << (8 * (index & 0x3));
2367 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(index >> 2), ring_mapping);
2368 }
2369
2370 /*
2371 * The Max Frame Size in MHADD/MAXFRS will be internally increased
2372 * by four bytes if the packet has a VLAN field, so includes MTU,
2373 * ethernet header and frame check sequence.
2374 * Register is MAXFRS in 82599.
2375 */
2376 reg_val = (ixgbe->default_mtu + sizeof (struct ether_header)
2377 + ETHERFCSL) << IXGBE_MHADD_MFS_SHIFT;
2378 IXGBE_WRITE_REG(hw, IXGBE_MHADD, reg_val);
2379
2380 /*
2381 * Setup Jumbo Frame enable bit
2382 */
2383 if (ixgbe->default_mtu > ETHERMTU) {
2384 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2385 reg_val |= IXGBE_HLREG0_JUMBOEN;
2386 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2387 }
2388
2389 /*
2390 * Setup RSC for multiple receive queues.
2391 */
2392 if (ixgbe->lro_enable) {
2393 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2394 /*
2395 * Make sure rx_buf_size * MAXDESC not greater
2396 * than 65535.
2397 * Intel recommends 4 for MAXDESC field value.
2398 */
2399 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCCTL(i));
2400 reg_val |= IXGBE_RSCCTL_RSCEN;
2401 if (ixgbe->rx_buf_size == IXGBE_PKG_BUF_16k)
2402 reg_val |= IXGBE_RSCCTL_MAXDESC_1;
2403 else
2404 reg_val |= IXGBE_RSCCTL_MAXDESC_4;
2405 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(i), reg_val);
2406 }
2407
2408 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCDBU);
2409 reg_val |= IXGBE_RSCDBU_RSCACKDIS;
2410 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, reg_val);
2411
2412 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2413 reg_val |= IXGBE_RDRXCTL_RSCACKC;
2414 reg_val |= IXGBE_RDRXCTL_FCOE_WRFIX;
2415 reg_val &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
2416
2417 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val);
2418 }
2419 }
2420
2421 static void
2422 ixgbe_setup_tx_ring(ixgbe_tx_ring_t *tx_ring)
2423 {
2424 ixgbe_t *ixgbe = tx_ring->ixgbe;
2425 struct ixgbe_hw *hw = &ixgbe->hw;
2426 uint32_t size;
2427 uint32_t buf_low;
2428 uint32_t buf_high;
2429 uint32_t reg_val;
2430
2431 ASSERT(mutex_owned(&tx_ring->tx_lock));
2432 ASSERT(mutex_owned(&ixgbe->gen_lock));
2433
2434 /*
2435 * Initialize the length register
2436 */
2437 size = tx_ring->ring_size * sizeof (union ixgbe_adv_tx_desc);
2438 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(tx_ring->index), size);
2439
2440 /*
2441 * Initialize the base address registers
2442 */
2443 buf_low = (uint32_t)tx_ring->tbd_area.dma_address;
2444 buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32);
2445 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(tx_ring->index), buf_low);
2446 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(tx_ring->index), buf_high);
2447
2448 /*
2449 * Setup head & tail pointers
2450 */
2451 IXGBE_WRITE_REG(hw, IXGBE_TDH(tx_ring->index), 0);
2452 IXGBE_WRITE_REG(hw, IXGBE_TDT(tx_ring->index), 0);
2453
2454 /*
2455 * Setup head write-back
2456 */
2457 if (ixgbe->tx_head_wb_enable) {
2458 /*
2459 * The memory of the head write-back is allocated using
2460 * the extra tbd beyond the tail of the tbd ring.
2461 */
2462 tx_ring->tbd_head_wb = (uint32_t *)
2463 ((uintptr_t)tx_ring->tbd_area.address + size);
2464 *tx_ring->tbd_head_wb = 0;
2465
2466 buf_low = (uint32_t)
2467 (tx_ring->tbd_area.dma_address + size);
2468 buf_high = (uint32_t)
2469 ((tx_ring->tbd_area.dma_address + size) >> 32);
2470
2471 /* Set the head write-back enable bit */
2472 buf_low |= IXGBE_TDWBAL_HEAD_WB_ENABLE;
2473
2474 IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(tx_ring->index), buf_low);
2475 IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(tx_ring->index), buf_high);
2476
2477 /*
2478 * Turn off relaxed ordering for head write back or it will
2479 * cause problems with the tx recycling
2480 */
2481
2482 reg_val = (hw->mac.type == ixgbe_mac_82598EB) ?
2483 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(tx_ring->index)) :
2484 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(tx_ring->index));
2485 reg_val &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2486 if (hw->mac.type == ixgbe_mac_82598EB) {
2487 IXGBE_WRITE_REG(hw,
2488 IXGBE_DCA_TXCTRL(tx_ring->index), reg_val);
2489 } else {
2490 IXGBE_WRITE_REG(hw,
2491 IXGBE_DCA_TXCTRL_82599(tx_ring->index), reg_val);
2492 }
2493 } else {
2494 tx_ring->tbd_head_wb = NULL;
2495 }
2496
2497 tx_ring->tbd_head = 0;
2498 tx_ring->tbd_tail = 0;
2499 tx_ring->tbd_free = tx_ring->ring_size;
2500
2501 if (ixgbe->tx_ring_init == B_TRUE) {
2502 tx_ring->tcb_head = 0;
2503 tx_ring->tcb_tail = 0;
2504 tx_ring->tcb_free = tx_ring->free_list_size;
2505 }
2506
2507 /*
2508 * Initialize the s/w context structure
2509 */
2510 bzero(&tx_ring->tx_context, sizeof (ixgbe_tx_context_t));
2511 }
2512
2513 static void
2514 ixgbe_setup_tx(ixgbe_t *ixgbe)
2515 {
2516 struct ixgbe_hw *hw = &ixgbe->hw;
2517 ixgbe_tx_ring_t *tx_ring;
2518 uint32_t reg_val;
2519 uint32_t ring_mapping;
2520 int i;
2521
2522 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2523 tx_ring = &ixgbe->tx_rings[i];
2524 ixgbe_setup_tx_ring(tx_ring);
2525 }
2526
2527 /*
2528 * Setup the per-ring statistics mapping.
2529 */
2530 ring_mapping = 0;
2531 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2532 ring_mapping |= (i & 0xF) << (8 * (i & 0x3));
2533 if ((i & 0x3) == 0x3) {
2534 switch (hw->mac.type) {
2535 case ixgbe_mac_82598EB:
2536 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2),
2537 ring_mapping);
2538 break;
2539
2540 case ixgbe_mac_82599EB:
2541 case ixgbe_mac_X540:
2542 case ixgbe_mac_X550:
2543 case ixgbe_mac_X550EM_x:
2544 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2),
2545 ring_mapping);
2546 break;
2547
2548 default:
2549 break;
2550 }
2551
2552 ring_mapping = 0;
2553 }
2554 }
2555 if (i & 0x3) {
2556 switch (hw->mac.type) {
2557 case ixgbe_mac_82598EB:
2558 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), ring_mapping);
2559 break;
2560
2561 case ixgbe_mac_82599EB:
2562 case ixgbe_mac_X540:
2563 case ixgbe_mac_X550:
2564 case ixgbe_mac_X550EM_x:
2565 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), ring_mapping);
2566 break;
2567
2568 default:
2569 break;
2570 }
2571 }
2572
2573 /*
2574 * Enable CRC appending and TX padding (for short tx frames)
2575 */
2576 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2577 reg_val |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN;
2578 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2579
2580 /*
2581 * enable DMA for 82599, X540 and X550 parts
2582 */
2583 if (hw->mac.type == ixgbe_mac_82599EB ||
2584 hw->mac.type == ixgbe_mac_X540 ||
2585 hw->mac.type == ixgbe_mac_X550 ||
2586 hw->mac.type == ixgbe_mac_X550EM_x) {
2587 /* DMATXCTL.TE must be set after all Tx config is complete */
2588 reg_val = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2589 reg_val |= IXGBE_DMATXCTL_TE;
2590 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg_val);
2591
2592 /* Disable arbiter to set MTQC */
2593 reg_val = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2594 reg_val |= IXGBE_RTTDCS_ARBDIS;
2595 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
2596 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
2597 reg_val &= ~IXGBE_RTTDCS_ARBDIS;
2598 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
2599 }
2600
2601 /*
2602 * Enabling tx queues ..
2603 * For 82599 must be done after DMATXCTL.TE is set
2604 */
2605 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2606 tx_ring = &ixgbe->tx_rings[i];
2607 reg_val = IXGBE_READ_REG(hw, IXGBE_TXDCTL(tx_ring->index));
2608 reg_val |= IXGBE_TXDCTL_ENABLE;
2609 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(tx_ring->index), reg_val);
2610 }
2611 }
2612
2613 /*
2614 * ixgbe_setup_rss - Setup receive-side scaling feature.
2615 */
2616 static void
2617 ixgbe_setup_rss(ixgbe_t *ixgbe)
2618 {
2619 struct ixgbe_hw *hw = &ixgbe->hw;
2620 uint32_t i, mrqc, rxcsum;
2621 uint32_t random;
2622 uint32_t reta;
2623 uint32_t ring_per_group;
2624
2625 /*
2626 * Fill out redirection table
2627 */
2628 reta = 0;
2629 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2630
2631 for (i = 0; i < 128; i++) {
2632 reta = (reta << 8) | (i % ring_per_group) |
2633 ((i % ring_per_group) << 4);
2634 if ((i & 3) == 3)
2635 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
2636 }
2637
2638 /*
2639 * Fill out hash function seeds with a random constant
2640 */
2641 for (i = 0; i < 10; i++) {
2642 (void) random_get_pseudo_bytes((uint8_t *)&random,
2643 sizeof (uint32_t));
2644 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random);
2645 }
2646
2647 /*
2648 * Enable RSS & perform hash on these packet types
2649 */
2650 mrqc = IXGBE_MRQC_RSSEN |
2651 IXGBE_MRQC_RSS_FIELD_IPV4 |
2652 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2653 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2654 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2655 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2656 IXGBE_MRQC_RSS_FIELD_IPV6 |
2657 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2658 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2659 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2660 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2661
2662 /*
2663 * Disable Packet Checksum to enable RSS for multiple receive queues.
2664 * It is an adapter hardware limitation that Packet Checksum is
2665 * mutually exclusive with RSS.
2666 */
2667 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2668 rxcsum |= IXGBE_RXCSUM_PCSD;
2669 rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
2670 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
2671 }
2672
2673 /*
2674 * ixgbe_setup_vmdq - Setup MAC classification feature
2675 */
2676 static void
2677 ixgbe_setup_vmdq(ixgbe_t *ixgbe)
2678 {
2679 struct ixgbe_hw *hw = &ixgbe->hw;
2680 uint32_t vmdctl, i, vtctl;
2681
2682 /*
2683 * Setup the VMDq Control register, enable VMDq based on
2684 * packet destination MAC address:
2685 */
2686 switch (hw->mac.type) {
2687 case ixgbe_mac_82598EB:
2688 /*
2689 * VMDq Enable = 1;
2690 * VMDq Filter = 0; MAC filtering
2691 * Default VMDq output index = 0;
2692 */
2693 vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2694 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2695 break;
2696
2697 case ixgbe_mac_82599EB:
2698 case ixgbe_mac_X540:
2699 case ixgbe_mac_X550:
2700 case ixgbe_mac_X550EM_x:
2701 /*
2702 * Enable VMDq-only.
2703 */
2704 vmdctl = IXGBE_MRQC_VMDQEN;
2705 IXGBE_WRITE_REG(hw, IXGBE_MRQC, vmdctl);
2706
2707 for (i = 0; i < hw->mac.num_rar_entries; i++) {
2708 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
2709 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
2710 }
2711
2712 /*
2713 * Enable Virtualization and Replication.
2714 */
2715 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
2716 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
2717
2718 /*
2719 * Enable receiving packets to all VFs
2720 */
2721 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
2722 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
2723 break;
2724
2725 default:
2726 break;
2727 }
2728 }
2729
2730 /*
2731 * ixgbe_setup_vmdq_rss - Setup both vmdq feature and rss feature.
2732 */
2733 static void
2734 ixgbe_setup_vmdq_rss(ixgbe_t *ixgbe)
2735 {
2736 struct ixgbe_hw *hw = &ixgbe->hw;
2737 uint32_t i, mrqc, rxcsum;
2738 uint32_t random;
2739 uint32_t reta;
2740 uint32_t ring_per_group;
2741 uint32_t vmdctl, vtctl;
2742
2743 /*
2744 * Fill out redirection table
2745 */
2746 reta = 0;
2747 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
2748 for (i = 0; i < 128; i++) {
2749 reta = (reta << 8) | (i % ring_per_group) |
2750 ((i % ring_per_group) << 4);
2751 if ((i & 3) == 3)
2752 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
2753 }
2754
2755 /*
2756 * Fill out hash function seeds with a random constant
2757 */
2758 for (i = 0; i < 10; i++) {
2759 (void) random_get_pseudo_bytes((uint8_t *)&random,
2760 sizeof (uint32_t));
2761 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random);
2762 }
2763
2764 /*
2765 * Enable and setup RSS and VMDq
2766 */
2767 switch (hw->mac.type) {
2768 case ixgbe_mac_82598EB:
2769 /*
2770 * Enable RSS & Setup RSS Hash functions
2771 */
2772 mrqc = IXGBE_MRQC_RSSEN |
2773 IXGBE_MRQC_RSS_FIELD_IPV4 |
2774 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2775 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2776 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2777 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2778 IXGBE_MRQC_RSS_FIELD_IPV6 |
2779 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2780 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2781 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2782 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2783
2784 /*
2785 * Enable and Setup VMDq
2786 * VMDq Filter = 0; MAC filtering
2787 * Default VMDq output index = 0;
2788 */
2789 vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2790 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2791 break;
2792
2793 case ixgbe_mac_82599EB:
2794 case ixgbe_mac_X540:
2795 case ixgbe_mac_X550:
2796 case ixgbe_mac_X550EM_x:
2797 /*
2798 * Enable RSS & Setup RSS Hash functions
2799 */
2800 mrqc = IXGBE_MRQC_RSS_FIELD_IPV4 |
2801 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2802 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2803 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2804 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2805 IXGBE_MRQC_RSS_FIELD_IPV6 |
2806 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2807 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2808 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2809
2810 /*
2811 * Enable VMDq+RSS.
2812 */
2813 if (ixgbe->num_rx_groups > 32) {
2814 mrqc = mrqc | IXGBE_MRQC_VMDQRSS64EN;
2815 } else {
2816 mrqc = mrqc | IXGBE_MRQC_VMDQRSS32EN;
2817 }
2818
2819 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2820
2821 for (i = 0; i < hw->mac.num_rar_entries; i++) {
2822 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
2823 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
2824 }
2825 break;
2826
2827 default:
2828 break;
2829
2830 }
2831
2832 /*
2833 * Disable Packet Checksum to enable RSS for multiple receive queues.
2834 * It is an adapter hardware limitation that Packet Checksum is
2835 * mutually exclusive with RSS.
2836 */
2837 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2838 rxcsum |= IXGBE_RXCSUM_PCSD;
2839 rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
2840 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
2841
2842 if (hw->mac.type == ixgbe_mac_82599EB ||
2843 hw->mac.type == ixgbe_mac_X540 ||
2844 hw->mac.type == ixgbe_mac_X550 ||
2845 hw->mac.type == ixgbe_mac_X550EM_x) {
2846 /*
2847 * Enable Virtualization and Replication.
2848 */
2849 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
2850 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
2851
2852 /*
2853 * Enable receiving packets to all VFs
2854 */
2855 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
2856 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
2857 }
2858 }
2859
2860 /*
2861 * ixgbe_init_unicst - Initialize the unicast addresses.
2862 */
2863 static void
2864 ixgbe_init_unicst(ixgbe_t *ixgbe)
2865 {
2866 struct ixgbe_hw *hw = &ixgbe->hw;
2867 uint8_t *mac_addr;
2868 int slot;
2869 /*
2870 * Here we should consider two situations:
2871 *
2872 * 1. Chipset is initialized at the first time,
2873 * Clear all the multiple unicast addresses.
2874 *
2875 * 2. Chipset is reset
2876 * Recover the multiple unicast addresses from the
2877 * software data structure to the RAR registers.
2878 */
2879 if (!ixgbe->unicst_init) {
2880 /*
2881 * Initialize the multiple unicast addresses
2882 */
2883 ixgbe->unicst_total = hw->mac.num_rar_entries;
2884 ixgbe->unicst_avail = ixgbe->unicst_total;
2885 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
2886 mac_addr = ixgbe->unicst_addr[slot].mac.addr;
2887 bzero(mac_addr, ETHERADDRL);
2888 (void) ixgbe_set_rar(hw, slot, mac_addr, NULL, NULL);
2889 ixgbe->unicst_addr[slot].mac.set = 0;
2890 }
2891 ixgbe->unicst_init = B_TRUE;
2892 } else {
2893 /* Re-configure the RAR registers */
2894 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
2895 mac_addr = ixgbe->unicst_addr[slot].mac.addr;
2896 if (ixgbe->unicst_addr[slot].mac.set == 1) {
2897 (void) ixgbe_set_rar(hw, slot, mac_addr,
2898 ixgbe->unicst_addr[slot].mac.group_index,
2899 IXGBE_RAH_AV);
2900 } else {
2901 bzero(mac_addr, ETHERADDRL);
2902 (void) ixgbe_set_rar(hw, slot, mac_addr,
2903 NULL, NULL);
2904 }
2905 }
2906 }
2907 }
2908
2909 /*
2910 * ixgbe_unicst_find - Find the slot for the specified unicast address
2911 */
2912 int
2913 ixgbe_unicst_find(ixgbe_t *ixgbe, const uint8_t *mac_addr)
2914 {
2915 int slot;
2916
2917 ASSERT(mutex_owned(&ixgbe->gen_lock));
2918
2919 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
2920 if (bcmp(ixgbe->unicst_addr[slot].mac.addr,
2921 mac_addr, ETHERADDRL) == 0)
2922 return (slot);
2923 }
2924
2925 return (-1);
2926 }
2927
2928 /*
2929 * ixgbe_multicst_add - Add a multicst address.
2930 */
2931 int
2932 ixgbe_multicst_add(ixgbe_t *ixgbe, const uint8_t *multiaddr)
2933 {
2934 ASSERT(mutex_owned(&ixgbe->gen_lock));
2935
2936 if ((multiaddr[0] & 01) == 0) {
2937 return (EINVAL);
2938 }
2939
2940 if (ixgbe->mcast_count >= MAX_NUM_MULTICAST_ADDRESSES) {
2941 return (ENOENT);
2942 }
2943
2944 bcopy(multiaddr,
2945 &ixgbe->mcast_table[ixgbe->mcast_count], ETHERADDRL);
2946 ixgbe->mcast_count++;
2947
2948 /*
2949 * Update the multicast table in the hardware
2950 */
2951 ixgbe_setup_multicst(ixgbe);
2952
2953 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
2954 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
2955 return (EIO);
2956 }
2957
2958 return (0);
2959 }
2960
2961 /*
2962 * ixgbe_multicst_remove - Remove a multicst address.
2963 */
2964 int
2965 ixgbe_multicst_remove(ixgbe_t *ixgbe, const uint8_t *multiaddr)
2966 {
2967 int i;
2968
2969 ASSERT(mutex_owned(&ixgbe->gen_lock));
2970
2971 for (i = 0; i < ixgbe->mcast_count; i++) {
2972 if (bcmp(multiaddr, &ixgbe->mcast_table[i],
2973 ETHERADDRL) == 0) {
2974 for (i++; i < ixgbe->mcast_count; i++) {
2975 ixgbe->mcast_table[i - 1] =
2976 ixgbe->mcast_table[i];
2977 }
2978 ixgbe->mcast_count--;
2979 break;
2980 }
2981 }
2982
2983 /*
2984 * Update the multicast table in the hardware
2985 */
2986 ixgbe_setup_multicst(ixgbe);
2987
2988 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
2989 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
2990 return (EIO);
2991 }
2992
2993 return (0);
2994 }
2995
2996 /*
2997 * ixgbe_setup_multicast - Setup multicast data structures.
2998 *
2999 * This routine initializes all of the multicast related structures
3000 * and save them in the hardware registers.
3001 */
3002 static void
3003 ixgbe_setup_multicst(ixgbe_t *ixgbe)
3004 {
3005 uint8_t *mc_addr_list;
3006 uint32_t mc_addr_count;
3007 struct ixgbe_hw *hw = &ixgbe->hw;
3008
3009 ASSERT(mutex_owned(&ixgbe->gen_lock));
3010
3011 ASSERT(ixgbe->mcast_count <= MAX_NUM_MULTICAST_ADDRESSES);
3012
3013 mc_addr_list = (uint8_t *)ixgbe->mcast_table;
3014 mc_addr_count = ixgbe->mcast_count;
3015
3016 /*
3017 * Update the multicast addresses to the MTA registers
3018 */
3019 (void) ixgbe_update_mc_addr_list(hw, mc_addr_list, mc_addr_count,
3020 ixgbe_mc_table_itr, TRUE);
3021 }
3022
3023 /*
3024 * ixgbe_setup_vmdq_rss_conf - Configure vmdq and rss (number and mode).
3025 *
3026 * Configure the rx classification mode (vmdq & rss) and vmdq & rss numbers.
3027 * Different chipsets may have different allowed configuration of vmdq and rss.
3028 */
3029 static void
3030 ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe)
3031 {
3032 struct ixgbe_hw *hw = &ixgbe->hw;
3033 uint32_t ring_per_group;
3034
3035 switch (hw->mac.type) {
3036 case ixgbe_mac_82598EB:
3037 /*
3038 * 82598 supports the following combination:
3039 * vmdq no. x rss no.
3040 * [5..16] x 1
3041 * [1..4] x [1..16]
3042 * However 8 rss queue per pool (vmdq) is sufficient for
3043 * most cases.
3044 */
3045 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3046 if (ixgbe->num_rx_groups > 4) {
3047 ixgbe->num_rx_rings = ixgbe->num_rx_groups;
3048 } else {
3049 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3050 min(8, ring_per_group);
3051 }
3052
3053 break;
3054
3055 case ixgbe_mac_82599EB:
3056 case ixgbe_mac_X540:
3057 case ixgbe_mac_X550:
3058 case ixgbe_mac_X550EM_x:
3059 /*
3060 * 82599 supports the following combination:
3061 * vmdq no. x rss no.
3062 * [33..64] x [1..2]
3063 * [2..32] x [1..4]
3064 * 1 x [1..16]
3065 * However 8 rss queue per pool (vmdq) is sufficient for
3066 * most cases.
3067 *
3068 * For now, treat X540 and X550 like the 82599.
3069 */
3070 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3071 if (ixgbe->num_rx_groups == 1) {
3072 ixgbe->num_rx_rings = min(8, ring_per_group);
3073 } else if (ixgbe->num_rx_groups <= 32) {
3074 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3075 min(4, ring_per_group);
3076 } else if (ixgbe->num_rx_groups <= 64) {
3077 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3078 min(2, ring_per_group);
3079 }
3080 break;
3081
3082 default:
3083 break;
3084 }
3085
3086 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3087
3088 if (ixgbe->num_rx_groups == 1 && ring_per_group == 1) {
3089 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
3090 } else if (ixgbe->num_rx_groups != 1 && ring_per_group == 1) {
3091 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ;
3092 } else if (ixgbe->num_rx_groups != 1 && ring_per_group != 1) {
3093 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ_RSS;
3094 } else {
3095 ixgbe->classify_mode = IXGBE_CLASSIFY_RSS;
3096 }
3097
3098 IXGBE_DEBUGLOG_2(ixgbe, "rx group number:%d, rx ring number:%d",
3099 ixgbe->num_rx_groups, ixgbe->num_rx_rings);
3100 }
3101
3102 /*
3103 * ixgbe_get_conf - Get driver configurations set in driver.conf.
3104 *
3105 * This routine gets user-configured values out of the configuration
3106 * file ixgbe.conf.
3107 *
3108 * For each configurable value, there is a minimum, a maximum, and a
3109 * default.
3110 * If user does not configure a value, use the default.
3111 * If user configures below the minimum, use the minumum.
3112 * If user configures above the maximum, use the maxumum.
3113 */
3114 static void
3115 ixgbe_get_conf(ixgbe_t *ixgbe)
3116 {
3117 struct ixgbe_hw *hw = &ixgbe->hw;
3118 uint32_t flow_control;
3119
3120 /*
3121 * ixgbe driver supports the following user configurations:
3122 *
3123 * Jumbo frame configuration:
3124 * default_mtu
3125 *
3126 * Ethernet flow control configuration:
3127 * flow_control
3128 *
3129 * Multiple rings configurations:
3130 * tx_queue_number
3131 * tx_ring_size
3132 * rx_queue_number
3133 * rx_ring_size
3134 *
3135 * Call ixgbe_get_prop() to get the value for a specific
3136 * configuration parameter.
3137 */
3138
3139 /*
3140 * Jumbo frame configuration - max_frame_size controls host buffer
3141 * allocation, so includes MTU, ethernet header, vlan tag and
3142 * frame check sequence.
3143 */
3144 ixgbe->default_mtu = ixgbe_get_prop(ixgbe, PROP_DEFAULT_MTU,
3145 MIN_MTU, ixgbe->capab->max_mtu, DEFAULT_MTU);
3146
3147 ixgbe->max_frame_size = ixgbe->default_mtu +
3148 sizeof (struct ether_vlan_header) + ETHERFCSL;
3149
3150 /*
3151 * Ethernet flow control configuration
3152 */
3153 flow_control = ixgbe_get_prop(ixgbe, PROP_FLOW_CONTROL,
3154 ixgbe_fc_none, 3, ixgbe_fc_none);
3155 if (flow_control == 3)
3156 flow_control = ixgbe_fc_default;
3157
3158 /*
3159 * fc.requested mode is what the user requests. After autoneg,
3160 * fc.current_mode will be the flow_control mode that was negotiated.
3161 */
3162 hw->fc.requested_mode = flow_control;
3163
3164 /*
3165 * Multiple rings configurations
3166 */
3167 ixgbe->num_tx_rings = ixgbe_get_prop(ixgbe, PROP_TX_QUEUE_NUM,
3168 ixgbe->capab->min_tx_que_num,
3169 ixgbe->capab->max_tx_que_num,
3170 ixgbe->capab->def_tx_que_num);
3171 ixgbe->tx_ring_size = ixgbe_get_prop(ixgbe, PROP_TX_RING_SIZE,
3172 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE);
3173
3174 ixgbe->num_rx_rings = ixgbe_get_prop(ixgbe, PROP_RX_QUEUE_NUM,
3175 ixgbe->capab->min_rx_que_num,
3176 ixgbe->capab->max_rx_que_num,
3177 ixgbe->capab->def_rx_que_num);
3178 ixgbe->rx_ring_size = ixgbe_get_prop(ixgbe, PROP_RX_RING_SIZE,
3179 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE);
3180
3181 /*
3182 * Multiple groups configuration
3183 */
3184 ixgbe->num_rx_groups = ixgbe_get_prop(ixgbe, PROP_RX_GROUP_NUM,
3185 ixgbe->capab->min_rx_grp_num, ixgbe->capab->max_rx_grp_num,
3186 ixgbe->capab->def_rx_grp_num);
3187
3188 ixgbe->mr_enable = ixgbe_get_prop(ixgbe, PROP_MR_ENABLE,
3189 0, 1, DEFAULT_MR_ENABLE);
3190
3191 if (ixgbe->mr_enable == B_FALSE) {
3192 ixgbe->num_tx_rings = 1;
3193 ixgbe->num_rx_rings = 1;
3194 ixgbe->num_rx_groups = 1;
3195 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
3196 } else {
3197 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3198 max(ixgbe->num_rx_rings / ixgbe->num_rx_groups, 1);
3199 /*
3200 * The combination of num_rx_rings and num_rx_groups
3201 * may be not supported by h/w. We need to adjust
3202 * them to appropriate values.
3203 */
3204 ixgbe_setup_vmdq_rss_conf(ixgbe);
3205 }
3206
3207 /*
3208 * Tunable used to force an interrupt type. The only use is
3209 * for testing of the lesser interrupt types.
3210 * 0 = don't force interrupt type
3211 * 1 = force interrupt type MSI-X
3212 * 2 = force interrupt type MSI
3213 * 3 = force interrupt type Legacy
3214 */
3215 ixgbe->intr_force = ixgbe_get_prop(ixgbe, PROP_INTR_FORCE,
3216 IXGBE_INTR_NONE, IXGBE_INTR_LEGACY, IXGBE_INTR_NONE);
3217
3218 ixgbe->tx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_TX_HCKSUM_ENABLE,
3219 0, 1, DEFAULT_TX_HCKSUM_ENABLE);
3220 ixgbe->rx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_RX_HCKSUM_ENABLE,
3221 0, 1, DEFAULT_RX_HCKSUM_ENABLE);
3222 ixgbe->lso_enable = ixgbe_get_prop(ixgbe, PROP_LSO_ENABLE,
3223 0, 1, DEFAULT_LSO_ENABLE);
3224 ixgbe->lro_enable = ixgbe_get_prop(ixgbe, PROP_LRO_ENABLE,
3225 0, 1, DEFAULT_LRO_ENABLE);
3226 ixgbe->tx_head_wb_enable = ixgbe_get_prop(ixgbe, PROP_TX_HEAD_WB_ENABLE,
3227 0, 1, DEFAULT_TX_HEAD_WB_ENABLE);
3228 ixgbe->relax_order_enable = ixgbe_get_prop(ixgbe,
3229 PROP_RELAX_ORDER_ENABLE, 0, 1, DEFAULT_RELAX_ORDER_ENABLE);
3230
3231 /* Head Write Back not recommended for 82599, X540 and X550 */
3232 if (hw->mac.type == ixgbe_mac_82599EB ||
3233 hw->mac.type == ixgbe_mac_X540 ||
3234 hw->mac.type == ixgbe_mac_X550 ||
3235 hw->mac.type == ixgbe_mac_X550EM_x) {
3236 ixgbe->tx_head_wb_enable = B_FALSE;
3237 }
3238
3239 /*
3240 * ixgbe LSO needs the tx h/w checksum support.
3241 * LSO will be disabled if tx h/w checksum is not
3242 * enabled.
3243 */
3244 if (ixgbe->tx_hcksum_enable == B_FALSE) {
3245 ixgbe->lso_enable = B_FALSE;
3246 }
3247
3248 /*
3249 * ixgbe LRO needs the rx h/w checksum support.
3250 * LRO will be disabled if rx h/w checksum is not
3251 * enabled.
3252 */
3253 if (ixgbe->rx_hcksum_enable == B_FALSE) {
3254 ixgbe->lro_enable = B_FALSE;
3255 }
3256
3257 /*
3258 * ixgbe LRO only supported by 82599, X540 and X550
3259 */
3260 if (hw->mac.type == ixgbe_mac_82598EB) {
3261 ixgbe->lro_enable = B_FALSE;
3262 }
3263 ixgbe->tx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_TX_COPY_THRESHOLD,
3264 MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD,
3265 DEFAULT_TX_COPY_THRESHOLD);
3266 ixgbe->tx_recycle_thresh = ixgbe_get_prop(ixgbe,
3267 PROP_TX_RECYCLE_THRESHOLD, MIN_TX_RECYCLE_THRESHOLD,
3268 MAX_TX_RECYCLE_THRESHOLD, DEFAULT_TX_RECYCLE_THRESHOLD);
3269 ixgbe->tx_overload_thresh = ixgbe_get_prop(ixgbe,
3270 PROP_TX_OVERLOAD_THRESHOLD, MIN_TX_OVERLOAD_THRESHOLD,
3271 MAX_TX_OVERLOAD_THRESHOLD, DEFAULT_TX_OVERLOAD_THRESHOLD);
3272 ixgbe->tx_resched_thresh = ixgbe_get_prop(ixgbe,
3273 PROP_TX_RESCHED_THRESHOLD, MIN_TX_RESCHED_THRESHOLD,
3274 MAX_TX_RESCHED_THRESHOLD, DEFAULT_TX_RESCHED_THRESHOLD);
3275
3276 ixgbe->rx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_RX_COPY_THRESHOLD,
3277 MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD,
3278 DEFAULT_RX_COPY_THRESHOLD);
3279 ixgbe->rx_limit_per_intr = ixgbe_get_prop(ixgbe, PROP_RX_LIMIT_PER_INTR,
3280 MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR,
3281 DEFAULT_RX_LIMIT_PER_INTR);
3282
3283 ixgbe->intr_throttling[0] = ixgbe_get_prop(ixgbe, PROP_INTR_THROTTLING,
3284 ixgbe->capab->min_intr_throttle,
3285 ixgbe->capab->max_intr_throttle,
3286 ixgbe->capab->def_intr_throttle);
3287 /*
3288 * 82599, X540 and X550 require the interrupt throttling rate is
3289 * a multiple of 8. This is enforced by the register definiton.
3290 */
3291 if (hw->mac.type == ixgbe_mac_82599EB ||
3292 hw->mac.type == ixgbe_mac_X540 ||
3293 hw->mac.type == ixgbe_mac_X550 ||
3294 hw->mac.type == ixgbe_mac_X550EM_x)
3295 ixgbe->intr_throttling[0] = ixgbe->intr_throttling[0] & 0xFF8;
3296
3297 hw->allow_unsupported_sfp = ixgbe_get_prop(ixgbe,
3298 PROP_ALLOW_UNSUPPORTED_SFP, 0, 1, DEFAULT_ALLOW_UNSUPPORTED_SFP);
3299 }
3300
3301 static void
3302 ixgbe_init_params(ixgbe_t *ixgbe)
3303 {
3304 ixgbe->param_en_10000fdx_cap = 1;
3305 ixgbe->param_en_1000fdx_cap = 1;
3306 ixgbe->param_en_100fdx_cap = 1;
3307 ixgbe->param_adv_10000fdx_cap = 1;
3308 ixgbe->param_adv_1000fdx_cap = 1;
3309 ixgbe->param_adv_100fdx_cap = 1;
3310
3311 ixgbe->param_pause_cap = 1;
3312 ixgbe->param_asym_pause_cap = 1;
3313 ixgbe->param_rem_fault = 0;
3314
3315 ixgbe->param_adv_autoneg_cap = 1;
3316 ixgbe->param_adv_pause_cap = 1;
3317 ixgbe->param_adv_asym_pause_cap = 1;
3318 ixgbe->param_adv_rem_fault = 0;
3319
3320 ixgbe->param_lp_10000fdx_cap = 0;
3321 ixgbe->param_lp_1000fdx_cap = 0;
3322 ixgbe->param_lp_100fdx_cap = 0;
3323 ixgbe->param_lp_autoneg_cap = 0;
3324 ixgbe->param_lp_pause_cap = 0;
3325 ixgbe->param_lp_asym_pause_cap = 0;
3326 ixgbe->param_lp_rem_fault = 0;
3327 }
3328
3329 /*
3330 * ixgbe_get_prop - Get a property value out of the configuration file
3331 * ixgbe.conf.
3332 *
3333 * Caller provides the name of the property, a default value, a minimum
3334 * value, and a maximum value.
3335 *
3336 * Return configured value of the property, with default, minimum and
3337 * maximum properly applied.
3338 */
3339 static int
3340 ixgbe_get_prop(ixgbe_t *ixgbe,
3341 char *propname, /* name of the property */
3342 int minval, /* minimum acceptable value */
3343 int maxval, /* maximim acceptable value */
3344 int defval) /* default value */
3345 {
3346 int value;
3347
3348 /*
3349 * Call ddi_prop_get_int() to read the conf settings
3350 */
3351 value = ddi_prop_get_int(DDI_DEV_T_ANY, ixgbe->dip,
3352 DDI_PROP_DONTPASS, propname, defval);
3353 if (value > maxval)
3354 value = maxval;
3355
3356 if (value < minval)
3357 value = minval;
3358
3359 return (value);
3360 }
3361
3362 /*
3363 * ixgbe_driver_setup_link - Using the link properties to setup the link.
3364 */
3365 int
3366 ixgbe_driver_setup_link(ixgbe_t *ixgbe, boolean_t setup_hw)
3367 {
3368 u32 autoneg_advertised = 0;
3369
3370 /*
3371 * No half duplex support with 10Gb parts
3372 */
3373 if (ixgbe->param_adv_10000fdx_cap == 1)
3374 autoneg_advertised |= IXGBE_LINK_SPEED_10GB_FULL;
3375
3376 if (ixgbe->param_adv_1000fdx_cap == 1)
3377 autoneg_advertised |= IXGBE_LINK_SPEED_1GB_FULL;
3378
3379 if (ixgbe->param_adv_100fdx_cap == 1)
3380 autoneg_advertised |= IXGBE_LINK_SPEED_100_FULL;
3381
3382 if (ixgbe->param_adv_autoneg_cap == 1 && autoneg_advertised == 0) {
3383 ixgbe_notice(ixgbe, "Invalid link settings. Setup link "
3384 "to autonegotiation with full link capabilities.");
3385
3386 autoneg_advertised = IXGBE_LINK_SPEED_10GB_FULL |
3387 IXGBE_LINK_SPEED_1GB_FULL |
3388 IXGBE_LINK_SPEED_100_FULL;
3389 }
3390
3391 if (setup_hw) {
3392 if (ixgbe_setup_link(&ixgbe->hw, autoneg_advertised,
3393 ixgbe->param_adv_autoneg_cap) != IXGBE_SUCCESS) {
3394 ixgbe_notice(ixgbe, "Setup link failed on this "
3395 "device.");
3396 return (IXGBE_FAILURE);
3397 }
3398 }
3399
3400 return (IXGBE_SUCCESS);
3401 }
3402
3403 /*
3404 * ixgbe_driver_link_check - Link status processing.
3405 *
3406 * This function can be called in both kernel context and interrupt context
3407 */
3408 static void
3409 ixgbe_driver_link_check(ixgbe_t *ixgbe)
3410 {
3411 struct ixgbe_hw *hw = &ixgbe->hw;
3412 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN;
3413 boolean_t link_up = B_FALSE;
3414 boolean_t link_changed = B_FALSE;
3415
3416 ASSERT(mutex_owned(&ixgbe->gen_lock));
3417
3418 (void) ixgbe_check_link(hw, &speed, &link_up, false);
3419 if (link_up) {
3420 ixgbe->link_check_complete = B_TRUE;
3421
3422 /* Link is up, enable flow control settings */
3423 (void) ixgbe_fc_enable(hw);
3424
3425 /*
3426 * The Link is up, check whether it was marked as down earlier
3427 */
3428 if (ixgbe->link_state != LINK_STATE_UP) {
3429 switch (speed) {
3430 case IXGBE_LINK_SPEED_10GB_FULL:
3431 ixgbe->link_speed = SPEED_10GB;
3432 break;
3433 case IXGBE_LINK_SPEED_1GB_FULL:
3434 ixgbe->link_speed = SPEED_1GB;
3435 break;
3436 case IXGBE_LINK_SPEED_100_FULL:
3437 ixgbe->link_speed = SPEED_100;
3438 }
3439 ixgbe->link_duplex = LINK_DUPLEX_FULL;
3440 ixgbe->link_state = LINK_STATE_UP;
3441 link_changed = B_TRUE;
3442 }
3443 } else {
3444 if (ixgbe->link_check_complete == B_TRUE ||
3445 (ixgbe->link_check_complete == B_FALSE &&
3446 gethrtime() >= ixgbe->link_check_hrtime)) {
3447 /*
3448 * The link is really down
3449 */
3450 ixgbe->link_check_complete = B_TRUE;
3451
3452 if (ixgbe->link_state != LINK_STATE_DOWN) {
3453 ixgbe->link_speed = 0;
3454 ixgbe->link_duplex = LINK_DUPLEX_UNKNOWN;
3455 ixgbe->link_state = LINK_STATE_DOWN;
3456 link_changed = B_TRUE;
3457 }
3458 }
3459 }
3460
3461 /*
3462 * If we are in an interrupt context, need to re-enable the
3463 * interrupt, which was automasked
3464 */
3465 if (servicing_interrupt() != 0) {
3466 ixgbe->eims |= IXGBE_EICR_LSC;
3467 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
3468 }
3469
3470 if (link_changed) {
3471 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
3472 }
3473 }
3474
3475 /*
3476 * ixgbe_sfp_check - sfp module processing done in taskq only for 82599.
3477 */
3478 static void
3479 ixgbe_sfp_check(void *arg)
3480 {
3481 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3482 uint32_t eicr = ixgbe->eicr;
3483 struct ixgbe_hw *hw = &ixgbe->hw;
3484
3485 mutex_enter(&ixgbe->gen_lock);
3486 if (eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw)) {
3487 /* clear the interrupt */
3488 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1_BY_MAC(hw));
3489
3490 /* if link up, do multispeed fiber setup */
3491 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3492 B_TRUE);
3493 ixgbe_driver_link_check(ixgbe);
3494 ixgbe_get_hw_state(ixgbe);
3495 } else if (eicr & IXGBE_EICR_GPI_SDP2_BY_MAC(hw)) {
3496 /* clear the interrupt */
3497 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2_BY_MAC(hw));
3498
3499 /* if link up, do sfp module setup */
3500 (void) hw->mac.ops.setup_sfp(hw);
3501
3502 /* do multispeed fiber setup */
3503 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3504 B_TRUE);
3505 ixgbe_driver_link_check(ixgbe);
3506 ixgbe_get_hw_state(ixgbe);
3507 }
3508 mutex_exit(&ixgbe->gen_lock);
3509
3510 /*
3511 * We need to fully re-check the link later.
3512 */
3513 ixgbe->link_check_complete = B_FALSE;
3514 ixgbe->link_check_hrtime = gethrtime() +
3515 (IXGBE_LINK_UP_TIME * 100000000ULL);
3516 }
3517
3518 /*
3519 * ixgbe_overtemp_check - overtemp module processing done in taskq
3520 *
3521 * This routine will only be called on adapters with temperature sensor.
3522 * The indication of over-temperature can be either SDP0 interrupt or the link
3523 * status change interrupt.
3524 */
3525 static void
3526 ixgbe_overtemp_check(void *arg)
3527 {
3528 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3529 struct ixgbe_hw *hw = &ixgbe->hw;
3530 uint32_t eicr = ixgbe->eicr;
3531 ixgbe_link_speed speed;
3532 boolean_t link_up;
3533
3534 mutex_enter(&ixgbe->gen_lock);
3535
3536 /* make sure we know current state of link */
3537 (void) ixgbe_check_link(hw, &speed, &link_up, false);
3538
3539 /* check over-temp condition */
3540 if (((eicr & IXGBE_EICR_GPI_SDP0_BY_MAC(hw)) && (!link_up)) ||
3541 (eicr & IXGBE_EICR_LSC)) {
3542 if (hw->phy.ops.check_overtemp(hw) == IXGBE_ERR_OVERTEMP) {
3543 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
3544
3545 /*
3546 * Disable the adapter interrupts
3547 */
3548 ixgbe_disable_adapter_interrupts(ixgbe);
3549
3550 /*
3551 * Disable Rx/Tx units
3552 */
3553 (void) ixgbe_stop_adapter(hw);
3554
3555 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
3556 ixgbe_error(ixgbe,
3557 "Problem: Network adapter has been stopped "
3558 "because it has overheated");
3559 ixgbe_error(ixgbe,
3560 "Action: Restart the computer. "
3561 "If the problem persists, power off the system "
3562 "and replace the adapter");
3563 }
3564 }
3565
3566 /* write to clear the interrupt */
3567 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
3568
3569 mutex_exit(&ixgbe->gen_lock);
3570 }
3571
3572 /*
3573 * ixgbe_link_timer - timer for link status detection
3574 */
3575 static void
3576 ixgbe_link_timer(void *arg)
3577 {
3578 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3579
3580 mutex_enter(&ixgbe->gen_lock);
3581 ixgbe_driver_link_check(ixgbe);
3582 mutex_exit(&ixgbe->gen_lock);
3583 }
3584
3585 /*
3586 * ixgbe_local_timer - Driver watchdog function.
3587 *
3588 * This function will handle the transmit stall check and other routines.
3589 */
3590 static void
3591 ixgbe_local_timer(void *arg)
3592 {
3593 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3594
3595 if (ixgbe->ixgbe_state & IXGBE_OVERTEMP)
3596 goto out;
3597
3598 if (ixgbe->ixgbe_state & IXGBE_ERROR) {
3599 ixgbe->reset_count++;
3600 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS)
3601 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED);
3602 goto out;
3603 }
3604
3605 if (ixgbe_stall_check(ixgbe)) {
3606 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STALL);
3607 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3608
3609 ixgbe->reset_count++;
3610 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS)
3611 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED);
3612 }
3613
3614 out:
3615 ixgbe_restart_watchdog_timer(ixgbe);
3616 }
3617
3618 /*
3619 * ixgbe_stall_check - Check for transmit stall.
3620 *
3621 * This function checks if the adapter is stalled (in transmit).
3622 *
3623 * It is called each time the watchdog timeout is invoked.
3624 * If the transmit descriptor reclaim continuously fails,
3625 * the watchdog value will increment by 1. If the watchdog
3626 * value exceeds the threshold, the ixgbe is assumed to
3627 * have stalled and need to be reset.
3628 */
3629 static boolean_t
3630 ixgbe_stall_check(ixgbe_t *ixgbe)
3631 {
3632 ixgbe_tx_ring_t *tx_ring;
3633 boolean_t result;
3634 int i;
3635
3636 if (ixgbe->link_state != LINK_STATE_UP)
3637 return (B_FALSE);
3638
3639 /*
3640 * If any tx ring is stalled, we'll reset the chipset
3641 */
3642 result = B_FALSE;
3643 for (i = 0; i < ixgbe->num_tx_rings; i++) {
3644 tx_ring = &ixgbe->tx_rings[i];
3645 if (tx_ring->tbd_free <= ixgbe->tx_recycle_thresh) {
3646 tx_ring->tx_recycle(tx_ring);
3647 }
3648
3649 if (tx_ring->recycle_fail > 0)
3650 tx_ring->stall_watchdog++;
3651 else
3652 tx_ring->stall_watchdog = 0;
3653
3654 if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) {
3655 result = B_TRUE;
3656 break;
3657 }
3658 }
3659
3660 if (result) {
3661 tx_ring->stall_watchdog = 0;
3662 tx_ring->recycle_fail = 0;
3663 }
3664
3665 return (result);
3666 }
3667
3668
3669 /*
3670 * is_valid_mac_addr - Check if the mac address is valid.
3671 */
3672 static boolean_t
3673 is_valid_mac_addr(uint8_t *mac_addr)
3674 {
3675 const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 };
3676 const uint8_t addr_test2[6] =
3677 { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF };
3678
3679 if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) ||
3680 !(bcmp(addr_test2, mac_addr, ETHERADDRL)))
3681 return (B_FALSE);
3682
3683 return (B_TRUE);
3684 }
3685
3686 static boolean_t
3687 ixgbe_find_mac_address(ixgbe_t *ixgbe)
3688 {
3689 #ifdef __sparc
3690 struct ixgbe_hw *hw = &ixgbe->hw;
3691 uchar_t *bytes;
3692 struct ether_addr sysaddr;
3693 uint_t nelts;
3694 int err;
3695 boolean_t found = B_FALSE;
3696
3697 /*
3698 * The "vendor's factory-set address" may already have
3699 * been extracted from the chip, but if the property
3700 * "local-mac-address" is set we use that instead.
3701 *
3702 * We check whether it looks like an array of 6
3703 * bytes (which it should, if OBP set it). If we can't
3704 * make sense of it this way, we'll ignore it.
3705 */
3706 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip,
3707 DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts);
3708 if (err == DDI_PROP_SUCCESS) {
3709 if (nelts == ETHERADDRL) {
3710 while (nelts--)
3711 hw->mac.addr[nelts] = bytes[nelts];
3712 found = B_TRUE;
3713 }
3714 ddi_prop_free(bytes);
3715 }
3716
3717 /*
3718 * Look up the OBP property "local-mac-address?". If the user has set
3719 * 'local-mac-address? = false', use "the system address" instead.
3720 */
3721 if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 0,
3722 "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) {
3723 if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) {
3724 if (localetheraddr(NULL, &sysaddr) != 0) {
3725 bcopy(&sysaddr, hw->mac.addr, ETHERADDRL);
3726 found = B_TRUE;
3727 }
3728 }
3729 ddi_prop_free(bytes);
3730 }
3731
3732 /*
3733 * Finally(!), if there's a valid "mac-address" property (created
3734 * if we netbooted from this interface), we must use this instead
3735 * of any of the above to ensure that the NFS/install server doesn't
3736 * get confused by the address changing as illumos takes over!
3737 */
3738 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip,
3739 DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts);
3740 if (err == DDI_PROP_SUCCESS) {
3741 if (nelts == ETHERADDRL) {
3742 while (nelts--)
3743 hw->mac.addr[nelts] = bytes[nelts];
3744 found = B_TRUE;
3745 }
3746 ddi_prop_free(bytes);
3747 }
3748
3749 if (found) {
3750 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL);
3751 return (B_TRUE);
3752 }
3753 #else
3754 _NOTE(ARGUNUSED(ixgbe));
3755 #endif
3756
3757 return (B_TRUE);
3758 }
3759
3760 #pragma inline(ixgbe_arm_watchdog_timer)
3761 static void
3762 ixgbe_arm_watchdog_timer(ixgbe_t *ixgbe)
3763 {
3764 /*
3765 * Fire a watchdog timer
3766 */
3767 ixgbe->watchdog_tid =
3768 timeout(ixgbe_local_timer,
3769 (void *)ixgbe, 1 * drv_usectohz(1000000));
3770
3771 }
3772
3773 /*
3774 * ixgbe_enable_watchdog_timer - Enable and start the driver watchdog timer.
3775 */
3776 void
3777 ixgbe_enable_watchdog_timer(ixgbe_t *ixgbe)
3778 {
3779 mutex_enter(&ixgbe->watchdog_lock);
3780
3781 if (!ixgbe->watchdog_enable) {
3782 ixgbe->watchdog_enable = B_TRUE;
3783 ixgbe->watchdog_start = B_TRUE;
3784 ixgbe_arm_watchdog_timer(ixgbe);
3785 }
3786
3787 mutex_exit(&ixgbe->watchdog_lock);
3788 }
3789
3790 /*
3791 * ixgbe_disable_watchdog_timer - Disable and stop the driver watchdog timer.
3792 */
3793 void
3794 ixgbe_disable_watchdog_timer(ixgbe_t *ixgbe)
3795 {
3796 timeout_id_t tid;
3797
3798 mutex_enter(&ixgbe->watchdog_lock);
3799
3800 ixgbe->watchdog_enable = B_FALSE;
3801 ixgbe->watchdog_start = B_FALSE;
3802 tid = ixgbe->watchdog_tid;
3803 ixgbe->watchdog_tid = 0;
3804
3805 mutex_exit(&ixgbe->watchdog_lock);
3806
3807 if (tid != 0)
3808 (void) untimeout(tid);
3809 }
3810
3811 /*
3812 * ixgbe_start_watchdog_timer - Start the driver watchdog timer.
3813 */
3814 void
3815 ixgbe_start_watchdog_timer(ixgbe_t *ixgbe)
3816 {
3817 mutex_enter(&ixgbe->watchdog_lock);
3818
3819 if (ixgbe->watchdog_enable) {
3820 if (!ixgbe->watchdog_start) {
3821 ixgbe->watchdog_start = B_TRUE;
3822 ixgbe_arm_watchdog_timer(ixgbe);
3823 }
3824 }
3825
3826 mutex_exit(&ixgbe->watchdog_lock);
3827 }
3828
3829 /*
3830 * ixgbe_restart_watchdog_timer - Restart the driver watchdog timer.
3831 */
3832 static void
3833 ixgbe_restart_watchdog_timer(ixgbe_t *ixgbe)
3834 {
3835 mutex_enter(&ixgbe->watchdog_lock);
3836
3837 if (ixgbe->watchdog_start)
3838 ixgbe_arm_watchdog_timer(ixgbe);
3839
3840 mutex_exit(&ixgbe->watchdog_lock);
3841 }
3842
3843 /*
3844 * ixgbe_stop_watchdog_timer - Stop the driver watchdog timer.
3845 */
3846 void
3847 ixgbe_stop_watchdog_timer(ixgbe_t *ixgbe)
3848 {
3849 timeout_id_t tid;
3850
3851 mutex_enter(&ixgbe->watchdog_lock);
3852
3853 ixgbe->watchdog_start = B_FALSE;
3854 tid = ixgbe->watchdog_tid;
3855 ixgbe->watchdog_tid = 0;
3856
3857 mutex_exit(&ixgbe->watchdog_lock);
3858
3859 if (tid != 0)
3860 (void) untimeout(tid);
3861 }
3862
3863 /*
3864 * ixgbe_disable_adapter_interrupts - Disable all adapter interrupts.
3865 */
3866 static void
3867 ixgbe_disable_adapter_interrupts(ixgbe_t *ixgbe)
3868 {
3869 struct ixgbe_hw *hw = &ixgbe->hw;
3870
3871 /*
3872 * mask all interrupts off
3873 */
3874 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xffffffff);
3875
3876 /*
3877 * for MSI-X, also disable autoclear
3878 */
3879 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
3880 IXGBE_WRITE_REG(hw, IXGBE_EIAC, 0x0);
3881 }
3882
3883 IXGBE_WRITE_FLUSH(hw);
3884 }
3885
3886 /*
3887 * ixgbe_enable_adapter_interrupts - Enable all hardware interrupts.
3888 */
3889 static void
3890 ixgbe_enable_adapter_interrupts(ixgbe_t *ixgbe)
3891 {
3892 struct ixgbe_hw *hw = &ixgbe->hw;
3893 uint32_t eiac, eiam;
3894 uint32_t gpie = IXGBE_READ_REG(hw, IXGBE_GPIE);
3895
3896 /* interrupt types to enable */
3897 ixgbe->eims = IXGBE_EIMS_ENABLE_MASK; /* shared code default */
3898 ixgbe->eims &= ~IXGBE_EIMS_TCP_TIMER; /* minus tcp timer */
3899 ixgbe->eims |= ixgbe->capab->other_intr; /* "other" interrupt types */
3900
3901 /* enable automask on "other" causes that this adapter can generate */
3902 eiam = ixgbe->capab->other_intr;
3903
3904 /*
3905 * msi-x mode
3906 */
3907 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
3908 /* enable autoclear but not on bits 29:20 */
3909 eiac = (ixgbe->eims & ~IXGBE_OTHER_INTR);
3910
3911 /* general purpose interrupt enable */
3912 gpie |= (IXGBE_GPIE_MSIX_MODE
3913 | IXGBE_GPIE_PBA_SUPPORT
3914 | IXGBE_GPIE_OCD
3915 | IXGBE_GPIE_EIAME);
3916 /*
3917 * non-msi-x mode
3918 */
3919 } else {
3920
3921 /* disable autoclear, leave gpie at default */
3922 eiac = 0;
3923
3924 /*
3925 * General purpose interrupt enable.
3926 * For 82599, X540 and X550, extended interrupt
3927 * automask enable only in MSI or MSI-X mode
3928 */
3929 if ((hw->mac.type == ixgbe_mac_82598EB) ||
3930 (ixgbe->intr_type == DDI_INTR_TYPE_MSI)) {
3931 gpie |= IXGBE_GPIE_EIAME;
3932 }
3933 }
3934
3935 /* Enable specific "other" interrupt types */
3936 switch (hw->mac.type) {
3937 case ixgbe_mac_82598EB:
3938 gpie |= ixgbe->capab->other_gpie;
3939 break;
3940
3941 case ixgbe_mac_82599EB:
3942 case ixgbe_mac_X540:
3943 case ixgbe_mac_X550:
3944 case ixgbe_mac_X550EM_x:
3945 gpie |= ixgbe->capab->other_gpie;
3946
3947 /* Enable RSC Delay 8us when LRO enabled */
3948 if (ixgbe->lro_enable) {
3949 gpie |= (1 << IXGBE_GPIE_RSC_DELAY_SHIFT);
3950 }
3951 break;
3952
3953 default:
3954 break;
3955 }
3956
3957 /* write to interrupt control registers */
3958 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
3959 IXGBE_WRITE_REG(hw, IXGBE_EIAC, eiac);
3960 IXGBE_WRITE_REG(hw, IXGBE_EIAM, eiam);
3961 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
3962 IXGBE_WRITE_FLUSH(hw);
3963 }
3964
3965 /*
3966 * ixgbe_loopback_ioctl - Loopback support.
3967 */
3968 enum ioc_reply
3969 ixgbe_loopback_ioctl(ixgbe_t *ixgbe, struct iocblk *iocp, mblk_t *mp)
3970 {
3971 lb_info_sz_t *lbsp;
3972 lb_property_t *lbpp;
3973 uint32_t *lbmp;
3974 uint32_t size;
3975 uint32_t value;
3976
3977 if (mp->b_cont == NULL)
3978 return (IOC_INVAL);
3979
3980 switch (iocp->ioc_cmd) {
3981 default:
3982 return (IOC_INVAL);
3983
3984 case LB_GET_INFO_SIZE:
3985 size = sizeof (lb_info_sz_t);
3986 if (iocp->ioc_count != size)
3987 return (IOC_INVAL);
3988
3989 value = sizeof (lb_normal);
3990 value += sizeof (lb_mac);
3991 value += sizeof (lb_external);
3992
3993 lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr;
3994 *lbsp = value;
3995 break;
3996
3997 case LB_GET_INFO:
3998 value = sizeof (lb_normal);
3999 value += sizeof (lb_mac);
4000 value += sizeof (lb_external);
4001
4002 size = value;
4003 if (iocp->ioc_count != size)
4004 return (IOC_INVAL);
4005
4006 value = 0;
4007 lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr;
4008
4009 lbpp[value++] = lb_normal;
4010 lbpp[value++] = lb_mac;
4011 lbpp[value++] = lb_external;
4012 break;
4013
4014 case LB_GET_MODE:
4015 size = sizeof (uint32_t);
4016 if (iocp->ioc_count != size)
4017 return (IOC_INVAL);
4018
4019 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
4020 *lbmp = ixgbe->loopback_mode;
4021 break;
4022
4023 case LB_SET_MODE:
4024 size = 0;
4025 if (iocp->ioc_count != sizeof (uint32_t))
4026 return (IOC_INVAL);
4027
4028 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
4029 if (!ixgbe_set_loopback_mode(ixgbe, *lbmp))
4030 return (IOC_INVAL);
4031 break;
4032 }
4033
4034 iocp->ioc_count = size;
4035 iocp->ioc_error = 0;
4036
4037 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4038 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4039 return (IOC_INVAL);
4040 }
4041
4042 return (IOC_REPLY);
4043 }
4044
4045 /*
4046 * ixgbe_set_loopback_mode - Setup loopback based on the loopback mode.
4047 */
4048 static boolean_t
4049 ixgbe_set_loopback_mode(ixgbe_t *ixgbe, uint32_t mode)
4050 {
4051 if (mode == ixgbe->loopback_mode)
4052 return (B_TRUE);
4053
4054 ixgbe->loopback_mode = mode;
4055
4056 if (mode == IXGBE_LB_NONE) {
4057 /*
4058 * Reset the chip
4059 */
4060 (void) ixgbe_reset(ixgbe);
4061 return (B_TRUE);
4062 }
4063
4064 mutex_enter(&ixgbe->gen_lock);
4065
4066 switch (mode) {
4067 default:
4068 mutex_exit(&ixgbe->gen_lock);
4069 return (B_FALSE);
4070
4071 case IXGBE_LB_EXTERNAL:
4072 break;
4073
4074 case IXGBE_LB_INTERNAL_MAC:
4075 ixgbe_set_internal_mac_loopback(ixgbe);
4076 break;
4077 }
4078
4079 mutex_exit(&ixgbe->gen_lock);
4080
4081 return (B_TRUE);
4082 }
4083
4084 /*
4085 * ixgbe_set_internal_mac_loopback - Set the internal MAC loopback mode.
4086 */
4087 static void
4088 ixgbe_set_internal_mac_loopback(ixgbe_t *ixgbe)
4089 {
4090 struct ixgbe_hw *hw;
4091 uint32_t reg;
4092 uint8_t atlas;
4093
4094 hw = &ixgbe->hw;
4095
4096 /*
4097 * Setup MAC loopback
4098 */
4099 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_HLREG0);
4100 reg |= IXGBE_HLREG0_LPBK;
4101 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_HLREG0, reg);
4102
4103 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
4104 reg &= ~IXGBE_AUTOC_LMS_MASK;
4105 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg);
4106
4107 /*
4108 * Disable Atlas Tx lanes to keep packets in loopback and not on wire
4109 */
4110 switch (hw->mac.type) {
4111 case ixgbe_mac_82598EB:
4112 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK,
4113 &atlas);
4114 atlas |= IXGBE_ATLAS_PDN_TX_REG_EN;
4115 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK,
4116 atlas);
4117
4118 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
4119 &atlas);
4120 atlas |= IXGBE_ATLAS_PDN_TX_10G_QL_ALL;
4121 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
4122 atlas);
4123
4124 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
4125 &atlas);
4126 atlas |= IXGBE_ATLAS_PDN_TX_1G_QL_ALL;
4127 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
4128 atlas);
4129
4130 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4131 &atlas);
4132 atlas |= IXGBE_ATLAS_PDN_TX_AN_QL_ALL;
4133 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4134 atlas);
4135 break;
4136
4137 case ixgbe_mac_82599EB:
4138 case ixgbe_mac_X540:
4139 case ixgbe_mac_X550:
4140 case ixgbe_mac_X550EM_x:
4141 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
4142 reg |= (IXGBE_AUTOC_FLU |
4143 IXGBE_AUTOC_10G_KX4);
4144 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg);
4145
4146 (void) ixgbe_setup_link(&ixgbe->hw, IXGBE_LINK_SPEED_10GB_FULL,
4147 B_FALSE);
4148 break;
4149
4150 default:
4151 break;
4152 }
4153 }
4154
4155 #pragma inline(ixgbe_intr_rx_work)
4156 /*
4157 * ixgbe_intr_rx_work - RX processing of ISR.
4158 */
4159 static void
4160 ixgbe_intr_rx_work(ixgbe_rx_ring_t *rx_ring)
4161 {
4162 mblk_t *mp;
4163
4164 mutex_enter(&rx_ring->rx_lock);
4165
4166 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
4167 mutex_exit(&rx_ring->rx_lock);
4168
4169 if (mp != NULL)
4170 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp,
4171 rx_ring->ring_gen_num);
4172 }
4173
4174 #pragma inline(ixgbe_intr_tx_work)
4175 /*
4176 * ixgbe_intr_tx_work - TX processing of ISR.
4177 */
4178 static void
4179 ixgbe_intr_tx_work(ixgbe_tx_ring_t *tx_ring)
4180 {
4181 ixgbe_t *ixgbe = tx_ring->ixgbe;
4182
4183 /*
4184 * Recycle the tx descriptors
4185 */
4186 tx_ring->tx_recycle(tx_ring);
4187
4188 /*
4189 * Schedule the re-transmit
4190 */
4191 if (tx_ring->reschedule &&
4192 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)) {
4193 tx_ring->reschedule = B_FALSE;
4194 mac_tx_ring_update(tx_ring->ixgbe->mac_hdl,
4195 tx_ring->ring_handle);
4196 IXGBE_DEBUG_STAT(tx_ring->stat_reschedule);
4197 }
4198 }
4199
4200 #pragma inline(ixgbe_intr_other_work)
4201 /*
4202 * ixgbe_intr_other_work - Process interrupt types other than tx/rx
4203 */
4204 static void
4205 ixgbe_intr_other_work(ixgbe_t *ixgbe, uint32_t eicr)
4206 {
4207 struct ixgbe_hw *hw = &ixgbe->hw;
4208
4209 ASSERT(mutex_owned(&ixgbe->gen_lock));
4210
4211 /*
4212 * handle link status change
4213 */
4214 if (eicr & IXGBE_EICR_LSC) {
4215 ixgbe_driver_link_check(ixgbe);
4216 ixgbe_get_hw_state(ixgbe);
4217 }
4218
4219 /*
4220 * check for fan failure on adapters with fans
4221 */
4222 if ((ixgbe->capab->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
4223 (eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw))) {
4224 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
4225
4226 /*
4227 * Disable the adapter interrupts
4228 */
4229 ixgbe_disable_adapter_interrupts(ixgbe);
4230
4231 /*
4232 * Disable Rx/Tx units
4233 */
4234 (void) ixgbe_stop_adapter(&ixgbe->hw);
4235
4236 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
4237 ixgbe_error(ixgbe,
4238 "Problem: Network adapter has been stopped "
4239 "because the fan has stopped.\n");
4240 ixgbe_error(ixgbe,
4241 "Action: Replace the adapter.\n");
4242
4243 /* re-enable the interrupt, which was automasked */
4244 ixgbe->eims |= IXGBE_EICR_GPI_SDP1_BY_MAC(hw);
4245 }
4246
4247 /*
4248 * Do SFP check for adapters with hot-plug capability
4249 */
4250 if ((ixgbe->capab->flags & IXGBE_FLAG_SFP_PLUG_CAPABLE) &&
4251 ((eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw)) ||
4252 (eicr & IXGBE_EICR_GPI_SDP2_BY_MAC(hw)))) {
4253 ixgbe->eicr = eicr;
4254 if ((ddi_taskq_dispatch(ixgbe->sfp_taskq,
4255 ixgbe_sfp_check, (void *)ixgbe,
4256 DDI_NOSLEEP)) != DDI_SUCCESS) {
4257 ixgbe_log(ixgbe, "No memory available to dispatch "
4258 "taskq for SFP check");
4259 }
4260 }
4261
4262 /*
4263 * Do over-temperature check for adapters with temp sensor
4264 */
4265 if ((ixgbe->capab->flags & IXGBE_FLAG_TEMP_SENSOR_CAPABLE) &&
4266 ((eicr & IXGBE_EICR_GPI_SDP0_BY_MAC(hw)) || (eicr & IXGBE_EICR_LSC))) {
4267 ixgbe->eicr = eicr;
4268 if ((ddi_taskq_dispatch(ixgbe->overtemp_taskq,
4269 ixgbe_overtemp_check, (void *)ixgbe,
4270 DDI_NOSLEEP)) != DDI_SUCCESS) {
4271 ixgbe_log(ixgbe, "No memory available to dispatch "
4272 "taskq for overtemp check");
4273 }
4274 }
4275 }
4276
4277 /*
4278 * ixgbe_intr_legacy - Interrupt handler for legacy interrupts.
4279 */
4280 static uint_t
4281 ixgbe_intr_legacy(void *arg1, void *arg2)
4282 {
4283 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
4284 struct ixgbe_hw *hw = &ixgbe->hw;
4285 ixgbe_tx_ring_t *tx_ring;
4286 ixgbe_rx_ring_t *rx_ring;
4287 uint32_t eicr;
4288 mblk_t *mp;
4289 boolean_t tx_reschedule;
4290 uint_t result;
4291
4292 _NOTE(ARGUNUSED(arg2));
4293
4294 mutex_enter(&ixgbe->gen_lock);
4295 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
4296 mutex_exit(&ixgbe->gen_lock);
4297 return (DDI_INTR_UNCLAIMED);
4298 }
4299
4300 mp = NULL;
4301 tx_reschedule = B_FALSE;
4302
4303 /*
4304 * Any bit set in eicr: claim this interrupt
4305 */
4306 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4307
4308 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4309 mutex_exit(&ixgbe->gen_lock);
4310 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4311 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4312 return (DDI_INTR_CLAIMED);
4313 }
4314
4315 if (eicr) {
4316 /*
4317 * For legacy interrupt, we have only one interrupt,
4318 * so we have only one rx ring and one tx ring enabled.
4319 */
4320 ASSERT(ixgbe->num_rx_rings == 1);
4321 ASSERT(ixgbe->num_tx_rings == 1);
4322
4323 /*
4324 * For legacy interrupt, rx rings[0] will use RTxQ[0].
4325 */
4326 if (eicr & 0x1) {
4327 ixgbe->eimc |= IXGBE_EICR_RTX_QUEUE;
4328 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4329 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE;
4330 /*
4331 * Clean the rx descriptors
4332 */
4333 rx_ring = &ixgbe->rx_rings[0];
4334 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
4335 }
4336
4337 /*
4338 * For legacy interrupt, tx rings[0] will use RTxQ[1].
4339 */
4340 if (eicr & 0x2) {
4341 /*
4342 * Recycle the tx descriptors
4343 */
4344 tx_ring = &ixgbe->tx_rings[0];
4345 tx_ring->tx_recycle(tx_ring);
4346
4347 /*
4348 * Schedule the re-transmit
4349 */
4350 tx_reschedule = (tx_ring->reschedule &&
4351 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh));
4352 }
4353
4354 /* any interrupt type other than tx/rx */
4355 if (eicr & ixgbe->capab->other_intr) {
4356 switch (hw->mac.type) {
4357 case ixgbe_mac_82598EB:
4358 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4359 break;
4360
4361 case ixgbe_mac_82599EB:
4362 case ixgbe_mac_X540:
4363 case ixgbe_mac_X550:
4364 case ixgbe_mac_X550EM_x:
4365 ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4366 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4367 break;
4368
4369 default:
4370 break;
4371 }
4372 ixgbe_intr_other_work(ixgbe, eicr);
4373 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4374 }
4375
4376 mutex_exit(&ixgbe->gen_lock);
4377
4378 result = DDI_INTR_CLAIMED;
4379 } else {
4380 mutex_exit(&ixgbe->gen_lock);
4381
4382 /*
4383 * No interrupt cause bits set: don't claim this interrupt.
4384 */
4385 result = DDI_INTR_UNCLAIMED;
4386 }
4387
4388 /* re-enable the interrupts which were automasked */
4389 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4390
4391 /*
4392 * Do the following work outside of the gen_lock
4393 */
4394 if (mp != NULL) {
4395 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp,
4396 rx_ring->ring_gen_num);
4397 }
4398
4399 if (tx_reschedule) {
4400 tx_ring->reschedule = B_FALSE;
4401 mac_tx_ring_update(ixgbe->mac_hdl, tx_ring->ring_handle);
4402 IXGBE_DEBUG_STAT(tx_ring->stat_reschedule);
4403 }
4404
4405 return (result);
4406 }
4407
4408 /*
4409 * ixgbe_intr_msi - Interrupt handler for MSI.
4410 */
4411 static uint_t
4412 ixgbe_intr_msi(void *arg1, void *arg2)
4413 {
4414 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
4415 struct ixgbe_hw *hw = &ixgbe->hw;
4416 uint32_t eicr;
4417
4418 _NOTE(ARGUNUSED(arg2));
4419
4420 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4421
4422 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4423 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4424 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4425 return (DDI_INTR_CLAIMED);
4426 }
4427
4428 /*
4429 * For MSI interrupt, we have only one vector,
4430 * so we have only one rx ring and one tx ring enabled.
4431 */
4432 ASSERT(ixgbe->num_rx_rings == 1);
4433 ASSERT(ixgbe->num_tx_rings == 1);
4434
4435 /*
4436 * For MSI interrupt, rx rings[0] will use RTxQ[0].
4437 */
4438 if (eicr & 0x1) {
4439 ixgbe_intr_rx_work(&ixgbe->rx_rings[0]);
4440 }
4441
4442 /*
4443 * For MSI interrupt, tx rings[0] will use RTxQ[1].
4444 */
4445 if (eicr & 0x2) {
4446 ixgbe_intr_tx_work(&ixgbe->tx_rings[0]);
4447 }
4448
4449 /* any interrupt type other than tx/rx */
4450 if (eicr & ixgbe->capab->other_intr) {
4451 mutex_enter(&ixgbe->gen_lock);
4452 switch (hw->mac.type) {
4453 case ixgbe_mac_82598EB:
4454 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4455 break;
4456
4457 case ixgbe_mac_82599EB:
4458 case ixgbe_mac_X540:
4459 case ixgbe_mac_X550:
4460 case ixgbe_mac_X550EM_x:
4461 ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4462 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4463 break;
4464
4465 default:
4466 break;
4467 }
4468 ixgbe_intr_other_work(ixgbe, eicr);
4469 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4470 mutex_exit(&ixgbe->gen_lock);
4471 }
4472
4473 /* re-enable the interrupts which were automasked */
4474 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4475
4476 return (DDI_INTR_CLAIMED);
4477 }
4478
4479 /*
4480 * ixgbe_intr_msix - Interrupt handler for MSI-X.
4481 */
4482 static uint_t
4483 ixgbe_intr_msix(void *arg1, void *arg2)
4484 {
4485 ixgbe_intr_vector_t *vect = (ixgbe_intr_vector_t *)arg1;
4486 ixgbe_t *ixgbe = vect->ixgbe;
4487 struct ixgbe_hw *hw = &ixgbe->hw;
4488 uint32_t eicr;
4489 int r_idx = 0;
4490
4491 _NOTE(ARGUNUSED(arg2));
4492
4493 /*
4494 * Clean each rx ring that has its bit set in the map
4495 */
4496 r_idx = bt_getlowbit(vect->rx_map, 0, (ixgbe->num_rx_rings - 1));
4497 while (r_idx >= 0) {
4498 ixgbe_intr_rx_work(&ixgbe->rx_rings[r_idx]);
4499 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1),
4500 (ixgbe->num_rx_rings - 1));
4501 }
4502
4503 /*
4504 * Clean each tx ring that has its bit set in the map
4505 */
4506 r_idx = bt_getlowbit(vect->tx_map, 0, (ixgbe->num_tx_rings - 1));
4507 while (r_idx >= 0) {
4508 ixgbe_intr_tx_work(&ixgbe->tx_rings[r_idx]);
4509 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1),
4510 (ixgbe->num_tx_rings - 1));
4511 }
4512
4513
4514 /*
4515 * Clean other interrupt (link change) that has its bit set in the map
4516 */
4517 if (BT_TEST(vect->other_map, 0) == 1) {
4518 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4519
4520 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) !=
4521 DDI_FM_OK) {
4522 ddi_fm_service_impact(ixgbe->dip,
4523 DDI_SERVICE_DEGRADED);
4524 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4525 return (DDI_INTR_CLAIMED);
4526 }
4527
4528 /*
4529 * Check "other" cause bits: any interrupt type other than tx/rx
4530 */
4531 if (eicr & ixgbe->capab->other_intr) {
4532 mutex_enter(&ixgbe->gen_lock);
4533 switch (hw->mac.type) {
4534 case ixgbe_mac_82598EB:
4535 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4536 ixgbe_intr_other_work(ixgbe, eicr);
4537 break;
4538
4539 case ixgbe_mac_82599EB:
4540 case ixgbe_mac_X540:
4541 case ixgbe_mac_X550:
4542 case ixgbe_mac_X550EM_x:
4543 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE;
4544 ixgbe_intr_other_work(ixgbe, eicr);
4545 break;
4546
4547 default:
4548 break;
4549 }
4550 mutex_exit(&ixgbe->gen_lock);
4551 }
4552
4553 /* re-enable the interrupts which were automasked */
4554 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4555 }
4556
4557 return (DDI_INTR_CLAIMED);
4558 }
4559
4560 /*
4561 * ixgbe_alloc_intrs - Allocate interrupts for the driver.
4562 *
4563 * Normal sequence is to try MSI-X; if not sucessful, try MSI;
4564 * if not successful, try Legacy.
4565 * ixgbe->intr_force can be used to force sequence to start with
4566 * any of the 3 types.
4567 * If MSI-X is not used, number of tx/rx rings is forced to 1.
4568 */
4569 static int
4570 ixgbe_alloc_intrs(ixgbe_t *ixgbe)
4571 {
4572 dev_info_t *devinfo;
4573 int intr_types;
4574 int rc;
4575
4576 devinfo = ixgbe->dip;
4577
4578 /*
4579 * Get supported interrupt types
4580 */
4581 rc = ddi_intr_get_supported_types(devinfo, &intr_types);
4582
4583 if (rc != DDI_SUCCESS) {
4584 ixgbe_log(ixgbe,
4585 "Get supported interrupt types failed: %d", rc);
4586 return (IXGBE_FAILURE);
4587 }
4588 IXGBE_DEBUGLOG_1(ixgbe, "Supported interrupt types: %x", intr_types);
4589
4590 ixgbe->intr_type = 0;
4591
4592 /*
4593 * Install MSI-X interrupts
4594 */
4595 if ((intr_types & DDI_INTR_TYPE_MSIX) &&
4596 (ixgbe->intr_force <= IXGBE_INTR_MSIX)) {
4597 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSIX);
4598 if (rc == IXGBE_SUCCESS)
4599 return (IXGBE_SUCCESS);
4600
4601 ixgbe_log(ixgbe,
4602 "Allocate MSI-X failed, trying MSI interrupts...");
4603 }
4604
4605 /*
4606 * MSI-X not used, force rings and groups to 1
4607 */
4608 ixgbe->num_rx_rings = 1;
4609 ixgbe->num_rx_groups = 1;
4610 ixgbe->num_tx_rings = 1;
4611 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
4612 ixgbe_log(ixgbe,
4613 "MSI-X not used, force rings and groups number to 1");
4614
4615 /*
4616 * Install MSI interrupts
4617 */
4618 if ((intr_types & DDI_INTR_TYPE_MSI) &&
4619 (ixgbe->intr_force <= IXGBE_INTR_MSI)) {
4620 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSI);
4621 if (rc == IXGBE_SUCCESS)
4622 return (IXGBE_SUCCESS);
4623
4624 ixgbe_log(ixgbe,
4625 "Allocate MSI failed, trying Legacy interrupts...");
4626 }
4627
4628 /*
4629 * Install legacy interrupts
4630 */
4631 if (intr_types & DDI_INTR_TYPE_FIXED) {
4632 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_FIXED);
4633 if (rc == IXGBE_SUCCESS)
4634 return (IXGBE_SUCCESS);
4635
4636 ixgbe_log(ixgbe,
4637 "Allocate Legacy interrupts failed");
4638 }
4639
4640 /*
4641 * If none of the 3 types succeeded, return failure
4642 */
4643 return (IXGBE_FAILURE);
4644 }
4645
4646 /*
4647 * ixgbe_alloc_intr_handles - Allocate interrupt handles.
4648 *
4649 * For legacy and MSI, only 1 handle is needed. For MSI-X,
4650 * if fewer than 2 handles are available, return failure.
4651 * Upon success, this maps the vectors to rx and tx rings for
4652 * interrupts.
4653 */
4654 static int
4655 ixgbe_alloc_intr_handles(ixgbe_t *ixgbe, int intr_type)
4656 {
4657 dev_info_t *devinfo;
4658 int request, count, actual;
4659 int minimum;
4660 int rc;
4661 uint32_t ring_per_group;
4662
4663 devinfo = ixgbe->dip;
4664
4665 switch (intr_type) {
4666 case DDI_INTR_TYPE_FIXED:
4667 request = 1; /* Request 1 legacy interrupt handle */
4668 minimum = 1;
4669 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: legacy");
4670 break;
4671
4672 case DDI_INTR_TYPE_MSI:
4673 request = 1; /* Request 1 MSI interrupt handle */
4674 minimum = 1;
4675 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI");
4676 break;
4677
4678 case DDI_INTR_TYPE_MSIX:
4679 /*
4680 * Best number of vectors for the adapter is
4681 * (# rx rings + # tx rings), however we will
4682 * limit the request number.
4683 */
4684 request = min(16, ixgbe->num_rx_rings + ixgbe->num_tx_rings);
4685 if (request > ixgbe->capab->max_ring_vect)
4686 request = ixgbe->capab->max_ring_vect;
4687 minimum = 1;
4688 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI-X");
4689 break;
4690
4691 default:
4692 ixgbe_log(ixgbe,
4693 "invalid call to ixgbe_alloc_intr_handles(): %d\n",
4694 intr_type);
4695 return (IXGBE_FAILURE);
4696 }
4697 IXGBE_DEBUGLOG_2(ixgbe, "interrupt handles requested: %d minimum: %d",
4698 request, minimum);
4699
4700 /*
4701 * Get number of supported interrupts
4702 */
4703 rc = ddi_intr_get_nintrs(devinfo, intr_type, &count);
4704 if ((rc != DDI_SUCCESS) || (count < minimum)) {
4705 ixgbe_log(ixgbe,
4706 "Get interrupt number failed. Return: %d, count: %d",
4707 rc, count);
4708 return (IXGBE_FAILURE);
4709 }
4710 IXGBE_DEBUGLOG_1(ixgbe, "interrupts supported: %d", count);
4711
4712 actual = 0;
4713 ixgbe->intr_cnt = 0;
4714 ixgbe->intr_cnt_max = 0;
4715 ixgbe->intr_cnt_min = 0;
4716
4717 /*
4718 * Allocate an array of interrupt handles
4719 */
4720 ixgbe->intr_size = request * sizeof (ddi_intr_handle_t);
4721 ixgbe->htable = kmem_alloc(ixgbe->intr_size, KM_SLEEP);
4722
4723 rc = ddi_intr_alloc(devinfo, ixgbe->htable, intr_type, 0,
4724 request, &actual, DDI_INTR_ALLOC_NORMAL);
4725 if (rc != DDI_SUCCESS) {
4726 ixgbe_log(ixgbe, "Allocate interrupts failed. "
4727 "return: %d, request: %d, actual: %d",
4728 rc, request, actual);
4729 goto alloc_handle_fail;
4730 }
4731 IXGBE_DEBUGLOG_1(ixgbe, "interrupts actually allocated: %d", actual);
4732
4733 /*
4734 * upper/lower limit of interrupts
4735 */
4736 ixgbe->intr_cnt = actual;
4737 ixgbe->intr_cnt_max = request;
4738 ixgbe->intr_cnt_min = minimum;
4739
4740 /*
4741 * rss number per group should not exceed the rx interrupt number,
4742 * else need to adjust rx ring number.
4743 */
4744 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
4745 ASSERT((ixgbe->num_rx_rings % ixgbe->num_rx_groups) == 0);
4746 if (actual < ring_per_group) {
4747 ixgbe->num_rx_rings = ixgbe->num_rx_groups * actual;
4748 ixgbe_setup_vmdq_rss_conf(ixgbe);
4749 }
4750
4751 /*
4752 * Now we know the actual number of vectors. Here we map the vector
4753 * to other, rx rings and tx ring.
4754 */
4755 if (actual < minimum) {
4756 ixgbe_log(ixgbe, "Insufficient interrupt handles available: %d",
4757 actual);
4758 goto alloc_handle_fail;
4759 }
4760
4761 /*
4762 * Get priority for first vector, assume remaining are all the same
4763 */
4764 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri);
4765 if (rc != DDI_SUCCESS) {
4766 ixgbe_log(ixgbe,
4767 "Get interrupt priority failed: %d", rc);
4768 goto alloc_handle_fail;
4769 }
4770
4771 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap);
4772 if (rc != DDI_SUCCESS) {
4773 ixgbe_log(ixgbe,
4774 "Get interrupt cap failed: %d", rc);
4775 goto alloc_handle_fail;
4776 }
4777
4778 ixgbe->intr_type = intr_type;
4779
4780 return (IXGBE_SUCCESS);
4781
4782 alloc_handle_fail:
4783 ixgbe_rem_intrs(ixgbe);
4784
4785 return (IXGBE_FAILURE);
4786 }
4787
4788 /*
4789 * ixgbe_add_intr_handlers - Add interrupt handlers based on the interrupt type.
4790 *
4791 * Before adding the interrupt handlers, the interrupt vectors have
4792 * been allocated, and the rx/tx rings have also been allocated.
4793 */
4794 static int
4795 ixgbe_add_intr_handlers(ixgbe_t *ixgbe)
4796 {
4797 int vector = 0;
4798 int rc;
4799
4800 switch (ixgbe->intr_type) {
4801 case DDI_INTR_TYPE_MSIX:
4802 /*
4803 * Add interrupt handler for all vectors
4804 */
4805 for (vector = 0; vector < ixgbe->intr_cnt; vector++) {
4806 /*
4807 * install pointer to vect_map[vector]
4808 */
4809 rc = ddi_intr_add_handler(ixgbe->htable[vector],
4810 (ddi_intr_handler_t *)ixgbe_intr_msix,
4811 (void *)&ixgbe->vect_map[vector], NULL);
4812
4813 if (rc != DDI_SUCCESS) {
4814 ixgbe_log(ixgbe,
4815 "Add interrupt handler failed. "
4816 "return: %d, vector: %d", rc, vector);
4817 for (vector--; vector >= 0; vector--) {
4818 (void) ddi_intr_remove_handler(
4819 ixgbe->htable[vector]);
4820 }
4821 return (IXGBE_FAILURE);
4822 }
4823 }
4824
4825 break;
4826
4827 case DDI_INTR_TYPE_MSI:
4828 /*
4829 * Add interrupt handlers for the only vector
4830 */
4831 rc = ddi_intr_add_handler(ixgbe->htable[vector],
4832 (ddi_intr_handler_t *)ixgbe_intr_msi,
4833 (void *)ixgbe, NULL);
4834
4835 if (rc != DDI_SUCCESS) {
4836 ixgbe_log(ixgbe,
4837 "Add MSI interrupt handler failed: %d", rc);
4838 return (IXGBE_FAILURE);
4839 }
4840
4841 break;
4842
4843 case DDI_INTR_TYPE_FIXED:
4844 /*
4845 * Add interrupt handlers for the only vector
4846 */
4847 rc = ddi_intr_add_handler(ixgbe->htable[vector],
4848 (ddi_intr_handler_t *)ixgbe_intr_legacy,
4849 (void *)ixgbe, NULL);
4850
4851 if (rc != DDI_SUCCESS) {
4852 ixgbe_log(ixgbe,
4853 "Add legacy interrupt handler failed: %d", rc);
4854 return (IXGBE_FAILURE);
4855 }
4856
4857 break;
4858
4859 default:
4860 return (IXGBE_FAILURE);
4861 }
4862
4863 return (IXGBE_SUCCESS);
4864 }
4865
4866 #pragma inline(ixgbe_map_rxring_to_vector)
4867 /*
4868 * ixgbe_map_rxring_to_vector - Map given rx ring to given interrupt vector.
4869 */
4870 static void
4871 ixgbe_map_rxring_to_vector(ixgbe_t *ixgbe, int r_idx, int v_idx)
4872 {
4873 /*
4874 * Set bit in map
4875 */
4876 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx);
4877
4878 /*
4879 * Count bits set
4880 */
4881 ixgbe->vect_map[v_idx].rxr_cnt++;
4882
4883 /*
4884 * Remember bit position
4885 */
4886 ixgbe->rx_rings[r_idx].intr_vector = v_idx;
4887 ixgbe->rx_rings[r_idx].vect_bit = 1 << v_idx;
4888 }
4889
4890 #pragma inline(ixgbe_map_txring_to_vector)
4891 /*
4892 * ixgbe_map_txring_to_vector - Map given tx ring to given interrupt vector.
4893 */
4894 static void
4895 ixgbe_map_txring_to_vector(ixgbe_t *ixgbe, int t_idx, int v_idx)
4896 {
4897 /*
4898 * Set bit in map
4899 */
4900 BT_SET(ixgbe->vect_map[v_idx].tx_map, t_idx);
4901
4902 /*
4903 * Count bits set
4904 */
4905 ixgbe->vect_map[v_idx].txr_cnt++;
4906
4907 /*
4908 * Remember bit position
4909 */
4910 ixgbe->tx_rings[t_idx].intr_vector = v_idx;
4911 ixgbe->tx_rings[t_idx].vect_bit = 1 << v_idx;
4912 }
4913
4914 /*
4915 * ixgbe_setup_ivar - Set the given entry in the given interrupt vector
4916 * allocation register (IVAR).
4917 * cause:
4918 * -1 : other cause
4919 * 0 : rx
4920 * 1 : tx
4921 */
4922 static void
4923 ixgbe_setup_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, uint8_t msix_vector,
4924 int8_t cause)
4925 {
4926 struct ixgbe_hw *hw = &ixgbe->hw;
4927 u32 ivar, index;
4928
4929 switch (hw->mac.type) {
4930 case ixgbe_mac_82598EB:
4931 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4932 if (cause == -1) {
4933 cause = 0;
4934 }
4935 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
4936 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4937 ivar &= ~(0xFF << (8 * (intr_alloc_entry & 0x3)));
4938 ivar |= (msix_vector << (8 * (intr_alloc_entry & 0x3)));
4939 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
4940 break;
4941
4942 case ixgbe_mac_82599EB:
4943 case ixgbe_mac_X540:
4944 case ixgbe_mac_X550:
4945 case ixgbe_mac_X550EM_x:
4946 if (cause == -1) {
4947 /* other causes */
4948 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4949 index = (intr_alloc_entry & 1) * 8;
4950 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4951 ivar &= ~(0xFF << index);
4952 ivar |= (msix_vector << index);
4953 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4954 } else {
4955 /* tx or rx causes */
4956 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
4957 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
4958 ivar = IXGBE_READ_REG(hw,
4959 IXGBE_IVAR(intr_alloc_entry >> 1));
4960 ivar &= ~(0xFF << index);
4961 ivar |= (msix_vector << index);
4962 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
4963 ivar);
4964 }
4965 break;
4966
4967 default:
4968 break;
4969 }
4970 }
4971
4972 /*
4973 * ixgbe_enable_ivar - Enable the given entry by setting the VAL bit of
4974 * given interrupt vector allocation register (IVAR).
4975 * cause:
4976 * -1 : other cause
4977 * 0 : rx
4978 * 1 : tx
4979 */
4980 static void
4981 ixgbe_enable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
4982 {
4983 struct ixgbe_hw *hw = &ixgbe->hw;
4984 u32 ivar, index;
4985
4986 switch (hw->mac.type) {
4987 case ixgbe_mac_82598EB:
4988 if (cause == -1) {
4989 cause = 0;
4990 }
4991 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
4992 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4993 ivar |= (IXGBE_IVAR_ALLOC_VAL << (8 *
4994 (intr_alloc_entry & 0x3)));
4995 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
4996 break;
4997
4998 case ixgbe_mac_82599EB:
4999 case ixgbe_mac_X540:
5000 case ixgbe_mac_X550:
5001 case ixgbe_mac_X550EM_x:
5002 if (cause == -1) {
5003 /* other causes */
5004 index = (intr_alloc_entry & 1) * 8;
5005 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5006 ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
5007 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5008 } else {
5009 /* tx or rx causes */
5010 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
5011 ivar = IXGBE_READ_REG(hw,
5012 IXGBE_IVAR(intr_alloc_entry >> 1));
5013 ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
5014 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
5015 ivar);
5016 }
5017 break;
5018
5019 default:
5020 break;
5021 }
5022 }
5023
5024 /*
5025 * ixgbe_disable_ivar - Disble the given entry by clearing the VAL bit of
5026 * given interrupt vector allocation register (IVAR).
5027 * cause:
5028 * -1 : other cause
5029 * 0 : rx
5030 * 1 : tx
5031 */
5032 static void
5033 ixgbe_disable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
5034 {
5035 struct ixgbe_hw *hw = &ixgbe->hw;
5036 u32 ivar, index;
5037
5038 switch (hw->mac.type) {
5039 case ixgbe_mac_82598EB:
5040 if (cause == -1) {
5041 cause = 0;
5042 }
5043 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
5044 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5045 ivar &= ~(IXGBE_IVAR_ALLOC_VAL<< (8 *
5046 (intr_alloc_entry & 0x3)));
5047 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
5048 break;
5049
5050 case ixgbe_mac_82599EB:
5051 case ixgbe_mac_X540:
5052 case ixgbe_mac_X550:
5053 case ixgbe_mac_X550EM_x:
5054 if (cause == -1) {
5055 /* other causes */
5056 index = (intr_alloc_entry & 1) * 8;
5057 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5058 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
5059 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5060 } else {
5061 /* tx or rx causes */
5062 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
5063 ivar = IXGBE_READ_REG(hw,
5064 IXGBE_IVAR(intr_alloc_entry >> 1));
5065 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
5066 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
5067 ivar);
5068 }
5069 break;
5070
5071 default:
5072 break;
5073 }
5074 }
5075
5076 /*
5077 * Convert the rx ring index driver maintained to the rx ring index
5078 * in h/w.
5079 */
5080 static uint32_t
5081 ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index)
5082 {
5083
5084 struct ixgbe_hw *hw = &ixgbe->hw;
5085 uint32_t rx_ring_per_group, hw_rx_index;
5086
5087 if (ixgbe->classify_mode == IXGBE_CLASSIFY_RSS ||
5088 ixgbe->classify_mode == IXGBE_CLASSIFY_NONE) {
5089 return (sw_rx_index);
5090 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ) {
5091 switch (hw->mac.type) {
5092 case ixgbe_mac_82598EB:
5093 return (sw_rx_index);
5094
5095 case ixgbe_mac_82599EB:
5096 case ixgbe_mac_X540:
5097 case ixgbe_mac_X550:
5098 case ixgbe_mac_X550EM_x:
5099 return (sw_rx_index * 2);
5100
5101 default:
5102 break;
5103 }
5104 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ_RSS) {
5105 rx_ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
5106
5107 switch (hw->mac.type) {
5108 case ixgbe_mac_82598EB:
5109 hw_rx_index = (sw_rx_index / rx_ring_per_group) *
5110 16 + (sw_rx_index % rx_ring_per_group);
5111 return (hw_rx_index);
5112
5113 case ixgbe_mac_82599EB:
5114 case ixgbe_mac_X540:
5115 case ixgbe_mac_X550:
5116 case ixgbe_mac_X550EM_x:
5117 if (ixgbe->num_rx_groups > 32) {
5118 hw_rx_index = (sw_rx_index /
5119 rx_ring_per_group) * 2 +
5120 (sw_rx_index % rx_ring_per_group);
5121 } else {
5122 hw_rx_index = (sw_rx_index /
5123 rx_ring_per_group) * 4 +
5124 (sw_rx_index % rx_ring_per_group);
5125 }
5126 return (hw_rx_index);
5127
5128 default:
5129 break;
5130 }
5131 }
5132
5133 /*
5134 * Should never reach. Just to make compiler happy.
5135 */
5136 return (sw_rx_index);
5137 }
5138
5139 /*
5140 * ixgbe_map_intrs_to_vectors - Map different interrupts to MSI-X vectors.
5141 *
5142 * For MSI-X, here will map rx interrupt, tx interrupt and other interrupt
5143 * to vector[0 - (intr_cnt -1)].
5144 */
5145 static int
5146 ixgbe_map_intrs_to_vectors(ixgbe_t *ixgbe)
5147 {
5148 int i, vector = 0;
5149
5150 /* initialize vector map */
5151 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map));
5152 for (i = 0; i < ixgbe->intr_cnt; i++) {
5153 ixgbe->vect_map[i].ixgbe = ixgbe;
5154 }
5155
5156 /*
5157 * non-MSI-X case is very simple: rx rings[0] on RTxQ[0],
5158 * tx rings[0] on RTxQ[1].
5159 */
5160 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
5161 ixgbe_map_rxring_to_vector(ixgbe, 0, 0);
5162 ixgbe_map_txring_to_vector(ixgbe, 0, 1);
5163 return (IXGBE_SUCCESS);
5164 }
5165
5166 /*
5167 * Interrupts/vectors mapping for MSI-X
5168 */
5169
5170 /*
5171 * Map other interrupt to vector 0,
5172 * Set bit in map and count the bits set.
5173 */
5174 BT_SET(ixgbe->vect_map[vector].other_map, 0);
5175 ixgbe->vect_map[vector].other_cnt++;
5176
5177 /*
5178 * Map rx ring interrupts to vectors
5179 */
5180 for (i = 0; i < ixgbe->num_rx_rings; i++) {
5181 ixgbe_map_rxring_to_vector(ixgbe, i, vector);
5182 vector = (vector +1) % ixgbe->intr_cnt;
5183 }
5184
5185 /*
5186 * Map tx ring interrupts to vectors
5187 */
5188 for (i = 0; i < ixgbe->num_tx_rings; i++) {
5189 ixgbe_map_txring_to_vector(ixgbe, i, vector);
5190 vector = (vector +1) % ixgbe->intr_cnt;
5191 }
5192
5193 return (IXGBE_SUCCESS);
5194 }
5195
5196 /*
5197 * ixgbe_setup_adapter_vector - Setup the adapter interrupt vector(s).
5198 *
5199 * This relies on ring/vector mapping already set up in the
5200 * vect_map[] structures
5201 */
5202 static void
5203 ixgbe_setup_adapter_vector(ixgbe_t *ixgbe)
5204 {
5205 struct ixgbe_hw *hw = &ixgbe->hw;
5206 ixgbe_intr_vector_t *vect; /* vector bitmap */
5207 int r_idx; /* ring index */
5208 int v_idx; /* vector index */
5209 uint32_t hw_index;
5210
5211 /*
5212 * Clear any previous entries
5213 */
5214 switch (hw->mac.type) {
5215 case ixgbe_mac_82598EB:
5216 for (v_idx = 0; v_idx < 25; v_idx++)
5217 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5218 break;
5219
5220 case ixgbe_mac_82599EB:
5221 case ixgbe_mac_X540:
5222 case ixgbe_mac_X550:
5223 case ixgbe_mac_X550EM_x:
5224 for (v_idx = 0; v_idx < 64; v_idx++)
5225 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5226 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, 0);
5227 break;
5228
5229 default:
5230 break;
5231 }
5232
5233 /*
5234 * For non MSI-X interrupt, rx rings[0] will use RTxQ[0], and
5235 * tx rings[0] will use RTxQ[1].
5236 */
5237 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
5238 ixgbe_setup_ivar(ixgbe, 0, 0, 0);
5239 ixgbe_setup_ivar(ixgbe, 0, 1, 1);
5240 return;
5241 }
5242
5243 /*
5244 * For MSI-X interrupt, "Other" is always on vector[0].
5245 */
5246 ixgbe_setup_ivar(ixgbe, IXGBE_IVAR_OTHER_CAUSES_INDEX, 0, -1);
5247
5248 /*
5249 * For each interrupt vector, populate the IVAR table
5250 */
5251 for (v_idx = 0; v_idx < ixgbe->intr_cnt; v_idx++) {
5252 vect = &ixgbe->vect_map[v_idx];
5253
5254 /*
5255 * For each rx ring bit set
5256 */
5257 r_idx = bt_getlowbit(vect->rx_map, 0,
5258 (ixgbe->num_rx_rings - 1));
5259
5260 while (r_idx >= 0) {
5261 hw_index = ixgbe->rx_rings[r_idx].hw_index;
5262 ixgbe_setup_ivar(ixgbe, hw_index, v_idx, 0);
5263 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1),
5264 (ixgbe->num_rx_rings - 1));
5265 }
5266
5267 /*
5268 * For each tx ring bit set
5269 */
5270 r_idx = bt_getlowbit(vect->tx_map, 0,
5271 (ixgbe->num_tx_rings - 1));
5272
5273 while (r_idx >= 0) {
5274 ixgbe_setup_ivar(ixgbe, r_idx, v_idx, 1);
5275 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1),
5276 (ixgbe->num_tx_rings - 1));
5277 }
5278 }
5279 }
5280
5281 /*
5282 * ixgbe_rem_intr_handlers - Remove the interrupt handlers.
5283 */
5284 static void
5285 ixgbe_rem_intr_handlers(ixgbe_t *ixgbe)
5286 {
5287 int i;
5288 int rc;
5289
5290 for (i = 0; i < ixgbe->intr_cnt; i++) {
5291 rc = ddi_intr_remove_handler(ixgbe->htable[i]);
5292 if (rc != DDI_SUCCESS) {
5293 IXGBE_DEBUGLOG_1(ixgbe,
5294 "Remove intr handler failed: %d", rc);
5295 }
5296 }
5297 }
5298
5299 /*
5300 * ixgbe_rem_intrs - Remove the allocated interrupts.
5301 */
5302 static void
5303 ixgbe_rem_intrs(ixgbe_t *ixgbe)
5304 {
5305 int i;
5306 int rc;
5307
5308 for (i = 0; i < ixgbe->intr_cnt; i++) {
5309 rc = ddi_intr_free(ixgbe->htable[i]);
5310 if (rc != DDI_SUCCESS) {
5311 IXGBE_DEBUGLOG_1(ixgbe,
5312 "Free intr failed: %d", rc);
5313 }
5314 }
5315
5316 kmem_free(ixgbe->htable, ixgbe->intr_size);
5317 ixgbe->htable = NULL;
5318 }
5319
5320 /*
5321 * ixgbe_enable_intrs - Enable all the ddi interrupts.
5322 */
5323 static int
5324 ixgbe_enable_intrs(ixgbe_t *ixgbe)
5325 {
5326 int i;
5327 int rc;
5328
5329 /*
5330 * Enable interrupts
5331 */
5332 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) {
5333 /*
5334 * Call ddi_intr_block_enable() for MSI
5335 */
5336 rc = ddi_intr_block_enable(ixgbe->htable, ixgbe->intr_cnt);
5337 if (rc != DDI_SUCCESS) {
5338 ixgbe_log(ixgbe,
5339 "Enable block intr failed: %d", rc);
5340 return (IXGBE_FAILURE);
5341 }
5342 } else {
5343 /*
5344 * Call ddi_intr_enable() for Legacy/MSI non block enable
5345 */
5346 for (i = 0; i < ixgbe->intr_cnt; i++) {
5347 rc = ddi_intr_enable(ixgbe->htable[i]);
5348 if (rc != DDI_SUCCESS) {
5349 ixgbe_log(ixgbe,
5350 "Enable intr failed: %d", rc);
5351 return (IXGBE_FAILURE);
5352 }
5353 }
5354 }
5355
5356 return (IXGBE_SUCCESS);
5357 }
5358
5359 /*
5360 * ixgbe_disable_intrs - Disable all the interrupts.
5361 */
5362 static int
5363 ixgbe_disable_intrs(ixgbe_t *ixgbe)
5364 {
5365 int i;
5366 int rc;
5367
5368 /*
5369 * Disable all interrupts
5370 */
5371 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) {
5372 rc = ddi_intr_block_disable(ixgbe->htable, ixgbe->intr_cnt);
5373 if (rc != DDI_SUCCESS) {
5374 ixgbe_log(ixgbe,
5375 "Disable block intr failed: %d", rc);
5376 return (IXGBE_FAILURE);
5377 }
5378 } else {
5379 for (i = 0; i < ixgbe->intr_cnt; i++) {
5380 rc = ddi_intr_disable(ixgbe->htable[i]);
5381 if (rc != DDI_SUCCESS) {
5382 ixgbe_log(ixgbe,
5383 "Disable intr failed: %d", rc);
5384 return (IXGBE_FAILURE);
5385 }
5386 }
5387 }
5388
5389 return (IXGBE_SUCCESS);
5390 }
5391
5392 /*
5393 * ixgbe_get_hw_state - Get and save parameters related to adapter hardware.
5394 */
5395 static void
5396 ixgbe_get_hw_state(ixgbe_t *ixgbe)
5397 {
5398 struct ixgbe_hw *hw = &ixgbe->hw;
5399 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN;
5400 boolean_t link_up = B_FALSE;
5401 uint32_t pcs1g_anlp = 0;
5402 uint32_t pcs1g_ana = 0;
5403 boolean_t autoneg = B_FALSE;
5404
5405 ASSERT(mutex_owned(&ixgbe->gen_lock));
5406 ixgbe->param_lp_1000fdx_cap = 0;
5407 ixgbe->param_lp_100fdx_cap = 0;
5408
5409 /* check for link, don't wait */
5410 (void) ixgbe_check_link(hw, &speed, &link_up, false);
5411 pcs1g_ana = IXGBE_READ_REG(hw, IXGBE_PCS1GANA);
5412
5413 if (link_up) {
5414 pcs1g_anlp = IXGBE_READ_REG(hw, IXGBE_PCS1GANLP);
5415
5416 ixgbe->param_lp_1000fdx_cap =
5417 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5418 ixgbe->param_lp_100fdx_cap =
5419 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5420 }
5421
5422 (void) ixgbe_get_link_capabilities(hw, &speed, &autoneg);
5423
5424 ixgbe->param_adv_1000fdx_cap = ((pcs1g_ana & IXGBE_PCS1GANA_FDC) &&
5425 (speed & IXGBE_LINK_SPEED_1GB_FULL)) ? 1 : 0;
5426 ixgbe->param_adv_100fdx_cap = ((pcs1g_ana & IXGBE_PCS1GANA_FDC) &&
5427 (speed & IXGBE_LINK_SPEED_100_FULL)) ? 1 : 0;
5428 }
5429
5430 /*
5431 * ixgbe_get_driver_control - Notify that driver is in control of device.
5432 */
5433 static void
5434 ixgbe_get_driver_control(struct ixgbe_hw *hw)
5435 {
5436 uint32_t ctrl_ext;
5437
5438 /*
5439 * Notify firmware that driver is in control of device
5440 */
5441 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
5442 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
5443 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
5444 }
5445
5446 /*
5447 * ixgbe_release_driver_control - Notify that driver is no longer in control
5448 * of device.
5449 */
5450 static void
5451 ixgbe_release_driver_control(struct ixgbe_hw *hw)
5452 {
5453 uint32_t ctrl_ext;
5454
5455 /*
5456 * Notify firmware that driver is no longer in control of device
5457 */
5458 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
5459 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
5460 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
5461 }
5462
5463 /*
5464 * ixgbe_atomic_reserve - Atomic decrease operation.
5465 */
5466 int
5467 ixgbe_atomic_reserve(uint32_t *count_p, uint32_t n)
5468 {
5469 uint32_t oldval;
5470 uint32_t newval;
5471
5472 /*
5473 * ATOMICALLY
5474 */
5475 do {
5476 oldval = *count_p;
5477 if (oldval < n)
5478 return (-1);
5479 newval = oldval - n;
5480 } while (atomic_cas_32(count_p, oldval, newval) != oldval);
5481
5482 return (newval);
5483 }
5484
5485 /*
5486 * ixgbe_mc_table_itr - Traverse the entries in the multicast table.
5487 */
5488 static uint8_t *
5489 ixgbe_mc_table_itr(struct ixgbe_hw *hw, uint8_t **upd_ptr, uint32_t *vmdq)
5490 {
5491 uint8_t *addr = *upd_ptr;
5492 uint8_t *new_ptr;
5493
5494 _NOTE(ARGUNUSED(hw));
5495 _NOTE(ARGUNUSED(vmdq));
5496
5497 new_ptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
5498 *upd_ptr = new_ptr;
5499 return (addr);
5500 }
5501
5502 /*
5503 * FMA support
5504 */
5505 int
5506 ixgbe_check_acc_handle(ddi_acc_handle_t handle)
5507 {
5508 ddi_fm_error_t de;
5509
5510 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
5511 ddi_fm_acc_err_clear(handle, DDI_FME_VERSION);
5512 return (de.fme_status);
5513 }
5514
5515 int
5516 ixgbe_check_dma_handle(ddi_dma_handle_t handle)
5517 {
5518 ddi_fm_error_t de;
5519
5520 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
5521 return (de.fme_status);
5522 }
5523
5524 /*
5525 * ixgbe_fm_error_cb - The IO fault service error handling callback function.
5526 */
5527 static int
5528 ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
5529 {
5530 _NOTE(ARGUNUSED(impl_data));
5531 /*
5532 * as the driver can always deal with an error in any dma or
5533 * access handle, we can just return the fme_status value.
5534 */
5535 pci_ereport_post(dip, err, NULL);
5536 return (err->fme_status);
5537 }
5538
5539 static void
5540 ixgbe_fm_init(ixgbe_t *ixgbe)
5541 {
5542 ddi_iblock_cookie_t iblk;
5543 int fma_dma_flag;
5544
5545 /*
5546 * Only register with IO Fault Services if we have some capability
5547 */
5548 if (ixgbe->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) {
5549 ixgbe_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC;
5550 } else {
5551 ixgbe_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC;
5552 }
5553
5554 if (ixgbe->fm_capabilities & DDI_FM_DMACHK_CAPABLE) {
5555 fma_dma_flag = 1;
5556 } else {
5557 fma_dma_flag = 0;
5558 }
5559
5560 ixgbe_set_fma_flags(fma_dma_flag);
5561
5562 if (ixgbe->fm_capabilities) {
5563
5564 /*
5565 * Register capabilities with IO Fault Services
5566 */
5567 ddi_fm_init(ixgbe->dip, &ixgbe->fm_capabilities, &iblk);
5568
5569 /*
5570 * Initialize pci ereport capabilities if ereport capable
5571 */
5572 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) ||
5573 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5574 pci_ereport_setup(ixgbe->dip);
5575
5576 /*
5577 * Register error callback if error callback capable
5578 */
5579 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5580 ddi_fm_handler_register(ixgbe->dip,
5581 ixgbe_fm_error_cb, (void*) ixgbe);
5582 }
5583 }
5584
5585 static void
5586 ixgbe_fm_fini(ixgbe_t *ixgbe)
5587 {
5588 /*
5589 * Only unregister FMA capabilities if they are registered
5590 */
5591 if (ixgbe->fm_capabilities) {
5592
5593 /*
5594 * Release any resources allocated by pci_ereport_setup()
5595 */
5596 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) ||
5597 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5598 pci_ereport_teardown(ixgbe->dip);
5599
5600 /*
5601 * Un-register error callback if error callback capable
5602 */
5603 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
5604 ddi_fm_handler_unregister(ixgbe->dip);
5605
5606 /*
5607 * Unregister from IO Fault Service
5608 */
5609 ddi_fm_fini(ixgbe->dip);
5610 }
5611 }
5612
5613 void
5614 ixgbe_fm_ereport(ixgbe_t *ixgbe, char *detail)
5615 {
5616 uint64_t ena;
5617 char buf[FM_MAX_CLASS];
5618
5619 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
5620 ena = fm_ena_generate(0, FM_ENA_FMT1);
5621 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities)) {
5622 ddi_fm_ereport_post(ixgbe->dip, buf, ena, DDI_NOSLEEP,
5623 FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL);
5624 }
5625 }
5626
5627 static int
5628 ixgbe_ring_start(mac_ring_driver_t rh, uint64_t mr_gen_num)
5629 {
5630 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)rh;
5631
5632 mutex_enter(&rx_ring->rx_lock);
5633 rx_ring->ring_gen_num = mr_gen_num;
5634 mutex_exit(&rx_ring->rx_lock);
5635 return (0);
5636 }
5637
5638 /*
5639 * Get the global ring index by a ring index within a group.
5640 */
5641 static int
5642 ixgbe_get_rx_ring_index(ixgbe_t *ixgbe, int gindex, int rindex)
5643 {
5644 ixgbe_rx_ring_t *rx_ring;
5645 int i;
5646
5647 for (i = 0; i < ixgbe->num_rx_rings; i++) {
5648 rx_ring = &ixgbe->rx_rings[i];
5649 if (rx_ring->group_index == gindex)
5650 rindex--;
5651 if (rindex < 0)
5652 return (i);
5653 }
5654
5655 return (-1);
5656 }
5657
5658 /*
5659 * Callback funtion for MAC layer to register all rings.
5660 */
5661 /* ARGSUSED */
5662 void
5663 ixgbe_fill_ring(void *arg, mac_ring_type_t rtype, const int group_index,
5664 const int ring_index, mac_ring_info_t *infop, mac_ring_handle_t rh)
5665 {
5666 ixgbe_t *ixgbe = (ixgbe_t *)arg;
5667 mac_intr_t *mintr = &infop->mri_intr;
5668
5669 switch (rtype) {
5670 case MAC_RING_TYPE_RX: {
5671 /*
5672 * 'index' is the ring index within the group.
5673 * Need to get the global ring index by searching in groups.
5674 */
5675 int global_ring_index = ixgbe_get_rx_ring_index(
5676 ixgbe, group_index, ring_index);
5677
5678 ASSERT(global_ring_index >= 0);
5679
5680 ixgbe_rx_ring_t *rx_ring = &ixgbe->rx_rings[global_ring_index];
5681 rx_ring->ring_handle = rh;
5682
5683 infop->mri_driver = (mac_ring_driver_t)rx_ring;
5684 infop->mri_start = ixgbe_ring_start;
5685 infop->mri_stop = NULL;
5686 infop->mri_poll = ixgbe_ring_rx_poll;
5687 infop->mri_stat = ixgbe_rx_ring_stat;
5688
5689 mintr->mi_handle = (mac_intr_handle_t)rx_ring;
5690 mintr->mi_enable = ixgbe_rx_ring_intr_enable;
5691 mintr->mi_disable = ixgbe_rx_ring_intr_disable;
5692 if (ixgbe->intr_type &
5693 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) {
5694 mintr->mi_ddi_handle =
5695 ixgbe->htable[rx_ring->intr_vector];
5696 }
5697
5698 break;
5699 }
5700 case MAC_RING_TYPE_TX: {
5701 ASSERT(group_index == -1);
5702 ASSERT(ring_index < ixgbe->num_tx_rings);
5703
5704 ixgbe_tx_ring_t *tx_ring = &ixgbe->tx_rings[ring_index];
5705 tx_ring->ring_handle = rh;
5706
5707 infop->mri_driver = (mac_ring_driver_t)tx_ring;
5708 infop->mri_start = NULL;
5709 infop->mri_stop = NULL;
5710 infop->mri_tx = ixgbe_ring_tx;
5711 infop->mri_stat = ixgbe_tx_ring_stat;
5712 if (ixgbe->intr_type &
5713 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) {
5714 mintr->mi_ddi_handle =
5715 ixgbe->htable[tx_ring->intr_vector];
5716 }
5717 break;
5718 }
5719 default:
5720 break;
5721 }
5722 }
5723
5724 /*
5725 * Callback funtion for MAC layer to register all groups.
5726 */
5727 void
5728 ixgbe_fill_group(void *arg, mac_ring_type_t rtype, const int index,
5729 mac_group_info_t *infop, mac_group_handle_t gh)
5730 {
5731 ixgbe_t *ixgbe = (ixgbe_t *)arg;
5732
5733 switch (rtype) {
5734 case MAC_RING_TYPE_RX: {
5735 ixgbe_rx_group_t *rx_group;
5736
5737 rx_group = &ixgbe->rx_groups[index];
5738 rx_group->group_handle = gh;
5739
5740 infop->mgi_driver = (mac_group_driver_t)rx_group;
5741 infop->mgi_start = NULL;
5742 infop->mgi_stop = NULL;
5743 infop->mgi_addmac = ixgbe_addmac;
5744 infop->mgi_remmac = ixgbe_remmac;
5745 infop->mgi_count = (ixgbe->num_rx_rings / ixgbe->num_rx_groups);
5746
5747 break;
5748 }
5749 case MAC_RING_TYPE_TX:
5750 break;
5751 default:
5752 break;
5753 }
5754 }
5755
5756 /*
5757 * Enable interrupt on the specificed rx ring.
5758 */
5759 int
5760 ixgbe_rx_ring_intr_enable(mac_intr_handle_t intrh)
5761 {
5762 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh;
5763 ixgbe_t *ixgbe = rx_ring->ixgbe;
5764 int r_idx = rx_ring->index;
5765 int hw_r_idx = rx_ring->hw_index;
5766 int v_idx = rx_ring->intr_vector;
5767
5768 mutex_enter(&ixgbe->gen_lock);
5769 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) {
5770 mutex_exit(&ixgbe->gen_lock);
5771 /*
5772 * Simply return 0.
5773 * Interrupts are being adjusted. ixgbe_intr_adjust()
5774 * will eventually re-enable the interrupt when it's
5775 * done with the adjustment.
5776 */
5777 return (0);
5778 }
5779
5780 /*
5781 * To enable interrupt by setting the VAL bit of given interrupt
5782 * vector allocation register (IVAR).
5783 */
5784 ixgbe_enable_ivar(ixgbe, hw_r_idx, 0);
5785
5786 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx);
5787
5788 /*
5789 * Trigger a Rx interrupt on this ring
5790 */
5791 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_EICS, (1 << v_idx));
5792 IXGBE_WRITE_FLUSH(&ixgbe->hw);
5793
5794 mutex_exit(&ixgbe->gen_lock);
5795
5796 return (0);
5797 }
5798
5799 /*
5800 * Disable interrupt on the specificed rx ring.
5801 */
5802 int
5803 ixgbe_rx_ring_intr_disable(mac_intr_handle_t intrh)
5804 {
5805 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh;
5806 ixgbe_t *ixgbe = rx_ring->ixgbe;
5807 int r_idx = rx_ring->index;
5808 int hw_r_idx = rx_ring->hw_index;
5809 int v_idx = rx_ring->intr_vector;
5810
5811 mutex_enter(&ixgbe->gen_lock);
5812 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) {
5813 mutex_exit(&ixgbe->gen_lock);
5814 /*
5815 * Simply return 0.
5816 * In the rare case where an interrupt is being
5817 * disabled while interrupts are being adjusted,
5818 * we don't fail the operation. No interrupts will
5819 * be generated while they are adjusted, and
5820 * ixgbe_intr_adjust() will cause the interrupts
5821 * to be re-enabled once it completes. Note that
5822 * in this case, packets may be delivered to the
5823 * stack via interrupts before xgbe_rx_ring_intr_enable()
5824 * is called again. This is acceptable since interrupt
5825 * adjustment is infrequent, and the stack will be
5826 * able to handle these packets.
5827 */
5828 return (0);
5829 }
5830
5831 /*
5832 * To disable interrupt by clearing the VAL bit of given interrupt
5833 * vector allocation register (IVAR).
5834 */
5835 ixgbe_disable_ivar(ixgbe, hw_r_idx, 0);
5836
5837 BT_CLEAR(ixgbe->vect_map[v_idx].rx_map, r_idx);
5838
5839 mutex_exit(&ixgbe->gen_lock);
5840
5841 return (0);
5842 }
5843
5844 /*
5845 * Add a mac address.
5846 */
5847 static int
5848 ixgbe_addmac(void *arg, const uint8_t *mac_addr)
5849 {
5850 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg;
5851 ixgbe_t *ixgbe = rx_group->ixgbe;
5852 struct ixgbe_hw *hw = &ixgbe->hw;
5853 int slot, i;
5854
5855 mutex_enter(&ixgbe->gen_lock);
5856
5857 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
5858 mutex_exit(&ixgbe->gen_lock);
5859 return (ECANCELED);
5860 }
5861
5862 if (ixgbe->unicst_avail == 0) {
5863 /* no slots available */
5864 mutex_exit(&ixgbe->gen_lock);
5865 return (ENOSPC);
5866 }
5867
5868 /*
5869 * The first ixgbe->num_rx_groups slots are reserved for each respective
5870 * group. The rest slots are shared by all groups. While adding a
5871 * MAC address, reserved slots are firstly checked then the shared
5872 * slots are searched.
5873 */
5874 slot = -1;
5875 if (ixgbe->unicst_addr[rx_group->index].mac.set == 1) {
5876 for (i = ixgbe->num_rx_groups; i < ixgbe->unicst_total; i++) {
5877 if (ixgbe->unicst_addr[i].mac.set == 0) {
5878 slot = i;
5879 break;
5880 }
5881 }
5882 } else {
5883 slot = rx_group->index;
5884 }
5885
5886 if (slot == -1) {
5887 /* no slots available */
5888 mutex_exit(&ixgbe->gen_lock);
5889 return (ENOSPC);
5890 }
5891
5892 bcopy(mac_addr, ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL);
5893 (void) ixgbe_set_rar(hw, slot, ixgbe->unicst_addr[slot].mac.addr,
5894 rx_group->index, IXGBE_RAH_AV);
5895 ixgbe->unicst_addr[slot].mac.set = 1;
5896 ixgbe->unicst_addr[slot].mac.group_index = rx_group->index;
5897 ixgbe->unicst_avail--;
5898
5899 mutex_exit(&ixgbe->gen_lock);
5900
5901 return (0);
5902 }
5903
5904 /*
5905 * Remove a mac address.
5906 */
5907 static int
5908 ixgbe_remmac(void *arg, const uint8_t *mac_addr)
5909 {
5910 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg;
5911 ixgbe_t *ixgbe = rx_group->ixgbe;
5912 struct ixgbe_hw *hw = &ixgbe->hw;
5913 int slot;
5914
5915 mutex_enter(&ixgbe->gen_lock);
5916
5917 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
5918 mutex_exit(&ixgbe->gen_lock);
5919 return (ECANCELED);
5920 }
5921
5922 slot = ixgbe_unicst_find(ixgbe, mac_addr);
5923 if (slot == -1) {
5924 mutex_exit(&ixgbe->gen_lock);
5925 return (EINVAL);
5926 }
5927
5928 if (ixgbe->unicst_addr[slot].mac.set == 0) {
5929 mutex_exit(&ixgbe->gen_lock);
5930 return (EINVAL);
5931 }
5932
5933 bzero(ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL);
5934 (void) ixgbe_clear_rar(hw, slot);
5935 ixgbe->unicst_addr[slot].mac.set = 0;
5936 ixgbe->unicst_avail++;
5937
5938 mutex_exit(&ixgbe->gen_lock);
5939
5940 return (0);
5941 }